Merge pull request #81 from zsogitbe/master

Logger correction and Seq2seq base training progress reporting
zhongkaifu · Nov 16, 2023 · da969bb · da969bb
2 parents 13ca642 + c855003
commit da969bb
Show file tree

Hide file tree

Showing 20 changed files with 498 additions and 50 deletions.
diff --git a/AdvUtils/Arg/ArgField.cs b/AdvUtils/Arg/ArgField.cs
@@ -45,7 +45,11 @@ public void Set(string val)
             }
             catch (Exception err)
             {
-                throw new ArgumentException($"Failed to set value of '{m_a.ToString()}', Error: '{err.Message}', Call Stack: '{err.StackTrace}'");
+                string stt = "";
+#if DEBUG
+                stt = $", Call Stack: '{err.StackTrace}'";
+#endif
+                throw new ArgumentException($"Failed to set value of '{m_a.ToString()}', Error: '{err.Message}'{stt}");
             }
         }
         public void Validate()

diff --git a/AdvUtils/Logger.cs b/AdvUtils/Logger.cs
@@ -63,11 +63,11 @@ public static ProgressCallback? Callback
             }
         }
 
-        static private List<TextWriterTraceListener> m_listeners = null;
-        static private string m_logFilePath = null;
-        static public string LogFilePath => m_logFilePath;
+        static private List<TextWriterTraceListener>? m_listeners = null;
+        static private string? m_logFilePath = null;
+        static public string? LogFilePath => m_logFilePath;
 
-        static public void Initialize(Destination dest, Level logLevel, string logFilePath = "", ProgressCallback callback = null)
+        static public void Initialize(Destination dest, Level logLevel, string logFilePath = "", ProgressCallback? callback = null)
         {
             if (m_listeners != null)
             {
@@ -119,23 +119,27 @@ public static void WriteLine(Level level, ConsoleColor color, string s, params o
                 Console.ForegroundColor = color;
 
                 StringBuilder sb = new StringBuilder();
-                sb.AppendFormat("{0},{1} ", level.ToString(), DateTime.Now.ToString());
 
-                if (args.Length == 0)
-                    sb.Append(s);
-                else
-                    sb.AppendFormat(s, args);
+                if (!String.IsNullOrEmpty(s))
+                {
+                    sb.AppendFormat("{0},{1} ", level.ToString(), DateTime.Now.ToString());
+
+                    if (args.Length == 0)
+                        sb.Append(s);
+                    else
+                        sb.AppendFormat(s, args);
 
-                string sLine = sb.ToString();
-                Trace.WriteLine(sLine);
-                Trace.Flush();
+                    string sLine = sb.ToString();
+                    Trace.WriteLine(sLine);
+                    Trace.Flush();
+                }
 
                 if (Callback != null)
                 {
                     int progress = 0;
                     if(args.Length > 0) 
                     {
-                        int.TryParse((string)args[0], out progress);
+                        int.TryParse(args[0].ToString(), out progress);
                     }
                     Callback(progress, sb, (int)level, (int)color);
                 }

diff --git a/Seq2SeqSharp/Corpus/MonoCorpus.cs b/Seq2SeqSharp/Corpus/MonoCorpus.cs
@@ -313,7 +313,8 @@ public void PrepareDataSet()
             }
             catch (Exception err)
             {
-                Logger.WriteLine($"Failed to prepare data set: '{err.Message}', Call Stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.info, $"Failed to prepare data set: '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
             }
         }
 

diff --git a/Seq2SeqSharp/MultiProcessorNetworkWrapper.cs b/Seq2SeqSharp/MultiProcessorNetworkWrapper.cs
@@ -112,7 +112,8 @@ public void SyncWeights()
                     }
                     catch (Exception err)
                     {
-                        Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}', Call Stack = '{err.StackTrace}'");
+                        Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}'.");
+                        Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
                         throw;
                     }
                 });
@@ -158,7 +159,8 @@ public void SumGradientsToNetworkOnDefaultDevice()
                     }
                     catch (Exception err)
                     {
-                        Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}', Call Stack = '{err.StackTrace}'");
+                        Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}'.");
+                        Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
                         throw;
                     }
                 });

diff --git a/Seq2SeqSharp/Optimizer/AdamOptimizer.cs b/Seq2SeqSharp/Optimizer/AdamOptimizer.cs
@@ -122,7 +122,7 @@ private void UpdateWeightsTensor(WeightTensor m, int batchSize, float step_size,
             catch (Exception err)
             {
                 Logger.WriteLine(Logger.Level.err, $"Exception: '{err.Message}'");
-                Logger.WriteLine(Logger.Level.err, $"Call stack: '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack: '{err.StackTrace}'");
 
                 throw;
             }

diff --git a/Seq2SeqSharp/Tools/BaseSeq2SeqFramework.cs b/Seq2SeqSharp/Tools/BaseSeq2SeqFramework.cs
@@ -227,7 +227,8 @@ protected bool SaveModelRoutine<ProtoBuf_T>(T model, Func<T, ProtoBuf_T> createM
             }
             catch (Exception ex)
             {
-                Logger.WriteLine(Logger.Level.warn, ConsoleColor.Yellow, $"Failed to save model to file. Exception = '{ex.Message}', Call stack = '{ex.StackTrace}'");
+                Logger.WriteLine(Logger.Level.warn, ConsoleColor.Yellow, $"Failed to save model to file. Exception = '{ex.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, ConsoleColor.Yellow, $"Call stack = '{ex.StackTrace}'");
                 return (false);
             }
         }
@@ -388,6 +389,9 @@ public void Train(int maxTrainingEpoch, ICorpus<IPairBatch> trainCorpus, ICorpus
                 // Train one epoch over given devices. Forward part is implemented in RunForwardOnSingleDevice function in below, 
                 // backward, weights updates and other parts are implemented in the framework. You can see them in BaseSeq2SeqFramework.cs
                 TrainOneEpoch(i, trainCorpus, validCorpusList, learningRate, optimizer, taskId2metrics, decodingOptions, RunForwardOnSingleDevice);
+
+                // send progress reporting in the form of a percentage value (0-100%)
+                Logger.WriteLine(Logger.Level.info, "", (int)(100 * (i + 1) / maxTrainingEpoch));
             }
 
             SaveModel(createBackupPrevious: false, suffix: $".{m_weightsUpdateCount}");
@@ -504,7 +508,8 @@ internal void TrainOneEpoch(int ep, ICorpus<IPairBatch> trainCorpus, ICorpus<IPa
                                     }
                                     else
                                     {
-                                        Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Inner Exception: {excep.Message}, Call stack: {excep.StackTrace}");
+                                        Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Inner Exception: {excep.Message}.");
+                                        Logger.WriteLine(Logger.Level.debug, ConsoleColor.Red, $"Call stack: {excep.StackTrace}");
                                         throw err;
                                     }
                                 }
@@ -519,13 +524,15 @@ internal void TrainOneEpoch(int ep, ICorpus<IPairBatch> trainCorpus, ICorpus<IPa
                                 }
                                 else
                                 {
-                                    Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Inner Exception: {err.Message}, Call stack: {err.StackTrace}");
+                                    Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Inner Exception: {err.Message}.");
+                                    Logger.WriteLine(Logger.Level.debug, ConsoleColor.Red, $"Call stack: {err.StackTrace}");
                                     throw err;
                                 }
                             }
                             else
                             {
-                                Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: {err.Message}, Call stack: {err.StackTrace}");
+                                Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: {err.Message}.");
+                                Logger.WriteLine(Logger.Level.debug, ConsoleColor.Red, $"Call stack: {err.StackTrace}");
                                 throw err;
                             }
 
@@ -541,12 +548,14 @@ internal void TrainOneEpoch(int ep, ICorpus<IPairBatch> trainCorpus, ICorpus<IPa
                         }
                         catch (WeightsCorruptedException err)
                         {
-                            Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: {err.Message}, Call stack: {err.StackTrace}");
+                            Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: {err.Message}.");
+                            Logger.WriteLine(Logger.Level.debug, ConsoleColor.Red, $"Call stack: {err.StackTrace}");
                             throw;
                         }
                         catch (Exception err)
                         {
-                            Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: {err.Message}, Call stack: {err.StackTrace}");
+                            Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: {err.Message}.");
+                            Logger.WriteLine(Logger.Level.debug, ConsoleColor.Red, $"Call stack: {err.StackTrace}");
                             throw;
                         }
                     }
@@ -706,7 +715,7 @@ private int TryToSplitBatchFactor(List<IPairBatch> sntPairBatchs, int batchSplit
                     catch (Exception err)
                     {
                         Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: '{err.Message}'");
-                        Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Call stack: '{err.StackTrace}'");
+                        Logger.WriteLine(Logger.Level.debug, ConsoleColor.Red, $"Call stack: '{err.StackTrace}'");
 
                         throw;
                     }
@@ -832,7 +841,8 @@ internal List<NetworkResult> RunTest(IPairBatch sntPairBatch, DecodingOptions de
                         }
                         catch (Exception err)
                         {
-                            Logger.WriteLine(Logger.Level.err, $"Test error at processor '{deviceIdx}'. Exception = '{err.Message}', Call Stack = '{err.StackTrace}'");
+                            Logger.WriteLine(Logger.Level.err, $"Test error at processor '{deviceIdx}'. Exception = '{err.Message}'.");
+                            Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
                             throw;
                         }
                     });
@@ -865,7 +875,8 @@ internal List<NetworkResult> RunTest(IPairBatch sntPairBatch, DecodingOptions de
             }
             catch (Exception err)
             {
-                Logger.WriteLine(Logger.Level.err, $"Exception = '{err.Message}', Call Stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Exception = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
                 throw;
             }
         }
@@ -934,7 +945,8 @@ internal List<NetworkResult> RunTest(IPairBatch sntPairBatch, DecodingOptions de
                     catch (Exception err)
                     {
                         runningGoodSoFar = false;
-                        Logger.WriteLine(Logger.Level.err, $"Test error at processor '{deviceIdx}'. Exception = '{err.Message}', Call Stack = '{err.StackTrace}'");
+                        Logger.WriteLine(Logger.Level.err, $"Test error at processor '{deviceIdx}'. Exception = '{err.Message}'.");
+                        Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
                         throw;
                     }
                 });
@@ -943,7 +955,8 @@ internal List<NetworkResult> RunTest(IPairBatch sntPairBatch, DecodingOptions de
             }
             catch (Exception err)
             {
-                Logger.WriteLine(Logger.Level.err, $"Exception = '{err.Message}', Call Stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Exception = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
                 throw;
             }
         }
@@ -1177,7 +1190,7 @@ private void RunValidParallel(Func<IComputeGraph, IPairBatch, DecodingOptions, b
                 }
                 catch (Exception err)
                 {
-                    Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: '{err.Message}'");
+                    Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: '{err.Message}'.");
                     Logger.WriteLine(Logger.Level.debug, ConsoleColor.Red, $"Call stack: '{err.StackTrace}'");
                 }
             });

diff --git a/TensorSharp.CUDA/ContextState/CudaKernelCache.cs b/TensorSharp.CUDA/ContextState/CudaKernelCache.cs
@@ -53,7 +53,7 @@ public CudaKernel Get(CudaContext context, byte[] ptx, string kernelName)
                 {
                     Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Failed to get kernel '{kernelName}'");
                     Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Exception: '{err.Message}'");
-                    Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Call stack: '{err.StackTrace}'");
+                    Logger.WriteLine(Logger.Level.debug, ConsoleColor.Red, $"Call stack: '{err.StackTrace}'");
 
                     throw;
                 }

diff --git a/TensorSharp.CUDA/CudaBasicOps.cs b/TensorSharp.CUDA/CudaBasicOps.cs
@@ -100,7 +100,8 @@ public void CopyGpu(
             }
             catch (Exception err)
             {
-                Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}', Call Stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
                 throw;
             }
         }
@@ -127,7 +128,8 @@ public void CopyCpuToGpu(
             }
             catch (Exception err)
             {
-                Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}', Call Stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
                 throw;
             }
         }
@@ -154,7 +156,8 @@ public void CopyGpuToCpu(
             }
             catch (Exception err)
             {
-                Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}', Call Stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Error Message = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call Stack = '{err.StackTrace}'");
                 throw;
             }
         }
@@ -318,7 +321,8 @@ public static Tensor AddmmBatch(Tensor result, float beta, Tensor src, float alp
             }
             catch (Exception err)
             {
-                Logger.WriteLine(Logger.Level.err, $"Failed to run AddmmBatch on GPU. Error message = '{err.Message}', Call stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Failed to run AddmmBatch on GPU. Error message = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack = '{err.StackTrace}'");
                 throw;
             }
         }

diff --git a/TensorSharp.CUDA/CudaStorage.cs b/TensorSharp.CUDA/CudaStorage.cs
@@ -89,9 +89,9 @@ public override float GetElementAsFloat(long index)
             }
             catch (Exception err)
             {
-                Logger.WriteLine($"Failed to get element as float from addr = '{ptr.Pointer}'");
-                Logger.WriteLine($"Exception: {err.Message}");
-                Logger.WriteLine($"Call stack: {err.StackTrace}");
+                Logger.WriteLine(Logger.Level.err, $"Failed to get element as float from addr = '{ptr.Pointer}'");
+                Logger.WriteLine(Logger.Level.err, $"Exception: {err.Message}");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack: {err.StackTrace}");
 
                 throw;
             }

diff --git a/TensorSharp.CUDA/DeviceCode/GatherScatterKernels.cs b/TensorSharp.CUDA/DeviceCode/GatherScatterKernels.cs
@@ -203,7 +203,8 @@ public Tensor Gather(Tensor result, Tensor src, int dim, Tensor indices)
             }
             catch (Exception err)
             {
-                Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Error = '{err.Message}', Call stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Error = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, ConsoleColor.Red, $"Call stack = '{err.StackTrace}'");
                 throw;
             }
         }
@@ -257,7 +258,8 @@ public Tensor Scatter(Tensor result, Tensor src, int dim, Tensor indices)
             }
             catch (Exception err)
             {
-                Logger.WriteLine($"Error = '{err.Message}', Call stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Error = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack = '{err.StackTrace}'");
                 throw;
             }
         }

diff --git a/TensorSharp.CUDA/KernelOps/ElementwiseTOp.cs b/TensorSharp.CUDA/KernelOps/ElementwiseTOp.cs
@@ -41,7 +41,9 @@ public static void Invoke(TSCudaContext context, CudaContext cudaContext, byte[]
             }
             catch (Exception ex)
             {
-                Logger.WriteLine($"Exception message = {ex.Message}, Call stack = {ex.StackTrace}");
+                Logger.WriteLine(Logger.Level.err, $"Exception message = '{ex.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack = '{ex.StackTrace}'");
+
                 throw;
             }
 
@@ -98,7 +100,8 @@ public static Tensor Invoke(CudaCode kernels, string funcName, Tensor result, Te
             }
             catch (Exception e)
             {
-                Logger.WriteLine($"Error = '{e.Message}', Call stack = '{e.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Error = '{e.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack = '{e.StackTrace}'");
                 throw;
             }
         }
@@ -268,7 +271,8 @@ public static Tensor Invoke(CudaCode kernels, string funcName, Tensor result, Te
             }
             catch (Exception e)
             {
-                Logger.WriteLine($"Exception message = '{e.Message}', Call stack = '{e.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Exception message = '{e.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack = '{e.StackTrace}'");
                 throw;
             }
         }
@@ -296,7 +300,8 @@ public static Tensor Invoke(CudaCode kernels, Tensor result, Tensor rhs)
             }
             catch (Exception e)
             {
-                Logger.WriteLine($"Exception message = '{e.Message}', Call stack = '{e.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Exception message = '{e.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack = '{e.StackTrace}'");
                 throw;
             }
         }

diff --git a/TensorSharp/Cpu/CpuBasicOps.cs b/TensorSharp/Cpu/CpuBasicOps.cs
@@ -104,7 +104,9 @@ public static Tensor Addmm(Tensor result, float beta, Tensor src, float alpha, T
             }
             catch (Exception err)
             {
-                Logger.WriteLine(Logger.Level.err, $"Exception = '{err.Message}', Call stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Exception = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack = '{err.StackTrace}'");
+
                 throw;
             }
         }

diff --git a/TensorSharp/Cpu/CpuFillCopyOps.cs b/TensorSharp/Cpu/CpuFillCopyOps.cs
@@ -44,7 +44,8 @@ public void Copy(Tensor result, Tensor src)
             }
             catch (Exception err)
             {
-                Logger.WriteLine($"Failed to run Copy operation on CPU. Message = '{err.Message}', Call stack = '{err.StackTrace}'");
+                Logger.WriteLine(Logger.Level.err, $"Failed to run Copy operation on CPU. Message = '{err.Message}'.");
+                Logger.WriteLine(Logger.Level.debug, $"Call stack = '{err.StackTrace}'");
                 throw;
             }
         }