Resolve conflicts

zhongkaifu · Nov 28, 2023 · e544a52 · e544a52
2 parents e11ed7b + fdc6a4e
commit e544a52
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 2 deletions.
diff --git a/Seq2SeqSharp/Corpus/ParallelCorpus.cs b/Seq2SeqSharp/Corpus/ParallelCorpus.cs
@@ -346,6 +346,11 @@ public void PrepareDataSet()
                 Logger.WriteLine(Logger.Level.debug, $"Start to sort and shuffle data set by length.");
 
                 m_sortedIndexedDataSetFilePath = tmpDataSetFilePath + ".sorted";
+
+#if DEBUG
+                string tmp_sortedIndexedDataSetFilePath = tmpDataSetFilePath + ".sorted.txt";
+                using (StreamWriter bwt = new StreamWriter(new FileStream(tmp_sortedIndexedDataSetFilePath, FileMode.Create, FileAccess.Write, FileShare.None, 40960000)))
+#endif
                 using (BinaryWriter bw = new BinaryWriter(new FileStream(m_sortedIndexedDataSetFilePath, FileMode.Create, FileAccess.Write, FileShare.None, 40960000)))
                 using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(tmpDataSetFilePath))
                 using (MemoryMappedViewStream mms = mmf.CreateViewStream())
@@ -388,6 +393,12 @@ public void PrepareDataSet()
                             bw.Write(String.Join("\n", srcLines));
                             bw.Write(String.Join("\n", tgtLines));
 
+#if DEBUG
+                            bwt.WriteLine(sentSize);
+                            bwt.WriteLine(String.Join("\n", srcLines));
+                            bwt.WriteLine(String.Join("\n", tgtLines));
+#endif
+
                             m_batchNumInTotal++;
                             if (m_batchNumInTotal % 10000 == 0)
                             {
@@ -457,7 +468,7 @@ public IEnumerator<T> GetEnumerator()
 
                         if (batchIdx % 10000 == 0)
                         {
-                            Logger.WriteLine($"Processing batch '{batchIdx}'");
+                            Logger.WriteLine(Logger.Level.debug, $"Processing batch '{batchIdx}'");
                         }
 
 

diff --git a/TensorSharp.CUDA/RuntimeCompiler/CudaCompiler.cs b/TensorSharp.CUDA/RuntimeCompiler/CudaCompiler.cs
@@ -49,7 +49,7 @@ private byte[] DoCompile(string fullSource)
                 }
                 else
                 {
-                    Logger.WriteLine($"Compiler Options: {string.Join(" ", m_options)}");
+                    Logger.WriteLine(Logger.Level.debug, $"Compiler Options: {string.Join(" ", m_options)}");
                     rtc.Compile(m_options);
                     //rtc.Compile(new string[] { "--use_fast_math", "--gpu-architecture=compute_60" });
                 }