Trunicated too long sentence while iterating indexed data set.

zhongkaifu · Dec 18, 2024 · 243f358 · 243f358
1 parent c54dc5f
commit 243f358
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/Seq2SeqSharp/Corpus/MonoCorpus.cs b/Seq2SeqSharp/Corpus/MonoCorpus.cs
@@ -416,7 +416,7 @@ public IEnumerator<T> GetEnumerator()
                             }
                         }
 
-                        IPair sntPair = new SntPair(tgtLine, tgtLine);
+                        IPair sntPair = new SntPair(tgtLine, tgtLine, maxSrcLength: m_maxTgtTokenSize, maxTgtLength: m_maxTgtTokenSize);
                         currentTokenCountsInBatch += sntPair.GetTgtTokenCount();
                         outputs.Add(sntPair);
 

diff --git a/Seq2SeqSharp/Corpus/Seq2SeqCorpusBatch.cs b/Seq2SeqSharp/Corpus/Seq2SeqCorpusBatch.cs
@@ -115,19 +115,27 @@ public class SntPair : IPair
         public List<string> SrcTokens;
         public List<string> TgtTokens;
 
-        public SntPair(string srcLine, string tgtLine)
+        public SntPair(string srcLine, string tgtLine, int maxSrcLength = -1, int maxTgtLength = -1)
         {
             SrcTokens = new List<string>();
             TgtTokens = new List<string>();
 
             if (String.IsNullOrEmpty(srcLine) == false)
             {
                 SrcTokens = srcLine.Split(' ').ToList();
+                if (maxSrcLength >= 0 && SrcTokens.Count > maxSrcLength)
+                {
+                    SrcTokens = SrcTokens.GetRange(0, maxSrcLength);
+                }
             }
 
             if (String.IsNullOrEmpty(tgtLine) == false)
             {
                 TgtTokens = tgtLine.Split(' ').ToList();
+                if (maxTgtLength >= 0 && TgtTokens.Count > maxTgtLength)
+                {
+                    TgtTokens = TgtTokens.GetRange(0, maxTgtLength);
+                }
             }
         }