minor update

zhongkaifu · Dec 1, 2023 · ce3f1d1 · ce3f1d1
1 parent ab5d708
commit ce3f1d1
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 34 deletions.
diff --git a/Seq2SeqSharp/Corpus/MonoCorpus.cs b/Seq2SeqSharp/Corpus/MonoCorpus.cs
@@ -226,23 +226,19 @@ public List<Dictionary<string, long>> CountTokenFreqs()
 
         public long GetNextLength(Dictionary<long, long> len2counts, long totalRecordsNum)
         {
-            long[] keys = len2counts.Keys.ToArray();
-            int rndIdx = rnd.Next(keys.Length);
-            return keys[rndIdx];
-
-            //long rndItems = rnd.NextInt64(totalRecordsNum);
-            //long totalItems = 0;
-            //foreach (var pair in len2counts)
-            //{
-            //    long length = pair.Value;
-            //    if (totalItems <= rndItems && totalItems + length >= rndItems)
-            //    {
-            //        return pair.Key;
-            //    }
-            //    totalItems += length;
-            //}
-
-            //return -1;
+            long rndItems = rnd.NextInt64(totalRecordsNum);
+            long totalItems = 0;
+            foreach (var pair in len2counts)
+            {
+                long length = pair.Value;
+                if (totalItems <= rndItems && totalItems + length >= rndItems)
+                {
+                    return pair.Key;
+                }
+                totalItems += length;
+            }
+
+            return -1;
         }
 
         public void PrepareDataSet()

diff --git a/Seq2SeqSharp/Corpus/ParallelCorpus.cs b/Seq2SeqSharp/Corpus/ParallelCorpus.cs
@@ -315,23 +315,19 @@ public interface ICorpus<out T> : IEnumerable<T>
 
         public long GetNextLength(Dictionary<long, long> len2counts, long totalRecordsNum)
         {
-            long[] keys = len2counts.Keys.ToArray();
-            int rndIdx = rnd.Next(keys.Length);
-            return keys[rndIdx];
-
-            //long rndItems = rnd.NextInt64(totalRecordsNum);
-            //long totalItems = 0;
-            //foreach (var pair in len2counts)
-            //{
-            //    long length = pair.Value;
-            //    if (totalItems <= rndItems && totalItems + length >= rndItems)
-            //    {
-            //        return pair.Key;
-            //    }
-            //    totalItems += length;
-            //}
-
-            //return -1;
+            long rndItems = rnd.NextInt64(totalRecordsNum);
+            long totalItems = 0;
+            foreach (var pair in len2counts)
+            {
+                long length = pair.Value;
+                if (totalItems <= rndItems && totalItems + length >= rndItems)
+                {
+                    return pair.Key;
+                }
+                totalItems += length;
+            }
+
+            return -1;
         }
 
         public void PrepareDataSet()