Tests

sillsdev · Feb 7, 2024 · 9061de4 · 9061de4
1 parent aac5fec
commit 9061de4
Show file tree

Hide file tree

Showing 7 changed files with 254 additions and 0 deletions.
diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/NmtPreprocessBuildJobTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/NmtPreprocessBuildJobTests.cs
@@ -10,12 +10,17 @@ public void SetUp()
                 Path.Combine("..", "..", "..", "Services", "data", "paratext"),
                 Path.Combine(Path.GetTempPath(), "Project.zip")
             );
+            ZipFile.CreateFromDirectory(
+                Path.Combine("..", "..", "..", "Services", "data", "paratext2"),
+                Path.Combine(Path.GetTempPath(), "Project2.zip")
+            );
         }
 
         [TearDown]
         public void TearDown()
         {
             File.Delete(Path.Combine(Path.GetTempPath(), "Project.zip"));
+            File.Delete(Path.Combine(Path.GetTempPath(), "Project2.zip"));
         }
 
         [Test]
@@ -143,6 +148,131 @@ int numEntriesWrittenToPretranslate
             }
         }
 
+        [Test]
+        [TestCase("MAT", "1CH", 23, 4)]
+        [TestCase("NT;LEV", "1CH", 25, 4)]
+        [TestCase("OT", "MRK", 10, 0)]
+        [TestCase("OT", "MLK", 0, 0, true)]
+        public async Task BuildJobTest_Chapterlevel(
+            string trainOnBiblicalRangeChapters,
+            string pretranslateBiblicalRangeChapters,
+            int numLinesWrittenToTrain,
+            int numEntriesWrittenToPretranslate,
+            bool throwsException = false
+        )
+        {
+            using var env = new TestEnvironment();
+            var parser = new ScriptureRangeParser();
+
+            Corpus corpus1 = new Corpus();
+            if (throwsException)
+            {
+                Assert.Throws<ArgumentException>(() =>
+                {
+                    corpus1 = new Corpus
+                    {
+                        Id = "corpusId1",
+                        SourceLanguage = "en",
+                        TargetLanguage = "es",
+                        PretranslateAll = false,
+                        TrainOnAll = false,
+                        PretranslateChapters = parser
+                            .GetChapters(pretranslateBiblicalRangeChapters)
+                            .Select(kvp => (kvp.Key, kvp.Value.ToHashSet()))
+                            .ToDictionary(),
+                        TrainOnChapters = parser
+                            .GetChapters(trainOnBiblicalRangeChapters)
+                            .Select(kvp => (kvp.Key, kvp.Value.ToHashSet()))
+                            .ToDictionary(),
+                        PretranslateTextIds = new HashSet<string>(),
+                        TrainOnTextIds = new HashSet<string>(),
+                        SourceFiles = new List<CorpusFile>
+                        {
+                            new CorpusFile
+                            {
+                                TextId = "textId1",
+                                Format = FileFormat.Paratext,
+                                Location = Path.Combine(Path.GetTempPath(), "Project.zip")
+                            }
+                        },
+                        TargetFiles = new List<CorpusFile>
+                        {
+                            new CorpusFile
+                            {
+                                TextId = "textId1",
+                                Format = FileFormat.Paratext,
+                                Location = Path.Combine(Path.GetTempPath(), "Project2.zip")
+                            }
+                        }
+                    };
+                });
+                return;
+            }
+            else
+            {
+                corpus1 = new Corpus
+                {
+                    Id = "corpusId1",
+                    SourceLanguage = "en",
+                    TargetLanguage = "es",
+                    PretranslateAll = false,
+                    TrainOnAll = false,
+                    PretranslateChapters = parser
+                        .GetChapters(pretranslateBiblicalRangeChapters)
+                        .Select(kvp => (kvp.Key, kvp.Value.ToHashSet()))
+                        .ToDictionary(),
+                    TrainOnChapters = parser
+                        .GetChapters(trainOnBiblicalRangeChapters)
+                        .Select(kvp => (kvp.Key, kvp.Value.ToHashSet()))
+                        .ToDictionary(),
+                    PretranslateTextIds = new HashSet<string>(),
+                    TrainOnTextIds = new HashSet<string>(),
+                    SourceFiles = new List<CorpusFile>
+                    {
+                        new CorpusFile
+                        {
+                            TextId = "textId1",
+                            Format = FileFormat.Paratext,
+                            Location = Path.Combine(Path.GetTempPath(), "Project.zip")
+                        }
+                    },
+                    TargetFiles = new List<CorpusFile>
+                    {
+                        new CorpusFile
+                        {
+                            TextId = "textId1",
+                            Format = FileFormat.Paratext,
+                            Location = Path.Combine(Path.GetTempPath(), "Project2.zip")
+                        }
+                    }
+                };
+            }
+            var corpora = new ReadOnlyList<Corpus>(new List<Corpus> { corpus1 });
+            await env.BuildJob.RunAsync("engine1", "build1", corpora, "{\"use_key_terms\":false}", default);
+            using (var stream = await env.SharedFileService.OpenReadAsync("builds/build1/train.src.txt"))
+            {
+                using (var reader = new StreamReader(stream))
+                {
+                    //Split yields one more segment that there are new lines; thus, the "- 1"
+                    string text = reader.ReadToEnd();
+                    Assert.That(text.Split("\n").Length - 1, Is.EqualTo(numLinesWrittenToTrain), text);
+                }
+            }
+            using (var stream = await env.SharedFileService.OpenReadAsync("builds/build1/pretranslate.src.json"))
+            {
+                using (var reader = new StreamReader(stream))
+                {
+                    JsonArray? pretranslationJsonObject = JsonSerializer.Deserialize<JsonArray>(reader.ReadToEnd());
+                    Assert.NotNull(pretranslationJsonObject);
+                    Assert.That(
+                        pretranslationJsonObject!.ToList().Count,
+                        Is.EqualTo(numEntriesWrittenToPretranslate),
+                        JsonSerializer.Serialize(pretranslationJsonObject)
+                    );
+                }
+            }
+        }
+
         private class TestEnvironment : DisposableBase
         {
             public ISharedFileService SharedFileService { get; }

diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/41MATTen.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/41MATTen.SFM
@@ -0,0 +1,40 @@
+\id MAT - Test
+\h Matthew
+\mt Matthew
+\ip An introduction to Matthew
+\c 1
+\s Chapter One
+\v 1 Chapter \pn one\+pro WON\+pro*\pn*, verse one.\f + \fr 1:1: \ft This is a footnote.\f*
+\li1
+\v 2 \bd C\bd*hapter one,
+\li2 verse\f + \fr 1:2: \ft This is a footnote.\f* two.
+\v 3 Chapter one,
+\li2 verse three.
+\v 4 Chapter one, 
+\li2 verse four,
+\v 5 Chapter one,
+\li2 verse \fig Figure 1|src="image1.png" size="col" ref="1:5"\fig* five.
+\c 2 
+\s1 Chapter Two
+\p
+\v 1 Chapter \add two\add*, verse \f + \fr 2:1: \ft This is a footnote.\f*one.
+\v 2-3 Chapter two, verse \fm ∆\fm*two.
+\v 3-4a Chapter two, verse \w three|lemma\w*.
+\v 4b Chapter two, verse four.
+\p
+\v 6 Chapter two, verse \w six|strong="12345" \w*.
+\v 6 Bad verse.
+\v 5 Chapter two, verse five \rq (MAT 3:1)\rq*.
+\v 7a Chapter two, verse seven A, 
+\s Section header
+\p
+\v 7b verse seven B.
+\p
+\v 8 This is a list:
+\b
+\tr \tc1
+\v 9 Chapter\tcr2 2\tc3 verse\tcr4 9
+\tr \tc1-2
+\v 10 \tc3-4 Chapter 2 verse 10
+\v 11-12
+\restore restore information
diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/42MRKTen.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/42MRKTen.SFM
@@ -0,0 +1,4 @@
+\id MRK - Test
+\h Mark
+\mt Mark
+\ip An introduction to Mark
diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/ProjectBiblicalTerms.xml b/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/ProjectBiblicalTerms.xml
@@ -0,0 +1,6 @@
+<BiblicalTermsList>
+  <Term Id="Abba">
+  <Category>PN</Category>
+    <Gloss>Abba</Gloss>
+  </Term>
+</BiblicalTermsList>
diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/Settings.xml b/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/Settings.xml
@@ -0,0 +1,34 @@
+<ScriptureText>
+  <StyleSheet>usfm.sty</StyleSheet>
+  <Versification>4</Versification>
+  <LanguageIsoCode>en:::</LanguageIsoCode>
+  <Language>English</Language>
+  <MinParatextVersion>8.0.100.76</MinParatextVersion>
+  <FullName>Test2</FullName>
+  <Encoding>65001</Encoding>
+  <Editable>T</Editable>
+  <Copyright />
+  <NormalizationForm>NFC</NormalizationForm>
+  <Name>Ten</Name>
+  <Guid>a7e0b3ce0200736062f9f810a444dbfbe64aca35</Guid>
+  <DefaultFont>Charis SIL</DefaultFont>
+  <DefaultFontSize>12</DefaultFontSize>
+  <FontFeatures />
+  <HtmlLanguage />
+  <AssociatedLexicalProject />
+  <FileNameBookNameForm>41MAT</FileNameBookNameForm>
+  <FileNamePrePart />
+  <FileNamePostPart>Ten.SFM</FileNamePostPart>
+  <BiblicalTermsListSetting>Project:Ten:ProjectBiblicalTerms.xml</BiblicalTermsListSetting>
+  <MatchBasedOnStems>F</MatchBasedOnStems>
+  <AllowReadAccess>F</AllowReadAccess>
+  <AllowSharingWithSLDR>F</AllowSharingWithSLDR>
+  <Visibility>Public</Visibility>
+  <TranslationInfo>Standard::</TranslationInfo>
+  <EncodingConverter />
+  <UsfmVersion>3</UsfmVersion>
+  <ParallelPassagesBooks>000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000</ParallelPassagesBooks>
+  <BooksPresent>000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000</BooksPresent>
+  <BibleModuleAssociations />
+  <Naming PrePart="" PostPart="Ten.SFM" BookNameForm="41MAT" />
+</ScriptureText>
diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/TermRenderings.xml b/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/TermRenderings.xml
@@ -0,0 +1,9 @@
+<TermRenderingsList>
+  <TermRendering Id="Abba" Guess="false">
+    <Renderings>Abba</Renderings>
+    <Glossary />
+    <Changes />
+    <Notes />
+    <Denials />
+  </TermRendering>
+</TermRenderingsList>
diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/custom.vrs b/tests/SIL.Machine.AspNetCore.Tests/Services/data/paratext2/custom.vrs
@@ -0,0 +1,31 @@
+# custom.vrs 
+
+LEV 14:56
+ROM 14:26
+REV 12:17
+TOB 5:22
+TOB 10:12
+SIR 23:28
+ESG 1:22
+ESG 3:15
+ESG 5:14
+ESG 8:17
+ESG 10:14
+SIR 33:33
+SIR 41:24
+BAR 1:22
+4MA 7:25
+4MA 12:20
+
+# deliberately missing verses
+-ROM 16:26
+-ROM 16:27
+-3JN 1:15
+-S3Y 1:49
+-ESG 4:6
+-ESG 9:5
+-ESG 9:30
+
+LEV 14:55 = LEV 14:55
+LEV 14:55 = LEV 14:56
+LEV 14:56 = LEV 14:57