From 67102badf5fd18c521ddd429ff6961192657f2ae Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 3 Jun 2024 23:31:11 -0700 Subject: [PATCH 001/105] Improve HTML integrity check to catch tag ordering issues The checker will now catch the following errors: - source: hello and target: hello - If tags are crossing: --- .../HtmlTagIntegrityChecker.java | 30 ++++++++++++++++++- .../HtmlTagIntegrityCheckerTest.java | 14 +++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java index 8f61b69441..09426ceca1 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -20,7 +21,7 @@ public class HtmlTagIntegrityChecker extends RegexIntegrityChecker { @Override public String getRegex() { - return "(<\\w+(\\s+\\w+(\\s*=\\s*('(\\\'|[^'])*?'|\"(\\\"|[^\"])*?\")))*?>|)"; + return "(<\\w+(\\s+\\w+(\\s*=\\s*('([^']*?)'|\"([^\"]*?)\"))?)*\\s*/?>|)"; } @Override @@ -38,6 +39,11 @@ public void check(String sourceContent, String targetContent) throws IntegrityCh || !targetHtmlTags.containsAll(sourceHtmlTags)) { throw new HtmlTagIntegrityCheckerException("HTML tags in source and target are different"); } + + logger.debug("Make sure the target tags are in valid order"); + if (!isValidTagOrder(targetHtmlTags)) { + throw new HtmlTagIntegrityCheckerException("HTML tags in target are not in valid order"); + } } /** @@ -60,4 +66,26 @@ List getHtmlTags(String string) { return tags; } + + public static boolean isValidTagOrder(List tags) { + + boolean res = true; + + ArrayDeque stack = new ArrayDeque<>(); + + for (String tag : tags) { + if (!tag.startsWith("%1 files"; + String target = "Il y a %1 fichiers et %2 dossiers"; + checker.check(source, target); + } + + @Test(expected = HtmlTagIntegrityCheckerException.class) + public void testHtmlTagCheckWorksWhenCrossing() { + String source = "There are %1 files and %2 folders"; + String target = "Il y a %1 fichiers et %2 dossiers"; + checker.check(source, target); + } + @Test(expected = HtmlTagIntegrityCheckerException.class) public void testHtmlTagCheckWorksWhenTagIsModified() { String source = "There are %1 files and %2 folders"; From fea0e2c5eb111d4b984ccd28ee245e321a031ab3 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 13:34:48 -0700 Subject: [PATCH 002/105] Add an integrity check for Markdown links Checks for Markdown links only, no other Markdown formatting is checked. --- .../rest/entity/IntegrityCheckerType.java | 3 +- .../IntegrityCheckerType.java | 3 +- .../MarkdownLinkIntegrityChecker.java | 35 ++++++++++++ ...MarkdownLinkIntegrityCheckerException.java | 7 +++ .../MarkdownLinkIntegrityCheckerTest.java | 54 +++++++++++++++++++ 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerException.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerTest.java diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java index ff6e3e33a2..71733a36e4 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java @@ -19,5 +19,6 @@ public enum IntegrityCheckerType { HTML_TAG, ELLIPSIS, BACKQUOTE, - EMPTY_TARGET_NOT_EMPTY_SOURCE; + EMPTY_TARGET_NOT_EMPTY_SOURCE, + MARKDOWN_LINKS; } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java index 0dbc6c02b7..6c8ac7e7bc 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java @@ -21,7 +21,8 @@ public enum IntegrityCheckerType { HTML_TAG(HtmlTagIntegrityChecker.class.getName()), ELLIPSIS(EllipsisIntegrityChecker.class.getName()), BACKQUOTE(BackquoteIntegrityChecker.class.getName()), - EMPTY_TARGET_NOT_EMPTY_SOURCE(EmptyTargetNotEmptySourceIntegrityChecker.class.getName()); + EMPTY_TARGET_NOT_EMPTY_SOURCE(EmptyTargetNotEmptySourceIntegrityChecker.class.getName()), + MARKDOWN_LINKS(MarkdownLinkIntegrityChecker.class.getName()); String className; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java new file mode 100644 index 0000000000..becdd96959 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java @@ -0,0 +1,35 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.regex.Matcher; + +public class MarkdownLinkIntegrityChecker extends RegexIntegrityChecker { + + @Override + public String getRegex() { + return "\\[(?.+?)]\\((?.+?)\\)"; + } + + @Override + Set getPlaceholders(String string) { + Set placeholders = new LinkedHashSet<>(); + + if (string != null) { + Matcher matcher = getPattern().matcher(string); + while (matcher.find()) { + placeholders.add("[%s](%s)".formatted("--translatable--", matcher.group("url"))); + } + } + return placeholders; + } + + @Override + public void check(String content, String target) { + try { + super.check(content, target); + } catch (RegexCheckerException ex) { + throw new MarkdownLinkIntegrityCheckerException("Variable types do not match."); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerException.java new file mode 100644 index 0000000000..f6a5db5907 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class MarkdownLinkIntegrityCheckerException extends IntegrityCheckException { + public MarkdownLinkIntegrityCheckerException(String message) { + super(message); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerTest.java new file mode 100644 index 0000000000..8ec6a5614e --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerTest.java @@ -0,0 +1,54 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; + +import java.util.Set; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public class MarkdownLinkIntegrityCheckerTest { + + MarkdownLinkIntegrityChecker checker = new MarkdownLinkIntegrityChecker(); + + @Test + public void testLink() { + String source = "This is [a link](http://localhost)"; + String target = "c'est [un lien](http://localhost)"; + checker.check(source, target); + } + + @Test + public void testBrokenLinkExtraSpace() { + String source = "This is [a link](http://localhost)"; + String target = "c'est [un lien] (http://localhost)"; + assertThrowsExactly( + MarkdownLinkIntegrityCheckerException.class, () -> checker.check(source, target)); + } + + @Test + public void testBrokenLinkTranslatedUrl() { + String source = "This is [a link](http://localhost)"; + String target = "c'est [un lien] (http://translated)"; + assertThrowsExactly( + MarkdownLinkIntegrityCheckerException.class, () -> checker.check(source, target)); + } + + @Test + public void testReorder() { + String source = + "This is the [first link](http://localhost/1) and this is the [second link](http://localhost/2)"; + String target = + "C'est le [dieuxieme lien](http://localhost/2) et c'est le [premier lien](http://localhost/1)"; + checker.check(source, target); + } + + @Test + public void getPlaceholder() { + String source = + "This is [a link](http://localhost/1) and another link [2nd link](http://localhost/2)"; + Set placeholders = checker.getPlaceholders(source); + Assertions.assertThat(placeholders) + .containsExactly( + "[--translatable--](http://localhost/1)", "[--translatable--](http://localhost/2)"); + } +} From 84f3cc26ae84d0cfd2b7f4373cd50e7c2613d58a Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 13:46:14 -0700 Subject: [PATCH 003/105] Add option to remove comments from iOS .strings files in SimpleFileEditorCommand --- .../cli/command/SimpleFileEditorCommand.java | 45 ++++++++++++++++++ .../command/SimpleFileEditorCommandTest.java | 15 ++++++ .../expected/en.lproj/Localizable.strings | Bin 0 -> 396 bytes .../input/en.lproj/Localizable.strings | Bin 0 -> 1100 bytes 4 files changed, 60 insertions(+) create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/expected/en.lproj/Localizable.strings create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/input/en.lproj/Localizable.strings diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java index 921b9645fe..a432892729 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java @@ -41,6 +41,8 @@ public class SimpleFileEditorCommand extends Command { /** logger */ static Logger logger = LoggerFactory.getLogger(SimpleFileEditorCommand.class); + static final String COMMENTS_PATTERN = "(?s)/*\\*.*?\\*/"; + @Autowired ConsoleWriter consoleWriter; @Autowired CommandHelper commandHelper; @@ -77,6 +79,12 @@ public class SimpleFileEditorCommand extends Command { description = "To remove location from both Mac strings and string dict files") boolean removeUsagesInMacStrings = false; + @Parameter( + names = {"--macstrings-remove-comments"}, + required = false, + description = "To remove comments from both Mac strings") + boolean removeCommentsInMacStrings = false; + @Parameter( names = {"--json-indent"}, required = false, @@ -107,6 +115,10 @@ public void execute() throws CommandException { removeUsagesInMacStringsAndStringDict(); } + if (removeCommentsInMacStrings) { + removeCommentsInMacStrings(); + } + if (indentJson) { indentJsonFiles(); } @@ -184,6 +196,39 @@ void removeUsagesInMacStringsAndStringDict() throws CommandException { }); } + void removeCommentsInMacStrings() throws CommandException { + MacStringsFileType fileType = new MacStringsFileType(); + + commandDirectories + .listFilesWithExtensionInSourceDirectory( + fileType.getSourceFileExtension(), fileType.getTargetFileExtension()) + .stream() + .filter( + path -> + fileType + .getSourceFilePattern() + .getPattern() + .matcher(path.toAbsolutePath().toString()) + .matches() + || fileType + .getTargetFilePattern() + .getPattern() + .matcher(path.toAbsolutePath().toString()) + .matches()) + .filter(getInputFilterMatch()) + .forEach( + inputPath -> { + consoleWriter + .a(" - Remove comments: ") + .fg(Ansi.Color.MAGENTA) + .a(inputPath.toString()) + .print(); + String modifiedContent = + commandHelper.getFileContent(inputPath).replaceAll(COMMENTS_PATTERN, ""); + writeOutputFile(inputPath, modifiedContent); + }); + } + Predicate getInputFilterMatch() { return path -> inputFilterPattern == null diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest.java index e3f1f4facb..da6531ab4a 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest.java @@ -47,4 +47,19 @@ public void macStringsRemoveUsages() { checkExpectedGeneratedResources(); } + + @Test + public void macStringsRemoveComments() { + + getL10nJCommander() + .run( + "simple-file-editor", + "-i", + getInputResourcesTestDir().getAbsolutePath(), + "-o", + getTargetTestDir().getAbsolutePath(), + "--macstrings-remove-comments"); + + checkExpectedGeneratedResources(); + } } diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/expected/en.lproj/Localizable.strings b/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/expected/en.lproj/Localizable.strings new file mode 100644 index 0000000000000000000000000000000000000000..c94f2639d180a0de2d2664ed71cd9d37717af384 GIT binary patch literal 396 zcmaiwO%8%E5QV=rr%!Il{Ax1C-1NGxMu{`taNfp z8VY(<$v~|xr_z1VyQk%>Y0HUD^MJ8e1rC#fclJfaeyD84?ZnxUjmML0qw=!vFvP literal 0 HcmV?d00001 diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/input/en.lproj/Localizable.strings b/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/input/en.lproj/Localizable.strings new file mode 100644 index 0000000000000000000000000000000000000000..dab1ad8c2c6458a3985efa549cf2472ffb5b7723 GIT binary patch literal 1100 zcmb`G!A`?45Jcykuds5=fznF7l){m(KtdJ)T0zn(2?u^2cx&UPtr|)xvJ%_t@yvSG z{`?rJkkgZvYSgM@{GgR8+N0_)FEmo7Tw`6*?pa^RgZ7L%Pz|w?zGdFPvnINPqb8oi zcg>gUZo^4i~A%!m1Nn7xGzQzjX)*yi7R z{3gp*H}IzQ)SDxwae)&v&OVHo9Kg)UGv>Ch)4Msvp9&T8snl-1y>DYp$t~i}^T6r1 zXHOoS+V6TcIpEo;j0vBZlh^;jJyfHOC^*e*twinHT^7 literal 0 HcmV?d00001 From 26acb6c7812a20b5310ea9032a48bed74430f239 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 6 May 2024 13:37:10 -0700 Subject: [PATCH 004/105] Add bn-BD locale --- webapp/src/main/resources/db/hsql/data.sql | 1 + webapp/src/main/resources/db/migration/V66__Add_bn_BD.sql | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 webapp/src/main/resources/db/migration/V66__Add_bn_BD.sql diff --git a/webapp/src/main/resources/db/hsql/data.sql b/webapp/src/main/resources/db/hsql/data.sql index 8fe1f5221c..b48f9d22e6 100644 --- a/webapp/src/main/resources/db/hsql/data.sql +++ b/webapp/src/main/resources/db/hsql/data.sql @@ -989,6 +989,7 @@ insert into locale (id, bcp47_tag) values (1637, 'zh-Hant-alt-long'); insert into locale (id, bcp47_tag) values (1638, 'zun'); insert into locale (id, bcp47_tag) values (1639, 'zxx'); insert into locale (id, bcp47_tag) values (1640, 'zza'); +insert into locale (id, bcp47_tag) values (1641, 'bn-BD'); insert into plural_form (id, name) values (1, 'one'); diff --git a/webapp/src/main/resources/db/migration/V66__Add_bn_BD.sql b/webapp/src/main/resources/db/migration/V66__Add_bn_BD.sql new file mode 100644 index 0000000000..6fd1882600 --- /dev/null +++ b/webapp/src/main/resources/db/migration/V66__Add_bn_BD.sql @@ -0,0 +1,4 @@ +insert into locale (id, bcp47_tag) values (1641, 'bn-BD'); + +insert into plural_form_for_locale (locale_id, plural_form_id) values (1641, 1); +insert into plural_form_for_locale (locale_id, plural_form_id) values (1641, 5); \ No newline at end of file From 19a36f712ac4400b48f476e3e39741a9b2d3e4eb Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 31 Jul 2024 17:34:05 -0700 Subject: [PATCH 005/105] Add RepositoryTmTranslateCommand to translate repository by source match Simple, low perf implementation to copy translation between repository. Can be useful to use the ImportLocalizedAssetCommand when some strings have been moved around. --- .../command/RepositoryTmTranslateCommand.java | 190 +++++++++++ .../mojito/rest/client/TextUnitClient.java | 313 ++++++++++++++++++ .../mojito/rest/script/LeverageScript.java | 138 ++++++++ 3 files changed, 641 insertions(+) create mode 100644 cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java create mode 100644 restclient/src/main/java/com/box/l10n/mojito/rest/client/TextUnitClient.java create mode 100644 restclient/src/test/java/com/box/l10n/mojito/rest/script/LeverageScript.java diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java new file mode 100644 index 0000000000..9a495adfa1 --- /dev/null +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java @@ -0,0 +1,190 @@ +package com.box.l10n.mojito.cli.command; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.box.l10n.mojito.cli.command.param.Param; +import com.box.l10n.mojito.cli.console.ConsoleWriter; +import com.box.l10n.mojito.rest.client.RepositoryClient; +import com.box.l10n.mojito.rest.client.TextUnitClient; +import com.box.l10n.mojito.rest.entity.Locale; +import com.box.l10n.mojito.rest.entity.PollableTask; +import com.box.l10n.mojito.rest.entity.Repository; +import com.box.l10n.mojito.rest.entity.RepositoryLocale; +import org.fusesource.jansi.Ansi; +import org.fusesource.jansi.Ansi.Color; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Command to populate untranslated string with translation from another repository translation memory. + * This meant to process small volumes but can be helpful to combine the with the import localized files command. + *

+ * Matches the text units by the source. Use text unit name and comment match as priority (moved assets), else just take the + * newest translation match. Has an option to not import string that have same source and target to avoid + * copy translation that might be from importing partly translated files. The drawback is that it could skip + * string that should be imported but the benefits is that the localized files will be same regardless. + *

+ * Limited to translating at most 1000 strings in a run + *

+ * + * @author jaurambault + */ +@Component +@Scope("prototype") +@Parameters( + commandNames = {"repository-translate"}, + commandDescription = "Translate a repository using another repository TM. Only untranslated text units" + + " will be processed, no overriding. Priorities text unit that have same and comment, else just look" + + "at the newest match") +public class RepositoryTmTranslateCommand extends Command { + + /** + * logger + */ + static Logger logger = LoggerFactory.getLogger(RepositoryTmTranslateCommand.class); + + @Autowired + ConsoleWriter consoleWriter; + + @Parameter( + names = {Param.SOURCE_REPOSITORY_LONG, Param.SOURCE_REPOSITORY_SHORT}, + arity = 1, + required = false, + description = Param.SOURCE_REPOSITORY_DESCRIPTION) + String sourceRepositoryParam; + + @Parameter( + names = {Param.TARGET_REPOSITORY_LONG, Param.TARGET_REPOSITORY_SHORT}, + arity = 1, + required = true, + description = Param.TARGET_REPOSITORY_DESCRIPTION) + String targetRepositoryParam; + + @Parameter( + names = {"--import-source-equals-target"}, + arity = 0, + description = "Import the translation if the source is equal to the target") + boolean importSourceEqualsTargetParam = false; + + @Parameter( + names = {Param.REPOSITORY_LOCALES_LONG, Param.REPOSITORY_LOCALES_SHORT}, + variableArity = true, + required = false, + description = "List of locales (bcp47 tags) to translate") + List localesParam = null; + + @Autowired + CommandHelper commandHelper; + + @Autowired + TextUnitClient textUnitClient; + + @Autowired + RepositoryClient repositoryClient; + + @Override + public void execute() throws CommandException { + + if (sourceRepositoryParam == null) { + sourceRepositoryParam = targetRepositoryParam; + } + + consoleWriter + .newLine() + .a("Translate repository: ") + .fg(Color.CYAN) + .a(targetRepositoryParam) + .reset() + .a(" with text units from repository: ") + .fg(Color.CYAN) + .a(sourceRepositoryParam) + .println(2); + + Repository sourceRepository = commandHelper.findRepositoryByName(sourceRepositoryParam); + Repository targetRepository = commandHelper.findRepositoryByName(targetRepositoryParam); + + List locales = localesParam; + if (locales == null) { + locales = targetRepository.getRepositoryLocales().stream().filter(rl -> rl.getParentLocale() != null).map(RepositoryLocale::getLocale).map(Locale::getBcp47Tag).toList(); + } + + for (String locale : locales) { + consoleWriter.a("Processing locale: ").fg(Color.MAGENTA).a(locale).println(); + List textUnitsToSave = new ArrayList<>(); + + List untranslatedTextUnitsForLocale = getUntranslatedTextUnitsForLocale(targetRepository.getName(), locale); + + for (TextUnitClient.TextUnit textUnit : untranslatedTextUnitsForLocale) { + consoleWriter.a("Processing source: ").fg(Color.CYAN).a(textUnit.source()).println(); + List existingTranslationWithSameSource = getTranslationsWithExactSource(sourceRepository.getName(), locale, textUnit.source()); + + Optional match = getMatchByNameAndComment(textUnit.source(), textUnit.comment(), existingTranslationWithSameSource); + if (match.isPresent()) { + match = getMatchByNewest(existingTranslationWithSameSource); + } + + if (match.isPresent()) { + var translation = match.get(); + consoleWriter.a("Found match: ").fg(Color.CYAN).a(translation.target()).println(); + + if (translation.source().equals(translation.target())) { + if (importSourceEqualsTargetParam) { + textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); + } else { + consoleWriter.a("Skip import, source = target").println(); + } + } else { + textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); + } + } else { + consoleWriter.a("No match found").println(); + } + consoleWriter.println(); + } + + consoleWriter.a("Saving translations for locale: ").fg(Color.MAGENTA).a(locale).reset().a("Text unit count: ").fg(Color.CYAN).a(textUnitsToSave.size()).println(); + TextUnitClient.ImportTextUnitsBatch importTextUnitsBatch = new TextUnitClient.ImportTextUnitsBatch(false, true, textUnitsToSave); + PollableTask pollableTask = textUnitClient.importTextUnitBatch(importTextUnitsBatch); + commandHelper.waitForPollableTask(pollableTask.getId()); + } + consoleWriter.fg(Ansi.Color.GREEN).newLine().a("Finished").println(2); + } + + private List getTranslationsWithExactSource(String repositoryName, String locale, String source) { + TextUnitClient.TextUnitSearchBody textUnitSearchBody = new TextUnitClient.TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.TRANSLATED); + textUnitSearchBody.setSource(source); + textUnitSearchBody.setLimit(20); + return textUnitClient.searchTextUnits(textUnitSearchBody); + + } + + private List getUntranslatedTextUnitsForLocale(String repositoryName, String locale) { + TextUnitClient.TextUnitSearchBody textUnitSearchBody = new TextUnitClient.TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setUsedFilter(TextUnitClient.UsedFilter.USED); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.UNTRANSLATED); + textUnitSearchBody.setLimit(1000); + return textUnitClient.searchTextUnits(textUnitSearchBody); + } + + private Optional getMatchByNewest(List candidates) { + return candidates.stream().max(Comparator.comparingLong(TextUnitClient.TextUnit::createdDate)); + } + + private static Optional getMatchByNameAndComment(String name, String comment, List candidates) { + return candidates.stream().filter(m -> Objects.equals(name, m.name()) && Objects.equals(comment, m.comment())).max(Comparator.comparingLong(TextUnitClient.TextUnit::createdDate)); + } +} diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/client/TextUnitClient.java b/restclient/src/main/java/com/box/l10n/mojito/rest/client/TextUnitClient.java new file mode 100644 index 0000000000..4abebde773 --- /dev/null +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/client/TextUnitClient.java @@ -0,0 +1,313 @@ +package com.box.l10n.mojito.rest.client; + +import com.box.l10n.mojito.rest.entity.PollableTask; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +@Component +public class TextUnitClient extends BaseClient { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(TextUnitClient.class); + + @Override + public String getEntityName() { + return "textunits"; + } + + public List searchTextUnits(TextUnitSearchBody textUnitSearchBody) { + TextUnit[] search = + authenticatedRestTemplate.postForObject( + getBasePathForEntity() + "/search", textUnitSearchBody, TextUnit[].class); + return Arrays.asList(search); + } + + public PollableTask importTextUnitBatch(ImportTextUnitsBatch importTextUnitsBatch) { + return authenticatedRestTemplate.postForObject( + getBasePath() + "/textunitsBatch", importTextUnitsBatch, PollableTask.class); + } + + public record ImportTextUnitsBatch( + boolean integrityCheckSkipped, + boolean integrityCheckKeepStatusIfFailedAndSameTarget, + List textUnits) {} + + public enum UsedFilter { + USED, + UNUSED + } + + public enum SearchType { + EXACT, + CONTAINS, + ILIKE + } + + public enum StatusFilter { + ALL, + TRANSLATED, + UNTRANSLATED, + TRANSLATED_AND_NOT_REJECTED, + APPROVED_OR_NEEDS_REVIEW_AND_NOT_REJECTED, + APPROVED_AND_NOT_REJECTED, + FOR_TRANSLATION, + REVIEW_NEEDED, + REVIEW_NOT_NEEDED, + TRANSLATION_NEEDED, + REJECTED, + NOT_REJECTED, + } + + public record TextUnit( + @JsonProperty("tmTextUnitId") Long tmTextUnitId, + @JsonProperty("tmTextUnitVariantId") Long tmTextUnitVariantId, + @JsonProperty("localeId") Long localeId, + @JsonProperty("name") String name, + @JsonProperty("source") String source, + @JsonProperty("comment") String comment, + @JsonProperty("target") String target, + @JsonProperty("targetLocale") String targetLocale, + @JsonProperty("targetComment") String targetComment, + @JsonProperty("assetId") Long assetId, + @JsonProperty("lastSuccessfulAssetExtractionId") Long lastSuccessfulAssetExtractionId, + @JsonProperty("assetExtractionId") Long assetExtractionId, + @JsonProperty("tmTextUnitCurrentVariantId") Long tmTextUnitCurrentVariantId, + @JsonProperty("status") Status status, + @JsonProperty("includedInLocalizedFile") Boolean includedInLocalizedFile, + @JsonProperty("createdDate") Long createdDate, + @JsonProperty("assetDeleted") Boolean assetDeleted, + @JsonProperty("pluralForm") String pluralForm, + @JsonProperty("pluralFormOther") String pluralFormOther, + @JsonProperty("repositoryName") String repositoryName, + @JsonProperty("assetPath") String assetPath, + @JsonProperty("assetTextUnitId") Long assetTextUnitId, + @JsonProperty("tmTextUnitCreatedDate") Long tmTextUnitCreatedDate, + @JsonProperty("doNotTranslate") Boolean doNotTranslate, + @JsonProperty("translated") Boolean translated, + @JsonProperty("used") Boolean used) { + + public TextUnit withTarget(String target, Status status) { + return new TextUnit( + tmTextUnitId, + tmTextUnitVariantId, + localeId, + name, + source, + comment, + target, + targetLocale, + targetComment, + assetId, + lastSuccessfulAssetExtractionId, + assetExtractionId, + tmTextUnitCurrentVariantId, + status, + includedInLocalizedFile, + createdDate, + assetDeleted, + pluralForm, + pluralFormOther, + repositoryName, + assetPath, + assetTextUnitId, + tmTextUnitCreatedDate, + doNotTranslate, + translated, + used); + } + } + + public enum Status { + TRANSLATION_NEEDED, + REVIEW_NEEDED, + APPROVED + } + + public static class TextUnitSearchBody { + List repositoryIds; + List repositoryNames; + List tmTextUnitIds; + String name; + String source; + String target; + String assetPath; + String pluralFormOther; + boolean pluralFormFiltered = true; + boolean pluralFormExcluded = false; + SearchType searchType = SearchType.EXACT; + List localeTags; + UsedFilter usedFilter; + StatusFilter statusFilter; + Boolean doNotTranslateFilter; + ZonedDateTime tmTextUnitCreatedBefore; + ZonedDateTime tmTextUnitCreatedAfter; + Long branchId; + Integer limit = 10; + Integer offset = 0; + + public List getRepositoryIds() { + return repositoryIds; + } + + public void setRepositoryIds(List repositoryIds) { + this.repositoryIds = repositoryIds; + } + + public List getRepositoryNames() { + return repositoryNames; + } + + public void setRepositoryNames(List repositoryNames) { + this.repositoryNames = repositoryNames; + } + + public List getTmTextUnitIds() { + return tmTextUnitIds; + } + + public void setTmTextUnitIds(List tmTextUnitIds) { + this.tmTextUnitIds = tmTextUnitIds; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getTarget() { + return target; + } + + public void setTarget(String target) { + this.target = target; + } + + public String getAssetPath() { + return assetPath; + } + + public void setAssetPath(String assetPath) { + this.assetPath = assetPath; + } + + public String getPluralFormOther() { + return pluralFormOther; + } + + public void setPluralFormOther(String pluralFormOther) { + this.pluralFormOther = pluralFormOther; + } + + public boolean isPluralFormFiltered() { + return pluralFormFiltered; + } + + public void setPluralFormFiltered(boolean pluralFormFiltered) { + this.pluralFormFiltered = pluralFormFiltered; + } + + public boolean isPluralFormExcluded() { + return pluralFormExcluded; + } + + public void setPluralFormExcluded(boolean pluralFormExcluded) { + this.pluralFormExcluded = pluralFormExcluded; + } + + public SearchType getSearchType() { + return searchType; + } + + public void setSearchType(SearchType searchType) { + this.searchType = searchType; + } + + public List getLocaleTags() { + return localeTags; + } + + public void setLocaleTags(List localeTags) { + this.localeTags = localeTags; + } + + public UsedFilter getUsedFilter() { + return usedFilter; + } + + public void setUsedFilter(UsedFilter usedFilter) { + this.usedFilter = usedFilter; + } + + public StatusFilter getStatusFilter() { + return statusFilter; + } + + public void setStatusFilter(StatusFilter statusFilter) { + this.statusFilter = statusFilter; + } + + public Boolean getDoNotTranslateFilter() { + return doNotTranslateFilter; + } + + public void setDoNotTranslateFilter(Boolean doNotTranslateFilter) { + this.doNotTranslateFilter = doNotTranslateFilter; + } + + public ZonedDateTime getTmTextUnitCreatedBefore() { + return tmTextUnitCreatedBefore; + } + + public void setTmTextUnitCreatedBefore(ZonedDateTime tmTextUnitCreatedBefore) { + this.tmTextUnitCreatedBefore = tmTextUnitCreatedBefore; + } + + public ZonedDateTime getTmTextUnitCreatedAfter() { + return tmTextUnitCreatedAfter; + } + + public void setTmTextUnitCreatedAfter(ZonedDateTime tmTextUnitCreatedAfter) { + this.tmTextUnitCreatedAfter = tmTextUnitCreatedAfter; + } + + public Long getBranchId() { + return branchId; + } + + public void setBranchId(Long branchId) { + this.branchId = branchId; + } + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + public Integer getOffset() { + return offset; + } + + public void setOffset(Integer offset) { + this.offset = offset; + } + } +} diff --git a/restclient/src/test/java/com/box/l10n/mojito/rest/script/LeverageScript.java b/restclient/src/test/java/com/box/l10n/mojito/rest/script/LeverageScript.java new file mode 100644 index 0000000000..aa3a11f96a --- /dev/null +++ b/restclient/src/test/java/com/box/l10n/mojito/rest/script/LeverageScript.java @@ -0,0 +1,138 @@ +package com.box.l10n.mojito.rest.script; + +import com.box.l10n.mojito.rest.client.PollableTaskClient; +import com.box.l10n.mojito.rest.client.RepositoryClient; +import com.box.l10n.mojito.rest.client.TextUnitClient; +import com.box.l10n.mojito.rest.client.TextUnitClient.TextUnit; +import com.box.l10n.mojito.rest.client.TextUnitClient.TextUnitSearchBody; +import com.box.l10n.mojito.rest.client.exception.RepositoryNotFoundException; +import com.box.l10n.mojito.rest.entity.PollableTask; +import com.box.l10n.mojito.rest.resttemplate.AuthenticatedRestTemplateTest; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@Configuration +@SpringBootTest( + classes = { + AuthenticatedRestTemplateTest.class, + RepositoryClient.class, + TextUnitClient.class, + PollableTaskClient.class + }) +@TestPropertySource(locations = "file:/Users/ja/.l10n/config/script/application.properties") +@EnableConfigurationProperties +// @Ignore +public class LeverageScript { + + static Logger logger = LoggerFactory.getLogger(LeverageScript.class); + + @Autowired TextUnitClient textUnitClient; + + @Autowired PollableTaskClient pollableTaskClient; + + @Autowired RepositoryClient repositoryClient; + + @Test + public void script() throws RepositoryNotFoundException { + + var targetRepository = "ios"; + var sourceRepository = "zzzzzzzzz_ios"; + var skipSourceEqTarget = true; + var locales = List.of("fr"); + + List textUnitsToSave = new ArrayList<>(); + + for (String locale : locales) { + logger.info("Processing locale: {}", locale); + List untranslatedTextUnitsForLocale = + getUntranslatedTextUnitsForLocale(targetRepository, locale); + + for (TextUnit textUnit : untranslatedTextUnitsForLocale) { + List existingTranslationWithSameSource = + getTranslationsWithExactSource(sourceRepository, locale, textUnit.source()); + Optional match = + getMatchByNameAndComment( + textUnit.source(), textUnit.comment(), existingTranslationWithSameSource); + + if (match.isPresent()) { + logger.info( + "Found a match (tuvid: {}) by name and comment for:\n {}\ntranslation is: {}", + match.get().tmTextUnitVariantId(), + textUnit.source(), + match.get().target()); + } else { + match = getMatchByNewest(existingTranslationWithSameSource); + if (match.isPresent()) { + logger.info( + "Found match (tuvid: {}) by newest date for:\n {}\ntranslation is: {}", + match.get().tmTextUnitVariantId(), + textUnit.source(), + match.get().target()); + } + } + + if (match.isPresent()) { + logger.info("found match for source: {}", textUnit.source()); + var translation = match.get(); + if (skipSourceEqTarget && translation.source().equals(translation.target())) { + logger.info("skipping because source and target are the same"); + } + textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); + } else { + logger.info("No match found for source: {}", textUnit.source()); + } + } + } + + TextUnitClient.ImportTextUnitsBatch importTextUnitsBatch = + new TextUnitClient.ImportTextUnitsBatch(false, true, textUnitsToSave); + PollableTask pollableTask = textUnitClient.importTextUnitBatch(importTextUnitsBatch); + pollableTaskClient.waitForPollableTask(pollableTask.getId()); + } + + private List getTranslationsWithExactSource( + String repositoryName, String locale, String source) { + TextUnitSearchBody textUnitSearchBody = new TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.TRANSLATED); + textUnitSearchBody.setSource(source); + textUnitSearchBody.setLimit(20); + return textUnitClient.searchTextUnits(textUnitSearchBody); + } + + private List getUntranslatedTextUnitsForLocale(String repositoryName, String locale) { + TextUnitSearchBody textUnitSearchBody = new TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setUsedFilter(TextUnitClient.UsedFilter.USED); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.UNTRANSLATED); + textUnitSearchBody.setLimit(1000); + return textUnitClient.searchTextUnits(textUnitSearchBody); + } + + private Optional getMatchByNewest(List candidates) { + return candidates.stream().max(Comparator.comparingLong(TextUnit::createdDate)); + } + + private static Optional getMatchByNameAndComment( + String name, String comment, List candidates) { + return candidates.stream() + .filter(m -> Objects.equals(name, m.name()) && Objects.equals(comment, m.comment())) + .max(Comparator.comparingLong(TextUnit::createdDate)); + } +} From d644419d1c8aea29b22c8d90b904f3f26a3bf676 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 15:32:08 -0700 Subject: [PATCH 006/105] Add Dockerfile for Java 21 and document K8s integration --- docker/Dockerfile-bk21 | 43 ++++++++++++++++++++++++++++++++++++++++++ docker/readme.md | 39 +++++++++++++++++++++++++++++--------- 2 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 docker/Dockerfile-bk21 diff --git a/docker/Dockerfile-bk21 b/docker/Dockerfile-bk21 new file mode 100644 index 0000000000..270669c8c2 --- /dev/null +++ b/docker/Dockerfile-bk21 @@ -0,0 +1,43 @@ +# syntax=docker/dockerfile:experimental + +FROM maven:3.9.6-eclipse-temurin-21 as build +VOLUME ["/tmp"] + +WORKDIR /mnt/mojito + +# copy source and make sure node* are not present (Mac version may conflict with Linux) +COPY . /mnt/mojito + +ENV PATH="/mnt/mojito/webapp/node/:${PATH}" +RUN --mount=type=cache,target=/root/.m2 --mount=type=cache,target=/mnt/mojito/node --mount=type=cache,target=/mnt/mojito/node_module mvn clean install -DskipTests + + +FROM eclipse-temurin:21-jdk-jammy as service +VOLUME /tmp + +ENV MOJITO_BIN=/usr/local/mojito/bin +ENV PATH $PATH:${MOJITO_BIN} +ENV MOJITO_HOST=localhost +ENV MOJITO_SCHEME=http +# MOJITO_PORT can conflict with Kubernetes, changing to MOJITO_SERVICE_PORT +ENV MOJITO_SERVICE_PORT=8080 + +COPY --from=build /mnt/mojito/webapp/target/mojito-webapp-*-exec.jar ${MOJITO_BIN}/mojito-webapp.jar +COPY --from=build /mnt/mojito/cli/target/mojito-cli-*-exec.jar ${MOJITO_BIN}/mojito-cli.jar +RUN sh -c 'touch ${MOJITO_BIN}/mojito-webapp.jar' +RUN sh -c 'touch ${MOJITO_BIN}/mojito-cli.jar' + +RUN apt-get update && apt-get install -y \ + mysql-client \ + socat + +# Create the shell wrapper for the jar +RUN /bin/echo -e "#!/bin/sh \n\ +java -Dl10n.resttemplate.host=\${MOJITO_HOST} \\\\\n \ + -Dl10n.resttemplate.scheme=\${MOJITO_SCHEME} \\\\\n \ + -Dl10n.resttemplate.port=\${MOJITO_PORT} \\\\\n \ + -jar $MOJITO_BIN/mojito-cli.jar \"\${@}\"" \ + >> /usr/local/mojito/bin/mojito && chmod +x $MOJITO_BIN/mojito + +# starting with "exec doesn't seem to be needed with openjdk:8-alpine. As per docker documentation, it is required in general +ENTRYPOINT exec java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar $MOJITO_BIN/mojito-webapp.jar diff --git a/docker/readme.md b/docker/readme.md index 02db69876f..c91bef3f87 100644 --- a/docker/readme.md +++ b/docker/readme.md @@ -1,9 +1,10 @@ # Docker compose -Run `docker-compose up` from within `mojito/docker/` or `docker-compose -f docker/docker-compose.yml up` from the +Run `docker-compose up` from within `mojito/docker/` or `docker-compose -f docker/docker-compose.yml up` from the project directory. It will start Mysql and build/start the Webapp. -In detached mode `docker compose up -d`. And to remove everything including volumes: `docker compose rm -s -v` to remove volumes. +In detached mode `docker compose up -d`. And to remove everything including volumes: `docker compose rm -s -v` to remove +volumes. To re-use a pre-built image, uncomment the `image` configuration in `docker-compose.yml`. @@ -39,18 +40,20 @@ services: Incompatibility Mac/Linux for `node/` `node_modules/`, remove the directories before calling docker commands. Some `Dockerfile` remove the directories explicitly. -# One-off build and push +# One-off build and push ## CLI image -Build: `docker build -t aurambaj/mojito-cli -f docker/Dockerfile-cli-bk8 .` and push: `docker push aurambaj/mojito-cli:latest` +Build: `docker build -t aurambaj/mojito-cli -f docker/Dockerfile-cli-bk8 .` and +push: `docker push aurambaj/mojito-cli:latest` -Example to run a command: +Example to run a command: `docker run --rm --name mojito-cli -it -e MOJITO_HOST="mojito.org" -e MOJITO_PORT="443" -e MOJITO_SCHEME="https" aurambaj/mojito-cli repo-view -n demo1` ## Webapp image -Build: `docker build -t aurambaj/mojito-webapp -f docker/Dockerfile-bk8 .` and push: `docker push aurambaj/mojito-webapp:latest` +Build: `docker build -t aurambaj/mojito-webapp -f docker/Dockerfile-bk8 .` and +push: `docker push aurambaj/mojito-webapp:latest` Start the webapp: `docker run --rm --name mojito-webapp -it aurambaj/mojito-webapp` and get a shell to try some command `docker exec -it mojito-webapp bash` @@ -87,7 +90,8 @@ This is not needed with Docker Desktop for Mac but was needed before so for the ### Find IP on Mac -Depending on how you instlled docker on Mac, `localhost` may not work. To get the IP to reach the service: `docker-machine ip default`. +Depending on how you instlled docker on Mac, `localhost` may not work. To get the IP to reach the +service: `docker-machine ip default`. `export l10n_resttemplate_host="192.168.99.111" ` and then whatever CLI command you have to run, eg. `mojito demo-create -n dockertest` @@ -113,7 +117,24 @@ services: - --innodb_use_native_aio=0 # needed on Mac ``` -## Alpine version +## Alpine version `FROM adoptopenjdk:8-jre` --> `FROM adoptopenjdk/openjdk8:alpine-jre` and change to `echo -e` for script generation -and make sure to use `/bin/echo` for consistent behavior between alpine and ubuntu. \ No newline at end of file +and make sure to use `/bin/echo` for consistent behavior between alpine and ubuntu. + +## Kubernetes + +Be aware that Kubernetes injects environment variables in the container, and that some could conflict with the +ones defined in the `Dockerfile`. For example, `MOJITO_PORT` was used in the `Dockerfile` but it is also injected by +Kubernetes with a value like `tcp://{ip}:{port}` which was making the CLI command fail in the container. Replaced +`MOJITO_PORT` with `MOJITO_SERVER_PORT` in the `Dockerfile` and the CLI command. + +### Port forwarding for MySQL + +First forward the pod port to the localhost port, then in the container use `socat` to forward pod port to the remote MySQL +server. Finally connect the SQL client to the localhost port: `33306`. + +``` +kubectl port-forward svc/mojito-webapp 33306:3306 +socat TCP-LISTEN:3306,reuseaddr,fork TCP:mojito-mysql-staging.mysql.database.azure.com:3306 +``` From 13f0fe58adbeee934eadb66ebc88617be000ac4f Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 15:56:02 -0700 Subject: [PATCH 007/105] Look for SESSION and JSESSIONID as session cookie This must have changed over time, the client still had JSESSIONID but by default spring configuration now returns SESSION. The server can be configured to return the old value JSESSIONID but it does not look like the best option. Accept both for now to be more flexible. --- .../FormLoginAuthenticationCsrfTokenInterceptor.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java b/restclient/src/main/java/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java index cc70321514..91b74f3708 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java @@ -6,6 +6,7 @@ import java.net.URI; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.hc.client5.http.cookie.Cookie; @@ -43,7 +44,7 @@ public class FormLoginAuthenticationCsrfTokenInterceptor implements ClientHttpRe public static final String CSRF_PARAM_NAME = "_csrf"; public static final String CSRF_HEADER_NAME = "X-CSRF-TOKEN"; - public static final String COOKIE_SESSION_NAME = "JSESSIONID"; + public static final Set COOKIE_SESSION_NAMES = Set.of("SESSION", "JSESSIONID"); @Autowired FormLoginConfig formLoginConfig; @@ -223,12 +224,13 @@ protected boolean doesSessionIdInCookieStoreExistAndMatchLatestSessionId() { } /** - * @return null if no sesson id cookie is found + * @return null if no session id cookie is found */ protected String getAuthenticationSessionIdFromCookieStore() { List cookies = cookieStore.getCookies(); + for (Cookie cookie : cookies) { - if (cookie.getName().equals(COOKIE_SESSION_NAME)) { + if (COOKIE_SESSION_NAMES.contains(cookie.getName())) { String cookieValue = cookie.getValue(); logger.debug("Found session cookie: {}", cookieValue); return cookieValue; From 5c9968ac0276b0eb90958f4dce958d8be02a481f Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 16:08:24 -0700 Subject: [PATCH 008/105] Android Locale to support old tag for Indonesian (in) This is needed in case we want to pull/push/import the old locale code --- .../filefinder/locale/AndroidLocaleType.java | 4 +++ .../locale/AndroidLocaleTypeTest.java | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleType.java b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleType.java index b36e146bfe..2d741a30ab 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleType.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleType.java @@ -19,6 +19,10 @@ public String getTargetLocaleRepresentation(String targetLocale) { String androidLocale = forLanguageTag.getLanguage(); + if (targetLocale.startsWith("in")) { + androidLocale = "in"; + } + if (!Strings.isNullOrEmpty(forLanguageTag.getCountry())) { androidLocale += "-r" + forLanguageTag.getCountry(); } diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleTypeTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleTypeTest.java index b24319d214..01b7de4d93 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleTypeTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleTypeTest.java @@ -26,4 +26,31 @@ public void testGetTargetLocaleRepresentationWithRegion() { String result = instance.getTargetLocaleRepresentation(targetLocale); assertEquals(expResult, result); } + + @Test + public void testGetTargetLocaleIndonesianOld() { + String targetLocale = "in"; + AndroidLocaleType instance = new AndroidLocaleType(); + String expResult = "in"; + String result = instance.getTargetLocaleRepresentation(targetLocale); + assertEquals(expResult, result); + } + + @Test + public void testGetTargetLocaleIndonesianIndonesiaOld() { + String targetLocale = "in-ID"; + AndroidLocaleType instance = new AndroidLocaleType(); + String expResult = "in-rID"; + String result = instance.getTargetLocaleRepresentation(targetLocale); + assertEquals(expResult, result); + } + + @Test + public void testGetTargetLocaleIndonesianNew() { + String targetLocale = "id"; + AndroidLocaleType instance = new AndroidLocaleType(); + String expResult = "id"; + String result = instance.getTargetLocaleRepresentation(targetLocale); + assertEquals(expResult, result); + } } From 897d44ee0036b7d44f95173a24d87301bd297f72 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 16:12:43 -0700 Subject: [PATCH 009/105] Fix bug in Project Request UI This must have been broken since the spring 3.x/Hibernate migration. --- .../main/java/com/box/l10n/mojito/entity/Drop.java | 13 ++++++++++--- .../l10n/mojito/service/drop/DropRepository.java | 7 +++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java b/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java index 81dabc59aa..5a6c9297d2 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java +++ b/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java @@ -33,14 +33,21 @@ @NamedEntityGraph( name = "Drop.legacy", attributeNodes = { - @NamedAttributeNode("repository"), + @NamedAttributeNode(value = "repository", subgraph = "Drop.legacy.repository"), @NamedAttributeNode(value = "importPollableTask", subgraph = "Drop.legacy.subTask"), - @NamedAttributeNode(value = "exportPollableTask", subgraph = "Drop.legacy.subTask") + @NamedAttributeNode(value = "exportPollableTask", subgraph = "Drop.legacy.subTask"), + @NamedAttributeNode(value = "translationKits", subgraph = "Drop.legacy.translationKits") }, subgraphs = { @NamedSubgraph( name = "Drop.legacy.subTask", - attributeNodes = {@NamedAttributeNode("subTasks")}) + attributeNodes = {@NamedAttributeNode("subTasks")}), + @NamedSubgraph( + name = "Drop.legacy.repository", + attributeNodes = {@NamedAttributeNode("createdByUser")}), + @NamedSubgraph( + name = "Drop.legacy.translationKits", + attributeNodes = {@NamedAttributeNode("locale")}) }) public class Drop extends AuditableEntity { diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java b/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java index 4a366a3017..f0d60d3580 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java @@ -2,6 +2,9 @@ import com.box.l10n.mojito.entity.Drop; import java.util.Optional; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; import org.springframework.data.jpa.repository.EntityGraph; import org.springframework.data.jpa.repository.EntityGraph.EntityGraphType; import org.springframework.data.jpa.repository.JpaRepository; @@ -14,6 +17,10 @@ @RepositoryRestResource(exported = false) public interface DropRepository extends JpaRepository, JpaSpecificationExecutor { + @Override + @EntityGraph(value = "Drop.legacy", type = EntityGraphType.FETCH) + Page findAll(Specification spec, Pageable pageable); + @Override @EntityGraph(value = "Drop.legacy", type = EntityGraphType.FETCH) Optional findById(Long aLong); From 7f2da0b247e717ae7d78f2c3f889bdc6a36ebd05 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 16:13:58 -0700 Subject: [PATCH 010/105] Fix performance issue when getting the repository --- .../src/main/java/com/box/l10n/mojito/rest/asset/AssetWS.java | 2 +- .../l10n/mojito/service/repository/RepositoryRepository.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/asset/AssetWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/asset/AssetWS.java index 04dff59e8d..c9d1c5fc9d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/rest/asset/AssetWS.java +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/asset/AssetWS.java @@ -124,7 +124,7 @@ public SourceAsset importSourceAsset(@RequestBody SourceAsset sourceAsset) throw Repository repository = repositoryRepository - .findById(sourceAsset.getRepositoryId()) + .findNoGraphById(sourceAsset.getRepositoryId()) .orElseThrow( () -> new RepositoryWithIdNotFoundException(sourceAsset.getRepositoryId())); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/repository/RepositoryRepository.java b/webapp/src/main/java/com/box/l10n/mojito/service/repository/RepositoryRepository.java index c28d7969b8..c014d74621 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/repository/RepositoryRepository.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/repository/RepositoryRepository.java @@ -27,6 +27,8 @@ public interface RepositoryRepository @Override Optional findById(Long aLong); + Optional findNoGraphById(Long aLong); + @Override List findAll(Specification s, Sort sort); From 4641a397e5a8f32187774c6abf88595a96bf5adf Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 16:14:13 -0700 Subject: [PATCH 011/105] Fix typo --- .../com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java index 7fe5b758ac..7adb810c07 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java @@ -28,7 +28,7 @@ @Parameters( commandNames = {"thirdparty-sync", "tps"}, commandDescription = - "Third-party command to sychronize text units and screenshots with third party TMS") + "Third-party command to synchronize text units and screenshots with third party TMS") public class ThirdPartySyncCommand extends Command { /** logger */ From 880386f096223f105a04185dfcb5c9597493a4b4 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 16:18:09 -0700 Subject: [PATCH 012/105] Add an option in the JSON filter to remove suffix in the text unit name Typically useful for processing JSON from FormatJS or similar. There the text unit name contain the json attribute name, which 1) is not super useful and 2) in case trying to import the translation from compiled files/KV json, it is not possible. --- .../ImportLocalizedAssetCommandTest.java | 52 ++++++++++++ .../expected/fr-CA.json | 21 +++++ .../expected/fr-FR.json | 21 +++++ .../expected/ja-JP.json | 21 +++++ .../input/source/en.json | 21 +++++ .../input/translations/fr-CA.json | 7 ++ .../input/translations/fr-FR.json | 7 ++ .../input/translations/ja-JP.json | 7 ++ .../l10n/mojito/okapi/filters/JSONFilter.java | 34 ++++++++ ...ortTranslationsFromLocalizedAssetStep.java | 29 +++++++ .../l10n/mojito/service/tm/TMServiceTest.java | 80 +++++++++++++++++++ 11 files changed, 300 insertions(+) create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-CA.json create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-FR.json create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/ja-JP.json create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/source/en.json create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-CA.json create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-FR.json create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/ja-JP.json diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java index ed34a12b57..96afcc68ba 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java @@ -678,6 +678,58 @@ public void importJsonDefaultFormatJs() throws Exception { checkExpectedGeneratedResources(); } + @Test + public void importJsonDefaultFormatJsCompiled() throws Exception { + + Repository repository = createTestRepoUsingRepoService(); + + getL10nJCommander() + .run( + "push", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-ft", + "JSON_NOBASENAME", + "-fo", + "noteKeyPattern=description", + "extractAllPairs=false", + "exceptions=defaultMessage", + "removeKeySuffix=/defaultMessage"); + + getL10nJCommander() + .run( + "import", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getInputResourcesTestDir("translations").getAbsolutePath(), + "-ft", + "JSON_NOBASENAME"); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir().getAbsolutePath(), + "-ft", + "JSON_NOBASENAME", + "-fo", + "noteKeyPattern=description", + "extractAllPairs=false", + "exceptions=defaultMessage", + "removeKeySuffix=/defaultMessage"); + + checkExpectedGeneratedResources(); + } + @Test public void importJsonI18NextParser() throws Exception { diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-CA.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-CA.json new file mode 100644 index 0000000000..bc67f94b3e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-CA.json @@ -0,0 +1,21 @@ +{ + "100_character_description_": { + "defaultMessage": "Description de 100 caractères :" + }, + "15_min_duration": { + "defaultMessage": "15 min", + "description": "File lock dialog duration" + }, + "1_day_duration": { + "defaultMessage": "1 jour", + "description": "File lock dialog duration" + }, + "1_hour_duration": { + "defaultMessage": "1 heure", + "description": "File lock dialog duration" + }, + "1_month_duration": { + "defaultMessage": "1 mois", + "description": "File lock dialog duration" + } +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-FR.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-FR.json new file mode 100644 index 0000000000..bc67f94b3e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-FR.json @@ -0,0 +1,21 @@ +{ + "100_character_description_": { + "defaultMessage": "Description de 100 caractères :" + }, + "15_min_duration": { + "defaultMessage": "15 min", + "description": "File lock dialog duration" + }, + "1_day_duration": { + "defaultMessage": "1 jour", + "description": "File lock dialog duration" + }, + "1_hour_duration": { + "defaultMessage": "1 heure", + "description": "File lock dialog duration" + }, + "1_month_duration": { + "defaultMessage": "1 mois", + "description": "File lock dialog duration" + } +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/ja-JP.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/ja-JP.json new file mode 100644 index 0000000000..f2a6578f59 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/ja-JP.json @@ -0,0 +1,21 @@ +{ + "100_character_description_": { + "defaultMessage": "100文字の説明:" + }, + "15_min_duration": { + "defaultMessage": "15分", + "description": "File lock dialog duration" + }, + "1_day_duration": { + "defaultMessage": "1日", + "description": "File lock dialog duration" + }, + "1_hour_duration": { + "defaultMessage": "1時間", + "description": "File lock dialog duration" + }, + "1_month_duration": { + "defaultMessage": "1か月", + "description": "File lock dialog duration" + } +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/source/en.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/source/en.json new file mode 100644 index 0000000000..e5e0f7149b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/source/en.json @@ -0,0 +1,21 @@ +{ + "100_character_description_": { + "defaultMessage": "100 character description:" + }, + "15_min_duration": { + "defaultMessage": "15 min", + "description": "File lock dialog duration" + }, + "1_day_duration": { + "defaultMessage": "1 day", + "description": "File lock dialog duration" + }, + "1_hour_duration": { + "defaultMessage": "1 hour", + "description": "File lock dialog duration" + }, + "1_month_duration": { + "defaultMessage": "1 month", + "description": "File lock dialog duration" + } +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-CA.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-CA.json new file mode 100644 index 0000000000..38918f96d2 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-CA.json @@ -0,0 +1,7 @@ +{ + "100_character_description_": "Description de 100 caractères :", + "15_min_duration": "15 min", + "1_day_duration": "1 jour", + "1_hour_duration": "1 heure", + "1_month_duration": "1 mois" +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-FR.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-FR.json new file mode 100644 index 0000000000..38918f96d2 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-FR.json @@ -0,0 +1,7 @@ +{ + "100_character_description_": "Description de 100 caractères :", + "15_min_duration": "15 min", + "1_day_duration": "1 jour", + "1_hour_duration": "1 heure", + "1_month_duration": "1 mois" +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/ja-JP.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/ja-JP.json new file mode 100644 index 0000000000..afa9c8925d --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/ja-JP.json @@ -0,0 +1,7 @@ +{ + "100_character_description_": "100文字の説明:", + "15_min_duration": "15分", + "1_day_duration": "1日", + "1_hour_duration": "1時間", + "1_month_duration": "1か月" +} \ No newline at end of file diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java index c3f33dc32b..f2d51a3705 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java @@ -48,6 +48,18 @@ public class JSONFilter extends net.sf.okapi.filters.json.JSONFilter { */ boolean noteKeepOrReplace = false; + /** + * Remove the suffix from the key. + * + *

Typically useful for FormatJS: + * + *

"text-unit-name": { "defaultMessage": "example", "description": "example description" } + * + *

text unit name would be text-unit-name/defaultMessage. With removeKeySuffix set, the + * "/defaultMessage" can be removed. + */ + String removeKeySuffix = null; + NoteAnnotation noteAnnotation; UsagesAnnotation usagesAnnotation; String currentKeyName; @@ -110,6 +122,7 @@ void applyFilterOptions(RawDocument input) { filterOptions.getString("usagesKeyPattern", s -> usagesKeyPattern = Pattern.compile(s)); filterOptions.getBoolean("noteKeepOrReplace", b -> noteKeepOrReplace = b); filterOptions.getBoolean("usagesKeepOrReplace", b -> usagesKeepOrReplace = b); + filterOptions.getString("removeKeySuffix", s -> removeKeySuffix = s); filterOptions.getBoolean( "convertToHtmlCodes", b -> { @@ -121,6 +134,18 @@ void applyFilterOptions(RawDocument input) { } } + @Override + public Event next() { + Event next = super.next(); + + if (next.isTextUnit()) { + ITextUnit textUnit = next.getTextUnit(); + textUnit.setName(removeKeySuffixIfMatch(textUnit.getName())); + } + + return next; + } + @Override public void handleComment(String c) { comment = c; @@ -170,6 +195,15 @@ void addNote(String value) { noteAnnotation.add(note); } + String removeKeySuffixIfMatch(String key) { + if (removeKeySuffix != null) { + if (key.endsWith(removeKeySuffix)) { + key = key.substring(0, key.length() - removeKeySuffix.length()); + } + } + return key; + } + void extractNoteIfMatch(String value) { if (noteKeyPattern != null) { Matcher m = noteKeyPattern.matcher(currentKeyName); diff --git a/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java b/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java index 778c092262..e7f92e0bae 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java +++ b/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java @@ -117,6 +117,35 @@ void initTextUnitsMapByName() { } } + @Override + protected TMTextUnitVariant importTextUnit( + TMTextUnit tmTextUnit, + TextContainer target, + TMTextUnitVariant.Status status, + ZonedDateTime createdDate) { + + for (TextUnitIntegrityChecker textUnitIntegrityChecker : textUnitIntegrityCheckers) { + try { + textUnitIntegrityChecker.check(tmTextUnit.getContent(), target.toString()); + } catch (IntegrityCheckException integrityCheckException) { + TMTextUnitVariantCommentAnnotation tmTextUnitVariantCommentAnnotation = + new TMTextUnitVariantCommentAnnotation(); + tmTextUnitVariantCommentAnnotation.setCommentType( + TMTextUnitVariantComment.Type.INTEGRITY_CHECK); + + tmTextUnitVariantCommentAnnotation.setMessage(integrityCheckException.getMessage()); + + tmTextUnitVariantCommentAnnotation.setSeverity( + TMTextUnitVariantComment.Severity.ERROR); + + new TMTextUnitVariantCommentAnnotations(target) + .addAnnotation(tmTextUnitVariantCommentAnnotation); + } + } + + return super.importTextUnit(tmTextUnit, target, status, createdDate); + } + void initTextUnitsMapByMd5() { logger.debug("initTextUnitsMapByMd5"); List textUnits = tmTextUnitRepository.findByAsset(asset); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java index 15c6f32a67..775cc5f7f1 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java @@ -3902,6 +3902,86 @@ public void testLocalizeJson() throws Exception { logger.debug("name=[{}], source=[{}]", textUnitDTO.getName(), textUnitDTO.getSource()); } + assertEquals("hello_world/string", textUnitDTOs.get(0).getName()); + + String localizedAsset = + tmService.generateLocalized( + asset, + assetContent, + repoLocale, + "en-GB", + null, + jsonFilterOptions, + Status.ALL, + InheritanceMode.USE_PARENT, + null); + logger.debug("localized=\n{}", localizedAsset); + assertEquals(expectedContent, localizedAsset); + } + + @Test + public void testLocalizeJsonRemoveKeySuffix() throws Exception { + Repository repo = repositoryService.createRepository(testIdWatcher.getEntityName("repository")); + RepositoryLocale repoLocale = repositoryService.addRepositoryLocale(repo, "en-GB"); + + List jsonFilterOptions = + Arrays.asList( + "useFullKeyPath=true", + "extractAllPairs=false", + "exceptions=.*/string", + "removeKeySuffix=/string"); + + String assetContent = + "{\n" + + " \"this to ignore\": {\n" + + " \"k1\": \"v1\"\n" + + " },\n" + + " \"hello_world\": {\n" + + " \"string\": \"Hello World\",\n" + + " \"note\": \"The start of every language book.\"\n" + + " },\n" + + " \"num_photos\": {\n" + + " \"string\": \"You have {numPhotos, plural, =0 {no photos.} =1 {one photo.} other {# photos.}}\",\n" + + " \"note\": \"A description that shows the number of photos a user has.\"\n" + + " }\n" + + "}"; + String expectedContent = assetContent; + + asset = assetService.createAssetWithContent(repo.getId(), "strings.json", assetContent); + asset = assetRepository.findById(asset.getId()).orElse(null); + assetId = asset.getId(); + tmId = repo.getTm().getId(); + + PollableFuture assetResult = + assetService.addOrUpdateAssetAndProcessIfNeeded( + repo.getId(), + asset.getPath(), + assetContent, + false, + null, + null, + null, + null, + null, + jsonFilterOptions); + try { + pollableTaskService.waitForPollableTask(assetResult.getPollableTask().getId()); + } catch (PollableTaskException | InterruptedException e) { + throw new RuntimeException(e); + } + assetResult.get(); + + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(repo.getId()); + textUnitSearcherParameters.setStatusFilter(StatusFilter.FOR_TRANSLATION); + List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); + assertEquals(2, textUnitDTOs.size()); + for (TextUnitDTO textUnitDTO : textUnitDTOs) { + logger.debug("name=[{}], source=[{}]", textUnitDTO.getName(), textUnitDTO.getSource()); + } + + assertEquals("hello_world", textUnitDTOs.get(0).getName()); + String localizedAsset = tmService.generateLocalized( asset, From 51ac6ad5102f41830540dd3186ebf54d9b1c58c5 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 16:18:23 -0700 Subject: [PATCH 013/105] Package JSON updates --- webapp/package-lock.json | 386 ++++++++++++++++++++++++++------------- 1 file changed, 262 insertions(+), 124 deletions(-) diff --git a/webapp/package-lock.json b/webapp/package-lock.json index 59d242bee5..b99b572bbd 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -150,13 +150,15 @@ "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", "dev": true, "requires": { - "kind-of": "^6.0.3", + "kind-of": "^3.0.2", "longest": "^1.0.1", "repeat-string": "^1.5.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -293,11 +295,13 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -308,11 +312,13 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -325,17 +331,21 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^6.0.3" + "kind-of": "^5.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -425,11 +435,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -440,11 +452,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -457,11 +471,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -472,11 +488,13 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -488,7 +506,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "micromatch": { "version": "3.1.10", @@ -503,7 +523,7 @@ "extend-shallow": "^3.0.2", "extglob": "^2.0.4", "fragment-cache": "^0.2.1", - "kind-of": "^6.0.3", + "kind-of": "^6.0.2", "nanomatch": "^1.2.9", "object.pick": "^1.3.0", "regex-not": "^1.0.0", @@ -512,7 +532,9 @@ }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1022,7 +1044,9 @@ }, "dependencies": { "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "mkdirp": { "version": "0.5.5", @@ -1030,11 +1054,13 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -1693,7 +1719,9 @@ "dev": true }, "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "mkdirp": { "version": "0.5.5", @@ -1701,11 +1729,13 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -1835,11 +1865,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1850,11 +1882,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1867,11 +1901,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1883,7 +1919,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -2561,17 +2599,21 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "normalize-path": { "version": "3.0.0", @@ -2649,18 +2691,22 @@ }, "dependencies": { "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "mkdirp": { "version": "0.5.0", "resolved": "https://box.jfrog.io/box/api/npm/boxnpm/mkdirp/-/mkdirp-0.5.0.tgz", "integrity": "sha1-HXMHam35hs2TROFecfzAWkyavxI=", "requires": { - "minimist": "^0.2.1" + "minimist": "0.0.8" }, "dependencies": { "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" } } } @@ -2707,7 +2753,7 @@ "requires": { "for-own": "^1.0.0", "is-plain-object": "^2.0.4", - "kind-of": "^6.0.3", + "kind-of": "^6.0.0", "shallow-clone": "^1.0.0" }, "dependencies": { @@ -2721,7 +2767,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -2870,13 +2918,16 @@ "dev": true, "optional": true, "requires": { - "ini": "^1.3.8", + "ini": "^1.3.4", "proto-list": "~1.2.1" }, "dependencies": { "ini": { - "version": "^1.3.8", - "dev": true + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "optional": true } } }, @@ -3425,11 +3476,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -3440,11 +3493,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -3457,11 +3512,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -3473,7 +3530,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -4358,7 +4417,9 @@ "optional": true }, "ini": { - "version": "^1.3.8" + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "is-fullwidth-code-point": { "version": "1.0.0", @@ -4385,7 +4446,9 @@ } }, "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "minipass": { "version": "2.9.0", @@ -4412,12 +4475,15 @@ "dev": true, "optional": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", - "dev": true + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "dev": true, + "optional": true } } }, @@ -4566,18 +4632,24 @@ "optional": true, "requires": { "deep-extend": "^0.6.0", - "ini": "^1.3.8", - "minimist": "^0.2.1", + "ini": "~1.3.0", + "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" }, "dependencies": { "ini": { - "version": "^1.3.8", - "dev": true + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "optional": true }, "minimist": { - "version": "^0.2.1", - "dev": true + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "dev": true, + "optional": true } } }, @@ -5082,7 +5154,7 @@ "dev": true, "requires": { "is-number": "^3.0.0", - "kind-of": "^6.0.3" + "kind-of": "^4.0.0" }, "dependencies": { "is-number": { @@ -5091,17 +5163,21 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5661,11 +5737,13 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5722,11 +5800,13 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5746,11 +5826,13 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^6.0.3" + "kind-of": "^5.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5960,12 +6042,14 @@ "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz", "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=", "requires": { - "node-fetch": "^2.6.1", + "node-fetch": "^1.0.1", "whatwg-fetch": ">=0.10.0" }, "dependencies": { "node-fetch": { - "version": "^2.6.1", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "requires": { "whatwg-url": "^5.0.0" } @@ -6108,7 +6192,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true }, "lazy-cache": { @@ -6170,17 +6256,21 @@ "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.0" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } }, "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" } } }, @@ -6380,7 +6470,7 @@ "decamelize": "^1.1.2", "loud-rejection": "^1.0.0", "map-obj": "^1.0.1", - "minimist": "^0.2.1", + "minimist": "^1.1.3", "normalize-package-data": "^2.3.4", "object-assign": "^4.0.1", "read-pkg-up": "^1.0.1", @@ -6389,7 +6479,9 @@ }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -6467,7 +6559,9 @@ } }, "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true }, "mixin-deep": { @@ -6515,11 +6609,13 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -6565,7 +6661,7 @@ "extend-shallow": "^3.0.2", "fragment-cache": "^0.2.1", "is-windows": "^1.0.2", - "kind-of": "^6.0.3", + "kind-of": "^6.0.2", "object.pick": "^1.3.0", "regex-not": "^1.0.0", "snapdragon": "^0.8.1", @@ -6604,7 +6700,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -7022,7 +7120,7 @@ "requires": { "copy-descriptor": "^0.1.0", "define-property": "^0.2.5", - "kind-of": "^6.0.3" + "kind-of": "^3.0.3" }, "dependencies": { "define-property": { @@ -7035,7 +7133,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8072,11 +8172,13 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8087,11 +8189,13 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8104,17 +8208,21 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^6.0.3" + "kind-of": "^5.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -8204,11 +8312,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8219,11 +8329,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8236,11 +8348,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8251,11 +8365,13 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8267,7 +8383,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "micromatch": { "version": "3.1.10", @@ -8282,7 +8400,7 @@ "extend-shallow": "^3.0.2", "extglob": "^2.0.4", "fragment-cache": "^0.2.1", - "kind-of": "^6.0.3", + "kind-of": "^6.0.2", "nanomatch": "^1.2.9", "object.pick": "^1.3.0", "regex-not": "^1.0.0", @@ -8291,7 +8409,9 @@ }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8796,12 +8916,14 @@ "dev": true, "requires": { "is-extendable": "^0.1.1", - "kind-of": "^6.0.3", + "kind-of": "^5.0.0", "mixin-object": "^2.0.1" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8888,11 +9010,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8903,11 +9027,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8920,11 +9046,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8936,7 +9064,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -8946,11 +9076,13 @@ "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.2.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -9593,11 +9725,13 @@ "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -9651,17 +9785,21 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -10173,7 +10311,7 @@ "string-width": "^1.0.2", "which-module": "^1.0.0", "y18n": "^3.2.1", - "yargs-parser": "18.1.2" + "yargs-parser": "^4.2.0" }, "dependencies": { "yargs-parser": { @@ -10325,7 +10463,7 @@ "string-width": "^1.0.2", "which-module": "^1.0.0", "y18n": "^3.2.1", - "yargs-parser": "18.1.2" + "yargs-parser": "^5.0.0" }, "dependencies": { "camelcase": { @@ -10366,4 +10504,4 @@ } } } -} \ No newline at end of file +} From d4afa022475a3ed9a806ce99aebb281a661de209 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 16:18:37 -0700 Subject: [PATCH 014/105] Update idea.md doc for symlinks --- idea.md | 1 + 1 file changed, 1 insertion(+) diff --git a/idea.md b/idea.md index 3580e105be..5ba1fec247 100644 --- a/idea.md +++ b/idea.md @@ -7,6 +7,7 @@ mkdir local/ cd local/ ln -s ~/.l10n/config ln -s ~/code/mojito-deploy/ +ln -s ~/tmp/mojito ``` ## Simplest way to get everything to work quickly (but slower that intellij incremental update) From b9a8efe22601ec2022a6ebea92244f9e94221128 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 5 Jun 2024 16:21:25 -0700 Subject: [PATCH 015/105] Improve the ImportLocalizedAssetCommand - Add an option to apply the integrity checks during import - Add parallel import support - Add option to continue on error eg. if some files are missing, try to limit the case of warnings by checking if source file is empty (which could imply there is no localized file in that case) --- .../command/ImportLocalizedAssetCommand.java | 88 ++++++++++++++----- ...ortTranslationsFromLocalizedAssetStep.java | 26 +++++- 2 files changed, 92 insertions(+), 22 deletions(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java index 896b2f59be..209ef3cd2d 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java @@ -11,7 +11,6 @@ import com.box.l10n.mojito.rest.client.LocaleClient; import com.box.l10n.mojito.rest.client.RepositoryClient; import com.box.l10n.mojito.rest.client.exception.AssetNotFoundException; -import com.box.l10n.mojito.rest.client.exception.PollableTaskException; import com.box.l10n.mojito.rest.entity.Asset; import com.box.l10n.mojito.rest.entity.ImportLocalizedAssetBody; import com.box.l10n.mojito.rest.entity.ImportLocalizedAssetBody.StatusForEqualTarget; @@ -22,6 +21,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import org.fusesource.jansi.Ansi; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -124,6 +124,12 @@ public class ImportLocalizedAssetCommand extends Command { converter = ImportLocalizedAssetBodyStatusForEqualTargetConverter.class) StatusForEqualTarget statusForEqualTarget = StatusForEqualTarget.APPROVED; + @Parameter( + names = {"--continue-on-error"}, + arity = 0, + description = "Continue import on errors") + boolean continueOnError = false; + @Autowired AssetClient assetClient; @Autowired LocaleClient localeClient; @@ -163,15 +169,37 @@ public void execute() throws CommandException { sourcePathFilterRegex, directoriesIncludePatterns, directoriesExcludePatterns)) { - for (Locale locale : getLocalesForImport()) { - doImportFileMatch(sourceFileMatch, locale); - } + + List list = + getLocalesForImport().stream() + .map(locale -> doImportFileMatch(sourceFileMatch, locale)) + .filter(Objects::nonNull) + .toList(); + + list.forEach( + importLocalizedAssetForContent -> { + try { + commandHelper.waitForPollableTask( + importLocalizedAssetForContent.getPollableTask().getId()); + } catch (CommandException e) { + if (continueOnError) { + consoleWriter + .a(" Error while importing: ") + .fg(Ansi.Color.RED) + .a(sourceFileMatch.getPath().toString()) + .println(); + } else { + throw e; + } + } + }); } consoleWriter.fg(Ansi.Color.GREEN).newLine().a("Finished").println(2); } - protected void doImportFileMatch(FileMatch fileMatch, Locale locale) throws CommandException { + protected ImportLocalizedAssetBody doImportFileMatch(FileMatch fileMatch, Locale locale) + throws CommandException { try { logger.info("Importing for locale: {}", locale.getBcp47Tag()); Path targetPath = getTargetPath(fileMatch, locale); @@ -185,26 +213,44 @@ protected void doImportFileMatch(FileMatch fileMatch, Locale locale) throws Comm Asset assetByPathAndRepositoryId = assetClient.getAssetByPathAndRepositoryId(fileMatch.getSourcePath(), repository.getId()); - ImportLocalizedAssetBody importLocalizedAssetForContent = - assetClient.importLocalizedAssetForContent( - assetByPathAndRepositoryId.getId(), - locale.getId(), - commandHelper.getFileContent(targetPath), - statusForEqualTarget, - fileMatch.getFileType().getFilterConfigIdOverride(), - commandHelper.getFilterOptionsOrDefaults( - fileMatch.getFileType(), filterOptionsParam)); - + String fileContent; try { - commandHelper.waitForPollableTask(importLocalizedAssetForContent.getPollableTask().getId()); - } catch (PollableTaskException e) { - throw new CommandException(e.getMessage(), e.getCause()); + fileContent = commandHelper.getFileContent(targetPath); + } catch (Exception e) { + if (continueOnError) { + if (!commandHelper.getFileContent(fileMatch.getPath()).isBlank()) { + consoleWriter + .a(" Missing file, skipping: ") + .fg(Ansi.Color.YELLOW) + .a(targetPath.toString()) + .println(); + } + return null; + } else { + throw e; + } } + return assetClient.importLocalizedAssetForContent( + assetByPathAndRepositoryId.getId(), + locale.getId(), + fileContent, + statusForEqualTarget, + fileMatch.getFileType().getFilterConfigIdOverride(), + commandHelper.getFilterOptionsOrDefaults(fileMatch.getFileType(), filterOptionsParam)); } catch (AssetNotFoundException ex) { - throw new CommandException( - "No asset for file [" + fileMatch.getPath() + "] into repo [" + repositoryParam + "]", - ex); + if (continueOnError) { + consoleWriter + .a(" Asset not found: ") + .fg(Ansi.Color.YELLOW) + .a(fileMatch.getPath().toString()) + .println(); + } else { + throw new CommandException( + "No asset for file [" + fileMatch.getPath() + "] into repo [" + repositoryParam + "]", + ex); + } } + return null; } public Collection getLocalesForImport() { diff --git a/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java b/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java index e7f92e0bae..6f97d972fb 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java +++ b/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java @@ -5,16 +5,25 @@ import com.box.l10n.mojito.entity.TMTextUnit; import com.box.l10n.mojito.entity.TMTextUnitCurrentVariant; import com.box.l10n.mojito.entity.TMTextUnitVariant; +import com.box.l10n.mojito.entity.TMTextUnitVariantComment; import com.box.l10n.mojito.service.NormalizationUtils; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckException; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckerFactory; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TMTextUnitVariantCommentAnnotation; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TMTextUnitVariantCommentAnnotations; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TextUnitIntegrityChecker; import com.box.l10n.mojito.service.tm.TranslatorWithInheritance; import com.box.l10n.mojito.service.tm.search.TextUnitDTO; import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; +import java.time.ZonedDateTime; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import net.sf.okapi.common.Event; import net.sf.okapi.common.resource.TextContainer; import org.slf4j.Logger; @@ -33,6 +42,8 @@ public class ImportTranslationsFromLocalizedAssetStep extends AbstractImportTran @Autowired TextUnitSearcher textUnitSearcher; + @Autowired IntegrityCheckerFactory integrityCheckerFactory; + Asset asset; RepositoryLocale repositoryLocale; StatusForEqualTarget statusForEqualTarget; @@ -43,6 +54,8 @@ public class ImportTranslationsFromLocalizedAssetStep extends AbstractImportTran TranslatorWithInheritance translatorWithInheritance; + private Set textUnitIntegrityCheckers = new HashSet<>(); + boolean hasTranslationWithoutInheritance; public enum StatusForEqualTarget { @@ -67,6 +80,17 @@ protected Event handleStartDocument(Event event) { translatorWithInheritance = new TranslatorWithInheritance(asset, repositoryLocale, InheritanceMode.USE_PARENT); hasTranslationWithoutInheritance = translatorWithInheritance.hasTranslationWithoutInheritance(); + + textUnitIntegrityCheckers = integrityCheckerFactory.getTextUnitCheckers(asset); + if (textUnitIntegrityCheckers.isEmpty()) { + logger.debug("There is no integrity checkers for asset id {}", asset.getId()); + } else { + logger.debug( + "Found {} integrity checker(s) for asset id {}", + textUnitIntegrityCheckers.size(), + asset.getId()); + } + return event; } @@ -136,7 +160,7 @@ protected TMTextUnitVariant importTextUnit( tmTextUnitVariantCommentAnnotation.setMessage(integrityCheckException.getMessage()); tmTextUnitVariantCommentAnnotation.setSeverity( - TMTextUnitVariantComment.Severity.ERROR); + TMTextUnitVariantComment.Severity.ERROR); //TODO(ja) dial it down for plural strings? new TMTextUnitVariantCommentAnnotations(target) .addAnnotation(tmTextUnitVariantCommentAnnotation); From 46f689703d089350d10fea5c9c5345f097aab1a6 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 7 Jun 2024 12:27:02 -0700 Subject: [PATCH 016/105] Make AndroidStringDocumentReader and AndroidStringDocumentWriter throw RuntimeException --- .../strings/AndroidStringDocumentReader.java | 11 +++-- .../AndroidStringDocumentReaderException.java | 7 +++ .../strings/AndroidStringDocumentUtils.java | 8 +++- .../strings/AndroidStringDocumentWriter.java | 24 ++++++---- .../service/thirdparty/ThirdPartyService.java | 2 +- .../thirdparty/ThirdPartyTMSSmartling.java | 45 ++++++------------- .../quartz/SmartlingPullLocaleFileJob.java | 14 +----- .../ThirdPartyTMSSmartlingTest.java | 12 +---- 8 files changed, 54 insertions(+), 69 deletions(-) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReaderException.java diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java index b5ea19e6a6..fa68788d1c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java @@ -21,10 +21,13 @@ public static AndroidStringDocument fromFile(String fileName) return buildFromDocument(documentBuilder().parse(new File(fileName))); } - public static AndroidStringDocument fromText(String text) - throws ParserConfigurationException, IOException, SAXException { - return buildFromDocument( - documentBuilder().parse(new InputSource(new StringReader(Strings.nullToEmpty(text))))); + public static AndroidStringDocument fromText(String text) { + try { + return buildFromDocument( + documentBuilder().parse(new InputSource(new StringReader(Strings.nullToEmpty(text))))); + } catch (SAXException | IOException e) { + throw new AndroidStringDocumentReaderException(e); + } } private static AndroidStringDocument buildFromDocument(Document source) { diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReaderException.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReaderException.java new file mode 100644 index 0000000000..6d8b4bf787 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReaderException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.android.strings; + +public class AndroidStringDocumentReaderException extends RuntimeException { + public AndroidStringDocumentReaderException(Exception e) { + super(e); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentUtils.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentUtils.java index cee751111e..1263050d3a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentUtils.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentUtils.java @@ -11,8 +11,12 @@ public final class AndroidStringDocumentUtils { DocumentBuilderFactory.newInstance(); static final TransformerFactory TRANSFORMER_FACTORY = TransformerFactory.newInstance(); - static DocumentBuilder documentBuilder() throws ParserConfigurationException { - return DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); + static DocumentBuilder documentBuilder() { + try { + return DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } } private AndroidStringDocumentUtils() { diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java index 0d02d862d4..c8538c2472 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java @@ -17,9 +17,9 @@ import java.io.IOException; import java.io.StringWriter; import java.io.Writer; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; @@ -34,13 +34,12 @@ public class AndroidStringDocumentWriter { private Document document; private Node root; - public AndroidStringDocumentWriter(final AndroidStringDocument source) - throws ParserConfigurationException { + public AndroidStringDocumentWriter(final AndroidStringDocument source) { this.source = requireNonNull(source); buildDomSource(); } - public void buildDomSource() throws ParserConfigurationException { + public void buildDomSource() { document = documentBuilder().newDocument(); root = document.createElement(ROOT_ELEMENT_NAME); document.setXmlStandalone(true); @@ -49,20 +48,29 @@ public void buildDomSource() throws ParserConfigurationException { domSource = new DOMSource(document); } - private W buildWriter(W writer) throws TransformerException { + private W buildWriter(W writer) { - Transformer transformer = TRANSFORMER_FACTORY.newTransformer(); + Transformer transformer = null; + try { + transformer = TRANSFORMER_FACTORY.newTransformer(); + } catch (TransformerConfigurationException e) { + throw new RuntimeException(e); + } transformer.setOutputProperty(OutputKeys.INDENT, "yes"); transformer.setOutputProperty(OutputKeys.METHOD, "xml"); transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "0"); - transformer.transform(domSource, new StreamResult(writer)); + try { + transformer.transform(domSource, new StreamResult(writer)); + } catch (TransformerException e) { + throw new RuntimeException(e); + } return writer; } - public String toText() throws TransformerException { + public String toText() { return buildWriter(new StringWriter()).toString(); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java index 281cdbb1d3..c1f8ef75f9 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java @@ -296,7 +296,7 @@ void mapThirdPartyTextUnitsToTextUnitDTOs( Set alreadyMappedTmTextUnitId = thirdPartyTextUnitRepository.findTmTextUnitIdsByAsset(asset); - Boolean allWithTmTextUnitId = + boolean allWithTmTextUnitId = thirdPartyTextUnitsToMap.stream() .map(ThirdPartyTextUnit::getTmTextUnitId) .allMatch(Objects::nonNull); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java index b2f0c951df..8457edc967 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java @@ -50,8 +50,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.TransformerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -472,17 +470,10 @@ private SmartlingFile uploadTextUnitsToSmartling( SmartlingFile file = new SmartlingFile(); file.setFileName(getOutputSourceFile(batchNumber, repository.getName(), filePrefix.getType())); - try { - - logger.debug("Save source file to: {}", file.getFileName()); - AndroidStringDocumentWriter writer = - new AndroidStringDocumentWriter(mapper.readFromSourceTextUnits(result)); - file.setFileContent(writer.toText()); - - } catch (ParserConfigurationException | TransformerException e) { - logger.error("An error occurred when processing a push batch", e); - throw new RuntimeException(e); - } + logger.debug("Save source file to: {}", file.getFileName()); + AndroidStringDocumentWriter writer = + new AndroidStringDocumentWriter(mapper.readFromSourceTextUnits(result)); + file.setFileContent(writer.toText()); if (!options.isDryRun()) { Mono.fromCallable( @@ -753,27 +744,19 @@ private SmartlingFile processTranslationBatch( SmartlingFile file = new SmartlingFile(); file.setFileName(targetFilename); - try { - - logger.debug("Save target file to: {}", file.getFileName()); + logger.debug("Save target file to: {}", file.getFileName()); - if (filterTmTextUnitIds != null) { - fileBatch = - fileBatch.stream() - .filter( - textUnitDTO -> filterTmTextUnitIds.contains(textUnitDTO.getTmTextUnitId())) - .collect(Collectors.toList()); - } - - AndroidStringDocumentWriter writer = - new AndroidStringDocumentWriter(mapper.readFromTargetTextUnits(batch)); - file.setFileContent(writer.toText()); - - } catch (ParserConfigurationException | TransformerException e) { - logger.error("An error occurred when processing a push_translations batch", e); - throw new RuntimeException(e); + if (filterTmTextUnitIds != null) { + fileBatch = + fileBatch.stream() + .filter(textUnitDTO -> filterTmTextUnitIds.contains(textUnitDTO.getTmTextUnitId())) + .collect(Collectors.toList()); } + AndroidStringDocumentWriter writer = + new AndroidStringDocumentWriter(mapper.readFromTargetTextUnits(batch)); + file.setFileContent(writer.toText()); + if (!options.isDryRun()) { logger.debug( "Push Android file to Smartling project: {} and locale: {}", projectId, localeTag); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java index 8f02159cb4..0f6d733b03 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java @@ -16,13 +16,10 @@ import com.box.l10n.mojito.smartling.SmartlingClientException; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; -import java.io.IOException; import java.util.List; -import javax.xml.parsers.ParserConfigurationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.xml.sax.SAXException; import reactor.core.publisher.Mono; public class SmartlingPullLocaleFileJob @@ -103,15 +100,8 @@ && matchesChecksumFromPreviousSync(input, localeTag, fileName, fileContent)) { return null; } - List textUnits; - - try { - textUnits = mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); - } catch (ParserConfigurationException | IOException | SAXException e) { - String msg = "An error occurred when processing a pull batch"; - logger.error(msg, e); - throw new RuntimeException(msg, e); - } + List textUnits = + mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); if (!textUnits.isEmpty() && input.getSmartlingFilePrefix().equalsIgnoreCase("plural") diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java index e79e185914..6986de05eb 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java @@ -66,7 +66,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Ints; import io.micrometer.core.instrument.MeterRegistry; -import java.io.IOException; import java.time.Duration; import java.time.ZonedDateTime; import java.util.ArrayList; @@ -77,7 +76,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import javax.xml.parsers.ParserConfigurationException; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -96,7 +94,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.HttpStatus; import org.springframework.web.client.HttpServerErrorException; -import org.xml.sax.SAXException; import reactor.util.retry.Retry; import reactor.util.retry.RetryBackoffSpec; @@ -1805,14 +1802,7 @@ private List readTextUnits(SmartlingFile file, String pluralSeparat AndroidStringDocumentMapper mapper = new AndroidStringDocumentMapper(pluralSeparator, null, null, null); List result; - - try { - result = mapper.mapToTextUnits(fromText(file.getFileContent())); - } catch (ParserConfigurationException | IOException | SAXException e) { - result = new ArrayList<>(); - e.printStackTrace(); - } - return result; + return mapper.mapToTextUnits(fromText(file.getFileContent())); } private void prepareAssetAndTextUnits(AssetExtraction assetExtraction, Asset asset, TM tm) { From b732b1ad79ecd0f6d8e3b5bf7942ddf8b252b05a Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 7 Jun 2024 12:30:25 -0700 Subject: [PATCH 017/105] Add readString() with UncheckedIOException --- common/src/main/java/com/box/l10n/mojito/io/Files.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/common/src/main/java/com/box/l10n/mojito/io/Files.java b/common/src/main/java/com/box/l10n/mojito/io/Files.java index ada5d8ed1e..a4679058b8 100644 --- a/common/src/main/java/com/box/l10n/mojito/io/Files.java +++ b/common/src/main/java/com/box/l10n/mojito/io/Files.java @@ -55,6 +55,14 @@ public static void write(Path path, String content) { } } + public static String readString(Path path) { + try { + return java.nio.file.Files.readString(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public static Stream lines(Path path) { try { return java.nio.file.Files.lines(path, StandardCharsets.UTF_8); From dac3f6c279e0af7e3461137a36c5b66f85a5c42a Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Sun, 9 Jun 2024 18:53:40 -0700 Subject: [PATCH 018/105] createTempDirectory with UncheckedIOException --- common/src/main/java/com/box/l10n/mojito/io/Files.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/common/src/main/java/com/box/l10n/mojito/io/Files.java b/common/src/main/java/com/box/l10n/mojito/io/Files.java index a4679058b8..3f4089398e 100644 --- a/common/src/main/java/com/box/l10n/mojito/io/Files.java +++ b/common/src/main/java/com/box/l10n/mojito/io/Files.java @@ -83,4 +83,12 @@ public static void write( throw new UncheckedIOException(ioe); } } + + public static Path createTempDirectory(String prefix, FileAttribute... attrs) { + try { + return java.nio.file.Files.createTempDirectory(prefix, attrs); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } From daba9db89cf10c657380aed9b231db78c7852cba Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 10 Jun 2024 14:54:37 -0700 Subject: [PATCH 019/105] AndroidStringDocumentMapper add optional id in text unit name --- .../mojito/android/strings/AndroidPlural.java | 10 ++ .../strings/AndroidStringDocumentMapper.java | 67 +++++++++-- .../AndroidStringDocumentMapperTest.java | 106 +++++++++++++++++- 3 files changed, 171 insertions(+), 12 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java index 646527ebb4..4fae3d20bf 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java @@ -69,6 +69,10 @@ public AndroidPluralBuilder addItem(AndroidPluralItem item) { return this; } + public String getName() { + return name; + } + public AndroidPluralBuilder setName(String name) { this.name = name; return this; @@ -79,6 +83,12 @@ public AndroidPluralBuilder setComment(String comment) { return this; } + public List getSortedItems() { + return items.stream() + .sorted(Comparator.comparing(item -> item.getQuantity().ordinal())) + .collect(Collectors.toList()); + } + public AndroidPlural build() { return new AndroidPlural(name, comment, items); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java index f13abd183b..62c961d3bc 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java @@ -6,9 +6,11 @@ import com.box.l10n.mojito.service.tm.search.TextUnitDTO; import com.google.common.base.CharMatcher; import com.google.common.base.Strings; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -29,15 +31,26 @@ public class AndroidStringDocumentMapper { private final String locale; private final String repositoryName; private final PluralNameParser pluralNameParser; + private final boolean addTextUnitIdInName; public AndroidStringDocumentMapper( - String pluralSeparator, String assetDelimiter, String locale, String repositoryName) { + String pluralSeparator, + String assetDelimiter, + String locale, + String repositoryName, + boolean addTextUnitIdInName) { this.pluralSeparator = pluralSeparator; this.assetDelimiter = Optional.ofNullable(Strings.emptyToNull(assetDelimiter)).orElse(DEFAULT_ASSET_DELIMITER); this.locale = locale; this.repositoryName = repositoryName; this.pluralNameParser = new PluralNameParser(); + this.addTextUnitIdInName = addTextUnitIdInName; + } + + public AndroidStringDocumentMapper( + String pluralSeparator, String assetDelimiter, String locale, String repositoryName) { + this(pluralSeparator, assetDelimiter, locale, repositoryName, false); } public AndroidStringDocumentMapper(String pluralSeparator, String assetDelimiter) { @@ -85,7 +98,19 @@ public AndroidStringDocument readFromTextUnits(List textUnits, bool } } - pluralByOther.forEach((pluralFormOther, builder) -> document.addPlural(builder.build())); + pluralByOther.forEach( + (pluralFormOther, builder) -> { + if (addTextUnitIdInName) { + String ids = + builder.getSortedItems().stream() + .map(AndroidPluralItem::getId) + .map(Objects::toString) + .collect(Collectors.joining(",")); + builder.setName(ids + "#@#" + builder.getName()); + } + + document.addPlural(builder.build()); + }); return document; } @@ -117,11 +142,34 @@ TextUnitDTO addTextUnitDTOAttributes(TextUnitDTO textUnit) { } if (textUnit.getName().contains(assetDelimiter)) { - String[] nameParts = textUnit.getName().split(assetDelimiter, 2); - if (nameParts.length > 1) { - textUnit.setAssetPath(nameParts[0]); - textUnit.setName(nameParts[1]); + if (addTextUnitIdInName) { + String[] nameParts = textUnit.getName().split(assetDelimiter, 3); + if (nameParts.length > 2) { + + if (textUnit.getPluralForm() == null) { + textUnit.setTmTextUnitId(Long.valueOf(nameParts[0])); + } else { + String idsString = nameParts[0]; + List ids = Arrays.stream(idsString.split(",")).map(Long::valueOf).toList(); + if (ids.size() != 6) { + throw new RuntimeException( + "There must be 6 ids, check that all the required text units are provided (typically TextUnitSearcherParameters#setPluralFormsFiltered(false)"); + } + + AndroidPluralQuantity androidPluralQuantity = + AndroidPluralQuantity.valueOf(textUnit.getPluralForm()); + textUnit.setTmTextUnitId(ids.get(androidPluralQuantity.ordinal())); + } + textUnit.setAssetPath(nameParts[1]); + textUnit.setName(nameParts[2]); + } + } else { + String[] nameParts = textUnit.getName().split(assetDelimiter, 2); + if (nameParts.length > 1) { + textUnit.setAssetPath(nameParts[0]); + textUnit.setName(nameParts[1]); + } } } @@ -142,7 +190,6 @@ Stream stringToTextUnits(AbstractAndroidString androidString) { Stream singularToTextUnit(AndroidSingular singular) { TextUnitDTO textUnit = new TextUnitDTO(); - textUnit.setName(singular.getName()); textUnit.setComment(singular.getComment()); textUnit.setTmTextUnitId(singular.getId()); @@ -190,9 +237,13 @@ String getKeyToGroupByPluralOtherAndComment(TextUnitDTO textUnit) { } AndroidSingular textUnitToAndroidSingular(TextUnitDTO textUnit, boolean useSource) { + return new AndroidSingular( textUnit.getTmTextUnitId(), - textUnit.getAssetPath() + assetDelimiter + textUnit.getName(), + (addTextUnitIdInName ? textUnit.getTmTextUnitId() + assetDelimiter : "") + + textUnit.getAssetPath() + + assetDelimiter + + textUnit.getName(), removeInvalidControlCharacter( Strings.nullToEmpty(useSource ? textUnit.getSource() : textUnit.getTarget())), removeInvalidControlCharacter(textUnit.getComment())); diff --git a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java index 25b35caf64..f7b342ed71 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java @@ -58,6 +58,29 @@ public void testReadFromSourceTextUnitsWithoutPluralForms() { assertThat(singular2.getComment()).isEqualTo("comment1"); } + @Test + public void testReadFromSourceTextUnitsWithoutPluralFormsAndWithTmTextUnitIdInName() { + mapper = new AndroidStringDocumentMapper("_", assetDelimiter, null, null, true); + textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); + textUnits.add(sourceTextUnitDTO(124L, "name1", "content1", "comment1", "my/path1", null, null)); + + document = mapper.readFromSourceTextUnits(textUnits); + + assertThat(document).isNotNull(); + assertThat(document.getPlurals()).isEmpty(); + assertThat(document.getSingulars()).hasSize(2); + AndroidSingular singular1 = document.getSingulars().get(0); + assertThat(singular1.getId()).isEqualTo(123L); + assertThat(singular1.getName()).isEqualTo("123#@#my/path0#@#name0"); + assertThat(singular1.getContent()).isEqualTo("content0"); + assertThat(singular1.getComment()).isEqualTo("comment0"); + AndroidSingular singular2 = document.getSingulars().get(1); + assertThat(singular2.getId()).isEqualTo(124L); + assertThat(singular2.getName()).isEqualTo("124#@#my/path1#@#name1"); + assertThat(singular2.getContent()).isEqualTo("content1"); + assertThat(singular2.getComment()).isEqualTo("comment1"); + } + @Test public void testReadFromSourceTextUnitsWithoutPluralFormsRemovesBadCharacters() { mapper = new AndroidStringDocumentMapper("_", assetDelimiter); @@ -134,6 +157,65 @@ public void testReadFromSourceTextUnitsWithPlurals() { assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); } + @Test + public void testReadFromSourceTextUnitsWithPluralsAndWithTmTextUnitIdInName() { + mapper = new AndroidStringDocumentMapper(" _", assetDelimiter, null, null, true); + textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); + + textUnits.add( + sourceTextUnitDTO( + 100L, "name1 _other", "content1_zero", "comment1", "my/path1", "zero", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 101L, "name1 _other", "content1_one", "comment1", "my/path1", "one", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 102L, "name1 _other", "content1_two", "comment1", "my/path1", "two", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 103L, "name1 _other", "content1_few", "comment1", "my/path1", "few", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 104L, "name1 _other", "content1_many", "comment1", "my/path1", "many", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 105L, + "name1 _other", + "content1_other", + "comment1", + "my/path1", + "other", + "name1_other")); + + document = mapper.readFromSourceTextUnits(textUnits); + + assertThat(document).isNotNull(); + assertThat(document.getSingulars()).hasSize(1); + AndroidSingular singular = document.getSingulars().get(0); + assertThat(singular.getId()).isEqualTo(123L); + assertThat(singular.getName()).isEqualTo("123#@#my/path0#@#name0"); + assertThat(singular.getContent()).isEqualTo("content0"); + assertThat(singular.getComment()).isEqualTo("comment0"); + + assertThat(document.getPlurals()).hasSize(1); + AndroidPlural plural = document.getPlurals().get(0); + assertThat(plural.getName()).isEqualTo("100,101,102,103,104,105#@#my/path1#@#name1"); + assertThat(plural.getComment()).isEqualTo("comment1"); + assertThat(plural.getItems()).hasSize(6); + assertThat(plural.getItems().get(ZERO).getId()).isEqualTo(100L); + assertThat(plural.getItems().get(ZERO).getContent()).isEqualTo("content1_zero"); + assertThat(plural.getItems().get(ONE).getId()).isEqualTo(101L); + assertThat(plural.getItems().get(ONE).getContent()).isEqualTo("content1_one"); + assertThat(plural.getItems().get(TWO).getId()).isEqualTo(102L); + assertThat(plural.getItems().get(TWO).getContent()).isEqualTo("content1_two"); + assertThat(plural.getItems().get(FEW).getId()).isEqualTo(103L); + assertThat(plural.getItems().get(FEW).getContent()).isEqualTo("content1_few"); + assertThat(plural.getItems().get(MANY).getId()).isEqualTo(104L); + assertThat(plural.getItems().get(MANY).getContent()).isEqualTo("content1_many"); + assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(105L); + assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); + } + @Test public void testReadFromSourceTextUnitsWithDuplicatePlurals() { mapper = new AndroidStringDocumentMapper(" _", assetDelimiter); @@ -667,13 +749,13 @@ public void testReadingTextUnitsFromFile() throws Exception { assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("some_string")) - .extracting("target", "comment") - .containsOnly(tuple("Dela...", "Some string")); + .extracting("name", "target", "comment") + .containsOnly(tuple("some_string", "Dela...", "Some string")); assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("show_options")) - .extracting("target", "comment") - .containsOnly(tuple("Mer \" dela", "Options")); + .extracting("name", "target", "comment") + .containsOnly(tuple("show_options", "Mer \" dela", "Options")); assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("without_comment")) @@ -734,6 +816,22 @@ public void testAddTextUnitDTOAttributesAssetPathAndName() { .containsExactly("asset_path", "name_part1#@#name_part2"); } + @Test + public void testAddTextUnitDTOAttributesTextUnitIdAndAssetPathAndName() { + mapper = new AndroidStringDocumentMapper("_", null, null, null, true); + TextUnitDTO textUnitDTO = new TextUnitDTO(); + + textUnitDTO.setName("156151#@#asset_path#@#name_part1"); + assertThat(mapper.addTextUnitDTOAttributes(textUnitDTO)) + .extracting(TextUnitDTO::getTmTextUnitId, TextUnitDTO::getAssetPath, TextUnitDTO::getName) + .containsExactly(156151L, "asset_path", "name_part1"); + + textUnitDTO.setName("156152#@#asset_path#@#name_part1#@#name_part2"); + assertThat(mapper.addTextUnitDTOAttributes(textUnitDTO)) + .extracting(TextUnitDTO::getTmTextUnitId, TextUnitDTO::getAssetPath, TextUnitDTO::getName) + .containsExactly(156152L, "asset_path", "name_part1#@#name_part2"); + } + private TextUnitDTO sourceTextUnitDTO( Long id, String name, From 5161ed0f6b02c2cbf0adc4b319711e495d1b0129 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 11 Jun 2024 17:11:47 -0700 Subject: [PATCH 020/105] Update Authentication documentation --- docs/_docs/guides/authentication_springboot3.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/_docs/guides/authentication_springboot3.md b/docs/_docs/guides/authentication_springboot3.md index 1d50be9860..772bf4099e 100644 --- a/docs/_docs/guides/authentication_springboot3.md +++ b/docs/_docs/guides/authentication_springboot3.md @@ -32,9 +32,10 @@ If the redirect is enabled, it is still possible to access {{ site.mojito_green #### Example with GitHub -Create a `GitHub OAuth app` with `Authorization callback URL`: `http://localhost:8080/login/oauth`. +Create a `GitHub OAuth app` with `Authorization callback URL`: `http://localhost:8080/login/oauth2/code/github`. This URI maps to the `redirect_uri` in OAuth and to `preEstablishedRedirectUri` in Spring settings. - The `clientId` and `clientSecret` are available once the app has been created. + The `clientId` and `clientSecret` are available once the app has been created. The homepage URL should not matter +but can be set to: `http://localhost:8080/` Settings to be added, substituting the client `id` and `secret`: From 577fde0c6d06904999f2d72f791aecddebeadbc2 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 11 Jun 2024 22:43:15 -0700 Subject: [PATCH 021/105] Add option locale mapping type in import command --- .../command/ImportLocalizedAssetCommand.java | 44 +- webapp/package-lock.json | 386 ++++++------------ 2 files changed, 149 insertions(+), 281 deletions(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java index 209ef3cd2d..5188f4c4f9 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java @@ -18,7 +18,6 @@ import com.box.l10n.mojito.rest.entity.Repository; import java.nio.file.Path; import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -72,6 +71,22 @@ public class ImportLocalizedAssetCommand extends Command { description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION) String localeMappingParam; + @Parameter( + names = {"-lmt", "--locale-mapping-type"}, + arity = 1, + required = false, + description = + "Specifies how to handle locale mappings when used with --locale-mapping. " + + "MAP_ONLY processes only the locales explicitly specified in the mapping. " + + "WITH_REPOSITORY generates a basic mapping from the repository's locales and " + + "supplements it with the provided mapping, potentially overriding existing entries.") + LocaleMappingType localeMappingTypeParam = LocaleMappingType.WITH_REPOSITORY; + + enum LocaleMappingType { + MAP_ONLY, + WITH_REPOSITORY + } + @Parameter( names = {Param.FILE_TYPE_LONG, Param.FILE_TYPE_SHORT}, variableArity = true, @@ -158,6 +173,7 @@ public void execute() throws CommandException { .println(2); repository = commandHelper.findRepositoryByName(repositoryParam); + commandDirectories = new CommandDirectories(sourceDirectoryParam, targetDirectoryParam); inverseLocaleMapping = localeMappingHelper.getInverseLocaleMapping(localeMappingParam); @@ -256,31 +272,21 @@ protected ImportLocalizedAssetBody doImportFileMatch(FileMatch fileMatch, Locale public Collection getLocalesForImport() { Collection sortedRepositoryLocales = commandHelper.getSortedRepositoryLocales(repository).values(); - filterLocalesWithMapping(sortedRepositoryLocales); - return sortedRepositoryLocales; - } - - private void filterLocalesWithMapping(Collection locales) { - - if (inverseLocaleMapping != null) { - Iterator iterator = locales.iterator(); - while (iterator.hasNext()) { - Locale l = iterator.next(); - if (!inverseLocaleMapping.containsKey(l.getBcp47Tag())) { - iterator.remove(); - } - } + if (LocaleMappingType.MAP_ONLY.equals(localeMappingTypeParam) && inverseLocaleMapping != null) { + sortedRepositoryLocales = + sortedRepositoryLocales.stream() + .filter(l -> inverseLocaleMapping.containsKey(l.getBcp47Tag())) + .toList(); } + return sortedRepositoryLocales; } private Path getTargetPath(FileMatch fileMatch, Locale locale) throws CommandException { - String targetLocale; + String targetLocale = locale.getBcp47Tag(); - if (inverseLocaleMapping != null) { + if (inverseLocaleMapping != null && inverseLocaleMapping.containsKey(locale.getBcp47Tag())) { targetLocale = inverseLocaleMapping.get(locale.getBcp47Tag()); - } else { - targetLocale = locale.getBcp47Tag(); } logger.info("processing locale for import: {}", targetLocale); diff --git a/webapp/package-lock.json b/webapp/package-lock.json index b99b572bbd..59d242bee5 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -150,15 +150,13 @@ "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", "dev": true, "requires": { - "kind-of": "^3.0.2", + "kind-of": "^6.0.3", "longest": "^1.0.1", "repeat-string": "^1.5.2" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -295,13 +293,11 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -312,13 +308,11 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -331,21 +325,17 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^5.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "version": "^6.0.3" } } }, @@ -435,13 +425,11 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -452,13 +440,11 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -471,13 +457,11 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -488,13 +472,11 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -506,9 +488,7 @@ "dev": true }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "version": "^6.0.3" }, "micromatch": { "version": "3.1.10", @@ -523,7 +503,7 @@ "extend-shallow": "^3.0.2", "extglob": "^2.0.4", "fragment-cache": "^0.2.1", - "kind-of": "^6.0.2", + "kind-of": "^6.0.3", "nanomatch": "^1.2.9", "object.pick": "^1.3.0", "regex-not": "^1.0.0", @@ -532,9 +512,7 @@ }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -1044,9 +1022,7 @@ }, "dependencies": { "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" + "version": "^0.2.1" }, "mkdirp": { "version": "0.5.5", @@ -1054,13 +1030,11 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^1.2.5" + "minimist": "^0.2.1" }, "dependencies": { "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "version": "^0.2.1", "dev": true } } @@ -1719,9 +1693,7 @@ "dev": true }, "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" + "version": "^0.2.1" }, "mkdirp": { "version": "0.5.5", @@ -1729,13 +1701,11 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^1.2.5" + "minimist": "^0.2.1" }, "dependencies": { "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "version": "^0.2.1", "dev": true } } @@ -1865,13 +1835,11 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -1882,13 +1850,11 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -1901,13 +1867,11 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -1919,9 +1883,7 @@ "dev": true }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "version": "^6.0.3" } } }, @@ -2599,21 +2561,17 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "version": "^6.0.3" }, "normalize-path": { "version": "3.0.0", @@ -2691,22 +2649,18 @@ }, "dependencies": { "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" + "version": "^0.2.1" }, "mkdirp": { "version": "0.5.0", "resolved": "https://box.jfrog.io/box/api/npm/boxnpm/mkdirp/-/mkdirp-0.5.0.tgz", "integrity": "sha1-HXMHam35hs2TROFecfzAWkyavxI=", "requires": { - "minimist": "0.0.8" + "minimist": "^0.2.1" }, "dependencies": { "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" + "version": "^0.2.1" } } } @@ -2753,7 +2707,7 @@ "requires": { "for-own": "^1.0.0", "is-plain-object": "^2.0.4", - "kind-of": "^6.0.0", + "kind-of": "^6.0.3", "shallow-clone": "^1.0.0" }, "dependencies": { @@ -2767,9 +2721,7 @@ } }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -2918,16 +2870,13 @@ "dev": true, "optional": true, "requires": { - "ini": "^1.3.4", + "ini": "^1.3.8", "proto-list": "~1.2.1" }, "dependencies": { "ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true, - "optional": true + "version": "^1.3.8", + "dev": true } } }, @@ -3476,13 +3425,11 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -3493,13 +3440,11 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -3512,13 +3457,11 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -3530,9 +3473,7 @@ "dev": true }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "version": "^6.0.3" } } }, @@ -4417,9 +4358,7 @@ "optional": true }, "ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + "version": "^1.3.8" }, "is-fullwidth-code-point": { "version": "1.0.0", @@ -4446,9 +4385,7 @@ } }, "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" + "version": "^0.2.1" }, "minipass": { "version": "2.9.0", @@ -4475,15 +4412,12 @@ "dev": true, "optional": true, "requires": { - "minimist": "^1.2.5" + "minimist": "^0.2.1" }, "dependencies": { "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", - "dev": true, - "optional": true + "version": "^0.2.1", + "dev": true } } }, @@ -4632,24 +4566,18 @@ "optional": true, "requires": { "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", + "ini": "^1.3.8", + "minimist": "^0.2.1", "strip-json-comments": "~2.0.1" }, "dependencies": { "ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true, - "optional": true + "version": "^1.3.8", + "dev": true }, "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", - "dev": true, - "optional": true + "version": "^0.2.1", + "dev": true } } }, @@ -5154,7 +5082,7 @@ "dev": true, "requires": { "is-number": "^3.0.0", - "kind-of": "^4.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "is-number": { @@ -5163,21 +5091,17 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -5737,13 +5661,11 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -5800,13 +5722,11 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -5826,13 +5746,11 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^5.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -6042,14 +5960,12 @@ "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz", "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=", "requires": { - "node-fetch": "^1.0.1", + "node-fetch": "^2.6.1", "whatwg-fetch": ">=0.10.0" }, "dependencies": { "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "version": "^2.6.1", "requires": { "whatwg-url": "^5.0.0" } @@ -6192,9 +6108,7 @@ } }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true }, "lazy-cache": { @@ -6256,21 +6170,17 @@ "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", "dev": true, "requires": { - "minimist": "^1.2.0" + "minimist": "^0.2.1" }, "dependencies": { "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "version": "^0.2.1", "dev": true } } }, "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" + "version": "^0.2.1" } } }, @@ -6470,7 +6380,7 @@ "decamelize": "^1.1.2", "loud-rejection": "^1.0.0", "map-obj": "^1.0.1", - "minimist": "^1.1.3", + "minimist": "^0.2.1", "normalize-package-data": "^2.3.4", "object-assign": "^4.0.1", "read-pkg-up": "^1.0.1", @@ -6479,9 +6389,7 @@ }, "dependencies": { "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "version": "^0.2.1", "dev": true } } @@ -6559,9 +6467,7 @@ } }, "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "version": "^0.2.1", "dev": true }, "mixin-deep": { @@ -6609,13 +6515,11 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^1.2.5" + "minimist": "^0.2.1" }, "dependencies": { "minimist": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", - "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "version": "^0.2.1", "dev": true } } @@ -6661,7 +6565,7 @@ "extend-shallow": "^3.0.2", "fragment-cache": "^0.2.1", "is-windows": "^1.0.2", - "kind-of": "^6.0.2", + "kind-of": "^6.0.3", "object.pick": "^1.3.0", "regex-not": "^1.0.0", "snapdragon": "^0.8.1", @@ -6700,9 +6604,7 @@ } }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -7120,7 +7022,7 @@ "requires": { "copy-descriptor": "^0.1.0", "define-property": "^0.2.5", - "kind-of": "^3.0.3" + "kind-of": "^6.0.3" }, "dependencies": { "define-property": { @@ -7133,9 +7035,7 @@ } }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -8172,13 +8072,11 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -8189,13 +8087,11 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -8208,21 +8104,17 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^5.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "version": "^6.0.3" } } }, @@ -8312,13 +8204,11 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -8329,13 +8219,11 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -8348,13 +8236,11 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -8365,13 +8251,11 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -8383,9 +8267,7 @@ "dev": true }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "version": "^6.0.3" }, "micromatch": { "version": "3.1.10", @@ -8400,7 +8282,7 @@ "extend-shallow": "^3.0.2", "extglob": "^2.0.4", "fragment-cache": "^0.2.1", - "kind-of": "^6.0.2", + "kind-of": "^6.0.3", "nanomatch": "^1.2.9", "object.pick": "^1.3.0", "regex-not": "^1.0.0", @@ -8409,9 +8291,7 @@ }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -8916,14 +8796,12 @@ "dev": true, "requires": { "is-extendable": "^0.1.1", - "kind-of": "^5.0.0", + "kind-of": "^6.0.3", "mixin-object": "^2.0.1" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -9010,13 +8888,11 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -9027,13 +8903,11 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -9046,13 +8920,11 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -9064,9 +8936,7 @@ "dev": true }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "version": "^6.0.3" } } }, @@ -9076,13 +8946,11 @@ "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", "dev": true, "requires": { - "kind-of": "^3.2.0" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -9725,13 +9593,11 @@ "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } @@ -9785,21 +9651,17 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^3.0.2" + "kind-of": "^6.0.3" }, "dependencies": { "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "version": "^6.0.3", "dev": true } } }, "kind-of": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "version": "^6.0.3" } } }, @@ -10311,7 +10173,7 @@ "string-width": "^1.0.2", "which-module": "^1.0.0", "y18n": "^3.2.1", - "yargs-parser": "^4.2.0" + "yargs-parser": "18.1.2" }, "dependencies": { "yargs-parser": { @@ -10463,7 +10325,7 @@ "string-width": "^1.0.2", "which-module": "^1.0.0", "y18n": "^3.2.1", - "yargs-parser": "^5.0.0" + "yargs-parser": "18.1.2" }, "dependencies": { "camelcase": { @@ -10504,4 +10366,4 @@ } } } -} +} \ No newline at end of file From 1d76ac50a36cdf7ead732665178f2b807b9f1c91 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 12 Jun 2024 11:37:02 -0700 Subject: [PATCH 022/105] Fix typo in comment - move to top of branch --- .../main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java index f2d51a3705..e8f67297c4 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java @@ -91,7 +91,7 @@ public void open(RawDocument input) { input.setAnnotation( new RemoveUntranslatedStategyAnnotation( RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); - // Post processing is disable for now, it will be enabled by the TranslsateStep if there are + // Post processing is disable for now, it will be enabled by the TranslateStep if there are // actual text unit to remove input.setAnnotation( new OutputDocumentPostProcessingAnnotation(JSONFilter::removeUntranslated, false)); From 909db032af3e4281321d8de2368757a8ca3c77fd Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 13 Jun 2024 23:47:09 -0700 Subject: [PATCH 023/105] Fix output ThirdPartySyncCommand --- .../com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java index 7adb810c07..1d6352ab80 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java @@ -156,7 +156,7 @@ public void execute() throws CommandException { .fg(CYAN) .a(skipAssetsWithPathPattern) .reset() - .a(" include-text-units-with-pattern") + .a(" include-text-units-with-pattern: ") .fg(CYAN) .a(includeTextUnitsWithPattern) .reset() From 5c5b12968410effbb6544fc72163016e9126f240 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 12 Jun 2024 18:31:14 -0700 Subject: [PATCH 024/105] AndroidStringDocumentMapper supports more Android escaping consider using the method from AndroidFilter --- .../strings/AndroidStringDocumentMapper.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java index 62c961d3bc..7d94a450c2 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java @@ -14,6 +14,8 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; + +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -266,11 +268,25 @@ static String removeInvalidControlCharacter(String str) { return withoutControlCharacters; } + /** + * should use {@link com.box.l10n.mojito.okapi.filters.AndroidFilter#unescape(String)} + */ static String unescape(String str) { - return Strings.nullToEmpty(str) + + String unescape = str; + + if (StringUtils.startsWith(unescape, "\"") + && StringUtils.endsWith(unescape, "\"")) { + unescape = + unescape.substring(1, unescape.length() - 1); + } + + unescape = Strings.nullToEmpty(unescape) .replaceAll("\\\\'", "'") .replaceAll("\\\\\"", "\"") .replaceAll("\\\\@", "@") .replaceAll("\\\\n", "\n"); + + return unescape; } } From c5ff3109958105eda9dc53790d2099be5ac6d9a2 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 13 Jun 2024 14:01:47 -0700 Subject: [PATCH 025/105] Improve removing comments from .strings - Remove multi blank line - Fix bug for when string had * --- .../cli/command/SimpleFileEditorCommand.java | 8 ++++++-- .../expected/en.lproj/Localizable.strings | Bin 396 -> 666 bytes .../input/en.lproj/Localizable.strings | Bin 1100 -> 1540 bytes 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java index a432892729..3306baff4e 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java @@ -41,7 +41,7 @@ public class SimpleFileEditorCommand extends Command { /** logger */ static Logger logger = LoggerFactory.getLogger(SimpleFileEditorCommand.class); - static final String COMMENTS_PATTERN = "(?s)/*\\*.*?\\*/"; + static final String COMMENTS_PATTERN = "(?s)/\\*.*?\\*/"; @Autowired ConsoleWriter consoleWriter; @@ -224,7 +224,11 @@ void removeCommentsInMacStrings() throws CommandException { .a(inputPath.toString()) .print(); String modifiedContent = - commandHelper.getFileContent(inputPath).replaceAll(COMMENTS_PATTERN, ""); + commandHelper + .getFileContent(inputPath) + .replaceAll(COMMENTS_PATTERN, "") + .replaceFirst("^\n+", "") + .replaceAll("\n{2,}", "\n\n"); writeOutputFile(inputPath, modifiedContent); }); } diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/expected/en.lproj/Localizable.strings b/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/expected/en.lproj/Localizable.strings index c94f2639d180a0de2d2664ed71cd9d37717af384..b096e6b6d90de97f1d91304bf365af98880d2724 100644 GIT binary patch literal 666 zcmchU&5FW65QOWzPtgRtco02!SrPO-1c@fB5sjL}Ltb9}dZJ{*u6mPZnC_XXuKu}R zB8@b}mRf70SOsUTN-5q_33o|cDASSGOb6w>M>0)0krtdLgM0Q*7JHKA8*UC~qA$#| zjLE!5!xa*pLoE-@R?%^G$NgqICszH2>(w}_%iPeH>%G=BCe+}%?0uL@#qT?Qi)=IW zG^3w_NuQ`83tkyIOzHAKToYAzrWGg)Y6I>$y=>u>{92fssgISQs<5qGKScgE2-|6P X_BqAJZhP8c2~{>7rJZX2{!MfPvaxC9 delta 37 rcmbQm+QZEI{~s5F5`!Uw0T9Md3>2Oymou@$Vd62HNsJDYZ5Uku&esb# diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/input/en.lproj/Localizable.strings b/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/input/en.lproj/Localizable.strings index dab1ad8c2c6458a3985efa549cf2472ffb5b7723..5fca63d2cf71596d4844018f62cf8522ecc371c2 100644 GIT binary patch literal 1540 zcmchX!A`dd<3!u@9N1wpN6k7p>7Tap#%d6k)x|9MH5n>wX%yjnu zXaC)q&yS9TV&qX~l1m{a_Jd@y;7KI4ekdIYq$OQB=eg(mR3fq?tdgvpSj@Y?&%s#| zoq>@OZ-IBguO)XY1htf96U8*33|O18W=ItQwS;m5{v1Zbe+JSsR!l6i{=|B(wnAS? zK+HX(?={h?%0e!|_22zl6;X~hN(|V0b4G7R&WJi)ZJRumntk+Hz@j?EQm?nttt)+M zb9u*UVC1)Z_Zl4fu4~rYq1mB`KHnpItxtj7bVk()>A6JmzUO@O?rGL>r?bvF_aD=D&8wI`_hMZVC22_A0Z^Z{!w??K!e!N142F<8@lZWQ Date: Mon, 17 Jun 2024 22:07:54 -0700 Subject: [PATCH 026/105] Add dependency for error handling in OAuth2 --- webapp/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/webapp/pom.xml b/webapp/pom.xml index 3a43625de3..ddcb6a4c96 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -40,6 +40,11 @@ spring-boot-starter-web + + org.springframework.boot + spring-boot-starter-oauth2-client + + org.springframework.boot spring-boot-starter-mustache From 541cc0164b761ecdd7bfc612c7e81df8235c693e Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 24 Jun 2024 09:57:27 -0700 Subject: [PATCH 027/105] Add simple Integrity check for Python FStrings --- .../command/RepositoryTmTranslateCommand.java | 316 +++++++++--------- .../strings/AndroidStringDocumentMapper.java | 22 +- ...ortTranslationsFromLocalizedAssetStep.java | 2 +- .../IntegrityCheckerType.java | 3 +- .../PythonFStringIntegrityChecker.java | 18 + ...ythonFStringIntegrityCheckerException.java | 7 + .../PythonFStringIntegrityCheckerTest.java | 40 +++ 7 files changed, 244 insertions(+), 164 deletions(-) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerException.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerTest.java diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java index 9a495adfa1..0ed54713b4 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java @@ -10,6 +10,11 @@ import com.box.l10n.mojito.rest.entity.PollableTask; import com.box.l10n.mojito.rest.entity.Repository; import com.box.l10n.mojito.rest.entity.RepositoryLocale; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; import org.fusesource.jansi.Ansi; import org.fusesource.jansi.Ansi.Color; import org.slf4j.Logger; @@ -18,173 +23,186 @@ import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - /** - * Command to populate untranslated string with translation from another repository translation memory. - * This meant to process small volumes but can be helpful to combine the with the import localized files command. - *

- * Matches the text units by the source. Use text unit name and comment match as priority (moved assets), else just take the - * newest translation match. Has an option to not import string that have same source and target to avoid - * copy translation that might be from importing partly translated files. The drawback is that it could skip - * string that should be imported but the benefits is that the localized files will be same regardless. - *

- * Limited to translating at most 1000 strings in a run - *

+ * Command to populate untranslated string with translation from another repository translation + * memory. This meant to process small volumes but can be helpful to combine the with the import + * localized files command. + * + *

Matches the text units by the source. Use text unit name and comment match as priority (moved + * assets), else just take the newest translation match. Has an option to not import string that + * have same source and target to avoid copy translation that might be from importing partly + * translated files. The drawback is that it could skip string that should be imported but the + * benefits is that the localized files will be same regardless. + * + *

Limited to translating at most 1000 strings in a run * * @author jaurambault */ @Component @Scope("prototype") @Parameters( - commandNames = {"repository-translate"}, - commandDescription = "Translate a repository using another repository TM. Only untranslated text units" + - " will be processed, no overriding. Priorities text unit that have same and comment, else just look" + - "at the newest match") + commandNames = {"repository-translate"}, + commandDescription = + "Translate a repository using another repository TM. Only untranslated text units" + + " will be processed, no overriding. Priorities text unit that have same and comment, else just look" + + "at the newest match") public class RepositoryTmTranslateCommand extends Command { - /** - * logger - */ - static Logger logger = LoggerFactory.getLogger(RepositoryTmTranslateCommand.class); - - @Autowired - ConsoleWriter consoleWriter; - - @Parameter( - names = {Param.SOURCE_REPOSITORY_LONG, Param.SOURCE_REPOSITORY_SHORT}, - arity = 1, - required = false, - description = Param.SOURCE_REPOSITORY_DESCRIPTION) - String sourceRepositoryParam; - - @Parameter( - names = {Param.TARGET_REPOSITORY_LONG, Param.TARGET_REPOSITORY_SHORT}, - arity = 1, - required = true, - description = Param.TARGET_REPOSITORY_DESCRIPTION) - String targetRepositoryParam; - - @Parameter( - names = {"--import-source-equals-target"}, - arity = 0, - description = "Import the translation if the source is equal to the target") - boolean importSourceEqualsTargetParam = false; - - @Parameter( - names = {Param.REPOSITORY_LOCALES_LONG, Param.REPOSITORY_LOCALES_SHORT}, - variableArity = true, - required = false, - description = "List of locales (bcp47 tags) to translate") - List localesParam = null; - - @Autowired - CommandHelper commandHelper; - - @Autowired - TextUnitClient textUnitClient; - - @Autowired - RepositoryClient repositoryClient; - - @Override - public void execute() throws CommandException { - - if (sourceRepositoryParam == null) { - sourceRepositoryParam = targetRepositoryParam; - } + /** logger */ + static Logger logger = LoggerFactory.getLogger(RepositoryTmTranslateCommand.class); - consoleWriter - .newLine() - .a("Translate repository: ") - .fg(Color.CYAN) - .a(targetRepositoryParam) - .reset() - .a(" with text units from repository: ") - .fg(Color.CYAN) - .a(sourceRepositoryParam) - .println(2); - - Repository sourceRepository = commandHelper.findRepositoryByName(sourceRepositoryParam); - Repository targetRepository = commandHelper.findRepositoryByName(targetRepositoryParam); - - List locales = localesParam; - if (locales == null) { - locales = targetRepository.getRepositoryLocales().stream().filter(rl -> rl.getParentLocale() != null).map(RepositoryLocale::getLocale).map(Locale::getBcp47Tag).toList(); - } + @Autowired ConsoleWriter consoleWriter; - for (String locale : locales) { - consoleWriter.a("Processing locale: ").fg(Color.MAGENTA).a(locale).println(); - List textUnitsToSave = new ArrayList<>(); - - List untranslatedTextUnitsForLocale = getUntranslatedTextUnitsForLocale(targetRepository.getName(), locale); - - for (TextUnitClient.TextUnit textUnit : untranslatedTextUnitsForLocale) { - consoleWriter.a("Processing source: ").fg(Color.CYAN).a(textUnit.source()).println(); - List existingTranslationWithSameSource = getTranslationsWithExactSource(sourceRepository.getName(), locale, textUnit.source()); - - Optional match = getMatchByNameAndComment(textUnit.source(), textUnit.comment(), existingTranslationWithSameSource); - if (match.isPresent()) { - match = getMatchByNewest(existingTranslationWithSameSource); - } - - if (match.isPresent()) { - var translation = match.get(); - consoleWriter.a("Found match: ").fg(Color.CYAN).a(translation.target()).println(); - - if (translation.source().equals(translation.target())) { - if (importSourceEqualsTargetParam) { - textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); - } else { - consoleWriter.a("Skip import, source = target").println(); - } - } else { - textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); - } - } else { - consoleWriter.a("No match found").println(); - } - consoleWriter.println(); - } + @Parameter( + names = {Param.SOURCE_REPOSITORY_LONG, Param.SOURCE_REPOSITORY_SHORT}, + arity = 1, + required = false, + description = Param.SOURCE_REPOSITORY_DESCRIPTION) + String sourceRepositoryParam; - consoleWriter.a("Saving translations for locale: ").fg(Color.MAGENTA).a(locale).reset().a("Text unit count: ").fg(Color.CYAN).a(textUnitsToSave.size()).println(); - TextUnitClient.ImportTextUnitsBatch importTextUnitsBatch = new TextUnitClient.ImportTextUnitsBatch(false, true, textUnitsToSave); - PollableTask pollableTask = textUnitClient.importTextUnitBatch(importTextUnitsBatch); - commandHelper.waitForPollableTask(pollableTask.getId()); - } - consoleWriter.fg(Ansi.Color.GREEN).newLine().a("Finished").println(2); - } + @Parameter( + names = {Param.TARGET_REPOSITORY_LONG, Param.TARGET_REPOSITORY_SHORT}, + arity = 1, + required = true, + description = Param.TARGET_REPOSITORY_DESCRIPTION) + String targetRepositoryParam; - private List getTranslationsWithExactSource(String repositoryName, String locale, String source) { - TextUnitClient.TextUnitSearchBody textUnitSearchBody = new TextUnitClient.TextUnitSearchBody(); - textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); - textUnitSearchBody.setLocaleTags(List.of(locale)); - textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.TRANSLATED); - textUnitSearchBody.setSource(source); - textUnitSearchBody.setLimit(20); - return textUnitClient.searchTextUnits(textUnitSearchBody); + @Parameter( + names = {"--import-source-equals-target"}, + arity = 0, + description = "Import the translation if the source is equal to the target") + boolean importSourceEqualsTargetParam = false; - } + @Parameter( + names = {Param.REPOSITORY_LOCALES_LONG, Param.REPOSITORY_LOCALES_SHORT}, + variableArity = true, + required = false, + description = "List of locales (bcp47 tags) to translate") + List localesParam = null; + + @Autowired CommandHelper commandHelper; - private List getUntranslatedTextUnitsForLocale(String repositoryName, String locale) { - TextUnitClient.TextUnitSearchBody textUnitSearchBody = new TextUnitClient.TextUnitSearchBody(); - textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); - textUnitSearchBody.setLocaleTags(List.of(locale)); - textUnitSearchBody.setUsedFilter(TextUnitClient.UsedFilter.USED); - textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.UNTRANSLATED); - textUnitSearchBody.setLimit(1000); - return textUnitClient.searchTextUnits(textUnitSearchBody); + @Autowired TextUnitClient textUnitClient; + + @Autowired RepositoryClient repositoryClient; + + @Override + public void execute() throws CommandException { + + if (sourceRepositoryParam == null) { + sourceRepositoryParam = targetRepositoryParam; } - private Optional getMatchByNewest(List candidates) { - return candidates.stream().max(Comparator.comparingLong(TextUnitClient.TextUnit::createdDate)); + consoleWriter + .newLine() + .a("Translate repository: ") + .fg(Color.CYAN) + .a(targetRepositoryParam) + .reset() + .a(" with text units from repository: ") + .fg(Color.CYAN) + .a(sourceRepositoryParam) + .println(2); + + Repository sourceRepository = commandHelper.findRepositoryByName(sourceRepositoryParam); + Repository targetRepository = commandHelper.findRepositoryByName(targetRepositoryParam); + + List locales = localesParam; + if (locales == null) { + locales = + targetRepository.getRepositoryLocales().stream() + .filter(rl -> rl.getParentLocale() != null) + .map(RepositoryLocale::getLocale) + .map(Locale::getBcp47Tag) + .toList(); } - private static Optional getMatchByNameAndComment(String name, String comment, List candidates) { - return candidates.stream().filter(m -> Objects.equals(name, m.name()) && Objects.equals(comment, m.comment())).max(Comparator.comparingLong(TextUnitClient.TextUnit::createdDate)); + for (String locale : locales) { + consoleWriter.a("Processing locale: ").fg(Color.MAGENTA).a(locale).println(); + List textUnitsToSave = new ArrayList<>(); + + List untranslatedTextUnitsForLocale = + getUntranslatedTextUnitsForLocale(targetRepository.getName(), locale); + + for (TextUnitClient.TextUnit textUnit : untranslatedTextUnitsForLocale) { + consoleWriter.a("Processing source: ").fg(Color.CYAN).a(textUnit.source()).println(); + List existingTranslationWithSameSource = + getTranslationsWithExactSource(sourceRepository.getName(), locale, textUnit.source()); + + Optional match = + getMatchByNameAndComment( + textUnit.source(), textUnit.comment(), existingTranslationWithSameSource); + if (match.isPresent()) { + match = getMatchByNewest(existingTranslationWithSameSource); + } + + if (match.isPresent()) { + var translation = match.get(); + consoleWriter.a("Found match: ").fg(Color.CYAN).a(translation.target()).println(); + + if (translation.source().equals(translation.target())) { + if (importSourceEqualsTargetParam) { + textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); + } else { + consoleWriter.a("Skip import, source = target").println(); + } + } else { + textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); + } + } else { + consoleWriter.a("No match found").println(); + } + consoleWriter.println(); + } + + consoleWriter + .a("Saving translations for locale: ") + .fg(Color.MAGENTA) + .a(locale) + .reset() + .a("Text unit count: ") + .fg(Color.CYAN) + .a(textUnitsToSave.size()) + .println(); + TextUnitClient.ImportTextUnitsBatch importTextUnitsBatch = + new TextUnitClient.ImportTextUnitsBatch(false, true, textUnitsToSave); + PollableTask pollableTask = textUnitClient.importTextUnitBatch(importTextUnitsBatch); + commandHelper.waitForPollableTask(pollableTask.getId()); } + consoleWriter.fg(Ansi.Color.GREEN).newLine().a("Finished").println(2); + } + + private List getTranslationsWithExactSource( + String repositoryName, String locale, String source) { + TextUnitClient.TextUnitSearchBody textUnitSearchBody = new TextUnitClient.TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.TRANSLATED); + textUnitSearchBody.setSource(source); + textUnitSearchBody.setLimit(20); + return textUnitClient.searchTextUnits(textUnitSearchBody); + } + + private List getUntranslatedTextUnitsForLocale( + String repositoryName, String locale) { + TextUnitClient.TextUnitSearchBody textUnitSearchBody = new TextUnitClient.TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setUsedFilter(TextUnitClient.UsedFilter.USED); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.UNTRANSLATED); + textUnitSearchBody.setLimit(1000); + return textUnitClient.searchTextUnits(textUnitSearchBody); + } + + private Optional getMatchByNewest( + List candidates) { + return candidates.stream().max(Comparator.comparingLong(TextUnitClient.TextUnit::createdDate)); + } + + private static Optional getMatchByNameAndComment( + String name, String comment, List candidates) { + return candidates.stream() + .filter(m -> Objects.equals(name, m.name()) && Objects.equals(comment, m.comment())) + .max(Comparator.comparingLong(TextUnitClient.TextUnit::createdDate)); + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java index 7d94a450c2..7a1b1bd227 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java @@ -14,7 +14,6 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; - import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -268,24 +267,21 @@ static String removeInvalidControlCharacter(String str) { return withoutControlCharacters; } - /** - * should use {@link com.box.l10n.mojito.okapi.filters.AndroidFilter#unescape(String)} - */ + /** should use {@link com.box.l10n.mojito.okapi.filters.AndroidFilter#unescape(String)} */ static String unescape(String str) { String unescape = str; - if (StringUtils.startsWith(unescape, "\"") - && StringUtils.endsWith(unescape, "\"")) { - unescape = - unescape.substring(1, unescape.length() - 1); + if (StringUtils.startsWith(unescape, "\"") && StringUtils.endsWith(unescape, "\"")) { + unescape = unescape.substring(1, unescape.length() - 1); } - unescape = Strings.nullToEmpty(unescape) - .replaceAll("\\\\'", "'") - .replaceAll("\\\\\"", "\"") - .replaceAll("\\\\@", "@") - .replaceAll("\\\\n", "\n"); + unescape = + Strings.nullToEmpty(unescape) + .replaceAll("\\\\'", "'") + .replaceAll("\\\\\"", "\"") + .replaceAll("\\\\@", "@") + .replaceAll("\\\\n", "\n"); return unescape; } diff --git a/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java b/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java index 6f97d972fb..c02ade6e2a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java +++ b/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java @@ -160,7 +160,7 @@ protected TMTextUnitVariant importTextUnit( tmTextUnitVariantCommentAnnotation.setMessage(integrityCheckException.getMessage()); tmTextUnitVariantCommentAnnotation.setSeverity( - TMTextUnitVariantComment.Severity.ERROR); //TODO(ja) dial it down for plural strings? + TMTextUnitVariantComment.Severity.ERROR); // TODO(ja) dial it down for plural strings? new TMTextUnitVariantCommentAnnotations(target) .addAnnotation(tmTextUnitVariantCommentAnnotation); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java index 6c8ac7e7bc..df8e85f24d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java @@ -22,7 +22,8 @@ public enum IntegrityCheckerType { ELLIPSIS(EllipsisIntegrityChecker.class.getName()), BACKQUOTE(BackquoteIntegrityChecker.class.getName()), EMPTY_TARGET_NOT_EMPTY_SOURCE(EmptyTargetNotEmptySourceIntegrityChecker.class.getName()), - MARKDOWN_LINKS(MarkdownLinkIntegrityChecker.class.getName()); + MARKDOWN_LINKS(MarkdownLinkIntegrityChecker.class.getName()), + PYTHON_FPRINT(PythonFStringIntegrityChecker.class.getName()); String className; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java new file mode 100644 index 0000000000..b4dea6bd3c --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java @@ -0,0 +1,18 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class PythonFStringIntegrityChecker extends RegexIntegrityChecker { + + @Override + public String getRegex() { + return "\\$\\{?[a-zA-Z_][a-zA-Z0-9_]*\\}?"; + } + + @Override + public void check(String content, String target) { + try { + super.check(content, target); + } catch (RegexCheckerException ex) { + throw new PythonFStringIntegrityCheckerException("Variable types do not match."); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerException.java new file mode 100644 index 0000000000..43724d91fa --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class PythonFStringIntegrityCheckerException extends IntegrityCheckException { + public PythonFStringIntegrityCheckerException(String message) { + super(message); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerTest.java new file mode 100644 index 0000000000..7a8c223800 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerTest.java @@ -0,0 +1,40 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.Test; + +public class PythonFStringIntegrityCheckerTest { + + PythonFStringIntegrityChecker checker = new PythonFStringIntegrityChecker(); + + @Test + public void testPlaceholderOK() { + String source = "This is a $placeholder"; + String target = "C'est un $placeholder"; + checker.check(source, target); + } + + @Test + public void testPlaceholderMissing() { + String source = "This is a $placeholder"; + String target = "C'est un $placehor"; + assertThrowsExactly( + PythonFStringIntegrityCheckerException.class, () -> checker.check(source, target)); + } + + @Test + public void testPlaceholderCurlyOK() { + String source = "This is a ${placeholder}"; + String target = "C'est un ${placeholder}"; + checker.check(source, target); + } + + @Test + public void testPlaceholderCurlyMissing() { + String source = "This is a ${placeholder}"; + String target = "C'est un ${placehor"; + assertThrowsExactly( + PythonFStringIntegrityCheckerException.class, () -> checker.check(source, target)); + } +} From ba9885636e85e1dfe5109fd5b50db5dbcef0a6d1 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 24 Jun 2024 13:45:09 -0700 Subject: [PATCH 028/105] Register Python fprint integrity checks --- .../com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java index 71733a36e4..85aef06aeb 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java @@ -20,5 +20,6 @@ public enum IntegrityCheckerType { ELLIPSIS, BACKQUOTE, EMPTY_TARGET_NOT_EMPTY_SOURCE, - MARKDOWN_LINKS; + MARKDOWN_LINKS, + PYTHON_FPRINT; } From 1f73d804c204376be0cf4c3786a5cb604538739a Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 24 Jun 2024 14:05:24 -0700 Subject: [PATCH 029/105] Add a specific type for FormatJS JSON --- .../mojito/cli/filefinder/file/FileTypes.java | 1 + .../file/FormatJSJSONNoBasenameFileType.java | 19 ++++++++++++ .../ImportLocalizedAssetCommandTest.java | 25 +++------------- .../mojito/cli/command/PullCommandTest.java | 30 ++++--------------- .../translations/source-xliff_fr-FR.xliff | 10 +++---- .../translations/source-xliff_ja-JP.xliff | 10 +++---- .../translations/source-xliff_fr-FR.xliff | 8 ++--- .../translations/source-xliff_ja-JP.xliff | 6 ++-- 8 files changed, 46 insertions(+), 63 deletions(-) create mode 100644 cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FormatJSJSONNoBasenameFileType.java diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FileTypes.java b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FileTypes.java index 59d53670eb..d3700ae6b1 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FileTypes.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FileTypes.java @@ -24,6 +24,7 @@ public enum FileTypes { JSON(JSONFileType.class), JSON_NOBASENAME(JSONNoBasenameFileType.class), CHROME_EXT_JSON(ChromeExtensionJSONFileType.class), + FORMATJS_JSON_NOBASENAME(FormatJSJSONNoBasenameFileType.class), I18NEXT_PARSER_JSON(I18NextFileType.class), TS(TSFileType.class), YAML(YamlFileType.class), diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FormatJSJSONNoBasenameFileType.java b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FormatJSJSONNoBasenameFileType.java new file mode 100644 index 0000000000..dd3ad2bee1 --- /dev/null +++ b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FormatJSJSONNoBasenameFileType.java @@ -0,0 +1,19 @@ +package com.box.l10n.mojito.cli.filefinder.file; + +import java.util.Arrays; + +/** + * @author jeanaurambault + */ +public class FormatJSJSONNoBasenameFileType extends LocaleAsFileNameType { + + public FormatJSJSONNoBasenameFileType() { + this.sourceFileExtension = "json"; + this.defaultFilterOptions = + Arrays.asList( + "noteKeyPattern=description", + "extractAllPairs=false", + "exceptions=defaultMessage", + "removeKeySuffix=/defaultMessage"); + } +} diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java index 96afcc68ba..e706d7f130 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java @@ -637,11 +637,7 @@ public void importJsonDefaultFormatJs() throws Exception { "-s", getInputResourcesTestDir("source").getAbsolutePath(), "-ft", - "JSON_NOBASENAME", - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage"); + "FORMATJS_JSON_NOBASENAME"); getL10nJCommander() .run( @@ -653,11 +649,7 @@ public void importJsonDefaultFormatJs() throws Exception { "-t", getInputResourcesTestDir("translations").getAbsolutePath(), "-ft", - "JSON_NOBASENAME", - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage"); + "FORMATJS_JSON_NOBASENAME"); getL10nJCommander() .run( @@ -669,11 +661,7 @@ public void importJsonDefaultFormatJs() throws Exception { "-t", getTargetTestDir().getAbsolutePath(), "-ft", - "JSON_NOBASENAME", - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage"); + "FORMATJS_JSON_NOBASENAME"); checkExpectedGeneratedResources(); } @@ -691,12 +679,7 @@ public void importJsonDefaultFormatJsCompiled() throws Exception { "-s", getInputResourcesTestDir("source").getAbsolutePath(), "-ft", - "JSON_NOBASENAME", - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", - "removeKeySuffix=/defaultMessage"); + "FORMATJS_JSON_NOBASENAME"); getL10nJCommander() .run( diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java index f66d48be26..c6ca8da260 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java @@ -1620,12 +1620,8 @@ public void pullJsonDefaultFormatJs() throws Exception { repository.getName(), "-s", getInputResourcesTestDir("source").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME"); + "FORMATJS_JSON_NOBASENAME"); Asset asset = assetClient.getAssetByPathAndRepositoryId("en.json", repository.getId()); importTranslations(asset.getId(), "source-xliff_", "fr-FR"); @@ -1640,12 +1636,8 @@ public void pullJsonDefaultFormatJs() throws Exception { getInputResourcesTestDir("source").getAbsolutePath(), "-t", getTargetTestDir("target").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME"); + "FORMATJS_JSON_NOBASENAME"); getL10nJCommander() .run( @@ -1656,12 +1648,8 @@ public void pullJsonDefaultFormatJs() throws Exception { getInputResourcesTestDir("source_modified").getAbsolutePath(), "-t", getTargetTestDir("target_modified").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME"); + "FORMATJS_JSON_NOBASENAME"); checkExpectedGeneratedResources(); } @@ -1677,12 +1665,8 @@ public void pullJsonDefaultFormatJsRemoveUntranslated() throws Exception { repository.getName(), "-s", getInputResourcesTestDir("source").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME"); + "FORMATJS_JSON_NOBASENAME"); Asset asset = assetClient.getAssetByPathAndRepositoryId("en.json", repository.getId()); importTranslations(asset.getId(), "source-xliff_", "fr-FR"); @@ -1697,12 +1681,8 @@ public void pullJsonDefaultFormatJsRemoveUntranslated() throws Exception { getInputResourcesTestDir("source").getAbsolutePath(), "-t", getTargetTestDir("target").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME", + "FORMATJS_JSON_NOBASENAME", "--inheritance-mode", "REMOVE_UNTRANSLATED"); diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_fr-FR.xliff b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_fr-FR.xliff index 270f4f848e..7364cce233 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_fr-FR.xliff +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_fr-FR.xliff @@ -2,26 +2,26 @@ - + 100 character description: Description de 100 caractères : - + 15 min 15 min File lock dialog duration - + 1 day 1 jour File lock dialog duration - + 1 hour 1 heure File lock dialog duration - + 1 month 1 mois File lock dialog duration diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_ja-JP.xliff b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_ja-JP.xliff index 7a27bc0549..0f9c2f5b5c 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_ja-JP.xliff +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_ja-JP.xliff @@ -2,26 +2,26 @@ - + 100 character description: 100文字の説明: - + 15 min 15分 File lock dialog duration - + 1 day 1日 File lock dialog duration - + 1 hour 1時間 File lock dialog duration - + 1 month 1か月 File lock dialog duration diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_fr-FR.xliff b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_fr-FR.xliff index 2420b4e634..89602f4de7 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_fr-FR.xliff +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_fr-FR.xliff @@ -2,21 +2,21 @@ - + 100 character description: Description de 100 caractères : - + 15 min 15 min File lock dialog duration - + 1 hour 1 heure File lock dialog duration - + 1 month 1 mois File lock dialog duration diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_ja-JP.xliff b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_ja-JP.xliff index cf041d39be..f383f05e1d 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_ja-JP.xliff +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_ja-JP.xliff @@ -2,16 +2,16 @@ - + 100 character description: 100文字の説明: - + 1 hour 1時間 File lock dialog duration - + 1 month 1か月 File lock dialog duration From b10c23266b176d7aadb764d00f9c10d5d4e6f6ac Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 10 Jun 2024 14:55:27 -0700 Subject: [PATCH 030/105] Initial version of the Phrase connector --- webapp/pom.xml | 6 + .../thirdparty/ThirdPartyTMSPhrase.java | 282 +++++++++++++ .../ThridPartyTMSPhraseException.java | 7 + .../thirdparty/phrase/PhraseClient.java | 390 ++++++++++++++++++ .../thirdparty/phrase/PhraseClientConfig.java | 23 ++ .../phrase/PhraseClientException.java | 22 + .../phrase/PhraseClientPropertiesConfig.java | 19 + .../thirdparty/ThirdPartyServiceTestData.java | 4 + .../thirdparty/ThirdPartyTMSPhraseTest.java | 91 ++++ .../thirdparty/phrase/PhraseClientTest.java | 113 +++++ 10 files changed, 957 insertions(+) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java diff --git a/webapp/pom.xml b/webapp/pom.xml index ddcb6a4c96..0bc0b1cfe4 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -60,6 +60,12 @@ spring-boot-starter-actuator + + com.phrase + phrase-java + 2.0.2 + + org.springframework.boot spring-boot-starter-data-jpa diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java new file mode 100644 index 0000000000..7d48480d52 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -0,0 +1,282 @@ +package com.box.l10n.mojito.service.thirdparty; + +import com.box.l10n.mojito.JSR310Migration; +import com.box.l10n.mojito.android.strings.AndroidStringDocument; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentReader; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter; +import com.box.l10n.mojito.entity.PollableTask; +import com.box.l10n.mojito.entity.Repository; +import com.box.l10n.mojito.entity.RepositoryLocale; +import com.box.l10n.mojito.service.pollableTask.PollableFuture; +import com.box.l10n.mojito.service.repository.RepositoryService; +import com.box.l10n.mojito.service.thirdparty.phrase.PhraseClient; +import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; +import com.box.l10n.mojito.service.tm.search.StatusFilter; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.box.l10n.mojito.service.tm.search.UsedFilter; +import com.google.common.collect.ImmutableList; +import com.phrase.client.model.Tag; +import com.phrase.client.model.TranslationKey; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@ConditionalOnProperty(value = "l10n.ThirdPartyTMS.impl", havingValue = "ThirdPartyTMSPhrase") +@Component +public class ThirdPartyTMSPhrase implements ThirdPartyTMS { + + static final String TAG_PREFIX = "push_"; + + static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhrase.class); + + @Autowired TextUnitSearcher textUnitSearcher = new TextUnitSearcher(); + + @Autowired TextUnitBatchImporterService textUnitBatchImporterService; + + @Autowired(required = false) + PhraseClient phraseClient; + + @Autowired RepositoryService repositoryService; + + @Override + public void removeImage(String projectId, String imageId) { + throw new UnsupportedOperationException("Remove image is not supported"); + } + + @Override + public ThirdPartyTMSImage uploadImage(String projectId, String name, byte[] content) { + throw new UnsupportedOperationException("Upload image is not supported"); + } + + @Override + public List getThirdPartyTextUnits( + Repository repository, String projectId, List optionList) { + + List thirdPartyTextUnits = new ArrayList<>(); + + List phraseTranslationKeys = phraseClient.getKeys(projectId); + + for (TranslationKey translationKey : phraseTranslationKeys) { + + String[] nameParts = translationKey.getName().split("#@#", 3); + + if (nameParts.length != 3) { + logger.error( + "Skipping entry. Name: {} should have 3 parts. Missing part could happen in old project", + translationKey.getName()); + continue; + } + + String idSection = nameParts[0]; + if (!idSection.contains(",")) { + ThirdPartyTextUnit thirdPartyTextUnit = new ThirdPartyTextUnit(); + thirdPartyTextUnit.setAssetPath(nameParts[1]); + thirdPartyTextUnit.setName(nameParts[2]); + thirdPartyTextUnit.setId(translationKey.getId()); + thirdPartyTextUnit.setTmTextUnitId(Long.valueOf(idSection)); + thirdPartyTextUnits.add(thirdPartyTextUnit); + } else { + List ids = Arrays.stream(idSection.split(",")).map(Long::valueOf).toList(); + for (Long id : ids) { + ThirdPartyTextUnit thirdPartyTextUnit = new ThirdPartyTextUnit(); + thirdPartyTextUnit.setAssetPath(nameParts[1]); + thirdPartyTextUnit.setName(nameParts[2]); + thirdPartyTextUnit.setId(translationKey.getId()); + thirdPartyTextUnit.setTmTextUnitId(id); + thirdPartyTextUnits.add(thirdPartyTextUnit); + } + } + } + + return thirdPartyTextUnits; + } + + @Override + public void createImageToTextUnitMappings( + String projectId, List thirdPartyImageToTextUnits) { + throw new UnsupportedOperationException("Create image to text units is not supported"); + } + + @Override + public void push( + Repository repository, + String projectId, + String pluralSeparator, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + List options) { + + List search = + getSourceTextUnitDTOs(repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); + + String text = getFileContent(pluralSeparator, search, true); + + String tagForUpload = getTagForUpload(); + phraseClient.uploadAndWait( + projectId, + repository.getSourceLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + text, + ImmutableList.of(tagForUpload)); + + phraseClient.removeKeysNotTaggedWith(projectId, tagForUpload); + + List tagsToDelete = + phraseClient.listTags(projectId).stream() + .filter( + tag -> + tag.getName() != null + && !tag.getName().equals(tagForUpload) + && tag.getName().startsWith(TAG_PREFIX)) + .toList(); + phraseClient.deleteTags(projectId, tagsToDelete); + } + + private List getSourceTextUnitDTOs( + Repository repository, String skipTextUnitsWithPattern, String skipAssetsWithPathPattern) { + TextUnitSearcherParameters parameters = new TextUnitSearcherParameters(); + + parameters.setRepositoryIds(repository.getId()); + parameters.setForRootLocale(true); + parameters.setDoNotTranslateFilter(false); + parameters.setUsedFilter(UsedFilter.USED); + parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); + parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + parameters.setPluralFormsFiltered(false); + parameters.setOrderByTextUnitID(true); + + return textUnitSearcher.search(parameters); + } + + public static String getTagForUpload() { + ZonedDateTime zonedDateTime = JSR310Migration.dateTimeNowInUTC(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy_MM_dd_HH_mm_ss_SSS"); + return ("%s%s_%s") + .formatted( + TAG_PREFIX, + formatter.format(zonedDateTime), + Math.abs(UUID.randomUUID().getLeastSignificantBits() % 1000)); + } + + @Override + public PollableFuture pull( + Repository repository, + String projectId, + String pluralSeparator, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + List optionList, + String schedulerName, + PollableTask currentTask) { + + Set repositoryLocalesWithoutRootLocale = + repositoryService.getRepositoryLocalesWithoutRootLocale(repository); + + for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale) { + String localeTag = repositoryLocale.getLocale().getBcp47Tag(); + logger.info("Downloading locale: {} from Phrase", localeTag); + String fileContent = phraseClient.localeDownload(projectId, localeTag, "xml"); + + AndroidStringDocumentMapper mapper = + new AndroidStringDocumentMapper( + pluralSeparator, null, localeTag, repository.getName(), true); + + List textUnitDTOS = + mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); + + textUnitBatchImporterService.importTextUnits(textUnitDTOS, false, true); + } + + return null; + } + + @Override + public void pushTranslations( + Repository repository, + String projectId, + String pluralSeparator, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + List optionList) { + + Set repositoryLocalesWithoutRootLocale = + repositoryService.getRepositoryLocalesWithoutRootLocale(repository); + + for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale) { + List textUnitDTOS = + getTextUnitDTOSForLocale( + repository, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern, + repositoryLocale); + + String fileContent = getFileContent(pluralSeparator, textUnitDTOS, false); + + phraseClient.uploadCreateFile( + projectId, + repositoryLocale.getLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + fileContent, + null); + } + } + + private List getTextUnitDTOSForLocale( + Repository repository, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + RepositoryLocale repositoryLocale) { + TextUnitSearcherParameters parameters = new TextUnitSearcherParameters(); + parameters.setRepositoryIds(repository.getId()); + parameters.setLocaleId(repositoryLocale.getLocale().getId()); + parameters.setDoNotTranslateFilter(false); + parameters.setStatusFilter(StatusFilter.TRANSLATED); + parameters.setUsedFilter(UsedFilter.USED); + parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); + parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + parameters.setIncludeTextUnitsWithPattern(includeTextUnitsWithPattern); + parameters.setPluralFormsFiltered(true); + return textUnitSearcher.search(parameters); + } + + private static String getFileContent( + String pluralSeparator, List textUnitDTOS, boolean useSource) { + + AndroidStringDocumentMapper androidStringDocumentMapper = + new AndroidStringDocumentMapper(pluralSeparator, null, null, null, true); + + AndroidStringDocument androidStringDocument = + androidStringDocumentMapper.readFromTextUnits(textUnitDTOS, useSource); + + return new AndroidStringDocumentWriter(androidStringDocument).toText(); + } + + @Override + public void pullSource( + Repository repository, + String projectId, + List optionList, + Map localeMapping) { + throw new UnsupportedOperationException("Pull source is not supported"); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java new file mode 100644 index 0000000000..835c16c069 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.service.thirdparty; + +public class ThridPartyTMSPhraseException extends RuntimeException { + public ThridPartyTMSPhraseException(String msg, Throwable e) { + super(msg, e); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java new file mode 100644 index 0000000000..7557ca3a99 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -0,0 +1,390 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import static com.box.l10n.mojito.io.Files.createDirectories; +import static com.box.l10n.mojito.io.Files.createTempDirectory; +import static com.box.l10n.mojito.io.Files.write; + +import com.box.l10n.mojito.json.ObjectMapper; +import com.google.common.collect.ImmutableSet; +import com.phrase.client.ApiClient; +import com.phrase.client.ApiException; +import com.phrase.client.api.KeysApi; +import com.phrase.client.api.LocalesApi; +import com.phrase.client.api.TagsApi; +import com.phrase.client.api.UploadsApi; +import com.phrase.client.model.Tag; +import com.phrase.client.model.TranslationKey; +import com.phrase.client.model.Upload; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +public class PhraseClient { + + static Logger logger = LoggerFactory.getLogger(PhraseClient.class); + + static final int BATCH_SIZE = 100; + + final ApiClient apiClient; + + final RetryBackoffSpec retryBackoffSpec; + + public PhraseClient(ApiClient apiClient) { + this.apiClient = apiClient; + this.retryBackoffSpec = + Retry.backoff(5, Duration.ofMillis(500)).maxBackoff(Duration.ofSeconds(5)); + } + + public Upload uploadAndWait( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags) { + + String uploadId = + uploadCreateFile(projectId, localeId, fileFormat, fileName, fileContent, tags); + return waitForUploadToFinish(projectId, uploadId); + } + + Upload waitForUploadToFinish(String projectId, String uploadId) { + UploadsApi uploadsApi = new UploadsApi(apiClient); + + try { + logger.debug("Waiting for upload to finish: {}", uploadId); + + Upload upload = uploadsApi.uploadShow(projectId, uploadId, null, null); + logger.debug( + "Upload info, first fetch: {}", new ObjectMapper().writeValueAsStringUnchecked(upload)); + + while (!ImmutableSet.of("success", "error").contains(upload.getState())) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + upload = uploadsApi.uploadShow(projectId, uploadId, null, null); + logger.debug( + "upload info after polling for success or error: {}", + new ObjectMapper().writeValueAsStringUnchecked(upload)); + } + + if ("error".equals(upload.getState())) { + throw new PhraseClientException( + "Upload failed: %s".formatted(new ObjectMapper().writeValueAsStringUnchecked(upload))); + } + + return upload; + } catch (ApiException e) { + logger.error("Error calling Phrase for waitForUploadToFinish: {}", e.getResponseBody()); + throw new PhraseClientException(e); + } + } + + public String uploadCreateFile( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags) { + + Path tmpWorkingDirectory = null; + + try { + tmpWorkingDirectory = createTempDirectory("phrase-integration"); + + if (tmpWorkingDirectory.toFile().exists()) { + logger.info("Created temporary working directory: {}", tmpWorkingDirectory); + } + + Path fileToUpload = tmpWorkingDirectory.resolve(fileName); + + logger.info("Create file: {}", fileToUpload); + createDirectories(fileToUpload.getParent()); + write(fileToUpload, fileContent); + + Upload upload = + uploadsApiUploadCreateWithRetry(projectId, localeId, fileFormat, tags, fileToUpload); + + return upload.getId(); + } finally { + if (tmpWorkingDirectory != null) { + com.box.l10n.mojito.io.Files.deleteRecursivelyIfExists(tmpWorkingDirectory); + } + } + } + + Upload uploadsApiUploadCreateWithRetry( + String projectId, String localeId, String fileFormat, List tags, Path fileToUpload) { + + return Mono.fromCallable( + () -> + new UploadsApi(apiClient) + .uploadCreate( + projectId, + fileToUpload.toFile(), + fileFormat, + localeId, + null, + null, + tags == null ? null : String.join(",", tags), + true, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null)) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to uploadCreate to Phrase, file: %s, project id: %s" + .formatted(fileToUpload.toAbsolutePath(), projectId)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error in UploadCreate from Phrase, file: %s, project id: %s" + .formatted(fileToUpload.toAbsolutePath(), projectId))) + .block(); + } + + public void removeKeysNotTaggedWith(String projectId, String tag) { + logger.info("Removing keys not tagged with: {}", tag); + + Mono.fromCallable( + () -> { + KeysApi keysApi = new KeysApi(apiClient); + keysApi.keysDeleteCollection(projectId, null, null, "-tags:%s".formatted(tag), null); + return null; + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to removeKeysNotTaggedWith from Phrase, project id: %s" + .formatted(projectId)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to removeKeysNotTaggedWith from Phrase, project id: %s" + .formatted(projectId))) + .block(); + } + + public String localeDownload(String projectId, String locale, String fileFormat) { + return Mono.fromCallable( + () -> { + LocalesApi localesApi = new LocalesApi(apiClient); + logger.info( + "Downloading locale: {} from project id: {} in file format: {}", + locale, + projectId, + fileFormat); + File file = + localesApi.localeDownload( + projectId, + locale, + null, + null, + null, + null, + fileFormat, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null); + + String localeDownloadContent = Files.readString(file.toPath()); + logger.debug("File: {}, Content: {}", file.toPath(), localeDownloadContent); + + return localeDownloadContent; + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale))) + .block(); + } + + public List getKeys(String projectId) { + KeysApi keysApi = new KeysApi(apiClient); + AtomicInteger page = new AtomicInteger(0); + int batchSize = BATCH_SIZE; + List translationKeys = new ArrayList<>(); + while (true) { + List translationKeysInPage = + Mono.fromCallable( + () -> { + logger.info("Fetching keys for project: {}, page: {}", projectId, page); + return keysApi.keysList( + projectId, null, page.get(), batchSize, null, null, null, null, null); + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to fetch keys for project: %s, page: %s" + .formatted(projectId, page.get())))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to fetch keys for project: %s, page: %s" + .formatted(projectId, page))) + .block(); + + translationKeys.addAll(translationKeysInPage); + + if (translationKeysInPage.size() < batchSize) { + break; + } else { + page.incrementAndGet(); + } + } + return translationKeys; + } + + public List listTags(String projectId) { + TagsApi tagsApi = new TagsApi(apiClient); + final AtomicInteger page = new AtomicInteger(0); + List tags = new ArrayList<>(); + while (true) { + List tagsInPage = + Mono.fromCallable( + () -> { + logger.info("Fetching tags for project: {}", projectId); + return tagsApi.tagsList(projectId, null, page.get(), BATCH_SIZE, null); + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to fetch tags for project: %s, page: %d" + .formatted(projectId, page.get())))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to fetch tags for project: %s, page: %s" + .formatted(projectId, page))) + .block(); + + tags.addAll(tagsInPage); + + if (tagsInPage.size() < BATCH_SIZE) { + break; + } else { + page.incrementAndGet(); + } + } + + return tags; + } + + public void deleteTags(String projectId, List tags) { + TagsApi tagsApi = new TagsApi(apiClient); + Map exceptions = new LinkedHashMap<>(); + for (Tag tag : tags) { + Mono.fromCallable( + () -> { + logger.debug( + "Deleting tag: %s in project id: %s".formatted(tag.getName(), projectId)); + tagsApi.tagDelete(projectId, tag.getName(), null, null); + return null; + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> { + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to delete tag: %s in project id: %s" + .formatted(tag.getName(), projectId)); + })) + .doOnError( + throwable -> { + exceptions.put(tag.getName(), throwable); + rethrowExceptionWithLog( + throwable, + "Final error to delete tag: %s in project id: %s" + .formatted(tag.getName(), projectId)); + }) + .block(); + } + + if (!exceptions.isEmpty()) { + List tagsWithErrors = exceptions.keySet().stream().limit(10).toList(); + String andMore = (tagsWithErrors.size() < exceptions.size()) ? " and more." : ""; + throw new PhraseClientException( + String.format("Can't delete tags: %s%s", tagsWithErrors, andMore)); + } + } + + private void logAttempt(Throwable throwable, String message) { + String errorMessage = getErrorMessageFromOptionalApiException(throwable); + logger.info("%s, error: %s".formatted(message, errorMessage), throwable); + } + + private void rethrowExceptionWithLog(Throwable throwable, String message) { + String errorMessage = getErrorMessageFromOptionalApiException(throwable); + logger.error("%s, error: %s".formatted(message, errorMessage)); + if (throwable.getCause() instanceof ApiException) { + throw new PhraseClientException(errorMessage, (ApiException) throwable.getCause()); + } else { + throw new RuntimeException(errorMessage, throwable); + } + } + + private String getErrorMessageFromOptionalApiException(Throwable t) { + String errorMessage; + if (t instanceof ApiException) { + errorMessage = ((ApiException) t).getResponseBody(); + } else { + errorMessage = t.getMessage(); + } + return errorMessage; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java new file mode 100644 index 0000000000..65e69823ab --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java @@ -0,0 +1,23 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import com.phrase.client.ApiClient; +import com.phrase.client.auth.ApiKeyAuth; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class PhraseClientConfig { + + @ConditionalOnProperty("l10n.phrase.client.token") + @Bean + public PhraseClient getPhraseClient(PhraseClientPropertiesConfig phraseClientPropertiesConfig) { + + ApiClient apiClient = new ApiClient(); + ApiKeyAuth authentication = (ApiKeyAuth) apiClient.getAuthentication("Token"); + authentication.setApiKey(phraseClientPropertiesConfig.getToken()); + authentication.setApiKeyPrefix("token"); + + return new PhraseClient(apiClient); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java new file mode 100644 index 0000000000..65f0ffc6c4 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java @@ -0,0 +1,22 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import com.phrase.client.ApiException; + +public class PhraseClientException extends RuntimeException { + + ApiException apiException; + + public PhraseClientException(String message) { + super(message); + } + + public PhraseClientException(ApiException e) { + super(e); + apiException = e; + } + + public PhraseClientException(String message, ApiException apiException) { + super(message); + this.apiException = apiException; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java new file mode 100644 index 0000000000..9baf70b304 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java @@ -0,0 +1,19 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties("l10n.phrase.client") +public class PhraseClientPropertiesConfig { + + String token; + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java index 1b081589f3..642bf18ca9 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java @@ -177,4 +177,8 @@ public ThirdPartyServiceTestData init() throws Exception { logger.debug("Finished init of ThirdPartyServiceTestData"); return this; } + + public String getPluralSeparator() { + return "_"; + } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java new file mode 100644 index 0000000000..f7ddc2ba61 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java @@ -0,0 +1,91 @@ +package com.box.l10n.mojito.service.thirdparty; + +import com.box.l10n.mojito.entity.Repository; +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.service.assetExtraction.ServiceTestBase; +import com.box.l10n.mojito.service.repository.RepositoryLocaleCreationException; +import com.box.l10n.mojito.service.repository.RepositoryService; +import com.box.l10n.mojito.service.tm.search.StatusFilter; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.box.l10n.mojito.test.TestIdWatcher; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.Assume; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +public class ThirdPartyTMSPhraseTest extends ServiceTestBase { + + static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhraseTest.class); + + @Autowired(required = false) + ThirdPartyTMSPhrase thirdPartyTMSPhrase; + + @Autowired RepositoryService repositoryService; + + @Autowired TextUnitSearcher textUnitSearcher; + + @Value("${test.phrase-client.projectId}") + String testProjectId; + + @Rule public TestIdWatcher testIdWatcher = new TestIdWatcher(); + + @Test + public void testBasics() throws RepositoryLocaleCreationException { + Assume.assumeNotNull(thirdPartyTMSPhrase); + Assume.assumeNotNull(testProjectId); + + ThirdPartyServiceTestData thirdPartyServiceTestData = + new ThirdPartyServiceTestData(testIdWatcher); + + Repository repository = thirdPartyServiceTestData.repository; + repositoryService.addRepositoryLocale(repository, "fr"); + + repository + .getRepositoryLocales() + .forEach(rl -> logger.info("repository locale: {}", rl.getLocale().getBcp47Tag())); + + thirdPartyTMSPhrase.push( + repository, + testProjectId, + thirdPartyServiceTestData.getPluralSeparator(), + null, + null, + null); + + thirdPartyTMSPhrase.pull( + repository, + testProjectId, + thirdPartyServiceTestData.getPluralSeparator(), + ImmutableMap.of("fr-FR", "fr"), + null, + null, + null, + null, + null); + + TextUnitSearcherParameters params = new TextUnitSearcherParameters(); + params.setRepositoryIds(repository.getId()); + params.setRootLocaleExcluded(false); + params.setStatusFilter(StatusFilter.TRANSLATED); + List search = textUnitSearcher.search(params); + + if (logger.isInfoEnabled()) { + ObjectMapper objectMapper = new ObjectMapper(); + search.stream().forEach(t -> logger.info(objectMapper.writeValueAsStringUnchecked(t))); + } + + // List thirdPartyTextUnits = + // thirdPartyTMSPhrase.getThirdPartyTextUnits(repository, testProjectId, null); + // logger.info("Get") + // thirdPartyTextUnits.stream().forEach(t -> logger.info("third party text unit: {}", + // t)); + + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java new file mode 100644 index 0000000000..c822d376de --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -0,0 +1,113 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSPhrase; +import com.google.common.collect.ImmutableList; +import com.phrase.client.model.Tag; +import com.phrase.client.model.TranslationKey; +import java.util.List; +import org.junit.Assume; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest( + classes = { + PhraseClientTest.class, + PhraseClientConfig.class, + PhraseClientPropertiesConfig.class + }) +@EnableConfigurationProperties +public class PhraseClientTest { + + static Logger logger = LoggerFactory.getLogger(PhraseClientTest.class); + + @Autowired(required = false) + PhraseClient phraseClient; + + @Value("${test.phrase-client.projectId}") + String testProjectId; + + @Test + public void testRemoveTag() { + String tagForUpload = "push_2024_06_10_07_17_00_089_122"; + List tagsToDelete = + phraseClient.listTags(testProjectId).stream() + .peek(tag -> logger.info("tag: {}", tag)) + .filter(tag -> tag.getName() != null && !tag.getName().equals(tagForUpload)) + .toList(); + + phraseClient.deleteTags(testProjectId, tagsToDelete); + } + + @Test + public void test() { + Assume.assumeNotNull(testProjectId); + + String tagForUpload = ThirdPartyTMSPhrase.getTagForUpload(); + + logger.info("tagForUpload: {}", tagForUpload); + + StringBuilder fileContentAndroidBuilder = generateFileContent(); + + String fileContentAndroid = fileContentAndroidBuilder.toString(); + phraseClient.uploadAndWait( + testProjectId, + "en", + "xml", + "strings.xml", + fileContentAndroid, + ImmutableList.of(tagForUpload)); + + phraseClient.removeKeysNotTaggedWith(testProjectId, tagForUpload); + + List translationKeys = phraseClient.getKeys(testProjectId); + for (TranslationKey translationKey : translationKeys) { + logger.info("{}", translationKey); + } + + // + // String fileContentAndroid2 = + // """ + // + // + // Locale Tester - fr + // Settings - fr + // Hello + // + // One thing - fr + // Multiple things - fr + // + // + // """; + // phraseClient.uploadCreateFile( + // testProjectId, "fr", "xml", "strings.xml", fileContentAndroid2, null); + // String s2 = phraseClient.localeDownload(testProjectId, "fr", "xml"); + // logger.info(s2); + } + + static StringBuilder generateFileContent() { + StringBuilder fileContentAndroidBuilder = new StringBuilder(); + + fileContentAndroidBuilder.append( + """ + + + Locale Tester + """); + + for (int i = 0; i < 2000; i++) { + fileContentAndroidBuilder.append( + String.format("Settings\n", i)); + } + + fileContentAndroidBuilder.append(""); + return fileContentAndroidBuilder; + } +} From 8310366e83696d4c70466f65877d3c41fa84f25f Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 12 Jun 2024 13:59:59 -0700 Subject: [PATCH 031/105] Refactoring the pullcommand locale mapping - not backward compatible. Backward compatibility could be achieved by setting LocaleMappingType.MAP_ONLY, but the new default is preferred. WITH_REPOSITORY generates a basic mapping using the repository's locales and supplements it with the provided mapping, potentially overriding existing entries. In practice, this means you can provide only a few locales to re-map. If you need to generate only a few locales (a rare use case, typically for testing), you can simply pass the MAP_ONLY option. --- .../l10n/mojito/cli/command/PullCommand.java | 103 ++++++++++------- .../cli/command/PullCommandParallel.java | 104 ++++++------------ .../mojito/cli/command/PullCommandTest.java | 78 ++++++++++--- .../box/l10n/mojito/LocaleMappingHelper.java | 7 +- 4 files changed, 166 insertions(+), 126 deletions(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommand.java index 3cc366311c..42de156dc2 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommand.java @@ -18,6 +18,7 @@ import com.box.l10n.mojito.rest.entity.RepositoryLocaleStatistic; import com.box.l10n.mojito.rest.entity.RepositoryStatistic; import java.nio.file.Path; +import java.util.AbstractMap.SimpleEntry; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -73,6 +74,22 @@ public class PullCommand extends Command { description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION) String localeMappingParam; + @Parameter( + names = {"-lmt", "--locale-mapping-type"}, + arity = 1, + required = false, + description = + "Specifies how to handle locale mappings when used with --locale-mapping. " + + "MAP_ONLY processes only the locales explicitly specified in the mapping. " + + "WITH_REPOSITORY generates a basic mapping from the repository's locales and " + + "supplements it with the provided mapping, potentially overriding existing entries.") + LocaleMappingType localeMappingTypeParam = LocaleMappingType.WITH_REPOSITORY; + + enum LocaleMappingType { + MAP_ONLY, + WITH_REPOSITORY + } + @Parameter( names = {Param.FILE_TYPE_LONG, Param.FILE_TYPE_SHORT}, variableArity = true, @@ -184,7 +201,10 @@ public class PullCommand extends Command { CommandDirectories commandDirectories; - /** Contains a map of locale for generating localized file a locales defined in the repository. */ + /** + * Contains a map of locale for generating localized file. Key: file output tag, value: the tag in + * the repository + */ Map localeMappings; /** Map of locale and the boolean value for fully translated status of the locale */ @@ -233,7 +253,7 @@ public void execute() throws CommandException { commandHelper.getFilterOptionsOrDefaults( sourceFileMatch.getFileType(), filterOptionsParam); - generateLocalizedFiles(sourceFileMatch, filterOptions); + generateLocalizedFiles(repository, sourceFileMatch, filterOptions); } } @@ -242,14 +262,6 @@ public void execute() throws CommandException { consoleWriter.fg(Color.GREEN).newLine().a("Finished").println(2); } - private void generateLocalizedFiles(FileMatch sourceFileMatch, List filterOptions) { - if (localeMappingParam != null) { - generateLocalizedFilesWithLocaleMaping(repository, sourceFileMatch, filterOptions); - } else { - generateLocalizedFilesWithoutLocaleMapping(repository, sourceFileMatch, filterOptions); - } - } - void initPullRunName() { if (recordPullRun) { pullRunName = UUID.randomUUID().toString(); @@ -270,51 +282,66 @@ void writePullRunFileIfNeeded() { } /** - * Default generation, uses the locales defined in the repository to generate the localized files. + * Default generation, uses the locales defined in the repository to generate the localized files + * and eventually use the locale mapping to override the output tags. * * @param repository * @param sourceFileMatch * @param filterOptions * @throws CommandException */ - void generateLocalizedFilesWithoutLocaleMapping( + void generateLocalizedFiles( Repository repository, FileMatch sourceFileMatch, List filterOptions) throws CommandException { - logger.debug("Generate localized files (without locale mapping)"); + logger.debug("Generate localized files"); + Map outputToRepo = getMapOutputTagToRepositoryLocale(); - for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale.values()) { - generateLocalizedFile(repository, sourceFileMatch, filterOptions, null, repositoryLocale); + for (Map.Entry e : outputToRepo.entrySet()) { + generateLocalizedFile(repository, sourceFileMatch, filterOptions, e.getKey(), e.getValue()); } } - /** - * Generation with locale mapping. The localized files are generated using specific output tags - * while still using the repository locale to fetch the proper translations. - * - * @param repository - * @param sourceFileMatch - * @param filterOptions - * @throws CommandException - */ - void generateLocalizedFilesWithLocaleMaping( - Repository repository, FileMatch sourceFileMatch, List filterOptions) - throws CommandException { + Map getMapOutputTagToRepositoryLocale() { + Map outputToRepo; + + if (LocaleMappingType.MAP_ONLY.equals(localeMappingTypeParam)) { + + if (localeMappings == null) { + throw new RuntimeException("MAP_ONLY must be used with an locale mapping"); + } + outputToRepo = getLocaleMapFromLocaleMappings(); + + } else { + if (localeMappings == null) { + outputToRepo = repositoryLocalesWithoutRootLocale; + } else { + outputToRepo = + repositoryLocalesWithoutRootLocale.entrySet().stream() + .filter(e -> !localeMappings.containsValue(e.getValue().getLocale().getBcp47Tag())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + outputToRepo.putAll(getLocaleMapFromLocaleMappings()); + } + } + return outputToRepo; + } - logger.debug("Generate localized files with locale mapping"); + private Map getLocaleMapFromLocaleMappings() { + return localeMappings.entrySet().stream() + .map(e -> new SimpleEntry<>(e.getKey(), getRepositoryLocaleByTag(e.getValue()))) + .collect(Collectors.toMap(SimpleEntry::getKey, SimpleEntry::getValue)); + } - List repositoryLocales = - localeMappings.entrySet().stream() - .map(entry -> getRepositoryLocaleForOutputBcp47Tag(entry.getKey())) - .distinct() - .collect(Collectors.toList()); + private RepositoryLocale getRepositoryLocaleByTag(String tag) { + RepositoryLocale repositoryLocale; - for (Map.Entry localeMapping : localeMappings.entrySet()) { - String outputBcp47tag = localeMapping.getKey(); - RepositoryLocale repositoryLocale = getRepositoryLocaleForOutputBcp47Tag(outputBcp47tag); - generateLocalizedFile( - repository, sourceFileMatch, filterOptions, outputBcp47tag, repositoryLocale); + if (rootRepositoryLocale.getLocale().getBcp47Tag().equals(tag)) { + repositoryLocale = rootRepositoryLocale; + } else { + repositoryLocale = repositoryLocalesWithoutRootLocale.get(tag); } + + return repositoryLocale; } void generateLocalizedFile( diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommandParallel.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommandParallel.java index a8452ce6e0..1221a5b75a 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommandParallel.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommandParallel.java @@ -8,7 +8,6 @@ import com.box.l10n.mojito.rest.entity.PollableTask; import com.box.l10n.mojito.rest.entity.Repository; import com.box.l10n.mojito.rest.entity.RepositoryLocale; -import com.google.common.collect.Lists; import java.nio.file.Path; import java.util.HashMap; import java.util.List; @@ -42,6 +41,7 @@ public PullCommandParallel(PullCommand pullCommand) { this.sourceLocale = pullCommand.sourceLocale; this.fileTypes = pullCommand.fileTypes; this.localeMappingParam = pullCommand.localeMappingParam; + this.localeMappingTypeParam = pullCommand.localeMappingTypeParam; this.repositoryParam = pullCommand.repositoryParam; this.sourceDirectoryParam = pullCommand.sourceDirectoryParam; this.targetDirectoryParam = pullCommand.targetDirectoryParam; @@ -82,19 +82,10 @@ public void pull() throws CommandException { private void sendContentForLocalizedGeneration( FileMatch sourceFileMatch, List filterOptions) { - if (localeMappingParam != null) { - pollableTaskIdToFileMatchMap.put( - generateLocalizedFilesWithLocaleMappingParallel( - repository, sourceFileMatch, filterOptions) - .getId(), - sourceFileMatch); - } else { - pollableTaskIdToFileMatchMap.put( - generateLocalizedFilesWithoutLocaleMappingParallel( - repository, sourceFileMatch, filterOptions) - .getId(), - sourceFileMatch); - } + + pollableTaskIdToFileMatchMap.put( + generateLocalizedFilesParallel(repository, sourceFileMatch, filterOptions).getId(), + sourceFileMatch); } private void pollForLocalizedFiles() { @@ -118,31 +109,6 @@ private void pollForLocalizedFiles() { }); } - private PollableTask generateLocalizedFilesWithLocaleMappingParallel( - Repository repository, FileMatch sourceFileMatch, List filterOptions) - throws CommandException { - - List repositoryLocales = - localeMappings.entrySet().stream() - .map(entry -> getRepositoryLocaleForOutputBcp47Tag(entry.getKey())) - .distinct() - .collect(Collectors.toList()); - return generateLocalizedFiles(repository, sourceFileMatch, filterOptions, repositoryLocales); - } - - private PollableTask generateLocalizedFilesWithoutLocaleMappingParallel( - Repository repository, FileMatch sourceFileMatch, List filterOptions) - throws CommandException { - - logger.debug("Generate localized files (without locale mapping)"); - - return generateLocalizedFiles( - repository, - sourceFileMatch, - filterOptions, - Lists.newArrayList(repositoryLocalesWithoutRootLocale.values())); - } - void writeLocalizedAssetToTargetDirectory( LocalizedAssetBody localizedAsset, FileMatch sourceFileMatch) throws CommandException { @@ -173,24 +139,8 @@ private synchronized void printFileGeneratedToConsole( .println(); } - void generateLocalizedFilesWithoutLocaleMapping( - Repository repository, FileMatch sourceFileMatch, List filterOptions) - throws CommandException { - - logger.debug("Generate localized files (without locale mapping)"); - - generateLocalizedFiles( - repository, - sourceFileMatch, - filterOptions, - Lists.newArrayList(repositoryLocalesWithoutRootLocale.values())); - } - - private PollableTask generateLocalizedFiles( - Repository repository, - FileMatch sourceFileMatch, - List filterOptions, - List repositoryLocales) { + private PollableTask generateLocalizedFilesParallel( + Repository repository, FileMatch sourceFileMatch, List filterOptions) { Asset assetByPathAndRepositoryId; String sourcePath = @@ -207,12 +157,16 @@ private PollableTask generateLocalizedFiles( e); } - return getLocalizedAssetBodyParallel( - sourceFileMatch, - repositoryLocales.stream() + List repositoryLocales = + getMapOutputTagToRepositoryLocale().values().stream() + .distinct() .filter( repoLocale -> shouldGenerateLocalizedFileWithCliOutput(sourceFileMatch, repoLocale)) - .collect(Collectors.toList()), + .toList(); + + return getLocalizedAssetBodyParallel( + sourceFileMatch, + repositoryLocales, getRepoLocaleToOutputTagsMap(), filterOptions, assetByPathAndRepositoryId, @@ -220,18 +174,26 @@ private PollableTask generateLocalizedFiles( } private Map> getRepoLocaleToOutputTagsMap() { - Map> localeIdToOutputTagsMap = new HashMap<>(); + Map> localeIdToOutputTagsMap; if (localeMappings != null) { - for (Map.Entry mapping : localeMappings.entrySet()) { - String outputBcp47tag = mapping.getKey(); - RepositoryLocale locale = getRepositoryLocaleForOutputBcp47Tag(outputBcp47tag); - if (localeIdToOutputTagsMap.containsKey(locale)) { - localeIdToOutputTagsMap.get(locale).add(outputBcp47tag); - } else { - localeIdToOutputTagsMap.put(locale, Lists.newArrayList(outputBcp47tag)); - } - } + + Map mapOutputTagToRepositoryLocale = + getMapOutputTagToRepositoryLocale(); + + localeIdToOutputTagsMap = + mapOutputTagToRepositoryLocale.entrySet().stream() + .collect( + Collectors.groupingBy( + Map.Entry::getValue, + Collectors.mapping(Map.Entry::getKey, Collectors.toList()))); + + } else { + // Adapted for backward compatibility - a deeper refactor could improve clarity. + // Note: If there is no mapping, it returns an empty map. The impact on the backend is + // unclear, + // so this behavior is being preserved. + localeIdToOutputTagsMap = new HashMap<>(); } return localeIdToOutputTagsMap; diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java index c6ca8da260..a9a5de0d09 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java @@ -516,7 +516,9 @@ public void localeMapping() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-FR:fr-FR,ja:ja-JP"); + "fr:fr-FR,fr-FR:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -550,6 +552,8 @@ public void localeMappingParallel() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,fr-FR:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--parallel"); checkExpectedGeneratedResources(); @@ -773,7 +777,9 @@ public void pullResw() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP"); + "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -785,7 +791,9 @@ public void pullResw() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP"); + "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -1004,6 +1012,8 @@ public void pullXcodeXliff() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-ft", "XCODE_XLIFF"); @@ -1018,6 +1028,8 @@ public void pullXcodeXliff() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-ft", "XCODE_XLIFF"); @@ -1052,7 +1064,9 @@ public void pullPo() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP"); + "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -1064,7 +1078,9 @@ public void pullPo() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP"); + "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -1101,6 +1117,8 @@ public void pullHtml() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-ft", "HTML_ALPHA", "-fo", @@ -1117,6 +1135,8 @@ public void pullHtml() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-ft", "HTML_ALPHA", "-fo", @@ -1154,6 +1174,8 @@ public void removeUntranslated() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--inheritance-mode", "REMOVE_UNTRANSLATED"); @@ -1168,6 +1190,8 @@ public void removeUntranslated() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--inheritance-mode", "REMOVE_UNTRANSLATED"); @@ -1205,6 +1229,8 @@ public void onlyApproved() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--inheritance-mode", "REMOVE_UNTRANSLATED", "--status", @@ -1224,6 +1250,8 @@ public void onlyApproved() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--inheritance-mode", "REMOVE_UNTRANSLATED", "--status", @@ -1397,7 +1425,9 @@ public void pullJS() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -1409,7 +1439,9 @@ public void pullJS() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -1441,7 +1473,9 @@ public void pullTS() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -1453,7 +1487,9 @@ public void pullTS() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -1963,7 +1999,9 @@ public void recordPullPoPlural() throws Exception { getTargetTestDir("target_baseline").getAbsolutePath(), "-lm", "ru-RU:ru-RU", - "--record-pull-run"); + "--record-pull-run", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -1975,7 +2013,9 @@ public void recordPullPoPlural() throws Exception { "-t", getInputResourcesTestDir("translations").getAbsolutePath(), "-lm", - "ru-RU:ru-RU"); + "ru-RU:ru-RU", + "-lmt", + "MAP_ONLY"); logger.debug("Record a second pull run after translation import"); getL10nJCommander() @@ -1989,7 +2029,9 @@ public void recordPullPoPlural() throws Exception { getTargetTestDir("target_translated").getAbsolutePath(), "-lm", "ru-RU:ru-RU", - "--record-pull-run"); + "--record-pull-run", + "-lmt", + "MAP_ONLY"); logger.debug("Simulate commit and linked to pull-run"); String pullRunHash1 = "ddaa11"; @@ -2461,7 +2503,9 @@ public void pullYaml() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -2473,7 +2517,9 @@ public void pullYaml() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -2509,6 +2555,8 @@ public void pullYamlWithFilterOptions() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-fo", "extractAllPairs=false", "exceptions=1_day_duration|1_year_duration"); @@ -2524,6 +2572,8 @@ public void pullYamlWithFilterOptions() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-fo", "extractAllPairs=false", "exceptions=1_day_duration|1_year_duration"); diff --git a/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java b/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java index 8601ea642c..3b78325577 100644 --- a/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java +++ b/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java @@ -9,10 +9,10 @@ public class LocaleMappingHelper { /** - * Gets the locale mapping given the locale mapping param + * Gets the locale mapping given the locale mapping param. * * @param localeMapppingParam locale mapping param coming from the CLI - * @return A map containing the locale mapping + * @return A map containing the locale mapping (key: output tag, value: the tag in the repository) */ public Map getLocaleMapping(String localeMapppingParam) { @@ -28,7 +28,8 @@ public Map getLocaleMapping(String localeMapppingParam) { * Gets the inverse locale mapping given the locale mapping param * * @param localeMapppingParam locale mapping param coming from the CLI - * @return A map containing the inverse locale mapping + * @return A map containing the inverse locale mapping (key: the tag in the repository, value: + * file output tag) */ public Map getInverseLocaleMapping(String localeMapppingParam) { From 9d9160699d5caa076546761a603d175c1c849280 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 13 Jun 2024 23:33:34 -0700 Subject: [PATCH 032/105] Add support for Plural String import initial version of the connector could only push/pull (maybe buggy), this is an attempt at supporting push translations too --- .../ImportLocalizedAssetCommandTest.java | 154 +++++++++++++++++- .../after-sync/res/values-fr-rCA/strings.xml | 25 +++ .../after-sync/res/values-fr-rFR/strings.xml | 25 +++ .../after-sync/res/values-ja-rJP/strings.xml | 22 +++ .../after-sync/res/values-ru-rRU/strings.xml | 31 ++++ .../before-sync/res/values-fr-rCA/strings.xml | 25 +++ .../before-sync/res/values-fr-rFR/strings.xml | 25 +++ .../before-sync/res/values-ja-rJP/strings.xml | 22 +++ .../before-sync/res/values-ru-rRU/strings.xml | 31 ++++ .../input/source/res/values/strings.xml | 25 +++ .../res/values-fr-rCA/strings.xml | 4 + .../res/values-fr-rFR/strings.xml | 25 +++ .../res/values-ja-rJP/strings.xml | 22 +++ .../res/values-ru-rRU/strings.xml | 31 ++++ .../strings/AndroidStringDocumentMapper.java | 28 ++-- .../thirdparty/ThirdPartyTMSPhrase.java | 118 ++++++++++++-- .../thirdparty/phrase/PhraseClient.java | 2 +- .../AndroidStringDocumentMapperTest.java | 6 +- 18 files changed, 594 insertions(+), 27 deletions(-) create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rCA/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rFR/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ja-rJP/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ru-rRU/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rCA/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rFR/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ja-rJP/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ru-rRU/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source/res/values/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rCA/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rFR/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ja-rJP/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ru-rRU/strings.xml diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java index e706d7f130..e457f97035 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java @@ -4,10 +4,12 @@ import com.box.l10n.mojito.entity.Locale; import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.service.locale.LocaleService; +import org.junit.Assume; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; /** * @author jeanaurambault @@ -19,6 +21,10 @@ public class ImportLocalizedAssetCommandTest extends CLITestBase { @Autowired LocaleService localeService; + // TODO(ja) move in its own class + @Value("${test.phrase-client.projectId:}") + String testProjectId; + @Test public void importAndroidStrings() throws Exception { @@ -92,6 +98,150 @@ public void importAndroidStringsPlural() throws Exception { checkExpectedGeneratedResources(); } + @Test + public void importAndroidStringsPostProcessing() throws Exception { + + Repository repository = createTestRepoUsingRepoService(); + repositoryService.addRepositoryLocale(repository, "ru-RU"); + + getL10nJCommander() + .run( + "push", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath()); + + getL10nJCommander() + .run( + "import", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getInputResourcesTestDir("translations").getAbsolutePath()); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("removeDescription").getAbsolutePath(), + "-fo", + "removeDescription=true"); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("removeUntranslated").getAbsolutePath(), + "--inheritance-mode", + "REMOVE_UNTRANSLATED"); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("removeUntranslatedAndDescription").getAbsolutePath(), + "--inheritance-mode", + "REMOVE_UNTRANSLATED", + "-fo", + "removeDescription=true"); + + checkExpectedGeneratedResources(); + } + @Test + public void importAndroidStringsPluralWithThirdPartySync() throws Exception { + Assume.assumeNotNull(testProjectId); + + Repository repository = createTestRepoUsingRepoService(); + repositoryService.addRepositoryLocale(repository, "ru-RU"); + + getL10nJCommander() + .run( + "push", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath()); + + getL10nJCommander() + .run( + "push", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source2").getAbsolutePath(), + "-b", + "source2"); + + getL10nJCommander() + .run( + "import", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getInputResourcesTestDir("translations").getAbsolutePath()); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("before-sync").getAbsolutePath()); + + getL10nJCommander() + .run( + "thirdparty-sync", + "-r", + repository.getName(), + "-p", + testProjectId, + "-a", + "PUSH,MAP_TEXTUNIT,PUSH_TRANSLATION,PULL"); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("after-sync").getAbsolutePath()); + + getL10nJCommander() + .run( + "thirdparty-sync", + "-r", + repository.getName(), + "-p", + testProjectId, + "-a", + "MAP_TEXTUNIT", + "-o", + "deleteCurrentMapping=true"); + + checkExpectedGeneratedResources(); + } + @Test public void importMacStrings() throws Exception { @@ -1023,7 +1173,9 @@ public void importDifferentSourceLocale() throws Exception { "-t", getTargetTestDir("withMapping").getAbsolutePath(), "-lm", - "en-US:en-US,en:en,fr-FR:fr-FR"); + "en-US:en-US,en:en,fr-FR:fr-FR", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..bf62334b5b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rCA/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..bf62334b5b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..9091b03269 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ja-rJP/strings.xml @@ -0,0 +1,22 @@ + + + + + 所要時間：%1$,d 時間 + + + + ユーザー + + + ボード参加者：%1$,d 人 + + + + + %1$d 個の材料 + + + + 削除 + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..0df87a4dfc --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ru-rRU/strings.xml @@ -0,0 +1,31 @@ + + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + + + Люди + + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + + Удалить + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..bf62334b5b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rCA/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..bf62334b5b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..9091b03269 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ja-rJP/strings.xml @@ -0,0 +1,22 @@ + + + + + 所要時間：%1$,d 時間 + + + + ユーザー + + + ボード参加者：%1$,d 人 + + + + + %1$d 個の材料 + + + + 削除 + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..0df87a4dfc --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ru-rRU/strings.xml @@ -0,0 +1,31 @@ + + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + + + Люди + + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + + Удалить + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source/res/values/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source/res/values/strings.xml new file mode 100644 index 0000000000..df1703cc80 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source/res/values/strings.xml @@ -0,0 +1,25 @@ + + + + + %,d hr to cook + %,d hrs to cook + + + + People + + + %1$,d collaborator + %1$,d collaborators + + + + + %d ingredient + %d ingredients + + + + Delete + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..0f85d2dd1e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rCA/strings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..0623031bdd --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..22a1c39bf1 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ja-rJP/strings.xml @@ -0,0 +1,22 @@ + + + + + 所要時間：%1$,d 時間 + + + + ユーザー + + + ボード参加者：%1$,d 人 + + + + + %1$d 個の材料 + + + + 削除 + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..0df87a4dfc --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ru-rRU/strings.xml @@ -0,0 +1,31 @@ + + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + + + Люди + + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + + Удалить + \ No newline at end of file diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java index 7a1b1bd227..f3a5b22142 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java @@ -9,6 +9,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -33,13 +34,15 @@ public class AndroidStringDocumentMapper { private final String repositoryName; private final PluralNameParser pluralNameParser; private final boolean addTextUnitIdInName; + private final Map pluralFormToCommaId; public AndroidStringDocumentMapper( String pluralSeparator, String assetDelimiter, String locale, String repositoryName, - boolean addTextUnitIdInName) { + boolean addTextUnitIdInName, + Map pluralFormToCommaId) { this.pluralSeparator = pluralSeparator; this.assetDelimiter = Optional.ofNullable(Strings.emptyToNull(assetDelimiter)).orElse(DEFAULT_ASSET_DELIMITER); @@ -47,11 +50,12 @@ public AndroidStringDocumentMapper( this.repositoryName = repositoryName; this.pluralNameParser = new PluralNameParser(); this.addTextUnitIdInName = addTextUnitIdInName; + this.pluralFormToCommaId = pluralFormToCommaId; } public AndroidStringDocumentMapper( String pluralSeparator, String assetDelimiter, String locale, String repositoryName) { - this(pluralSeparator, assetDelimiter, locale, repositoryName, false); + this(pluralSeparator, assetDelimiter, locale, repositoryName, false, null); } public AndroidStringDocumentMapper(String pluralSeparator, String assetDelimiter) { @@ -102,14 +106,18 @@ public AndroidStringDocument readFromTextUnits(List textUnits, bool pluralByOther.forEach( (pluralFormOther, builder) -> { if (addTextUnitIdInName) { - String ids = - builder.getSortedItems().stream() - .map(AndroidPluralItem::getId) - .map(Objects::toString) - .collect(Collectors.joining(",")); + String ids; + if (pluralFormToCommaId != null) { + ids = this.pluralFormToCommaId.get(pluralFormOther); + } else { + ids = + builder.getSortedItems().stream() + .map(AndroidPluralItem::getId) + .map(Objects::toString) + .collect(Collectors.joining(",")); + } builder.setName(ids + "#@#" + builder.getName()); } - document.addPlural(builder.build()); }); @@ -159,7 +167,7 @@ TextUnitDTO addTextUnitDTOAttributes(TextUnitDTO textUnit) { } AndroidPluralQuantity androidPluralQuantity = - AndroidPluralQuantity.valueOf(textUnit.getPluralForm()); + AndroidPluralQuantity.valueOf(textUnit.getPluralForm().toUpperCase(Locale.ROOT)); textUnit.setTmTextUnitId(ids.get(androidPluralQuantity.ordinal())); } textUnit.setAssetPath(nameParts[1]); @@ -229,7 +237,7 @@ boolean isSingularTextUnit(TextUnitDTO textUnit) { return Strings.isNullOrEmpty(textUnit.getPluralForm()); } - String getKeyToGroupByPluralOtherAndComment(TextUnitDTO textUnit) { + public static String getKeyToGroupByPluralOtherAndComment(TextUnitDTO textUnit) { return textUnit.getAssetPath() + DEFAULT_ASSET_DELIMITER + textUnit.getPluralFormOther() diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 7d48480d52..152b4236b4 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -8,10 +8,12 @@ import com.box.l10n.mojito.entity.PollableTask; import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.entity.RepositoryLocale; +import com.box.l10n.mojito.json.ObjectMapper; import com.box.l10n.mojito.service.pollableTask.PollableFuture; import com.box.l10n.mojito.service.repository.RepositoryService; import com.box.l10n.mojito.service.thirdparty.phrase.PhraseClient; import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; +import com.box.l10n.mojito.service.tm.search.SearchType; import com.box.l10n.mojito.service.tm.search.StatusFilter; import com.box.l10n.mojito.service.tm.search.TextUnitDTO; import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; @@ -22,12 +24,16 @@ import com.phrase.client.model.TranslationKey; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -122,7 +128,7 @@ public void push( List search = getSourceTextUnitDTOs(repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); - String text = getFileContent(pluralSeparator, search, true); + String text = getFileContent(pluralSeparator, search, true, null); String tagForUpload = getTagForUpload(); phraseClient.uploadAndWait( @@ -162,6 +168,24 @@ private List getSourceTextUnitDTOs( return textUnitSearcher.search(parameters); } + private List getSourceTextUnitDTOsPluralOnly( + Repository repository, String skipTextUnitsWithPattern, String skipAssetsWithPathPattern) { + + TextUnitSearcherParameters parameters = new TextUnitSearcherParameters(); + + parameters.setRepositoryIds(repository.getId()); + parameters.setForRootLocale(true); + parameters.setDoNotTranslateFilter(false); + parameters.setUsedFilter(UsedFilter.USED); + parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); + parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + parameters.setSearchType(SearchType.ILIKE); + parameters.setPluralFormsFiltered(false); + parameters.setPluralFormOther("%"); + + return textUnitSearcher.search(parameters); + } + public static String getTagForUpload() { ZonedDateTime zonedDateTime = JSR310Migration.dateTimeNowInUTC(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy_MM_dd_HH_mm_ss_SSS"); @@ -192,9 +216,11 @@ public PollableFuture pull( logger.info("Downloading locale: {} from Phrase", localeTag); String fileContent = phraseClient.localeDownload(projectId, localeTag, "xml"); + logger.info("file content from pull: {}", fileContent); + AndroidStringDocumentMapper mapper = new AndroidStringDocumentMapper( - pluralSeparator, null, localeTag, repository.getName(), true); + pluralSeparator, null, localeTag, repository.getName(), true, null); List textUnitDTOS = mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); @@ -216,6 +242,36 @@ public void pushTranslations( String includeTextUnitsWithPattern, List optionList) { + List pluralTextUnitDTOs = + getSourceTextUnitDTOsPluralOnly( + repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); + + Map> pluralFormOtherToTextUnitDTO = + pluralTextUnitDTOs.stream() + .collect( + Collectors.groupingBy( + AndroidStringDocumentMapper::getKeyToGroupByPluralOtherAndComment)); + + pluralFormOtherToTextUnitDTO.forEach( + (key, value) -> { + if (value.size() != 6) { + throw new RuntimeException("there must be only 6 text units per PluralFormOther value"); + } + }); + + Map pluralFormToCommaId = + pluralFormOtherToTextUnitDTO.entrySet().stream() + .map( + e -> + new SimpleEntry<>( + e.getKey(), + e.getValue().stream() + .sorted(new ByPluralFormComparator()) + .map(TextUnitDTO::getTmTextUnitId) + .map(String::valueOf) + .collect(Collectors.joining(",")))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Set repositoryLocalesWithoutRootLocale = repositoryService.getRepositoryLocalesWithoutRootLocale(repository); @@ -228,15 +284,29 @@ public void pushTranslations( includeTextUnitsWithPattern, repositoryLocale); - String fileContent = getFileContent(pluralSeparator, textUnitDTOS, false); + if (textUnitDTOS.isEmpty()) { + logger.info("Not translation, skip upload"); + } else { - phraseClient.uploadCreateFile( - projectId, - repositoryLocale.getLocale().getBcp47Tag(), - "xml", - repository.getName() + "-strings.xml", - fileContent, - null); + logger.info("Print text unit for {}", repositoryLocale.getLocale().getBcp47Tag()); + textUnitDTOS.forEach( + textUnitDTO -> + logger.info( + "Textunit: {}", + ObjectMapper.withIndentedOutput().writeValueAsStringUnchecked(textUnitDTO))); + + String fileContent = + getFileContent(pluralSeparator, textUnitDTOS, false, pluralFormToCommaId); + logger.info("Push translation to phrase:\n{}", fileContent); + + phraseClient.uploadAndWait( + projectId, + repositoryLocale.getLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + fileContent, + null); + } } } @@ -260,10 +330,14 @@ private List getTextUnitDTOSForLocale( } private static String getFileContent( - String pluralSeparator, List textUnitDTOS, boolean useSource) { + String pluralSeparator, + List textUnitDTOS, + boolean useSource, + Map pluralFormToCommaId) { AndroidStringDocumentMapper androidStringDocumentMapper = - new AndroidStringDocumentMapper(pluralSeparator, null, null, null, true); + new AndroidStringDocumentMapper( + pluralSeparator, null, null, null, true, pluralFormToCommaId); AndroidStringDocument androidStringDocument = androidStringDocumentMapper.readFromTextUnits(textUnitDTOS, useSource); @@ -279,4 +353,24 @@ public void pullSource( Map localeMapping) { throw new UnsupportedOperationException("Pull source is not supported"); } + + static class ByPluralFormComparator implements Comparator { + + private final Map orderMap; + + public ByPluralFormComparator() { + this.orderMap = new HashMap<>(); + int i = 0; + for (String v : Arrays.asList("zero", "one", "two", "few", "many", "other")) { + this.orderMap.put(v, i++); + } + } + + @Override + public int compare(TextUnitDTO o1, TextUnitDTO o2) { + return Integer.compare( + orderMap.getOrDefault(o1.getPluralForm(), -1), + orderMap.getOrDefault(o2.getPluralForm(), -1)); + } + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java index 7557ca3a99..6acc3c824e 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -93,7 +93,7 @@ Upload waitForUploadToFinish(String projectId, String uploadId) { } } - public String uploadCreateFile( + String uploadCreateFile( String projectId, String localeId, String fileFormat, diff --git a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java index f7b342ed71..4a8b96ba4b 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java @@ -60,7 +60,7 @@ public void testReadFromSourceTextUnitsWithoutPluralForms() { @Test public void testReadFromSourceTextUnitsWithoutPluralFormsAndWithTmTextUnitIdInName() { - mapper = new AndroidStringDocumentMapper("_", assetDelimiter, null, null, true); + mapper = new AndroidStringDocumentMapper("_", assetDelimiter, null, null, true, null); textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); textUnits.add(sourceTextUnitDTO(124L, "name1", "content1", "comment1", "my/path1", null, null)); @@ -159,7 +159,7 @@ public void testReadFromSourceTextUnitsWithPlurals() { @Test public void testReadFromSourceTextUnitsWithPluralsAndWithTmTextUnitIdInName() { - mapper = new AndroidStringDocumentMapper(" _", assetDelimiter, null, null, true); + mapper = new AndroidStringDocumentMapper(" _", assetDelimiter, null, null, true, null); textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); textUnits.add( @@ -818,7 +818,7 @@ public void testAddTextUnitDTOAttributesAssetPathAndName() { @Test public void testAddTextUnitDTOAttributesTextUnitIdAndAssetPathAndName() { - mapper = new AndroidStringDocumentMapper("_", null, null, null, true); + mapper = new AndroidStringDocumentMapper("_", null, null, null, true, null); TextUnitDTO textUnitDTO = new TextUnitDTO(); textUnitDTO.setName("156151#@#asset_path#@#name_part1"); From 145c056d2190026f219c299d4e65a906efd690c3 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 19 Jun 2024 12:51:01 -0700 Subject: [PATCH 033/105] Phrase Connector support N Mojito repositories to 1 project mapping --- .../thirdparty/ThirdPartyTMSPhrase.java | 93 ++++++++++++++++--- .../thirdparty/phrase/PhraseClient.java | 86 +++++++++++++---- .../thirdparty/ThirdPartyTMSPhraseTest.java | 2 +- .../thirdparty/phrase/PhraseClientTest.java | 90 +++++++++++------- 4 files changed, 207 insertions(+), 64 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 152b4236b4..2b53a58a26 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -45,6 +46,7 @@ public class ThirdPartyTMSPhrase implements ThirdPartyTMS { static final String TAG_PREFIX = "push_"; + static final String TAG_PREFIX_WITH_REPOSITORY = "push_%s"; static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhrase.class); @@ -57,6 +59,12 @@ public class ThirdPartyTMSPhrase implements ThirdPartyTMS { @Autowired RepositoryService repositoryService; + public ThirdPartyTMSPhrase() {} + + public ThirdPartyTMSPhrase(PhraseClient phraseClient) { + this.phraseClient = phraseClient; + } + @Override public void removeImage(String projectId, String imageId) { throw new UnsupportedOperationException("Remove image is not supported"); @@ -130,7 +138,7 @@ public void push( String text = getFileContent(pluralSeparator, search, true, null); - String tagForUpload = getTagForUpload(); + String tagForUpload = getTagForUpload(repository.getName()); phraseClient.uploadAndWait( projectId, repository.getSourceLocale().getBcp47Tag(), @@ -139,17 +147,62 @@ public void push( text, ImmutableList.of(tagForUpload)); - phraseClient.removeKeysNotTaggedWith(projectId, tagForUpload); + removeUnusedKeysAndTags(projectId, repository.getName(), tagForUpload); + } - List tagsToDelete = + /** + * Remove unused keys and tags + * + *

    + *
  • Remove Unused Keys: + *
      + *
    • Unused keys in a Phrase project are defined as keys that are not tagged with the + * latest push tag from any Mojito repository. + *
    • To identify unused keys, fetch the current push tags of all repositories, excluding + * the push tag related to the Mojito repository being processed, and include the + * current upload tag. + *
    + *
  • Manage Tags: + *
      + *
    • Ensure that there is only one active "push" tag per repository. + *
    • Remove old tags that are prefixed with "push" but not active + *
    + *
+ * + *

Explanation: + * + *

The N:1 relationship between Mojito and Phrase is maintained through this logic, allowing + * for efficient management of keys and tags. + */ + public void removeUnusedKeysAndTags( + String projectId, String repositoryName, String tagForUpload) { + + List tagsForOtherRepositories = phraseClient.listTags(projectId).stream() + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> tagName.startsWith(TAG_PREFIX)) .filter( - tag -> - tag.getName() != null - && !tag.getName().equals(tagForUpload) - && tag.getName().startsWith(TAG_PREFIX)) + tagName -> + !tagName.startsWith(TAG_PREFIX_WITH_REPOSITORY.formatted(repositoryName))) + .toList(); + + List allActiveTags = new ArrayList<>(tagsForOtherRepositories); + allActiveTags.add(tagForUpload); + + logger.info("All active tags: {}", allActiveTags); + phraseClient.removeKeysNotTaggedWith(projectId, allActiveTags); + + List pushTagsToDelete = + phraseClient.listTags(projectId).stream() + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> tagName.startsWith(TAG_PREFIX)) + .filter(tagName -> !allActiveTags.contains(tagName)) .toList(); - phraseClient.deleteTags(projectId, tagsToDelete); + + logger.info("Tags to delete: {}", pushTagsToDelete); + phraseClient.deleteTags(projectId, pushTagsToDelete); } private List getSourceTextUnitDTOs( @@ -186,12 +239,13 @@ private List getSourceTextUnitDTOsPluralOnly( return textUnitSearcher.search(parameters); } - public static String getTagForUpload() { + public static String getTagForUpload(String repositoryName) { ZonedDateTime zonedDateTime = JSR310Migration.dateTimeNowInUTC(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy_MM_dd_HH_mm_ss_SSS"); - return ("%s%s_%s") + return ("%s%s_%s_%s") .formatted( TAG_PREFIX, + repositoryName, formatter.format(zonedDateTime), Math.abs(UUID.randomUUID().getLeastSignificantBits() % 1000)); } @@ -211,10 +265,19 @@ public PollableFuture pull( Set repositoryLocalesWithoutRootLocale = repositoryService.getRepositoryLocalesWithoutRootLocale(repository); + String currentTags = getCurrentTagsForRepository(repository, projectId); + for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale) { String localeTag = repositoryLocale.getLocale().getBcp47Tag(); logger.info("Downloading locale: {} from Phrase", localeTag); - String fileContent = phraseClient.localeDownload(projectId, localeTag, "xml"); + + String fileContent = + phraseClient.localeDownload( + projectId, + localeTag, + "xml", + currentTags, + () -> getCurrentTagsForRepository(repository, projectId)); logger.info("file content from pull: {}", fileContent); @@ -231,6 +294,14 @@ public PollableFuture pull( return null; } + private String getCurrentTagsForRepository(Repository repository, String projectId) { + return phraseClient.listTags(projectId).stream() + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> tagName.startsWith(repository.getName())) + .collect(Collectors.joining(",")); + } + @Override public void pushTranslations( Repository repository, diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java index 6acc3c824e..d5ffd75a27 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Mono; @@ -168,13 +170,35 @@ Upload uploadsApiUploadCreateWithRetry( .block(); } - public void removeKeysNotTaggedWith(String projectId, String tag) { - logger.info("Removing keys not tagged with: {}", tag); + /** + * Conducted tests on keysDeleteCollection using the -tags:tag1,tag2 option, and it + * operates as a filter for keys "not in any of the tags". + * + *

It removes keys that are not tagged with any of the provided tags. + * + *

For example: + * + *

    + *
  • If key ka has tag tag1, + *
  • If key kb has tag tag2, + *
  • If key kc has tag tag3, + *
+ * + *

Calling keysDeleteCollection with -tags:tag1,tag3 will remove + * kb. + */ + public void removeKeysNotTaggedWith(String projectId, List anyOfTheseTags) { + logger.info("Removing keys not tagged with any of the following tags: {}", anyOfTheseTags); Mono.fromCallable( () -> { KeysApi keysApi = new KeysApi(apiClient); - keysApi.keysDeleteCollection(projectId, null, null, "-tags:%s".formatted(tag), null); + keysApi.keysDeleteCollection( + projectId, + null, + null, + "-tags:%s".formatted(String.join(",", anyOfTheseTags)), + null); return null; }) .retryWhen( @@ -193,7 +217,18 @@ public void removeKeysNotTaggedWith(String projectId, String tag) { .block(); } - public String localeDownload(String projectId, String locale, String fileFormat) { + /** + * @param onTagErrorRefreshCallback with concurrent update, the tags could be updated during + * download. We don't want to retry the whole logic, so we provide a callback to refresh the + * tags + */ + public String localeDownload( + String projectId, + String locale, + String fileFormat, + String tags, + Supplier onTagErrorRefreshCallback) { + AtomicReference refTags = new AtomicReference<>(tags); return Mono.fromCallable( () -> { LocalesApi localesApi = new LocalesApi(apiClient); @@ -211,7 +246,7 @@ public String localeDownload(String projectId, String locale, String fileFormat) null, null, fileFormat, - null, + refTags.get(), null, null, null, @@ -234,11 +269,22 @@ public String localeDownload(String projectId, String locale, String fileFormat) }) .retryWhen( retryBackoffSpec.doBeforeRetry( - doBeforeRetry -> - logAttempt( - doBeforeRetry.failure(), - "Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s" - .formatted(projectId, locale)))) + doBeforeRetry -> { + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale)); + + if (getErrorMessageFromOptionalApiException(doBeforeRetry.failure()) + .contains("Invalid Download Options. Parameter tags ")) { + String newTags = onTagErrorRefreshCallback.get(); + logger.warn( + "Replacing old tags: {} with new tags: {} for download locale", + refTags.get(), + newTags); + refTags.set(newTags); + } + })) .doOnError( throwable -> rethrowExceptionWithLog( @@ -325,15 +371,17 @@ public List listTags(String projectId) { return tags; } - public void deleteTags(String projectId, List tags) { + public void deleteTags(String projectId, List tagNames) { + + logger.debug("Delete tags: {}", tagNames); + TagsApi tagsApi = new TagsApi(apiClient); Map exceptions = new LinkedHashMap<>(); - for (Tag tag : tags) { + for (String tagName : tagNames) { Mono.fromCallable( () -> { - logger.debug( - "Deleting tag: %s in project id: %s".formatted(tag.getName(), projectId)); - tagsApi.tagDelete(projectId, tag.getName(), null, null); + logger.debug("Deleting tag: %s in project id: %s".formatted(tagName, projectId)); + tagsApi.tagDelete(projectId, tagName, null, null); return null; }) .retryWhen( @@ -342,15 +390,15 @@ public void deleteTags(String projectId, List tags) { logAttempt( doBeforeRetry.failure(), "Retrying failed attempt to delete tag: %s in project id: %s" - .formatted(tag.getName(), projectId)); + .formatted(tagName, projectId)); })) .doOnError( throwable -> { - exceptions.put(tag.getName(), throwable); + exceptions.put(tagName, throwable); rethrowExceptionWithLog( throwable, "Final error to delete tag: %s in project id: %s" - .formatted(tag.getName(), projectId)); + .formatted(tagName, projectId)); }) .block(); } @@ -359,7 +407,7 @@ public void deleteTags(String projectId, List tags) { List tagsWithErrors = exceptions.keySet().stream().limit(10).toList(); String andMore = (tagsWithErrors.size() < exceptions.size()) ? " and more." : ""; throw new PhraseClientException( - String.format("Can't delete tags: %s%s", tagsWithErrors, andMore)); + String.format("Can't delete tagNames: %s%s", tagsWithErrors, andMore)); } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java index f7ddc2ba61..55f896d3b6 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java @@ -31,7 +31,7 @@ public class ThirdPartyTMSPhraseTest extends ServiceTestBase { @Autowired TextUnitSearcher textUnitSearcher; - @Value("${test.phrase-client.projectId}") + @Value("${test.phrase-client.projectId:}") String testProjectId; @Rule public TestIdWatcher testIdWatcher = new TestIdWatcher(); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java index c822d376de..b929a4903e 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -1,10 +1,11 @@ package com.box.l10n.mojito.service.thirdparty.phrase; -import com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSPhrase; -import com.google.common.collect.ImmutableList; +import com.box.l10n.mojito.JSR310Migration; import com.phrase.client.model.Tag; -import com.phrase.client.model.TranslationKey; +import java.time.ZonedDateTime; import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import org.junit.Assume; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,16 +32,18 @@ public class PhraseClientTest { @Autowired(required = false) PhraseClient phraseClient; - @Value("${test.phrase-client.projectId}") + @Value("${test.phrase-client.projectId:}") String testProjectId; @Test public void testRemoveTag() { String tagForUpload = "push_2024_06_10_07_17_00_089_122"; - List tagsToDelete = + List tagsToDelete = phraseClient.listTags(testProjectId).stream() .peek(tag -> logger.info("tag: {}", tag)) - .filter(tag -> tag.getName() != null && !tag.getName().equals(tagForUpload)) + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> !tagName.equals(tagForUpload)) .toList(); phraseClient.deleteTags(testProjectId, tagsToDelete); @@ -50,27 +53,29 @@ public void testRemoveTag() { public void test() { Assume.assumeNotNull(testProjectId); - String tagForUpload = ThirdPartyTMSPhrase.getTagForUpload(); - - logger.info("tagForUpload: {}", tagForUpload); - - StringBuilder fileContentAndroidBuilder = generateFileContent(); - - String fileContentAndroid = fileContentAndroidBuilder.toString(); - phraseClient.uploadAndWait( - testProjectId, - "en", - "xml", - "strings.xml", - fileContentAndroid, - ImmutableList.of(tagForUpload)); - - phraseClient.removeKeysNotTaggedWith(testProjectId, tagForUpload); - - List translationKeys = phraseClient.getKeys(testProjectId); - for (TranslationKey translationKey : translationKeys) { - logger.info("{}", translationKey); - } + // for (int i = 0; i < 3; i++) { + // String repoName = "repo_%d".formatted(i); + // String tagForUpload = ThirdPartyTMSPhrase.getTagForUpload(repoName); + // + // logger.info("tagForUpload: {}", tagForUpload); + // + // String fileContentAndroid = generateFileContent(repoName).toString(); + // phraseClient.uploadAndWait( + // testProjectId, + // "en", + // "xml", + // "strings.xml", + // fileContentAndroid, + // ImmutableList.of(tagForUpload)); + // + // new ThirdPartyTMSPhrase(phraseClient) + // .removeUnusedKeysAndTags(testProjectId, repoName, tagForUpload); + // } + // + // List translationKeys = phraseClient.getKeys(testProjectId); + // for (TranslationKey translationKey : translationKeys) { + // logger.info("{}", translationKey); + // } // // String fileContentAndroid2 = @@ -88,23 +93,42 @@ public void test() { // """; // phraseClient.uploadCreateFile( // testProjectId, "fr", "xml", "strings.xml", fileContentAndroid2, null); - // String s2 = phraseClient.localeDownload(testProjectId, "fr", "xml"); - // logger.info(s2); + + String s2 = + phraseClient.localeDownload( + testProjectId, + "en", + "xml", + "startWithABadTag", + () -> + phraseClient.listTags(testProjectId).stream() + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> tagName.startsWith("push_repo_2")) + .collect(Collectors.joining(","))); + + logger.info(s2); } - static StringBuilder generateFileContent() { + static StringBuilder generateFileContent(String repositoryName) { StringBuilder fileContentAndroidBuilder = new StringBuilder(); fileContentAndroidBuilder.append( """ - Locale Tester - """); + Locale Tester + """ + .formatted(repositoryName)); - for (int i = 0; i < 2000; i++) { + ZonedDateTime now = JSR310Migration.dateTimeNowInUTC(); + for (int i = 0; i < 1; i++) { fileContentAndroidBuilder.append( String.format("Settings\n", i)); + fileContentAndroidBuilder.append( + String.format( + "Settings\n", + repositoryName, i, now.toString())); } fileContentAndroidBuilder.append(""); From ac73c8bb96959bbd2a441f24ec00d8cb8b26113c Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Sat, 22 Jun 2024 14:26:40 -0700 Subject: [PATCH 034/105] Fix issues related to tag names having some character converted --- .../service/thirdparty/ThirdPartyService.java | 9 ++- .../thirdparty/ThirdPartyTMSPhrase.java | 59 ++++++++++++++----- .../ThridPartyTMSPhraseException.java | 4 ++ .../thirdparty/phrase/PhraseClient.java | 23 ++++++-- 4 files changed, 74 insertions(+), 21 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java index c1f8ef75f9..07ba5f3f06 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java @@ -278,7 +278,14 @@ void mapMojitoAndThirdPartyTextUnits( thirdPartyTextUnits.stream() .collect( groupingBy( - o -> assetCache.getUnchecked(o.getAssetPath()).orElse(null), + o -> + assetCache + .getUnchecked(o.getAssetPath()) + .orElseThrow( + () -> + new RuntimeException( + "Trying to map a third party text unit for an asset (%s) that does not exist in the repository" + .formatted(o.getAssetPath()))), LinkedHashMap::new, toList())); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 2b53a58a26..c09c8a4ed3 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -81,7 +81,10 @@ public List getThirdPartyTextUnits( List thirdPartyTextUnits = new ArrayList<>(); - List phraseTranslationKeys = phraseClient.getKeys(projectId); + String currentTagsForRepository = getCurrentTagsForRepository(repository, projectId); + + List phraseTranslationKeys = + phraseClient.getKeys(projectId, currentTagsForRepository); for (TranslationKey translationKey : phraseTranslationKeys) { @@ -182,9 +185,7 @@ public void removeUnusedKeysAndTags( .map(Tag::getName) .filter(Objects::nonNull) .filter(tagName -> tagName.startsWith(TAG_PREFIX)) - .filter( - tagName -> - !tagName.startsWith(TAG_PREFIX_WITH_REPOSITORY.formatted(repositoryName))) + .filter(tagName -> !tagName.startsWith(getTagNamePrefixForRepository(repositoryName))) .toList(); List allActiveTags = new ArrayList<>(tagsForOtherRepositories); @@ -242,12 +243,25 @@ private List getSourceTextUnitDTOsPluralOnly( public static String getTagForUpload(String repositoryName) { ZonedDateTime zonedDateTime = JSR310Migration.dateTimeNowInUTC(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy_MM_dd_HH_mm_ss_SSS"); - return ("%s%s_%s_%s") - .formatted( - TAG_PREFIX, - repositoryName, - formatter.format(zonedDateTime), - Math.abs(UUID.randomUUID().getLeastSignificantBits() % 1000)); + return normalizeTagName( + "%s%s_%s_%s" + .formatted( + TAG_PREFIX, + repositoryName, + formatter.format(zonedDateTime), + Math.abs(UUID.randomUUID().getLeastSignificantBits() % 1000))); + } + + private static String getTagNamePrefixForRepository(String repositoryName) { + return normalizeTagName(TAG_PREFIX_WITH_REPOSITORY.formatted(repositoryName)); + } + + /** + * At least "/" are getting converted by phrase into "_", apply that logic on our side to have + * consistent filtering + */ + private static String normalizeTagName(String repositoryName) { + return repositoryName.replace("/", "_"); } @Override @@ -269,7 +283,7 @@ public PollableFuture pull( for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale) { String localeTag = repositoryLocale.getLocale().getBcp47Tag(); - logger.info("Downloading locale: {} from Phrase", localeTag); + logger.info("Downloading locale: {} from Phrase with tags: {}", localeTag, currentTags); String fileContent = phraseClient.localeDownload( @@ -294,12 +308,25 @@ public PollableFuture pull( return null; } + /** + * It should typically return a single tag, but if concurrent syncs, it could return more. That + * should not be a problem when downloading or keys for mapping or pulling strings + */ private String getCurrentTagsForRepository(Repository repository, String projectId) { - return phraseClient.listTags(projectId).stream() - .map(Tag::getName) - .filter(Objects::nonNull) - .filter(tagName -> tagName.startsWith(repository.getName())) - .collect(Collectors.joining(",")); + String tagNamePrefixForRepository = getTagNamePrefixForRepository(repository.getName()); + String tags = + phraseClient.listTags(projectId).stream() + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> tagName.startsWith(tagNamePrefixForRepository)) + .collect(Collectors.joining(",")); + + if (tags.isBlank()) { + throw new RuntimeException( + "There are no current tags for the repository, make sure push was run first or that the repository name does not contain special character (which will need to get normalized)"); + } + + return tags; } @Override diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java index 835c16c069..1756f5b2a6 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java @@ -4,4 +4,8 @@ public class ThridPartyTMSPhraseException extends RuntimeException { public ThridPartyTMSPhraseException(String msg, Throwable e) { super(msg, e); } + + public ThridPartyTMSPhraseException(String message) { + super(message); + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java index d5ffd75a27..831cca200b 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -105,16 +105,23 @@ String uploadCreateFile( Path tmpWorkingDirectory = null; + logger.info( + "uploadCreateFile: projectId: {}, localeId: {}, fileName: {}, tags: {}", + projectId, + localeId, + fileName, + tags); + try { tmpWorkingDirectory = createTempDirectory("phrase-integration"); if (tmpWorkingDirectory.toFile().exists()) { - logger.info("Created temporary working directory: {}", tmpWorkingDirectory); + logger.debug("Created temporary working directory: {}", tmpWorkingDirectory); } Path fileToUpload = tmpWorkingDirectory.resolve(fileName); - logger.info("Create file: {}", fileToUpload); + logger.debug("Create file: {}", fileToUpload); createDirectories(fileToUpload.getParent()); write(fileToUpload, fileContent); @@ -294,7 +301,7 @@ public String localeDownload( .block(); } - public List getKeys(String projectId) { + public List getKeys(String projectId, String tags) { KeysApi keysApi = new KeysApi(apiClient); AtomicInteger page = new AtomicInteger(0); int batchSize = BATCH_SIZE; @@ -305,7 +312,15 @@ public List getKeys(String projectId) { () -> { logger.info("Fetching keys for project: {}, page: {}", projectId, page); return keysApi.keysList( - projectId, null, page.get(), batchSize, null, null, null, null, null); + projectId, + null, + page.get(), + batchSize, + null, + null, + null, + "tags:%s".formatted(tags), + null); }) .retryWhen( retryBackoffSpec.doBeforeRetry( From 1a43216c2571c1773b568775330cf7ea23dfe31d Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Sat, 22 Jun 2024 15:03:14 -0700 Subject: [PATCH 035/105] Make sure to not import the source comment when import translations from Phrase TMS That happened because the localized files contain the original string description from the Android source file, and those were getting re-imported with the translation --- .../l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index c09c8a4ed3..ba278cf1b5 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -302,6 +302,11 @@ public PollableFuture pull( List textUnitDTOS = mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); + // make sure that the source comment does not get replicated to all translations. Eventually + // we may want to + // communicate some comments, but anyway it should not be the source comment + textUnitDTOS.forEach(t -> t.setComment(null)); + textUnitBatchImporterService.importTextUnits(textUnitDTOS, false, true); } From 867d88285ce9d598b240a5571f596920bfaa3dcf Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 27 Jun 2024 14:22:46 -0700 Subject: [PATCH 036/105] Make Phrase TMS Pull Parallel --- webapp/package-lock.json | 386 ++++++++++++------ .../thirdparty/ThirdPartyTMSPhrase.java | 103 +++-- .../thirdparty/phrase/PhraseClient.java | 5 +- .../TextUnitBatchImporterService.java | 13 +- .../thirdparty/phrase/PhraseClientTest.java | 31 ++ 5 files changed, 378 insertions(+), 160 deletions(-) diff --git a/webapp/package-lock.json b/webapp/package-lock.json index 59d242bee5..b99b572bbd 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -150,13 +150,15 @@ "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", "dev": true, "requires": { - "kind-of": "^6.0.3", + "kind-of": "^3.0.2", "longest": "^1.0.1", "repeat-string": "^1.5.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -293,11 +295,13 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -308,11 +312,13 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -325,17 +331,21 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^6.0.3" + "kind-of": "^5.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -425,11 +435,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -440,11 +452,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -457,11 +471,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -472,11 +488,13 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -488,7 +506,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "micromatch": { "version": "3.1.10", @@ -503,7 +523,7 @@ "extend-shallow": "^3.0.2", "extglob": "^2.0.4", "fragment-cache": "^0.2.1", - "kind-of": "^6.0.3", + "kind-of": "^6.0.2", "nanomatch": "^1.2.9", "object.pick": "^1.3.0", "regex-not": "^1.0.0", @@ -512,7 +532,9 @@ }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1022,7 +1044,9 @@ }, "dependencies": { "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "mkdirp": { "version": "0.5.5", @@ -1030,11 +1054,13 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -1693,7 +1719,9 @@ "dev": true }, "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "mkdirp": { "version": "0.5.5", @@ -1701,11 +1729,13 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -1835,11 +1865,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1850,11 +1882,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1867,11 +1901,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1883,7 +1919,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -2561,17 +2599,21 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "normalize-path": { "version": "3.0.0", @@ -2649,18 +2691,22 @@ }, "dependencies": { "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "mkdirp": { "version": "0.5.0", "resolved": "https://box.jfrog.io/box/api/npm/boxnpm/mkdirp/-/mkdirp-0.5.0.tgz", "integrity": "sha1-HXMHam35hs2TROFecfzAWkyavxI=", "requires": { - "minimist": "^0.2.1" + "minimist": "0.0.8" }, "dependencies": { "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" } } } @@ -2707,7 +2753,7 @@ "requires": { "for-own": "^1.0.0", "is-plain-object": "^2.0.4", - "kind-of": "^6.0.3", + "kind-of": "^6.0.0", "shallow-clone": "^1.0.0" }, "dependencies": { @@ -2721,7 +2767,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -2870,13 +2918,16 @@ "dev": true, "optional": true, "requires": { - "ini": "^1.3.8", + "ini": "^1.3.4", "proto-list": "~1.2.1" }, "dependencies": { "ini": { - "version": "^1.3.8", - "dev": true + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "optional": true } } }, @@ -3425,11 +3476,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -3440,11 +3493,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -3457,11 +3512,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -3473,7 +3530,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -4358,7 +4417,9 @@ "optional": true }, "ini": { - "version": "^1.3.8" + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "is-fullwidth-code-point": { "version": "1.0.0", @@ -4385,7 +4446,9 @@ } }, "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "minipass": { "version": "2.9.0", @@ -4412,12 +4475,15 @@ "dev": true, "optional": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", - "dev": true + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "dev": true, + "optional": true } } }, @@ -4566,18 +4632,24 @@ "optional": true, "requires": { "deep-extend": "^0.6.0", - "ini": "^1.3.8", - "minimist": "^0.2.1", + "ini": "~1.3.0", + "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" }, "dependencies": { "ini": { - "version": "^1.3.8", - "dev": true + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "optional": true }, "minimist": { - "version": "^0.2.1", - "dev": true + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "dev": true, + "optional": true } } }, @@ -5082,7 +5154,7 @@ "dev": true, "requires": { "is-number": "^3.0.0", - "kind-of": "^6.0.3" + "kind-of": "^4.0.0" }, "dependencies": { "is-number": { @@ -5091,17 +5163,21 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5661,11 +5737,13 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5722,11 +5800,13 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5746,11 +5826,13 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^6.0.3" + "kind-of": "^5.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5960,12 +6042,14 @@ "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz", "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=", "requires": { - "node-fetch": "^2.6.1", + "node-fetch": "^1.0.1", "whatwg-fetch": ">=0.10.0" }, "dependencies": { "node-fetch": { - "version": "^2.6.1", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "requires": { "whatwg-url": "^5.0.0" } @@ -6108,7 +6192,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true }, "lazy-cache": { @@ -6170,17 +6256,21 @@ "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.0" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } }, "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" } } }, @@ -6380,7 +6470,7 @@ "decamelize": "^1.1.2", "loud-rejection": "^1.0.0", "map-obj": "^1.0.1", - "minimist": "^0.2.1", + "minimist": "^1.1.3", "normalize-package-data": "^2.3.4", "object-assign": "^4.0.1", "read-pkg-up": "^1.0.1", @@ -6389,7 +6479,9 @@ }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -6467,7 +6559,9 @@ } }, "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true }, "mixin-deep": { @@ -6515,11 +6609,13 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -6565,7 +6661,7 @@ "extend-shallow": "^3.0.2", "fragment-cache": "^0.2.1", "is-windows": "^1.0.2", - "kind-of": "^6.0.3", + "kind-of": "^6.0.2", "object.pick": "^1.3.0", "regex-not": "^1.0.0", "snapdragon": "^0.8.1", @@ -6604,7 +6700,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -7022,7 +7120,7 @@ "requires": { "copy-descriptor": "^0.1.0", "define-property": "^0.2.5", - "kind-of": "^6.0.3" + "kind-of": "^3.0.3" }, "dependencies": { "define-property": { @@ -7035,7 +7133,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8072,11 +8172,13 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8087,11 +8189,13 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8104,17 +8208,21 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^6.0.3" + "kind-of": "^5.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -8204,11 +8312,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8219,11 +8329,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8236,11 +8348,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8251,11 +8365,13 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8267,7 +8383,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "micromatch": { "version": "3.1.10", @@ -8282,7 +8400,7 @@ "extend-shallow": "^3.0.2", "extglob": "^2.0.4", "fragment-cache": "^0.2.1", - "kind-of": "^6.0.3", + "kind-of": "^6.0.2", "nanomatch": "^1.2.9", "object.pick": "^1.3.0", "regex-not": "^1.0.0", @@ -8291,7 +8409,9 @@ }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8796,12 +8916,14 @@ "dev": true, "requires": { "is-extendable": "^0.1.1", - "kind-of": "^6.0.3", + "kind-of": "^5.0.0", "mixin-object": "^2.0.1" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8888,11 +9010,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8903,11 +9027,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8920,11 +9046,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8936,7 +9064,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -8946,11 +9076,13 @@ "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.2.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -9593,11 +9725,13 @@ "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -9651,17 +9785,21 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -10173,7 +10311,7 @@ "string-width": "^1.0.2", "which-module": "^1.0.0", "y18n": "^3.2.1", - "yargs-parser": "18.1.2" + "yargs-parser": "^4.2.0" }, "dependencies": { "yargs-parser": { @@ -10325,7 +10463,7 @@ "string-width": "^1.0.2", "which-module": "^1.0.0", "y18n": "^3.2.1", - "yargs-parser": "18.1.2" + "yargs-parser": "^5.0.0" }, "dependencies": { "camelcase": { @@ -10366,4 +10504,4 @@ } } } -} \ No newline at end of file +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index ba278cf1b5..7fbbbbb639 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -19,9 +19,12 @@ import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; import com.box.l10n.mojito.service.tm.search.UsedFilter; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.phrase.client.model.Tag; import com.phrase.client.model.TranslationKey; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.AbstractMap.SimpleEntry; @@ -37,7 +40,6 @@ import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @@ -50,19 +52,28 @@ public class ThirdPartyTMSPhrase implements ThirdPartyTMS { static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhrase.class); - @Autowired TextUnitSearcher textUnitSearcher = new TextUnitSearcher(); + TextUnitSearcher textUnitSearcher = new TextUnitSearcher(); - @Autowired TextUnitBatchImporterService textUnitBatchImporterService; + TextUnitBatchImporterService textUnitBatchImporterService; - @Autowired(required = false) PhraseClient phraseClient; - @Autowired RepositoryService repositoryService; + RepositoryService repositoryService; - public ThirdPartyTMSPhrase() {} + MeterRegistry meterRegistry; - public ThirdPartyTMSPhrase(PhraseClient phraseClient) { + public ThirdPartyTMSPhrase( + TextUnitSearcher textUnitSearcher, + TextUnitBatchImporterService textUnitBatchImporterService, + PhraseClient phraseClient, + RepositoryService repositoryService, + MeterRegistry meterRegistry) { + + this.textUnitSearcher = textUnitSearcher; + this.textUnitBatchImporterService = textUnitBatchImporterService; this.phraseClient = phraseClient; + this.repositoryService = repositoryService; + this.meterRegistry = meterRegistry; } @Override @@ -281,36 +292,68 @@ public PollableFuture pull( String currentTags = getCurrentTagsForRepository(repository, projectId); - for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale) { - String localeTag = repositoryLocale.getLocale().getBcp47Tag(); - logger.info("Downloading locale: {} from Phrase with tags: {}", localeTag, currentTags); + // may already hit rate limit, according it is 4 qps ... there is a retry in the locale client + // though. + // but 4 qps is very low to download 64 locales. + repositoryLocalesWithoutRootLocale.parallelStream() + .forEach( + locale -> pullLocaleTimed(repository, projectId, locale, pluralSeparator, currentTags)); - String fileContent = - phraseClient.localeDownload( - projectId, - localeTag, - "xml", - currentTags, - () -> getCurrentTagsForRepository(repository, projectId)); + return null; + } - logger.info("file content from pull: {}", fileContent); + private void pullLocaleTimed( + Repository repository, + String projectId, + RepositoryLocale repositoryLocale, + String pluralSeparator, + String currentTags) { + try (var timer = + Timer.resource(meterRegistry, "ThirdPartyTMSPhrase.pullLocale") + .tag("repository", repository.getName())) { - AndroidStringDocumentMapper mapper = - new AndroidStringDocumentMapper( - pluralSeparator, null, localeTag, repository.getName(), true, null); + pullLocale(repository, projectId, repositoryLocale, pluralSeparator, currentTags); + } + } - List textUnitDTOS = - mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); + private void pullLocale( + Repository repository, + String projectId, + RepositoryLocale repositoryLocale, + String pluralSeparator, + String currentTags) { - // make sure that the source comment does not get replicated to all translations. Eventually - // we may want to - // communicate some comments, but anyway it should not be the source comment - textUnitDTOS.forEach(t -> t.setComment(null)); + String localeTag = repositoryLocale.getLocale().getBcp47Tag(); + logger.info("Downloading locale: {} from Phrase with tags: {}", localeTag, currentTags); - textUnitBatchImporterService.importTextUnits(textUnitDTOS, false, true); - } + Stopwatch localeDownloadStopWatch = Stopwatch.createStarted(); + String fileContent = + phraseClient.localeDownload( + projectId, + localeTag, + "xml", + currentTags, + () -> getCurrentTagsForRepository(repository, projectId)); - return null; + logger.info("Phrase locale download took: {}", localeDownloadStopWatch.elapsed()); + + logger.debug("file content from pull: {}", fileContent); + + AndroidStringDocumentMapper mapper = + new AndroidStringDocumentMapper( + pluralSeparator, null, localeTag, repository.getName(), true, null); + + List textUnitDTOS = + mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); + + // make sure that the source comment does not get replicated to all translations. Eventually + // we may want to + // communicate some comments, but anyway it should not be the source comment + textUnitDTOS.forEach(t -> t.setComment(null)); + + Stopwatch importStopWatch = Stopwatch.createStarted(); + textUnitBatchImporterService.importTextUnits(textUnitDTOS, false, true); + logger.info("Time importing text units: {}", importStopWatch.elapsed()); } /** diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java index 831cca200b..ddf6181be4 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -282,8 +282,9 @@ public String localeDownload( "Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s" .formatted(projectId, locale)); - if (getErrorMessageFromOptionalApiException(doBeforeRetry.failure()) - .contains("Invalid Download Options. Parameter tags ")) { + if (onTagErrorRefreshCallback != null + && getErrorMessageFromOptionalApiException(doBeforeRetry.failure()) + .contains("Invalid Download Options. Parameter tags ")) { String newTags = onTagErrorRefreshCallback.get(); logger.warn( "Replacing old tags: {} with new tags: {} for download locale", diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java index 5c7e3c3b0e..188dbf8173 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java @@ -5,6 +5,7 @@ import static com.box.l10n.mojito.utils.Predicates.logIfFalse; import com.box.l10n.mojito.JSR310Migration; +import com.box.l10n.mojito.aspect.StopWatch; import com.box.l10n.mojito.entity.Asset; import com.box.l10n.mojito.entity.Locale; import com.box.l10n.mojito.entity.Repository; @@ -128,6 +129,7 @@ public PollableFuture asyncImportTextUnits( ImportTextUnitJob.class, importTextUnitJobInput, schedulerName); } + @StopWatch public PollableFuture importTextUnits( List textUnitDTOs, boolean integrityCheckSkipped, @@ -194,6 +196,7 @@ public PollableFuture importTextUnits( * @param asset * @param textUnitsToImport text units to which the current text units must be added */ + @StopWatch void mapTextUnitsToImportWithExistingTextUnits( Locale locale, Asset asset, List textUnitsToImport) { logger.debug( @@ -205,14 +208,16 @@ void mapTextUnitsToImportWithExistingTextUnits( textUnitsToImport.forEach(tu -> match.apply(tu).ifPresent(m -> tu.setCurrentTextUnit(m))); } + @StopWatch @Transactional void importTextUnitsOfLocaleAndAsset( Locale locale, Asset asset, List textUnitsToImport) { ZonedDateTime importTime = JSR310Migration.newDateTimeEmptyCtor(); - logger.debug( - "Start import text units for asset: {} and locale: {}", + logger.info( + "Start import text units for asset: {}, locale: {}, count: {}", asset.getPath(), - locale.getBcp47Tag()); + locale.getBcp47Tag(), + textUnitsToImport.size()); textUnitsToImport.stream() .filter( @@ -244,7 +249,7 @@ void importTextUnitsOfLocaleAndAsset( TMTextUnitCurrentVariant tmTextUnitCurrentVariant = null; if (currentTextUnit.getTmTextUnitCurrentVariantId() != null) { - logger.debug("Looking up current variant"); + // this is making many calls! tmTextUnitCurrentVariant = tmTextUnitCurrentVariantRepository.findByLocale_IdAndTmTextUnit_Id( currentTextUnit.getLocaleId(), currentTextUnit.getTmTextUnitId()); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java index b929a4903e..bdf64f52e6 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -1,6 +1,7 @@ package com.box.l10n.mojito.service.thirdparty.phrase; import com.box.l10n.mojito.JSR310Migration; +import com.google.common.base.Stopwatch; import com.phrase.client.model.Tag; import java.time.ZonedDateTime; import java.util.List; @@ -49,6 +50,36 @@ public void testRemoveTag() { phraseClient.deleteTags(testProjectId, tagsToDelete); } + @Test + public void testParallelDownload() { + // measure time of following call + Stopwatch total = Stopwatch.createStarted(); + + List locales = + List.of( + "bg", "bn", "bs", "ca", "cs", "da", "de", "el", "es-419", "es", "et", "fi", "fr-CA", + "fr", "gu", "hi", "hr", "hu", "hy", "id", "is", "it", "ja", "ka", "kn", "ko", "lv", + "mk", "mr", "ms", "my", "nb", "nl", "pl", "pt", "pt-PT", "ro", "ru", "sk", "sl", "so", + "sq", "sr", "sv", "sw", "ta", "te", "th", "tr", "uk", "vi", "zh", "zh-HK", "zh-Hant"); + + locales.parallelStream() + .forEach( + l -> { + Stopwatch started = Stopwatch.createStarted(); + String s = + phraseClient.localeDownload( + "9b6f4e167397549fe7ec1fe82497159b", + l, + "xml", + "push_chatgpt-web_2024_06_27_05_14_58_356_208", + null); + // logger.info(s); + logger.info("Download: {} in {}", l, started.elapsed()); + }); + + logger.info("total time: {}", total.elapsed()); + } + @Test public void test() { Assume.assumeNotNull(testProjectId); From c172bdbb97e79fb96eb5eb849c9524ab467cec29 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 3 Jul 2024 22:28:28 -0700 Subject: [PATCH 037/105] Non-backward compatible - Fixes batch import logic to save target comments Previously, the source comment (text unit) was systemically copied into the target comment (text unit variant) and that for all imported translations. This seems to be the behavior since day 1 of the TextUnitBatchImporterService, yet it is a bug. It affects ThirdPartySync and all other batch imports. This fix may cause issue to production instance as it will attempt remove all the invalid comments, potentially leading to a massive amount of translation to be changed, ie. putting to much load on the system. --- .../tm/importer/TextUnitBatchImporterService.java | 7 ++++--- .../tm/importer/TextUnitForBatchMatcherImport.java | 9 +++++++++ .../tm/importer/TextUnitBatchImporterServiceTest.java | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java index 188dbf8173..6a185f45c0 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java @@ -264,7 +264,7 @@ void importTextUnitsOfLocaleAndAsset( currentTextUnit.getTmTextUnitId(), locale.getId(), textUnitForBatchImport.getContent(), - textUnitForBatchImport.getComment(), + textUnitForBatchImport.getTargetComment(), textUnitForBatchImport.getStatus(), textUnitForBatchImport.isIncludedInLocalizedFile(), importTime, @@ -299,11 +299,11 @@ boolean isUpdateNeeded(TextUnitForBatchMatcherImport textUnitForBatchImport) { currentTextUnit.getStatus(), DigestUtils.md5Hex(currentTextUnit.getTarget()), currentTextUnit.isIncludedInLocalizedFile(), - currentTextUnit.getComment(), + currentTextUnit.getTargetComment(), textUnitForBatchImport.getStatus(), DigestUtils.md5Hex(textUnitForBatchImport.getContent()), textUnitForBatchImport.isIncludedInLocalizedFile(), - textUnitForBatchImport.getComment()); + textUnitForBatchImport.getTargetComment()); } List getTextUnitTDOsForLocaleAndAsset(Locale locale, Asset asset) { @@ -430,6 +430,7 @@ List skipInvalidAndConvertToTextUnitForBatchImpor textUnitForBatchImport.setLocale(localeService.findByBcp47Tag(t.getTargetLocale())); textUnitForBatchImport.setContent(NormalizationUtils.normalize(t.getTarget())); textUnitForBatchImport.setComment(t.getComment()); + textUnitForBatchImport.setTargetComment(t.getTargetComment()); textUnitForBatchImport.setIncludedInLocalizedFile(t.isIncludedInLocalizedFile()); textUnitForBatchImport.setStatus(t.getStatus() == null ? APPROVED : t.getStatus()); return textUnitForBatchImport; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitForBatchMatcherImport.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitForBatchMatcherImport.java index e5f81a23ff..32a7dc55fe 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitForBatchMatcherImport.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitForBatchMatcherImport.java @@ -21,6 +21,7 @@ public class TextUnitForBatchMatcherImport implements TextUnitForBatchMatcher { String content; String name; String comment; + String targetComment; Long tmTextUnitId; TextUnitDTO currentTextUnit; boolean includedInLocalizedFile; @@ -78,6 +79,14 @@ public void setComment(String comment) { this.comment = comment; } + public String getTargetComment() { + return targetComment; + } + + public void setTargetComment(String targetComment) { + this.targetComment = targetComment; + } + @Override public Long getTmTextUnitId() { return tmTextUnitId; diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java index 970fe254fe..8f0e16aaf7 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java @@ -82,6 +82,7 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException { textUnitDTO.setAssetPath(tmTestData.asset.getPath()); textUnitDTO.setName("TEST2"); textUnitDTO.setTarget("TEST2 translation for fr"); + textUnitDTO.setComment("Comment2"); TextUnitDTO textUnitDTO2 = new TextUnitDTO(); textUnitDTO2.setRepositoryName(tmTestData.repository.getName()); @@ -89,6 +90,7 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException { textUnitDTO2.setAssetPath(tmTestData.asset.getPath()); textUnitDTO2.setName("TEST3"); textUnitDTO2.setTarget("TEST3 translation for fr"); + textUnitDTO2.setTargetComment("TEST3 target comment"); TextUnitDTO textUnitDTO3 = new TextUnitDTO(); textUnitDTO3.setRepositoryName(tmTestData.repository.getName()); @@ -123,9 +125,11 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException { i++; assertEquals("TEST2", textUnitDTOsFromSearch.get(i).getName()); assertEquals("TEST2 translation for fr", textUnitDTOsFromSearch.get(i).getTarget()); + assertNull(textUnitDTOsFromSearch.get(i).getTargetComment()); i++; assertEquals("TEST3", textUnitDTOsFromSearch.get(i).getName()); assertEquals("TEST3 translation for fr", textUnitDTOsFromSearch.get(i).getTarget()); + assertEquals("TEST3 target comment", textUnitDTOsFromSearch.get(i).getTargetComment()); i++; } From 5b608e4ad54281692d8640a3d7b6de31964898b6 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 5 Aug 2024 13:38:51 -0700 Subject: [PATCH 038/105] Ignore PhraseClientTest --- .../mojito/service/thirdparty/phrase/PhraseClientTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java index bdf64f52e6..796fbfe355 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -8,6 +8,7 @@ import java.util.Objects; import java.util.stream.Collectors; import org.junit.Assume; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.slf4j.Logger; @@ -26,6 +27,7 @@ PhraseClientPropertiesConfig.class }) @EnableConfigurationProperties +@Ignore public class PhraseClientTest { static Logger logger = LoggerFactory.getLogger(PhraseClientTest.class); @@ -52,6 +54,8 @@ public void testRemoveTag() { @Test public void testParallelDownload() { + + Assume.assumeNotNull(testProjectId); // measure time of following call Stopwatch total = Stopwatch.createStarted(); From 5ccca9e716d3694f21450c4cf5eec2de5342d1d3 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 9 Aug 2024 12:36:16 -0700 Subject: [PATCH 039/105] Improve remove untranslated string in AndroidFilter Also add an option to remove descriptions --- .../ImportLocalizedAssetCommandTest.java | 1 + .../res/values-fr-rCA/strings.xml | 30 ++++ .../res/values-fr-rFR/strings.xml | 30 ++++ .../res/values-ja-rJP/strings.xml | 25 +++ .../res/values-ru-rRU/strings.xml | 40 +++++ .../res/values-fr-rCA/strings.xml | 7 + .../res/values-fr-rFR/strings.xml | 25 +++ .../res/values-ja-rJP/strings.xml | 18 ++ .../res/values-ru-rRU/strings.xml | 27 +++ .../res/values-fr-rCA/strings.xml | 7 + .../res/values-fr-rFR/strings.xml | 25 +++ .../res/values-ja-rJP/strings.xml | 18 ++ .../res/values-ru-rRU/strings.xml | 27 +++ .../input/source/res/values/strings.xml | 38 +++++ .../res/values-fr-rCA/strings.xml | 4 + .../res/values-fr-rFR/strings.xml | 31 ++++ .../res/values-ja-rJP/strings.xml | 20 +++ .../res/values-ru-rRU/strings.xml | 30 ++++ .../mojito/okapi/filters/AndroidFilter.java | 161 +++++++++++++++++- .../mojito/okapi/filters/FilterOptions.java | 6 + .../okapi/filters/AndroidFilterTest.java | 119 +++++++++++++ .../box/l10n/mojito/okapi/TranslateStep.java | 28 ++- .../l10n/mojito/service/tm/TMServiceTest.java | 9 +- 23 files changed, 704 insertions(+), 22 deletions(-) create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/source/res/values/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rCA/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rFR/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ja-rJP/strings.xml create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ru-rRU/strings.xml diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java index e457f97035..150d95395c 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java @@ -162,6 +162,7 @@ public void importAndroidStringsPostProcessing() throws Exception { checkExpectedGeneratedResources(); } + @Test public void importAndroidStringsPluralWithThirdPartySync() throws Exception { Assume.assumeNotNull(testProjectId); diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..626d7f70a2 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml @@ -0,0 +1,30 @@ + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + Membres + + %1$,d collaborateur + %1$,d collaborateurs + + + + %1$d ingrédient + %1$d ingrédients + + + Supprimer + Untranslated + + plural_untranslated_one + plural_untranslated_other + + do_not_translate + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..626d7f70a2 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml @@ -0,0 +1,30 @@ + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + Membres + + %1$,d collaborateur + %1$,d collaborateurs + + + + %1$d ingrédient + %1$d ingrédients + + + Supprimer + Untranslated + + plural_untranslated_one + plural_untranslated_other + + do_not_translate + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..f6d60b9ef8 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml @@ -0,0 +1,25 @@ + + + + 所要時間：%1$,d 時間 + + ユーザー + + ボード参加者：%1$,d 人 + + + + %1$d 個の材料 + + + 削除 + Untranslated + + plural_untranslated_other + + do_not_translate + + + plural_do_not_translate_other + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..9ea8e22ee0 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml @@ -0,0 +1,40 @@ + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + Люди + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + Удалить + Untranslated + + plural_untranslated_one + plural_untranslated_other + plural_untranslated_other + plural_untranslated_other + + do_not_translate + + + plural_do_not_translate_one + plural_do_not_translate_other + plural_do_not_translate_other + plural_do_not_translate_other + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..7f8da59742 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml @@ -0,0 +1,7 @@ + + + + + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..ae2676f2b4 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + Membres + + %1$,d collaborateur + %1$,d collaborateurs + + + + %1$d ingrédient + %1$d ingrédients + + + Supprimer + do_not_translate + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..bfc2d7d33f --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml @@ -0,0 +1,18 @@ + + + + 所要時間：%1$,d 時間 + + ユーザー + + ボード参加者：%1$,d 人 + + + + %1$d 個の材料 + + + 削除 + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..4185c07f6e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml @@ -0,0 +1,27 @@ + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + Люди + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + Удалить + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..7f8da59742 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml @@ -0,0 +1,7 @@ + + + + + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..26960ff3f8 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + Membres + + %1$,d collaborateur + %1$,d collaborateurs + + + + %1$d ingrédient + %1$d ingrédients + + + Supprimer + do_not_translate + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..8f5a230089 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml @@ -0,0 +1,18 @@ + + + + 所要時間：%1$,d 時間 + + ユーザー + + ボード参加者：%1$,d 人 + + + + %1$d 個の材料 + + + 削除 + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..487445f368 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml @@ -0,0 +1,27 @@ + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + Люди + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + Удалить + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/source/res/values/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/source/res/values/strings.xml new file mode 100644 index 0000000000..75700cc771 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/source/res/values/strings.xml @@ -0,0 +1,38 @@ + + + + + %,d hr to cook + %,d hrs to cook + + + People + + + %1$,d collaborator + %1$,d collaborators + + + + + %d ingredient + %d ingredients + + + + Delete + + Untranslated + + plural_untranslated_one + plural_untranslated_other + + + do_not_translate + + + + plural_do_not_translate_one + plural_do_not_translate_other + + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..0f85d2dd1e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rCA/strings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..92f1912ff1 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rFR/strings.xml @@ -0,0 +1,31 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + + do_not_translate_fr + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..b4d3cad434 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ja-rJP/strings.xml @@ -0,0 +1,20 @@ + + + + 所要時間：%1$,d 時間 + + + ユーザー + + + ボード参加者：%1$,d 人 + + + + + %1$d 個の材料 + + + + 削除 + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..7cb534e72b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ru-rRU/strings.xml @@ -0,0 +1,30 @@ + + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + + Люди + + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + + Удалить + \ No newline at end of file diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java index f60b6ed9ff..e784403620 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java @@ -1,10 +1,23 @@ package com.box.l10n.mojito.okapi.filters; import com.box.l10n.mojito.okapi.TextUnitUtils; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import net.sf.okapi.common.Event; import net.sf.okapi.common.IResource; import net.sf.okapi.common.LocaleId; @@ -19,6 +32,13 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; /** * @author jaurambault @@ -35,6 +55,10 @@ public class AndroidFilter extends XMLFilter { private static final String OPTION_OLD_ESCAPING = "oldEscaping"; + private static final String REMOVE_DESCRIPTION = "removeDescription"; + + private static final String POST_PROCESS_INDENT = "postProcessIndent"; + private static final Pattern PATTERN_PLURAL_START = Pattern.compile(""); private static final Pattern PATTERN_XML_COMMENT = Pattern.compile(""); @@ -84,12 +108,28 @@ public List getConfigurations() { List eventQueue = new ArrayList<>(); + boolean removeDescription = false; + + int postProcessIndent = 2; + @Override public void open(RawDocument input) { super.open(input); targetLocale = input.getTargetLocale(); hasAnnotation = input.getAnnotation(CopyFormsOnImport.class) != null; applyFilterOptions(input); + input.setAnnotation( + new RemoveUntranslatedStategyAnnotation( + RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); + // The post processing is optionally enable based on the removed description option. Contrary to + // a filter like + // JsonFilter the post-processing where the post processing is always disable by default, and + // then activated + // in the TranslateStep + input.setAnnotation( + new OutputDocumentPostProcessingAnnotation( + new AndroidFilePostProcessing(removeDescription, postProcessIndent)::execute, + removeDescription)); } void applyFilterOptions(RawDocument input) { @@ -104,9 +144,20 @@ void applyFilterOptions(RawDocument input) { androidXMLEncoder.oldEscaping = oldEscaping; } }); - } + logger.debug("filter option, old escaping: {}", oldEscaping); - logger.debug("filter option, old escaping: {}", oldEscaping); + filterOptions.getBoolean( + REMOVE_DESCRIPTION, + b -> { + removeDescription = b; + }); + + filterOptions.getInteger( + POST_PROCESS_INDENT, + i -> { + postProcessIndent = i; + }); + } } @Override @@ -369,4 +420,110 @@ void updateFormInSkeleton(ITextUnit textUnit) { } } } + + static class AndroidFilePostProcessing { + static final String DESCRIPTION_ATTRIBUTE = "description"; + boolean removeDescription; + int indent; + + AndroidFilePostProcessing(boolean removeDescription, int indent) { + this.removeDescription = removeDescription; + this.indent = indent; + } + + String execute(String xmlContent) { + + if (xmlContent == null || xmlContent.isBlank()) { + return xmlContent; + } + + try { + DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); + Document document = documentBuilder.parse(new InputSource(new StringReader(xmlContent))); + document.getDocumentElement().normalize(); + + NodeList stringElements = document.getElementsByTagName("string"); + for (int i = 0; i < stringElements.getLength(); i++) { + Node node = stringElements.item(i); + if (node.getNodeType() == Node.ELEMENT_NODE) { + Element element = (Element) node; + if (element + .getTextContent() + .equals(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { + element.getParentNode().removeChild(element); + i--; + } + + if (element.hasAttribute(DESCRIPTION_ATTRIBUTE)) { + if (removeDescription) { + element.removeAttribute(DESCRIPTION_ATTRIBUTE); + } + } + } + } + + NodeList pluralsElements = document.getElementsByTagName("plurals"); + for (int i = 0; i < pluralsElements.getLength(); i++) { + Element plurals = (Element) pluralsElements.item(i); + NodeList items = plurals.getElementsByTagName("item"); + boolean hasTranslated = false; + + for (int j = 0; j < items.getLength(); j++) { + Element item = (Element) items.item(j); + if (item.getTextContent().equals(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { + item.getParentNode().removeChild(item); + j--; + } else { + hasTranslated = true; + } + } + + if (!hasTranslated) { + plurals.getParentNode().removeChild(plurals); + i--; + } + + if (plurals.hasAttribute(DESCRIPTION_ATTRIBUTE)) { + if (removeDescription) { + plurals.removeAttribute(DESCRIPTION_ATTRIBUTE); + } + } + } + + removeWhitespaceNodes(document); + + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + Transformer transformer = transformerFactory.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty( + "{http://xml.apache.org/xslt}indent-amount", String.valueOf(indent)); + + boolean hasDeclaration = xmlContent.trim().startsWith("= 0; i--) { + Node childNode = childNodes.item(i); + if (childNode instanceof Text && childNode.getNodeValue().isBlank()) { + node.removeChild(childNode); + } else if (childNode instanceof Element) { + removeWhitespaceNodes(childNode); + } + } + } + } } diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java index 5d55a3aabf..21e40dc2ea 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java @@ -33,4 +33,10 @@ public void getBoolean(String key, Consumer consumer) { consumer.accept(Boolean.valueOf(this.options.get(key))); } } + + public void getInteger(String key, Consumer consumer) { + if (this.options.containsKey(key)) { + consumer.accept(Integer.valueOf(this.options.get(key))); + } + } } diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java index 29a7187e91..e8f7397a6f 100644 --- a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java +++ b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java @@ -109,4 +109,123 @@ void testUnescaping(String input, String expected) { logger.debug("> Input:\n{}\n> Expected:\n{}\n> Actual:\n{}\n>>>", input, expected, s); assertEquals(expected, s); } + + @Test + public void testPostProcessingKeepDescription() { + AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = + new AndroidFilter.AndroidFilePostProcessing(false, 2); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + translated + @#$untranslated$#@ + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessing.execute(input); + String expected = + """ + + + somestring to keep + + + translated + + + pin fr + pins fr + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingRemoveDescription() { + AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = + new AndroidFilter.AndroidFilePostProcessing(true, 2); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + translated + @#$untranslated$#@ + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessing.execute(input); + String expected = + """ + + + somestring to keep + + + translated + + + pin fr + pins fr + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingEmptyFile() { + AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = + new AndroidFilter.AndroidFilePostProcessing(true, 2); + String input = ""; + String output = androidFilePostProcessing.execute(input); + String expected = ""; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingNoProlog() { + AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = + new AndroidFilter.AndroidFilePostProcessing(true, 2); + String input = + """ + + somestring to keep + @#$untranslated$#@ + + """; + String output = androidFilePostProcessing.execute(input); + String expected = + """ + + somestring to keep + + """; + assertEquals(expected, output); + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java b/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java index 8ade9e401d..cd63cd5657 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java +++ b/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java @@ -137,22 +137,20 @@ protected Event handleTextUnit(Event event) { if (translation == null && InheritanceMode.REMOVE_UNTRANSLATED.equals(inheritanceMode)) { logger.debug("Remove untranslated text unit"); - Event androidEvent = getEventForAndroidFirstTextUnit(textUnit); - - if (androidEvent == null) { - switch (getRemoveUntranslatedStrategyFromAnnotation()) { - case NOOP_EVENT: - event = Event.createNoopEvent(); - break; - case PLACEHOLDER_AND_POST_PROCESSING: - logger.debug("Set untranslated placeholder for text unit with name: {}", name); - textUnit.setTarget( - targetLocale, - new TextContainer(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)); - enableOutputDocumentPostProcessing(); - break; - } + + switch (getRemoveUntranslatedStrategyFromAnnotation()) { + case NOOP_EVENT: + event = Event.createNoopEvent(); + break; + case PLACEHOLDER_AND_POST_PROCESSING: + logger.debug("Set untranslated placeholder for text unit with name: {}", name); + textUnit.setTarget( + targetLocale, + new TextContainer(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)); + enableOutputDocumentPostProcessing(); + break; } + } else { if (!shouldConvertToHtmlCodes) { diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java index 775cc5f7f1..8831e9b2de 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java @@ -2034,7 +2034,7 @@ public void testLocalizeAndroidStringsRemoveUntranslatedOldEsaping() throws Exce assetResult.get(); String forImport = - "\n" + " Le test\n" + ""; + "\n" + " Le test\n" + "\n"; tmService .importLocalizedAssetAsync( @@ -2053,7 +2053,7 @@ public void testLocalizeAndroidStringsRemoveUntranslatedOldEsaping() throws Exce repoLocale, "en-GB", null, - null, + List.of("postProcessIndent=4"), Status.ALL, InheritanceMode.REMOVE_UNTRANSLATED, null); @@ -2097,10 +2097,9 @@ public void testLocalizeAndroidStringsRemoveUntranslatedSingleItem() throws Exce assetResult.get(); String expectedLocalized = - "\n" + "\n" + "" - + "\n" - + ""; + + "\n"; String localizedAsset = tmService.generateLocalized( From f426dc0d0fbd4ef487bb576e52e1af92a1a14398 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 21 Aug 2024 21:22:49 -0700 Subject: [PATCH 040/105] HtmlTagIntegrityChecker check for duplicate non balanced tag --- .../integritychecker/HtmlTagIntegrityChecker.java | 3 ++- .../integritychecker/HtmlTagIntegrityCheckerTest.java | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java index 09426ceca1..f5d8a533c8 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java @@ -35,7 +35,8 @@ public void check(String sourceContent, String targetContent) throws IntegrityCh logger.debug("Source Html tags: {}", sourceHtmlTags); logger.debug("Make sure the target has the same Html tags as the source"); - if (!sourceHtmlTags.containsAll(targetHtmlTags) + if (sourceHtmlTags.size() != targetHtmlTags.size() + || !sourceHtmlTags.containsAll(targetHtmlTags) || !targetHtmlTags.containsAll(sourceHtmlTags)) { throw new HtmlTagIntegrityCheckerException("HTML tags in source and target are different"); } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java index 4eb5b019dc..df48ae696e 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java @@ -60,6 +60,13 @@ public void testHtmlTagCheckWorksWhenCrossing() { checker.check(source, target); } + @Test(expected = HtmlTagIntegrityCheckerException.class) + public void testHtmlTagCheckFailsWithDuplicatedOpening() { + String source = "There are %1 files"; + String target = "Il y a %1 fichiers"; + checker.check(source, target); + } + @Test(expected = HtmlTagIntegrityCheckerException.class) public void testHtmlTagCheckWorksWhenTagIsModified() { String source = "There are %1 files and %2 folders"; From 3ad81ff68a2d0e81e420c2aedee1f834c0cd66ac Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 22 Aug 2024 17:48:46 -0700 Subject: [PATCH 041/105] Improve error message Just to make the order of file explicit --- .../src/main/java/com/box/l10n/mojito/test/IOTestBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-common/src/main/java/com/box/l10n/mojito/test/IOTestBase.java b/test-common/src/main/java/com/box/l10n/mojito/test/IOTestBase.java index 77086d6ec9..9de797bb72 100644 --- a/test-common/src/main/java/com/box/l10n/mojito/test/IOTestBase.java +++ b/test-common/src/main/java/com/box/l10n/mojito/test/IOTestBase.java @@ -254,9 +254,9 @@ protected void checkDirectoriesContainSameContent(File dir1, File dir2) // If that's a file, check that the both files have the same content if (file2.isFile() && !Files.equal(file1, file2)) { throw new DifferentDirectoryContentException( - "File: " + "File1: " + file1.toString() - + " and file: " + + " and file2: " + file2.toString() + " have different content." + "\n\nfile1 content:\n\n" From f0021d472eedef4f97e13f085c2355e47335d6ff Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 22 Aug 2024 18:03:07 -0700 Subject: [PATCH 042/105] Fix concurrency issue by isolating putBase in a separate transaction The putBase operation is now executed in an isolated transaction to ensure it is retryable without impacting the parent transaction. This prevents concurrency issues that were previously causing conflicts in the parent transaction during import. --- .../service/blobstorage/database/DatabaseBlobStorage.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/database/DatabaseBlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/database/DatabaseBlobStorage.java index 313d397eea..cabda6db83 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/database/DatabaseBlobStorage.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/database/DatabaseBlobStorage.java @@ -11,6 +11,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.domain.PageRequest; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; /** * Implementation that use the database to store the blobs. @@ -60,6 +62,7 @@ public void put(String name, byte[] content, Retention retention) { }); } + @Transactional(propagation = Propagation.REQUIRES_NEW) void putBase(String name, byte[] content, Retention retention) { MBlob mBlob = mBlobRepository From c13d1f7b197bbda784b0a1f99b03e23462fa9f13 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 22 Aug 2024 23:02:32 -0700 Subject: [PATCH 043/105] Support Plural Strings with the Same Name but Different Content Across Branches Warning: this is modifying the text unit searcher to add the "branchId" to the TextUnitDTO, this will basically impact everything. Under normal circumstances, there can only be one active text unit for a given name. However, with multiple branches, it's possible to have multiple text units with the same name. While this isn't an issue for singular strings, it becomes problematic for plural strings, as we group by name when building the AndroidStringDocument. This can lead to too many or duplicated entries for one or more plural forms. To resolve this, we now include the branch ID in the group by logic. This ensures that plural strings are properly grouped, allowing for correct document generation. With the Phrase connector, each group will be clearly identified since the ID of each text unit is included in the key. For other connectors that don't use IDs, results may vary. --- .../input/source2/res/values/strings.xml | 7 + .../mojito/android/strings/AndroidPlural.java | 2 +- .../strings/AndroidStringDocumentMapper.java | 14 +- .../mojito/service/tm/search/TextUnitDTO.java | 9 + .../search/TextUnitDTONativeObjectMapper.java | 3 +- .../service/tm/search/TextUnitSearcher.java | 3 +- .../AndroidStringDocumentMapperTest.java | 179 +++++++++++++++++- 7 files changed, 208 insertions(+), 9 deletions(-) create mode 100644 cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source2/res/values/strings.xml diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source2/res/values/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source2/res/values/strings.xml new file mode 100644 index 0000000000..8be79e5824 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source2/res/values/strings.xml @@ -0,0 +1,7 @@ + + + + %1$,d collaborator in branch + %1$,d collaborators in branch + + \ No newline at end of file diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java index 4fae3d20bf..ca846803cf 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java @@ -46,7 +46,7 @@ private Map buildItemMap( throw new AndroidPluralDuplicateKeyException( "A duplicate was found when building an Android Plural. " + androidPluralItem - + " dupplicates: " + + " duplicates: " + androidPluralItem2); })); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java index f3a5b22142..bdf49ec225 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java @@ -242,7 +242,19 @@ public static String getKeyToGroupByPluralOtherAndComment(TextUnitDTO textUnit) + DEFAULT_ASSET_DELIMITER + textUnit.getPluralFormOther() + "_" - + textUnit.getComment(); + + textUnit.getComment() + + "-" + // Support plural strings with the same name but different content across branches + // + // Under normal circumstances, there can only be one active text unit for a given name. + // However, with multiple branches, it's possible to have multiple text units with the + // same name. While this isn't an issue for singular strings, it becomes problematic for + // plural strings, as we group by name when building the AndroidStringDocument. This can + // lead to too many or duplicated entries for one or more plural forms. + // + // To resolve this, we now include the branch ID in the group by logic. This ensures + // that plural strings are properly grouped, allowing for correct document generation. + + textUnit.getBranchId(); } AndroidSingular textUnitToAndroidSingular(TextUnitDTO textUnit, boolean useSource) { diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java index 025ad75599..0be8393b04 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java @@ -34,6 +34,7 @@ public class TextUnitDTO { private Long assetTextUnitId; private ZonedDateTime tmTextUnitCreatedDate; private boolean doNotTranslate; + private Long branchId; public Long getTmTextUnitId() { return tmTextUnitId; @@ -242,4 +243,12 @@ public boolean isDoNotTranslate() { public void setDoNotTranslate(boolean doNotTranslate) { this.doNotTranslate = doNotTranslate; } + + public Long getBranchId() { + return branchId; + } + + public void setBranchId(Long branchId) { + this.branchId = branchId; + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java index dd2c35bd25..75bcfcfd02 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java @@ -51,11 +51,12 @@ public TextUnitDTO mapObject(CriteriaResult cr) { t.setAssetPath(cr.getString(idx++)); t.setAssetTextUnitId(cr.getLong(idx++)); t.setTmTextUnitCreatedDate(JSR310Migration.newDateTimeCtorWithDate(cr.getDate(idx++))); - t.setDoNotTranslate(Boolean.valueOf(includedInLocalizedFile)); String doNotTranslate = cr.getString(idx++); t.setDoNotTranslate(Boolean.valueOf(doNotTranslate)); + t.setBranchId(cr.getLong(idx++)); + return t; } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java index e5e737debe..515fa38798 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java @@ -219,7 +219,8 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) .addProjection("a.path", "assetPath") .addProjection("atu.id", "assetTextUnitId") .addProjection("tu.created_date", "tmTextUnitCreatedDate") - .addProjection("atu.do_not_translate", "doNotTranslate")); + .addProjection("atu.do_not_translate", "doNotTranslate") + .addProjection("atu.branch_id", "branch_id")); logger.debug("Add search filters"); NativeJunctionExp conjunction = NativeExps.conjunction(); diff --git a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java index 4a8b96ba4b..0148b53931 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java @@ -217,7 +217,7 @@ public void testReadFromSourceTextUnitsWithPluralsAndWithTmTextUnitIdInName() { } @Test - public void testReadFromSourceTextUnitsWithDuplicatePlurals() { + public void testReadFromSourceTextUnitsWithDuplicatePluralsDifferentAsset() { mapper = new AndroidStringDocumentMapper(" _", assetDelimiter); textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); @@ -342,7 +342,7 @@ public void testReadFromSourceTextUnitsWithDuplicatePlurals() { assertThat(singular.getComment()).isEqualTo("comment0"); assertThat(document.getPlurals()).hasSize(2); - AndroidPlural plural = document.getPlurals().get(1); + AndroidPlural plural = document.getPlurals().get(0); assertThat(plural.getName()).isEqualTo("my/path0.xml#@#name1"); assertThat(plural.getComment()).isEqualTo("comment1"); assertThat(plural.getItems()).hasSize(6); @@ -359,7 +359,7 @@ public void testReadFromSourceTextUnitsWithDuplicatePlurals() { assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(105L); assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); - plural = document.getPlurals().get(0); + plural = document.getPlurals().get(1); assertThat(plural.getName()).isEqualTo("my/path1.xml#@#name1"); assertThat(plural.getComment()).isEqualTo("comment1"); assertThat(plural.getItems()).hasSize(6); @@ -377,6 +377,175 @@ public void testReadFromSourceTextUnitsWithDuplicatePlurals() { assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); } + @Test + public void testReadFromSourceTextUnitsWithDuplicatePluralsDifferentBranch() { + mapper = new AndroidStringDocumentMapper(" _", assetDelimiter); + textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); + + // first group + textUnits.add( + sourceTextUnitDTO( + 100L, + "name1 _other", + "content1_zero", + "comment1", + "my/path0.xml", + "zero", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 101L, + "name1 _other", + "content1_one", + "comment1", + "my/path0.xml", + "one", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 102L, + "name1 _other", + "content1_two", + "comment1", + "my/path0.xml", + "two", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 103L, + "name1 _other", + "content1_few", + "comment1", + "my/path0.xml", + "few", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 104L, + "name1 _other", + "content1_many", + "comment1", + "my/path0.xml", + "many", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 105L, + "name1 _other", + "content1_other", + "comment1", + "my/path0.xml", + "other", + "name1_other")); + + // second group + textUnits.add( + sourceTextUnitDTO( + 200L, + "name1 _other", + "content1_zero_b", + "comment1", + "my/path0.xml", + "zero", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 201L, + "name1 _other", + "content1_one_b", + "comment1", + "my/path0.xml", + "one", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 202L, + "name1 _other", + "content1_two_b", + "comment1", + "my/path0.xml", + "two", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 203L, + "name1 _other", + "content1_few_b", + "comment1", + "my/path0.xml", + "few", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 204L, + "name1 _other", + "content1_many_b", + "comment1", + "my/path0.xml", + "many", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 205L, + "name1 _other", + "content1_other_b", + "comment1", + "my/path0.xml", + "other", + "name1_other")); + + // Set different asset extraction id ~= branch to the text units. Keep null on first one + int group2StartIdx = 7; + for (int i = group2StartIdx; i < group2StartIdx + 6; i++) { + textUnits.get(i).setBranchId(123L); + } + + document = mapper.readFromSourceTextUnits(textUnits); + + assertThat(document).isNotNull(); + assertThat(document.getSingulars()).hasSize(1); + AndroidSingular singular = document.getSingulars().get(0); + assertThat(singular.getId()).isEqualTo(123L); + assertThat(singular.getName()).isEqualTo("my/path0#@#name0"); + assertThat(singular.getContent()).isEqualTo("content0"); + assertThat(singular.getComment()).isEqualTo("comment0"); + + assertThat(document.getPlurals()).hasSize(2); + AndroidPlural plural = document.getPlurals().get(0); + assertThat(plural.getName()).isEqualTo("my/path0.xml#@#name1"); + assertThat(plural.getComment()).isEqualTo("comment1"); + assertThat(plural.getItems()).hasSize(6); + assertThat(plural.getItems().get(ZERO).getId()).isEqualTo(100L); + assertThat(plural.getItems().get(ZERO).getContent()).isEqualTo("content1_zero"); + assertThat(plural.getItems().get(ONE).getId()).isEqualTo(101L); + assertThat(plural.getItems().get(ONE).getContent()).isEqualTo("content1_one"); + assertThat(plural.getItems().get(TWO).getId()).isEqualTo(102L); + assertThat(plural.getItems().get(TWO).getContent()).isEqualTo("content1_two"); + assertThat(plural.getItems().get(FEW).getId()).isEqualTo(103L); + assertThat(plural.getItems().get(FEW).getContent()).isEqualTo("content1_few"); + assertThat(plural.getItems().get(MANY).getId()).isEqualTo(104L); + assertThat(plural.getItems().get(MANY).getContent()).isEqualTo("content1_many"); + assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(105L); + assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); + + plural = document.getPlurals().get(1); + assertThat(plural.getName()).isEqualTo("my/path0.xml#@#name1"); + assertThat(plural.getComment()).isEqualTo("comment1"); + assertThat(plural.getItems()).hasSize(6); + assertThat(plural.getItems().get(ZERO).getId()).isEqualTo(200L); + assertThat(plural.getItems().get(ZERO).getContent()).isEqualTo("content1_zero_b"); + assertThat(plural.getItems().get(ONE).getId()).isEqualTo(201L); + assertThat(plural.getItems().get(ONE).getContent()).isEqualTo("content1_one_b"); + assertThat(plural.getItems().get(TWO).getId()).isEqualTo(202L); + assertThat(plural.getItems().get(TWO).getContent()).isEqualTo("content1_two_b"); + assertThat(plural.getItems().get(FEW).getId()).isEqualTo(203L); + assertThat(plural.getItems().get(FEW).getContent()).isEqualTo("content1_few_b"); + assertThat(plural.getItems().get(MANY).getId()).isEqualTo(204L); + assertThat(plural.getItems().get(MANY).getContent()).isEqualTo("content1_many_b"); + assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(205L); + assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other_b"); + } + @Test public void testReadFromTargetTextUnitsForEmptyList() { mapper = new AndroidStringDocumentMapper("", assetDelimiter); @@ -613,7 +782,7 @@ public void testReadFromTargetTextUnitsWithPluralsDuplicated() { assertThat(singular.getComment()).isEqualTo("comment0"); assertThat(document.getPlurals()).hasSize(2); - AndroidPlural plural = document.getPlurals().get(1); + AndroidPlural plural = document.getPlurals().get(0); assertThat(plural.getName()).isEqualTo("my/path0.xml#@#name1"); assertThat(plural.getComment()).isEqualTo("comment1"); assertThat(plural.getItems()).hasSize(6); @@ -630,7 +799,7 @@ public void testReadFromTargetTextUnitsWithPluralsDuplicated() { assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(105L); assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); - plural = document.getPlurals().get(0); + plural = document.getPlurals().get(1); assertThat(plural.getName()).isEqualTo("my/path1.xml#@#name1"); assertThat(plural.getComment()).isEqualTo("comment1"); assertThat(plural.getItems()).hasSize(6); From 9391ecadcb462316796ebf6ad45c346eb65d5a3c Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 23 Aug 2024 08:37:38 -0700 Subject: [PATCH 044/105] Doc to re-indent list of commits --- idea.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/idea.md b/idea.md index 5ba1fec247..f2a715870f 100644 --- a/idea.md +++ b/idea.md @@ -28,6 +28,12 @@ Some useful command This can be configured in Maven UI too. +## Re-indent list of commits + +``` +git rebase -i HEAD~20 --exec 'mvn spotless:apply && git add -A && git commit --amend --no-edit' +``` + ## Debug commands with Maven ### Debug the CLI From fc4387b1189ed8921cdbfa1cf670b47d1305f680 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 23 Aug 2024 14:09:38 -0700 Subject: [PATCH 045/105] Add test for translatable=false in Android plurals string Contrary to simple strings, "plural" strings are not properly skipped --- .../l10n/mojito/service/tm/TMServiceTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java index 8831e9b2de..fd3601e085 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java @@ -1702,6 +1702,52 @@ public void testLocalizeAndroidCommentWithTranslatableFalse() throws Exception { assertEquals(assetContent, localizedAsset); } + @Test + public void testLocalizeAndroidTranslatableFalse() throws Exception { + + Repository repo = repositoryService.createRepository(testIdWatcher.getEntityName("repository")); + RepositoryLocale repoLocale = repositoryService.addRepositoryLocale(repo, "en-GB"); + + String assetContent = + """ + + + do_not_translate + + + plural_do_not_translate_one + plural_do_not_translate_other + + """; + + asset = + assetService.createAssetWithContent(repo.getId(), "res/values/strings.xml", assetContent); + asset = assetRepository.findById(asset.getId()).orElse(null); + assetId = asset.getId(); + tmId = repo.getTm().getId(); + + PollableFuture assetResult = + assetService.addOrUpdateAssetAndProcessIfNeeded( + repo.getId(), asset.getPath(), assetContent, false, null, null, null, null, null, null); + try { + pollableTaskService.waitForPollableTask(assetResult.getPollableTask().getId()); + } catch (PollableTaskException | InterruptedException e) { + throw new RuntimeException(e); + } + assetResult.get(); + + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(repo.getId()); + textUnitSearcherParameters.setStatusFilter(StatusFilter.FOR_TRANSLATION); + List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); + for (TextUnitDTO textUnitDTO : textUnitDTOs) { + logger.info("name=[{}]", textUnitDTO.getName()); + } + assertEquals(2, textUnitDTOs.size()); + // TODO translatable on plural strings don't work + + } + @Test public void testLocalizeAndroidUnicodeEscape() throws Exception { From 92b7a0d304d015658097ea877e2ad89818f722ab Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 23 Aug 2024 14:48:20 -0700 Subject: [PATCH 046/105] Add integrity checker for emails --- .../rest/entity/IntegrityCheckerType.java | 3 +- .../EmailIntegrityChecker.java | 20 ++++++++++ .../EmailIntegrityCheckerException.java | 7 ++++ .../IntegrityCheckerType.java | 3 +- .../EmailIntegrityCheckerTest.java | 38 +++++++++++++++++++ 5 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityChecker.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerException.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerTest.java diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java index 85aef06aeb..866d0f39cc 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java @@ -21,5 +21,6 @@ public enum IntegrityCheckerType { BACKQUOTE, EMPTY_TARGET_NOT_EMPTY_SOURCE, MARKDOWN_LINKS, - PYTHON_FPRINT; + PYTHON_FPRINT, + EMAIL; } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityChecker.java new file mode 100644 index 0000000000..81467203cb --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityChecker.java @@ -0,0 +1,20 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class EmailIntegrityChecker extends RegexIntegrityChecker { + + static final String EMAIL_REGEX = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; + + @Override + public String getRegex() { + return EMAIL_REGEX; + } + + @Override + public void check(String content, String target) { + try { + super.check(content, target); + } catch (RegexCheckerException ex) { + throw new EmailIntegrityCheckerException("Emails are changed."); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerException.java new file mode 100644 index 0000000000..07b78bcd2b --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class EmailIntegrityCheckerException extends IntegrityCheckException { + public EmailIntegrityCheckerException(String message) { + super(message); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java index df8e85f24d..2af8281d56 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java @@ -23,7 +23,8 @@ public enum IntegrityCheckerType { BACKQUOTE(BackquoteIntegrityChecker.class.getName()), EMPTY_TARGET_NOT_EMPTY_SOURCE(EmptyTargetNotEmptySourceIntegrityChecker.class.getName()), MARKDOWN_LINKS(MarkdownLinkIntegrityChecker.class.getName()), - PYTHON_FPRINT(PythonFStringIntegrityChecker.class.getName()); + PYTHON_FPRINT(PythonFStringIntegrityChecker.class.getName()), + EMAIL(EmailIntegrityChecker.class.getName()); String className; diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerTest.java new file mode 100644 index 0000000000..c924428e7f --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerTest.java @@ -0,0 +1,38 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import org.junit.Ignore; +import org.junit.Test; + +public class EmailIntegrityCheckerTest { + + EmailIntegrityChecker checker = new EmailIntegrityChecker(); + + @Test + public void testNoEmail() { + String source = "There is no email"; + String target = "Il n'y a pas d'email"; + checker.check(source, target); + } + + @Test(expected = EmailIntegrityCheckerException.class) + public void testMissingInTarget() { + String source = "There is an ja@test.com"; + String target = "Il n'y a pas d'email"; + checker.check(source, target); + } + + @Test(expected = EmailIntegrityCheckerException.class) + public void testAddedInTarget() { + String source = "There is no email"; + String target = "Il n'y a un email ja@test.com"; + checker.check(source, target); + } + + @Ignore // not supported yet + @Test(expected = EmailIntegrityCheckerException.class) + public void testDuplicates() { + String source = "There is an ja@test.com"; + String target = "Il n'y a un email ja@test.com ja@test.com"; + checker.check(source, target); + } +} From e61b5c92201bc18a2d4cbdbb779e99f5282b890a Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 26 Aug 2024 18:10:57 -0700 Subject: [PATCH 047/105] Support unmatched double curly braces in CompositeFormatIntegrityChecker This will now complain if source has "{{ }}" and target "{{ }" --- .../CompositeFormatIntegrityChecker.java | 2 +- .../CompositeFormatIntegrityCheckerTest.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java index b8b5486d23..a69cf40eda 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java @@ -9,7 +9,7 @@ public class CompositeFormatIntegrityChecker extends RegexIntegrityChecker { @Override public String getRegex() { - return "\\{.*?\\}"; + return "\\{\\{[^{}]*?\\}\\}|\\{[^{}]*?\\}"; } @Override diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java index 6c8c97ea4a..19a1ad12e5 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java @@ -62,6 +62,16 @@ public void testMustache() { checker.check(source, target); } + @Test(expected = CompositeFormatIntegrityCheckerException.class) + public void testMustacheUnmatchedBrace() { + CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); + + String source = "A {{ mustache }} template."; + String target = "Un modėle {{ mustache }."; + + checker.check(source, target); + } + @Test(expected = CompositeFormatIntegrityCheckerException.class) public void testMustacheInvalidMissingCurlyBraces() { CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); From 180fb4d593caa9c8dc89710577093d161bc7a09c Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 26 Aug 2024 18:46:10 -0700 Subject: [PATCH 048/105] Add URLIntegrityChecker for standalone URLs Basic URL matcher for plain text, or for simple format. --- .../rest/entity/IntegrityCheckerType.java | 3 +- .../IntegrityCheckerType.java | 3 +- .../integritychecker/URLIntegrityChecker.java | 29 ++++++ .../URLIntegrityCheckerException.java | 11 +++ .../URLIntegrityCheckerTest.java | 92 +++++++++++++++++++ 5 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java index 866d0f39cc..73b4d8b7eb 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java @@ -22,5 +22,6 @@ public enum IntegrityCheckerType { EMPTY_TARGET_NOT_EMPTY_SOURCE, MARKDOWN_LINKS, PYTHON_FPRINT, - EMAIL; + EMAIL, + URL; } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java index 2af8281d56..3e959413a7 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckerType.java @@ -24,7 +24,8 @@ public enum IntegrityCheckerType { EMPTY_TARGET_NOT_EMPTY_SOURCE(EmptyTargetNotEmptySourceIntegrityChecker.class.getName()), MARKDOWN_LINKS(MarkdownLinkIntegrityChecker.class.getName()), PYTHON_FPRINT(PythonFStringIntegrityChecker.class.getName()), - EMAIL(EmailIntegrityChecker.class.getName()); + EMAIL(EmailIntegrityChecker.class.getName()), + URL(URLIntegrityChecker.class.getName()); String className; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java new file mode 100644 index 0000000000..6d65594a06 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java @@ -0,0 +1,29 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +/** + * Checks for URLs in plain text. + * + *

Regex is simple and might under match but that's preferable too triggering false positive + * + *

Consider using another checker for format with links like Markdown: + * MarkdownLinkIntegrityChecker + * + * @author jaurambault + */ +public class URLIntegrityChecker extends RegexIntegrityChecker { + + @Override + public String getRegex() { + return "(https?|ftp)://(?:www\\.)?[a-zA-Z0-9-]+\\.[a-zA-Z]{2,}(?:/[a-zA-Z0-9/_\\-?#+%]*)*|mailto:[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; + } + + @Override + public void check(String sourceContent, String targetContent) + throws CompositeFormatIntegrityCheckerException { + try { + super.check(sourceContent, targetContent); + } catch (RegexCheckerException rce) { + throw new URLIntegrityCheckerException(rce); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java new file mode 100644 index 0000000000..d34c6e0c91 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java @@ -0,0 +1,11 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +/** + * @author jaurambault + */ +public class URLIntegrityCheckerException extends RegexCheckerException { + + public URLIntegrityCheckerException(RegexCheckerException rce) { + super(rce.getMessage()); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java new file mode 100644 index 0000000000..34e1e0e6d2 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java @@ -0,0 +1,92 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import org.junit.Test; + +public class URLIntegrityCheckerTest { + + URLIntegrityChecker checker = new URLIntegrityChecker(); + + @Test + public void noURL() { + String source = "No URL"; + String target = "Pas d'URL"; + checker.check(source, target); + } + + @Test + public void validUrl() { + String source = "Visit our site at https://example.org"; + String target = "Visitez notre site à https://example.org"; + checker.check(source, target); + } + + @Test + public void validUrlCJK() { + String source = "Please visit our website at http://example.org for more information."; + String target = "詳細については、こちらのウェブサイトhttp://example.orgをご覧ください。"; + checker.check(source, target); + } + + @Test + public void invalidUrlCJK() { + String source = "Please visit our website at http://example.org for more information."; + String target = "詳細については、こちらのウェブサイトhttp://example.org-をご覧ください。"; + checker.check(source, target); + } + + @Test + public void validUrlDifferentQuotes() { + String source = "Visit our site at \"https://example.org\""; + String target = "Visitez notre site à 'https://example.org'"; + checker.check(source, target); + } + + @Test + public void validUrlParam() { + String source = "Visit our site at \"https://example.org?test=value1\""; + String target = "Visitez notre site à 'https://example.org?test=value1'"; + checker.check(source, target); + } + + @Test(expected = URLIntegrityCheckerException.class) + public void missingUrl() { + String source = "Visit our site at https://example.org"; + String target = "Visitez notre site à"; + checker.check(source, target); + } + + @Test(expected = URLIntegrityCheckerException.class) + public void changedUrl() { + String source = "The website is https://example.org"; + String target = "Le site web est https://bad.org"; + checker.check(source, target); + } + + @Test + public void validHttpUrl() { + String source = "Visit our site at http://example.org"; + String target = "Visitez notre site à http://example.org"; + checker.check(source, target); + } + + @Test + public void validEmail() { + String source = "Send message to mailto:test@test.org"; + String target = "Envoyer un message a mailto:test@test.org"; + checker.check(source, target); + } + + @Test(expected = URLIntegrityCheckerException.class) + public void missingEmail() { + String source = "Send message to mailto:test@test.org"; + String target = "Envoyer un message a"; + checker.check(source, target); + } + + @Test(expected = URLIntegrityCheckerException.class) + public void chanagedEmail() { + String source = "There is no mailto:test@test.org"; + String target = "Il n'y a pas d'email mailto:tedst@test.org"; + checker.check(source, target); + } +} From f1d48acc57a3de687a5e5af351deb817e478642e Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 26 Aug 2024 22:28:53 -0700 Subject: [PATCH 049/105] Introduce generic OptionsParser --- .../mojito/okapi/filters/FilterOptions.java | 36 ++----------- .../box/l10n/mojito/utils/OptionsParser.java | 51 +++++++++++++++++++ .../OptionsParserTest.java} | 26 +++++----- .../thirdparty/ThirdPartyTMSPhrase.java | 31 +++++++++-- 4 files changed, 93 insertions(+), 51 deletions(-) create mode 100644 common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java rename common/src/test/java/com/box/l10n/mojito/{okapi/filters/FilterOptionsTest.java => utils/OptionsParserTest.java} (67%) diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java index 21e40dc2ea..8451451281 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java @@ -1,42 +1,12 @@ package com.box.l10n.mojito.okapi.filters; -import com.google.common.base.Splitter; -import java.util.LinkedHashMap; +import com.box.l10n.mojito.utils.OptionsParser; import java.util.List; -import java.util.Map; -import java.util.function.Consumer; import net.sf.okapi.common.annotation.IAnnotation; -public class FilterOptions implements IAnnotation { - - Map options = new LinkedHashMap<>(); +public class FilterOptions extends OptionsParser implements IAnnotation { public FilterOptions(List options) { - if (options != null) { - for (String option : options) { - List optionKeyAndValue = Splitter.on("=").limit(2).splitToList(option); - if (optionKeyAndValue.size() == 2) { - this.options.put(optionKeyAndValue.get(0), optionKeyAndValue.get(1)); - } - } - } - } - - public void getString(String key, Consumer consumer) { - if (this.options.containsKey(key)) { - consumer.accept(this.options.get(key)); - } - } - - public void getBoolean(String key, Consumer consumer) { - if (this.options.containsKey(key)) { - consumer.accept(Boolean.valueOf(this.options.get(key))); - } - } - - public void getInteger(String key, Consumer consumer) { - if (this.options.containsKey(key)) { - consumer.accept(Integer.valueOf(this.options.get(key))); - } + super(options); } } diff --git a/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java b/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java new file mode 100644 index 0000000000..9128d4bda7 --- /dev/null +++ b/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java @@ -0,0 +1,51 @@ +package com.box.l10n.mojito.utils; + +import com.google.common.base.Splitter; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +public class OptionsParser { + + Map options = new LinkedHashMap<>(); + + public OptionsParser(List options) { + if (options != null) { + for (String option : options) { + List optionKeyAndValue = Splitter.on("=").limit(2).splitToList(option); + if (optionKeyAndValue.size() == 2) { + this.options.put(optionKeyAndValue.get(0), optionKeyAndValue.get(1)); + } + } + } + } + + public void getString(String key, Consumer consumer) { + if (this.options.containsKey(key)) { + consumer.accept(this.options.get(key)); + } + } + + public void getBoolean(String key, Consumer consumer) { + if (this.options.containsKey(key)) { + consumer.accept(Boolean.valueOf(this.options.get(key))); + } + } + + public Boolean getBoolean(String key) { + Boolean value = null; + + if (this.options.containsKey(key)) { + value = Boolean.valueOf(this.options.get(key)); + } + + return value; + } + + public void getInteger(String key, Consumer consumer) { + if (this.options.containsKey(key)) { + consumer.accept(Integer.valueOf(this.options.get(key))); + } + } +} diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/FilterOptionsTest.java b/common/src/test/java/com/box/l10n/mojito/utils/OptionsParserTest.java similarity index 67% rename from common/src/test/java/com/box/l10n/mojito/okapi/filters/FilterOptionsTest.java rename to common/src/test/java/com/box/l10n/mojito/utils/OptionsParserTest.java index 31d293b6e7..cd9452c7b5 100644 --- a/common/src/test/java/com/box/l10n/mojito/okapi/filters/FilterOptionsTest.java +++ b/common/src/test/java/com/box/l10n/mojito/utils/OptionsParserTest.java @@ -1,4 +1,4 @@ -package com.box.l10n.mojito.okapi.filters; +package com.box.l10n.mojito.utils; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -8,26 +8,26 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Test; -public class FilterOptionsTest { +public class OptionsParserTest { @Test public void testNullGivesEmptyMap() { - FilterOptions filterOptions = new FilterOptions(null); - assertTrue(filterOptions.options.isEmpty()); + OptionsParser optionsParser = new OptionsParser(null); + assertTrue(optionsParser.options.isEmpty()); } @Test public void testSplit() { - FilterOptions filterOptions = - new FilterOptions(Arrays.asList("option1=value1", "option2=value2")); + OptionsParser optionsParser = + new OptionsParser(Arrays.asList("option1=value1", "option2=value2")); AtomicBoolean called = new AtomicBoolean(false); - filterOptions.getString( + optionsParser.getString( "option1", v -> { called.set(true); assertEquals("value1", v); }); - filterOptions.getString( + optionsParser.getString( "option2", v -> { called.set(true); @@ -38,16 +38,16 @@ public void testSplit() { @Test public void testGetBoolean() { - FilterOptions filterOptions = new FilterOptions(Arrays.asList("option1=true", "option2=false")); + OptionsParser optionsParser = new OptionsParser(Arrays.asList("option1=true", "option2=false")); AtomicBoolean called = new AtomicBoolean(false); - filterOptions.getBoolean( + optionsParser.getBoolean( "option1", v -> { called.set(true); assertTrue(v); }); - filterOptions.getBoolean( + optionsParser.getBoolean( "option2", v -> { called.set(true); @@ -60,8 +60,8 @@ public void testGetBoolean() { @Test public void testSetObjectVariable() { - FilterOptions filterOptions = new FilterOptions(Arrays.asList("option1=someobjectvalue")); - filterOptions.getString("option1", v -> forTestSetObjectVariable = v); + OptionsParser optionsParser = new OptionsParser(Arrays.asList("option1=someobjectvalue")); + optionsParser.getString("option1", v -> forTestSetObjectVariable = v); assertEquals("someobjectvalue", forTestSetObjectVariable); } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 7fbbbbb639..b5223bb6af 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -19,6 +19,7 @@ import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; import com.box.l10n.mojito.service.tm.search.UsedFilter; +import com.box.l10n.mojito.utils.OptionsParser; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.phrase.client.model.Tag; @@ -292,12 +293,23 @@ public PollableFuture pull( String currentTags = getCurrentTagsForRepository(repository, projectId); + OptionsParser optionsParser = new OptionsParser(optionList); + Boolean integrityCheckKeepStatusIfFailedAndSameTarget = + optionsParser.getBoolean("integrityCheckKeepStatusIfFailedAndSameTarget"); + // may already hit rate limit, according it is 4 qps ... there is a retry in the locale client // though. // but 4 qps is very low to download 64 locales. repositoryLocalesWithoutRootLocale.parallelStream() .forEach( - locale -> pullLocaleTimed(repository, projectId, locale, pluralSeparator, currentTags)); + locale -> + pullLocaleTimed( + repository, + projectId, + locale, + pluralSeparator, + currentTags, + integrityCheckKeepStatusIfFailedAndSameTarget)); return null; } @@ -307,12 +319,19 @@ private void pullLocaleTimed( String projectId, RepositoryLocale repositoryLocale, String pluralSeparator, - String currentTags) { + String currentTags, + boolean integrityCheckKeepStatusIfFailedAndSameTarget) { try (var timer = Timer.resource(meterRegistry, "ThirdPartyTMSPhrase.pullLocale") .tag("repository", repository.getName())) { - pullLocale(repository, projectId, repositoryLocale, pluralSeparator, currentTags); + pullLocale( + repository, + projectId, + repositoryLocale, + pluralSeparator, + currentTags, + integrityCheckKeepStatusIfFailedAndSameTarget); } } @@ -321,7 +340,8 @@ private void pullLocale( String projectId, RepositoryLocale repositoryLocale, String pluralSeparator, - String currentTags) { + String currentTags, + boolean integrityCheckKeepStatusIfFailedAndSameTarget) { String localeTag = repositoryLocale.getLocale().getBcp47Tag(); logger.info("Downloading locale: {} from Phrase with tags: {}", localeTag, currentTags); @@ -352,7 +372,8 @@ private void pullLocale( textUnitDTOS.forEach(t -> t.setComment(null)); Stopwatch importStopWatch = Stopwatch.createStarted(); - textUnitBatchImporterService.importTextUnits(textUnitDTOS, false, true); + textUnitBatchImporterService.importTextUnits( + textUnitDTOS, false, integrityCheckKeepStatusIfFailedAndSameTarget); logger.info("Time importing text units: {}", importStopWatch.elapsed()); } From 11287330576a0bcb549cce5d8300b670104f189e Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 27 Aug 2024 09:11:57 -0700 Subject: [PATCH 050/105] Add option to override translation status during import Currently, if the target has not changed, we retain the status to ensure that a false positive "rejected" string can be marked as "accepted." However, this approach has a drawback: if a new integrity check is added that would cause a failure, the strings remain marked as "accepted" rather than being updated to "rejected." We are adding this option to allow for a complete re-application of the checks. The downside is that any false positives will need to be re-marked as "accepted." --- .../main/java/com/box/l10n/mojito/utils/OptionsParser.java | 4 ++-- .../l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java b/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java index 9128d4bda7..662a23a7c0 100644 --- a/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java +++ b/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java @@ -33,8 +33,8 @@ public void getBoolean(String key, Consumer consumer) { } } - public Boolean getBoolean(String key) { - Boolean value = null; + public Boolean getBoolean(String key, Boolean defaultValue) { + Boolean value = defaultValue; if (this.options.containsKey(key)) { value = Boolean.valueOf(this.options.get(key)); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index b5223bb6af..9379d2befa 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -295,7 +295,7 @@ public PollableFuture pull( OptionsParser optionsParser = new OptionsParser(optionList); Boolean integrityCheckKeepStatusIfFailedAndSameTarget = - optionsParser.getBoolean("integrityCheckKeepStatusIfFailedAndSameTarget"); + optionsParser.getBoolean("integrityCheckKeepStatusIfFailedAndSameTarget", true); // may already hit rate limit, according it is 4 qps ... there is a retry in the locale client // though. From de2ad61d16647d9b714b262fda33711d8b6e2106 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 28 Aug 2024 07:55:26 -0700 Subject: [PATCH 051/105] Add option to remove existing third party mapping when synchronizing In some Translation Management Systems (TMS), string IDs may need to be remapped. This is particularly necessary when strings with the same "key" are removed and then recreated, often due to incorrect handling. Different TMS systems handle this differently: some use soft deletion, while others create a completely new entry with a new ID. In these cases, a new mapping is required. --- .../service/thirdparty/ThirdPartyService.java | 20 +++++++++++++++++-- .../ThirdPartyTextUnitRepository.java | 5 +++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java index 07ba5f3f06..6defd73973 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java @@ -24,6 +24,7 @@ import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; import com.box.l10n.mojito.service.tm.textunitdtocache.TextUnitDTOsCacheService; import com.box.l10n.mojito.service.tm.textunitdtocache.UpdateType; +import com.box.l10n.mojito.utils.OptionsParser; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -268,6 +269,9 @@ void mapMojitoAndThirdPartyTextUnits( repository.getName(), projectId); + OptionsParser optionsParser = new OptionsParser(options); + Boolean deleteCurrentMapping = optionsParser.getBoolean("deleteCurrentMapping", false); + logger.debug("Get the text units of the third party TMS"); List thirdPartyTextUnits = thirdPartyTMS.getThirdPartyTextUnits(repository, projectId, options); @@ -294,15 +298,27 @@ void mapMojitoAndThirdPartyTextUnits( thirdPartyTextUnitsByAsset.entrySet().stream() .filter(e -> e.getKey() != null) .forEach( - e -> mapThirdPartyTextUnitsToTextUnitDTOs(e.getKey(), e.getValue(), pluralSeparator)); + e -> + mapThirdPartyTextUnitsToTextUnitDTOs( + e.getKey(), e.getValue(), pluralSeparator, deleteCurrentMapping)); } void mapThirdPartyTextUnitsToTextUnitDTOs( - Asset asset, List thirdPartyTextUnitsToMap, String pluralSeparator) { + Asset asset, + List thirdPartyTextUnitsToMap, + String pluralSeparator, + boolean deleteCurrentMapping) { logger.debug("Map third party text units to text unit DTOs for asset: {}", asset.getId()); Set alreadyMappedTmTextUnitId = thirdPartyTextUnitRepository.findTmTextUnitIdsByAsset(asset); + if (deleteCurrentMapping) { + logger.info("Delete existing ThirdPartyTextUnit mapping for asset id: {}", asset.getId()); + int deletedCount = thirdPartyTextUnitRepository.deleteByAssetId(asset.getId()); + logger.info( + "Deleted {} ThirdPartyTextUnit mappings for asset id: {}", deletedCount, asset.getId()); + } + boolean allWithTmTextUnitId = thirdPartyTextUnitsToMap.stream() .map(ThirdPartyTextUnit::getTmTextUnitId) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnitRepository.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnitRepository.java index 7bad9752ec..68e798a00c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnitRepository.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnitRepository.java @@ -12,7 +12,9 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.data.rest.core.annotation.RepositoryRestResource; +import org.springframework.transaction.annotation.Transactional; /** * @author jeanaurambault @@ -32,4 +34,7 @@ public interface ThirdPartyTextUnitRepository @Override @EntityGraph(value = "ThirdPartyTextUnit.legacy", type = EntityGraphType.FETCH) List findAll(); + + @Transactional + int deleteByAssetId(@Param("assetId") long assetId); } From 3c6af3e89cc05a49b98c8fea0b2addd9e37296f9 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 28 Aug 2024 11:43:28 -0700 Subject: [PATCH 052/105] Make integrity check messages more specific --- .../BackquoteIntegrityChecker.java | 3 ++- .../CompositeFormatIntegrityChecker.java | 3 ++- ...mpositeFormatIntegrityCheckerException.java | 6 +++--- .../MarkdownLinkIntegrityChecker.java | 2 +- .../PrintfLikeIntegrityChecker.java | 3 ++- ...PrintfLikeVariableTypeIntegrityChecker.java | 3 ++- .../PythonFStringIntegrityChecker.java | 3 ++- .../SimplePrintfLikeIntegrityChecker.java | 3 ++- .../integritychecker/URLIntegrityChecker.java | 2 +- .../URLIntegrityCheckerException.java | 6 +++--- ...ntLikeVariableTypeIntegrityCheckerTest.java | 18 ++++++++++++------ ...ParameterSpecifierIntegrityCheckerTest.java | 14 +++++++------- .../PrintfLikeIntegrityCheckerTest.java | 14 +++++++------- .../SimplePrintfLikeIntegrityCheckerTest.java | 6 ++++-- 14 files changed, 50 insertions(+), 36 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/BackquoteIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/BackquoteIntegrityChecker.java index bccdd4dbbb..2cf281350a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/BackquoteIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/BackquoteIntegrityChecker.java @@ -25,7 +25,8 @@ public void check(String sourceContent, String targetContent) try { super.check(sourceContent, targetContent); } catch (RegexCheckerException rce) { - throw new BackquoteIntegrityCheckerException((rce.getMessage())); + throw new BackquoteIntegrityCheckerException( + "Backquoted stings are different in source and target"); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java index a69cf40eda..426be785e0 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java @@ -18,7 +18,8 @@ public void check(String sourceContent, String targetContent) try { super.check(sourceContent, targetContent); } catch (RegexCheckerException rce) { - throw new CompositeFormatIntegrityCheckerException(rce); + throw new CompositeFormatIntegrityCheckerException( + "Composite Format placeholders in source and target are different"); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerException.java index 0349800d67..a2e6b4abe1 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerException.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerException.java @@ -3,9 +3,9 @@ /** * @author jaurambault */ -public class CompositeFormatIntegrityCheckerException extends RegexCheckerException { +public class CompositeFormatIntegrityCheckerException extends IntegrityCheckException { - public CompositeFormatIntegrityCheckerException(RegexCheckerException rce) { - super(rce.getMessage()); + public CompositeFormatIntegrityCheckerException(String message) { + super(message); } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java index becdd96959..980dcc9f72 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java @@ -29,7 +29,7 @@ public void check(String content, String target) { try { super.check(content, target); } catch (RegexCheckerException ex) { - throw new MarkdownLinkIntegrityCheckerException("Variable types do not match."); + throw new MarkdownLinkIntegrityCheckerException("Markdown Links do not match."); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityChecker.java index 0031cc4ec2..9270cae383 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityChecker.java @@ -29,7 +29,8 @@ public void check(String sourceContent, String targetContent) try { super.check(sourceContent, targetContent); } catch (RegexCheckerException rce) { - throw new PrintfLikeIntegrityCheckerException((rce.getMessage())); + throw new PrintfLikeIntegrityCheckerException( + "PrintfLike placeholders are different in source and target"); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeVariableTypeIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeVariableTypeIntegrityChecker.java index afe8dbc687..5fd2ebcca6 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeVariableTypeIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeVariableTypeIntegrityChecker.java @@ -22,7 +22,8 @@ public void check(String content, String target) { try { super.check(content, target); } catch (RegexCheckerException ex) { - throw new PrintfLikeVariableTypeIntegrityCheckerException("Variable types do not match."); + throw new PrintfLikeVariableTypeIntegrityCheckerException( + "PrintfLikeVariableType placeholder are different in source and target."); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java index b4dea6bd3c..bbc216d483 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java @@ -12,7 +12,8 @@ public void check(String content, String target) { try { super.check(content, target); } catch (RegexCheckerException ex) { - throw new PythonFStringIntegrityCheckerException("Variable types do not match."); + throw new PythonFStringIntegrityCheckerException( + "PythonFString placeholders are different in source and target."); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityChecker.java index df0f36e243..c6d19c264c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityChecker.java @@ -22,7 +22,8 @@ public void check(String sourceContent, String targetContent) try { super.check(sourceContent, targetContent); } catch (RegexCheckerException rce) { - throw new SimplePrintfLikeIntegrityCheckerException((rce.getMessage())); + throw new SimplePrintfLikeIntegrityCheckerException( + "SimplePrintfLike placeholders are different in source and target."); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java index 6d65594a06..e321906f7a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java @@ -23,7 +23,7 @@ public void check(String sourceContent, String targetContent) try { super.check(sourceContent, targetContent); } catch (RegexCheckerException rce) { - throw new URLIntegrityCheckerException(rce); + throw new URLIntegrityCheckerException("URLs in source and target are different"); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java index d34c6e0c91..aa1a13948e 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java @@ -3,9 +3,9 @@ /** * @author jaurambault */ -public class URLIntegrityCheckerException extends RegexCheckerException { +public class URLIntegrityCheckerException extends IntegrityCheckException { - public URLIntegrityCheckerException(RegexCheckerException rce) { - super(rce.getMessage()); + public URLIntegrityCheckerException(String message) { + super(message); } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintLikeVariableTypeIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintLikeVariableTypeIntegrityCheckerTest.java index 2f47fe968d..33a8f69fac 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintLikeVariableTypeIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintLikeVariableTypeIntegrityCheckerTest.java @@ -27,7 +27,8 @@ public void testMissingVariableTypeCausesIntegrityViolation() checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } } @@ -42,7 +43,8 @@ public void testModifiedVariableTypeCausesIntegrityViolation() checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } } @@ -56,7 +58,8 @@ public void testVariableWithFormattingFlagChecked() { checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } source = "%(count) s view"; @@ -66,7 +69,8 @@ public void testVariableWithFormattingFlagChecked() { checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } source = "%(count).1f view"; @@ -76,7 +80,8 @@ public void testVariableWithFormattingFlagChecked() { checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } } @@ -90,7 +95,8 @@ public void testCurlyBracketsAreChecked() { checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeAddParameterSpecifierIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeAddParameterSpecifierIntegrityCheckerTest.java index 82ebce5fad..1417f0cfaa 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeAddParameterSpecifierIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeAddParameterSpecifierIntegrityCheckerTest.java @@ -133,7 +133,7 @@ public void testMacPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -150,7 +150,7 @@ public void testMacPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeatedOr checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -167,7 +167,7 @@ public void testMacPlaceholderCheckFailsIfSamePlaceholdersCountButSpecifierModif checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -243,7 +243,7 @@ public void testAndroidPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -260,7 +260,7 @@ public void testAndroidPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeat checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -277,7 +277,7 @@ public void testAndroidPlaceholderCheckFailsIfSamePlaceholdersCountButSpecifierM checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -304,7 +304,7 @@ public void testIncorrectlyModifiedSpecifier() throws PrintfLikeIntegrityChecker checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityCheckerTest.java index 8bba1b6b16..48ab2a28e3 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityCheckerTest.java @@ -123,7 +123,7 @@ public void testMacPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -139,7 +139,7 @@ public void testMacPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeatedOr checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -155,7 +155,7 @@ public void testMacPlaceholderCheckFailsIfSamePlaceholdersCountButSpecifierModif checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -225,7 +225,7 @@ public void testAndroidPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -241,7 +241,7 @@ public void testAndroidPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeat checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -257,7 +257,7 @@ public void testAndroidPlaceholderCheckFailsIfSamePlaceholdersCountButSpecifierM checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -282,7 +282,7 @@ public void testIncorrectlyModifiedSpecifier() throws PrintfLikeIntegrityChecker checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityCheckerTest.java index ab514561ef..11cb8ed38d 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityCheckerTest.java @@ -67,7 +67,8 @@ public void testPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("SimplePrintfLikeIntegrityCheckerException must be thrown"); } catch (SimplePrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals( + e.getMessage(), "SimplePrintfLike placeholders are different in source and target."); } } @@ -82,7 +83,8 @@ public void testPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeatedOrMis checker.check(source, target); fail("SimplePrintfLikeIntegrityCheckerException must be thrown"); } catch (SimplePrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals( + e.getMessage(), "SimplePrintfLike placeholders are different in source and target."); } } From b1e14959342f4511b927759da84474e962a11ea3 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 29 Aug 2024 15:57:38 -0700 Subject: [PATCH 053/105] Add "standalone" attribute to output document based on input This update modifies the Android filter to apply the "standalone" attribute in the output XML document according to the presence of the "standalone" attribute in the input document. If the input document specifies "standalone," the output will also include this attribute with the same value. Otherwise, the output document will omit the "standalone" attribute. --- .../com/box/l10n/mojito/okapi/filters/AndroidFilter.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java index e784403620..b48ae4d12a 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java @@ -500,9 +500,16 @@ String execute(String xmlContent) { "{http://xml.apache.org/xslt}indent-amount", String.valueOf(indent)); boolean hasDeclaration = xmlContent.trim().startsWith(" Date: Fri, 30 Aug 2024 20:05:42 -0700 Subject: [PATCH 054/105] Add support for PKCS#1 and PKCS#8 private keys in GithubClient Enhanced GithubClient to accept both PKCS#1 and PKCS#8 private key formats. The update also allows keys to be provided with or without PEM headers and footers. --- .../box/l10n/mojito/github/GithubClient.java | 68 ++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java index de087a6e47..023c11f7fb 100644 --- a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java +++ b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java @@ -69,12 +69,78 @@ public GithubClient(String appId, String key, String owner) { private PrivateKey createPrivateKey(String key) throws NoSuchAlgorithmException, InvalidKeySpecException { - byte[] encodedKey = Base64.decodeBase64(key); + byte[] encodedKey = keyToBytes(key); PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(encodedKey); KeyFactory keyFactory = KeyFactory.getInstance("RSA"); return keyFactory.generatePrivate(spec); } + /** + * Converts a PEM-formatted private key string to a byte array. + * + *

This method handles both PKCS#1 and PKCS#8 formatted private keys. It strips the PEM headers + * and footers, decodes the base64 content, and, if necessary, converts a PKCS#1 key to a PKCS#8 + * format. + */ + private static byte[] keyToBytes(String formattedKey) { + boolean pkcs1Header = formattedKey.startsWith("-----BEGIN RSA PRIVATE KEY-----"); + boolean pkcs8Header = formattedKey.startsWith("-----BEGIN PRIVATE KEY-----"); + + if (pkcs1Header || pkcs8Header) { + formattedKey = + formattedKey + .replaceAll("-----BEGIN.*?-----", "") + .replaceAll("-----END.*?-----", "") + .replaceAll("\\s+", ""); + } + + byte[] encodedKey = Base64.decodeBase64(formattedKey); + + if (pkcs1Header) { + encodedKey = convertPkcs1ToPkcs8(encodedKey); + } + + return encodedKey; + } + + private static byte[] convertPkcs1ToPkcs8(byte[] pkcs1Bytes) { + // PKCS#8 header for RSA encryption + byte[] pkcs8Header = { + 0x30, + (byte) 0x82, // SEQUENCE + LENGTH + (byte) ((pkcs1Bytes.length + 22) >> 8), + (byte) ((pkcs1Bytes.length + 22) & 0xff), // PKCS#8 LENGTH + 0x02, + 0x01, + 0x00, // INTEGER (0) + 0x30, + 0x0d, // SEQUENCE + 0x06, + 0x09, // OID + 0x2a, + (byte) 0x86, + 0x48, + (byte) 0x86, + (byte) 0xf7, + 0x0d, + 0x01, + 0x01, + 0x01, // rsaEncryption OID + 0x05, + 0x00, // NULL + 0x04, + (byte) 0x82, // OCTET STRING + LENGTH + (byte) (pkcs1Bytes.length >> 8), + (byte) (pkcs1Bytes.length & 0xff) // PKCS#1 LENGTH + }; + + byte[] pkcs8Bytes = new byte[pkcs8Header.length + pkcs1Bytes.length]; + System.arraycopy(pkcs8Header, 0, pkcs8Bytes, 0, pkcs8Header.length); + System.arraycopy(pkcs1Bytes, 0, pkcs8Bytes, pkcs8Header.length, pkcs1Bytes.length); + + return pkcs8Bytes; + } + public void addCommentToPR(String repository, int prNumber, String comment) { String repoFullPath = getRepositoryPath(repository); try { From 4142480c280c9b21eff68fcb27241524eca16bb2 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 30 Aug 2024 20:22:49 -0700 Subject: [PATCH 055/105] Add post-processing to AndroidFilter to remove non-translatable elements Introduced a new post-processing option, postRemoveTranslatableFalse, in AndroidFilter. This option removes elements with the attribute translatable="false", ensuring only translatable elements are written to the output. Note: This does not prevent these elements from being pushed to Mojito, this is a post-processing step. Due to an existing bug in the filter, singular elements are properly excluded from translation, but plural ones are not (to fix later). But excluded elements from translation in Mojito are still outputted, while this new option ensures they are removed from the final output. Typically used with pull -fo postRemoveTranslatableFalse=true. --- .../mojito/okapi/filters/AndroidFilter.java | 41 ++++++++++++++-- .../okapi/filters/AndroidFilterTest.java | 47 +++++++++++++++++-- 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java index b48ae4d12a..656b059381 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java @@ -59,6 +59,9 @@ public class AndroidFilter extends XMLFilter { private static final String POST_PROCESS_INDENT = "postProcessIndent"; + private static final String POST_PROCESS_REMOVE_TRANSLATABLE_FALSE = + "postRemoveTranslatableFalse"; + private static final Pattern PATTERN_PLURAL_START = Pattern.compile(""); private static final Pattern PATTERN_XML_COMMENT = Pattern.compile(""); @@ -110,6 +113,8 @@ public List getConfigurations() { boolean removeDescription = false; + boolean removeTranslatableFalse = false; + int postProcessIndent = 2; @Override @@ -128,7 +133,9 @@ public void open(RawDocument input) { // in the TranslateStep input.setAnnotation( new OutputDocumentPostProcessingAnnotation( - new AndroidFilePostProcessing(removeDescription, postProcessIndent)::execute, + new AndroidFilePostProcessing( + removeDescription, postProcessIndent, removeTranslatableFalse) + ::execute, removeDescription)); } @@ -152,6 +159,12 @@ void applyFilterOptions(RawDocument input) { removeDescription = b; }); + filterOptions.getBoolean( + POST_PROCESS_REMOVE_TRANSLATABLE_FALSE, + b -> { + removeTranslatableFalse = b; + }); + filterOptions.getInteger( POST_PROCESS_INDENT, i -> { @@ -424,10 +437,13 @@ void updateFormInSkeleton(ITextUnit textUnit) { static class AndroidFilePostProcessing { static final String DESCRIPTION_ATTRIBUTE = "description"; boolean removeDescription; + boolean removeTranslatableFalse; int indent; - AndroidFilePostProcessing(boolean removeDescription, int indent) { + AndroidFilePostProcessing( + boolean removeDescription, int indent, boolean removeTranslatableFalse) { this.removeDescription = removeDescription; + this.removeTranslatableFalse = removeTranslatableFalse; this.indent = indent; } @@ -491,6 +507,9 @@ String execute(String xmlContent) { } } + if (removeTranslatableFalse) { + removeTranslatableFalseElements(document); + } removeWhitespaceNodes(document); TransformerFactory transformerFactory = TransformerFactory.newInstance(); @@ -521,7 +540,7 @@ String execute(String xmlContent) { } } - public void removeWhitespaceNodes(Node node) { + void removeWhitespaceNodes(Node node) { NodeList childNodes = node.getChildNodes(); for (int i = childNodes.getLength() - 1; i >= 0; i--) { Node childNode = childNodes.item(i); @@ -532,5 +551,21 @@ public void removeWhitespaceNodes(Node node) { } } } + + void removeTranslatableFalseElements(Node node) { + NodeList childNodes = node.getChildNodes(); + for (int i = childNodes.getLength() - 1; i >= 0; i--) { + Node childNode = childNodes.item(i); + if (childNode.getNodeType() == Node.ELEMENT_NODE) { + Element element = (Element) childNode; + if (element.hasAttribute("translatable") + && element.getAttribute("translatable").equals("false")) { + node.removeChild(element); + } else { + removeTranslatableFalseElements(element); + } + } + } + } } } diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java index e8f7397a6f..99dd4e0d08 100644 --- a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java +++ b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java @@ -113,7 +113,7 @@ void testUnescaping(String input, String expected) { @Test public void testPostProcessingKeepDescription() { AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = - new AndroidFilter.AndroidFilePostProcessing(false, 2); + new AndroidFilter.AndroidFilePostProcessing(false, 2, false); String input = """ @@ -157,7 +157,7 @@ public void testPostProcessingKeepDescription() { @Test public void testPostProcessingRemoveDescription() { AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = - new AndroidFilter.AndroidFilePostProcessing(true, 2); + new AndroidFilter.AndroidFilePostProcessing(true, 2, false); String input = """ @@ -201,7 +201,7 @@ public void testPostProcessingRemoveDescription() { @Test public void testPostProcessingEmptyFile() { AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = - new AndroidFilter.AndroidFilePostProcessing(true, 2); + new AndroidFilter.AndroidFilePostProcessing(true, 2, false); String input = ""; String output = androidFilePostProcessing.execute(input); String expected = ""; @@ -211,7 +211,7 @@ public void testPostProcessingEmptyFile() { @Test public void testPostProcessingNoProlog() { AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = - new AndroidFilter.AndroidFilePostProcessing(true, 2); + new AndroidFilter.AndroidFilePostProcessing(true, 2, false); String input = """ @@ -228,4 +228,43 @@ public void testPostProcessingNoProlog() { """; assertEquals(expected, output); } + + @Test + public void testPostProcessingRemoveTranslatableFalse() { + AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = + new AndroidFilter.AndroidFilePostProcessing(true, 2, true); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + translated + @#$untranslated$#@ + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessing.execute(input); + String expected = + """ + + + + + translated + + + """; + assertEquals(expected, output); + } } From 05ea7902d029d12f3daa471b7378819360865041 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 30 Aug 2024 21:24:41 -0700 Subject: [PATCH 056/105] Add MessageFormatIntegrityCheckerTest test for quote with curly braces --- .../MessageFormatIntegrityCheckerTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java index 12d0d51124..1d421305ab 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java @@ -3,6 +3,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import com.google.common.collect.ImmutableMap; +import com.ibm.icu.text.MessageFormat; import org.junit.Test; public class MessageFormatIntegrityCheckerTest { @@ -128,4 +130,18 @@ public void testNamedParametersChangedButWithDuplicates() // assertEquals("Different placeholder name in source and target", e.getMessage()); } } + + @Test(expected = IntegrityCheckException.class) + public void testQuoteCurlyEscaping() throws MessageFormatIntegrityCheckerException { + // ' with character are rendered by if it is a special character like {, it will escape it .... + MessageFormatIntegrityChecker checker = new MessageFormatIntegrityChecker(); + String source = "This is a \"{placeholder}\""; + String target = "C'est un '{placeholder}'"; + checker.check(source, target); + + MessageFormat messageFormat = new MessageFormat(target); + String format = messageFormat.format(ImmutableMap.of("placeholder", "stuff")); + assertEquals("C'est un {placeholder}", format); + } + } From a249f60f500edcfc3f0438be13b0fc81a4837404 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 3 Sep 2024 15:22:46 -0700 Subject: [PATCH 057/105] Support "dash" in HtmlTagIntegrityChecker --- .../integritychecker/HtmlTagIntegrityChecker.java | 2 +- .../integritychecker/RegexIntegrityChecker.java | 4 ++-- .../integritychecker/HtmlTagIntegrityCheckerTest.java | 10 ++++++++++ .../MessageFormatIntegrityCheckerTest.java | 1 - 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java index f5d8a533c8..896f50bba4 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java @@ -21,7 +21,7 @@ public class HtmlTagIntegrityChecker extends RegexIntegrityChecker { @Override public String getRegex() { - return "(<\\w+(\\s+\\w+(\\s*=\\s*('([^']*?)'|\"([^\"]*?)\"))?)*\\s*/?>|)"; + return "(<[a-zA-Z][\\w-]*(\\s+\\w+(\\s*=\\s*('([^']*?)'|\"([^\"]*?)\"))?)*\\s*/?>|<\\/[a-zA-Z][\\w-]*>)"; } @Override diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/RegexIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/RegexIntegrityChecker.java index e0ccc3552f..6a8d900882 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/RegexIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/RegexIntegrityChecker.java @@ -1,6 +1,6 @@ package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -68,7 +68,7 @@ Pattern getPattern() { */ Set getPlaceholders(String string) { - Set placeholders = new HashSet<>(); + Set placeholders = new LinkedHashSet<>(); if (string != null) { diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java index df48ae696e..29b288e1d1 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; +import java.util.Set; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -191,4 +193,12 @@ public void testHtmlTagCheckNonTagLessThanDoesntConfuseThings() { checker.check(source, target); } + + @Test + public void testHtmlTagCheckWithDash() { + String source = "Mag-sign up o upang "; + Set placeholders = checker.getPlaceholders(source); + Assertions.assertThat(placeholders) + .containsExactly("", "", "", ""); + } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java index 1d421305ab..faf288f852 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java @@ -143,5 +143,4 @@ public void testQuoteCurlyEscaping() throws MessageFormatIntegrityCheckerExcepti String format = messageFormat.format(ImmutableMap.of("placeholder", "stuff")); assertEquals("C'est un {placeholder}", format); } - } From 5f165d76f972cab69a808970ecc63734f0f229d0 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 4 Sep 2024 21:48:25 -0700 Subject: [PATCH 058/105] Add command to create pull request in Github This is exploratory to see if we can avoid having to install the GH CLI or make curl command --- .../cli/command/GithubCreatePRCommand.java | 95 +++++++++++++++++++ .../box/l10n/mojito/github/GithubClient.java | 41 ++++++++ 2 files changed, 136 insertions(+) create mode 100644 cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java new file mode 100644 index 0000000000..52de84d993 --- /dev/null +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java @@ -0,0 +1,95 @@ +package com.box.l10n.mojito.cli.command; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.box.l10n.mojito.cli.console.ConsoleWriter; +import com.box.l10n.mojito.github.GithubClients; +import com.box.l10n.mojito.github.GithubException; +import java.util.List; +import org.fusesource.jansi.Ansi; +import org.kohsuke.github.GHPullRequest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +@Component +@Scope("prototype") +@Parameters( + commandNames = {"github-pr-create"}, + commandDescription = "Create a Github PR") +public class GithubCreatePRCommand extends Command { + + @Autowired GithubClients githubClients; + + @Qualifier("ansiCodeEnabledFalse") + @Autowired + ConsoleWriter consoleWriter; + + @Parameter( + names = {"--owner", "-o"}, + required = true, + arity = 1, + description = "The Github repository owner") + String owner; + + @Parameter( + names = {"--repository", "-r"}, + required = true, + arity = 1, + description = "The Github repository name") + String repository; + + @Parameter( + names = {"--title"}, + required = true, + arity = 1, + description = "The PR title") + String title; + + @Parameter( + names = {"--head"}, + required = true, + arity = 1, + description = "The PR head") + String head; + + @Parameter( + names = {"--base"}, + required = true, + arity = 1, + description = "The PR base") + String base; + + @Parameter( + names = {"--body"}, + required = false, + arity = 1, + description = "The PR body") + String body; + + @Parameter( + names = {"--reviewers"}, + required = false, + variableArity = true, + description = "The PR reviewers") + List reviewers; + + @Override + public boolean shouldShowInCommandList() { + return false; + } + + @Override + protected void execute() throws CommandException { + try { + + GHPullRequest pr = + githubClients.getClient(owner).createPR(repository, title, head, base, body, reviewers); + + consoleWriter.a("PR created: ").fg(Ansi.Color.CYAN).a(pr.getHtmlUrl().toString()).println(); + } catch (GithubException e) { + throw new CommandException(e); + } + } +} diff --git a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java index 023c11f7fb..4fd471ff1c 100644 --- a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java +++ b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java @@ -15,6 +15,8 @@ import org.kohsuke.github.GHAppInstallationToken; import org.kohsuke.github.GHCommitState; import org.kohsuke.github.GHIssueComment; +import org.kohsuke.github.GHPullRequest; +import org.kohsuke.github.GHUser; import org.kohsuke.github.GitHub; import org.kohsuke.github.GitHubBuilder; import org.slf4j.Logger; @@ -304,6 +306,45 @@ public List getPRComments(String repository, int prNumber) { } } + public GHPullRequest createPR( + String repository, + String title, + String head, + String base, + String body, + List reviewers) { + String repoFullPath = getRepositoryPath(repository); + try { + GHPullRequest pullRequest = + getGithubClient(repository) + .getRepository(repoFullPath) + .createPullRequest(title, head, base, body); + + if (reviewers != null) { + List reviewersGH = + reviewers.stream() + .map( + s -> { + try { + return gitHubClient.getUser(s); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .toList(); + + pullRequest.requestReviewers(reviewersGH); + } + + return pullRequest; + } catch (IOException | NoSuchAlgorithmException | InvalidKeySpecException e) { + String message = + String.format("Error creating a PR in repository '%s': %s", repoFullPath, e.getMessage()); + logger.error(message, e); + throw new GithubException(message, e); + } + } + public String getOwner() { return owner; } From 4c275f805304087e95e34a341b379ecd3b101214 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 10 Sep 2024 22:28:51 -0700 Subject: [PATCH 059/105] Add test for the AndroidStringDocumentWriter XML escaping We should consider removing the escaping for Phrase, if that is causing issue to process. --- .../ImportLocalizedAssetCommandTest.java | 16 ++++---- .../AndroidStringDocumentWriterTest.java | 38 +++++++++++++++++++ .../URLIntegrityCheckerTest.java | 7 ++++ 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java index 150d95395c..4e455abbf6 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java @@ -219,14 +219,14 @@ public void importAndroidStringsPluralWithThirdPartySync() throws Exception { "PUSH,MAP_TEXTUNIT,PUSH_TRANSLATION,PULL"); getL10nJCommander() - .run( - "pull", - "-r", - repository.getName(), - "-s", - getInputResourcesTestDir("source").getAbsolutePath(), - "-t", - getTargetTestDir("after-sync").getAbsolutePath()); + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("after-sync").getAbsolutePath()); getL10nJCommander() .run( diff --git a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java index fc215103d6..969f5e9fba 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java @@ -140,6 +140,44 @@ public void testWriteChars() throws Exception { assertThat(getTempFileContent()).isEqualTo(result); } + @Test + public void testEscaping() throws Exception { + + // Not sure why the current implementation uses escapeQuotes(). Intuitively, for + // machine-to-machine communication, pure XML + // seems to be the best option. However, it appears that escapeQuotes() tries to mimic the + // Android format + // overloads. I'm wondering if this was required by Smartling. + result = + """ + + with a <a href=\\"http://test.org\\"> link</a>. + <i>i</i> could be unescape, but currently is escaped. + a string with\\n return line + a string with \\" quote. + <annotation url=\\"http://test.com\\">test</annotation> + + """; + + source = new AndroidStringDocument(); + source.addSingular( + new AndroidSingular(120L, "a_href", "with a link.", "")); + source.addSingular( + new AndroidSingular( + 121L, "i", "i could be unescape, but currently is escaped.", "")); + source.addSingular(new AndroidSingular(122L, "return_line", "a string with\n return line", "")); + source.addSingular(new AndroidSingular(123L, "with_quotes", "a string with \" quote.", "")); + source.addSingular( + new AndroidSingular( + 124L, "with_annotation", "test", "")); + + writer = new AndroidStringDocumentWriter(source); + assertThat(writer.toText()).isEqualTo(result); + + writer.toFile(tmpFile.getPath()); + assertThat(getTempFileContent()).isEqualTo(result); + } + @Test public void testWritePlurals() throws Exception { diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java index 34e1e0e6d2..2ac1809f14 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java @@ -89,4 +89,11 @@ public void chanagedEmail() { String target = "Il n'y a pas d'email mailto:tedst@test.org"; checker.check(source, target); } + + @Test(expected = URLIntegrityCheckerException.class) + public void urlWithDash() { + String source = "A url https://test.com/manage."; + String target = "Une url https://test.com/account/manage-."; + checker.check(source, target); + } } From 89933e209fda9a12ccb4e6f57b651252c57546a3 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 12 Sep 2024 14:19:53 -0700 Subject: [PATCH 060/105] Update install documentation for java/mysql --- docs/_docs/guides/002-install_springboot3.md | 8 ++++---- docs/_docs/guides/open-source-contributors.md | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/_docs/guides/002-install_springboot3.md b/docs/_docs/guides/002-install_springboot3.md index 3a0247b399..1e23124afb 100644 --- a/docs/_docs/guides/002-install_springboot3.md +++ b/docs/_docs/guides/002-install_springboot3.md @@ -9,13 +9,13 @@ permalink: /docs/guides/install-springboot3/ No binaries are available for the `Spring Boot 3` at the moment. It should be built from the `master` branch. -Assuming `Java 17` is installed, `./mvnw install -DskipTests` should be enough to build the `jar` files. +Assuming `Java 21` is installed, `./mvnw install -DskipTests` should be enough to build the `jar` files. For detail instructions on development environment setup, [see here]({{ site.url }}/docs/guides/open-source-contributors/). ### Using Executable Jars -`Java 17` is required. +`Java 21` is required. Run the Webapp with: @@ -78,8 +78,8 @@ The port can be changed with the `server.port` property. ### MySQL -[Install MySQL 5.7](http://dev.mysql.com/doc/refman/5.7/en/installing.html) and then create a database for {{ site.mojito_green }} -(with Brew: `brew install mysql@5.7`). +[Install MySQL 8](https://dev.mysql.com/doc/mysql-installation-excerpt/8.0/en/) and then create a database for {{ site.mojito_green }} +(with Brew: `brew install mysql@8`). Connect to MySQL DB as root user diff --git a/docs/_docs/guides/open-source-contributors.md b/docs/_docs/guides/open-source-contributors.md index 772acee666..b80c05a92b 100644 --- a/docs/_docs/guides/open-source-contributors.md +++ b/docs/_docs/guides/open-source-contributors.md @@ -32,13 +32,13 @@ Ensure that you have added the `cask-versions` [tap](https://github.com/Homebrew brew tap homebrew/cask-versions ``` -Install `java 17` from `Temurin`: +Install `java 21` from `Temurin`: ```sh brew install --cask temurin17 ``` -Check that `java 17` is now in use with `java -version`. If you need multiple versions of `java` consider using +Check that `java 21` is now in use with `java -version`. If you need multiple versions of `java` consider using something like `jenv` (don't forget the plugins: `jenv enable-plugin maven` & `jenv enable-plugin export` ) or `sdkman`. Install `maven` (latest version should be fine): @@ -58,13 +58,13 @@ Note, to install the exact same `maven` version as the wrapper: `brew install ma ### Install on Ubuntu 18.4 LTS -Install `java 17` from `OpenJDK`: +Install `java 21` from `OpenJDK`: ```sh -sudo apt-get install openjdk-17-jdk +sudo apt-get install openjdk-21-jdk ``` -Check that `java 17` is now in use with `java -version`. If not you, can set it as default with -`sudo update-java-alternatives -s /usr/lib/jvm/java-1.17.0-openjdk-amd64` (To check the list +Check that `java 21` is now in use with `java -version`. If not you, can set it as default with +`sudo update-java-alternatives -s /usr/lib/jvm/java-1.21.0-openjdk-amd64` (To check the list `sudo update-java-alternatives -l`). If you need multiple versions of `java` consider using something like `jenv` or `sdkman`. @@ -140,8 +140,8 @@ Run the following commands to create the databases. Change the user name, databa CREATE USER 'mojito'@'localhost' IDENTIFIED BY 'mojito'; CREATE DATABASE IF NOT EXISTS mojito CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_bin'; CREATE DATABASE IF NOT EXISTS mojito_dev CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_bin'; -GRANT ALL ON mojito.* TO 'mojito'@'localhost' IDENTIFIED BY 'mojito'; -GRANT ALL ON mojito_dev.* TO 'mojito'@'localhost' IDENTIFIED BY 'mojito'; +GRANT ALL PRIVILEGES ON mojito.* TO 'mojito'@'localhost'; +GRANT ALL PRIVILEGES ON mojito_dev.* TO 'mojito'@'localhost'; FLUSH PRIVILEGES; ``` From a690350ec8777e01ccee0491dc7ca1d8d7a28eba Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 13 Sep 2024 00:56:01 -0700 Subject: [PATCH 061/105] Support user creation for Oicd authentication This is similar to UserDetailImplOAuth2UserService though in that case it is simpler and just reuses the parent class to get an OicdUser and then convert it into a UserDetailsImpl. Add a username-from-email config. In case the principal ID form the auth provider is a UUID, it is of little use on the Mojito side for tasks such as sending notifications, associating with Git commit, PRs, etc. With this option, the username is extracted from the email. This assumes a 1:1 mapping between the email and the Unix username Considered saving the email in the user, but for simplicity we just need a username that is unix like for now. Note: UserDetailImplOAuth2UserService probably duplicated the code to support the "unwrapping" of the user attribute. The logic was probably failing before being able to do the conversion --- .../mojito/security/OidcUserDetailsImpl.java | 42 ++++++++++++++++ .../l10n/mojito/security/SecurityConfig.java | 19 +++++++ .../UserDetailImplOidcUserService.java | 50 +++++++++++++++++++ .../mojito/security/WebSecurityConfig.java | 3 ++ 4 files changed, 114 insertions(+) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/security/OidcUserDetailsImpl.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/security/UserDetailImplOidcUserService.java diff --git a/webapp/src/main/java/com/box/l10n/mojito/security/OidcUserDetailsImpl.java b/webapp/src/main/java/com/box/l10n/mojito/security/OidcUserDetailsImpl.java new file mode 100644 index 0000000000..87f14d247b --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/security/OidcUserDetailsImpl.java @@ -0,0 +1,42 @@ +package com.box.l10n.mojito.security; + +import com.box.l10n.mojito.entity.security.user.User; +import java.util.Map; +import org.springframework.security.oauth2.core.oidc.OidcIdToken; +import org.springframework.security.oauth2.core.oidc.OidcUserInfo; +import org.springframework.security.oauth2.core.oidc.user.OidcUser; + +public class OidcUserDetailsImpl extends UserDetailsImpl implements OidcUser { + + private final OidcUser oidcUser; + + public OidcUserDetailsImpl(User user, OidcUser oidcUser) { + super(user); + this.oidcUser = oidcUser; + } + + @Override + public Map getClaims() { + return oidcUser.getClaims(); + } + + @Override + public OidcUserInfo getUserInfo() { + return oidcUser.getUserInfo(); + } + + @Override + public OidcIdToken getIdToken() { + return oidcUser.getIdToken(); + } + + @Override + public Map getAttributes() { + return oidcUser.getAttributes(); + } + + @Override + public String getName() { + return oidcUser.getName(); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/security/SecurityConfig.java b/webapp/src/main/java/com/box/l10n/mojito/security/SecurityConfig.java index 8863f92e22..db3388374a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/security/SecurityConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/security/SecurityConfig.java @@ -72,6 +72,17 @@ public static class OAuth2 { String unwrapUserAttributes; + /** + * In case the principal ID form the auth provider is a UUID, it is of little use on the Mojito + * side for tasks such as sending notifications, associating with Git commit, PRs, etc. + * + *

With this option, the username is extracted from the email. This assumes a 1:1 mapping + * between the email and the Unix username + * + *

Currently only implement in {@link UserDetailImplOidcUserService} + */ + boolean usernameFromEmail = false; + String givenNameAttribute = "first_name"; String surnameAttribute = "last_name"; String commonNameAttribute = "name"; @@ -115,5 +126,13 @@ public String getUnwrapUserAttributes() { public void setUnwrapUserAttributes(String unwrapUserAttributes) { this.unwrapUserAttributes = unwrapUserAttributes; } + + public boolean isUsernameFromEmail() { + return usernameFromEmail; + } + + public void setUsernameFromEmail(boolean usernameFromEmail) { + this.usernameFromEmail = usernameFromEmail; + } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/security/UserDetailImplOidcUserService.java b/webapp/src/main/java/com/box/l10n/mojito/security/UserDetailImplOidcUserService.java new file mode 100644 index 0000000000..54665fcf77 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/security/UserDetailImplOidcUserService.java @@ -0,0 +1,50 @@ +package com.box.l10n.mojito.security; + +import com.box.l10n.mojito.entity.security.user.User; +import com.box.l10n.mojito.service.security.user.UserService; +import java.util.Objects; +import org.springframework.security.oauth2.client.oidc.userinfo.OidcUserRequest; +import org.springframework.security.oauth2.client.oidc.userinfo.OidcUserService; +import org.springframework.security.oauth2.core.OAuth2AuthenticationException; +import org.springframework.security.oauth2.core.oidc.user.OidcUser; + +class UserDetailImplOidcUserService extends OidcUserService { + + SecurityConfig securityConfig; + UserService userService; + + public UserDetailImplOidcUserService(SecurityConfig securityConfig, UserService userService) { + this.securityConfig = Objects.requireNonNull(securityConfig); + this.userService = Objects.requireNonNull(userService); + } + + @Override + public OidcUser loadUser(OidcUserRequest userRequest) throws OAuth2AuthenticationException { + OidcUser oidcUser = super.loadUser(userRequest); + + SecurityConfig.OAuth2 securityConfigOAuth2 = + securityConfig + .getoAuth2() + .getOrDefault( + userRequest.getClientRegistration().getRegistrationId(), + new SecurityConfig.OAuth2()); + + String username; + if (securityConfigOAuth2.usernameFromEmail) { + String email = oidcUser.getEmail(); + if (email == null) { + throw new RuntimeException( + "OidcUser's email must not be null since it used to extract username"); + } + username = email.split("@")[0]; + } else { + username = oidcUser.getName(); + } + + User user = + userService.getOrCreateOrUpdateBasicUser( + username, oidcUser.getGivenName(), oidcUser.getFamilyName(), oidcUser.getFullName()); + + return new OidcUserDetailsImpl(user, oidcUser); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/security/WebSecurityConfig.java b/webapp/src/main/java/com/box/l10n/mojito/security/WebSecurityConfig.java index 1c05bbf271..9f0b1c3f2f 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/security/WebSecurityConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/security/WebSecurityConfig.java @@ -268,6 +268,9 @@ public SecurityFilterChain configure(HttpSecurity http) throws Exception { userInfoEndpoint -> { userInfoEndpoint.userService( new UserDetailImplOAuth2UserService(securityConfig, userService)); + + userInfoEndpoint.oidcUserService( + new UserDetailImplOidcUserService(securityConfig, userService)); }); }); break; From 20d7c986b3cf6818bf118ac3f6b05245f8d7ff2b Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 13 Sep 2024 12:00:16 -0700 Subject: [PATCH 062/105] Improve HtmlTagIntegrityChecker and reporting --- .../HtmlTagIntegrityChecker.java | 36 ++++++++++++++++--- .../HtmlTagIntegrityCheckerTest.java | 7 ++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java index 896f50bba4..beb2c66388 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java @@ -2,8 +2,13 @@ import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.regex.Matcher; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,11 +39,34 @@ public void check(String sourceContent, String targetContent) throws IntegrityCh List sourceHtmlTags = getHtmlTags(sourceContent); logger.debug("Source Html tags: {}", sourceHtmlTags); + Map sourceTagCount = + sourceHtmlTags.stream() + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); + Map targetTagCount = + targetHtmlTags.stream() + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); + logger.debug("Make sure the target has the same Html tags as the source"); - if (sourceHtmlTags.size() != targetHtmlTags.size() - || !sourceHtmlTags.containsAll(targetHtmlTags) - || !targetHtmlTags.containsAll(sourceHtmlTags)) { - throw new HtmlTagIntegrityCheckerException("HTML tags in source and target are different"); + if (!sourceTagCount.equals(targetTagCount)) { + StringBuilder differences = new StringBuilder(); + + Set allTags = new HashSet<>(); + allTags.addAll(sourceTagCount.keySet()); + allTags.addAll(targetTagCount.keySet()); + + for (String tag : allTags) { + Long sourceCount = sourceTagCount.getOrDefault(tag, 0L); + Long targetCount = targetTagCount.getOrDefault(tag, 0L); + if (sourceCount > targetCount) { + differences.append( + String.format("Target is missing %d tag(s) '%s'%n", sourceCount - targetCount, tag)); + } else if (sourceCount < targetCount) { + differences.append( + String.format("Target has extra %d tag(s) '%s'%n", targetCount - sourceCount, tag)); + } + } + throw new HtmlTagIntegrityCheckerException( + "HTML tag counts in source and target are different:\n" + differences.toString().trim()); } logger.debug("Make sure the target tags are in valid order"); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java index 29b288e1d1..4dcadf6d41 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java @@ -69,6 +69,13 @@ public void testHtmlTagCheckFailsWithDuplicatedOpening() { checker.check(source, target); } + @Test(expected = HtmlTagIntegrityCheckerException.class) + public void testHtmlTagCheckFailsWithDuplicatedButSameCount() { + String source = "

some text

some other text

"; + String target = "

some text

some other text

"; + checker.check(source, target); + } + @Test(expected = HtmlTagIntegrityCheckerException.class) public void testHtmlTagCheckWorksWhenTagIsModified() { String source = "There are %1 files and %2 folders"; From e47b90e87ebe7c418441254202227f2e50170f09 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Sun, 15 Sep 2024 22:05:33 -0700 Subject: [PATCH 063/105] Improve URLIntegrityChecker Support sub domains --- .../integritychecker/URLIntegrityChecker.java | 2 +- .../integritychecker/URLIntegrityCheckerTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java index e321906f7a..23f447ccaf 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java @@ -14,7 +14,7 @@ public class URLIntegrityChecker extends RegexIntegrityChecker { @Override public String getRegex() { - return "(https?|ftp)://(?:www\\.)?[a-zA-Z0-9-]+\\.[a-zA-Z]{2,}(?:/[a-zA-Z0-9/_\\-?#+%]*)*|mailto:[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; + return "(https?|ftp)://(?:www\\.)?(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}(?:/[a-zA-Z0-9/_\\.\\-?#+%]*)*|mailto:[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; } @Override diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java index 2ac1809f14..aa0687a685 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java @@ -92,8 +92,8 @@ public void chanagedEmail() { @Test(expected = URLIntegrityCheckerException.class) public void urlWithDash() { - String source = "A url https://test.com/manage."; - String target = "Une url https://test.com/account/manage-."; + String source = "A url https://test.more.com/account/manage."; + String target = "Une url https://test.more.com/account/manage-."; checker.check(source, target); } } From a755c880544c1fff38545f4dfd5a489d33d58180 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 16 Sep 2024 11:34:10 -0700 Subject: [PATCH 064/105] Improve CompositeFormatIntegrityChecker for imbalanced curly braces Looking to check for source "{{placeholder}}" and target "{{{placeholder}}" and reject it --- .../CompositeFormatIntegrityChecker.java | 2 +- .../CompositeFormatIntegrityCheckerTest.java | 26 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java index 426be785e0..c3621b4935 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java @@ -9,7 +9,7 @@ public class CompositeFormatIntegrityChecker extends RegexIntegrityChecker { @Override public String getRegex() { - return "\\{\\{[^{}]*?\\}\\}|\\{[^{}]*?\\}"; + return "(\\{){1,3}[^\\{\\}]*\\}+"; } @Override diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java index 19a1ad12e5..1503212bcc 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java @@ -17,7 +17,7 @@ public class CompositeFormatIntegrityCheckerTest { public void testGetPlaceholder() throws CompositeFormatIntegrityCheckerException { CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); - String string = "{0} '{2}' {0:0.00}% \"{1}\""; + String string = "{0} '{2}' {0:0.00}% \"{1}\" {{3}} {{{4}}} {{{5}} {{6}}}"; Set placeholders = checker.getPlaceholders(string); @@ -26,6 +26,10 @@ public void testGetPlaceholder() throws CompositeFormatIntegrityCheckerException expected.add("{2}"); expected.add("{0:0.00}"); expected.add("{1}"); + expected.add("{{3}}"); + expected.add("{{{4}}}"); + expected.add("{{{5}}"); + expected.add("{{6}}}"); assertEquals(expected, placeholders); } @@ -82,6 +86,26 @@ public void testMustacheInvalidMissingCurlyBraces() { checker.check(source, target); } + @Test(expected = CompositeFormatIntegrityCheckerException.class) + public void testMustacheInvalidMissingCurlyBraces2() { + CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); + + String source = "A {{mustache} template"; + String target = "Un modėle {{{mustache}"; + + checker.check(source, target); + } + + @Test(expected = CompositeFormatIntegrityCheckerException.class) + public void testMustacheInvalidMissingCurlyBraces3() { + CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); + + String source = "A {{mustache}} template"; + String target = "Un modėle {{{mustache}}"; + + checker.check(source, target); + } + @Test(expected = CompositeFormatIntegrityCheckerException.class) public void testMustacheInvalidMissingPlaceholder() { CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); From 0c2a437f8e3eb3c5ab52cd69ff357373e094b0b5 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 16 Sep 2024 11:34:32 -0700 Subject: [PATCH 065/105] Bump github-api to 1.325 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 531224df72..629f491990 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ 2.8.2 1.22.0 0.10.5 - 1.313 + 1.325 64.2 true true From efbafb6c8e32dc1a31d5b4b9b749da5ee2931900 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 16 Sep 2024 22:21:47 -0700 Subject: [PATCH 066/105] Add option to not escape in AndroidStringDocumentWriter --- .../strings/AndroidStringDocumentWriter.java | 25 ++++++++++++- .../AndroidStringDocumentWriterTest.java | 37 +++++++++++++++---- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java index c8538c2472..722b585b7e 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java @@ -33,9 +33,15 @@ public class AndroidStringDocumentWriter { private DOMSource domSource; private Document document; private Node root; + private EscapeType escapeType; public AndroidStringDocumentWriter(final AndroidStringDocument source) { + this(source, EscapeType.QUOTE_AND_NEW_LINE); + } + + public AndroidStringDocumentWriter(final AndroidStringDocument source, EscapeType escapeType) { this.source = requireNonNull(source); + this.escapeType = escapeType; buildDomSource(); } @@ -153,7 +159,22 @@ private void setAttribute(Element element, String name, String value) { } } - static String escapeQuotes(String str) { - return Strings.nullToEmpty(str).replaceAll("\"", "\\\\\"").replaceAll("\n", "\\\\n"); + String escapeQuotes(String str) { + String escaped = str; + if (!Strings.isNullOrEmpty(str)) { + escaped = + switch (escapeType) { + case QUOTE_AND_NEW_LINE -> str.replaceAll("\"", "\\\\\"").replaceAll("\n", "\\\\n"); + case NEW_LINE -> str.replaceAll("\n", "\\\\n"); + case NONE -> str; + }; + } + return escaped; + } + + public enum EscapeType { + QUOTE_AND_NEW_LINE, // Legacy + NEW_LINE, + NONE } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java index 969f5e9fba..417309ce57 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java @@ -1,6 +1,5 @@ package com.box.l10n.mojito.android.strings; -import static com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter.escapeQuotes; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.ImmutableList; @@ -264,14 +263,38 @@ public void testWritePreservesOrdering() throws Exception { assertThat(getTempFileContent()).isEqualTo(result); } + @Test + public void testGenerateWithHTML() { + source = new AndroidStringDocument(); + source.addSingular( + new AndroidSingular(10L, "string1", "some link to a page", "test comment1")); + source.addSingular( + new AndroidSingular( + 11L, + "string2", + "some link to a page", + "test comment2")); + source.addSingular( + new AndroidSingular( + 12L, + "string3", + "If that is your IP address click here to unblock it.", + "test comment2")); + + writer = new AndroidStringDocumentWriter(source); + // TODO(ja) must finsih + System.out.println(writer.toText()); + // assertThat(writer.toText()).isEqualTo(result); + } + @Test public void testEscapeQuotes() { - assertThat(escapeQuotes(null)).isEqualTo(""); - assertThat(escapeQuotes("")).isEqualTo(""); - assertThat(escapeQuotes("String")).isEqualTo("String"); - assertThat(escapeQuotes("second\\\"String")).isEqualTo("second\\\\\"String"); - assertThat(escapeQuotes("third\nString")).isEqualTo("third\\nString"); - assertThat(escapeQuotes("fourth\ntest\\\"String\\\"")) + assertThat(writer.escapeQuotes(null)).isEqualTo(""); + assertThat(writer.escapeQuotes("")).isEqualTo(""); + assertThat(writer.escapeQuotes("String")).isEqualTo("String"); + assertThat(writer.escapeQuotes("second\\\"String")).isEqualTo("second\\\\\"String"); + assertThat(writer.escapeQuotes("third\nString")).isEqualTo("third\\nString"); + assertThat(writer.escapeQuotes("fourth\ntest\\\"String\\\"")) .isEqualTo("fourth\\ntest\\\\\"String\\\\\""); } From 33a0c2eb1308e7e3b1d3f468586b919ca443e0ca Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 16 Sep 2024 22:22:41 -0700 Subject: [PATCH 067/105] Test escaping with Phrase --- .../thirdparty/phrase/PhraseClientTest.java | 92 ++++++++++++++++--- 1 file changed, 77 insertions(+), 15 deletions(-) diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java index 796fbfe355..2c70bdda0a 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -1,12 +1,15 @@ package com.box.l10n.mojito.service.thirdparty.phrase; import com.box.l10n.mojito.JSR310Migration; +import com.box.l10n.mojito.android.strings.AndroidSingular; +import com.box.l10n.mojito.android.strings.AndroidStringDocument; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter; import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; import com.phrase.client.model.Tag; import java.time.ZonedDateTime; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; import org.junit.Assume; import org.junit.Ignore; import org.junit.Test; @@ -88,6 +91,65 @@ public void testParallelDownload() { public void test() { Assume.assumeNotNull(testProjectId); + for (int i = 0; i < 2; i++) { + AndroidStringDocument source = new AndroidStringDocument(); + source.addSingular( + new AndroidSingular(11L, i + "string1", "some link to a page", "test comment1")); + source.addSingular( + new AndroidSingular( + 12L, + i + "string2", + "some link to a page", + "test comment2")); + source.addSingular( + new AndroidSingular( + 13L, + i + "string3", + "If that is your IP address click here to unblock it.", + "test comment2")); + + source.addSingular( + new AndroidSingular( + 14L, + i + "string4", + "If that is your IP address click here to unblock it.", + "test comment2")); + + source.addSingular( + new AndroidSingular( + 15L, + i + "string5", + "If that is your IP address click here to unblock it.", + "test comment2")); + + source.addSingular( + new AndroidSingular( + 16L, + i + "string6", + "visit your settings", + "test comment2")); + + String androidFile = new AndroidStringDocumentWriter(source, i % 2 == 0).toText(); + System.out.println(androidFile); + + phraseClient.uploadAndWait( + testProjectId, + "en", + "xml", + "strings.xml", + androidFile, + ImmutableList.of(i % 2 == 0 ? "test-escaping" : "test-no-escaping")); + + String s = + phraseClient.localeDownload( + testProjectId, + "en", + "xml", + i % 2 == 0 ? "test-escaping" : "test-no-escaping", + () -> null); + System.out.println(s); + } + // for (int i = 0; i < 3; i++) { // String repoName = "repo_%d".formatted(i); // String tagForUpload = ThirdPartyTMSPhrase.getTagForUpload(repoName); @@ -129,20 +191,20 @@ public void test() { // phraseClient.uploadCreateFile( // testProjectId, "fr", "xml", "strings.xml", fileContentAndroid2, null); - String s2 = - phraseClient.localeDownload( - testProjectId, - "en", - "xml", - "startWithABadTag", - () -> - phraseClient.listTags(testProjectId).stream() - .map(Tag::getName) - .filter(Objects::nonNull) - .filter(tagName -> tagName.startsWith("push_repo_2")) - .collect(Collectors.joining(","))); - - logger.info(s2); + // String s2 = + // phraseClient.localeDownload( + // testProjectId, + // "en", + // "xml", + // "startWithABadTag", + // () -> + // phraseClient.listTags(testProjectId).stream() + // .map(Tag::getName) + // .filter(Objects::nonNull) + // .filter(tagName -> tagName.startsWith("push_repo_2")) + // .collect(Collectors.joining(","))); + // + // logger.info(s2); } static StringBuilder generateFileContent(String repositoryName) { From e43cb0e628311a77276e90742957f77bef29cd36 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 17 Sep 2024 16:51:11 -0700 Subject: [PATCH 068/105] Add Github "enable auto merge" PR option This is using the GraphQL API since it is not available in the Java client. --- .../cli/command/GithubCreatePRCommand.java | 18 ++- .../GithubGetInstallationTokenCommand.java | 5 +- .../box/l10n/mojito/github/GithubClient.java | 127 +++++++++++++++--- 3 files changed, 127 insertions(+), 23 deletions(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java index 52de84d993..936691750c 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java @@ -3,6 +3,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.box.l10n.mojito.cli.console.ConsoleWriter; +import com.box.l10n.mojito.github.GithubClient; import com.box.l10n.mojito.github.GithubClients; import com.box.l10n.mojito.github.GithubException; import java.util.List; @@ -75,6 +76,19 @@ public class GithubCreatePRCommand extends Command { description = "The PR reviewers") List reviewers; + @Parameter( + names = {"--enable-auto-merge"}, + required = false, + arity = 1, + description = "Enable auto-merge with the specified method") + EnableAutoMergeType enableAutoMerge = EnableAutoMergeType.NONE; + + enum EnableAutoMergeType { + SQUASH, + MERGE, + NONE + } + @Override public boolean shouldShowInCommandList() { return false; @@ -86,8 +100,10 @@ protected void execute() throws CommandException { GHPullRequest pr = githubClients.getClient(owner).createPR(repository, title, head, base, body, reviewers); - consoleWriter.a("PR created: ").fg(Ansi.Color.CYAN).a(pr.getHtmlUrl().toString()).println(); + if (!EnableAutoMergeType.NONE.equals(enableAutoMerge)) { + githubClients.getClient(owner).enableAutoMerge(pr, GithubClient.AutoMergeType.SQUASH); + } } catch (GithubException e) { throw new CommandException(e); } diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubGetInstallationTokenCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubGetInstallationTokenCommand.java index 3620e58251..78418c5f59 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubGetInstallationTokenCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubGetInstallationTokenCommand.java @@ -4,9 +4,6 @@ import com.beust.jcommander.Parameters; import com.box.l10n.mojito.cli.console.ConsoleWriter; import com.box.l10n.mojito.github.GithubClients; -import java.io.IOException; -import java.security.NoSuchAlgorithmException; -import java.security.spec.InvalidKeySpecException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Scope; @@ -50,7 +47,7 @@ protected void execute() throws CommandException { consoleWriter .a(githubClients.getClient(owner).getGithubAppInstallationToken(repository).getToken()) .print(); - } catch (IOException | NoSuchAlgorithmException | InvalidKeySpecException e) { + } catch (Exception e) { throw new CommandException(e); } } diff --git a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java index 4fd471ff1c..0b2e5036cd 100644 --- a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java +++ b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java @@ -1,9 +1,16 @@ package com.box.l10n.mojito.github; +import com.box.l10n.mojito.json.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.ImmutableMap; import io.jsonwebtoken.JwtBuilder; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; @@ -11,6 +18,7 @@ import java.security.spec.PKCS8EncodedKeySpec; import java.util.Date; import java.util.List; +import java.util.Map; import org.apache.commons.codec.binary.Base64; import org.kohsuke.github.GHAppInstallationToken; import org.kohsuke.github.GHCommitState; @@ -306,6 +314,15 @@ public List getPRComments(String repository, int prNumber) { } } + public void listPR(String repository) { + try { + GitHub gc = getGithubClient(repository); + gc.getRepository(repository); + } catch (IOException | NoSuchAlgorithmException | InvalidKeySpecException e) { + throw new RuntimeException(e); + } + } + public GHPullRequest createPR( String repository, String title, @@ -335,9 +352,8 @@ public GHPullRequest createPR( pullRequest.requestReviewers(reviewersGH); } - return pullRequest; - } catch (IOException | NoSuchAlgorithmException | InvalidKeySpecException e) { + } catch (Exception e) { String message = String.format("Error creating a PR in repository '%s': %s", repoFullPath, e.getMessage()); logger.error(message, e); @@ -345,6 +361,78 @@ public GHPullRequest createPR( } } + public enum AutoMergeType { + SQUASH, + MERGE + } + + /** + * This is implemented using the GraphQL API since, the Java client does not support this method + * yet. + */ + public void enableAutoMerge(GHPullRequest pullRequest, AutoMergeType autoMergeType) { + + logger.debug( + "Enable Auto Merge on PR: {}, with type: {}", + pullRequest.getNumber(), + autoMergeType.name()); + + ObjectMapper objectMapper = new ObjectMapper(); + + HttpResponse mutationResponse; + + try (HttpClient client = HttpClient.newHttpClient()) { + + String mutation = + """ + mutation($pullRequestId: ID!, $mergeMethod: PullRequestMergeMethod!) { + enablePullRequestAutoMerge(input: { + pullRequestId: $pullRequestId, + mergeMethod: $mergeMethod, + }) { + clientMutationId + } + } + """; + + ImmutableMap variables = + ImmutableMap.of( + "pullRequestId", pullRequest.getNodeId(), "mergeMethod", autoMergeType.name()); + + ImmutableMap payload = + ImmutableMap.of("query", mutation, "variables", variables); + + String jsonPayload = objectMapper.writeValueAsStringUnchecked(payload); + + String authToken = + getGithubAppInstallationToken(pullRequest.getRepository().getName()).getToken(); + + HttpRequest mutationRequest = + HttpRequest.newBuilder() + .uri(URI.create("https://api.github.com/graphql")) + .header("Authorization", "Bearer " + authToken) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .build(); + try { + mutationResponse = client.send(mutationRequest, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + if (mutationResponse.statusCode() != 200) { + throw new GithubException("Error enabling auto-merge: " + mutationResponse.body()); + } + + Map mutationResponseMap = + objectMapper.readValueUnchecked(mutationResponse.body(), new TypeReference<>() {}); + + if (mutationResponseMap.containsKey("errors")) { + throw new GithubException("Error enabling auto-merge: " + mutationResponse.body()); + } + } + } + public String getOwner() { return owner; } @@ -357,22 +445,25 @@ public String getEndpoint() { return endpoint; } - public GHAppInstallationToken getGithubAppInstallationToken(String repository) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { - if (githubAppInstallationToken == null - || githubAppInstallationToken.getExpiresAt().getTime() - <= System.currentTimeMillis() - 30000) { - // Existing installation token has less than 30 seconds before expiry, get new token - GitHub gitHub = - new GitHubBuilder() - .withEndpoint(getEndpoint()) - .withJwtToken(getGithubJWT(tokenTTL).getToken()) - .build(); - githubAppInstallationToken = - gitHub.getApp().getInstallationByRepository(owner, repository).createToken().create(); - } + public GHAppInstallationToken getGithubAppInstallationToken(String repository) { + try { + if (githubAppInstallationToken == null + || githubAppInstallationToken.getExpiresAt().getTime() + <= System.currentTimeMillis() - 30000) { + // Existing installation token has less than 30 seconds before expiry, get new token + GitHub gitHub = + new GitHubBuilder() + .withEndpoint(getEndpoint()) + .withJwtToken(getGithubJWT(tokenTTL).getToken()) + .build(); + githubAppInstallationToken = + gitHub.getApp().getInstallationByRepository(owner, repository).createToken().create(); + } - return githubAppInstallationToken; + return githubAppInstallationToken; + } catch (Exception e) { + throw new RuntimeException("Can't get the App installation token", e); + } } protected GitHub createGithubClient(String repository) @@ -383,7 +474,7 @@ protected GitHub createGithubClient(String repository) .build(); } - private String getRepositoryPath(String repository) { + String getRepositoryPath(String repository) { return owner != null && !owner.isEmpty() ? owner + "/" + repository : repository; } From 85981dde2bb8d79e2e522246242808e746774261 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 20 Sep 2024 00:02:12 -0700 Subject: [PATCH 069/105] Re-implement Phrase String file upload since SDK does not accept format options --- .../box/l10n/mojito/utils/OptionsParser.java | 10 + .../thirdparty/ThirdPartyTMSPhrase.java | 47 ++++- .../thirdparty/phrase/PhraseClient.java | 179 +++++++++++++++++- .../thirdparty/phrase/PhraseClientTest.java | 62 ++++-- 4 files changed, 270 insertions(+), 28 deletions(-) diff --git a/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java b/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java index 662a23a7c0..f18489c6dd 100644 --- a/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java +++ b/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java @@ -43,6 +43,16 @@ public Boolean getBoolean(String key, Boolean defaultValue) { return value; } + public String getString(String key, String defaultValue) { + String value = defaultValue; + + if (this.options.containsKey(key)) { + value = this.options.get(key); + } + + return value; + } + public void getInteger(String key, Consumer consumer) { if (this.options.containsKey(key)) { consumer.accept(Integer.valueOf(this.options.get(key))); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 9379d2befa..8b24bed708 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -154,13 +154,45 @@ public void push( String text = getFileContent(pluralSeparator, search, true, null); String tagForUpload = getTagForUpload(repository.getName()); - phraseClient.uploadAndWait( - projectId, - repository.getSourceLocale().getBcp47Tag(), - "xml", - repository.getName() + "-strings.xml", - text, - ImmutableList.of(tagForUpload)); + + OptionsParser optionsParser = new OptionsParser(options); + Boolean nativeUpload = optionsParser.getBoolean("nativeUpload", false); + Boolean nativeUploadUnescapeTags = optionsParser.getBoolean("nativeUploadUnescapeTags", true); + Boolean nativeUploadUnescapeLineBreaks = + optionsParser.getBoolean("nativeUploadUnescapeLineBreaks", false); + + if (nativeUpload) { + // convert_placeholder + // escape_linebreaks + // unescape_linebreaks + // unescape_tags + Map formatOptions = new HashMap<>(); + if (nativeUploadUnescapeTags) { + formatOptions.put("unescape_tags", "true"); + } + + if (nativeUploadUnescapeLineBreaks) { + formatOptions.put("unescape_linebreaks", "true"); + } + + phraseClient.nativeUploadAndWait( + projectId, + repository.getSourceLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + text, + ImmutableList.of(tagForUpload), + formatOptions.isEmpty() ? null : formatOptions); + } else { + phraseClient.uploadAndWait( + projectId, + repository.getSourceLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + text, + ImmutableList.of(tagForUpload), + null); + } removeUnusedKeysAndTags(projectId, repository.getName(), tagForUpload); } @@ -472,6 +504,7 @@ public void pushTranslations( "xml", repository.getName() + "-strings.xml", fileContent, + null, null); } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java index ddf6181be4..4bb365cdf9 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -5,6 +5,7 @@ import static com.box.l10n.mojito.io.Files.write; import com.box.l10n.mojito.json.ObjectMapper; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableSet; import com.phrase.client.ApiClient; import com.phrase.client.ApiException; @@ -12,10 +13,17 @@ import com.phrase.client.api.LocalesApi; import com.phrase.client.api.TagsApi; import com.phrase.client.api.UploadsApi; +import com.phrase.client.auth.ApiKeyAuth; import com.phrase.client.model.Tag; import com.phrase.client.model.TranslationKey; import com.phrase.client.model.Upload; import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; @@ -23,6 +31,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; @@ -48,16 +57,33 @@ public PhraseClient(ApiClient apiClient) { Retry.backoff(5, Duration.ofMillis(500)).maxBackoff(Duration.ofSeconds(5)); } + public Upload nativeUploadAndWait( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + Map formatOptions) { + + String uploadId = + nativeUploadCreateFileWithRetry( + projectId, localeId, fileFormat, fileName, fileContent, tags, formatOptions); + return waitForUploadToFinish(projectId, uploadId); + } + public Upload uploadAndWait( String projectId, String localeId, String fileFormat, String fileName, String fileContent, - List tags) { + List tags, + String formatOptions) { String uploadId = - uploadCreateFile(projectId, localeId, fileFormat, fileName, fileContent, tags); + uploadCreateFile( + projectId, localeId, fileFormat, fileName, fileContent, tags, formatOptions); return waitForUploadToFinish(projectId, uploadId); } @@ -101,7 +127,8 @@ String uploadCreateFile( String fileFormat, String fileName, String fileContent, - List tags) { + List tags, + String formatOptions) { Path tmpWorkingDirectory = null; @@ -126,7 +153,8 @@ String uploadCreateFile( write(fileToUpload, fileContent); Upload upload = - uploadsApiUploadCreateWithRetry(projectId, localeId, fileFormat, tags, fileToUpload); + uploadsApiUploadCreateWithRetry( + projectId, localeId, fileFormat, tags, fileToUpload, formatOptions); return upload.getId(); } finally { @@ -136,8 +164,147 @@ String uploadCreateFile( } } + public String nativeUploadCreateFileWithRetry( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + Map formatOptions) { + + logger.info( + "nativeUploadCreateFile: projectId: {}, localeId: {}, fileName: {}, tags: {}", + projectId, + localeId, + fileName, + tags); + + return Mono.fromCallable( + () -> + nativeUploadCreateFile( + projectId, localeId, fileFormat, fileName, fileContent, tags, formatOptions)) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to uploadCreate to Phrase, file: %s, project id: %s" + .formatted(fileName, projectId)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error in UploadCreate from Phrase, file: %s, project id: %s" + .formatted(fileName, projectId))) + .block(); + } + + /** + * The official SDK does not support format_options properly, so adding a replacement method base + * on pure Java client + */ + public String nativeUploadCreateFile( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + Map formatOptions) { + + String urlString = String.format("%s/projects/%s/uploads", apiClient.getBasePath(), projectId); + String boundary = UUID.randomUUID().toString(); + final String LINE_FEED = "\r\n"; + + StringBuilder multipartBody = new StringBuilder(); + + multipartBody.append("--").append(boundary).append(LINE_FEED); + multipartBody + .append("Content-Disposition: form-data; name=\"file\"; filename=\"") + .append(fileName) + .append("\"") + .append(LINE_FEED); + multipartBody.append("Content-Type: application/xml").append(LINE_FEED); + multipartBody.append(LINE_FEED); + multipartBody.append(fileContent).append(LINE_FEED); + + addFormField(multipartBody, boundary, "locale_id", localeId); + addFormField(multipartBody, boundary, "file_format", fileFormat); + addFormField(multipartBody, boundary, "update_translations", "true"); + addFormField(multipartBody, boundary, "update_descriptions", "true"); + + if (tags != null) { + String tagsString = String.join(",", tags); + addFormField(multipartBody, boundary, "tags", tagsString); + } + + if (formatOptions != null) { + for (Map.Entry e : formatOptions.entrySet()) { + addFormField( + multipartBody, boundary, "format_options[%s]".formatted(e.getKey()), e.getValue()); + } + } + multipartBody.append("--").append(boundary).append("--").append(LINE_FEED); + + String token = ((ApiKeyAuth) apiClient.getAuthentication("Token")).getApiKey(); + + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(urlString)) + .header("Authorization", "token " + token) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST( + HttpRequest.BodyPublishers.ofString( + multipartBody.toString(), StandardCharsets.UTF_8)) + .build(); + + HttpResponse response; + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + int statusCode = response.statusCode(); + String responseBody = response.body(); + + if (statusCode == 201) { + JsonNode rootNode = new ObjectMapper().readTreeUnchecked(responseBody); + return rootNode.path("id").asText(); + } else { + throw new RuntimeException("Server returned status code " + statusCode + ": " + responseBody); + } + } + + /** + * Helper method to add a form field to the multipart body. + * + * @param builder The StringBuilder for the multipart body. + * @param boundary The boundary string. + * @param name The name of the form field. + * @param value The value of the form field. + */ + private static void addFormField( + StringBuilder builder, String boundary, String name, String value) { + String LINE_FEED = "\r\n"; + builder.append("--").append(boundary).append(LINE_FEED); + builder + .append("Content-Disposition: form-data; name=\"") + .append(name) + .append("\"") + .append(LINE_FEED); + builder.append(LINE_FEED); + builder.append(value).append(LINE_FEED); + } + Upload uploadsApiUploadCreateWithRetry( - String projectId, String localeId, String fileFormat, List tags, Path fileToUpload) { + String projectId, + String localeId, + String fileFormat, + List tags, + Path fileToUpload, + String formatOptions) { return Mono.fromCallable( () -> @@ -157,7 +324,7 @@ Upload uploadsApiUploadCreateWithRetry( null, null, null, - null, + formatOptions, null, null, null)) diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java index 2c70bdda0a..5ed24ba05e 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -6,7 +6,10 @@ import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.phrase.client.model.Tag; +import com.phrase.client.model.Upload; +import java.io.IOException; import java.time.ZonedDateTime; import java.util.List; import java.util.Objects; @@ -41,6 +44,8 @@ public class PhraseClientTest { @Value("${test.phrase-client.projectId:}") String testProjectId; + @Autowired private PhraseClientConfig phraseClientConfig; + @Test public void testRemoveTag() { String tagForUpload = "push_2024_06_10_07_17_00_089_122"; @@ -88,57 +93,84 @@ public void testParallelDownload() { } @Test - public void test() { + public void test() throws IOException, InterruptedException { Assume.assumeNotNull(testProjectId); - for (int i = 0; i < 2; i++) { + for (int i = 1; i < 2; i++) { AndroidStringDocument source = new AndroidStringDocument(); source.addSingular( - new AndroidSingular(11L, i + "string1", "some link to a page", "test comment1")); + new AndroidSingular(11L, i + "-string1", "some link to a page", "test comment1")); source.addSingular( new AndroidSingular( 12L, - i + "string2", + i + "-string2", "some link to a page", "test comment2")); source.addSingular( new AndroidSingular( 13L, - i + "string3", + i + "-string3", "If that is your IP address click here to unblock it.", "test comment2")); source.addSingular( new AndroidSingular( 14L, - i + "string4", + i + "-string4", "If that is your IP address click here to unblock it.", "test comment2")); source.addSingular( new AndroidSingular( 15L, - i + "string5", + i + "-string5", "If that is your IP address click here to unblock it.", "test comment2")); source.addSingular( new AndroidSingular( 16L, - i + "string6", + i + "-string6", "visit your settings", "test comment2")); + source.addSingular( + new AndroidSingular(17L, i + "-string7", "a string\nwith return line", "test comment2")); + + source.addSingular( + new AndroidSingular( + 18L, i + "-string9", "a string\n\nwith two return line", "test comment2")); + + source.addSingular( + new AndroidSingular( + 19L, i + "-string10", "a string & and the & escape", "test comment2")); + + source.addSingular( + new AndroidSingular(20L, i + "-string11", "simple tag", "test comment2")); + String androidFile = new AndroidStringDocumentWriter(source, i % 2 == 0).toText(); System.out.println(androidFile); + Upload upload = + phraseClient.nativeUploadAndWait( + testProjectId, + "en", + "xml", + "strings.xml", + androidFile, + ImmutableList.of(i % 2 == 0 ? "test-escaping" : "test-no-escaping"), + ImmutableMap.of("unescape_tags", "true")); + + // Upload upload = + // phraseClient.uploadAndWait( + // testProjectId, + // "en", + // "xml", + // "strings.xml", + // androidFile, + // ImmutableList.of(i % 2 == 0 ? "test-escaping" : "test-no-escaping"), + // null); - phraseClient.uploadAndWait( - testProjectId, - "en", - "xml", - "strings.xml", - androidFile, - ImmutableList.of(i % 2 == 0 ? "test-escaping" : "test-no-escaping")); + System.out.println(upload); String s = phraseClient.localeDownload( From b030aa560966939f10f19c7dffb22039d3461f6d Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 23 Sep 2024 14:39:54 -0700 Subject: [PATCH 070/105] Add an option to GithubCreatePRCommand to add labels A list of labels can be added when the PR is created --- .../cli/command/GithubCreatePRCommand.java | 18 +++++++++++++++--- .../box/l10n/mojito/github/GithubClient.java | 10 ++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java index 936691750c..6bf6a14ceb 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java @@ -83,6 +83,13 @@ public class GithubCreatePRCommand extends Command { description = "Enable auto-merge with the specified method") EnableAutoMergeType enableAutoMerge = EnableAutoMergeType.NONE; + @Parameter( + names = {"--labels"}, + required = false, + variableArity = true, + description = "The PR labels") + List labels; + enum EnableAutoMergeType { SQUASH, MERGE, @@ -98,12 +105,17 @@ public boolean shouldShowInCommandList() { protected void execute() throws CommandException { try { - GHPullRequest pr = - githubClients.getClient(owner).createPR(repository, title, head, base, body, reviewers); + GithubClient githubClient = githubClients.getClient(owner); + + GHPullRequest pr = githubClient.createPR(repository, title, head, base, body, reviewers); + consoleWriter.a("PR created: ").fg(Ansi.Color.CYAN).a(pr.getHtmlUrl().toString()).println(); if (!EnableAutoMergeType.NONE.equals(enableAutoMerge)) { - githubClients.getClient(owner).enableAutoMerge(pr, GithubClient.AutoMergeType.SQUASH); + githubClient.enableAutoMerge(pr, GithubClient.AutoMergeType.SQUASH); } + + githubClient.addLabelsToPR(pr, labels); + } catch (GithubException e) { throw new CommandException(e); } diff --git a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java index 0b2e5036cd..2414b791a7 100644 --- a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java +++ b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java @@ -433,6 +433,16 @@ public void enableAutoMerge(GHPullRequest pullRequest, AutoMergeType autoMergeTy } } + public void addLabelsToPR(GHPullRequest pullRequest, List labels) { + if (labels != null) { + try { + pullRequest.addLabels(labels.toArray(new String[0])); + } catch (IOException e) { + throw new GithubException("Can't add labels to PR", e); + } + } + } + public String getOwner() { return owner; } From 1fa4ac515bd53a4e56f4bbd82698e37ee8e11e71 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 24 Sep 2024 21:22:14 -0700 Subject: [PATCH 071/105] MacStringsFilter remove untranslated test Add test for current behavior when using REMOVED_UNTRANSLATED. Note it just puts an empty value instead of removing the entry, which is not usable. --- .../l10n/mojito/service/tm/TMServiceTest.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java index fd3601e085..88d4f7a2e7 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java @@ -3646,6 +3646,100 @@ public void testLocalizePoPluralAr() throws Exception { assertEquals(forImport, localizedAsset); } + @Test + public void testLocalizeMacStringsRemoveUntranslated() throws Exception { + + Repository repo = repositoryService.createRepository(testIdWatcher.getEntityName("repository")); + RepositoryLocale repoLocale; + try { + repoLocale = repositoryService.addRepositoryLocale(repo, "ja-JP"); + } catch (RepositoryLocaleCreationException e) { + throw new RuntimeException(e); + } + + String assetContent = + """ + /* comment 1 */ + "key1" = "value1"; + + /* comment 2 */ + "key2" = "value2"; + """; + + String expectedLocalizedAsset = + """ + /* comment 1 */ + "key1" = ""; + + /* comment 2 */ + "key2" = ""; + """; + + asset = assetService.createAssetWithContent(repo.getId(), "Localizable.strings", assetContent); + asset = assetRepository.findById(asset.getId()).orElse(null); + assetId = asset.getId(); + tmId = repo.getTm().getId(); + + PollableFuture assetResult = + assetService.addOrUpdateAssetAndProcessIfNeeded( + repo.getId(), asset.getPath(), assetContent, false, null, null, null, null, null, null); + try { + pollableTaskService.waitForPollableTask(assetResult.getPollableTask().getId()); + } catch (PollableTaskException | InterruptedException e) { + throw new RuntimeException(e); + } + assetResult.get(); + + String localizedAsset = + tmService.generateLocalized( + asset, + assetContent, + repoLocale, + "ja-JP", + null, + null, + Status.ALL, + InheritanceMode.REMOVE_UNTRANSLATED, + null); + logger.debug("localized=\n{}\nEOL", localizedAsset); + assertEquals(expectedLocalizedAsset, localizedAsset); + + String forImport = + """ + /* comment 1 */ + "key1" = "value1-jp"; + + /* comment 2 */ + "key2" = "value2-jp"; + """; + + logger.debug("formimport=\n{}", forImport); + + tmService + .importLocalizedAssetAsync( + assetId, + forImport, + repoLocale.getLocale().getId(), + StatusForEqualTarget.TRANSLATION_NEEDED, + null, + null) + .get(); + + localizedAsset = + tmService.generateLocalized( + asset, + assetContent, + repoLocale, + "ja-JP", + null, + null, + Status.ALL, + InheritanceMode.REMOVE_UNTRANSLATED, + null); + logger.info("localized after import=\n{}", localizedAsset); + assertEquals(forImport, localizedAsset); + } + @Test public void testLocalizeMacStringsNamessNotEnclosedInDoubleQuotes() throws Exception { From da916f43beaf3081f6e352c934e9f82018e77f33 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 25 Sep 2024 00:54:15 -0700 Subject: [PATCH 072/105] Refactor filter post processing --- .../mojito/okapi/filters/AndroidFilter.java | 58 +++++++++++++------ .../l10n/mojito/okapi/filters/JSONFilter.java | 16 ++++- .../l10n/mojito/okapi/filters/POFilter.java | 14 ++++- ...FilterEventsToInMemoryRawDocumentStep.java | 6 +- ...utputDocumentPostProcessingAnnotation.java | 37 +++++++----- .../okapi/filters/AndroidFilterTest.java | 30 +++++----- .../box/l10n/mojito/okapi/TranslateStep.java | 10 ++-- .../AndroidStringDocumentMapperTest.java | 12 +++- .../l10n/mojito/service/tm/TMServiceTest.java | 2 +- 9 files changed, 120 insertions(+), 65 deletions(-) diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java index 656b059381..e3de012438 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java @@ -2,6 +2,7 @@ import com.box.l10n.mojito.okapi.TextUnitUtils; import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation.OutputDocumentPostProcessorBase; import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; @@ -117,6 +118,12 @@ public List getConfigurations() { int postProcessIndent = 2; + /** + * Historically there was no processing if no option was passed. We want to keep this behavior to + * avoid output change. + */ + boolean shouldApplyPostProcessingRemoveUntranslatedExcluded = false; + @Override public void open(RawDocument input) { super.open(input); @@ -126,17 +133,14 @@ public void open(RawDocument input) { input.setAnnotation( new RemoveUntranslatedStategyAnnotation( RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); - // The post processing is optionally enable based on the removed description option. Contrary to - // a filter like - // JsonFilter the post-processing where the post processing is always disable by default, and - // then activated - // in the TranslateStep input.setAnnotation( new OutputDocumentPostProcessingAnnotation( - new AndroidFilePostProcessing( - removeDescription, postProcessIndent, removeTranslatableFalse) - ::execute, - removeDescription)); + new AndroidFilePostProcessor( + false, + removeDescription, + postProcessIndent, + removeTranslatableFalse, + shouldApplyPostProcessingRemoveUntranslatedExcluded))); } void applyFilterOptions(RawDocument input) { @@ -157,18 +161,21 @@ void applyFilterOptions(RawDocument input) { REMOVE_DESCRIPTION, b -> { removeDescription = b; + shouldApplyPostProcessingRemoveUntranslatedExcluded = true; }); filterOptions.getBoolean( POST_PROCESS_REMOVE_TRANSLATABLE_FALSE, b -> { removeTranslatableFalse = b; + shouldApplyPostProcessingRemoveUntranslatedExcluded = true; }); filterOptions.getInteger( POST_PROCESS_INDENT, i -> { postProcessIndent = i; + shouldApplyPostProcessingRemoveUntranslatedExcluded = true; }); } } @@ -434,22 +441,32 @@ void updateFormInSkeleton(ITextUnit textUnit) { } } - static class AndroidFilePostProcessing { + static class AndroidFilePostProcessor extends OutputDocumentPostProcessorBase { static final String DESCRIPTION_ATTRIBUTE = "description"; boolean removeDescription; boolean removeTranslatableFalse; int indent; - - AndroidFilePostProcessing( - boolean removeDescription, int indent, boolean removeTranslatableFalse) { + boolean shouldApplyPostProcessingRemoveUntranslatedExcluded; + + AndroidFilePostProcessor( + boolean removeUntranslated, + boolean removeDescription, + int indent, + boolean removeTranslatableFalse, + boolean shouldApplyPostProcessingRemoveUntranslatedExcluded) { + this.setRemoveUntranslated(removeUntranslated); this.removeDescription = removeDescription; this.removeTranslatableFalse = removeTranslatableFalse; this.indent = indent; + this.shouldApplyPostProcessingRemoveUntranslatedExcluded = + shouldApplyPostProcessingRemoveUntranslatedExcluded; } - String execute(String xmlContent) { + public String execute(String xmlContent) { - if (xmlContent == null || xmlContent.isBlank()) { + if (xmlContent == null + || xmlContent.isBlank() + || (!shouldApplyPostProcessingRemoveUntranslatedExcluded && !removeTranslatableFalse)) { return xmlContent; } @@ -464,9 +481,10 @@ String execute(String xmlContent) { Node node = stringElements.item(i); if (node.getNodeType() == Node.ELEMENT_NODE) { Element element = (Element) node; - if (element - .getTextContent() - .equals(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { + if (hasRemoveUntranslated() + && element + .getTextContent() + .equals(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { element.getParentNode().removeChild(element); i--; } @@ -487,7 +505,9 @@ String execute(String xmlContent) { for (int j = 0; j < items.getLength(); j++) { Element item = (Element) items.item(j); - if (item.getTextContent().equals(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { + if (hasRemoveUntranslated() + && item.getTextContent() + .equals(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { item.getParentNode().removeChild(item); j--; } else { diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java index e8f67297c4..0c9bec5d81 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java @@ -2,6 +2,7 @@ import com.box.l10n.mojito.json.JsonObjectRemoverByValue; import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation.OutputDocumentPostProcessorBase; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.HashSet; @@ -91,10 +92,19 @@ public void open(RawDocument input) { input.setAnnotation( new RemoveUntranslatedStategyAnnotation( RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); - // Post processing is disable for now, it will be enabled by the TranslateStep if there are - // actual text unit to remove input.setAnnotation( - new OutputDocumentPostProcessingAnnotation(JSONFilter::removeUntranslated, false)); + new OutputDocumentPostProcessingAnnotation( + new OutputDocumentPostProcessorBase() { + @Override + public String execute(String content) { + + if (hasRemoveUntranslated()) { + content = removeUntranslated(content); + } + + return content; + } + })); } static String removeUntranslated(String content) { diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/POFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/POFilter.java index e65bf342a9..73f8670851 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/POFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/POFilter.java @@ -106,10 +106,18 @@ public void open(RawDocument input) { input.setAnnotation( new RemoveUntranslatedStategyAnnotation( RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); - // Post processing is disable for now, it will be enabled by the TranslsateStep if there are - // actual text unit to remove + input.setAnnotation( - new OutputDocumentPostProcessingAnnotation(POFilter::removeUntranslated, false)); + new OutputDocumentPostProcessingAnnotation( + new OutputDocumentPostProcessingAnnotation.OutputDocumentPostProcessorBase() { + @Override + public String execute(String content) { + if (hasRemoveUntranslated()) { + content = removeUntranslated(content); + } + return content; + } + })); } @Override diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/steps/FilterEventsToInMemoryRawDocumentStep.java b/common/src/main/java/com/box/l10n/mojito/okapi/steps/FilterEventsToInMemoryRawDocumentStep.java index 5c3c93c401..7baa13477d 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/steps/FilterEventsToInMemoryRawDocumentStep.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/steps/FilterEventsToInMemoryRawDocumentStep.java @@ -126,9 +126,11 @@ String getOutputDocument() throws UnsupportedEncodingException { OutputDocumentPostProcessingAnnotation outputDocumentPostProcessingAnnotation = rawDocument.getAnnotation(OutputDocumentPostProcessingAnnotation.class); if (outputDocumentPostProcessingAnnotation != null - && outputDocumentPostProcessingAnnotation.isEnabled()) { + && outputDocumentPostProcessingAnnotation.getOutputDocumentPostProcessor() != null) { outputDocument = - outputDocumentPostProcessingAnnotation.getPostProcessing().apply(outputDocument); + outputDocumentPostProcessingAnnotation + .getOutputDocumentPostProcessor() + .execute(outputDocument); } return outputDocument; diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/steps/OutputDocumentPostProcessingAnnotation.java b/common/src/main/java/com/box/l10n/mojito/okapi/steps/OutputDocumentPostProcessingAnnotation.java index 3f8b59b906..02366aae38 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/steps/OutputDocumentPostProcessingAnnotation.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/steps/OutputDocumentPostProcessingAnnotation.java @@ -1,31 +1,38 @@ package com.box.l10n.mojito.okapi.steps; -import java.util.function.Function; import net.sf.okapi.common.annotation.IAnnotation; public class OutputDocumentPostProcessingAnnotation implements IAnnotation { - Function postProcessing; - boolean enabled; + OutputDocumentPostProcessor outputDocumentPostProcessor; public OutputDocumentPostProcessingAnnotation( - Function postProcessing, boolean enabled) { - this.postProcessing = postProcessing; - this.enabled = enabled; + OutputDocumentPostProcessor outputDocumentPostProcessor) { + this.outputDocumentPostProcessor = outputDocumentPostProcessor; } - public Function getPostProcessing() { - return postProcessing; + public OutputDocumentPostProcessor getOutputDocumentPostProcessor() { + return outputDocumentPostProcessor; } - public void setPostProcessing(Function postProcessing) { - this.postProcessing = postProcessing; - } + public interface OutputDocumentPostProcessor { + String execute(String content); + + boolean hasRemoveUntranslated(); - public boolean isEnabled() { - return enabled; + void setRemoveUntranslated(boolean removeUntranslated); } - public void setEnabled(boolean enabled) { - this.enabled = enabled; + public abstract static class OutputDocumentPostProcessorBase + implements OutputDocumentPostProcessor { + boolean removeUntranslated = false; + + public boolean hasRemoveUntranslated() { + return removeUntranslated; + } + + @Override + public void setRemoveUntranslated(boolean removeUntranslated) { + this.removeUntranslated = removeUntranslated; + } } } diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java index 99dd4e0d08..af14a80365 100644 --- a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java +++ b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java @@ -112,8 +112,8 @@ void testUnescaping(String input, String expected) { @Test public void testPostProcessingKeepDescription() { - AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = - new AndroidFilter.AndroidFilePostProcessing(false, 2, false); + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, false, 2, false, false); String input = """ @@ -135,7 +135,7 @@ public void testPostProcessingKeepDescription() { """; - String output = androidFilePostProcessing.execute(input); + String output = androidFilePostProcessor.execute(input); String expected = """ @@ -156,8 +156,8 @@ public void testPostProcessingKeepDescription() { @Test public void testPostProcessingRemoveDescription() { - AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = - new AndroidFilter.AndroidFilePostProcessing(true, 2, false); + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, false, false); String input = """ @@ -179,7 +179,7 @@ public void testPostProcessingRemoveDescription() { """; - String output = androidFilePostProcessing.execute(input); + String output = androidFilePostProcessor.execute(input); String expected = """ @@ -200,18 +200,18 @@ public void testPostProcessingRemoveDescription() { @Test public void testPostProcessingEmptyFile() { - AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = - new AndroidFilter.AndroidFilePostProcessing(true, 2, false); + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, false, false); String input = ""; - String output = androidFilePostProcessing.execute(input); + String output = androidFilePostProcessor.execute(input); String expected = ""; assertEquals(expected, output); } @Test public void testPostProcessingNoProlog() { - AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = - new AndroidFilter.AndroidFilePostProcessing(true, 2, false); + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, false, false); String input = """ @@ -219,7 +219,7 @@ public void testPostProcessingNoProlog() { @#$untranslated$#@ """; - String output = androidFilePostProcessing.execute(input); + String output = androidFilePostProcessor.execute(input); String expected = """ @@ -231,8 +231,8 @@ public void testPostProcessingNoProlog() { @Test public void testPostProcessingRemoveTranslatableFalse() { - AndroidFilter.AndroidFilePostProcessing androidFilePostProcessing = - new AndroidFilter.AndroidFilePostProcessing(true, 2, true); + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); String input = """ @@ -254,7 +254,7 @@ public void testPostProcessingRemoveTranslatableFalse() { """; - String output = androidFilePostProcessing.execute(input); + String output = androidFilePostProcessor.execute(input); String expected = """ diff --git a/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java b/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java index cd63cd5657..cf9ec7afd4 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java +++ b/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java @@ -43,7 +43,6 @@ public class TranslateStep extends AbstractMd5ComputationStep { Status status; InheritanceMode inheritanceMode; RawDocument rawDocument; - boolean rawDocumentProcessingEnabled = false; boolean saveUsedTmTextUnitVariantIds = false; /** @@ -147,7 +146,7 @@ protected Event handleTextUnit(Event event) { textUnit.setTarget( targetLocale, new TextContainer(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)); - enableOutputDocumentPostProcessing(); + setOutputDocumentPostProcessingRemoveUntranslated(); break; } @@ -195,12 +194,11 @@ protected Event handleDocumentPart(Event event) { return event; } - void enableOutputDocumentPostProcessing() { + void setOutputDocumentPostProcessingRemoveUntranslated() { OutputDocumentPostProcessingAnnotation annotation = rawDocument.getAnnotation(OutputDocumentPostProcessingAnnotation.class); - if (annotation != null && !rawDocumentProcessingEnabled) { - annotation.setEnabled(true); - rawDocumentProcessingEnabled = true; + if (annotation != null && annotation.getOutputDocumentPostProcessor() != null) { + annotation.getOutputDocumentPostProcessor().setRemoveUntranslated(true); } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java index 0148b53931..aff676de04 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java @@ -924,13 +924,23 @@ public void testReadingTextUnitsFromFile() throws Exception { assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("show_options")) .extracting("name", "target", "comment") - .containsOnly(tuple("show_options", "Mer \" dela", "Options")); + .containsOnly(tuple("show_options", "Mer \" dela", null)); assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("without_comment")) .extracting("target", "comment") .containsOnly(tuple("ei ' kommentteja", null)); + assertThat(textUnits) + .filteredOn(tu -> tu.getName().equalsIgnoreCase("show_options2")) + .extracting("name", "target", "comment") + .containsOnly(tuple("show_options2", "Mer \\\" dela", "Options")); + + assertThat(textUnits) + .filteredOn(tu -> tu.getName().equalsIgnoreCase("without_comment2")) + .extracting("target", "comment") + .containsOnly(tuple("ei \\' kommentteja", null)); + assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("line_break")) .extracting("target", "comment") diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java index 88d4f7a2e7..67e8b8d1e0 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java @@ -2143,7 +2143,7 @@ public void testLocalizeAndroidStringsRemoveUntranslatedSingleItem() throws Exce assetResult.get(); String expectedLocalized = - "\n" + "\n" + "" + "\n"; From c7dab1799f2797bab3b22fe5926c2a7f0a886629 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 25 Sep 2024 16:50:44 -0700 Subject: [PATCH 073/105] Add post-processing to MacStringsFilter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the following features: - Properly Remove Untranslated Strings: Previously, the filter would add a blank string for untranslated entries. Now, we will remove the entire entry if it’s untranslated. - Remove Comments: Instead of using an external simple-file-editor command, we will incorporate comment removal directly into the MacStringsFilter. --- .../okapi/filters/MacStringsFilter.java | 123 +++++++ .../okapi/filters/MacStringsFilterTest.java | 306 ++++++++++++++++++ .../l10n/mojito/service/tm/TMServiceTest.java | 9 +- 3 files changed, 430 insertions(+), 8 deletions(-) create mode 100644 common/src/test/java/com/box/l10n/mojito/okapi/filters/MacStringsFilterTest.java diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/MacStringsFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/MacStringsFilter.java index 5c86aa0adf..ed18693d61 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/MacStringsFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/MacStringsFilter.java @@ -1,11 +1,17 @@ package com.box.l10n.mojito.okapi.filters; import com.box.l10n.mojito.okapi.ExtractUsagesFromTextUnitComments; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation.OutputDocumentPostProcessorBase; +import com.google.common.base.Strings; import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import net.sf.okapi.common.Event; import net.sf.okapi.common.EventType; import net.sf.okapi.common.filters.FilterConfiguration; +import net.sf.okapi.common.resource.RawDocument; import net.sf.okapi.common.resource.TextUnit; import net.sf.okapi.filters.regex.RegexFilter; import org.springframework.beans.factory.annotation.Autowired; @@ -18,14 +24,41 @@ public class MacStringsFilter extends RegexEscapeDoubleQuoteFilter { public static final String FILTER_CONFIG_ID = "okf_regex@mojito"; + private static final String REMOVE_COMMENTS = "removeComment"; @Autowired ExtractUsagesFromTextUnitComments extractUsagesFromTextUnitComments; + boolean removeComment = false; + @Override public String getName() { return FILTER_CONFIG_ID; } + @Override + public void open(RawDocument input) { + super.open(input); + applyFilterOptions(input); + input.setAnnotation( + new RemoveUntranslatedStategyAnnotation( + RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); + input.setAnnotation( + new OutputDocumentPostProcessingAnnotation( + new MacStringsFilterPostProcessor(removeComment))); + } + + void applyFilterOptions(RawDocument input) { + FilterOptions filterOptions = input.getAnnotation(FilterOptions.class); + + if (filterOptions != null) { + filterOptions.getBoolean( + REMOVE_COMMENTS, + b -> { + removeComment = b; + }); + } + } + @Override public List getConfigurations() { List list = new ArrayList<>(); @@ -51,4 +84,94 @@ public Event next() { return event; } + + static class MacStringsFilterPostProcessor extends OutputDocumentPostProcessorBase { + + static final Pattern COMMENT_KEY_VALUE_PATTERN = + Pattern.compile("(?s)(/\\*.*?\\*/\\s*)?(\".*?\"\\s*=\\s*\".*?\";)"); + static final Pattern COMMENT_PATTERN = Pattern.compile("(?s)/\\*.*?\\*/"); + + boolean removeComment = false; + + public MacStringsFilterPostProcessor(boolean removeComment) { + this.removeComment = removeComment; + } + + public String execute(String fileContent) { + String result = fileContent; + + if (hasRemoveUntranslated()) { + result = removeUntranslated(fileContent); + } + + if (removeComment) { + result = removeComments(fileContent); + } + + return result; + } + + public static String removeUntranslated(String fileContent) { + + Matcher matcher = COMMENT_KEY_VALUE_PATTERN.matcher(fileContent); + StringBuilder stringBuilder = new StringBuilder(); + + while (matcher.find()) { + String entry = matcher.group(); + if (entry.contains(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { + matcher.appendReplacement(stringBuilder, ""); + } else { + matcher.appendReplacement(stringBuilder, Matcher.quoteReplacement(entry)); + } + } + matcher.appendTail(stringBuilder); + + String result = collapseBlankLines(stringBuilder.toString()); + result = ensureEndLineAsInInput(result, fileContent); + + return result; + } + + public static String removeComments(String fileContent) { + Matcher matcher = COMMENT_PATTERN.matcher(fileContent); + String result = matcher.replaceAll(""); + result = collapseBlankLines(result); + result = ensureEndLineAsInInput(result, fileContent); + return result; + } + + static String ensureEndLineAsInInput(String output, String input) { + String result = output.trim(); + + if (input.isBlank()) { + return "\n".repeat(input.length()) + result; + } else { + + int leadingNewlines = 0; + int trailingNewlines = 0; + int length = input.length(); + + int index = 0; + while (index < length && input.charAt(index) == '\n') { + leadingNewlines++; + index++; + } + + index = length - 1; + while (index >= 0 && input.charAt(index) == '\n') { + trailingNewlines++; + index--; + } + return "\n".repeat(leadingNewlines) + result + "\n".repeat(trailingNewlines); + } + } + + static String collapseBlankLines(String input) { + String result = input; + if (!Strings.isNullOrEmpty(input)) { + result = input.replaceAll("(?m)^(\\s*\\n){2,}", "\n"); + } + return result; + } + } } diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/MacStringsFilterTest.java b/common/src/test/java/com/box/l10n/mojito/okapi/filters/MacStringsFilterTest.java new file mode 100644 index 0000000000..c74c1d5a48 --- /dev/null +++ b/common/src/test/java/com/box/l10n/mojito/okapi/filters/MacStringsFilterTest.java @@ -0,0 +1,306 @@ +package com.box.l10n.mojito.okapi.filters; + +import static com.box.l10n.mojito.okapi.filters.MacStringsFilter.MacStringsFilterPostProcessor.collapseBlankLines; +import static com.box.l10n.mojito.okapi.filters.MacStringsFilter.MacStringsFilterPostProcessor.ensureEndLineAsInInput; +import static com.box.l10n.mojito.okapi.filters.MacStringsFilter.MacStringsFilterPostProcessor.removeComments; +import static com.box.l10n.mojito.okapi.filters.MacStringsFilter.MacStringsFilterPostProcessor.removeUntranslated; +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class MacStringsFilterTest { + + @Test + public void testRemoveUntranslated_NoUntranslatedEntries() { + String input = + """ + /* Welcome message */ + "welcome_message" = "Welcome to our app!"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String output = + """ + /* Welcome message */ + "welcome_message" = "Welcome to our app!"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_WithUntranslatedEntry() { + String input = + """ + /* Welcome message */ + "welcome_message" = "@#$untranslated$#@"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String output = + """ + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_UntranslatedEntryWithoutComment() { + String input = + """ + "welcome_message" = "@#$untranslated$#@"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + + String output = + """ + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_MultipleUntranslatedEntries() { + String input = + """ + /* Welcome message */ + "welcome_message" = "@#$untranslated$#@"; + + /* Farewell message */ + "farewell_message" = "@#$untranslated$#@"; + + /* Info message */ + "info_message" = "Information available."; + """; + String output = + """ + /* Info message */ + "info_message" = "Information available."; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_EscapedCharacters() { + String input = + """ + /* Message with escaped characters */ + "escaped_message" = "Line1\\nLine2\\tTabbed"; + + /* Untranslated message */ + "untranslated_key" = "@#$untranslated$#@"; + """; + String output = + """ + /* Message with escaped characters */ + "escaped_message" = "Line1\\nLine2\\tTabbed"; + """; + String result = removeUntranslated(input); + System.out.println("[" + input + "]"); + System.out.println("[" + result + "]"); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_MultilineValue() { + String input = + """ + /* Multiline message */ + "multiline_message" = "This is a long message that spans multiple lines \\ + in the .strings file, but will be rendered as a single line \\ + when displayed in the application."; + + /* Untranslated message */ + "untranslated_key" = "@#$untranslated$#@"; + """; + String output = + """ + /* Multiline message */ + "multiline_message" = "This is a long message that spans multiple lines \\ + in the .strings file, but will be rendered as a single line \\ + when displayed in the application."; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_OnlyUntranslatedEntries() { + String input = + """ + /* Untranslated message */ + "untranslated_key1" = "@#$untranslated$#@"; + + /* Untranslated message */ + "untranslated_key2" = "@#$untranslated$#@"; + """; + String output = "\n"; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_EmptyFile() { + String input = ""; + String output = ""; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_UntranslatedAtEnd() { + String input = + """ + /* Welcome message */ + "welcome_message" = "Welcome!"; + + /* Untranslated message */ + "untranslated_key" = "@#$untranslated$#@"; + """; + String output = + """ + /* Welcome message */ + "welcome_message" = "Welcome!"; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveComments_NoComments() { + String input = + """ + "welcome_message" = "Welcome to our app!"; + "farewell_message" = "Goodbye!"; + """; + String expected = + """ + "welcome_message" = "Welcome to our app!"; + "farewell_message" = "Goodbye!"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_WithBlockComments() { + String input = + """ + /* Welcome message */ + "welcome_message" = "Welcome to our app!"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String expected = + """ + "welcome_message" = "Welcome to our app!"; + + "farewell_message" = "Goodbye!"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_UntranslatedEntryWithComments() { + String input = + """ + /* Untranslated message */ + "untranslated_key" = "@#$untranslated$#@"; + """; + String expected = + """ + "untranslated_key" = "@#$untranslated$#@"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_EscapedCharacters() { + String input = + """ + /* Message with escaped characters */ + "escaped_message" = "Line1\\nLine2\\tTabbed"; + """; + String expected = + """ + "escaped_message" = "Line1\\nLine2\\tTabbed"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_EmptyFile() { + String input = ""; + String expected = ""; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_MultilineBlockComment() { + String input = + """ + /* Multiline + block comment */ + "message" = "Hello!"; + """; + String expected = + """ + "message" = "Hello!"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_NestedBlockComments() { + String input = + """ + /* Outer comment + /* Inner comment */ + */ + "message" = "Hello!"; + """; + String expected = + """ + */ + "message" = "Hello!"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testAddEndLineAsInInput() { + assertEquals("", ensureEndLineAsInInput("", "")); + assertEquals("\n", ensureEndLineAsInInput("", "\n")); + assertEquals("\n", ensureEndLineAsInInput("\n", "\n")); + assertEquals("", ensureEndLineAsInInput("\n", "")); + } + + @Test + public void testCollapseBlankLines() { + assertEquals("", collapseBlankLines("")); + assertEquals("\n", collapseBlankLines("\n")); + assertEquals("\n", collapseBlankLines("\n\n")); + assertEquals("\na\nb\n\nc\n\n", collapseBlankLines("\na\nb\n\nc\n\n")); + assertEquals("a\nb\n\nc\n\n", collapseBlankLines("a\nb\n\nc\n\n")); + assertEquals("\na\nb\n\nc", collapseBlankLines("\na\nb\n\nc")); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java index 67e8b8d1e0..5ee946df5c 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java @@ -3666,14 +3666,7 @@ public void testLocalizeMacStringsRemoveUntranslated() throws Exception { "key2" = "value2"; """; - String expectedLocalizedAsset = - """ - /* comment 1 */ - "key1" = ""; - - /* comment 2 */ - "key2" = ""; - """; + String expectedLocalizedAsset = "\n"; asset = assetService.createAssetWithContent(repo.getId(), "Localizable.strings", assetContent); asset = assetRepository.findById(asset.getId()).orElse(null); From 711d83aa5820a2618b6f1de732428e88d2e50a7e Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 25 Sep 2024 16:51:55 -0700 Subject: [PATCH 074/105] Implement native Phrase locale download Since the SDK does not support the formatOptions --- .../thirdparty/phrase/PhraseClient.java | 97 +++++++++++++++++++ .../thirdparty/phrase/PhraseClientTest.java | 18 +++- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java index 4bb365cdf9..f9264726e2 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; @@ -28,9 +29,11 @@ import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.StringJoiner; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -469,6 +472,100 @@ && getErrorMessageFromOptionalApiException(doBeforeRetry.failure()) .block(); } + public String nativeLocaleDownload( + String projectId, + String locale, + String fileFormat, + String tags, + Map formatOptions, + Supplier onTagErrorRefreshCallback) { + AtomicReference refTags = new AtomicReference<>(tags); + return Mono.fromCallable( + () -> { + logger.info( + "Native Downloading locale: {} from project id: {} in file format: {}", + locale, + projectId, + fileFormat); + + Map parameters = new HashMap<>(); + parameters.put("file_format", fileFormat); + parameters.put("tags", refTags.get()); + for (Map.Entry e : formatOptions.entrySet()) { + parameters.put("format_options[%s]".formatted(e.getKey()), e.getValue()); + } + + StringJoiner queryJoiner = new StringJoiner("&"); + for (Map.Entry entry : parameters.entrySet()) { + queryJoiner.add( + URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8) + + "=" + + URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8)); + } + + String url = + String.format( + "%s/projects/%s/locales/%s/download?%s", + apiClient.getBasePath(), + URLEncoder.encode(projectId, StandardCharsets.UTF_8), + URLEncoder.encode(locale, StandardCharsets.UTF_8), + queryJoiner); + + String token = ((ApiKeyAuth) apiClient.getAuthentication("Token")).getApiKey(); + + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Authorization", "token " + token) + .header("Accept", "*") + .GET() + .build(); + + HttpResponse response; + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + int statusCode = response.statusCode(); + String responseBody = response.body(); + + if (statusCode == 200) { + return responseBody; + } else { + throw new RuntimeException( + "Can't download locale. status code " + statusCode + ": " + responseBody); + } + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> { + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale)); + + if (onTagErrorRefreshCallback != null + && getErrorMessageFromOptionalApiException(doBeforeRetry.failure()) + .contains("Invalid Download Options. Parameter tags ")) { + String newTags = onTagErrorRefreshCallback.get(); + logger.warn( + "Replacing old tags: {} with new tags: {} for download locale", + refTags.get(), + newTags); + refTags.set(newTags); + } + })) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale))) + .block(); + } + public List getKeys(String projectId, String tags) { KeysApi keysApi = new KeysApi(apiClient); AtomicInteger page = new AtomicInteger(0); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java index 5ed24ba05e..5f4ae75a73 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.service.thirdparty.phrase; +import static com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter.EscapeType.NONE; + import com.box.l10n.mojito.JSR310Migration; import com.box.l10n.mojito.android.strings.AndroidSingular; import com.box.l10n.mojito.android.strings.AndroidStringDocument; @@ -148,8 +150,9 @@ public void test() throws IOException, InterruptedException { source.addSingular( new AndroidSingular(20L, i + "-string11", "simple tag", "test comment2")); - String androidFile = new AndroidStringDocumentWriter(source, i % 2 == 0).toText(); + String androidFile = new AndroidStringDocumentWriter(source, NONE).toText(); System.out.println(androidFile); + Upload upload = phraseClient.nativeUploadAndWait( testProjectId, @@ -180,6 +183,19 @@ public void test() throws IOException, InterruptedException { i % 2 == 0 ? "test-escaping" : "test-no-escaping", () -> null); System.out.println(s); + + String ns = + phraseClient.nativeLocaleDownload( + testProjectId, + "en", + "xml", + i % 2 == 0 ? "test-escaping" : "test-no-escaping", + ImmutableMap.of("escape_tags", "true"), + () -> null); + + System.out.println(ns); + + // Assert.assertEquals(androidFile, ns); } // for (int i = 0; i < 3; i++) { From 5c36e7eb71b6a8205e05e90666b9552d75bd4031 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 26 Sep 2024 13:28:26 -0700 Subject: [PATCH 075/105] Support format options using native http call in ThirdPartyTMSPhrase we're using native call since the SDK does not work. Put a few customizable option to test different setup. --- .../thirdparty/ThirdPartyTMSPhrase.java | 169 ++++++++++++------ 1 file changed, 116 insertions(+), 53 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 8b24bed708..59d9064ea6 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -5,10 +5,10 @@ import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; import com.box.l10n.mojito.android.strings.AndroidStringDocumentReader; import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter.EscapeType; import com.box.l10n.mojito.entity.PollableTask; import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.entity.RepositoryLocale; -import com.box.l10n.mojito.json.ObjectMapper; import com.box.l10n.mojito.service.pollableTask.PollableFuture; import com.box.l10n.mojito.service.repository.RepositoryService; import com.box.l10n.mojito.service.thirdparty.phrase.PhraseClient; @@ -38,6 +38,7 @@ import java.util.Objects; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -148,33 +149,20 @@ public void push( String skipAssetsWithPathPattern, List options) { + OptionsParser optionsParser = new OptionsParser(options); + Boolean nativeClient = optionsParser.getBoolean("nativeClient", false); + Map formatOptions = getFormatOptions(optionsParser); + final AtomicReference escapeType = + getEscapeTypeAtomicReference(nativeClient, optionsParser); + List search = getSourceTextUnitDTOs(repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); - - String text = getFileContent(pluralSeparator, search, true, null); + String text = getFileContent(pluralSeparator, search, true, null, escapeType.get()); String tagForUpload = getTagForUpload(repository.getName()); - OptionsParser optionsParser = new OptionsParser(options); - Boolean nativeUpload = optionsParser.getBoolean("nativeUpload", false); - Boolean nativeUploadUnescapeTags = optionsParser.getBoolean("nativeUploadUnescapeTags", true); - Boolean nativeUploadUnescapeLineBreaks = - optionsParser.getBoolean("nativeUploadUnescapeLineBreaks", false); - - if (nativeUpload) { - // convert_placeholder - // escape_linebreaks - // unescape_linebreaks - // unescape_tags - Map formatOptions = new HashMap<>(); - if (nativeUploadUnescapeTags) { - formatOptions.put("unescape_tags", "true"); - } - - if (nativeUploadUnescapeLineBreaks) { - formatOptions.put("unescape_linebreaks", "true"); - } - + if (nativeClient) { + logger.info("Pushing with native and options: {}", formatOptions); phraseClient.nativeUploadAndWait( projectId, repository.getSourceLocale().getBcp47Tag(), @@ -341,7 +329,8 @@ public PollableFuture pull( locale, pluralSeparator, currentTags, - integrityCheckKeepStatusIfFailedAndSameTarget)); + integrityCheckKeepStatusIfFailedAndSameTarget, + optionList)); return null; } @@ -352,7 +341,8 @@ private void pullLocaleTimed( RepositoryLocale repositoryLocale, String pluralSeparator, String currentTags, - boolean integrityCheckKeepStatusIfFailedAndSameTarget) { + boolean integrityCheckKeepStatusIfFailedAndSameTarget, + List optionList) { try (var timer = Timer.resource(meterRegistry, "ThirdPartyTMSPhrase.pullLocale") .tag("repository", repository.getName())) { @@ -363,7 +353,8 @@ private void pullLocaleTimed( repositoryLocale, pluralSeparator, currentTags, - integrityCheckKeepStatusIfFailedAndSameTarget); + integrityCheckKeepStatusIfFailedAndSameTarget, + optionList); } } @@ -373,19 +364,49 @@ private void pullLocale( RepositoryLocale repositoryLocale, String pluralSeparator, String currentTags, - boolean integrityCheckKeepStatusIfFailedAndSameTarget) { + boolean integrityCheckKeepStatusIfFailedAndSameTarget, + List optionList) { String localeTag = repositoryLocale.getLocale().getBcp47Tag(); logger.info("Downloading locale: {} from Phrase with tags: {}", localeTag, currentTags); + OptionsParser optionsParser = new OptionsParser(optionList); + Boolean nativeClient = optionsParser.getBoolean("nativeClient", false); + Boolean escapeTags = optionsParser.getBoolean("escapeTags", true); + Boolean escapeLineBreaks = optionsParser.getBoolean("escapeLineBreaks", false); + Map formatOptions = new HashMap<>(); + if (escapeTags) { + formatOptions.put("escape_tags", "true"); + } + if (escapeLineBreaks) { + formatOptions.put("escape_linebreaks", "true"); + } + final AtomicReference escapeTypeAtomicReference = + getEscapeTypeAtomicReference(nativeClient, optionsParser); + Stopwatch localeDownloadStopWatch = Stopwatch.createStarted(); - String fileContent = - phraseClient.localeDownload( - projectId, - localeTag, - "xml", - currentTags, - () -> getCurrentTagsForRepository(repository, projectId)); + String fileContent; + + if (nativeClient) { + logger.info("Pulling locale with native and options: {}", formatOptions); + + fileContent = + phraseClient.nativeLocaleDownload( + projectId, + localeTag, + "xml", + currentTags, + formatOptions, + () -> getCurrentTagsForRepository(repository, projectId)); + } else { + fileContent = + phraseClient.localeDownload( + projectId, + localeTag, + "xml", + currentTags, + () -> getCurrentTagsForRepository(repository, projectId)); + } logger.info("Phrase locale download took: {}", localeDownloadStopWatch.elapsed()); @@ -398,6 +419,7 @@ private void pullLocale( List textUnitDTOS = mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); + // TODO(ja) that should not be necessary since the TextUnitBatchImporterService was fixed? // make sure that the source comment does not get replicated to all translations. Eventually // we may want to // communicate some comments, but anyway it should not be the source comment @@ -441,6 +463,16 @@ public void pushTranslations( String includeTextUnitsWithPattern, List optionList) { + OptionsParser optionsParser = new OptionsParser(optionList); + Boolean nativeClient = optionsParser.getBoolean("nativeClient", false); + Map formatOptions = getFormatOptions(optionsParser); + + final AtomicReference escapeType = + getEscapeTypeAtomicReference(nativeClient, optionsParser); + + List search = + getSourceTextUnitDTOs(repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); + List pluralTextUnitDTOs = getSourceTextUnitDTOsPluralOnly( repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); @@ -486,26 +518,31 @@ public void pushTranslations( if (textUnitDTOS.isEmpty()) { logger.info("Not translation, skip upload"); } else { - - logger.info("Print text unit for {}", repositoryLocale.getLocale().getBcp47Tag()); - textUnitDTOS.forEach( - textUnitDTO -> - logger.info( - "Textunit: {}", - ObjectMapper.withIndentedOutput().writeValueAsStringUnchecked(textUnitDTO))); - String fileContent = - getFileContent(pluralSeparator, textUnitDTOS, false, pluralFormToCommaId); + getFileContent( + pluralSeparator, textUnitDTOS, false, pluralFormToCommaId, escapeType.get()); logger.info("Push translation to phrase:\n{}", fileContent); - phraseClient.uploadAndWait( - projectId, - repositoryLocale.getLocale().getBcp47Tag(), - "xml", - repository.getName() + "-strings.xml", - fileContent, - null, - null); + if (nativeClient) { + logger.info("Pushing translations with native and options: {}", formatOptions); + phraseClient.nativeUploadAndWait( + projectId, + repositoryLocale.getLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + fileContent, + null, + formatOptions.isEmpty() ? null : formatOptions); + } else { + phraseClient.uploadAndWait( + projectId, + repositoryLocale.getLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + fileContent, + null, + null); + } } } } @@ -533,7 +570,8 @@ private static String getFileContent( String pluralSeparator, List textUnitDTOS, boolean useSource, - Map pluralFormToCommaId) { + Map pluralFormToCommaId, + EscapeType escapeType) { AndroidStringDocumentMapper androidStringDocumentMapper = new AndroidStringDocumentMapper( @@ -542,7 +580,7 @@ private static String getFileContent( AndroidStringDocument androidStringDocument = androidStringDocumentMapper.readFromTextUnits(textUnitDTOS, useSource); - return new AndroidStringDocumentWriter(androidStringDocument).toText(); + return new AndroidStringDocumentWriter(androidStringDocument, escapeType).toText(); } @Override @@ -554,6 +592,31 @@ public void pullSource( throw new UnsupportedOperationException("Pull source is not supported"); } + private static Map getFormatOptions(OptionsParser optionsParser) { + Boolean unescapeTags = optionsParser.getBoolean("unescapeTags", true); + Boolean unescapeLineBreaks = optionsParser.getBoolean("unescapeLineBreaks", false); + Map formatOptions = new HashMap<>(); + if (unescapeTags) { + formatOptions.put("unescape_tags", "true"); + } + + if (unescapeLineBreaks) { + formatOptions.put("unescape_linebreaks", "true"); + } + return formatOptions; + } + + private static AtomicReference getEscapeTypeAtomicReference( + Boolean nativeClient, OptionsParser optionsParser) { + final AtomicReference escapeType = + new AtomicReference<>(EscapeType.QUOTE_AND_NEW_LINE); + if (nativeClient) { + escapeType.set(EscapeType.NEW_LINE); + } + optionsParser.getString("escapeType", s -> escapeType.set(EscapeType.valueOf(s))); + return escapeType; + } + static class ByPluralFormComparator implements Comparator { private final Map orderMap; From 05164415a69860c110311481781f76e77a0ebdfc Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 27 Sep 2024 00:39:15 -0700 Subject: [PATCH 076/105] Refactor AndroidString classes to be able to apply proper and consistent escaping. This also greatly simplify the logic --- .../res/values-fr-rCA/strings.xml | 2 +- .../res/values-fr-rFR/strings.xml | 2 +- .../res/values-ja-rJP/strings.xml | 2 +- .../res/values-ru-rRU/strings.xml | 2 +- .../res/values-fr-rCA/strings.xml | 2 +- .../res/values-fr-rFR/strings.xml | 2 +- .../res/values-ja-rJP/strings.xml | 2 +- .../res/values-ru-rRU/strings.xml | 2 +- .../res/values-fr-rCA/strings.xml | 2 +- .../res/values-fr-rFR/strings.xml | 2 +- .../res/values-ja-rJP/strings.xml | 2 +- .../res/values-ru-rRU/strings.xml | 2 +- .../mojito/okapi/filters/AndroidFilter.java | 7 +- .../okapi/filters/AndroidFilterTest.java | 84 ++++++++++++++++- .../strings/AndroidStringDocument.java | 41 --------- .../strings/AndroidStringDocumentMapper.java | 24 +---- .../strings/AndroidStringDocumentReader.java | 91 +++++++++++++++++- .../strings/AndroidStringDocumentWriter.java | 4 +- .../android/strings/AndroidStringElement.java | 92 ------------------- .../AndroidStringDocumentWriterTest.java | 15 +-- .../android/strings/test_resources_file.xml | 6 +- 21 files changed, 203 insertions(+), 185 deletions(-) diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml index 626d7f70a2..06d4ae0878 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml @@ -1,4 +1,4 @@ - + %1$,d heure à cuisiner diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml index 626d7f70a2..06d4ae0878 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml @@ -1,4 +1,4 @@ - + %1$,d heure à cuisiner diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml index f6d60b9ef8..fdec8b186e 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml @@ -1,4 +1,4 @@ - + 所要時間：%1$,d 時間 diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml index 9ea8e22ee0..db73c90d99 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml @@ -1,4 +1,4 @@ - + Приготовление: %1$,d час diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml index 7f8da59742..57d77bf1f7 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml @@ -1,4 +1,4 @@ - + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml index ae2676f2b4..dd671274a9 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml @@ -1,4 +1,4 @@ - + %1$,d heure à cuisiner diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml index bfc2d7d33f..b6dc7ce899 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml @@ -1,4 +1,4 @@ - + 所要時間：%1$,d 時間 diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml index 4185c07f6e..4c831f9553 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml @@ -1,4 +1,4 @@ - + Приготовление: %1$,d час diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml index 7f8da59742..57d77bf1f7 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml @@ -1,4 +1,4 @@ - + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml index 26960ff3f8..aaddcd25e0 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml @@ -1,4 +1,4 @@ - + %1$,d heure à cuisiner diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml index 8f5a230089..e95e22548c 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml @@ -1,4 +1,4 @@ - + 所要時間：%1$,d 時間 diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml index 487445f368..a08ef21339 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml @@ -1,4 +1,4 @@ - + Приготовление: %1$,d час diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java index e3de012438..349e21a3e1 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java @@ -466,7 +466,7 @@ public String execute(String xmlContent) { if (xmlContent == null || xmlContent.isBlank() - || (!shouldApplyPostProcessingRemoveUntranslatedExcluded && !removeTranslatableFalse)) { + || (!shouldApplyPostProcessingRemoveUntranslatedExcluded && !hasRemoveUntranslated())) { return xmlContent; } @@ -553,6 +553,11 @@ public String execute(String xmlContent) { StreamResult streamResult = new StreamResult(new StringWriter()); transformer.transform(domSource, streamResult); String processedXmlContent = streamResult.getWriter().toString(); + + if (!hasStandalone) { + processedXmlContent = processedXmlContent.replace(" standalone=\"no\"", ""); + } + return processedXmlContent; } catch (ParserConfigurationException | SAXException | IOException | TransformerException e) { diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java index af14a80365..2f247934a2 100644 --- a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java +++ b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java @@ -138,7 +138,7 @@ public void testPostProcessingKeepDescription() { String output = androidFilePostProcessor.execute(input); String expected = """ - + somestring to keep @@ -182,7 +182,7 @@ public void testPostProcessingRemoveDescription() { String output = androidFilePostProcessor.execute(input); String expected = """ - + somestring to keep @@ -257,7 +257,7 @@ public void testPostProcessingRemoveTranslatableFalse() { String output = androidFilePostProcessor.execute(input); String expected = """ - + @@ -267,4 +267,82 @@ public void testPostProcessingRemoveTranslatableFalse() { """; assertEquals(expected, output); } + + @Test + public void testPostProcessingStandaloneNo() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + translated + @#$untranslated$#@ + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + + + translated + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingStandaloneYes() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + translated + @#$untranslated$#@ + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + + + translated + + + """; + assertEquals(expected, output); + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java index 5919a27bf5..4ae2ebb2a8 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java @@ -1,10 +1,7 @@ package com.box.l10n.mojito.android.strings; -import static com.google.common.base.Preconditions.checkArgument; - import java.util.ArrayList; import java.util.List; -import org.w3c.dom.Node; public class AndroidStringDocument { @@ -29,46 +26,8 @@ public void addSingular(AndroidSingular string) { strings.add(string); } - public void addSingular(AndroidStringElement element, Node comment) { - - checkArgument(element.isSingular(), "element should be singular"); - - addSingular( - new AndroidSingular( - element.getIdAttribute(), - element.getNameAttribute(), - element.getUnescapedContent(), - comment != null ? comment.getTextContent() : null)); - } - public void addPlural(AndroidPlural plural) { plurals.add(plural); strings.add(plural); } - - public void addPlural(AndroidStringElement element, Node comment) { - - checkArgument(element.isPlural(), "element should be plural"); - - AndroidPlural.AndroidPluralBuilder builder = AndroidPlural.builder(); - builder.setName(element.getNameAttribute()); - if (comment != null) { - builder.setComment(comment.getTextContent()); - } - - element.forEachPluralItem(builder::addItem); - - addPlural(builder.build()); - } - - public void addElement(Node node, Node comment) { - - AndroidStringElement element = new AndroidStringElement(node); - - if (element.isSingular()) { - addSingular(element, comment); - } else if (element.isPlural()) { - addPlural(element, comment); - } - } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java index bdf49ec225..0cc53bbc0e 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java @@ -15,7 +15,6 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -202,7 +201,7 @@ Stream singularToTextUnit(AndroidSingular singular) { textUnit.setName(singular.getName()); textUnit.setComment(singular.getComment()); textUnit.setTmTextUnitId(singular.getId()); - textUnit.setTarget(unescape(singular.getContent())); + textUnit.setTarget(singular.getContent()); addTextUnitDTOAttributes(textUnit); return Stream.of(textUnit); @@ -226,7 +225,7 @@ Stream pluralToTextUnits(AndroidPlural plural) { textUnit.setTmTextUnitId(item.getId()); textUnit.setPluralForm(quantity); textUnit.setPluralFormOther(pluralFormOther); - textUnit.setTarget(unescape(item.getContent())); + textUnit.setTarget(item.getContent()); addTextUnitDTOAttributes(textUnit); return textUnit; @@ -286,23 +285,4 @@ static String removeInvalidControlCharacter(String str) { return withoutControlCharacters; } - - /** should use {@link com.box.l10n.mojito.okapi.filters.AndroidFilter#unescape(String)} */ - static String unescape(String str) { - - String unescape = str; - - if (StringUtils.startsWith(unescape, "\"") && StringUtils.endsWith(unescape, "\"")) { - unescape = unescape.substring(1, unescape.length() - 1); - } - - unescape = - Strings.nullToEmpty(unescape) - .replaceAll("\\\\'", "'") - .replaceAll("\\\\\"", "\"") - .replaceAll("\\\\@", "@") - .replaceAll("\\\\n", "\n"); - - return unescape; - } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java index fa68788d1c..ebfd371e2d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java @@ -7,17 +7,20 @@ import java.io.IOException; import java.io.StringReader; import java.util.LinkedList; +import java.util.Optional; import java.util.Queue; -import javax.xml.parsers.ParserConfigurationException; +import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Document; +import org.w3c.dom.Element; import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; public class AndroidStringDocumentReader { - public static AndroidStringDocument fromFile(String fileName) - throws ParserConfigurationException, IOException, SAXException { + public static AndroidStringDocument fromFile(String fileName) throws IOException, SAXException { return buildFromDocument(documentBuilder().parse(new File(fileName))); } @@ -44,10 +47,90 @@ private static AndroidStringDocument buildFromDocument(Document source) { commentNodes.offer(node); } else if (Node.ELEMENT_NODE == node.getNodeType()) { - document.addElement(node, commentNodes.poll()); + Element nodeAsElement = (Element) node; + Node comment = commentNodes.poll(); + + if (nodeAsElement.getTagName().equals(AndroidStringElement.SINGULAR_ELEMENT_NAME)) { + document.addSingular( + new AndroidSingular( + getAttributeAs( + nodeAsElement, AndroidStringElement.ID_ATTRIBUTE_NAME, Long::valueOf), + getNameAttribute(nodeAsElement), + unescape(nodeAsElement.getTextContent()), + comment != null ? comment.getTextContent() : null)); + } else if (nodeAsElement.getTagName().equals(AndroidStringElement.PLURAL_ELEMENT_NAME)) { + + AndroidPlural.AndroidPluralBuilder builder = AndroidPlural.builder(); + + builder.setName(nodeAsElement.getAttribute(AndroidStringElement.NAME_ATTRIBUTE_NAME)); + if (comment != null) { + builder.setComment(comment.getTextContent()); + } + + NodeList nodeList = + nodeAsElement.getElementsByTagName(AndroidStringElement.PLURAL_ITEM_ELEMENT_NAME); + + for (int j = 0; j < nodeList.getLength(); j++) { + Element item = (Element) nodeList.item(j); + + builder.addItem( + new AndroidPluralItem( + getAttribute(item, AndroidStringElement.QUANTITY_ATTRIBUTE_NAME), + getAttributeAs(item, AndroidStringElement.ID_ATTRIBUTE_NAME, Long::valueOf), + unescape(getTextContent(item)))); + } + + document.addPlural(builder.build()); + } } } return document; } + + public static String getAttribute(Element element, String name) { + // looks like ofNullable is useless and then the orElse + return Optional.ofNullable(element.getAttribute(name)).orElse(""); + } + + public static T getAttributeAs( + Element element, String attributeName, Function converter) { + + T result = null; + + if (element.hasAttribute(attributeName)) { + result = converter.apply(element.getAttribute(attributeName)); + } + + return result; + } + + public static String getTextContent(Element element) { + return Optional.ofNullable(element).map(Element::getTextContent).orElse(""); + } + + public static String getNameAttribute(Element element) { + return element.getAttribute(AndroidStringElement.NAME_ATTRIBUTE_NAME); + } + + /** should use {@link com.box.l10n.mojito.okapi.filters.AndroidFilter#unescape(String)} */ + static String unescape(String str) { + + String unescape = str; + + if (!Strings.isNullOrEmpty(str)) { + + if (StringUtils.startsWith(unescape, "\"") && StringUtils.endsWith(unescape, "\"")) { + unescape = unescape.substring(1, unescape.length() - 1); + } + + unescape = + Strings.nullToEmpty(unescape) + .replaceAll("\\\\'", "'") + .replaceAll("\\\\\"", "\"") + .replaceAll("\\\\@", "@") + .replaceAll("\\\\n", "\n"); + } + return unescape; + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java index 722b585b7e..ec23b2ebb7 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java @@ -137,7 +137,7 @@ private Element addChild(Node node, String name) { private void setContent(Element element, String content) { if (!Strings.isNullOrEmpty(content)) { - element.setTextContent(escapeQuotes(content)); + element.setTextContent(escapeQuotes(content, escapeType)); } } @@ -159,7 +159,7 @@ private void setAttribute(Element element, String name, String value) { } } - String escapeQuotes(String str) { + static String escapeQuotes(String str, EscapeType escapeType) { String escaped = str; if (!Strings.isNullOrEmpty(str)) { escaped = diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringElement.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringElement.java index 4833b113f6..96fd248fdf 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringElement.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringElement.java @@ -1,13 +1,5 @@ package com.box.l10n.mojito.android.strings; -import com.google.common.base.Strings; -import java.util.Optional; -import java.util.function.Consumer; -import java.util.function.Function; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - public class AndroidStringElement { static final String ROOT_ELEMENT_NAME = "resources"; @@ -17,88 +9,4 @@ public class AndroidStringElement { static final String NAME_ATTRIBUTE_NAME = "name"; static final String QUANTITY_ATTRIBUTE_NAME = "quantity"; static final String ID_ATTRIBUTE_NAME = "tmTextUnitId"; - - private final Element element; - - public AndroidStringElement(Element element) { - this.element = element; - } - - public AndroidStringElement(Node node) { - this((Element) node); - } - - public boolean isSingular() { - return element.getTagName().equals(SINGULAR_ELEMENT_NAME); - } - - public boolean isPlural() { - return element.getTagName().equals(PLURAL_ELEMENT_NAME); - } - - public boolean isPluralItem() { - return element.getTagName().equals(PLURAL_ITEM_ELEMENT_NAME); - } - - public String getTextContent() { - return Optional.ofNullable(element).map(Element::getTextContent).orElse(""); - } - - public Long getIdAttribute() { - return getLongAttribute(ID_ATTRIBUTE_NAME); - } - - public String getNameAttribute() { - return getAttribute(NAME_ATTRIBUTE_NAME); - } - - public String getAttribute(String name) { - return Optional.ofNullable(element.getAttribute(name)).orElse(""); - } - - public String getUnescapedContent() { - return removeEscape(element.getTextContent()); - } - - public Long getLongAttribute(String attributeName) { - return getAttributeAs(attributeName, Long::valueOf); - } - - public T getAttributeAs(String attributeName, Function converter) { - - T result = null; - - if (element.hasAttribute(attributeName)) { - result = converter.apply(element.getAttribute(attributeName)); - } - - return result; - } - - public void forEachPluralItem(Consumer consumer) { - - AndroidStringElement node; - NodeList nodeList = element.getElementsByTagName(PLURAL_ITEM_ELEMENT_NAME); - - for (int i = 0; i < nodeList.getLength(); i++) { - - node = new AndroidStringElement((Element) nodeList.item(i)); - - if (node.isPluralItem()) { - consumer.accept( - new AndroidPluralItem( - node.getAttribute(QUANTITY_ATTRIBUTE_NAME), - node.getLongAttribute(ID_ATTRIBUTE_NAME), - node.getTextContent())); - } - } - } - - private static String removeEscape(String str) { - return Strings.nullToEmpty(str) - .replaceAll("\\\\'", "'") - .replaceAll("\\\\\"", "\"") - .replace("\\\\\n", "\n") - .replaceAll("\\\\@", "@"); - } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java index 417309ce57..214e3660dc 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.android.strings; +import static com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter.escapeQuotes; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.ImmutableList; @@ -289,12 +290,14 @@ public void testGenerateWithHTML() { @Test public void testEscapeQuotes() { - assertThat(writer.escapeQuotes(null)).isEqualTo(""); - assertThat(writer.escapeQuotes("")).isEqualTo(""); - assertThat(writer.escapeQuotes("String")).isEqualTo("String"); - assertThat(writer.escapeQuotes("second\\\"String")).isEqualTo("second\\\\\"String"); - assertThat(writer.escapeQuotes("third\nString")).isEqualTo("third\\nString"); - assertThat(writer.escapeQuotes("fourth\ntest\\\"String\\\"")) + AndroidStringDocumentWriter.EscapeType escapeType = + AndroidStringDocumentWriter.EscapeType.QUOTE_AND_NEW_LINE; + assertThat(escapeQuotes(null, escapeType)).isEqualTo(null); + assertThat(escapeQuotes("", escapeType)).isEqualTo(""); + assertThat(escapeQuotes("String", escapeType)).isEqualTo("String"); + assertThat(escapeQuotes("second\\\"String", escapeType)).isEqualTo("second\\\\\"String"); + assertThat(escapeQuotes("third\nString", escapeType)).isEqualTo("third\\nString"); + assertThat(escapeQuotes("fourth\ntest\\\"String\\\"", escapeType)) .isEqualTo("fourth\\ntest\\\\\"String\\\\\""); } diff --git a/webapp/src/test/resources/com/box/l10n/mojito/android/strings/test_resources_file.xml b/webapp/src/test/resources/com/box/l10n/mojito/android/strings/test_resources_file.xml index 6f877f2698..1556c0f2c2 100644 --- a/webapp/src/test/resources/com/box/l10n/mojito/android/strings/test_resources_file.xml +++ b/webapp/src/test/resources/com/box/l10n/mojito/android/strings/test_resources_file.xml @@ -3,8 +3,10 @@ Dela... - Mer \\" dela - ei \\' kommentteja + Mer \\" dela + ei \\' kommentteja + Mer \" dela + ei \' kommentteja salto de \nlinea From cc793dde6ffdba07a17d78cad24a73c02a799bff Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 30 Sep 2024 11:53:50 -0700 Subject: [PATCH 077/105] Add error log when the Android XML post processing fails --- .../java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java index 349e21a3e1..786b714af9 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java @@ -561,6 +561,7 @@ public String execute(String xmlContent) { return processedXmlContent; } catch (ParserConfigurationException | SAXException | IOException | TransformerException e) { + logger.error("Can't post-process Android XML:\n{}", xmlContent); throw new RuntimeException(e); } } From 17d47301af9197773c47898ad742ed10435ace1b Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 30 Sep 2024 17:03:56 -0700 Subject: [PATCH 078/105] Add an adhoc check in the HtmlTagIntegrityChecker This is very adhoc and will require a better implementation if there are more of those type of issue. Also expecting for most of them to go away. --- .../integritychecker/HtmlTagIntegrityChecker.java | 13 +++++++++++++ .../HtmlTagIntegrityCheckerTest.java | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java index beb2c66388..58ef8249e9 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java @@ -73,6 +73,19 @@ public void check(String sourceContent, String targetContent) throws IntegrityCh if (!isValidTagOrder(targetHtmlTags)) { throw new HtmlTagIntegrityCheckerException("HTML tags in target are not in valid order"); } + + logger.debug("Ad-hoc checks"); + checkDoubleAnnotationElements(sourceContent, targetContent); + } + + /** Adhoc check to unblock. Eventually needs a better solution. */ + void checkDoubleAnnotationElements(String sourceContent, String targetContent) { + String doubleAnnotationString = "text"; + String target = "text"; + + checker.check(source, target); + } + @Test public void testHtmlTagCheckNonTagLessThanDoesntConfuseThings() { String source = "Upload is <10% complete."; From cd2d8539ae9fc904ec4af25dfbc2b6ca6633c29e Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 1 Oct 2024 09:56:34 -0700 Subject: [PATCH 079/105] Refactor integrity checks in TextUnitBatchImporterService MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is preparatory work to implement the new mode: KEEP_STATUS_IF_SAME_TARGET. That mode will be similar to the legacy KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET, but it will also retain the status when the integrity checker does not fail. This is an extension of the legacy behavior to allow marking a translation as invalid when the integrity check didn’t catch the issue, eventually breaking a build. --- .../l10n/mojito/rest/textunit/TextUnitWS.java | 8 ++- .../service/asset/ImportTextUnitJob.java | 9 +-- .../service/asset/ImportTextUnitJobInput.java | 23 +++----- .../service/asset/VirtualAssetService.java | 4 +- .../RepositoryMachineTranslationService.java | 4 +- .../thirdparty/ThirdPartyTMSPhrase.java | 5 +- .../ThirdPartyTMSSmartlingWithJson.java | 5 +- .../quartz/SmartlingPullLocaleFileJob.java | 4 +- .../TextUnitBatchImporterService.java | 56 ++++++++++++++----- .../BranchNotificationServiceTest.java | 7 ++- .../ThirdPartyTMSSmartlingTest.java | 4 +- .../ThirdPartyTMSSmartlingWithJsonTest.java | 9 ++- .../SmartlingPullLocaleFileJobTest.java | 15 ++--- .../TextUnitBatchImporterServiceTest.java | 29 ++++++---- 14 files changed, 115 insertions(+), 67 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/TextUnitWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/TextUnitWS.java index f39cab94d1..3740d4df2f 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/TextUnitWS.java +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/TextUnitWS.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.rest.textunit; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; + import com.box.l10n.mojito.entity.Asset; import com.box.l10n.mojito.entity.AssetTextUnit; import com.box.l10n.mojito.entity.BaseEntity; @@ -310,9 +312,9 @@ public PollableTask importTextUnitBatch(@RequestBody String string) { PollableFuture pollableFuture = textUnitBatchImporterService.asyncImportTextUnits( importTextUnitsBatch.getTextUnits(), - importTextUnitsBatch.isIntegrityCheckSkipped(), - importTextUnitsBatch.isIntegrityCheckKeepStatusIfFailedAndSameTarget()); - + fromLegacy( + importTextUnitsBatch.isIntegrityCheckSkipped(), + importTextUnitsBatch.isIntegrityCheckKeepStatusIfFailedAndSameTarget())); return pollableFuture.getPollableTask(); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJob.java b/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJob.java index 0d0b914d26..152605adf3 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJob.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJob.java @@ -3,6 +3,8 @@ import com.box.l10n.mojito.quartz.QuartzPollableJob; import com.box.l10n.mojito.service.pollableTask.PollableTaskService; import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -24,10 +26,9 @@ public class ImportTextUnitJob extends QuartzPollableJob textUnitDTOs = input.getTextUnitDTOs(); + + textUnitBatchImporterService.importTextUnits(textUnitDTOs, input.getIntegrityChecksType()); return null; } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJobInput.java b/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJobInput.java index 901c0aaf67..ea84d95fee 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJobInput.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJobInput.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.service.asset; +import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; import com.box.l10n.mojito.service.tm.search.TextUnitDTO; import java.util.List; @@ -8,26 +9,16 @@ */ public class ImportTextUnitJobInput { - boolean integrityCheckSkipped; - boolean integrityCheckKeepStatusIfFailedAndSameTarget; List textUnitDTOs; + TextUnitBatchImporterService.IntegrityChecksType integrityChecksType; - public boolean isIntegrityCheckSkipped() { - return integrityCheckSkipped; + public TextUnitBatchImporterService.IntegrityChecksType getIntegrityChecksType() { + return integrityChecksType; } - public void setIntegrityCheckSkipped(boolean integrityCheckSkipped) { - this.integrityCheckSkipped = integrityCheckSkipped; - } - - public boolean isIntegrityCheckKeepStatusIfFailedAndSameTarget() { - return integrityCheckKeepStatusIfFailedAndSameTarget; - } - - public void setIntegrityCheckKeepStatusIfFailedAndSameTarget( - boolean integrityCheckKeepStatusIfFailedAndSameTarget) { - this.integrityCheckKeepStatusIfFailedAndSameTarget = - integrityCheckKeepStatusIfFailedAndSameTarget; + public void setIntegrityChecksType( + TextUnitBatchImporterService.IntegrityChecksType integrityChecksType) { + this.integrityChecksType = integrityChecksType; } public List getTextUnitDTOs() { diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/asset/VirtualAssetService.java b/webapp/src/main/java/com/box/l10n/mojito/service/asset/VirtualAssetService.java index b939f6a967..37e27ddbd9 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/asset/VirtualAssetService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/asset/VirtualAssetService.java @@ -2,6 +2,7 @@ import static com.box.l10n.mojito.entity.TMTextUnitVariant.Status.APPROVED; import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.slf4j.LoggerFactory.getLogger; import com.box.l10n.mojito.entity.Asset; @@ -341,7 +342,8 @@ public PollableFuture importLocalizedTextUnits( textUnitDTOs.add(textUnitDTO); } - return textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOs, false, false); + return textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOs, fromLegacy(false, false)); } @Transactional diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/machinetranslation/RepositoryMachineTranslationService.java b/webapp/src/main/java/com/box/l10n/mojito/service/machinetranslation/RepositoryMachineTranslationService.java index 155c1d260a..dedb5ec65d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/machinetranslation/RepositoryMachineTranslationService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/machinetranslation/RepositoryMachineTranslationService.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.service.machinetranslation; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; + import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.service.pollableTask.Pollable; import com.box.l10n.mojito.service.pollableTask.PollableFuture; @@ -127,7 +129,7 @@ public PollableFuture translateRepository( .collect(Collectors.toList()); textUnitBatchImporterService.importTextUnits( - machineTranslatedTextUnitDTOs, false, true); + machineTranslatedTextUnitDTOs, fromLegacy(false, true)); }); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 59d9064ea6..bccaea6194 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.service.thirdparty; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; + import com.box.l10n.mojito.JSR310Migration; import com.box.l10n.mojito.android.strings.AndroidStringDocument; import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; @@ -426,8 +428,9 @@ private void pullLocale( textUnitDTOS.forEach(t -> t.setComment(null)); Stopwatch importStopWatch = Stopwatch.createStarted(); + textUnitBatchImporterService.importTextUnits( - textUnitDTOS, false, integrityCheckKeepStatusIfFailedAndSameTarget); + textUnitDTOS, fromLegacy(false, integrityCheckKeepStatusIfFailedAndSameTarget)); logger.info("Time importing text units: {}", importStopWatch.elapsed()); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java index 4343368218..682479d36f 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java @@ -2,6 +2,7 @@ import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSUtils.isFileEqualToPreviousRun; import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.isPluralFile; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.entity.RepositoryLocale; @@ -250,7 +251,9 @@ && isFileEqualToPreviousRun( t.setRepositoryName(repository.getName()); t.setTargetLocale(repositoryLocale.getLocale().getBcp47Tag()); }); - textUnitBatchImporterService.importTextUnits(textUnitDTOS, false, true); + + textUnitBatchImporterService.importTextUnits( + textUnitDTOS, fromLegacy(false, true)); }); }); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java index 0f6d733b03..024bb802f0 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java @@ -1,6 +1,7 @@ package com.box.l10n.mojito.service.thirdparty.smartling.quartz; import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSUtils.isFileEqualToPreviousRun; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import com.box.l10n.mojito.LocaleMappingHelper; import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; @@ -111,7 +112,8 @@ && matchesChecksumFromPreviousSync(input, localeTag, fileName, fileContent)) { if (!input.isDryRun()) { logger.debug("Importing text units for locale: {}", smartlingLocale); - textUnitBatchImporterService.importTextUnits(textUnits, false, true); + + textUnitBatchImporterService.importTextUnits(textUnits, fromLegacy(false, true)); } } return null; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java index 6a185f45c0..e19f0e094d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java @@ -96,6 +96,40 @@ public class TextUnitBatchImporterService { @Value("${l10n.textUnitBatchImporterService.quartz.schedulerName:" + DEFAULT_SCHEDULER_NAME + "}") String schedulerName; + public enum IntegrityChecksType { + /** Don't run integrity checks */ + SKIP, + /** Always use the status from the integrity checker (legacy behavior 1) */ + ALWAYS, + /** + * Run integrity checks. If it fails and the target is the same, keep the current status; + * otherwise, reject (legacy behavior 2). + */ + KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET, + /** + * Run integrity checks. If the target is the same, keep the current status. + * + *

This is an extension of the legacy behavior that allows marking a translation as invalid + * when the integrity check did not catch the issue, eventually causing a build failure. + */ + KEEP_STATUS_IF_SAME_TARGET; + + public static IntegrityChecksType fromLegacy( + boolean integrityCheckSkipped, boolean integrityCheckKeepStatusIfFailedAndSameTarget) { + IntegrityChecksType legacy = IntegrityChecksType.SKIP; + + if (!integrityCheckSkipped) { + if (!integrityCheckKeepStatusIfFailedAndSameTarget) { + legacy = ALWAYS; + } else { + legacy = KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET; + } + } + + return legacy; + } + } + /** * Imports a batch of text units. * @@ -115,15 +149,11 @@ public class TextUnitBatchImporterService { * @return */ public PollableFuture asyncImportTextUnits( - List textUnitDTOs, - boolean integrityCheckSkipped, - boolean integrityCheckKeepStatusIfFailedAndSameTarget) { + List textUnitDTOs, IntegrityChecksType integrityChecksType) { ImportTextUnitJobInput importTextUnitJobInput = new ImportTextUnitJobInput(); importTextUnitJobInput.setTextUnitDTOs(textUnitDTOs); - importTextUnitJobInput.setIntegrityCheckSkipped(integrityCheckSkipped); - importTextUnitJobInput.setIntegrityCheckKeepStatusIfFailedAndSameTarget( - integrityCheckKeepStatusIfFailedAndSameTarget); + importTextUnitJobInput.setIntegrityChecksType(integrityChecksType); return quartzPollableTaskScheduler.scheduleJob( ImportTextUnitJob.class, importTextUnitJobInput, schedulerName); @@ -131,9 +161,7 @@ public PollableFuture asyncImportTextUnits( @StopWatch public PollableFuture importTextUnits( - List textUnitDTOs, - boolean integrityCheckSkipped, - boolean integrityCheckKeepStatusIfFailedAndSameTarget) { + List textUnitDTOs, IntegrityChecksType integrityChecksType) { return meterRegistry .timer("TextUnitBatchImporterService.importTextUnits") @@ -164,7 +192,7 @@ public PollableFuture importTextUnits( mapTextUnitsToImportWithExistingTextUnits( locale, asset, textUnitsForBatchImport); - if (!integrityCheckSkipped) { + if (!IntegrityChecksType.SKIP.equals(integrityChecksType)) { try (var timer2 = Timer.resource( meterRegistry, @@ -173,9 +201,7 @@ public PollableFuture importTextUnits( .tag("asset", asset.getPath())) { applyIntegrityChecks( - asset, - textUnitsForBatchImport, - integrityCheckKeepStatusIfFailedAndSameTarget); + asset, textUnitsForBatchImport, integrityChecksType); } } importTextUnitsOfLocaleAndAsset(locale, asset, textUnitsForBatchImport); @@ -314,7 +340,7 @@ List getTextUnitTDOsForLocaleAndAsset(Locale locale, Asset asset) { void applyIntegrityChecks( Asset asset, List textUnitsForBatchImport, - boolean keepStatusIfCheckFailedAndSameTarget) { + IntegrityChecksType integrityChecksType) { Set textUnitCheckers = integrityCheckerFactory.getTextUnitCheckers(asset); @@ -338,7 +364,7 @@ void applyIntegrityChecks( boolean hasSameTarget = textUnitForBatchImport.getContent().equals(currentTextUnit.getTarget()); - if (hasSameTarget && keepStatusIfCheckFailedAndSameTarget) { + if (hasSameTarget && !IntegrityChecksType.ALWAYS.equals(integrityChecksType)) { textUnitForBatchImport.setIncludedInLocalizedFile( currentTextUnit.isIncludedInLocalizedFile()); textUnitForBatchImport.setStatus(currentTextUnit.getStatus()); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/branch/notification/BranchNotificationServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/branch/notification/BranchNotificationServiceTest.java index 98dd0c1e62..feec95d81b 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/branch/notification/BranchNotificationServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/branch/notification/BranchNotificationServiceTest.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.service.branch.notification; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; + import com.box.l10n.mojito.entity.AssetContent; import com.box.l10n.mojito.entity.Branch; import com.box.l10n.mojito.entity.BranchNotification; @@ -152,8 +154,9 @@ void translateBranch(Branch branch) throws ExecutionException, InterruptedExcept textUnitBatchImporterService .asyncImportTextUnits( importTextUnitsBatch.getTextUnits(), - importTextUnitsBatch.isIntegrityCheckSkipped(), - importTextUnitsBatch.isIntegrityCheckKeepStatusIfFailedAndSameTarget()) + fromLegacy( + importTextUnitsBatch.isIntegrityCheckSkipped(), + importTextUnitsBatch.isIntegrityCheckKeepStatusIfFailedAndSameTarget())) .get(); // make sure the stats are ready for whatever is done next in notificaiton diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java index 6986de05eb..eeaa00ef98 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java @@ -2,6 +2,7 @@ import static com.box.l10n.mojito.android.strings.AndroidStringDocumentReader.fromText; import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -199,8 +200,7 @@ public void setUp() throws SchedulerException { anyString()); doReturn(null) .when(mockTextUnitBatchImporterService) - .importTextUnits(any(), eq(false), eq(true)); - + .importTextUnits(any(), eq(fromLegacy(false, true))); resultProcessor = new StubSmartlingResultProcessor(); tmsSmartling = new ThirdPartyTMSSmartling( diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java index 636f45b2bb..f32b77e2e2 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.service.thirdparty; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; import static org.mockito.ArgumentMatchers.*; @@ -147,7 +148,9 @@ public void testJsonWithICUMessagFormats() throws Exception { return textUnitDTO; }) .collect(ImmutableList.toImmutableList()); - textUnitBatchImporterService.importTextUnits(localizedTextUnitDTOs, false, false); + + textUnitBatchImporterService.importTextUnits( + localizedTextUnitDTOs, fromLegacy(false, false)); }); SmartlingOptions smartlingOptions = SmartlingOptions.parseList(ImmutableList.of()); @@ -386,7 +389,7 @@ public void testPullWithUntranslatedUnits() throws Exception { ArgumentCaptor> dtoListCaptor = ArgumentCaptor.forClass(ImmutableList.class); Mockito.verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(dtoListCaptor.capture(), anyBoolean(), anyBoolean()); + .importTextUnits(dtoListCaptor.capture(), any()); ImmutableList translatedUnits = dtoListCaptor.getValue(); // Expecting two fully translated units @@ -409,7 +412,7 @@ public void testPullWithUntranslatedUnits() throws Exception { dtoListCaptor = ArgumentCaptor.forClass(ImmutableList.class); Mockito.verify(textUnitBatchImporterServiceMock, times(2)) - .importTextUnits(dtoListCaptor.capture(), anyBoolean(), anyBoolean()); + .importTextUnits(dtoListCaptor.capture(), any()); translatedUnits = dtoListCaptor.getValue(); // Expecting two fully translated units diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJobTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJobTest.java index 95b99bfafd..99b7524108 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJobTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJobTest.java @@ -1,6 +1,7 @@ package com.box.l10n.mojito.service.thirdparty.smartling.quartz; import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.groups.Tuple.tuple; import static org.mockito.ArgumentMatchers.any; @@ -85,7 +86,7 @@ public void setup() { anyString()); doReturn(null) .when(textUnitBatchImporterServiceMock) - .importTextUnits(any(), eq(false), eq(true)); + .importTextUnits(any(), eq(fromLegacy(false, true))); RetryBackoffSpec retryConfiguration = Retry.backoff(10, Duration.ofMillis(1)).maxBackoff(Duration.ofMillis(10)); @@ -132,7 +133,7 @@ public void testPullSingular() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); List captured = textUnitListCaptor.getValue(); @@ -186,7 +187,7 @@ public void testPullPlural() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); List captured = textUnitListCaptor.getValue(); @@ -251,7 +252,7 @@ public void testPullPluralsFix() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); List captured = textUnitListCaptor.getValue(); @@ -319,7 +320,7 @@ public void testPullDryRun() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(0)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); } @Test(expected = SmartlingClientException.class) @@ -370,7 +371,7 @@ public void testPullShortCircuitIfNoChanges() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(0)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); verify(thirdPartyFileChecksumRepositoryMock, times(0)).save(any(ThirdPartyFileChecksum.class)); } @@ -412,7 +413,7 @@ public void testPullShortCircuitIfChanges() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); verify(thirdPartyFileChecksumRepositoryMock, times(1)).save(any(ThirdPartyFileChecksum.class)); } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java index 8f0e16aaf7..3566f5b61d 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.service.tm.importer; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -103,7 +104,8 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException { Arrays.asList(textUnitDTO, textUnitDTO2, textUnitDTO3); PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false); + textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOsForImport, fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); @@ -150,7 +152,8 @@ public void testAsyncImportTextUnitsFromSearch() throws InterruptedException { } PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false); + textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOsForImport, fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -200,7 +203,8 @@ public void testAsyncImportTextUnitsDuplicatedNames() throws InterruptedExceptio } PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false); + textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOsForImport, fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -254,7 +258,8 @@ public void testAsyncImportTextUnitsDuplicatedEntries() throws InterruptedExcept textUnitDTOsForImport.add(duplicatedEntry); PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false); + textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOsForImport, fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -326,7 +331,9 @@ public void testImportMulipleRepositoryAssetAndLocale() throws Exception { + textUnitDTO.getName()); } - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false).get(); + textUnitBatchImporterService + .asyncImportTextUnits(textUnitDTOsForImport, fromLegacy(false, false)) + .get(); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); assertFalse(textUnitDTOs.isEmpty()); @@ -385,7 +392,7 @@ public void testUnused() textUnitDTO.setTarget("v1"); textUnitBatchImporterService - .asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false) + .asyncImportTextUnits(Arrays.asList(textUnitDTO), fromLegacy(false, false)) .get(); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -417,7 +424,7 @@ public void testUnused() textUnitDTO.setTarget("v2"); textUnitBatchImporterService - .asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false) + .asyncImportTextUnits(Arrays.asList(textUnitDTO), fromLegacy(false, false)) .get(); textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -432,7 +439,7 @@ public void testUnused() .addTextUnits(virtualAsset1.getId(), Arrays.asList(virtualAssetTextUnit)) .get(); textUnitBatchImporterService - .asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false) + .asyncImportTextUnits(Arrays.asList(textUnitDTO), fromLegacy(false, false)) .get(); textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -477,7 +484,8 @@ public void testIntegirtyChecker() throws Exception { textUnitDTO.setTarget("with some broken {placeholder"); PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false); + textUnitBatchImporterService.asyncImportTextUnits( + Arrays.asList(textUnitDTO), fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); TextUnitSearcherParameters textUnitSearcherParameters = @@ -495,7 +503,8 @@ public void testIntegirtyChecker() throws Exception { textUnitDTO.setTarget("with fixed {placeholder}"); asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false); + textUnitBatchImporterService.asyncImportTextUnits( + Arrays.asList(textUnitDTO), fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); From 314f202fda59c4b1095caac3bbe09e8900ca3cb3 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 1 Oct 2024 10:14:24 -0700 Subject: [PATCH 080/105] Add constant for using native client in ThirdPartyTMSPhrase --- .../mojito/service/thirdparty/ThirdPartyTMSPhrase.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index bccaea6194..0b66300543 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -53,6 +53,7 @@ public class ThirdPartyTMSPhrase implements ThirdPartyTMS { static final String TAG_PREFIX = "push_"; static final String TAG_PREFIX_WITH_REPOSITORY = "push_%s"; + static final boolean NATIVE_CLIENT_DEFAULT_VALUE = false; static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhrase.class); @@ -152,7 +153,7 @@ public void push( List options) { OptionsParser optionsParser = new OptionsParser(options); - Boolean nativeClient = optionsParser.getBoolean("nativeClient", false); + Boolean nativeClient = optionsParser.getBoolean("nativeClient", NATIVE_CLIENT_DEFAULT_VALUE); Map formatOptions = getFormatOptions(optionsParser); final AtomicReference escapeType = getEscapeTypeAtomicReference(nativeClient, optionsParser); @@ -373,7 +374,7 @@ private void pullLocale( logger.info("Downloading locale: {} from Phrase with tags: {}", localeTag, currentTags); OptionsParser optionsParser = new OptionsParser(optionList); - Boolean nativeClient = optionsParser.getBoolean("nativeClient", false); + Boolean nativeClient = optionsParser.getBoolean("nativeClient", NATIVE_CLIENT_DEFAULT_VALUE); Boolean escapeTags = optionsParser.getBoolean("escapeTags", true); Boolean escapeLineBreaks = optionsParser.getBoolean("escapeLineBreaks", false); Map formatOptions = new HashMap<>(); @@ -467,7 +468,7 @@ public void pushTranslations( List optionList) { OptionsParser optionsParser = new OptionsParser(optionList); - Boolean nativeClient = optionsParser.getBoolean("nativeClient", false); + Boolean nativeClient = optionsParser.getBoolean("nativeClient", NATIVE_CLIENT_DEFAULT_VALUE); Map formatOptions = getFormatOptions(optionsParser); final AtomicReference escapeType = From 6316016a0d6ccd3661710bed05b64757cb73548f Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 1 Oct 2024 17:01:22 -0700 Subject: [PATCH 081/105] Fix third party mapping when delete current mapping option is used The already mapped set must be empty when using the delete option. And there is no need to call the service to get what is already mapped --- .../l10n/mojito/service/thirdparty/ThirdPartyService.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java index 6defd73973..0e8c07dc9c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java @@ -309,14 +309,17 @@ void mapThirdPartyTextUnitsToTextUnitDTOs( String pluralSeparator, boolean deleteCurrentMapping) { logger.debug("Map third party text units to text unit DTOs for asset: {}", asset.getId()); - Set alreadyMappedTmTextUnitId = - thirdPartyTextUnitRepository.findTmTextUnitIdsByAsset(asset); + + Set alreadyMappedTmTextUnitId; if (deleteCurrentMapping) { logger.info("Delete existing ThirdPartyTextUnit mapping for asset id: {}", asset.getId()); int deletedCount = thirdPartyTextUnitRepository.deleteByAssetId(asset.getId()); logger.info( "Deleted {} ThirdPartyTextUnit mappings for asset id: {}", deletedCount, asset.getId()); + alreadyMappedTmTextUnitId = Collections.emptySet(); + } else { + alreadyMappedTmTextUnitId = thirdPartyTextUnitRepository.findTmTextUnitIdsByAsset(asset); } boolean allWithTmTextUnitId = From a53f4ff212d9edfb7d9a4e2c48214d8a6f3a89b4 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 1 Oct 2024 19:16:05 -0700 Subject: [PATCH 082/105] Improve integrity checker for batch import the new mode IntegrityChecksType.KEEP_STATUS_IF_SAME_TARGET could become default, if no test have issues. --- .../TextUnitBatchImporterService.java | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java index e19f0e094d..e69ea6a870 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java @@ -100,7 +100,7 @@ public enum IntegrityChecksType { /** Don't run integrity checks */ SKIP, /** Always use the status from the integrity checker (legacy behavior 1) */ - ALWAYS, + ALWAYS_USE_INTEGRITY_CHECKER_STATUS, /** * Run integrity checks. If it fails and the target is the same, keep the current status; * otherwise, reject (legacy behavior 2). @@ -120,7 +120,7 @@ public static IntegrityChecksType fromLegacy( if (!integrityCheckSkipped) { if (!integrityCheckKeepStatusIfFailedAndSameTarget) { - legacy = ALWAYS; + legacy = ALWAYS_USE_INTEGRITY_CHECKER_STATUS; } else { legacy = KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET; } @@ -356,34 +356,43 @@ void applyIntegrityChecks( textUnitForBatchImport.setIncludedInLocalizedFile(true); textUnitForBatchImport.setStatus(APPROVED); + boolean hasSameTarget = + textUnitForBatchImport.getContent().equals(currentTextUnit.getTarget()); + for (TextUnitIntegrityChecker textUnitChecker : textUnitCheckers) { try { textUnitChecker.check(currentTextUnit.getSource(), textUnitForBatchImport.getContent()); - } catch (IntegrityCheckException ice) { - - boolean hasSameTarget = - textUnitForBatchImport.getContent().equals(currentTextUnit.getTarget()); - if (hasSameTarget && !IntegrityChecksType.ALWAYS.equals(integrityChecksType)) { + if (hasSameTarget + && !IntegrityChecksType.KEEP_STATUS_IF_SAME_TARGET.equals(integrityChecksType)) { textUnitForBatchImport.setIncludedInLocalizedFile( currentTextUnit.isIncludedInLocalizedFile()); textUnitForBatchImport.setStatus(currentTextUnit.getStatus()); - } else { - logger.info( - "Integrity check failed for string with source:\n{}\n\nand content:\n{}", - currentTextUnit.getSource(), - textUnitForBatchImport.getContent(), - ice); - textUnitForBatchImport.setIncludedInLocalizedFile(false); - textUnitForBatchImport.setStatus(Status.TRANSLATION_NEEDED); - - TMTextUnitVariantComment tmTextUnitVariantComment = new TMTextUnitVariantComment(); - tmTextUnitVariantComment.setSeverity(Severity.ERROR); - tmTextUnitVariantComment.setContent(ice.getMessage()); - textUnitForBatchImport.getTmTextUnitVariantComments().add(tmTextUnitVariantComment); + } + } catch (IntegrityCheckException ice) { + logger.info( + "Integrity check failed for string with source:\n{}\n\nand content:\n{}", + currentTextUnit.getSource(), + textUnitForBatchImport.getContent(), + ice); + textUnitForBatchImport.setIncludedInLocalizedFile(false); + textUnitForBatchImport.setStatus(Status.TRANSLATION_NEEDED); + + if (hasSameTarget) { + switch (integrityChecksType) { + case KEEP_STATUS_IF_SAME_TARGET: + case KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET: + textUnitForBatchImport.setIncludedInLocalizedFile( + currentTextUnit.isIncludedInLocalizedFile()); + textUnitForBatchImport.setStatus(currentTextUnit.getStatus()); + break; + } } - break; + TMTextUnitVariantComment tmTextUnitVariantComment = new TMTextUnitVariantComment(); + tmTextUnitVariantComment.setSeverity(Severity.ERROR); + tmTextUnitVariantComment.setContent(ice.getMessage()); + textUnitForBatchImport.getTmTextUnitVariantComments().add(tmTextUnitVariantComment); } } } From 66ed7b509a93189df618d4dcbb1c97feaf4f8b00 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 1 Oct 2024 19:46:05 -0700 Subject: [PATCH 083/105] Add an option to choose the batch import integrity check type in the phrase connector Use KEEP_STATUS_IF_SAME_TARGET in Phrase Connector as default, which different from current behavior --- .../thirdparty/ThirdPartyTMSPhrase.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 0b66300543..c3211bc942 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -1,7 +1,5 @@ package com.box.l10n.mojito.service.thirdparty; -import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; - import com.box.l10n.mojito.JSR310Migration; import com.box.l10n.mojito.android.strings.AndroidStringDocument; import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; @@ -15,6 +13,7 @@ import com.box.l10n.mojito.service.repository.RepositoryService; import com.box.l10n.mojito.service.thirdparty.phrase.PhraseClient; import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; +import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType; import com.box.l10n.mojito.service.tm.search.SearchType; import com.box.l10n.mojito.service.tm.search.StatusFilter; import com.box.l10n.mojito.service.tm.search.TextUnitDTO; @@ -320,6 +319,11 @@ public PollableFuture pull( Boolean integrityCheckKeepStatusIfFailedAndSameTarget = optionsParser.getBoolean("integrityCheckKeepStatusIfFailedAndSameTarget", true); + AtomicReference integrityChecksType = + new AtomicReference<>(IntegrityChecksType.KEEP_STATUS_IF_SAME_TARGET); + optionsParser.getString( + "integrityChecksType", s -> integrityChecksType.set(IntegrityChecksType.valueOf(s))); + // may already hit rate limit, according it is 4 qps ... there is a retry in the locale client // though. // but 4 qps is very low to download 64 locales. @@ -332,7 +336,7 @@ public PollableFuture pull( locale, pluralSeparator, currentTags, - integrityCheckKeepStatusIfFailedAndSameTarget, + integrityChecksType.get(), optionList)); return null; @@ -344,7 +348,7 @@ private void pullLocaleTimed( RepositoryLocale repositoryLocale, String pluralSeparator, String currentTags, - boolean integrityCheckKeepStatusIfFailedAndSameTarget, + IntegrityChecksType integrityChecksType, List optionList) { try (var timer = Timer.resource(meterRegistry, "ThirdPartyTMSPhrase.pullLocale") @@ -356,7 +360,7 @@ private void pullLocaleTimed( repositoryLocale, pluralSeparator, currentTags, - integrityCheckKeepStatusIfFailedAndSameTarget, + integrityChecksType, optionList); } } @@ -367,7 +371,7 @@ private void pullLocale( RepositoryLocale repositoryLocale, String pluralSeparator, String currentTags, - boolean integrityCheckKeepStatusIfFailedAndSameTarget, + IntegrityChecksType integrityChecksType, List optionList) { String localeTag = repositoryLocale.getLocale().getBcp47Tag(); @@ -430,8 +434,7 @@ private void pullLocale( Stopwatch importStopWatch = Stopwatch.createStarted(); - textUnitBatchImporterService.importTextUnits( - textUnitDTOS, fromLegacy(false, integrityCheckKeepStatusIfFailedAndSameTarget)); + textUnitBatchImporterService.importTextUnits(textUnitDTOS, integrityChecksType); logger.info("Time importing text units: {}", importStopWatch.elapsed()); } From baab8f96cbc6dc2371c9019632842355901182d5 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 1 Oct 2024 21:09:49 -0700 Subject: [PATCH 084/105] Add an option to re-apply integrity status unless the target are tagged with "false positive" The idea is to tag properly the "false positive" so that we can easily re-apply integrity checks when bugs fixes are released. Use a very basic "false positive" tag for now --- .../importer/TextUnitBatchImporterService.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java index e69ea6a870..564250faa0 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java @@ -37,6 +37,7 @@ import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; import com.box.l10n.mojito.service.tm.textunitdtocache.TextUnitDTOsCacheService; import com.box.l10n.mojito.service.tm.textunitdtocache.UpdateType; +import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -64,6 +65,8 @@ @Component public class TextUnitBatchImporterService { + static final String FALSE_POSITIVE_TAG_FOR_STATUS = "false positive"; + /** logger */ static Logger logger = LoggerFactory.getLogger(TextUnitBatchImporterService.class); @@ -101,6 +104,14 @@ public enum IntegrityChecksType { SKIP, /** Always use the status from the integrity checker (legacy behavior 1) */ ALWAYS_USE_INTEGRITY_CHECKER_STATUS, + + /** + * Use the status from the integrity check unless the current translation has a tag in the + * comment saying this was a false positive. + * + *

In case of false positive, we don't want to change the status back + */ + UNLESS_TAGGED_USE_INTEGRITY_CHECKER_STATUS, /** * Run integrity checks. If it fails and the target is the same, keep the current status; * otherwise, reject (legacy behavior 2). @@ -380,6 +391,12 @@ void applyIntegrityChecks( if (hasSameTarget) { switch (integrityChecksType) { + case UNLESS_TAGGED_USE_INTEGRITY_CHECKER_STATUS: + if (!Strings.nullToEmpty(currentTextUnit.getTargetComment()) + .toLowerCase() + .contains(FALSE_POSITIVE_TAG_FOR_STATUS)) { + break; + } case KEEP_STATUS_IF_SAME_TARGET: case KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET: textUnitForBatchImport.setIncludedInLocalizedFile( From 2d3a90c778921456c0dc572e1a04b34b51f9a091 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 28 Oct 2024 11:28:55 -0700 Subject: [PATCH 085/105] Add measurement to PhraseClient wait for upload to finish --- .../mojito/service/thirdparty/phrase/PhraseClient.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java index f9264726e2..71d5259543 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -6,6 +6,7 @@ import com.box.l10n.mojito.json.ObjectMapper; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableSet; import com.phrase.client.ApiClient; import com.phrase.client.ApiException; @@ -92,10 +93,11 @@ public Upload uploadAndWait( Upload waitForUploadToFinish(String projectId, String uploadId) { UploadsApi uploadsApi = new UploadsApi(apiClient); - try { logger.debug("Waiting for upload to finish: {}", uploadId); + Stopwatch stopwatch = Stopwatch.createStarted(); + Upload upload = uploadsApi.uploadShow(projectId, uploadId, null, null); logger.debug( "Upload info, first fetch: {}", new ObjectMapper().writeValueAsStringUnchecked(upload)); @@ -117,6 +119,8 @@ Upload waitForUploadToFinish(String projectId, String uploadId) { "Upload failed: %s".formatted(new ObjectMapper().writeValueAsStringUnchecked(upload))); } + logger.info("Waited: {} for upload: {} to finish", stopwatch.elapsed(), uploadId); + return upload; } catch (ApiException e) { logger.error("Error calling Phrase for waitForUploadToFinish: {}", e.getResponseBody()); @@ -216,6 +220,8 @@ public String nativeUploadCreateFile( List tags, Map formatOptions) { + Stopwatch stopwatch = Stopwatch.createStarted(); + String urlString = String.format("%s/projects/%s/uploads", apiClient.getBasePath(), projectId); String boundary = UUID.randomUUID().toString(); final String LINE_FEED = "\r\n"; @@ -269,6 +275,8 @@ public String nativeUploadCreateFile( throw new RuntimeException(e); } + logger.info("nativeUploadCreateFile took: {}", stopwatch.elapsed()); + int statusCode = response.statusCode(); String responseBody = response.body(); From a95e7bf86b2aa224deb4cf7f4fc5c41872bc353a Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 28 Oct 2024 11:29:44 -0700 Subject: [PATCH 086/105] Add Json format output in OpenAIClient --- common/pom.xml | 5 + .../box/l10n/mojito/openai/OpenAIClient.java | 92 ++++++++++++++++++- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 94e7c73393..38ac5f7840 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -60,6 +60,11 @@ jackson-module-blackbird + + com.fasterxml.jackson.module + jackson-module-jsonSchema + + org.springframework.boot spring-boot-starter diff --git a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java index fdcf1facf0..6fb19b640c 100644 --- a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java +++ b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java @@ -5,15 +5,22 @@ import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; @@ -214,7 +221,8 @@ public record ChatCompletionsRequest( @JsonProperty("max_tokens") int maxTokens, @JsonProperty("top_p") float topP, @JsonProperty("frequency_penalty") float frequencyPenalty, - @JsonProperty("presence_penalty") float presencePenalty) { + @JsonProperty("presence_penalty") float presencePenalty, + @JsonProperty("response_format") ResponseFormat responseFormat) { static String ENDPOINT = "/v1/chat/completions"; @@ -228,6 +236,79 @@ public enum Models { } } + public interface ResponseFormat {} + + public record JsonFormat(String type, @JsonProperty("json_schema") JsonSchema jsonSchema) + implements ResponseFormat { + + public record JsonSchema(boolean strict, String name, Object schema) { + + public static ObjectNode createJsonSchema(Class type) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(objectMapper); + com.fasterxml.jackson.module.jsonSchema.JsonSchema baseSchema = null; + try { + baseSchema = schemaGen.generateSchema(type); + } catch (JsonMappingException e) { + throw new RuntimeException(e); + } + JsonNode schemaNode = objectMapper.valueToTree(baseSchema); + ObjectNode rootNode = (ObjectNode) schemaNode; + enhanceSchema(rootNode); + return rootNode; + } + + private static void enhanceSchema(ObjectNode objectNode) { + + if (!objectNode.has("type")) { + objectNode.put("type", "object"); + } + objectNode.put("additionalProperties", false); + + if (objectNode.has("properties")) { + ObjectNode propertiesNode = (ObjectNode) objectNode.get("properties"); + ArrayNode requiredFields = objectNode.putArray("required"); + + Iterator> fields = propertiesNode.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + String fieldName = field.getKey(); + requiredFields.add(fieldName); + + JsonNode fieldSchema = field.getValue(); + if (fieldSchema.isObject()) { + ObjectNode fieldObjectNode = (ObjectNode) fieldSchema; + + String fieldType = + fieldObjectNode.has("type") ? fieldObjectNode.get("type").asText() : null; + + if ("object".equals(fieldType) && fieldObjectNode.has("properties")) { + enhanceSchema(fieldObjectNode); + } else if ("array".equals(fieldType) && fieldObjectNode.has("items")) { + enhanceArrayItems(fieldObjectNode); + } + } + } + } + } + + private static void enhanceArrayItems(ObjectNode arrayNode) { + JsonNode itemsNode = arrayNode.get("items"); + if (itemsNode != null && itemsNode.isObject()) { + ObjectNode itemsObjectNode = (ObjectNode) itemsNode; + + if (itemsObjectNode.has("properties")) { + enhanceSchema(itemsObjectNode); + } + + if (!itemsObjectNode.has("additionalProperties")) { + itemsObjectNode.put("additionalProperties", false); + } + } + } + } + } + public interface Message { String role(); @@ -314,6 +395,7 @@ public static class Builder { private float topP; private float frequencyPenalty; private float presencePenalty; + private ResponseFormat responseFormat; public Builder model(Models model) { return model(model.name); @@ -364,6 +446,11 @@ public Builder presencePenalty(float presencePenalty) { return this; } + public Builder responseFormat(ResponseFormat responseFormat) { + this.responseFormat = responseFormat; + return this; + } + public ChatCompletionsRequest build() { return new ChatCompletionsRequest( model, @@ -374,7 +461,8 @@ public ChatCompletionsRequest build() { maxTokens, topP, frequencyPenalty, - presencePenalty); + presencePenalty, + responseFormat); } } From 4bb6edf5913cb587fa430fed255de57a944b08af Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 28 Oct 2024 11:31:11 -0700 Subject: [PATCH 087/105] Add measurement to ThirdPartyTMSPhrase --- .../service/thirdparty/ThirdPartyTMSPhrase.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index c3211bc942..30c904e666 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -157,14 +157,17 @@ public void push( final AtomicReference escapeType = getEscapeTypeAtomicReference(nativeClient, optionsParser); + Stopwatch stopwatchGetTextUnitDTO = Stopwatch.createStarted(); List search = getSourceTextUnitDTOs(repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); + logger.info("Get Source TextUnitDTO for push took: {}", stopwatchGetTextUnitDTO.elapsed()); String text = getFileContent(pluralSeparator, search, true, null, escapeType.get()); String tagForUpload = getTagForUpload(repository.getName()); if (nativeClient) { - logger.info("Pushing with native and options: {}", formatOptions); + logger.debug("Pushing with native and options: {}", formatOptions); + Stopwatch stopWatchPhaseNativePush = Stopwatch.createStarted(); phraseClient.nativeUploadAndWait( projectId, repository.getSourceLocale().getBcp47Tag(), @@ -173,6 +176,10 @@ public void push( text, ImmutableList.of(tagForUpload), formatOptions.isEmpty() ? null : formatOptions); + logger.info( + "Pushing with native and options: {}, took: {}", + formatOptions, + stopWatchPhaseNativePush.elapsed()); } else { phraseClient.uploadAndWait( projectId, @@ -214,6 +221,8 @@ public void push( public void removeUnusedKeysAndTags( String projectId, String repositoryName, String tagForUpload) { + Stopwatch stopwatchRemoveUnusedKeysAndTags = Stopwatch.createStarted(); + List tagsForOtherRepositories = phraseClient.listTags(projectId).stream() .map(Tag::getName) @@ -225,7 +234,7 @@ public void removeUnusedKeysAndTags( List allActiveTags = new ArrayList<>(tagsForOtherRepositories); allActiveTags.add(tagForUpload); - logger.info("All active tags: {}", allActiveTags); + logger.debug("All active tags: {}", allActiveTags); phraseClient.removeKeysNotTaggedWith(projectId, allActiveTags); List pushTagsToDelete = @@ -238,6 +247,8 @@ public void removeUnusedKeysAndTags( logger.info("Tags to delete: {}", pushTagsToDelete); phraseClient.deleteTags(projectId, pushTagsToDelete); + + logger.info("removeUnusedKeysAndTags took: {}", stopwatchRemoveUnusedKeysAndTags); } private List getSourceTextUnitDTOs( From 6d7b9d7de22c075db39e695cdee3d89d5ffd7fbb Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 28 Oct 2024 11:40:00 -0700 Subject: [PATCH 088/105] Add AI Review webservice The WS allows to review a string at a time using OpenAI chat completion API with Json format output. It grades current translation and string comment, propose alternative translations. --- .../AiReviewConfigurationProperties.java | 18 ++ .../l10n/mojito/rest/textunit/AiReviewWS.java | 189 ++++++++++++++++++ .../service/tm/search/TextUnitSearcher.java | 4 + .../tm/search/TextUnitSearcherParameters.java | 10 + 4 files changed, 221 insertions(+) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewConfigurationProperties.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewWS.java diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewConfigurationProperties.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewConfigurationProperties.java new file mode 100644 index 0000000000..11584eae63 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewConfigurationProperties.java @@ -0,0 +1,18 @@ +package com.box.l10n.mojito.rest.textunit; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties("l10n.ai-review") +public class AiReviewConfigurationProperties { + String openaiClientToken; + + public String getOpenaiClientToken() { + return openaiClientToken; + } + + public void setOpenaiClientToken(String openaiClientToken) { + this.openaiClientToken = openaiClientToken; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewWS.java new file mode 100644 index 0000000000..28ccd9a962 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewWS.java @@ -0,0 +1,189 @@ +package com.box.l10n.mojito.rest.textunit; + +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.JsonFormat.JsonSchema.createJsonSchema; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.SystemMessage.systemMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.UserMessage.userMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.chatCompletionsRequest; + +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.openai.OpenAIClient; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class AiReviewWS { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(AiReviewWS.class); + + TextUnitSearcher textUnitSearcher; + + AiReviewConfigurationProperties aiReviewConfigurationProperties; + + public AiReviewWS( + TextUnitSearcher textUnitSearcher, + AiReviewConfigurationProperties aiReviewConfigurationProperties) { + this.textUnitSearcher = textUnitSearcher; + this.aiReviewConfigurationProperties = aiReviewConfigurationProperties; + } + + @RequestMapping(method = RequestMethod.GET, value = "/api/proto-ai-review") + @ResponseStatus(HttpStatus.OK) + public ProtoAiReviewResponse getTextUnitsWithGet(ProtoAiReviewRequest protoAiReviewRequest) { + + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setTmTextUnitVariantId(protoAiReviewRequest.tmTextUnitVariantId); + + List search = textUnitSearcher.search(textUnitSearcherParameters); + if (search.isEmpty()) { + throw new RuntimeException("Wrong tmTextUnitVariantId"); + } + + TextUnitDTO textUnit = search.getFirst(); + + AiReviewInput input = + new AiReviewInput( + textUnit.getTargetLocale(), + textUnit.getSource(), + textUnit.getComment(), + new AiReviewInput.ExistingTarget( + textUnit.getTarget(), !textUnit.isIncludedInLocalizedFile())); + + ObjectMapper objectMapper = ObjectMapper.withIndentedOutput(); + String inputAsJsonString = objectMapper.writeValueAsStringUnchecked(input); + + ObjectNode jsonSchema = createJsonSchema(AiReviewOutput.class); + + OpenAIClient.ChatCompletionsRequest chatCompletionsRequest = + chatCompletionsRequest() + .model("gpt-4o-2024-08-06") + .maxTokens(16384) + .messages( + List.of( + systemMessageBuilder().content(PROMPT).build(), + userMessageBuilder().content(inputAsJsonString).build())) + .responseFormat( + new OpenAIClient.ChatCompletionsRequest.JsonFormat( + "json_schema", + new OpenAIClient.ChatCompletionsRequest.JsonFormat.JsonSchema( + true, "request_json_format", jsonSchema))) + .build(); + + logger.info(objectMapper.writeValueAsStringUnchecked(chatCompletionsRequest)); + + OpenAIClient openAIClient = + OpenAIClient.builder() + .apiKey(aiReviewConfigurationProperties.getOpenaiClientToken()) + .build(); + + OpenAIClient.ChatCompletionsResponse chatCompletionsResponse = + openAIClient.getChatCompletions(chatCompletionsRequest).join(); + + logger.info(objectMapper.writeValueAsStringUnchecked(chatCompletionsResponse)); + + String jsonResponse = chatCompletionsResponse.choices().getFirst().message().content(); + AiReviewOutput review = objectMapper.readValueUnchecked(jsonResponse, AiReviewOutput.class); + + return new ProtoAiReviewResponse(textUnit, review); + } + + public record ProtoAiReviewRequest(long tmTextUnitVariantId) {} + + public record ProtoAiReviewResponse(TextUnitDTO textUnitDTO, AiReviewOutput aiReviewOutput) {} + + record AiReviewOutput( + String source, + Target target, + DescriptionRating descriptionRating, + AltTarget altTarget, + ExistingTargetRating existingTargetRating, + ReviewRequired reviewRequired) { + record Target(String content, String explanation, int confidenceLevel) {} + + record AltTarget(String content, String explanation, int confidenceLevel) {} + + record DescriptionRating(String explanation, int score) {} + + record ExistingTargetRating(String explanation, int score) {} + + record ReviewRequired(boolean required, String reason) {} + } + + record AiReviewInput( + String locale, String source, String sourceDescription, ExistingTarget existingTarget) { + record ExistingTarget(String content, boolean hasBrokenPlaceholders) {} + } + + static final String PROMPT = + """ + Your role is to act as a translator. + You are tasked with translating provided source strings while preserving both the tone and the technical structure of the string. This includes protecting any tags, placeholders, or code elements that should not be translated. + + The input will be provided in JSON format with the following fields: + + • "source": The source text to be translated. + • "locale": The target language locale, following the BCP47 standard (e.g., “fr”, “es-419”). + • "sourceDescription": A description providing context for the source text. + • "existingTarget" (optional): An existing translation to review. + + Instructions: + + • If the source is colloquial, keep the translation colloquial; if it’s formal, maintain formality in the translation. + • Pay attention to regional variations specified in the "locale" field (e.g., “es” vs. “es-419”, “fr” vs. “fr-CA”, “zh” vs. “zh-Hant”), and ensure the translation length remains similar to the source text. + • Aim to provide the best translation, while compromising on length to ensure it remains close to the original text length + + Handling Tags and Code: + + Some strings contain code elements such as tags (e.g., {atag}, ICU message format, or HTML tags). You are provided with a inputs of tags that need to be protected. Ensure that: + + • Tags like {atag} remain untouched. + • In cases of nested content (e.g., text that needs translation), only translate the inner text while preserving the outer structure. + • Complex structures like ICU message formats should have placeholders or variables left intact (e.g., {count, plural, one {# item} other {# items}}), but translate any inner translatable text. + + Ambiguity and Context: + + After translating, assess the usefulness of the "sourceDescription" field: + + • Rate its usefulness on a scale of 0 to 2: + • 0 – Not helpful at all; irrelevant or misleading. + • 1 – Somewhat helpful; provides partial or unclear context but is useful to some extent. + • 2 – Very helpful; provides clear and sufficient guidance for the translation. + + If the source is ambiguous—for example, if it could be interpreted as a noun or a verb—you must: + + • Indicate the ambiguity in your explanation. + • Provide translations for all possible interpretations. + • Set "reviewRequired" to true, and explain the need for review due to the ambiguity. + + You will provide an output in JSON format with the following fields: + + • "source": The original source text. + • "target": An object containing: + • "content": The best translation. + • "explanation": A brief explanation of your translation choices. + • "confidenceLevel": Your confidence level (0-100%) in the translation. + • "descriptionRating": An object containing: + • "explanation": An explanation of how the "sourceDescription" aided your translation. + • "score": The usefulness score (0-2). + • "altTarget": An object containing: + • "content": An alternative translation, if applicable. Focus on showcasing grammar differences, + • "explanation": Explanation for the alternative translation. + • "confidenceLevel": Your confidence level (0-100%) in the alternative translation. + • "existingTargetRating" (if "existingTarget" is provided): An object containing: + • "explanation": Feedback on the existing translation’s accuracy and quality. + • "score": A rating score (0-2). + • "reviewRequired": An object containing: + • "required": true or false, indicating if review is needed. + • "reason": A detailed explanation of why review is or isn’t needed. + """; +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java index 515fa38798..216d6f8e95 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java @@ -430,6 +430,10 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) new NativeDateGteExp("tu.created_date", searchParameters.getTmTextUnitCreatedAfter())); } + if (searchParameters.getTmTextUnitVariantId() != null) { + conjunction.add(new NativeEqExpFix("tuv.id", searchParameters.getTmTextUnitVariantId())); + } + if (!conjunction.toSQL().isEmpty()) { c.add(conjunction); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java index cc062c6afc..9792093ff0 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java @@ -50,6 +50,8 @@ public class TextUnitSearcherParameters { String includeTextUnitsWithPattern; String skipAssetPathWithPattern; + Long tmTextUnitVariantId; + public String getName() { return name; } @@ -287,6 +289,14 @@ public void setTmTextUnitIds(Long... tmTextUnitIds) { } } + public Long getTmTextUnitVariantId() { + return tmTextUnitVariantId; + } + + public void setTmTextUnitVariantId(Long tmTextUnitVariantId) { + this.tmTextUnitVariantId = tmTextUnitVariantId; + } + public String getSkipTextUnitWithPattern() { return skipTextUnitWithPattern; } From ec85151b65e61ffcf5b15dd02cfbddde41abdcc4 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 28 Oct 2024 11:41:15 -0700 Subject: [PATCH 089/105] Add a workbench modal to review translation using AI This is one string at a time. Review the current translation, the comment/description and suggest alternative translations --- .../main/resources/properties/en.properties | 6 + .../js/actions/workbench/AiReviewActions.js | 15 + .../actions/workbench/TextUnitDataSource.js | 9 + .../js/components/workbench/AIReviewModal.js | 291 ++++++++++++++++++ .../js/components/workbench/TextUnit.js | 17 + .../js/components/workbench/Workbench.js | 8 + .../resources/public/js/sdk/TextUnitClient.js | 6 + .../js/stores/workbench/AiReviewStore.js | 46 +++ webapp/src/main/resources/sass/mojito.scss | 167 ++++++++++ 9 files changed, 565 insertions(+) create mode 100644 webapp/src/main/resources/public/js/actions/workbench/AiReviewActions.js create mode 100644 webapp/src/main/resources/public/js/components/workbench/AIReviewModal.js create mode 100644 webapp/src/main/resources/public/js/stores/workbench/AiReviewStore.js diff --git a/webapp/src/main/resources/properties/en.properties b/webapp/src/main/resources/properties/en.properties index 58bca97d29..26555aa4d3 100644 --- a/webapp/src/main/resources/properties/en.properties +++ b/webapp/src/main/resources/properties/en.properties @@ -758,6 +758,9 @@ textUnit.gitBlameModal.screenshotModalOpen=View # Tooltip for button to open the translation history modal workbench.translationHistoryModal.info=Text unit history +# Tooltip for button to open the translation history modal +workbench.aiReviewModal.info=AI Review + # Title of the translation history modal dialog workbench.translationHistoryModal.title=Translation History @@ -851,3 +854,6 @@ userErrorModal.delete=The user could not be deleted userErrorModal.forbidden=You do not have the necessary permissions for this operation. users.userInformationAlert=Administrators and Project Managers have write access to the user management as well as creating and deleting repositories. Translators can only translate. Users do not have any write access. + + +aiReviewModal.title=AI Review diff --git a/webapp/src/main/resources/public/js/actions/workbench/AiReviewActions.js b/webapp/src/main/resources/public/js/actions/workbench/AiReviewActions.js new file mode 100644 index 0000000000..9c6e5bbbe1 --- /dev/null +++ b/webapp/src/main/resources/public/js/actions/workbench/AiReviewActions.js @@ -0,0 +1,15 @@ +import alt from "../../alt"; + +class AiReviewActions { + + constructor() { + this.generateActions( + "openWithTextUnit", + "getAiReviewSuccess", + "getAiReviewError", + "close" + ); + } +} + +export default alt.createActions(AiReviewActions); diff --git a/webapp/src/main/resources/public/js/actions/workbench/TextUnitDataSource.js b/webapp/src/main/resources/public/js/actions/workbench/TextUnitDataSource.js index 77d6cbb513..30c0d7ef0d 100644 --- a/webapp/src/main/resources/public/js/actions/workbench/TextUnitDataSource.js +++ b/webapp/src/main/resources/public/js/actions/workbench/TextUnitDataSource.js @@ -4,6 +4,7 @@ import TextUnitClient from "../../sdk/TextUnitClient"; import WorkbenchActions from "./WorkbenchActions"; import GitBlameActions from "./GitBlameActions"; import TranslationHistoryActions from "./TranslationHistoryActions"; +import AiReviewActions from "./AiReviewActions"; const TextUnitDataSource = { performSaveTextUnit: { @@ -69,6 +70,14 @@ const TextUnitDataSource = { }, success: TranslationHistoryActions.getTranslationHistorySuccess, error: TranslationHistoryActions.getTranslationHistoryError + }, + + getAiReview: { + remote(aiReviewStoreState, textUnit) { + return TextUnitClient.getAiReview(textUnit); + }, + success: AiReviewActions.getAiReviewSuccess, + error: AiReviewActions.getAiReviewError } }; diff --git a/webapp/src/main/resources/public/js/components/workbench/AIReviewModal.js b/webapp/src/main/resources/public/js/components/workbench/AIReviewModal.js new file mode 100644 index 0000000000..51980b0f89 --- /dev/null +++ b/webapp/src/main/resources/public/js/components/workbench/AIReviewModal.js @@ -0,0 +1,291 @@ +import PropTypes from 'prop-types'; +import React from "react"; +import {FormattedMessage, injectIntl} from "react-intl"; +import {Button, Glyphicon, Modal} from "react-bootstrap"; +import {withAppConfig} from "../../utils/AppConfig"; +import TextUnitSDK from "../../sdk/TextUnit"; + +class AIReviewModal extends React.Component { + static propTypes() { + return { + "show": PropTypes.bool.isRequired, + "onCloseModal": PropTypes.func.isRequired, + }; + } + + closeModal = () => { + this.props.onCloseModal(); + }; + + getTitle = () => { + return this.props.intl.formatMessage({id: "aiReviewModal.title"}); + }; + + renderRating = (rating) => { + let label; + switch (rating) { + case 0: + label = bad; + break; + case 2: + label = good; + break; + default: + label = average; + } + + return label; + }; + + renderHeader() { + return + {this.props.textUnit.getTargetLocale()} + {this.props.textUnit.getName()} + + } + + renderHeaderRight() { + return + asset: {this.props.textUnit.getAssetPath()} + repo: {this.props.textUnit.getRepositoryName()} + + } + + + renderSource() { + return {this.props.textUnit.getSource()} + } + + renderTarget() { + return {this.props.textUnit.getTarget()} + } + + renderTargetStatus() { + return

{this.renderReviewGlyph()}
+ } + + renderComment() { + return {this.props.textUnit.getComment()} + } + + renderCommentRating() { + return this.props.review && + +
{this.renderRating(this.props.review.descriptionRating.score)}
+
{this.props.review.descriptionRating.explanation}
+ + } + + renderTargetRating() { + return this.props.review && + +
{this.renderRating(this.props.review.existingTargetRating.score)}
+
{this.props.review.existingTargetRating.explanation}
+ + } + + renderSuggestion1() { + return this.props.review && +
{this.props.review.target.content} {this.props.review.target.confidenceLevel}
+ + + } + + renderSuggestion2() { + return this.props.review && +
{this.props.review.altTarget.content} {this.props.review.altTarget.confidenceLevel}
+ + } + + /* TODO(ja) make a component instead of copy/paste */ + renderReviewGlyph() { + + let ui = ""; + if (this.props.textUnit.isTranslated()) { + + let glyphType = "ok"; + let glyphTitle = this.props.intl.formatMessage({id: "textUnit.reviewModal.accepted"}); + + if (!this.props.textUnit.isIncludedInLocalizedFile()) { + + glyphType = "alert"; + glyphTitle = this.props.intl.formatMessage({id: "textUnit.reviewModal.rejected"}); + + } else if (this.props.textUnit.getStatus() === TextUnitSDK.STATUS.REVIEW_NEEDED) { + + glyphType = "eye-open"; + glyphTitle = this.props.intl.formatMessage({id: "textUnit.reviewModal.needsReview"}); + + } else if (this.props.textUnit.getStatus() === TextUnitSDK.STATUS.TRANSLATION_NEEDED) { + + glyphType = "edit"; + glyphTitle = this.props.intl.formatMessage({id: "textUnit.reviewModal.translationNeeded"}); + } + ui = ( + + ); + } + + return ui; + } + + render() { + if (!this.props.show) return null; + + return ( + + + {this.getTitle()} + + + +
+
Text Unit
+
+
+ {this.props.textUnit.getTargetLocale()} + {this.props.textUnit.getName()} + {this.props.textUnit.getAssetPath()} + {this.props.textUnit.getRepositoryName()} +
+
+
{this.props.textUnit.getSource()}
+
{this.props.textUnit.getComment()}
+
+
+
{this.renderTarget()}
+
{this.renderReviewGlyph()}
+
+
+ + {this.props.review == null ? +
+ {this.props.loading && + } +
+ : +
Target Analysis
+
+
+ {this.renderRating(this.props.review.existingTargetRating.score)} +
+
+ {this.props.review.existingTargetRating.explanation} +
+
+ +
Target Suggestions
+
+
+ {this.props.review.target.confidenceLevel} +
+
+ {this.props.review.target.content} +
+
+
+
+ {this.props.review.altTarget.confidenceLevel} +
+
+ {this.props.review.altTarget.content} +
+
+ +
Comment Analysis
+
+
+ {this.renderRating(this.props.review.descriptionRating.score)} +
+
+ {this.props.review.descriptionRating.explanation} +
+
+ + } +
+ {/*
*/} + {/*
{this.renderHeader()}
*/} + {/*
{this.renderHeaderRight()}
*/} + {/*
{this.renderSource()}
*/} + {/*
{this.renderTarget()}
*/} + {/* /!*
{this.renderTargetStatus()}
*!/*/} + {/*
{this.renderComment()}
*/} + {/*
Comment and Target Ratings
*/} + {/*
{this.renderCommentRating()}
*/} + {/*
{this.renderTargetRating()}
*/} + {/*
Target Suggestions
*/} + {/*
{this.renderSuggestion1()}
*/} + {/*
{this.renderSuggestion2()}
*/} + {/*
*/} + + + {/*
*/} + {/*
*/} + {/*
{this.props.textUnit.getSource()}
*/} + {/*
{this.props.textUnit.getComment()}
*/} + {/*
*/} + {/*
*/} + {/*
{this.props.textUnit.getTarget()}
*/} + + {/* {this.props.loading ? :*/} + {/*
*/} + {/*
{this.renderRating(this.getRating())}*/} + {/*
*/} + {/*{this.props.review.existingTargetRating.explanation}
*/} + + {/*
*/} + {/*
*/} + {/*
*/} + + + {/*
Suggestions
*/} + {/*
{this.props.review.target.content}
*/} + {/*{this.props.review.target.explanation}*/} + {/*
{this.props.review.target.confidenceLevel}*/} + {/*
*/} + + {/*
*/} + {/*
{this.props.review.altTarget.content}
*/} + {/*{this.props.review.altTarget.explanation}*/} + {/*
{this.props.review.altTarget.confidenceLevel}
*/} + + {/*
*/} + {/* {this.props.review.descriptionRating.explanation}
*/} + {/**/} + + {/*{this.renderRating(this.props.review.descriptionRating.score)}*/} + {/* */} + {/* }*/} + {/**/} + + + + + + + + ); + } + + getRating() { + return this.props.review.existingTargetRating.score; + } +} + +export default withAppConfig(injectIntl(AIReviewModal)); diff --git a/webapp/src/main/resources/public/js/components/workbench/TextUnit.js b/webapp/src/main/resources/public/js/components/workbench/TextUnit.js index deeae25400..89c7a734f8 100644 --- a/webapp/src/main/resources/public/js/components/workbench/TextUnit.js +++ b/webapp/src/main/resources/public/js/components/workbench/TextUnit.js @@ -34,6 +34,7 @@ import { Tooltip } from "react-bootstrap"; import ViewModeStore from "../../stores/workbench/ViewModeStore"; +import AiReviewActions from "../../actions/workbench/AiReviewActions"; let TextUnit = createReactClass({ @@ -658,6 +659,13 @@ let TextUnit = createReactClass({ TranslationHistoryActions.openWithTextUnit(this.props.textUnit); }, + onAiReviewClick(e){ + + e.stopPropagation(); + + AiReviewActions.openWithTextUnit(this.props.textUnit); + }, + /** * Handle click on the asset path icon: stop event propagation (no need to bubble * up as we're reloading the workbench with new data) and update the search @@ -859,6 +867,10 @@ let TextUnit = createReactClass({ let assetPathTranslationHistoryTooltip = {this.props.intl.formatMessage( {id: 'workbench.translationHistoryModal.info'} )}; + let aiReviewTooltip = + {this.props.intl.formatMessage( {id: 'workbench.aiReviewModal.info'} )}; + + // Only show the overlay trigger for the translation history if there is a current text unit variant (ie. there is // at least one translation) If not, we have no history to show anyways! return ( + + + + ); }, diff --git a/webapp/src/main/resources/public/js/components/workbench/Workbench.js b/webapp/src/main/resources/public/js/components/workbench/Workbench.js index d7a9616deb..47d90d9295 100644 --- a/webapp/src/main/resources/public/js/components/workbench/Workbench.js +++ b/webapp/src/main/resources/public/js/components/workbench/Workbench.js @@ -23,6 +23,9 @@ import ShareSearchParamsModal from "./ShareSearchParamsModal"; import ShareSearchParamsModalActions from "../../actions/workbench/ShareSearchParamsModalActions"; import ShareSearchParamsButton from "./ShareSearchParamsButton"; import AuthorityService from "../../utils/AuthorityService"; +import AIReviewModal from "./AIReviewModal"; +import AiReviewActions from "../../actions/workbench/AiReviewActions"; +import AIReviewStore from "../../stores/workbench/AiReviewStore"; let Workbench = createReactClass({ displayName: 'Workbench', @@ -49,6 +52,11 @@ let Workbench = createReactClass({ GitBlameScreenshotViewerActions.openScreenshotsViewer(branchScreenshots); }}/> + + + { diff --git a/webapp/src/main/resources/public/js/sdk/TextUnitClient.js b/webapp/src/main/resources/public/js/sdk/TextUnitClient.js index 57ef104ea6..eaea836a59 100644 --- a/webapp/src/main/resources/public/js/sdk/TextUnitClient.js +++ b/webapp/src/main/resources/public/js/sdk/TextUnitClient.js @@ -110,6 +110,12 @@ class TextUnitClient extends BaseClient { }); } + getAiReview(textUnit) { + return this.get(this.baseUrl + "proto-ai-review", { + "tmTextUnitVariantId": textUnit.getTmTextUnitVariantId() + }); + } + getEntityName() { return 'textunits'; } diff --git a/webapp/src/main/resources/public/js/stores/workbench/AiReviewStore.js b/webapp/src/main/resources/public/js/stores/workbench/AiReviewStore.js new file mode 100644 index 0000000000..fb212fe0c9 --- /dev/null +++ b/webapp/src/main/resources/public/js/stores/workbench/AiReviewStore.js @@ -0,0 +1,46 @@ +import alt from "../../alt"; +import TextUnitDataSource from "../../actions/workbench/TextUnitDataSource"; +import AIReviewActions from "../../actions/workbench/AiReviewActions"; + +class AIReviewStore { + + constructor() { + this.setDefaultState(); + this.bindActions(AIReviewActions); + + this.registerAsync(TextUnitDataSource); + } + + setDefaultState() { + this.show = false; + this.textUnit = null; + this.review = null; + this.loading = false; + } + + close() { + this.show = false; + } + + openWithTextUnit(textUnit) { + this.show = true; + this.textUnit = textUnit; + this.review = null; + this.loading = true; + this.getInstance().getAiReview(textUnit); + } + + onGetAiReviewSuccess(protoAiReviewResponse) { + console.log("AIReviewStore::onGetAiReviewInfoSuccess"); + this.review = protoAiReviewResponse.aiReviewOutput; + console.log("\n\n\n\n" + this.review + "\n\n\n\n") + this.loading = false; + } + + onGetAiReviewError(errorResponse) { + console.log("AIReviewStore::onGetAiReviewInfoError"); + this.loading = false; + } +} + +export default alt.createStore(AIReviewStore, 'AIReviewStore'); diff --git a/webapp/src/main/resources/sass/mojito.scss b/webapp/src/main/resources/sass/mojito.scss index f2e197e201..f54082b5a6 100644 --- a/webapp/src/main/resources/sass/mojito.scss +++ b/webapp/src/main/resources/sass/mojito.scss @@ -442,6 +442,15 @@ div.textunit label { color: $brand-success; } +.textunit:hover .textunit-assetpath, .textunit:hover .textunit-ai-review { + visibility: visible; +} + +.textunit-name .textunit-assetpath, .textunit-name .textunit-ai-review { + visibility: hidden; + color: $brand-success; +} + .textunit-returnline { color: $brand-primary; font-size: 10px; @@ -466,6 +475,164 @@ div.textunit label { white-space: pre-wrap; } +.ai-review-modal { +} + +.ai-review-modal .modal-dialog { + max-width: 100%; + width: 90%; +} + + +.ai-review-root2 { + +} + +.ai-review-root2 .section-title { + font-weight: bold; + margin-top: 10px; + margin-bottom: 10px; +} + +.ai-review-root2 .text-unit { + display: grid; + grid-template-columns: 1fr 1fr; + grid-template-areas: + "header header" + "left right"; +} + +.ai-review-root2 .text-unit .right { + align-self: center; + display: grid; + grid-template-columns: 1fr min-content 10px; + grid-template-areas: + "target status ." +} + +.ai-review-root2 .grid-2c { + display: grid; + grid-template-columns: 1fr 11fr; + grid-template-areas: "left right"; +} + +.ai-review-root2 .grid-2c .left { + grid-area: left; + text-align: center; +} + +.ai-review-root2 .grid-2c .right { + grid-area: right; +} + + + +////// + + +.ai-review-root { + display: grid; + grid-gap: 20px; + grid-template-columns: min-content 2fr 1fr; + grid-template: + "header header-right" + "source target" + "comment target" + "ratings ratings" + "comment-rating target-rating" + "suggestions suggestions" + "suggestion1 suggestion2" + +} + +.ai-review-root .locale { + grid-area: locale; +} + +.ai-review-root .header { + grid-area: header; + display: flex; + gap: 10px; +} + +.ai-review-root .header-right { + grid-area: header-right; + display: flex; + gap: 5px; + flex-wrap: wrap; + align-content: space-around; +} + +.ai-review-root .source { + grid-area: source; +} + +.ai-review-root .target { + grid-area: target; + align-content: center; +} + +.ai-review-root .target-status { + grid-area: target-status; + text-align: right; + align-content: center; +} + + +.ai-review-root .comment { + grid-area: comment; +} + +.ai-review-root .comment-rating { + grid-area: comment-rating; + align-content: center; +} + +.ai-review-root .target-rating { + grid-area: target-rating; + align-content: center; +} + +.ai-review-root .ratings { + grid-area: ratings; + align-content: center; + font-weight: bold; + text-align: center; + margin-top: 10px; +} + +.ai-review-root .rating { + display: grid; + grid-template-columns: 1fr 11fr; + grid-gap: 5px; +} + + +.ai-review-root .suggestions { + display: grid; + grid-area: suggestions; + margin-top: 10px; + text-align: center; + font-weight: bold; + margin-bottom: 10px; +} + +.ai-review-root .suggestion1 { + grid-area: suggestion1; +} + +.ai-review-root .suggestion2 { + grid-area: suggestion2; +} + + +.ai-review-modal .loading { + display: flex; + align-items: center; + justify-content: center; + min-height: 75px; +} + .text-unit-root { display: grid; align-items: center; From c64e65241cfec3a344721c627a359ecd9be53ada Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Sat, 5 Oct 2024 15:21:19 -0700 Subject: [PATCH 090/105] Add option to GithubCreatePRCommand for closing PRs with a specific prefix Adds the --also-close-prefixed option in the GithubCreatePRCommand. It enables the command to close open PRs that start with the provided prefix. --- .../cli/command/GithubCreatePRCommand.java | 31 +++++++++++++++++++ .../box/l10n/mojito/github/GithubClient.java | 9 ++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java index 6bf6a14ceb..fa392b2e7c 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java @@ -6,8 +6,10 @@ import com.box.l10n.mojito.github.GithubClient; import com.box.l10n.mojito.github.GithubClients; import com.box.l10n.mojito.github.GithubException; +import java.io.IOException; import java.util.List; import org.fusesource.jansi.Ansi; +import org.kohsuke.github.GHIssueState; import org.kohsuke.github.GHPullRequest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -90,6 +92,13 @@ public class GithubCreatePRCommand extends Command { description = "The PR labels") List labels; + @Parameter( + names = {"--also-close-prefixed"}, + description = + "Closes all PRs that start with the specified prefix. Use this to manage PRs that are part of a batch or related by a common feature.", + required = false) + String alsoClosePrefixed = null; + enum EnableAutoMergeType { SQUASH, MERGE, @@ -107,6 +116,10 @@ protected void execute() throws CommandException { GithubClient githubClient = githubClients.getClient(owner); + if (alsoClosePrefixed != null) { + closePRPrefixedWith(githubClient, alsoClosePrefixed); + } + GHPullRequest pr = githubClient.createPR(repository, title, head, base, body, reviewers); consoleWriter.a("PR created: ").fg(Ansi.Color.CYAN).a(pr.getHtmlUrl().toString()).println(); @@ -120,4 +133,22 @@ protected void execute() throws CommandException { throw new CommandException(e); } } + + void closePRPrefixedWith(GithubClient githubClient, String alsoClosePrefixed) { + githubClient.listPR(repository, GHIssueState.OPEN).stream() + .filter(pr -> pr.getTitle().startsWith(alsoClosePrefixed)) + .forEach( + pullRequest -> { + try { + consoleWriter + .a("Closing: ") + .fg(Ansi.Color.CYAN) + .a(pullRequest.getHtmlUrl().toString()) + .println(); + pullRequest.close(); + } catch (IOException e) { + throw new CommandException(e); + } + }); + } } diff --git a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java index 2414b791a7..a9de13a22c 100644 --- a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java +++ b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java @@ -23,6 +23,7 @@ import org.kohsuke.github.GHAppInstallationToken; import org.kohsuke.github.GHCommitState; import org.kohsuke.github.GHIssueComment; +import org.kohsuke.github.GHIssueState; import org.kohsuke.github.GHPullRequest; import org.kohsuke.github.GHUser; import org.kohsuke.github.GitHub; @@ -314,10 +315,12 @@ public List getPRComments(String repository, int prNumber) { } } - public void listPR(String repository) { + public List listPR(String repository, GHIssueState state) { try { - GitHub gc = getGithubClient(repository); - gc.getRepository(repository); + String repoFullPath = getRepositoryPath(repository); + return getGithubClient(repository) + .getRepository(repoFullPath) + .getPullRequests(GHIssueState.OPEN); } catch (IOException | NoSuchAlgorithmException | InvalidKeySpecException e) { throw new RuntimeException(e); } From 5951e456ec3ae21e33154c54c2edf5e8973dbeae Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 23 Oct 2024 15:08:43 -0700 Subject: [PATCH 091/105] Document Quartz multi scheduler configuration --- .../resources/config/application.properties | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/webapp/src/main/resources/config/application.properties b/webapp/src/main/resources/config/application.properties index 8d8f46baba..8745bea184 100644 --- a/webapp/src/main/resources/config/application.properties +++ b/webapp/src/main/resources/config/application.properties @@ -112,6 +112,55 @@ l10n.org.quartz.scheduler.skipUpdateCheck=true #l10n.org.quartz.dataSource.myDS.maxConnections=27 #l10n.org.quartz.dataSource.myDS.validationQuery=select 1 +### for multiple Quartz schedulers +#l10n.org.multi-quartz.enabled=true +### Copy/paste the following section as many time as needed, renaming "default" to scheduler names eg. "ai-translate". +### By default, different services use the default scheduler (search for DEFAULT_SCHEDULER_NAME usages) +### configure services as needed to use a different scheduler eg. + +#l10n.ai-translate.scheduler-name=ai-translate + +#l10n.assetWS.quartz.schedulerName= +#l10n.assetService.quartz.schedulerName= +#l10n.assetExtraction.quartz.schedulerName= +#l10n.branchService.quartz.schedulerName= +#l10n.branchStatistic.quartz.schedulerName= +#l10n.branchNotification.quartz.schedulerName= +#l10n.repositoryStatistics.quartz.schedulerName= +#l10n.thirdPartyService.quartz.schedulerName= +#l10n.tmService.quartz.schedulerName= +#l10n.textUnitBatchImporterService.quartz.schedulerName +#l10n.machineTranslation.quartz.schedulerName= + +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.useProperties=true +#l10n.org.multi-quartz.schedulers.default.quartz.scheduler.instanceId=AUTO +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.isClustered=true +#l10n.org.multi-quartz.schedulers.default.quartz.threadPool.threadCount=10 +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.class=org.quartz.impl.jdbcjobstore.JobStoreTX +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.dataSource=myDS +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.provider=hikaricp +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.driver=com.mysql.jdbc.Driver +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.URL=jdbc:mysql://localhost:3306/mojito?characterEncoding=UTF-8&useUnicode=true +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.user=mojito +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.password=mojito +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.maxConnections=12 +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.validationQuery=select 1 +# +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.useProperties=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.useProperties} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.scheduler.instanceId=${l10n.org.multi-quartz.schedulers.default.quartz.scheduler.instanceId} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.isClustered=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.isClustered} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.threadPool.threadCount=10 +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.class=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.class} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.driverDelegateClass=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.driverDelegateClass} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.dataSource=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.dataSource} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.provider=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.provider} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.driver=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.driver} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.URL=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.URL} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.user=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.user} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.password=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.password} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.maxConnections=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.maxConnections} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.validationQuery=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.validationQuery} ### Settings to enable/disable upgrade jobs. "true" by default to make upgrades easy. # Once upgrades are done it can be set to false to stop running the logic. From 210d9cceedd38753d096ee658891b4ac4bf6dee6 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 23 Oct 2024 17:46:39 -0700 Subject: [PATCH 092/105] Add upload/download file and create batch in OpenAIClient --- .../box/l10n/mojito/openai/OpenAIClient.java | 298 ++++++++++++++- .../l10n/mojito/openai/OpenAIClientTest.java | 347 +++++++++++++++++- 2 files changed, 636 insertions(+), 9 deletions(-) diff --git a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java index 6fb19b640c..a804929bb4 100644 --- a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java +++ b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator; +import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -22,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -611,19 +613,307 @@ public record Usage( @JsonProperty("total_tokens") int totalTokens) {} } + public UploadFileResponse uploadFile(UploadFileRequest uploadFileRequest) { + + final String boundary = UUID.randomUUID().toString(); + + String body = uploadFileRequest.getMultipartBody(boundary); + + HttpRequest request = + HttpRequest.newBuilder() + .uri(getUriForEndpoint(UploadFileRequest.ENDPOINT)) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(HttpRequest.BodyPublishers.ofString(body)) + .build(); + + HttpResponse response; + try { + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + if (response.statusCode() != 200) { + throw new OpenAIClientResponseException("Can't upload file", response); + } + + UploadFileResponse uploadFileResponse; + try { + uploadFileResponse = objectMapper.readValue(response.body(), UploadFileResponse.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + return uploadFileResponse; + } + + public record UploadFileRequest( + String purpose, String filename, String content, String contentType) { + + static final String ENDPOINT = "/v1/files"; + + public static UploadFileRequest forBatch(String filename, String content) { + return new UploadFileRequest(Purpose.BATCH.toString(), filename, content, "application/json"); + } + + enum Purpose { + BATCH("batch"), + ASSISTANTS("assistants"), + FINE_TUNE("fine-tune"), + VISION("vision"); + + private final String purposeCode; + + Purpose(String purposeCode) { + this.purposeCode = purposeCode; + } + + public String getPurposeCode() { + return purposeCode; + } + + @Override + public String toString() { + return purposeCode; + } + + public static Purpose fromCode(String purposeCode) { + for (Purpose purpose : Purpose.values()) { + if (purpose.purposeCode.equalsIgnoreCase(purposeCode)) { + return purpose; + } + } + throw new IllegalArgumentException("Unknown purpose: " + purposeCode); + } + } + + String getMultipartBody(String boundary) { + String body = + """ + --%1$s\r + Content-Disposition: form-data; name="purpose"\r + \r + %5$s\r + --%1$s\r + Content-Disposition: form-data; name="file"; filename="%2$s"\r + Content-Type: %3$s\r + \r + %4$s\r + --%1$s--\r + """ + .formatted(boundary, filename, contentType, content, purpose); + return body; + } + } + + public record UploadFileResponse( + String object, + String id, + String purpose, + String filename, + int bytes, + @JsonProperty("created_at") long createdAt, + String status, + @JsonProperty("status_details") String statusDetails) {} + + public record RequestBatchFileLine( + @JsonProperty("custom_id") String customId, String method, String url, Object body) { + + public static RequestBatchFileLine forChatCompletion( + String customId, ChatCompletionsRequest chatCompletionsRequest) { + return new RequestBatchFileLine( + customId, "POST", "/v1/chat/completions", chatCompletionsRequest); + } + } + + public record ChatCompletionResponseBatchFileLine( + String id, @JsonProperty("custom_id") String customId, Response response) { + public record Response( + @JsonProperty("status_code") int statusCode, + @JsonProperty("request_id") String requestId, + @JsonProperty("body") ChatCompletionsResponse chatCompletionsResponse) {} + } + + public DownloadFileContentResponse downloadFileContent( + DownloadFileContentRequest downloadFileContentRequest) { + HttpResponse response; + HttpRequest request = + HttpRequest.newBuilder() + .uri( + getUriForEndpoint( + DownloadFileContentRequest.ENDPOINT.formatted( + downloadFileContentRequest.fileId()))) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .GET() + .build(); + + try { + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException( + "Error while sending the request to download the file: " + + downloadFileContentRequest.fileId(), + e); + } + + if (response.statusCode() != 200) { + throw new OpenAIClientResponseException("Can't download file content", response); + } + + return new DownloadFileContentResponse(response.body()); + } + + public record DownloadFileContentRequest(String fileId) { + static final String ENDPOINT = "/v1/files/%s/content"; + } + + public record DownloadFileContentResponse(String content) {} + + public CreateBatchResponse createBatch(CreateBatchRequest createBatchRequest) { + + String jsonBody; + try { + jsonBody = objectMapper.writeValueAsString(createBatchRequest); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + HttpRequest request = + HttpRequest.newBuilder() + .uri(getUriForEndpoint(CreateBatchRequest.ENDPOINT)) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(jsonBody)) + .build(); + + HttpResponse response; + try { + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + if (response.statusCode() != 200) { + throw new OpenAIClientResponseException("Can't create batch", response); + } + + CreateBatchResponse createBatchResponse; + try { + createBatchResponse = objectMapper.readValue(response.body(), CreateBatchResponse.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + return createBatchResponse; + } + + public record CreateBatchRequest( + @JsonProperty("input_file_id") String inputFileId, + String endpoint, + @JsonProperty("completion_window") String completionWindow, + Map metadata) { + + public static final String ENDPOINT = "/v1/batches"; + + public static CreateBatchRequest forChatCompletion( + String fileId, Map metadata) { + return new CreateBatchRequest(fileId, "/v1/chat/completions", "24h", metadata); + } + } + + public record CreateBatchResponse( + String id, + String object, + String endpoint, + String errors, + @JsonProperty("input_file_id") String inputFileId, + @JsonProperty("completion_window") String completionWindow, + String status, + @JsonProperty("output_file_id") String outputFileId, + @JsonProperty("error_file_id") String errorFileId, + @JsonProperty("created_at") long createdAt, + @JsonProperty("in_progress_at") Long inProgressAt, + @JsonProperty("expires_at") long expiresAt, + @JsonProperty("completed_at") Long completedAt, + @JsonProperty("failed_at") Long failedAt, + @JsonProperty("expired_at") Long expiredAt, + @JsonProperty("request_counts") RequestCounts requestCounts, + Map metadata) { + record RequestCounts(int total, int completed, int failed) {} + } + + public RetrieveBatchResponse retrieveBatch(RetrieveBatchRequest retrieveBatchRequest) { + HttpRequest request = + HttpRequest.newBuilder() + .uri( + getUriForEndpoint( + RetrieveBatchRequest.ENDPOINT.formatted(retrieveBatchRequest.batchId()))) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .GET() + .build(); + + HttpResponse response; + try { + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + if (response.statusCode() != 200) { + throw new OpenAIClientResponseException("Can't retrieve batch", response); + } + + RetrieveBatchResponse retrieveBatchResponse; + try { + retrieveBatchResponse = objectMapper.readValue(response.body(), RetrieveBatchResponse.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + return retrieveBatchResponse; + } + + public record RetrieveBatchRequest(String batchId) { + public static final String ENDPOINT = "/v1/batches/%s"; + } + + public record RetrieveBatchResponse( + String id, + String object, + String endpoint, + String errors, + @JsonProperty("input_file_id") String inputFileId, + @JsonProperty("completion_window") String completionWindow, + String status, + @JsonProperty("output_file_id") String outputFileId, + @JsonProperty("error_file_id") String errorFileId, + @JsonProperty("created_at") long createdAt, + @JsonProperty("in_progress_at") Long inProgressAt, + @JsonProperty("expires_at") long expiresAt, + @JsonProperty("completed_at") Long completedAt, + @JsonProperty("failed_at") Long failedAt, + @JsonProperty("expired_at") Long expiredAt, + @JsonProperty("request_counts") RequestCounts requestCounts, + Map metadata) { + record RequestCounts(int total, int completed, int failed) {} + } + private URI getUriForEndpoint(String endpoint) { return URI.create(host).resolve(endpoint); } - public class OpenAIClientResponseException extends RuntimeException { - HttpResponse httpResponse; + public static class OpenAIClientResponseException extends RuntimeException { + HttpResponse httpResponse; - public OpenAIClientResponseException(String message, HttpResponse httpResponse) { + public OpenAIClientResponseException(String message, HttpResponse httpResponse) { super(message); this.httpResponse = Objects.requireNonNull(httpResponse); } - public OpenAIClientResponseException(String message, Exception e, HttpResponse httpResponse) { + public OpenAIClientResponseException( + String message, Exception e, HttpResponse httpResponse) { super(message, e); this.httpResponse = Objects.requireNonNull(httpResponse); } diff --git a/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java index cceebc5056..dfc0391cc5 100644 --- a/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java +++ b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java @@ -13,11 +13,18 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.box.l10n.mojito.io.Files; import com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsResponse; +import com.box.l10n.mojito.openai.OpenAIClient.OpenAIClientResponseException; +import com.box.l10n.mojito.openai.OpenAIClient.UploadFileRequest; +import com.box.l10n.mojito.openai.OpenAIClient.UploadFileResponse; +import java.io.IOException; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.file.Paths; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import org.junit.jupiter.api.Test; @@ -28,11 +35,10 @@ class OpenAIClientTest { static { try { - // API_KEY = - // - // Files.readString(Paths.get(System.getProperty("user.home")).resolve(".keys/openai")) - // .trim(); - API_KEY = "test-api-key"; + API_KEY = + Files.readString(Paths.get(System.getProperty("user.home")).resolve(".keys/openai")) + .trim(); + // API_KEY = "test-api-key"; } catch (Throwable e) { throw new RuntimeException(e); } @@ -222,4 +228,335 @@ public void testGetChatCompletionsDeserializationError() throws Exception { assertEquals( "Can't deserialize ChatCompletionsResponse", completionException.getCause().getMessage()); } + + @Test + public void testUploadFileSuccess() throws Exception { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(200); + when(mockResponse.body()) + .thenReturn( + """ +{ + "id": "file-123", + "filename": "example.jsonl", + "status": "uploaded", + "created_at": 1690000000 +}"""); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + UploadFileRequest fileUploadRequest = UploadFileRequest.forBatch("example.jsonl", "{}\n{}\n"); + + UploadFileResponse uploadFileResponse = openAIClient.uploadFile(fileUploadRequest); + + assertNotNull(uploadFileResponse); + assertEquals("file-123", uploadFileResponse.id()); + assertEquals("example.jsonl", uploadFileResponse.filename()); + assertEquals("uploaded", uploadFileResponse.status()); + assertEquals(1690000000, uploadFileResponse.createdAt()); + } + + @Test + public void testUploadFileError() throws Exception { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(400); + String errorMessage = + """ + { + "error": { + "message": "Invalid file format for Batch API. Must be .jsonl", + "type": "invalid_request_error", + "param": null, + "code": null + } + } + """; + when(mockResponse.body()).thenReturn(errorMessage); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + UploadFileRequest fileUploadRequest = + UploadFileRequest.forBatch( + "example.jsonl", + """ + { + "a" : "b" + } + """); + + OpenAIClientResponseException openAIClientResponseException = + assertThrows( + OpenAIClientResponseException.class, () -> openAIClient.uploadFile(fileUploadRequest)); + assertEquals(openAIClientResponseException.httpResponse.statusCode(), 400); + assertEquals(openAIClientResponseException.httpResponse.body(), errorMessage); + } + + @Test + public void testFileUploadRequestMultiPartBody() { + UploadFileRequest uploadFileRequest = UploadFileRequest.forBatch("test.jsonl", "{}\n{}"); + String actual = uploadFileRequest.getMultipartBody("test-boundary"); + assertEquals( + """ + --test-boundary\r + Content-Disposition: form-data; name="purpose"\r + \r + batch\r + --test-boundary\r + Content-Disposition: form-data; name="file"; filename="test.jsonl"\r + Content-Type: application/json\r + \r + {} + {}\r + --test-boundary--\r + """, + actual); + } + + @Test + public void testDownloadFileContentSuccess() throws IOException, InterruptedException { + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(200); + String fileContent = + """ + {"a" : "b"} + {"c" : "d"} + """; + when(mockResponse.body()).thenReturn(fileContent); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClient.DownloadFileContentResponse downloadFileContentResponse = + openAIClient.downloadFileContent( + new OpenAIClient.DownloadFileContentRequest("id-for-test")); + + assertEquals(fileContent, downloadFileContentResponse.content()); + } + + @Test + public void testDownloadFileContentError() throws IOException, InterruptedException { + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(404); + String body = + """ + { + "error": { + "message": "No such File object: id-for-test", + "type": "invalid_request_error", + "param": "id", + "code": null + } + } + """; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClientResponseException openAIClientResponseException = + assertThrows( + OpenAIClientResponseException.class, + () -> + openAIClient.downloadFileContent( + new OpenAIClient.DownloadFileContentRequest("id-for-test"))); + assertEquals(body, openAIClientResponseException.httpResponse.body()); + assertEquals(404, openAIClientResponseException.httpResponse.statusCode()); + } + + @Test + public void testCreateBatchSuccess() throws IOException, InterruptedException { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(200); + String body = + """ + { + "id": "batch_67199315c20081909074e442115938a2", + "object": "batch", + "endpoint": "/v1/chat/completions", + "errors": null, + "input_file_id": "file-pp1I2zv79eAnm47wt6rCNL5a", + "completion_window": "24h", + "status": "validating", + "output_file_id": null, + "error_file_id": null, + "created_at": 1729729301, + "in_progress_at": null, + "expires_at": 1729815701, + "finalizing_at": null, + "completed_at": null, + "failed_at": null, + "expired_at": null, + "cancelling_at": null, + "cancelled_at": null, + "request_counts": { + "total": 0, + "completed": 0, + "failed": 0 + }, + "metadata": { + "k1": "v1", + "k2": "v2" + } + } + """; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClient.CreateBatchResponse batch = + openAIClient.createBatch( + OpenAIClient.CreateBatchRequest.forChatCompletion( + "file-pp1I2zv79eAnm47wt6rCNL5a", Map.of("k1", "v1", "k2", "v2"))); + assertEquals("batch_67199315c20081909074e442115938a2", batch.id()); + assertEquals("file-pp1I2zv79eAnm47wt6rCNL5a", batch.inputFileId()); + assertEquals("v1", batch.metadata().get("k1")); + } + + @Test + public void testCreateBatchError() throws IOException, InterruptedException { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(400); + String body = + """ + { + "error": { + "message": "Invalid 'input_file_id': 'wrong-id'. Expected an ID that begins with 'file'.", + "type": "invalid_request_error", + "param": "input_file_id", + "code": "invalid_value" + } + }"""; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClientResponseException openAIClientResponseException = + assertThrows( + OpenAIClientResponseException.class, + () -> + openAIClient.createBatch( + OpenAIClient.CreateBatchRequest.forChatCompletion("wrong-id", null))); + assertEquals(body, openAIClientResponseException.httpResponse.body()); + assertEquals(400, openAIClientResponseException.httpResponse.statusCode()); + } + + @Test + public void testRetrieveBatchSuccess() throws IOException, InterruptedException { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(200); + String body = + """ + { + "id": "batch_67199315c20081909074e442115938a2", + "object": "batch", + "endpoint": "/v1/chat/completions", + "errors": null, + "input_file_id": "file-pp1I2zv79eAnm47wt6rCNL5a", + "completion_window": "24h", + "status": "validating", + "output_file_id": null, + "error_file_id": null, + "created_at": 1729729301, + "in_progress_at": null, + "expires_at": 1729815701, + "finalizing_at": null, + "completed_at": null, + "failed_at": null, + "expired_at": null, + "cancelling_at": null, + "cancelled_at": null, + "request_counts": { + "total": 0, + "completed": 0, + "failed": 0 + }, + "metadata": { + "k1": "v1", + "k2": "v2" + } + } + """; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClient.RetrieveBatchResponse batch = + openAIClient.retrieveBatch( + new OpenAIClient.RetrieveBatchRequest("batch_67199315c20081909074e442115938a2")); + assertEquals("batch_67199315c20081909074e442115938a2", batch.id()); + assertEquals("file-pp1I2zv79eAnm47wt6rCNL5a", batch.inputFileId()); + assertEquals("v1", batch.metadata().get("k1")); + } + + @Test + public void testRetrieveBatchError() throws IOException, InterruptedException { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(400); + String body = + """ + { + "error": { + "message": "Invalid 'input_file_id': 'wrong-id'. Expected an ID that begins with 'file'.", + "type": "invalid_request_error", + "param": "input_file_id", + "code": "invalid_value" + } + }"""; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClientResponseException openAIClientResponseException = + assertThrows( + OpenAIClientResponseException.class, + () -> + openAIClient.createBatch( + OpenAIClient.CreateBatchRequest.forChatCompletion("wrong-id", null))); + assertEquals(body, openAIClientResponseException.httpResponse.body()); + assertEquals(400, openAIClientResponseException.httpResponse.statusCode()); + } } From 2cd6ce30b56b374e6b5af5f3fd10657401cc2a14 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 23 Oct 2024 17:47:09 -0700 Subject: [PATCH 093/105] AI Translate Web Service This service adds AI translation capabilities to a repository using the OpenAI Batch Chat Completion API. The process involves a one-pass translation similar to the prompt used for review. If some strings are rejected by the integrity check, they will be saved as rejected. The follow-up process involves re-translating these strings until a valid translation is achieved. --- .../mojito/rest/textunit/AiTranslateWS.java | 54 ++ .../blobstorage/StructuredBlobStorage.java | 1 + .../oaitranslate/AiTranslateConfig.java | 44 ++ .../AiTranslateConfigurationProperties.java | 28 + .../service/oaitranslate/AiTranslateJob.java | 27 + .../oaitranslate/AiTranslateService.java | 487 ++++++++++++++++++ .../oaitranslate/AiTranslateServiceTest.java | 29 ++ 7 files changed, 670 insertions(+) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfigurationProperties.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateJob.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java new file mode 100644 index 0000000000..2eddf2929d --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java @@ -0,0 +1,54 @@ +package com.box.l10n.mojito.rest.textunit; + +import com.box.l10n.mojito.entity.PollableTask; +import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateConfigurationProperties; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateService; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateService.AiTranslateInput; +import com.box.l10n.mojito.service.pollableTask.PollableFuture; +import com.box.l10n.mojito.service.repository.RepositoryRepository; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class AiTranslateWS { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(AiTranslateWS.class); + + @Autowired AiTranslateService aiTranslateService; + + @Autowired RepositoryRepository repositoryRepository; + + @Autowired QuartzPollableTaskScheduler quartzPollableTaskScheduler; + + @Autowired AiTranslateConfigurationProperties aiTranslateConfigurationProperties; + + @RequestMapping(method = RequestMethod.POST, value = "/api/proto-ai-translate") + @ResponseStatus(HttpStatus.OK) + public ProtoAiTranslateResponse aiTranslate( + @RequestBody ProtoAiTranslateRequest protoAiTranslateRequest) { + + PollableFuture pollableFuture = + aiTranslateService.aiTranslateAsync( + new AiTranslateInput( + protoAiTranslateRequest.repositoryName(), + protoAiTranslateRequest.targetBcp47tags(), + protoAiTranslateRequest.sourceTextMaxCountPerLocale())); + + return new ProtoAiTranslateResponse(pollableFuture.getPollableTask()); + } + + public record ProtoAiTranslateRequest( + String repositoryName, List targetBcp47tags, int sourceTextMaxCountPerLocale) {} + + public record ProtoAiTranslateResponse(PollableTask pollableTask) {} +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java index 32ef760e50..bfea907c06 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java @@ -48,5 +48,6 @@ public enum Prefix { MULTI_BRANCH_STATE, TEXT_UNIT_DTOS_CACHE, CLOB_STORAGE_WS, + AI_TRANSLATE_WS, } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java new file mode 100644 index 0000000000..a54f98a9ba --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java @@ -0,0 +1,44 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.openai.OpenAIClient; +import java.time.Duration; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +@Configuration +public class AiTranslateConfig { + + AiTranslateConfigurationProperties aiTranslateConfigurationProperties; + + public AiTranslateConfig(AiTranslateConfigurationProperties aiTranslateConfigurationProperties) { + this.aiTranslateConfigurationProperties = aiTranslateConfigurationProperties; + } + + @Bean + @Qualifier("AiTranslate") + OpenAIClient openAIClient() { + String openaiClientToken = aiTranslateConfigurationProperties.getOpenaiClientToken(); + if (openaiClientToken == null) { + return null; + } + return new OpenAIClient.Builder().apiKey(openaiClientToken).build(); + } + + @Bean + @Qualifier("AiTranslate") + ObjectMapper objectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + AiTranslateService.configureObjectMapper(objectMapper); + return objectMapper; + } + + @Bean + @Qualifier("AiTranslate") + RetryBackoffSpec retryBackoffSpec() { + return Retry.backoff(5, Duration.ofMillis(500)).maxBackoff(Duration.ofSeconds(5)); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfigurationProperties.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfigurationProperties.java new file mode 100644 index 0000000000..7810883643 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfigurationProperties.java @@ -0,0 +1,28 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import com.box.l10n.mojito.quartz.QuartzSchedulerManager; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties("l10n.ai-translate") +public class AiTranslateConfigurationProperties { + String openaiClientToken; + String schedulerName = QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; + + public String getOpenaiClientToken() { + return openaiClientToken; + } + + public void setOpenaiClientToken(String openaiClientToken) { + this.openaiClientToken = openaiClientToken; + } + + public String getSchedulerName() { + return schedulerName; + } + + public void setSchedulerName(String schedulerName) { + this.schedulerName = schedulerName; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateJob.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateJob.java new file mode 100644 index 0000000000..30d08a8155 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateJob.java @@ -0,0 +1,27 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import com.box.l10n.mojito.quartz.QuartzPollableJob; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateService.AiTranslateInput; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Class to process a batch of strings for machine translation against a set of target languages. + * + * @author garion + */ +@Component +public class AiTranslateJob extends QuartzPollableJob { + + static Logger logger = LoggerFactory.getLogger(AiTranslateJob.class); + + @Autowired AiTranslateService aiTranslateService; + + @Override + public Void call(AiTranslateInput aiTranslateJobInput) throws Exception { + aiTranslateService.aiTranslate(aiTranslateJobInput); + return null; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java new file mode 100644 index 0000000000..d9014f856a --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java @@ -0,0 +1,487 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionResponseBatchFileLine; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.JsonFormat.JsonSchema.createJsonSchema; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.SystemMessage.systemMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.UserMessage.userMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.chatCompletionsRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.CreateBatchRequest.forChatCompletion; +import static com.box.l10n.mojito.openai.OpenAIClient.DownloadFileContentRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.DownloadFileContentResponse; +import static com.box.l10n.mojito.openai.OpenAIClient.RetrieveBatchRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.RetrieveBatchResponse; +import static com.box.l10n.mojito.openai.OpenAIClient.UploadFileRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.UploadFileResponse; +import static com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage.Prefix.AI_TRANSLATE_WS; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toMap; + +import com.box.l10n.mojito.entity.Repository; +import com.box.l10n.mojito.entity.RepositoryLocale; +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.openai.OpenAIClient; +import com.box.l10n.mojito.openai.OpenAIClient.CreateBatchResponse; +import com.box.l10n.mojito.openai.OpenAIClient.RequestBatchFileLine; +import com.box.l10n.mojito.quartz.QuartzJobInfo; +import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; +import com.box.l10n.mojito.service.blobstorage.Retention; +import com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage; +import com.box.l10n.mojito.service.pollableTask.PollableFuture; +import com.box.l10n.mojito.service.repository.RepositoryNameNotFoundException; +import com.box.l10n.mojito.service.repository.RepositoryRepository; +import com.box.l10n.mojito.service.repository.RepositoryService; +import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; +import com.box.l10n.mojito.service.tm.search.StatusFilter; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.util.ArrayDeque; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import reactor.core.publisher.Mono; +import reactor.util.retry.RetryBackoffSpec; + +@Service +public class AiTranslateService { + + static final String METADATA__TEXT_UNIT_DTOS__BLOB_ID = "textUnitDTOs"; + + /** logger */ + static Logger logger = LoggerFactory.getLogger(AiTranslateService.class); + + TextUnitSearcher textUnitSearcher; + + RepositoryRepository repositoryRepository; + + RepositoryService repositoryService; + + AiTranslateConfigurationProperties aiTranslateConfigurationProperties; + + OpenAIClient openAIClient; + + TextUnitBatchImporterService textUnitBatchImporterService; + + StructuredBlobStorage structuredBlobStorage; + + ObjectMapper objectMapper; + + RetryBackoffSpec retryBackoffSpec; + + QuartzPollableTaskScheduler quartzPollableTaskScheduler; + + public AiTranslateService( + TextUnitSearcher textUnitSearcher, + RepositoryRepository repositoryRepository, + RepositoryService repositoryService, + TextUnitBatchImporterService textUnitBatchImporterService, + StructuredBlobStorage structuredBlobStorage, + AiTranslateConfigurationProperties aiTranslateConfigurationProperties, + @Qualifier("AiTranslate") @Autowired(required = false) OpenAIClient openAIClient, + @Qualifier("AiTranslate") ObjectMapper objectMapper, + @Qualifier("AiTranslate") RetryBackoffSpec retryBackoffSpec, + QuartzPollableTaskScheduler quartzPollableTaskScheduler) { + this.textUnitSearcher = textUnitSearcher; + this.repositoryRepository = repositoryRepository; + this.repositoryService = repositoryService; + this.textUnitBatchImporterService = textUnitBatchImporterService; + this.structuredBlobStorage = structuredBlobStorage; + this.aiTranslateConfigurationProperties = aiTranslateConfigurationProperties; + this.objectMapper = objectMapper; + this.openAIClient = openAIClient; + this.retryBackoffSpec = retryBackoffSpec; + this.quartzPollableTaskScheduler = quartzPollableTaskScheduler; + } + + public record AiTranslateInput( + String repositoryName, List targetBcp47tags, int sourceTextMaxCountPerLocale) {} + + public PollableFuture aiTranslateAsync(AiTranslateInput aiTranslateInput) { + + QuartzJobInfo quartzJobInfo = + QuartzJobInfo.newBuilder(AiTranslateJob.class) + .withInlineInput(false) + .withInput(aiTranslateInput) + .withScheduler(aiTranslateConfigurationProperties.getSchedulerName()) + .build(); + + return quartzPollableTaskScheduler.scheduleJob(quartzJobInfo); + } + + public void aiTranslate(AiTranslateInput aiTranslateInput) throws AiTranslateException { + + Repository repository = repositoryRepository.findByName(aiTranslateInput.repositoryName()); + + if (repository == null) { + throw new RepositoryNameNotFoundException( + String.format( + "Repository with name '%s' can not be found!", aiTranslateInput.repositoryName())); + } + + logger.debug("Start AI Translation for repository: {}", repository.getName()); + + try { + Set repositoryLocalesWithoutRootLocale = + repositoryService.getRepositoryLocalesWithoutRootLocale(repository).stream() + .filter( + rl -> + aiTranslateInput.targetBcp47tags == null + || aiTranslateInput.targetBcp47tags.contains( + rl.getLocale().getBcp47Tag())) + .collect(Collectors.toSet()); + + logger.debug("Create batches for repository: {}", repository.getName()); + ArrayDeque batches = + repositoryLocalesWithoutRootLocale.stream() + .map( + createBatchForRepositoryLocale( + repository, aiTranslateInput.sourceTextMaxCountPerLocale())) + .filter(Objects::nonNull) + .collect(Collectors.toCollection(ArrayDeque::new)); + + logger.debug("Import batches for repository: {}", repository.getName()); + while (!batches.isEmpty()) { + RetrieveBatchResponse retrieveBatchResponse = getNextFinishedBatch(batches); + importBatch(retrieveBatchResponse); + } + } catch (OpenAIClient.OpenAIClientResponseException openAIClientResponseException) { + logger.error( + "Failed to ai translate: %s".formatted(openAIClientResponseException), + openAIClientResponseException); + throw new AiTranslateException(openAIClientResponseException); + } + } + + void importBatch(RetrieveBatchResponse retrieveBatchResponse) { + + logger.info("Importing batch: {}", retrieveBatchResponse.id()); + + String textUnitDTOsBlobId = + retrieveBatchResponse.metadata().get(METADATA__TEXT_UNIT_DTOS__BLOB_ID); + + logger.info("Trying to load textUnitDTOs from blob: {}", textUnitDTOsBlobId); + AiTranslateBlobStorage aiTranslateBlobStorage = + structuredBlobStorage + .getString(AI_TRANSLATE_WS, textUnitDTOsBlobId) + .map(s -> objectMapper.readValueUnchecked(s, AiTranslateBlobStorage.class)) + .orElseThrow( + () -> + new RuntimeException( + "There must be an entry for textUnitDTOsBlobId: " + textUnitDTOsBlobId)); + + Map tmTextUnitIdToTextUnitDTOs = + aiTranslateBlobStorage.textUnitDTOS().stream() + .collect(toMap(TextUnitDTO::getTmTextUnitId, Function.identity())); + + DownloadFileContentResponse downloadFileContentResponse = + getOpenAIClient() + .downloadFileContent( + new DownloadFileContentRequest(retrieveBatchResponse.outputFileId())); + + List forImport = + downloadFileContentResponse + .content() + .lines() + .map( + line -> { + ChatCompletionResponseBatchFileLine chatCompletionResponseBatchFileLine = + objectMapper.readValueUnchecked( + line, ChatCompletionResponseBatchFileLine.class); + + if (chatCompletionResponseBatchFileLine.response().statusCode() != 200) { + throw new RuntimeException( + "Response batch file line failed: " + chatCompletionResponseBatchFileLine); + } + + String aiTranslateOutputAsJson = + chatCompletionResponseBatchFileLine + .response() + .chatCompletionsResponse() + .choices() + .getFirst() + .message() + .content(); + + AiTranslateOutput aiTranslateOutput = + objectMapper.readValueUnchecked( + aiTranslateOutputAsJson, AiTranslateOutput.class); + + TextUnitDTO textUnitDTO = + tmTextUnitIdToTextUnitDTOs.get( + Long.valueOf(chatCompletionResponseBatchFileLine.customId())); + textUnitDTO.setTarget(aiTranslateOutput.target().content()); + textUnitDTO.setTargetComment("ai-translate"); + return textUnitDTO; + }) + .toList(); + + textUnitBatchImporterService.importTextUnits( + forImport, + TextUnitBatchImporterService.IntegrityChecksType.ALWAYS_USE_INTEGRITY_CHECKER_STATUS); + } + + Function createBatchForRepositoryLocale( + Repository repository, int sourceTextMaxCountPerLocale) { + + return repositoryLocale -> { + logger.debug( + "Get untranslated string for locale: '{}' in repository: '{}'", + repositoryLocale.getLocale().getBcp47Tag(), + repository.getName()); + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(repository.getId()); + textUnitSearcherParameters.setStatusFilter(StatusFilter.UNTRANSLATED); + textUnitSearcherParameters.setLocaleId(repositoryLocale.getLocale().getId()); + textUnitSearcherParameters.setLimit(sourceTextMaxCountPerLocale); + List textUnitDTOS = textUnitSearcher.search(textUnitSearcherParameters); + + CreateBatchResponse createBatchResponse = null; + if (textUnitDTOS.isEmpty()) { + logger.debug("Nothing to translate, don't create a batch"); + } else { + logger.debug("Save the TextUnitDTOs in blob storage for later batch import"); + String batchId = + "%s_%s".formatted(repositoryLocale.getLocale().getBcp47Tag(), UUID.randomUUID()); + structuredBlobStorage.put( + AI_TRANSLATE_WS, + batchId, + objectMapper.writeValueAsStringUnchecked(new AiTranslateBlobStorage(textUnitDTOS)), + Retention.MIN_1_DAY); + + logger.debug("Generate the batch file content"); + String batchFileContent = generateBatchFileContent(textUnitDTOS); + + logger.debug("Upload batch file content: {}", batchFileContent); + UploadFileResponse uploadFileResponse = + getOpenAIClient() + .uploadFile( + UploadFileRequest.forBatch("%s.jsonl".formatted(batchId), batchFileContent)); + + logger.debug("Create the batch using file: {}", uploadFileResponse); + createBatchResponse = + getOpenAIClient() + .createBatch( + forChatCompletion( + uploadFileResponse.id(), + Map.of(METADATA__TEXT_UNIT_DTOS__BLOB_ID, batchId))); + } + + logger.info( + "Created batch for locale: {} with {} text units", + repositoryLocale.getLocale().getBcp47Tag(), + textUnitDTOS.size()); + return createBatchResponse; + }; + } + + String generateBatchFileContent(List textUnitDTOS) { + return textUnitDTOS.stream() + .map( + textUnitDTO -> { + CompletionInput completionInput = + new CompletionInput( + textUnitDTO.getTargetLocale(), + textUnitDTO.getSource(), + textUnitDTO.getComment()); + + String inputAsJsonString = objectMapper.writeValueAsStringUnchecked(completionInput); + + ObjectNode jsonSchema = createJsonSchema(AiTranslateOutput.class); + + ChatCompletionsRequest chatCompletionsRequest = + chatCompletionsRequest() + .model("gpt-4o-2024-08-06") + .maxTokens(16384) + .messages( + List.of( + systemMessageBuilder().content(PROMPT).build(), + userMessageBuilder().content(inputAsJsonString).build())) + .responseFormat( + new ChatCompletionsRequest.JsonFormat( + "json_schema", + new ChatCompletionsRequest.JsonFormat.JsonSchema( + true, "request_json_format", jsonSchema))) + .build(); + + return RequestBatchFileLine.forChatCompletion( + textUnitDTO.getTmTextUnitId().toString(), chatCompletionsRequest); + }) + .map(objectMapper::writeValueAsStringUnchecked) + .collect(joining("\n")); + } + + /** + * Use a queue to not stay stuck on a slow job, and try to import faster. Batch are imported + * sequentially. + * + *

Note: This is an active blocking pooling which blocks the thread but is isolated in a thread + * pool. + */ + RetrieveBatchResponse getNextFinishedBatch(ArrayDeque batches) { + while (true) { + int size = batches.size(); + + for (int i = 0; i < size; i++) { + CreateBatchResponse batch = batches.removeFirst(); + + logger.debug("Retrieve current status of batch: {}", batch.id()); + RetrieveBatchResponse retrieveBatchResponse = retrieveBatchWithRetry(batch); + + if ("completed".equals(retrieveBatchResponse.status())) { + logger.info("Next completed batch is: {}", retrieveBatchResponse.id()); + return retrieveBatchResponse; + } else if ("failed".equals(retrieveBatchResponse.status())) { + logger.error("Batch failed, skipping it: {}", retrieveBatchResponse); + } else { + logger.debug( + "Batch is still processing append to the end of the queue: {}", + retrieveBatchResponse); + batches.offerLast(batch); + } + } + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + RetrieveBatchResponse retrieveBatchWithRetry(CreateBatchResponse batch) { + + return Mono.fromCallable( + () -> getOpenAIClient().retrieveBatch(new RetrieveBatchRequest(batch.id()))) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> { + logger.info("Retrying retrieving batch: {}", batch.id()); + })) + .doOnError( + throwable -> new RuntimeException("Failed to retrieve batch: " + batch.id(), throwable)) + .block(); + } + + record CompletionInput(String locale, String source, String sourceDescription) {} + + record AiTranslateOutput( + String source, + Target target, + DescriptionRating descriptionRating, + AltTarget altTarget, + ExistingTargetRating existingTargetRating, + ReviewRequired reviewRequired) { + record Target(String content, String explanation, int confidenceLevel) {} + + record AltTarget(String content, String explanation, int confidenceLevel) {} + + record DescriptionRating(String explanation, int score) {} + + record ExistingTargetRating(String explanation, int score) {} + + record ReviewRequired(boolean required, String reason) {} + } + + record AiTranslateBlobStorage(List textUnitDTOS) {} + + static final String PROMPT = + """ + Your role is to act as a translator. + You are tasked with translating provided source strings while preserving both the tone and the technical structure of the string. This includes protecting any tags, placeholders, or code elements that should not be translated. + + The input will be provided in JSON format with the following fields: + + • "source": The source text to be translated. + • "locale": The target language locale, following the BCP47 standard (e.g., “fr”, “es-419”). + • "sourceDescription": A description providing context for the source text. + • "existingTarget" (optional): An existing translation to review. + + Instructions: + + • If the source is colloquial, keep the translation colloquial; if it’s formal, maintain formality in the translation. + • Pay attention to regional variations specified in the "locale" field (e.g., “es” vs. “es-419”, “fr” vs. “fr-CA”, “zh” vs. “zh-Hant”), and ensure the translation length remains similar to the source text. + + Handling Tags and Code: + + Some strings contain code elements such as tags (e.g., {atag}, ICU message format, or HTML tags). You are provided with a inputs of tags that need to be protected. Ensure that: + + • Tags like {atag} remain untouched. + • In cases of nested content (e.g., text that needs translation), only translate the inner text while preserving the outer structure. + • Complex structures like ICU message formats should have placeholders or variables left intact (e.g., {count, plural, one {# item} other {# items}}), but translate any inner translatable text. + + Ambiguity and Context: + + After translating, assess the usefulness of the "sourceDescription" field: + + • Rate its usefulness on a scale of 0 to 2: + • 0 – Not helpful at all; irrelevant or misleading. + • 1 – Somewhat helpful; provides partial or unclear context but is useful to some extent. + • 2 – Very helpful; provides clear and sufficient guidance for the translation. + + If the source is ambiguous—for example, if it could be interpreted as a noun or a verb—you must: + + • Indicate the ambiguity in your explanation. + • Provide translations for all possible interpretations. + • Set "reviewRequired" to true, and explain the need for review due to the ambiguity. + + You will provide an output in JSON format with the following fields: + + • "source": The original source text. + • "target": An object containing: + • "content": The best translation. + • "explanation": A brief explanation of your translation choices. + • "confidenceLevel": Your confidence level (0-100%) in the translation. + • "descriptionRating": An object containing: + • "explanation": An explanation of how the "sourceDescription" aided your translation. + • "score": The usefulness score (0-2). + • "altTarget": An object containing: + • "content": An alternative translation, if applicable. Focus on showcasing grammar differences, + • "explanation": Explanation for the alternative translation. + • "confidenceLevel": Your confidence level (0-100%) in the alternative translation. + • "reviewRequired": An object containing: + • "required": true or false, indicating if review is needed. + • "reason": A detailed explanation of why review is or isn’t needed. + """; + + /** + * Typical configuration for the ObjectMapper needed by this class. + * + *

The ObjectMapper must not use indentation else Jsonl serialization will fail. + */ + public static void configureObjectMapper(ObjectMapper objectMapper) { + objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + objectMapper.disable(SerializationFeature.INDENT_OUTPUT); + objectMapper.registerModule(new JavaTimeModule()); + } + + OpenAIClient getOpenAIClient() { + if (openAIClient == null) { + String msg = + "OpenAI client is not configured for AiTranslateService. Ensure that the OpenAI API key is provided in the configuration (qualifier='aiTranslate')."; + logger.error(msg); + throw new RuntimeException(msg); + } + return openAIClient; + } + + public class AiTranslateException extends Exception { + public AiTranslateException(Throwable cause) { + super(cause); + } + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java new file mode 100644 index 0000000000..4c389bceab --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java @@ -0,0 +1,29 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import com.box.l10n.mojito.service.assetExtraction.ServiceTestBase; +import com.box.l10n.mojito.service.tm.TMTestData; +import com.box.l10n.mojito.test.TestIdWatcher; +import java.util.concurrent.ExecutionException; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +public class AiTranslateServiceTest extends ServiceTestBase { + + static Logger logger = LoggerFactory.getLogger(AiTranslateServiceTest.class); + + @Rule public TestIdWatcher testIdWatcher = new TestIdWatcher(); + + @Autowired AiTranslateService aiTranslateService; + + @Test + public void aiTranslate() throws ExecutionException, InterruptedException { + TMTestData tmTestData = new TMTestData(testIdWatcher); + aiTranslateService + .aiTranslateAsync( + new AiTranslateService.AiTranslateInput(tmTestData.repository.getName(), null, 100)) + .get(); + } +} From 660ae666ced649b430c8975ae4af2cdc89d0ccd7 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 25 Oct 2024 11:02:33 -0700 Subject: [PATCH 094/105] Add command line to translate repository using the AiTranslateService --- .../RepositoryAiTranslationCommand.java | 89 +++++++++++++++++++ .../client/RepositoryAiTranslateClient.java | 35 ++++++++ 2 files changed, 124 insertions(+) create mode 100644 cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java create mode 100644 restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java new file mode 100644 index 0000000000..d4b431c934 --- /dev/null +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java @@ -0,0 +1,89 @@ +package com.box.l10n.mojito.cli.command; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.box.l10n.mojito.cli.command.param.Param; +import com.box.l10n.mojito.cli.console.ConsoleWriter; +import com.box.l10n.mojito.rest.client.RepositoryAiTranslateClient; +import com.box.l10n.mojito.rest.client.RepositoryAiTranslateClient.ProtoAiTranslateResponse; +import com.box.l10n.mojito.rest.entity.PollableTask; +import java.util.List; +import java.util.stream.Collectors; +import org.fusesource.jansi.Ansi.Color; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +/** + * Command to machine translate untranslated strings in a repository. + * + * @author jaurambault + */ +@Component +@Scope("prototype") +@Parameters( + commandNames = {"repository-ai-translate"}, + commandDescription = "Ai translate untranslated and rejected strings in a repository") +public class RepositoryAiTranslationCommand extends Command { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(RepositoryAiTranslationCommand.class); + + @Autowired ConsoleWriter consoleWriter; + + @Parameter( + names = {Param.REPOSITORY_LONG, Param.REPOSITORY_SHORT}, + arity = 1, + required = true, + description = Param.REPOSITORY_DESCRIPTION) + String repositoryParam; + + @Parameter( + names = {Param.REPOSITORY_LOCALES_LONG, Param.REPOSITORY_LOCALES_SHORT}, + variableArity = true, + required = true, + description = "List of locales (bcp47 tags) to machine translate") + List locales; + + @Parameter( + names = {"--source-text-max-count"}, + arity = 1, + description = + "Source text max count per locale sent to MT (this param is used to avoid " + + "sending too many strings to MT)") + int sourceTextMaxCount = 100; + + @Autowired CommandHelper commandHelper; + + @Autowired RepositoryAiTranslateClient repositoryAiTranslateClient; + + @Override + public boolean shouldShowInCommandList() { + return false; + } + + @Override + public void execute() throws CommandException { + + consoleWriter + .newLine() + .a("Ai translate repository: ") + .fg(Color.CYAN) + .a(repositoryParam) + .reset() + .a(" for locales: ") + .fg(Color.CYAN) + .a(locales.stream().collect(Collectors.joining(", ", "[", "]"))) + .println(2); + + ProtoAiTranslateResponse protoAiTranslateResponse = + repositoryAiTranslateClient.translateRepository( + new RepositoryAiTranslateClient.ProtoAiTranslateRequest( + repositoryParam, locales, sourceTextMaxCount)); + + PollableTask pollableTask = protoAiTranslateResponse.pollableTask(); + commandHelper.waitForPollableTask(pollableTask.getId()); + } +} diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java b/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java new file mode 100644 index 0000000000..6262f484ff --- /dev/null +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java @@ -0,0 +1,35 @@ +package com.box.l10n.mojito.rest.client; + +import com.box.l10n.mojito.rest.entity.PollableTask; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** + * @author jaurambault + */ +@Component +public class RepositoryAiTranslateClient extends BaseClient { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(RepositoryAiTranslateClient.class); + + @Override + public String getEntityName() { + return "proto-ai-translate"; + } + + /** Ai translate untranslated and rejected strings in a repository for a given list of locales */ + public ProtoAiTranslateResponse translateRepository( + ProtoAiTranslateRequest protoAiTranslateRequest) { + + return authenticatedRestTemplate.postForObject( + getBasePathForEntity(), protoAiTranslateRequest, ProtoAiTranslateResponse.class); + } + + public record ProtoAiTranslateRequest( + String repositoryName, List targetBcp47tags, int sourceTextMaxCountPerLocale) {} + + public record ProtoAiTranslateResponse(PollableTask pollableTask) {} +} From ec6c1683bb7acab31a819a7f82b911d719f44ac5 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Mon, 28 Oct 2024 10:30:48 -0700 Subject: [PATCH 095/105] Fix doc/properties for logging --- webapp/src/main/resources/config/application.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/resources/config/application.properties b/webapp/src/main/resources/config/application.properties index 8745bea184..6db15e2df0 100644 --- a/webapp/src/main/resources/config/application.properties +++ b/webapp/src/main/resources/config/application.properties @@ -12,7 +12,8 @@ info.build.version=@project.version@ #logging.file.path=/var/logs #logging.file.name=mojito.log #logging.config= # location of config file (default classpath:logback.xml for logback) -#logging.level.*=ERROR +#logging.level.root=ERROR +#logging.level.com.box.l10n.*=ERROR #logging.level.org.apache.http.wire=DEBUG logging.level.net.sf.okapi.common.pipelinedriver.PipelineDriver=ERROR From 043b4f3d409ee7729f552938c21826443e9cd4d8 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 12 Nov 2024 09:40:40 -0800 Subject: [PATCH 096/105] Add script to build web and its dependencies --- webapp/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/package.json b/webapp/package.json index f6104db8f4..6c8fb88698 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -60,6 +60,7 @@ "start-server-nofe": "mvn spring-boot:run -P '!frontend' -Dspring-boot.run.jvmArguments=\"-Dspring.config.additional-location=optional:file://$HOME/.l10n/config/webapp/ -Dspring.profiles.active=$USER,npm -Duser.timezone=UTC\"", "start-server-nofe-debug": "mvn spring-boot:run -P '!frontend' -Dspring-boot.run.jvmArguments=\"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005 -Dspring.config.additional-location=optional:file://$HOME/.l10n/config/webapp/ -Dspring.profiles.active=$USER,npm -Duser.timezone=UTC\"", "start-server-nofe-debugn": "mvn spring-boot:run -P '!frontend' -Dspring-boot.run.jvmArguments=\"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -Dspring.config.additional-location=optional:file://$HOME/.l10n/config/webapp/ -Dspring.profiles.active=$USER,npm -Duser.timezone=UTC\"", + "start-server-nofe-dep": "cd .. && mvn install -DskipTests -P'!frontend' && cd webapp && mvn spring-boot:run -P '!frontend' -Dspring-boot.run.jvmArguments=\"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -Dspring.config.additional-location=optional:file://$HOME/.l10n/config/webapp/ -Dspring.profiles.active=$USER,npm -Duser.timezone=UTC\"", "watch": "webpack --config webpack.config.js --progress --colors --watch --env.inlineSourceMap", "watch-ict": "webpack --config webpack.config_ict.js --progress --colors --watch --env.inlineSourceMap", "build": "webpack --config webpack.config.js --progress --env.production", From 23b8feb1802888d96523efcc53e623b6e8190b14 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 29 Oct 2024 15:27:50 -0700 Subject: [PATCH 097/105] Add an option to skip removing unused keys and tags --- .../mojito/service/thirdparty/ThirdPartyTMSPhrase.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 30c904e666..774a6e840e 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -191,7 +191,13 @@ public void push( null); } - removeUnusedKeysAndTags(projectId, repository.getName(), tagForUpload); + boolean skipRemoveUnusedKeysAndTags = + optionsParser.getBoolean("skipRemoveUnusedKeysAndTags", false); + + if (!skipRemoveUnusedKeysAndTags) { + logger.info("Skipping removing unused keys and tags"); + removeUnusedKeysAndTags(projectId, repository.getName(), tagForUpload); + } } /** From 1c768c6a34b26fe5d2ff6a076209c95f98a9d169 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 29 Oct 2024 15:38:23 -0700 Subject: [PATCH 098/105] Remove Plural Entries Missing the 'Other' Form Not having the 'other' form in plural resources can cause the app to crash. This commit ensures that any plural entries lacking the 'other' form are removed to prevent crashes. --- .../mojito/okapi/filters/AndroidFilter.java | 10 ++- .../okapi/filters/AndroidFilterTest.java | 75 +++++++++++++++---- 2 files changed, 69 insertions(+), 16 deletions(-) diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java index 786b714af9..06ff9c6d59 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java @@ -502,9 +502,17 @@ public String execute(String xmlContent) { Element plurals = (Element) pluralsElements.item(i); NodeList items = plurals.getElementsByTagName("item"); boolean hasTranslated = false; + boolean hasOther = false; for (int j = 0; j < items.getLength(); j++) { Element item = (Element) items.item(j); + + if ("other".equals(item.getAttribute("quantity"))) { + hasOther = + !RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER.equals( + item.getTextContent()); + } + if (hasRemoveUntranslated() && item.getTextContent() .equals(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { @@ -515,7 +523,7 @@ public String execute(String xmlContent) { } } - if (!hasTranslated) { + if (!hasOther || !hasTranslated) { plurals.getParentNode().removeChild(plurals); i--; } diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java index 2f247934a2..90874d2bce 100644 --- a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java +++ b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java @@ -126,8 +126,8 @@ public void testPostProcessingKeepDescription() { @#$untranslated$#@ - translated - @#$untranslated$#@ + @#$untranslated$#@ + translated pin fr @@ -143,7 +143,7 @@ public void testPostProcessingKeepDescription() { somestring to keep - translated + translated pin fr @@ -170,8 +170,8 @@ public void testPostProcessingRemoveDescription() { @#$untranslated$#@ - translated - @#$untranslated$#@ + @#$untranslated$#@ + translated pin fr @@ -187,7 +187,7 @@ public void testPostProcessingRemoveDescription() { somestring to keep - translated + translated pin fr @@ -245,8 +245,8 @@ public void testPostProcessingRemoveTranslatableFalse() { @#$untranslated$#@ - translated - @#$untranslated$#@ + @#$untranslated$#@ + translated pin fr @@ -261,13 +261,58 @@ public void testPostProcessingRemoveTranslatableFalse() { - translated + translated """; assertEquals(expected, output); } + @Test + public void testPostProcessingRemoveMissingOther() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); + String input = + """ + + + + pin fr + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingRemoveMissingOtherUntranslated() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); + String input = + """ + + + + pin fr + @#$untranslated$#@ + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + """; + assertEquals(expected, output); + } + @Test public void testPostProcessingStandaloneNo() { AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = @@ -284,8 +329,8 @@ public void testPostProcessingStandaloneNo() { @#$untranslated$#@ - translated - @#$untranslated$#@ + @#$untranslated$#@ + translated pin fr @@ -300,7 +345,7 @@ public void testPostProcessingStandaloneNo() { - translated + translated """; @@ -323,8 +368,8 @@ public void testPostProcessingStandaloneYes() { @#$untranslated$#@ - translated - @#$untranslated$#@ + @#$untranslated$#@ + translated pin fr @@ -339,7 +384,7 @@ public void testPostProcessingStandaloneYes() { - translated + translated """; From 5cdacf60c10d9c4df0c306015c8e6aaac09218bd Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Tue, 29 Oct 2024 15:26:03 -0700 Subject: [PATCH 099/105] Implement first version of no-batch ai translate api --- .../RepositoryAiTranslationCommand.java | 14 +- .../box/l10n/mojito/openai/OpenAIClient.java | 33 +- .../l10n/mojito/openai/OpenAIClientPool.java | 77 ++++ .../mojito/openai/OpenAIClientPoolTest.java | 134 +++++++ .../l10n/mojito/openai/OpenAIClientTest.java | 363 +++++++++--------- .../client/RepositoryAiTranslateClient.java | 5 +- .../mojito/rest/textunit/AiTranslateWS.java | 9 +- .../oaitranslate/AiTranslateConfig.java | 12 + .../oaitranslate/AiTranslateService.java | 246 ++++++++++-- .../oaitranslate/AiTranslateServiceTest.java | 28 +- 10 files changed, 702 insertions(+), 219 deletions(-) create mode 100644 common/src/main/java/com/box/l10n/mojito/openai/OpenAIClientPool.java create mode 100644 common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientPoolTest.java diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java index d4b431c934..7986d5ab84 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java @@ -43,8 +43,8 @@ public class RepositoryAiTranslationCommand extends Command { @Parameter( names = {Param.REPOSITORY_LOCALES_LONG, Param.REPOSITORY_LOCALES_SHORT}, variableArity = true, - required = true, - description = "List of locales (bcp47 tags) to machine translate") + description = + "List of locales (bcp47 tags) to translate, if not provided translate all locales in the repository") List locales; @Parameter( @@ -55,6 +55,12 @@ public class RepositoryAiTranslationCommand extends Command { + "sending too many strings to MT)") int sourceTextMaxCount = 100; + @Parameter( + names = {"--use-batch"}, + arity = 1, + description = "To use the batch API or not") + boolean useBatch = false; + @Autowired CommandHelper commandHelper; @Autowired RepositoryAiTranslateClient repositoryAiTranslateClient; @@ -75,13 +81,13 @@ public void execute() throws CommandException { .reset() .a(" for locales: ") .fg(Color.CYAN) - .a(locales.stream().collect(Collectors.joining(", ", "[", "]"))) + .a(locales == null ? "" : locales.stream().collect(Collectors.joining(", ", "[", "]"))) .println(2); ProtoAiTranslateResponse protoAiTranslateResponse = repositoryAiTranslateClient.translateRepository( new RepositoryAiTranslateClient.ProtoAiTranslateRequest( - repositoryParam, locales, sourceTextMaxCount)); + repositoryParam, locales, sourceTextMaxCount, useBatch)); PollableTask pollableTask = protoAiTranslateResponse.pollableTask(); commandHelper.waitForPollableTask(pollableTask.getId()); diff --git a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java index a804929bb4..35290cd880 100644 --- a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java +++ b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java @@ -25,6 +25,8 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.ForkJoinPool; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -39,11 +41,19 @@ public class OpenAIClient { final HttpClient httpClient; - OpenAIClient(String apiKey, String host, ObjectMapper objectMapper, HttpClient httpClient) { + final Executor asyncExecutor; + + OpenAIClient( + String apiKey, + String host, + ObjectMapper objectMapper, + HttpClient httpClient, + Executor asyncExecutor) { this.apiKey = Objects.requireNonNull(apiKey); this.host = Objects.requireNonNull(host); this.objectMapper = Objects.requireNonNull(objectMapper); this.httpClient = Objects.requireNonNull(httpClient); + this.asyncExecutor = Objects.requireNonNull(asyncExecutor); } public static class Builder { @@ -56,6 +66,8 @@ public static class Builder { private HttpClient httpClient; + private Executor asyncExecutor; + public Builder() {} public Builder apiKey(String apiKey) { @@ -78,6 +90,11 @@ public Builder httpClient(HttpClient httpClient) { return this; } + public Builder asyncExecutor(Executor asyncExecutor) { + this.asyncExecutor = asyncExecutor; + return this; + } + public OpenAIClient build() { if (apiKey == null) { throw new IllegalStateException("API key must be provided"); @@ -89,11 +106,16 @@ public OpenAIClient build() { if (httpClient == null) { httpClient = createHttpClient(); } - return new OpenAIClient(apiKey, host, objectMapper, httpClient); + + if (asyncExecutor == null) { + asyncExecutor = ForkJoinPool.commonPool(); + } + + return new OpenAIClient(apiKey, host, objectMapper, httpClient, asyncExecutor); } private HttpClient createHttpClient() { - return HttpClient.newHttpClient(); + return HttpClient.newBuilder().build(); } private ObjectMapper createObjectMapper() { @@ -135,7 +157,7 @@ public CompletableFuture getChatCompletions( CompletableFuture chatCompletionsResponse = httpClient .sendAsync(request, HttpResponse.BodyHandlers.ofString()) - .thenApply( + .thenApplyAsync( httpResponse -> { if (httpResponse.statusCode() != 200) { throw new OpenAIClientResponseException("ChatCompletion failed", httpResponse); @@ -148,7 +170,8 @@ public CompletableFuture getChatCompletions( "Can't deserialize ChatCompletionsResponse", e, httpResponse); } } - }); + }, + asyncExecutor); return chatCompletionsResponse; } diff --git a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClientPool.java b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClientPool.java new file mode 100644 index 0000000000..f18ed0c433 --- /dev/null +++ b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClientPool.java @@ -0,0 +1,77 @@ +package com.box.l10n.mojito.openai; + +import com.google.common.base.Function; +import java.net.http.HttpClient; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadLocalRandom; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OpenAIClientPool { + + static Logger logger = LoggerFactory.getLogger(OpenAIClientPool.class); + + int numberOfClients; + OpenAIClientWithSemaphore[] openAIClientWithSemaphores; + + /** + * Pool to parallelize slower requests (1s+) over HTTP/2 connections. + * + * @param numberOfClients Number of OpenAIClient instances with independent HttpClients. + * @param numberOfParallelRequestPerClient Maximum parallel requests per client, controlled by a + * semaphore to prevent overload. + * @param sizeOfAsyncProcessors Shared async processors across all HttpClients to limit threads, + * as request time is the main bottleneck. + * @param apiKey API key for authentication. + */ + public OpenAIClientPool( + int numberOfClients, + int numberOfParallelRequestPerClient, + int sizeOfAsyncProcessors, + String apiKey) { + ExecutorService asyncExecutor = Executors.newWorkStealingPool(sizeOfAsyncProcessors); + this.numberOfClients = numberOfClients; + this.openAIClientWithSemaphores = new OpenAIClientWithSemaphore[numberOfClients]; + for (int i = 0; i < numberOfClients; i++) { + this.openAIClientWithSemaphores[i] = + new OpenAIClientWithSemaphore( + OpenAIClient.builder() + .apiKey(apiKey) + .asyncExecutor(asyncExecutor) + .httpClient(HttpClient.newBuilder().executor(asyncExecutor).build()) + .build(), + new Semaphore(numberOfParallelRequestPerClient)); + } + } + + public CompletableFuture submit(Function> f) { + + while (true) { + for (OpenAIClientWithSemaphore openAIClientWithSemaphore : openAIClientWithSemaphores) { + if (openAIClientWithSemaphore.semaphore().tryAcquire()) { + return f.apply(openAIClientWithSemaphore.openAIClient()) + .whenComplete((o, e) -> openAIClientWithSemaphore.semaphore().release()); + } + } + + try { + logger.debug("can't directly acquire any semaphore, do blocking"); + int randomSemaphoreIndex = + ThreadLocalRandom.current().nextInt(openAIClientWithSemaphores.length); + OpenAIClientWithSemaphore randomClientWithSemaphore = + this.openAIClientWithSemaphores[randomSemaphoreIndex]; + randomClientWithSemaphore.semaphore().acquire(); + return f.apply(randomClientWithSemaphore.openAIClient()) + .whenComplete((o, e) -> randomClientWithSemaphore.semaphore().release()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Can't submit task to the OpenAIClientPool", e); + } + } + } + + record OpenAIClientWithSemaphore(OpenAIClient openAIClient, Semaphore semaphore) {} +} diff --git a/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientPoolTest.java b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientPoolTest.java new file mode 100644 index 0000000000..3ddbbeb15b --- /dev/null +++ b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientPoolTest.java @@ -0,0 +1,134 @@ +package com.box.l10n.mojito.openai; + +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.SystemMessage.systemMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.UserMessage.userMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.chatCompletionsRequest; + +import com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsResponse; +import com.google.common.base.Stopwatch; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Disabled +public class OpenAIClientPoolTest { + + static Logger logger = LoggerFactory.getLogger(OpenAIClientPoolTest.class); + + static final String API_KEY; + + static { + try { + // API_KEY = + // + // Files.readString(Paths.get(System.getProperty("user.home")).resolve(".keys/openai")) + // .trim(); + API_KEY = "test-api-key"; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Test + public void test() { + int numberOfClients = 10; + int numberOfParallelRequestPerClient = 50; + int numberOfRequests = 10000; + int sizeOfAsyncProcessors = 10; + int totalExecutions = numberOfClients * numberOfParallelRequestPerClient; + + OpenAIClientPool openAIClientPool = + new OpenAIClientPool( + numberOfClients, numberOfParallelRequestPerClient, sizeOfAsyncProcessors, API_KEY); + + AtomicInteger responseCounter = new AtomicInteger(); + AtomicInteger submitted = new AtomicInteger(); + Stopwatch stopwatch = Stopwatch.createStarted(); + + ArrayList submissionTimes = new ArrayList<>(); + ArrayList responseTimes = new ArrayList<>(); + + List> responses = new ArrayList<>(); + for (int i = 0; i < numberOfRequests; i++) { + String message = "Is %d prime?".formatted(i); + Stopwatch requestStopwatch = Stopwatch.createStarted(); + OpenAIClient.ChatCompletionsRequest chatCompletionsRequest = + chatCompletionsRequest() + .model("gpt-4o-2024-08-06") + .messages( + List.of( + systemMessageBuilder() + .content("You're an engine designed to check prime numbers") + .build(), + userMessageBuilder().content(message).build())) + .build(); + + CompletableFuture response = + openAIClientPool.submit( + openAIClient -> { + CompletableFuture chatCompletions = + openAIClient.getChatCompletions(chatCompletionsRequest); + submissionTimes.add(requestStopwatch.elapsed(TimeUnit.SECONDS)); + if (submitted.incrementAndGet() % 100 == 0) { + logger.info( + "--> request per second: " + + submitted.get() / (stopwatch.elapsed(TimeUnit.SECONDS) + 0.00001) + + ", submission count: " + + submitted.get() + + ", future response count: " + + responses.size() + + ", last submissions took: " + + submissionTimes.subList( + Math.max(0, submissionTimes.size() - 100), submissionTimes.size())); + } + return chatCompletions; + }); + + response.thenApply( + chatCompletionsResponse -> { + responseTimes.add(requestStopwatch.elapsed(TimeUnit.MILLISECONDS)); + if (responseCounter.incrementAndGet() % 10 == 0) { + double avg = + responseTimes.stream().collect(Collectors.averagingLong(Long::longValue)); + logger.info( + "<-- response per second: " + + responseCounter.get() / stopwatch.elapsed(TimeUnit.SECONDS) + + ", average response time: " + + Math.round(avg) + + " (rps: " + + Math.round(totalExecutions / (avg / 1000.0)) + + "), response count from counter: " + + responseCounter.get() + + ", last elapsed times: " + + responseTimes.subList(responseTimes.size() - 20, responseTimes.size())); + } + return chatCompletionsResponse; + }); + + responses.add(response); + } + + Stopwatch started = Stopwatch.createStarted(); + CompletableFuture.allOf(responses.toArray(new CompletableFuture[responses.size()])).join(); + logger.info("Waiting for join: " + started.elapsed()); + + double avg = responseTimes.stream().collect(Collectors.averagingLong(Long::longValue)); + logger.info( + "Total time: " + + stopwatch.elapsed().toString() + + ", request per second: " + + Math.round((double) numberOfRequests / stopwatch.elapsed(TimeUnit.SECONDS)) + + ", average response time: " + + Math.round(avg) + + " (theory rps: " + + Math.round(totalExecutions / (avg / 1000.0)) + + ")"); + } +} diff --git a/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java index dfc0391cc5..d538cb45b3 100644 --- a/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java +++ b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java @@ -13,7 +13,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.box.l10n.mojito.io.Files; import com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsResponse; import com.box.l10n.mojito.openai.OpenAIClient.OpenAIClientResponseException; import com.box.l10n.mojito.openai.OpenAIClient.UploadFileRequest; @@ -22,23 +21,23 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.nio.file.Paths; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import org.junit.jupiter.api.Test; -class OpenAIClientTest { +public class OpenAIClientTest { static final String API_KEY; static { try { - API_KEY = - Files.readString(Paths.get(System.getProperty("user.home")).resolve(".keys/openai")) - .trim(); - // API_KEY = "test-api-key"; + // API_KEY = + // + // Files.readString(Paths.get(System.getProperty("user.home")).resolve(".keys/openai")) + // .trim(); + API_KEY = "test-api-key"; } catch (Throwable e) { throw new RuntimeException(e); } @@ -66,29 +65,29 @@ public void testGetChatCompletionsSuccess() throws Exception { String jsonResponse = """ - { - "id": "chatcmpl-9DNYjOkXJxILUK3NXFv9MCZV0P8jZ", - "object": "chat.completion", - "created": 1712975853, - "model": "gpt-3.5-turbo-0125", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Il s'agit d'un test unitaire" - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 24, - "completion_tokens": 9, - "total_tokens": 33 - }, - "system_fingerprint": "fp_c2295e73ad" - }"""; + { + "id": "chatcmpl-9DNYjOkXJxILUK3NXFv9MCZV0P8jZ", + "object": "chat.completion", + "created": 1712975853, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Il s'agit d'un test unitaire" + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 9, + "total_tokens": 33 + }, + "system_fingerprint": "fp_c2295e73ad" + }"""; HttpResponse mockResponse = mock(HttpResponse.class); when(mockResponse.statusCode()).thenReturn(200); @@ -132,14 +131,14 @@ public void testGetChatCompletionsRequestError() throws Exception { when(mockResponse.statusCode()).thenReturn(400); String errorMsg = """ - { - "error": { - "message": "The model `invalid-model` does not exist or you do not have access to it.", - "type": "invalid_request_error", - "param": null, - "code": "model_not_found" - } - }"""; + { + "error": { + "message": "The model `invalid-model` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": null, + "code": "model_not_found" + } + }"""; when(mockResponse.body()).thenReturn(errorMsg); HttpClient mockHttpClient = mock(HttpClient.class); @@ -159,12 +158,12 @@ public void testGetChatCompletionsRequestError() throws Exception { .getMessage() .contains( """ - "error": { - "message": "The model `invalid-model` does not exist or you do not have access to it.", - "type": "invalid_request_error", - "param": null, - "code": "model_not_found" - }""")); + "error": { + "message": "The model `invalid-model` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": null, + "code": "model_not_found" + }""")); } @Test @@ -186,29 +185,29 @@ public void testGetChatCompletionsDeserializationError() throws Exception { String jsonResponse = """ - { - "id": "chatcmpl-9DNYjOkXJxILUK3NXFv9MCZV0P8jZ", - "object": "chat.completion", - "created": "invalid date to break deserialization", - "model": "gpt-3.5-turbo-0125", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Il s'agit d'un test unitaire" - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 24, - "completion_tokens": 9, - "total_tokens": 33 - }, - "system_fingerprint": "fp_c2295e73ad" - }"""; + { + "id": "chatcmpl-9DNYjOkXJxILUK3NXFv9MCZV0P8jZ", + "object": "chat.completion", + "created": "invalid date to break deserialization", + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Il s'agit d'un test unitaire" + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 9, + "total_tokens": 33 + }, + "system_fingerprint": "fp_c2295e73ad" + }"""; HttpResponse mockResponse = mock(HttpResponse.class); when(mockResponse.statusCode()).thenReturn(200); @@ -239,12 +238,12 @@ public void testUploadFileSuccess() throws Exception { when(mockResponse.body()) .thenReturn( """ -{ - "id": "file-123", - "filename": "example.jsonl", - "status": "uploaded", - "created_at": 1690000000 -}"""); + { + "id": "file-123", + "filename": "example.jsonl", + "status": "uploaded", + "created_at": 1690000000 + }"""); when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) .thenReturn(mockResponse); @@ -271,15 +270,15 @@ public void testUploadFileError() throws Exception { when(mockResponse.statusCode()).thenReturn(400); String errorMessage = """ - { - "error": { - "message": "Invalid file format for Batch API. Must be .jsonl", - "type": "invalid_request_error", - "param": null, - "code": null + { + "error": { + "message": "Invalid file format for Batch API. Must be .jsonl", + "type": "invalid_request_error", + "param": null, + "code": null + } } - } - """; + """; when(mockResponse.body()).thenReturn(errorMessage); when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) .thenReturn(mockResponse); @@ -291,10 +290,10 @@ public void testUploadFileError() throws Exception { UploadFileRequest.forBatch( "example.jsonl", """ - { - "a" : "b" - } - """); + { + "a" : "b" + } + """); OpenAIClientResponseException openAIClientResponseException = assertThrows( @@ -309,18 +308,18 @@ public void testFileUploadRequestMultiPartBody() { String actual = uploadFileRequest.getMultipartBody("test-boundary"); assertEquals( """ - --test-boundary\r - Content-Disposition: form-data; name="purpose"\r - \r - batch\r - --test-boundary\r - Content-Disposition: form-data; name="file"; filename="test.jsonl"\r - Content-Type: application/json\r - \r - {} - {}\r - --test-boundary--\r - """, + --test-boundary\r + Content-Disposition: form-data; name="purpose"\r + \r + batch\r + --test-boundary\r + Content-Disposition: form-data; name="file"; filename="test.jsonl"\r + Content-Type: application/json\r + \r + {} + {}\r + --test-boundary--\r + """, actual); } @@ -332,9 +331,9 @@ public void testDownloadFileContentSuccess() throws IOException, InterruptedExce when(mockResponse.statusCode()).thenReturn(200); String fileContent = """ - {"a" : "b"} - {"c" : "d"} - """; + {"a" : "b"} + {"c" : "d"} + """; when(mockResponse.body()).thenReturn(fileContent); when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) .thenReturn(mockResponse); @@ -357,15 +356,15 @@ public void testDownloadFileContentError() throws IOException, InterruptedExcept when(mockResponse.statusCode()).thenReturn(404); String body = """ - { - "error": { - "message": "No such File object: id-for-test", - "type": "invalid_request_error", - "param": "id", - "code": null + { + "error": { + "message": "No such File object: id-for-test", + "type": "invalid_request_error", + "param": "id", + "code": null + } } - } - """; + """; when(mockResponse.body()).thenReturn(body); when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) .thenReturn(mockResponse); @@ -392,36 +391,36 @@ public void testCreateBatchSuccess() throws IOException, InterruptedException { when(mockResponse.statusCode()).thenReturn(200); String body = """ - { - "id": "batch_67199315c20081909074e442115938a2", - "object": "batch", - "endpoint": "/v1/chat/completions", - "errors": null, - "input_file_id": "file-pp1I2zv79eAnm47wt6rCNL5a", - "completion_window": "24h", - "status": "validating", - "output_file_id": null, - "error_file_id": null, - "created_at": 1729729301, - "in_progress_at": null, - "expires_at": 1729815701, - "finalizing_at": null, - "completed_at": null, - "failed_at": null, - "expired_at": null, - "cancelling_at": null, - "cancelled_at": null, - "request_counts": { - "total": 0, - "completed": 0, - "failed": 0 - }, - "metadata": { - "k1": "v1", - "k2": "v2" + { + "id": "batch_67199315c20081909074e442115938a2", + "object": "batch", + "endpoint": "/v1/chat/completions", + "errors": null, + "input_file_id": "file-pp1I2zv79eAnm47wt6rCNL5a", + "completion_window": "24h", + "status": "validating", + "output_file_id": null, + "error_file_id": null, + "created_at": 1729729301, + "in_progress_at": null, + "expires_at": 1729815701, + "finalizing_at": null, + "completed_at": null, + "failed_at": null, + "expired_at": null, + "cancelling_at": null, + "cancelled_at": null, + "request_counts": { + "total": 0, + "completed": 0, + "failed": 0 + }, + "metadata": { + "k1": "v1", + "k2": "v2" + } } - } - """; + """; when(mockResponse.body()).thenReturn(body); when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) .thenReturn(mockResponse); @@ -447,14 +446,14 @@ public void testCreateBatchError() throws IOException, InterruptedException { when(mockResponse.statusCode()).thenReturn(400); String body = """ - { - "error": { - "message": "Invalid 'input_file_id': 'wrong-id'. Expected an ID that begins with 'file'.", - "type": "invalid_request_error", - "param": "input_file_id", - "code": "invalid_value" - } - }"""; + { + "error": { + "message": "Invalid 'input_file_id': 'wrong-id'. Expected an ID that begins with 'file'.", + "type": "invalid_request_error", + "param": "input_file_id", + "code": "invalid_value" + } + }"""; when(mockResponse.body()).thenReturn(body); when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) .thenReturn(mockResponse); @@ -481,36 +480,36 @@ public void testRetrieveBatchSuccess() throws IOException, InterruptedException when(mockResponse.statusCode()).thenReturn(200); String body = """ - { - "id": "batch_67199315c20081909074e442115938a2", - "object": "batch", - "endpoint": "/v1/chat/completions", - "errors": null, - "input_file_id": "file-pp1I2zv79eAnm47wt6rCNL5a", - "completion_window": "24h", - "status": "validating", - "output_file_id": null, - "error_file_id": null, - "created_at": 1729729301, - "in_progress_at": null, - "expires_at": 1729815701, - "finalizing_at": null, - "completed_at": null, - "failed_at": null, - "expired_at": null, - "cancelling_at": null, - "cancelled_at": null, - "request_counts": { - "total": 0, - "completed": 0, - "failed": 0 - }, - "metadata": { - "k1": "v1", - "k2": "v2" + { + "id": "batch_67199315c20081909074e442115938a2", + "object": "batch", + "endpoint": "/v1/chat/completions", + "errors": null, + "input_file_id": "file-pp1I2zv79eAnm47wt6rCNL5a", + "completion_window": "24h", + "status": "validating", + "output_file_id": null, + "error_file_id": null, + "created_at": 1729729301, + "in_progress_at": null, + "expires_at": 1729815701, + "finalizing_at": null, + "completed_at": null, + "failed_at": null, + "expired_at": null, + "cancelling_at": null, + "cancelled_at": null, + "request_counts": { + "total": 0, + "completed": 0, + "failed": 0 + }, + "metadata": { + "k1": "v1", + "k2": "v2" + } } - } - """; + """; when(mockResponse.body()).thenReturn(body); when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) .thenReturn(mockResponse); @@ -535,14 +534,14 @@ public void testRetrieveBatchError() throws IOException, InterruptedException { when(mockResponse.statusCode()).thenReturn(400); String body = """ - { - "error": { - "message": "Invalid 'input_file_id': 'wrong-id'. Expected an ID that begins with 'file'.", - "type": "invalid_request_error", - "param": "input_file_id", - "code": "invalid_value" - } - }"""; + { + "error": { + "message": "Invalid 'input_file_id': 'wrong-id'. Expected an ID that begins with 'file'.", + "type": "invalid_request_error", + "param": "input_file_id", + "code": "invalid_value" + } + }"""; when(mockResponse.body()).thenReturn(body); when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) .thenReturn(mockResponse); diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java b/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java index 6262f484ff..f0f5207675 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java @@ -29,7 +29,10 @@ public ProtoAiTranslateResponse translateRepository( } public record ProtoAiTranslateRequest( - String repositoryName, List targetBcp47tags, int sourceTextMaxCountPerLocale) {} + String repositoryName, + List targetBcp47tags, + int sourceTextMaxCountPerLocale, + boolean useBatch) {} public record ProtoAiTranslateResponse(PollableTask pollableTask) {} } diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java index 2eddf2929d..7f33de3211 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java @@ -42,13 +42,18 @@ public ProtoAiTranslateResponse aiTranslate( new AiTranslateInput( protoAiTranslateRequest.repositoryName(), protoAiTranslateRequest.targetBcp47tags(), - protoAiTranslateRequest.sourceTextMaxCountPerLocale())); + protoAiTranslateRequest.sourceTextMaxCountPerLocale(), + protoAiTranslateRequest.useBatch())); return new ProtoAiTranslateResponse(pollableFuture.getPollableTask()); } public record ProtoAiTranslateRequest( - String repositoryName, List targetBcp47tags, int sourceTextMaxCountPerLocale) {} + String repositoryName, + List targetBcp47tags, + int sourceTextMaxCountPerLocale, + boolean useBatch, + boolean allLocales) {} public record ProtoAiTranslateResponse(PollableTask pollableTask) {} } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java index a54f98a9ba..4ef75a3db5 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java @@ -2,6 +2,7 @@ import com.box.l10n.mojito.json.ObjectMapper; import com.box.l10n.mojito.openai.OpenAIClient; +import com.box.l10n.mojito.openai.OpenAIClientPool; import java.time.Duration; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -28,6 +29,17 @@ OpenAIClient openAIClient() { return new OpenAIClient.Builder().apiKey(openaiClientToken).build(); } + @Bean + @Qualifier("AiTranslate") + OpenAIClientPool openAIClientPool() { + String openaiClientToken = aiTranslateConfigurationProperties.getOpenaiClientToken(); + if (openaiClientToken == null) { + return null; + } + return new OpenAIClientPool( + 10, 50, 5, aiTranslateConfigurationProperties.getOpenaiClientToken()); + } + @Bean @Qualifier("AiTranslate") ObjectMapper objectMapper() { diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java index d9014f856a..8d9b962398 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java @@ -21,12 +21,15 @@ import com.box.l10n.mojito.entity.RepositoryLocale; import com.box.l10n.mojito.json.ObjectMapper; import com.box.l10n.mojito.openai.OpenAIClient; +import com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsResponse; import com.box.l10n.mojito.openai.OpenAIClient.CreateBatchResponse; import com.box.l10n.mojito.openai.OpenAIClient.RequestBatchFileLine; +import com.box.l10n.mojito.openai.OpenAIClientPool; import com.box.l10n.mojito.quartz.QuartzJobInfo; import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; import com.box.l10n.mojito.service.blobstorage.Retention; import com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateService.CompletionInput.ExistingTarget; import com.box.l10n.mojito.service.pollableTask.PollableFuture; import com.box.l10n.mojito.service.repository.RepositoryNameNotFoundException; import com.box.l10n.mojito.service.repository.RepositoryRepository; @@ -41,12 +44,17 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.io.IOException; +import java.time.Duration; import java.util.ArrayDeque; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -54,7 +62,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; import reactor.util.retry.RetryBackoffSpec; @Service @@ -75,6 +85,8 @@ public class AiTranslateService { OpenAIClient openAIClient; + OpenAIClientPool openAIClientPool; + TextUnitBatchImporterService textUnitBatchImporterService; StructuredBlobStorage structuredBlobStorage; @@ -93,6 +105,7 @@ public AiTranslateService( StructuredBlobStorage structuredBlobStorage, AiTranslateConfigurationProperties aiTranslateConfigurationProperties, @Qualifier("AiTranslate") @Autowired(required = false) OpenAIClient openAIClient, + @Qualifier("AiTranslate") @Autowired(required = false) OpenAIClientPool openAIClientPool, @Qualifier("AiTranslate") ObjectMapper objectMapper, @Qualifier("AiTranslate") RetryBackoffSpec retryBackoffSpec, QuartzPollableTaskScheduler quartzPollableTaskScheduler) { @@ -104,12 +117,16 @@ public AiTranslateService( this.aiTranslateConfigurationProperties = aiTranslateConfigurationProperties; this.objectMapper = objectMapper; this.openAIClient = openAIClient; + this.openAIClientPool = openAIClientPool; this.retryBackoffSpec = retryBackoffSpec; this.quartzPollableTaskScheduler = quartzPollableTaskScheduler; } public record AiTranslateInput( - String repositoryName, List targetBcp47tags, int sourceTextMaxCountPerLocale) {} + String repositoryName, + List targetBcp47tags, + int sourceTextMaxCountPerLocale, + boolean useBatch) {} public PollableFuture aiTranslateAsync(AiTranslateInput aiTranslateInput) { @@ -124,26 +141,183 @@ public PollableFuture aiTranslateAsync(AiTranslateInput aiTranslateInput) } public void aiTranslate(AiTranslateInput aiTranslateInput) throws AiTranslateException { + if (aiTranslateInput.useBatch()) { + aiTranslateBatch(aiTranslateInput); + } else { + aiTranslateNoBatch(aiTranslateInput); + } + } - Repository repository = repositoryRepository.findByName(aiTranslateInput.repositoryName()); + public void aiTranslateNoBatch(AiTranslateInput aiTranslateInput) { - if (repository == null) { - throw new RepositoryNameNotFoundException( - String.format( - "Repository with name '%s' can not be found!", aiTranslateInput.repositoryName())); + Repository repository = getRepository(aiTranslateInput); + + logger.info("Start AI Translation (no batch) for repository: {}", repository.getName()); + + Set filteredRepositoryLocales = + getFilteredRepositoryLocales(aiTranslateInput, repository); + + Flux.fromIterable(filteredRepositoryLocales) + .flatMap( + rl -> + asyncProcessLocale( + rl, aiTranslateInput.sourceTextMaxCountPerLocale(), openAIClientPool), + 10) + .then() + .doOnTerminate( + () -> + logger.info( + "Done with AI Translation (no batch) for repository: {}", repository.getName())) + .block(); + } + + Mono asyncProcessLocale( + RepositoryLocale repositoryLocale, + int sourceTextMaxCountPerLocale, + OpenAIClientPool openAIClientPool) { + + Repository repository = repositoryLocale.getRepository(); + + logger.info( + "Get untranslated strings for locale: '{}' in repository: '{}'", + repositoryLocale.getLocale().getBcp47Tag(), + repository.getName()); + + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(repository.getId()); + textUnitSearcherParameters.setStatusFilter(StatusFilter.FOR_TRANSLATION); + textUnitSearcherParameters.setLocaleId(repositoryLocale.getLocale().getId()); + textUnitSearcherParameters.setLimit(sourceTextMaxCountPerLocale); + + List textUnitDTOS = textUnitSearcher.search(textUnitSearcherParameters); + + if (textUnitDTOS.isEmpty()) { + logger.debug( + "Nothing to translate for locale: {}", repositoryLocale.getLocale().getBcp47Tag()); + return Mono.empty(); } + logger.info( + "Starting parallel processing for each string in locale: {}, count: {}", + repositoryLocale.getLocale().getBcp47Tag(), + textUnitDTOS.size()); + + return Flux.fromIterable(textUnitDTOS) + .buffer(500) + .concatMap( + batch -> + Flux.fromIterable(batch) + .flatMap( + textUnitDTO -> + getChatCompletionForTextUnitDTO(textUnitDTO, openAIClientPool) + .retryWhen( + Retry.backoff(5, Duration.ofSeconds(1)) + .filter(this::isRetriableException) + .doBeforeRetry( + retrySignal -> { + logger.warn( + "Retrying request for TextUnitDTO {} due to exception of type {}", + textUnitDTO.getTmTextUnitId(), + retrySignal.failure().getMessage()); + })) + .onErrorResume( + error -> { + logger.error( + "Request for TextUnitDTO {} failed after retries: {}", + textUnitDTO.getTmTextUnitId(), + error.getMessage()); + return Mono.empty(); + })) + .collectList() + .flatMap(this::submitForImport) + .doOnTerminate(() -> logger.info("Done submitting for processing"))) + .then(); + } + + record MyRecord(TextUnitDTO textUnitDTO, ChatCompletionsResponse chatCompletionsResponse) {} + + private Mono submitForImport(List results) { + logger.info("Submit for import for locale {}", results.get(0).textUnitDTO().getTargetLocale()); + List forImport = + results.stream() + .map( + myRecord -> { + TextUnitDTO textUnitDTO = myRecord.textUnitDTO(); + ChatCompletionsResponse chatCompletionsResponse = + myRecord.chatCompletionsResponse(); + + String completionOutputAsJson = + chatCompletionsResponse.choices().getFirst().message().content(); + + CompletionOutput completionOutput = + objectMapper.readValueUnchecked( + completionOutputAsJson, CompletionOutput.class); + + textUnitDTO.setTarget(completionOutput.target().content()); + textUnitDTO.setTargetComment("ai-translate"); + return textUnitDTO; + }) + .collect(Collectors.toList()); + + textUnitBatchImporterService.importTextUnits( + forImport, + TextUnitBatchImporterService.IntegrityChecksType.ALWAYS_USE_INTEGRITY_CHECKER_STATUS); + + return Mono.empty(); + } + + private Mono getChatCompletionForTextUnitDTO( + TextUnitDTO textUnitDTO, OpenAIClientPool openAIClientPool) { + + CompletionInput completionInput = + new CompletionInput( + textUnitDTO.getTargetLocale(), + textUnitDTO.getSource(), + textUnitDTO.getComment(), + textUnitDTO.getTarget() == null + ? null + : new ExistingTarget( + textUnitDTO.getTarget(), !textUnitDTO.isIncludedInLocalizedFile())); + + String inputAsJsonString = objectMapper.writeValueAsStringUnchecked(completionInput); + ObjectNode jsonSchema = createJsonSchema(CompletionOutput.class); + + ChatCompletionsRequest chatCompletionsRequest = + chatCompletionsRequest() + .model("gpt-4o-2024-08-06") + .maxTokens(16384) + .messages( + List.of( + systemMessageBuilder().content(PROMPT).build(), + userMessageBuilder().content(inputAsJsonString).build())) + .responseFormat( + new ChatCompletionsRequest.JsonFormat( + "json_schema", + new ChatCompletionsRequest.JsonFormat.JsonSchema( + true, "request_json_format", jsonSchema))) + .build(); + + CompletableFuture futureResult = + openAIClientPool.submit( + (openAIClient) -> openAIClient.getChatCompletions(chatCompletionsRequest)); + return Mono.fromFuture(futureResult) + .map(chatCompletionsResponse -> new MyRecord(textUnitDTO, chatCompletionsResponse)); + } + + private boolean isRetriableException(Throwable throwable) { + Throwable cause = throwable instanceof CompletionException ? throwable.getCause() : throwable; + return cause instanceof IOException || cause instanceof TimeoutException; + } + + public void aiTranslateBatch(AiTranslateInput aiTranslateInput) throws AiTranslateException { + + Repository repository = getRepository(aiTranslateInput); + logger.debug("Start AI Translation for repository: {}", repository.getName()); try { Set repositoryLocalesWithoutRootLocale = - repositoryService.getRepositoryLocalesWithoutRootLocale(repository).stream() - .filter( - rl -> - aiTranslateInput.targetBcp47tags == null - || aiTranslateInput.targetBcp47tags.contains( - rl.getLocale().getBcp47Tag())) - .collect(Collectors.toSet()); + getFilteredRepositoryLocales(aiTranslateInput, repository); logger.debug("Create batches for repository: {}", repository.getName()); ArrayDeque batches = @@ -167,6 +341,27 @@ public void aiTranslate(AiTranslateInput aiTranslateInput) throws AiTranslateExc } } + private Set getFilteredRepositoryLocales( + AiTranslateInput aiTranslateInput, Repository repository) { + return repositoryService.getRepositoryLocalesWithoutRootLocale(repository).stream() + .filter( + rl -> + aiTranslateInput.targetBcp47tags == null + || aiTranslateInput.targetBcp47tags.contains(rl.getLocale().getBcp47Tag())) + .collect(Collectors.toSet()); + } + + private Repository getRepository(AiTranslateInput aiTranslateInput) { + Repository repository = repositoryRepository.findByName(aiTranslateInput.repositoryName()); + + if (repository == null) { + throw new RepositoryNameNotFoundException( + String.format( + "Repository with name '%s' can not be found!", aiTranslateInput.repositoryName())); + } + return repository; + } + void importBatch(RetrieveBatchResponse retrieveBatchResponse) { logger.info("Importing batch: {}", retrieveBatchResponse.id()); @@ -208,7 +403,7 @@ void importBatch(RetrieveBatchResponse retrieveBatchResponse) { "Response batch file line failed: " + chatCompletionResponseBatchFileLine); } - String aiTranslateOutputAsJson = + String completionOutputAsJson = chatCompletionResponseBatchFileLine .response() .chatCompletionsResponse() @@ -217,14 +412,14 @@ void importBatch(RetrieveBatchResponse retrieveBatchResponse) { .message() .content(); - AiTranslateOutput aiTranslateOutput = + CompletionOutput completionOutput = objectMapper.readValueUnchecked( - aiTranslateOutputAsJson, AiTranslateOutput.class); + completionOutputAsJson, CompletionOutput.class); TextUnitDTO textUnitDTO = tmTextUnitIdToTextUnitDTOs.get( Long.valueOf(chatCompletionResponseBatchFileLine.customId())); - textUnitDTO.setTarget(aiTranslateOutput.target().content()); + textUnitDTO.setTarget(completionOutput.target().content()); textUnitDTO.setTargetComment("ai-translate"); return textUnitDTO; }) @@ -266,7 +461,6 @@ Function createBatchForRepositoryLocale( logger.debug("Generate the batch file content"); String batchFileContent = generateBatchFileContent(textUnitDTOS); - logger.debug("Upload batch file content: {}", batchFileContent); UploadFileResponse uploadFileResponse = getOpenAIClient() .uploadFile( @@ -297,11 +491,13 @@ String generateBatchFileContent(List textUnitDTOS) { new CompletionInput( textUnitDTO.getTargetLocale(), textUnitDTO.getSource(), - textUnitDTO.getComment()); + textUnitDTO.getComment(), + new ExistingTarget( + textUnitDTO.getTarget(), !textUnitDTO.isIncludedInLocalizedFile())); String inputAsJsonString = objectMapper.writeValueAsStringUnchecked(completionInput); - ObjectNode jsonSchema = createJsonSchema(AiTranslateOutput.class); + ObjectNode jsonSchema = createJsonSchema(CompletionOutput.class); ChatCompletionsRequest chatCompletionsRequest = chatCompletionsRequest() @@ -376,9 +572,12 @@ RetrieveBatchResponse retrieveBatchWithRetry(CreateBatchResponse batch) { .block(); } - record CompletionInput(String locale, String source, String sourceDescription) {} + record CompletionInput( + String locale, String source, String sourceDescription, ExistingTarget existingTarget) { + record ExistingTarget(String content, boolean hasBrokenPlaceholders) {} + } - record AiTranslateOutput( + record CompletionOutput( String source, Target target, DescriptionRating descriptionRating, @@ -408,7 +607,7 @@ record AiTranslateBlobStorage(List textUnitDTOS) {} • "source": The source text to be translated. • "locale": The target language locale, following the BCP47 standard (e.g., “fr”, “es-419”). • "sourceDescription": A description providing context for the source text. - • "existingTarget" (optional): An existing translation to review. + • "existingTarget" (optional): An existing translation to review. Indicates if it has broken placeholders. Instructions: @@ -422,6 +621,7 @@ Some strings contain code elements such as tags (e.g., {atag}, ICU message forma • Tags like {atag} remain untouched. • In cases of nested content (e.g., text that needs translation), only translate the inner text while preserving the outer structure. • Complex structures like ICU message formats should have placeholders or variables left intact (e.g., {count, plural, one {# item} other {# items}}), but translate any inner translatable text. + • If an existing translation is provided and has broken placeholder, make sure to fix them in the new translation. Ambiguity and Context: diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java index 4c389bceab..1ef12ede94 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java @@ -1,9 +1,12 @@ package com.box.l10n.mojito.service.oaitranslate; import com.box.l10n.mojito.service.assetExtraction.ServiceTestBase; +import com.box.l10n.mojito.service.repository.RepositoryNameAlreadyUsedException; +import com.box.l10n.mojito.service.repository.RepositoryService; import com.box.l10n.mojito.service.tm.TMTestData; import com.box.l10n.mojito.test.TestIdWatcher; import java.util.concurrent.ExecutionException; +import org.junit.Assume; import org.junit.Rule; import org.junit.Test; import org.slf4j.Logger; @@ -18,12 +21,33 @@ public class AiTranslateServiceTest extends ServiceTestBase { @Autowired AiTranslateService aiTranslateService; + @Autowired AiTranslateConfigurationProperties aiTranslateConfigurationProperties; + + @Autowired RepositoryService repositoryService; + + @Test + public void aiTranslateBatch() throws ExecutionException, InterruptedException { + Assume.assumeNotNull(aiTranslateConfigurationProperties.getOpenaiClientToken()); + + TMTestData tmTestData = new TMTestData(testIdWatcher); + aiTranslateService + .aiTranslateAsync( + new AiTranslateService.AiTranslateInput( + tmTestData.repository.getName(), null, 100, true)) + .get(); + } + @Test - public void aiTranslate() throws ExecutionException, InterruptedException { + public void aiTranslateNoBatch() + throws ExecutionException, InterruptedException, RepositoryNameAlreadyUsedException { + Assume.assumeNotNull(aiTranslateConfigurationProperties.getOpenaiClientToken()); + TMTestData tmTestData = new TMTestData(testIdWatcher); + aiTranslateService .aiTranslateAsync( - new AiTranslateService.AiTranslateInput(tmTestData.repository.getName(), null, 100)) + new AiTranslateService.AiTranslateInput( + tmTestData.repository.getName(), null, 100, false)) .get(); } } From a1b8fcb8f7cd1f357a59dbdb59b6d654b0c42096 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 21 Nov 2024 11:58:48 -0800 Subject: [PATCH 100/105] Handle concurrent run of PhraseTMS sync Remove Phrase tags only they are older than 5 minutes, this way concurrent sync don't end up in a state where there are no tags, which break the "pull" logic. The time window needs to be higher than the time "push" takes, putting 5 mintues for now. --- .../thirdparty/ThirdPartyTMSPhrase.java | 38 +++++++++++++- .../thirdparty/ThirdPartyTMSPhraseTest.java | 50 +++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 774a6e840e..c95db2112c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -27,6 +27,8 @@ import com.phrase.client.model.TranslationKey; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; +import java.time.Duration; +import java.time.LocalDateTime; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.AbstractMap.SimpleEntry; @@ -52,6 +54,7 @@ public class ThirdPartyTMSPhrase implements ThirdPartyTMS { static final String TAG_PREFIX = "push_"; static final String TAG_PREFIX_WITH_REPOSITORY = "push_%s"; + static final String TAG_DATE_FORMAT = "yyyy_MM_dd_HH_mm_ss_SSS"; static final boolean NATIVE_CLIENT_DEFAULT_VALUE = false; static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhrase.class); @@ -249,12 +252,22 @@ public void removeUnusedKeysAndTags( .filter(Objects::nonNull) .filter(tagName -> tagName.startsWith(TAG_PREFIX)) .filter(tagName -> !allActiveTags.contains(tagName)) + // That's to handle concurrent sync and make sure a tag that was just pushed is not + // deleted before the pull from the same sync is finished. + // We don't prevent syncs to run concurrently + .filter(tagName -> !areTagsWithin5Minutes(tagForUpload, tagName)) .toList(); logger.info("Tags to delete: {}", pushTagsToDelete); phraseClient.deleteTags(projectId, pushTagsToDelete); + logger.info("RemoveUnusedKeysAndTags took: {}", stopwatchRemoveUnusedKeysAndTags); + } - logger.info("removeUnusedKeysAndTags took: {}", stopwatchRemoveUnusedKeysAndTags); + static boolean areTagsWithin5Minutes(String tagName1, String tagName2) { + return Duration.between(uploadTagToLocalDateTime(tagName2), uploadTagToLocalDateTime(tagName1)) + .abs() + .getSeconds() + < (60 * 5); } private List getSourceTextUnitDTOs( @@ -293,7 +306,7 @@ private List getSourceTextUnitDTOsPluralOnly( public static String getTagForUpload(String repositoryName) { ZonedDateTime zonedDateTime = JSR310Migration.dateTimeNowInUTC(); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy_MM_dd_HH_mm_ss_SSS"); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(TAG_DATE_FORMAT); return normalizeTagName( "%s%s_%s_%s" .formatted( @@ -303,6 +316,27 @@ public static String getTagForUpload(String repositoryName) { Math.abs(UUID.randomUUID().getLeastSignificantBits() % 1000))); } + public static LocalDateTime uploadTagToLocalDateTime(String tag) { + + if (tag == null || !tag.contains("_")) { + throw new IllegalArgumentException("Invalid tag format: " + tag); + } + + int dateEndIndex = tag.lastIndexOf('_'); // last part is a random number + int dateStartIndex = dateEndIndex; + + for (int i = 0; i < 7; i++) { + dateStartIndex = tag.lastIndexOf('_', dateStartIndex - 1); + if (dateStartIndex == -1) { + throw new IllegalArgumentException("Invalid tag format: " + tag); + } + } + + String dateTimePart = tag.substring(dateStartIndex + 1, dateEndIndex); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(TAG_DATE_FORMAT); + return LocalDateTime.parse(dateTimePart, formatter); + } + private static String getTagNamePrefixForRepository(String repositoryName) { return normalizeTagName(TAG_PREFIX_WITH_REPOSITORY.formatted(repositoryName)); } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java index 55f896d3b6..37b20a1e31 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java @@ -1,5 +1,12 @@ package com.box.l10n.mojito.service.thirdparty; +import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSPhrase.areTagsWithin5Minutes; +import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSPhrase.uploadTagToLocalDateTime; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.json.ObjectMapper; import com.box.l10n.mojito.service.assetExtraction.ServiceTestBase; @@ -11,6 +18,8 @@ import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; import com.box.l10n.mojito.test.TestIdWatcher; import com.google.common.collect.ImmutableMap; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.List; import org.junit.Assume; import org.junit.Rule; @@ -59,6 +68,14 @@ public void testBasics() throws RepositoryLocaleCreationException { null, null); + thirdPartyTMSPhrase.push( + repository, + testProjectId, + thirdPartyServiceTestData.getPluralSeparator(), + null, + null, + null); + thirdPartyTMSPhrase.pull( repository, testProjectId, @@ -88,4 +105,37 @@ public void testBasics() throws RepositoryLocaleCreationException { // t)); } + + @Test + public void testUploadTagToLocalDateTimeValid() { + LocalDateTime localDateTime = uploadTagToLocalDateTime("push_test_2024_11_21_18_55_38_004_502"); + assertEquals( + "Parsed LocalDateTime does not match the expected value", + "2024-11-21T18:55:38.004", + localDateTime.format(DateTimeFormatter.ISO_DATE_TIME) + ); + } + + @Test + public void testUploadTagToLocalDateTimeInvalid() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> uploadTagToLocalDateTime("invalid_tag_2024_11_21_18_55_004")); + assertTrue(exception.getMessage().contains("Invalid tag format")); + } + + @Test + public void testTagsWithin5Minutes() { + String tag1 = "push_test_2024_11_21_18_55_38_004_502"; + String tag2 = "push_test_2024_11_21_18_53_30_123_456"; + assertTrue(areTagsWithin5Minutes(tag1, tag2)); + } + + @Test + public void testTagsOutside5Minutes() { + String tag1 = "push_test_2024_11_21_18_55_38_004_502"; + String tag2 = "push_test_2024_11_21_18_49_30_123_456"; + assertFalse(areTagsWithin5Minutes(tag1, tag2)); + } } From 0ce43e34f860f2a0d0e7e8ede2dd6d5d3472a9c6 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 22 Nov 2024 15:28:50 -0800 Subject: [PATCH 101/105] Add an option to AiTranslate only a specific list of text units --- .../command/RepositoryAiTranslationCommand.java | 8 +++++++- .../client/RepositoryAiTranslateClient.java | 1 + .../mojito/rest/textunit/AiTranslateWS.java | 2 ++ .../oaitranslate/AiTranslateService.java | 17 +++++++++++++++-- .../oaitranslate/AiTranslateServiceTest.java | 4 ++-- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java index 7986d5ab84..e77a52ce4b 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java @@ -55,6 +55,12 @@ public class RepositoryAiTranslationCommand extends Command { + "sending too many strings to MT)") int sourceTextMaxCount = 100; + @Parameter( + names = {"--text-unit-ids"}, + arity = 1, + description = "The list of TmTextUnitIds to translate") + List textUnitIds; + @Parameter( names = {"--use-batch"}, arity = 1, @@ -87,7 +93,7 @@ public void execute() throws CommandException { ProtoAiTranslateResponse protoAiTranslateResponse = repositoryAiTranslateClient.translateRepository( new RepositoryAiTranslateClient.ProtoAiTranslateRequest( - repositoryParam, locales, sourceTextMaxCount, useBatch)); + repositoryParam, locales, sourceTextMaxCount, textUnitIds, useBatch)); PollableTask pollableTask = protoAiTranslateResponse.pollableTask(); commandHelper.waitForPollableTask(pollableTask.getId()); diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java b/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java index f0f5207675..27986eb8a4 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java @@ -32,6 +32,7 @@ public record ProtoAiTranslateRequest( String repositoryName, List targetBcp47tags, int sourceTextMaxCountPerLocale, + List tmTextUnitIds, boolean useBatch) {} public record ProtoAiTranslateResponse(PollableTask pollableTask) {} diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java index 7f33de3211..6f600b0ed5 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java @@ -43,6 +43,7 @@ public ProtoAiTranslateResponse aiTranslate( protoAiTranslateRequest.repositoryName(), protoAiTranslateRequest.targetBcp47tags(), protoAiTranslateRequest.sourceTextMaxCountPerLocale(), + protoAiTranslateRequest.tmTextUnitIds(), protoAiTranslateRequest.useBatch())); return new ProtoAiTranslateResponse(pollableFuture.getPollableTask()); @@ -53,6 +54,7 @@ public record ProtoAiTranslateRequest( List targetBcp47tags, int sourceTextMaxCountPerLocale, boolean useBatch, + List tmTextUnitIds, boolean allLocales) {} public record ProtoAiTranslateResponse(PollableTask pollableTask) {} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java index 8d9b962398..93ad54f81a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java @@ -126,6 +126,7 @@ public record AiTranslateInput( String repositoryName, List targetBcp47tags, int sourceTextMaxCountPerLocale, + List tmTextUnitIds, boolean useBatch) {} public PollableFuture aiTranslateAsync(AiTranslateInput aiTranslateInput) { @@ -161,7 +162,10 @@ public void aiTranslateNoBatch(AiTranslateInput aiTranslateInput) { .flatMap( rl -> asyncProcessLocale( - rl, aiTranslateInput.sourceTextMaxCountPerLocale(), openAIClientPool), + rl, + aiTranslateInput.sourceTextMaxCountPerLocale(), + aiTranslateInput.tmTextUnitIds(), + openAIClientPool), 10) .then() .doOnTerminate( @@ -174,6 +178,7 @@ public void aiTranslateNoBatch(AiTranslateInput aiTranslateInput) { Mono asyncProcessLocale( RepositoryLocale repositoryLocale, int sourceTextMaxCountPerLocale, + List tmTextUnitIds, OpenAIClientPool openAIClientPool) { Repository repository = repositoryLocale.getRepository(); @@ -187,7 +192,15 @@ Mono asyncProcessLocale( textUnitSearcherParameters.setRepositoryIds(repository.getId()); textUnitSearcherParameters.setStatusFilter(StatusFilter.FOR_TRANSLATION); textUnitSearcherParameters.setLocaleId(repositoryLocale.getLocale().getId()); - textUnitSearcherParameters.setLimit(sourceTextMaxCountPerLocale); + if (tmTextUnitIds != null) { + logger.debug( + "Using tmTextUnitIds: {} for ai translate repository: {}", + tmTextUnitIds, + repository.getName()); + textUnitSearcherParameters.setTmTextUnitIds(tmTextUnitIds); + } else { + textUnitSearcherParameters.setLimit(sourceTextMaxCountPerLocale); + } List textUnitDTOS = textUnitSearcher.search(textUnitSearcherParameters); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java index 1ef12ede94..afad21f8c3 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java @@ -33,7 +33,7 @@ public void aiTranslateBatch() throws ExecutionException, InterruptedException { aiTranslateService .aiTranslateAsync( new AiTranslateService.AiTranslateInput( - tmTestData.repository.getName(), null, 100, true)) + tmTestData.repository.getName(), null, 100, null, true)) .get(); } @@ -47,7 +47,7 @@ public void aiTranslateNoBatch() aiTranslateService .aiTranslateAsync( new AiTranslateService.AiTranslateInput( - tmTestData.repository.getName(), null, 100, false)) + tmTestData.repository.getName(), null, 100, null, false)) .get(); } } From a6f71438b1843aaac8d3f44737d42808b6f414fb Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Wed, 27 Nov 2024 14:01:12 -0800 Subject: [PATCH 102/105] Ad hoc relaxation of the integrity checker for plural strings Only the CLDR 'other' form must have the exact same number of placeholders. For other forms, we allow one placeholder to be removed. Note that placeholders are stored in a set, so duplicate placeholders will count as one. This only target printflike check, so mainly targeting Android, and some gettext. This is pretty adhoc anyway --- .../PluralIntegrityCheckerRelaxer.java | 41 ++++++++++++ .../tm/TMTextUnitIntegrityCheckService.java | 19 +++++- .../TextUnitBatchImporterService.java | 56 ++++++++++------ .../PluralIntegrityCheckerRelaxerTest.java | 64 +++++++++++++++++++ 4 files changed, 158 insertions(+), 22 deletions(-) create mode 100644 webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxer.java create mode 100644 webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxerTest.java diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxer.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxer.java new file mode 100644 index 0000000000..13d08b167c --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxer.java @@ -0,0 +1,41 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import java.util.Set; +import org.springframework.stereotype.Component; + +@Component +public class PluralIntegrityCheckerRelaxer { + + /** + * This is very ad hoc! + * + *

There are cases where the plural form doesn’t retain the number. Right now, those are + * wrongly marked as rejected. Ideally, we should clearly identify which placeholder might be + * missing from the string, but that’s another level of complexity. Also, use the class name to + * target certain integrity checks to keep this hack constrained. + */ + public boolean shouldRelaxIntegrityCheck( + String source, String target, String pluralForm, TextUnitIntegrityChecker textUnitChecker) { + + boolean shouldRelax = false; + + if (pluralForm != null && !"other".equals(pluralForm)) { + if (textUnitChecker instanceof PrintfLikeIntegrityChecker + || textUnitChecker instanceof PrintfLikeIgnorePercentageAfterBracketsIntegrityChecker + || textUnitChecker instanceof PrintfLikeVariableTypeIntegrityChecker + || textUnitChecker instanceof SimplePrintfLikeIntegrityChecker) { + + RegexIntegrityChecker regexIntegrityChecker = (RegexIntegrityChecker) textUnitChecker; + + Set sourcePlaceholders = regexIntegrityChecker.getPlaceholders(source); + Set targetPlaceholders = regexIntegrityChecker.getPlaceholders(target); + + if (sourcePlaceholders.size() - targetPlaceholders.size() <= 1) { + shouldRelax = true; + } + } + } + + return shouldRelax; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java index ad45b4c186..587528acfe 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java @@ -7,6 +7,7 @@ import com.box.l10n.mojito.service.asset.AssetRepository; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckException; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckerFactory; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.PluralIntegrityCheckerRelaxer; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TextUnitIntegrityChecker; import java.util.Set; import org.slf4j.Logger; @@ -27,6 +28,8 @@ public class TMTextUnitIntegrityCheckService { @Autowired TMTextUnitRepository tmTextUnitRepository; + @Autowired PluralIntegrityCheckerRelaxer pluralIntegrityCheckerRelaxer; + /** * Checks the integrity of the content given the {@link com.box.l10n.mojito.entity.TMTextUnit#id} * @@ -46,7 +49,21 @@ public void checkTMTextUnitIntegrity(Long tmTextUnitId, String contentToCheck) logger.debug("No designated checker for this asset. Nothing to do"); } else { for (TextUnitIntegrityChecker textUnitChecker : textUnitCheckers) { - textUnitChecker.check(tmTextUnit.getContent(), contentToCheck); + try { + textUnitChecker.check(tmTextUnit.getContent(), contentToCheck); + } catch (IntegrityCheckException e) { + if (tmTextUnit.getPluralForm() != null && pluralIntegrityCheckerRelaxer.shouldRelaxIntegrityCheck( + tmTextUnit.getContent(), + contentToCheck, + tmTextUnit.getPluralForm().getName(), + textUnitChecker)) { + logger.debug( + "Relaxing the check for plural string with form: {}", + tmTextUnit.getPluralForm().getName()); + } else { + throw e; + } + } } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java index 564250faa0..3402a3ca4b 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java @@ -22,6 +22,7 @@ import com.box.l10n.mojito.service.asset.ImportTextUnitJobInput; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckException; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckerFactory; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.PluralIntegrityCheckerRelaxer; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TextUnitIntegrityChecker; import com.box.l10n.mojito.service.locale.LocaleService; import com.box.l10n.mojito.service.pollableTask.PollableFuture; @@ -96,6 +97,8 @@ public class TextUnitBatchImporterService { @Autowired MeterRegistry meterRegistry; + @Autowired PluralIntegrityCheckerRelaxer pluralIntegrityCheckerRelaxer; + @Value("${l10n.textUnitBatchImporterService.quartz.schedulerName:" + DEFAULT_SCHEDULER_NAME + "}") String schedulerName; @@ -386,30 +389,41 @@ void applyIntegrityChecks( currentTextUnit.getSource(), textUnitForBatchImport.getContent(), ice); - textUnitForBatchImport.setIncludedInLocalizedFile(false); - textUnitForBatchImport.setStatus(Status.TRANSLATION_NEEDED); - - if (hasSameTarget) { - switch (integrityChecksType) { - case UNLESS_TAGGED_USE_INTEGRITY_CHECKER_STATUS: - if (!Strings.nullToEmpty(currentTextUnit.getTargetComment()) - .toLowerCase() - .contains(FALSE_POSITIVE_TAG_FOR_STATUS)) { + + if (pluralIntegrityCheckerRelaxer.shouldRelaxIntegrityCheck( + currentTextUnit.getSource(), + textUnitForBatchImport.getContent(), + textUnitForBatchImport.getCurrentTextUnit().getPluralForm(), + textUnitChecker)) { + logger.debug( + "Relaxing the check for plural string with form: {}", + textUnitForBatchImport.getCurrentTextUnit().getPluralForm()); + } else { + textUnitForBatchImport.setIncludedInLocalizedFile(false); + textUnitForBatchImport.setStatus(Status.TRANSLATION_NEEDED); + + if (hasSameTarget) { + switch (integrityChecksType) { + case UNLESS_TAGGED_USE_INTEGRITY_CHECKER_STATUS: + if (!Strings.nullToEmpty(currentTextUnit.getTargetComment()) + .toLowerCase() + .contains(FALSE_POSITIVE_TAG_FOR_STATUS)) { + break; + } + case KEEP_STATUS_IF_SAME_TARGET: + case KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET: + textUnitForBatchImport.setIncludedInLocalizedFile( + currentTextUnit.isIncludedInLocalizedFile()); + textUnitForBatchImport.setStatus(currentTextUnit.getStatus()); break; - } - case KEEP_STATUS_IF_SAME_TARGET: - case KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET: - textUnitForBatchImport.setIncludedInLocalizedFile( - currentTextUnit.isIncludedInLocalizedFile()); - textUnitForBatchImport.setStatus(currentTextUnit.getStatus()); - break; + } } - } - TMTextUnitVariantComment tmTextUnitVariantComment = new TMTextUnitVariantComment(); - tmTextUnitVariantComment.setSeverity(Severity.ERROR); - tmTextUnitVariantComment.setContent(ice.getMessage()); - textUnitForBatchImport.getTmTextUnitVariantComments().add(tmTextUnitVariantComment); + TMTextUnitVariantComment tmTextUnitVariantComment = new TMTextUnitVariantComment(); + tmTextUnitVariantComment.setSeverity(Severity.ERROR); + tmTextUnitVariantComment.setContent(ice.getMessage()); + textUnitForBatchImport.getTmTextUnitVariantComments().add(tmTextUnitVariantComment); + } } } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxerTest.java new file mode 100644 index 0000000000..7afeb4961d --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxerTest.java @@ -0,0 +1,64 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.Test; + +public class PluralIntegrityCheckerRelaxerTest { + + @Test + public void testShouldRelaxIntegrityCheck_whenPluralFormIsOther() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + boolean result = + relaxer.shouldRelaxIntegrityCheck( + "source %d", "target", "other", new PrintfLikeIntegrityChecker()); + + assertFalse(result, "Integrity check should not be relaxed for plural form 'other'."); + } + + @Test + public void testShouldRelaxIntegrityCheck_whenPlaceholdersDifferByOne_One() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + boolean result = + relaxer.shouldRelaxIntegrityCheck( + "source %d", "target", "one", new PrintfLikeIntegrityChecker()); + + assertTrue( + result, "Integrity check should be relaxed when placeholders differ by one for 'one' form"); + } + + @Test + public void testShouldRelaxIntegrityCheck_whenPlaceholdersDifferByOne_One_DuplicatePlaceholder() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + boolean result = + relaxer.shouldRelaxIntegrityCheck( + "source %d %d", "target", "one", new PrintfLikeIntegrityChecker()); + + assertTrue( + result, + "Integrity check should be relaxed when placeholders differ by one for 'one' form (as placeholders are stored in a set, ignoring duplicates)"); + } + + @Test + public void testShouldRelaxIntegrityCheck_whenPlaceholdersDifferByTwo_One() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + boolean result = + relaxer.shouldRelaxIntegrityCheck( + "source %1d %2d", "target", "one", new PrintfLikeIntegrityChecker()); + + assertFalse( + result, + "Integrity check should not be relaxed when placeholders differ by two for 'one' form"); + } + + @Test + public void testShouldRelaxIntegrityCheck_withUnsupportedChecker() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + + boolean result = + relaxer.shouldRelaxIntegrityCheck("source %d", "target", "one", new URLIntegrityChecker()); + + assertFalse(result, "Integrity check should not be relaxed for unsupported checker types."); + } +} From 7954f3d9187445ef6f44dd9d273c3d627db505e7 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Thu, 5 Dec 2024 17:59:59 -0800 Subject: [PATCH 103/105] Remove System.out.println --- .../java/com/box/l10n/mojito/service/commit/CommitService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/commit/CommitService.java b/webapp/src/main/java/com/box/l10n/mojito/service/commit/CommitService.java index e9f5eafb72..a5ac52163c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/commit/CommitService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/commit/CommitService.java @@ -148,7 +148,7 @@ public Commit getOrCreateCommit( commit.setAuthorName(authorName); commit.setSourceCreationDate(sourceCreationDate); - System.out.printf("setSourceCreationDate: %s%n", commit.getSourceCreationDate()); + logger.debug("setSourceCreationDate: {}", commit.getSourceCreationDate()); commit = commitRepository.save(commit); } From 5571379078f8d0d0170471d7e743453945df2b94 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 13 Dec 2024 10:13:32 -0800 Subject: [PATCH 104/105] Ai translate only uesd strings --- .../service/oaitranslate/AiTranslateService.java | 2 ++ .../service/tm/TMTextUnitIntegrityCheckService.java | 11 ++++++----- .../service/thirdparty/ThirdPartyTMSPhraseTest.java | 7 +++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java index 93ad54f81a..cc0014290b 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java @@ -39,6 +39,7 @@ import com.box.l10n.mojito.service.tm.search.TextUnitDTO; import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.box.l10n.mojito.service.tm.search.UsedFilter; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.SerializationFeature; @@ -456,6 +457,7 @@ Function createBatchForRepositoryLocale( textUnitSearcherParameters.setStatusFilter(StatusFilter.UNTRANSLATED); textUnitSearcherParameters.setLocaleId(repositoryLocale.getLocale().getId()); textUnitSearcherParameters.setLimit(sourceTextMaxCountPerLocale); + textUnitSearcherParameters.setUsedFilter(UsedFilter.USED); List textUnitDTOS = textUnitSearcher.search(textUnitSearcherParameters); CreateBatchResponse createBatchResponse = null; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java index 587528acfe..531e29876a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java @@ -52,11 +52,12 @@ public void checkTMTextUnitIntegrity(Long tmTextUnitId, String contentToCheck) try { textUnitChecker.check(tmTextUnit.getContent(), contentToCheck); } catch (IntegrityCheckException e) { - if (tmTextUnit.getPluralForm() != null && pluralIntegrityCheckerRelaxer.shouldRelaxIntegrityCheck( - tmTextUnit.getContent(), - contentToCheck, - tmTextUnit.getPluralForm().getName(), - textUnitChecker)) { + if (tmTextUnit.getPluralForm() != null + && pluralIntegrityCheckerRelaxer.shouldRelaxIntegrityCheck( + tmTextUnit.getContent(), + contentToCheck, + tmTextUnit.getPluralForm().getName(), + textUnitChecker)) { logger.debug( "Relaxing the check for plural string with form: {}", tmTextUnit.getPluralForm().getName()); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java index 37b20a1e31..77af92b666 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java @@ -110,10 +110,9 @@ public void testBasics() throws RepositoryLocaleCreationException { public void testUploadTagToLocalDateTimeValid() { LocalDateTime localDateTime = uploadTagToLocalDateTime("push_test_2024_11_21_18_55_38_004_502"); assertEquals( - "Parsed LocalDateTime does not match the expected value", - "2024-11-21T18:55:38.004", - localDateTime.format(DateTimeFormatter.ISO_DATE_TIME) - ); + "Parsed LocalDateTime does not match the expected value", + "2024-11-21T18:55:38.004", + localDateTime.format(DateTimeFormatter.ISO_DATE_TIME)); } @Test From 8bdb00857a7d27d9145073687bcb5822ecbb93c9 Mon Sep 17 00:00:00 2001 From: Jean Aurambault Date: Fri, 13 Dec 2024 11:03:13 -0800 Subject: [PATCH 105/105] Ai translate only used for batch --- .../box/l10n/mojito/service/oaitranslate/AiTranslateService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java index cc0014290b..d36483330a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java @@ -193,6 +193,7 @@ Mono asyncProcessLocale( textUnitSearcherParameters.setRepositoryIds(repository.getId()); textUnitSearcherParameters.setStatusFilter(StatusFilter.FOR_TRANSLATION); textUnitSearcherParameters.setLocaleId(repositoryLocale.getLocale().getId()); + textUnitSearcherParameters.setUsedFilter(UsedFilter.USED); if (tmTextUnitIds != null) { logger.debug( "Using tmTextUnitIds: {} for ai translate repository: {}",