diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java new file mode 100644 index 0000000000..fa392b2e7c --- /dev/null +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubCreatePRCommand.java @@ -0,0 +1,154 @@ +package com.box.l10n.mojito.cli.command; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.box.l10n.mojito.cli.console.ConsoleWriter; +import com.box.l10n.mojito.github.GithubClient; +import com.box.l10n.mojito.github.GithubClients; +import com.box.l10n.mojito.github.GithubException; +import java.io.IOException; +import java.util.List; +import org.fusesource.jansi.Ansi; +import org.kohsuke.github.GHIssueState; +import org.kohsuke.github.GHPullRequest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +@Component +@Scope("prototype") +@Parameters( + commandNames = {"github-pr-create"}, + commandDescription = "Create a Github PR") +public class GithubCreatePRCommand extends Command { + + @Autowired GithubClients githubClients; + + @Qualifier("ansiCodeEnabledFalse") + @Autowired + ConsoleWriter consoleWriter; + + @Parameter( + names = {"--owner", "-o"}, + required = true, + arity = 1, + description = "The Github repository owner") + String owner; + + @Parameter( + names = {"--repository", "-r"}, + required = true, + arity = 1, + description = "The Github repository name") + String repository; + + @Parameter( + names = {"--title"}, + required = true, + arity = 1, + description = "The PR title") + String title; + + @Parameter( + names = {"--head"}, + required = true, + arity = 1, + description = "The PR head") + String head; + + @Parameter( + names = {"--base"}, + required = true, + arity = 1, + description = "The PR base") + String base; + + @Parameter( + names = {"--body"}, + required = false, + arity = 1, + description = "The PR body") + String body; + + @Parameter( + names = {"--reviewers"}, + required = false, + variableArity = true, + description = "The PR reviewers") + List reviewers; + + @Parameter( + names = {"--enable-auto-merge"}, + required = false, + arity = 1, + description = "Enable auto-merge with the specified method") + EnableAutoMergeType enableAutoMerge = EnableAutoMergeType.NONE; + + @Parameter( + names = {"--labels"}, + required = false, + variableArity = true, + description = "The PR labels") + List labels; + + @Parameter( + names = {"--also-close-prefixed"}, + description = + "Closes all PRs that start with the specified prefix. Use this to manage PRs that are part of a batch or related by a common feature.", + required = false) + String alsoClosePrefixed = null; + + enum EnableAutoMergeType { + SQUASH, + MERGE, + NONE + } + + @Override + public boolean shouldShowInCommandList() { + return false; + } + + @Override + protected void execute() throws CommandException { + try { + + GithubClient githubClient = githubClients.getClient(owner); + + if (alsoClosePrefixed != null) { + closePRPrefixedWith(githubClient, alsoClosePrefixed); + } + + GHPullRequest pr = githubClient.createPR(repository, title, head, base, body, reviewers); + + consoleWriter.a("PR created: ").fg(Ansi.Color.CYAN).a(pr.getHtmlUrl().toString()).println(); + if (!EnableAutoMergeType.NONE.equals(enableAutoMerge)) { + githubClient.enableAutoMerge(pr, GithubClient.AutoMergeType.SQUASH); + } + + githubClient.addLabelsToPR(pr, labels); + + } catch (GithubException e) { + throw new CommandException(e); + } + } + + void closePRPrefixedWith(GithubClient githubClient, String alsoClosePrefixed) { + githubClient.listPR(repository, GHIssueState.OPEN).stream() + .filter(pr -> pr.getTitle().startsWith(alsoClosePrefixed)) + .forEach( + pullRequest -> { + try { + consoleWriter + .a("Closing: ") + .fg(Ansi.Color.CYAN) + .a(pullRequest.getHtmlUrl().toString()) + .println(); + pullRequest.close(); + } catch (IOException e) { + throw new CommandException(e); + } + }); + } +} diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubGetInstallationTokenCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubGetInstallationTokenCommand.java index 3620e58251..78418c5f59 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubGetInstallationTokenCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/GithubGetInstallationTokenCommand.java @@ -4,9 +4,6 @@ import com.beust.jcommander.Parameters; import com.box.l10n.mojito.cli.console.ConsoleWriter; import com.box.l10n.mojito.github.GithubClients; -import java.io.IOException; -import java.security.NoSuchAlgorithmException; -import java.security.spec.InvalidKeySpecException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Scope; @@ -50,7 +47,7 @@ protected void execute() throws CommandException { consoleWriter .a(githubClients.getClient(owner).getGithubAppInstallationToken(repository).getToken()) .print(); - } catch (IOException | NoSuchAlgorithmException | InvalidKeySpecException e) { + } catch (Exception e) { throw new CommandException(e); } } diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java index 896b2f59be..5188f4c4f9 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java @@ -11,7 +11,6 @@ import com.box.l10n.mojito.rest.client.LocaleClient; import com.box.l10n.mojito.rest.client.RepositoryClient; import com.box.l10n.mojito.rest.client.exception.AssetNotFoundException; -import com.box.l10n.mojito.rest.client.exception.PollableTaskException; import com.box.l10n.mojito.rest.entity.Asset; import com.box.l10n.mojito.rest.entity.ImportLocalizedAssetBody; import com.box.l10n.mojito.rest.entity.ImportLocalizedAssetBody.StatusForEqualTarget; @@ -19,9 +18,9 @@ import com.box.l10n.mojito.rest.entity.Repository; import java.nio.file.Path; import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import org.fusesource.jansi.Ansi; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,6 +71,22 @@ public class ImportLocalizedAssetCommand extends Command { description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION) String localeMappingParam; + @Parameter( + names = {"-lmt", "--locale-mapping-type"}, + arity = 1, + required = false, + description = + "Specifies how to handle locale mappings when used with --locale-mapping. " + + "MAP_ONLY processes only the locales explicitly specified in the mapping. " + + "WITH_REPOSITORY generates a basic mapping from the repository's locales and " + + "supplements it with the provided mapping, potentially overriding existing entries.") + LocaleMappingType localeMappingTypeParam = LocaleMappingType.WITH_REPOSITORY; + + enum LocaleMappingType { + MAP_ONLY, + WITH_REPOSITORY + } + @Parameter( names = {Param.FILE_TYPE_LONG, Param.FILE_TYPE_SHORT}, variableArity = true, @@ -124,6 +139,12 @@ public class ImportLocalizedAssetCommand extends Command { converter = ImportLocalizedAssetBodyStatusForEqualTargetConverter.class) StatusForEqualTarget statusForEqualTarget = StatusForEqualTarget.APPROVED; + @Parameter( + names = {"--continue-on-error"}, + arity = 0, + description = "Continue import on errors") + boolean continueOnError = false; + @Autowired AssetClient assetClient; @Autowired LocaleClient localeClient; @@ -152,6 +173,7 @@ public void execute() throws CommandException { .println(2); repository = commandHelper.findRepositoryByName(repositoryParam); + commandDirectories = new CommandDirectories(sourceDirectoryParam, targetDirectoryParam); inverseLocaleMapping = localeMappingHelper.getInverseLocaleMapping(localeMappingParam); @@ -163,15 +185,37 @@ public void execute() throws CommandException { sourcePathFilterRegex, directoriesIncludePatterns, directoriesExcludePatterns)) { - for (Locale locale : getLocalesForImport()) { - doImportFileMatch(sourceFileMatch, locale); - } + + List list = + getLocalesForImport().stream() + .map(locale -> doImportFileMatch(sourceFileMatch, locale)) + .filter(Objects::nonNull) + .toList(); + + list.forEach( + importLocalizedAssetForContent -> { + try { + commandHelper.waitForPollableTask( + importLocalizedAssetForContent.getPollableTask().getId()); + } catch (CommandException e) { + if (continueOnError) { + consoleWriter + .a(" Error while importing: ") + .fg(Ansi.Color.RED) + .a(sourceFileMatch.getPath().toString()) + .println(); + } else { + throw e; + } + } + }); } consoleWriter.fg(Ansi.Color.GREEN).newLine().a("Finished").println(2); } - protected void doImportFileMatch(FileMatch fileMatch, Locale locale) throws CommandException { + protected ImportLocalizedAssetBody doImportFileMatch(FileMatch fileMatch, Locale locale) + throws CommandException { try { logger.info("Importing for locale: {}", locale.getBcp47Tag()); Path targetPath = getTargetPath(fileMatch, locale); @@ -185,56 +229,64 @@ protected void doImportFileMatch(FileMatch fileMatch, Locale locale) throws Comm Asset assetByPathAndRepositoryId = assetClient.getAssetByPathAndRepositoryId(fileMatch.getSourcePath(), repository.getId()); - ImportLocalizedAssetBody importLocalizedAssetForContent = - assetClient.importLocalizedAssetForContent( - assetByPathAndRepositoryId.getId(), - locale.getId(), - commandHelper.getFileContent(targetPath), - statusForEqualTarget, - fileMatch.getFileType().getFilterConfigIdOverride(), - commandHelper.getFilterOptionsOrDefaults( - fileMatch.getFileType(), filterOptionsParam)); - + String fileContent; try { - commandHelper.waitForPollableTask(importLocalizedAssetForContent.getPollableTask().getId()); - } catch (PollableTaskException e) { - throw new CommandException(e.getMessage(), e.getCause()); + fileContent = commandHelper.getFileContent(targetPath); + } catch (Exception e) { + if (continueOnError) { + if (!commandHelper.getFileContent(fileMatch.getPath()).isBlank()) { + consoleWriter + .a(" Missing file, skipping: ") + .fg(Ansi.Color.YELLOW) + .a(targetPath.toString()) + .println(); + } + return null; + } else { + throw e; + } } + return assetClient.importLocalizedAssetForContent( + assetByPathAndRepositoryId.getId(), + locale.getId(), + fileContent, + statusForEqualTarget, + fileMatch.getFileType().getFilterConfigIdOverride(), + commandHelper.getFilterOptionsOrDefaults(fileMatch.getFileType(), filterOptionsParam)); } catch (AssetNotFoundException ex) { - throw new CommandException( - "No asset for file [" + fileMatch.getPath() + "] into repo [" + repositoryParam + "]", - ex); + if (continueOnError) { + consoleWriter + .a(" Asset not found: ") + .fg(Ansi.Color.YELLOW) + .a(fileMatch.getPath().toString()) + .println(); + } else { + throw new CommandException( + "No asset for file [" + fileMatch.getPath() + "] into repo [" + repositoryParam + "]", + ex); + } } + return null; } public Collection getLocalesForImport() { Collection sortedRepositoryLocales = commandHelper.getSortedRepositoryLocales(repository).values(); - filterLocalesWithMapping(sortedRepositoryLocales); - return sortedRepositoryLocales; - } - - private void filterLocalesWithMapping(Collection locales) { - - if (inverseLocaleMapping != null) { - Iterator iterator = locales.iterator(); - while (iterator.hasNext()) { - Locale l = iterator.next(); - if (!inverseLocaleMapping.containsKey(l.getBcp47Tag())) { - iterator.remove(); - } - } + if (LocaleMappingType.MAP_ONLY.equals(localeMappingTypeParam) && inverseLocaleMapping != null) { + sortedRepositoryLocales = + sortedRepositoryLocales.stream() + .filter(l -> inverseLocaleMapping.containsKey(l.getBcp47Tag())) + .toList(); } + return sortedRepositoryLocales; } private Path getTargetPath(FileMatch fileMatch, Locale locale) throws CommandException { - String targetLocale; + String targetLocale = locale.getBcp47Tag(); - if (inverseLocaleMapping != null) { + if (inverseLocaleMapping != null && inverseLocaleMapping.containsKey(locale.getBcp47Tag())) { targetLocale = inverseLocaleMapping.get(locale.getBcp47Tag()); - } else { - targetLocale = locale.getBcp47Tag(); } logger.info("processing locale for import: {}", targetLocale); diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommand.java index 3cc366311c..42de156dc2 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommand.java @@ -18,6 +18,7 @@ import com.box.l10n.mojito.rest.entity.RepositoryLocaleStatistic; import com.box.l10n.mojito.rest.entity.RepositoryStatistic; import java.nio.file.Path; +import java.util.AbstractMap.SimpleEntry; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -73,6 +74,22 @@ public class PullCommand extends Command { description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION) String localeMappingParam; + @Parameter( + names = {"-lmt", "--locale-mapping-type"}, + arity = 1, + required = false, + description = + "Specifies how to handle locale mappings when used with --locale-mapping. " + + "MAP_ONLY processes only the locales explicitly specified in the mapping. " + + "WITH_REPOSITORY generates a basic mapping from the repository's locales and " + + "supplements it with the provided mapping, potentially overriding existing entries.") + LocaleMappingType localeMappingTypeParam = LocaleMappingType.WITH_REPOSITORY; + + enum LocaleMappingType { + MAP_ONLY, + WITH_REPOSITORY + } + @Parameter( names = {Param.FILE_TYPE_LONG, Param.FILE_TYPE_SHORT}, variableArity = true, @@ -184,7 +201,10 @@ public class PullCommand extends Command { CommandDirectories commandDirectories; - /** Contains a map of locale for generating localized file a locales defined in the repository. */ + /** + * Contains a map of locale for generating localized file. Key: file output tag, value: the tag in + * the repository + */ Map localeMappings; /** Map of locale and the boolean value for fully translated status of the locale */ @@ -233,7 +253,7 @@ public void execute() throws CommandException { commandHelper.getFilterOptionsOrDefaults( sourceFileMatch.getFileType(), filterOptionsParam); - generateLocalizedFiles(sourceFileMatch, filterOptions); + generateLocalizedFiles(repository, sourceFileMatch, filterOptions); } } @@ -242,14 +262,6 @@ public void execute() throws CommandException { consoleWriter.fg(Color.GREEN).newLine().a("Finished").println(2); } - private void generateLocalizedFiles(FileMatch sourceFileMatch, List filterOptions) { - if (localeMappingParam != null) { - generateLocalizedFilesWithLocaleMaping(repository, sourceFileMatch, filterOptions); - } else { - generateLocalizedFilesWithoutLocaleMapping(repository, sourceFileMatch, filterOptions); - } - } - void initPullRunName() { if (recordPullRun) { pullRunName = UUID.randomUUID().toString(); @@ -270,51 +282,66 @@ void writePullRunFileIfNeeded() { } /** - * Default generation, uses the locales defined in the repository to generate the localized files. + * Default generation, uses the locales defined in the repository to generate the localized files + * and eventually use the locale mapping to override the output tags. * * @param repository * @param sourceFileMatch * @param filterOptions * @throws CommandException */ - void generateLocalizedFilesWithoutLocaleMapping( + void generateLocalizedFiles( Repository repository, FileMatch sourceFileMatch, List filterOptions) throws CommandException { - logger.debug("Generate localized files (without locale mapping)"); + logger.debug("Generate localized files"); + Map outputToRepo = getMapOutputTagToRepositoryLocale(); - for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale.values()) { - generateLocalizedFile(repository, sourceFileMatch, filterOptions, null, repositoryLocale); + for (Map.Entry e : outputToRepo.entrySet()) { + generateLocalizedFile(repository, sourceFileMatch, filterOptions, e.getKey(), e.getValue()); } } - /** - * Generation with locale mapping. The localized files are generated using specific output tags - * while still using the repository locale to fetch the proper translations. - * - * @param repository - * @param sourceFileMatch - * @param filterOptions - * @throws CommandException - */ - void generateLocalizedFilesWithLocaleMaping( - Repository repository, FileMatch sourceFileMatch, List filterOptions) - throws CommandException { + Map getMapOutputTagToRepositoryLocale() { + Map outputToRepo; + + if (LocaleMappingType.MAP_ONLY.equals(localeMappingTypeParam)) { + + if (localeMappings == null) { + throw new RuntimeException("MAP_ONLY must be used with an locale mapping"); + } + outputToRepo = getLocaleMapFromLocaleMappings(); + + } else { + if (localeMappings == null) { + outputToRepo = repositoryLocalesWithoutRootLocale; + } else { + outputToRepo = + repositoryLocalesWithoutRootLocale.entrySet().stream() + .filter(e -> !localeMappings.containsValue(e.getValue().getLocale().getBcp47Tag())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + outputToRepo.putAll(getLocaleMapFromLocaleMappings()); + } + } + return outputToRepo; + } - logger.debug("Generate localized files with locale mapping"); + private Map getLocaleMapFromLocaleMappings() { + return localeMappings.entrySet().stream() + .map(e -> new SimpleEntry<>(e.getKey(), getRepositoryLocaleByTag(e.getValue()))) + .collect(Collectors.toMap(SimpleEntry::getKey, SimpleEntry::getValue)); + } - List repositoryLocales = - localeMappings.entrySet().stream() - .map(entry -> getRepositoryLocaleForOutputBcp47Tag(entry.getKey())) - .distinct() - .collect(Collectors.toList()); + private RepositoryLocale getRepositoryLocaleByTag(String tag) { + RepositoryLocale repositoryLocale; - for (Map.Entry localeMapping : localeMappings.entrySet()) { - String outputBcp47tag = localeMapping.getKey(); - RepositoryLocale repositoryLocale = getRepositoryLocaleForOutputBcp47Tag(outputBcp47tag); - generateLocalizedFile( - repository, sourceFileMatch, filterOptions, outputBcp47tag, repositoryLocale); + if (rootRepositoryLocale.getLocale().getBcp47Tag().equals(tag)) { + repositoryLocale = rootRepositoryLocale; + } else { + repositoryLocale = repositoryLocalesWithoutRootLocale.get(tag); } + + return repositoryLocale; } void generateLocalizedFile( diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommandParallel.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommandParallel.java index a8452ce6e0..1221a5b75a 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommandParallel.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/PullCommandParallel.java @@ -8,7 +8,6 @@ import com.box.l10n.mojito.rest.entity.PollableTask; import com.box.l10n.mojito.rest.entity.Repository; import com.box.l10n.mojito.rest.entity.RepositoryLocale; -import com.google.common.collect.Lists; import java.nio.file.Path; import java.util.HashMap; import java.util.List; @@ -42,6 +41,7 @@ public PullCommandParallel(PullCommand pullCommand) { this.sourceLocale = pullCommand.sourceLocale; this.fileTypes = pullCommand.fileTypes; this.localeMappingParam = pullCommand.localeMappingParam; + this.localeMappingTypeParam = pullCommand.localeMappingTypeParam; this.repositoryParam = pullCommand.repositoryParam; this.sourceDirectoryParam = pullCommand.sourceDirectoryParam; this.targetDirectoryParam = pullCommand.targetDirectoryParam; @@ -82,19 +82,10 @@ public void pull() throws CommandException { private void sendContentForLocalizedGeneration( FileMatch sourceFileMatch, List filterOptions) { - if (localeMappingParam != null) { - pollableTaskIdToFileMatchMap.put( - generateLocalizedFilesWithLocaleMappingParallel( - repository, sourceFileMatch, filterOptions) - .getId(), - sourceFileMatch); - } else { - pollableTaskIdToFileMatchMap.put( - generateLocalizedFilesWithoutLocaleMappingParallel( - repository, sourceFileMatch, filterOptions) - .getId(), - sourceFileMatch); - } + + pollableTaskIdToFileMatchMap.put( + generateLocalizedFilesParallel(repository, sourceFileMatch, filterOptions).getId(), + sourceFileMatch); } private void pollForLocalizedFiles() { @@ -118,31 +109,6 @@ private void pollForLocalizedFiles() { }); } - private PollableTask generateLocalizedFilesWithLocaleMappingParallel( - Repository repository, FileMatch sourceFileMatch, List filterOptions) - throws CommandException { - - List repositoryLocales = - localeMappings.entrySet().stream() - .map(entry -> getRepositoryLocaleForOutputBcp47Tag(entry.getKey())) - .distinct() - .collect(Collectors.toList()); - return generateLocalizedFiles(repository, sourceFileMatch, filterOptions, repositoryLocales); - } - - private PollableTask generateLocalizedFilesWithoutLocaleMappingParallel( - Repository repository, FileMatch sourceFileMatch, List filterOptions) - throws CommandException { - - logger.debug("Generate localized files (without locale mapping)"); - - return generateLocalizedFiles( - repository, - sourceFileMatch, - filterOptions, - Lists.newArrayList(repositoryLocalesWithoutRootLocale.values())); - } - void writeLocalizedAssetToTargetDirectory( LocalizedAssetBody localizedAsset, FileMatch sourceFileMatch) throws CommandException { @@ -173,24 +139,8 @@ private synchronized void printFileGeneratedToConsole( .println(); } - void generateLocalizedFilesWithoutLocaleMapping( - Repository repository, FileMatch sourceFileMatch, List filterOptions) - throws CommandException { - - logger.debug("Generate localized files (without locale mapping)"); - - generateLocalizedFiles( - repository, - sourceFileMatch, - filterOptions, - Lists.newArrayList(repositoryLocalesWithoutRootLocale.values())); - } - - private PollableTask generateLocalizedFiles( - Repository repository, - FileMatch sourceFileMatch, - List filterOptions, - List repositoryLocales) { + private PollableTask generateLocalizedFilesParallel( + Repository repository, FileMatch sourceFileMatch, List filterOptions) { Asset assetByPathAndRepositoryId; String sourcePath = @@ -207,12 +157,16 @@ private PollableTask generateLocalizedFiles( e); } - return getLocalizedAssetBodyParallel( - sourceFileMatch, - repositoryLocales.stream() + List repositoryLocales = + getMapOutputTagToRepositoryLocale().values().stream() + .distinct() .filter( repoLocale -> shouldGenerateLocalizedFileWithCliOutput(sourceFileMatch, repoLocale)) - .collect(Collectors.toList()), + .toList(); + + return getLocalizedAssetBodyParallel( + sourceFileMatch, + repositoryLocales, getRepoLocaleToOutputTagsMap(), filterOptions, assetByPathAndRepositoryId, @@ -220,18 +174,26 @@ private PollableTask generateLocalizedFiles( } private Map> getRepoLocaleToOutputTagsMap() { - Map> localeIdToOutputTagsMap = new HashMap<>(); + Map> localeIdToOutputTagsMap; if (localeMappings != null) { - for (Map.Entry mapping : localeMappings.entrySet()) { - String outputBcp47tag = mapping.getKey(); - RepositoryLocale locale = getRepositoryLocaleForOutputBcp47Tag(outputBcp47tag); - if (localeIdToOutputTagsMap.containsKey(locale)) { - localeIdToOutputTagsMap.get(locale).add(outputBcp47tag); - } else { - localeIdToOutputTagsMap.put(locale, Lists.newArrayList(outputBcp47tag)); - } - } + + Map mapOutputTagToRepositoryLocale = + getMapOutputTagToRepositoryLocale(); + + localeIdToOutputTagsMap = + mapOutputTagToRepositoryLocale.entrySet().stream() + .collect( + Collectors.groupingBy( + Map.Entry::getValue, + Collectors.mapping(Map.Entry::getKey, Collectors.toList()))); + + } else { + // Adapted for backward compatibility - a deeper refactor could improve clarity. + // Note: If there is no mapping, it returns an empty map. The impact on the backend is + // unclear, + // so this behavior is being preserved. + localeIdToOutputTagsMap = new HashMap<>(); } return localeIdToOutputTagsMap; diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java new file mode 100644 index 0000000000..e77a52ce4b --- /dev/null +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiTranslationCommand.java @@ -0,0 +1,101 @@ +package com.box.l10n.mojito.cli.command; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.box.l10n.mojito.cli.command.param.Param; +import com.box.l10n.mojito.cli.console.ConsoleWriter; +import com.box.l10n.mojito.rest.client.RepositoryAiTranslateClient; +import com.box.l10n.mojito.rest.client.RepositoryAiTranslateClient.ProtoAiTranslateResponse; +import com.box.l10n.mojito.rest.entity.PollableTask; +import java.util.List; +import java.util.stream.Collectors; +import org.fusesource.jansi.Ansi.Color; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +/** + * Command to machine translate untranslated strings in a repository. + * + * @author jaurambault + */ +@Component +@Scope("prototype") +@Parameters( + commandNames = {"repository-ai-translate"}, + commandDescription = "Ai translate untranslated and rejected strings in a repository") +public class RepositoryAiTranslationCommand extends Command { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(RepositoryAiTranslationCommand.class); + + @Autowired ConsoleWriter consoleWriter; + + @Parameter( + names = {Param.REPOSITORY_LONG, Param.REPOSITORY_SHORT}, + arity = 1, + required = true, + description = Param.REPOSITORY_DESCRIPTION) + String repositoryParam; + + @Parameter( + names = {Param.REPOSITORY_LOCALES_LONG, Param.REPOSITORY_LOCALES_SHORT}, + variableArity = true, + description = + "List of locales (bcp47 tags) to translate, if not provided translate all locales in the repository") + List locales; + + @Parameter( + names = {"--source-text-max-count"}, + arity = 1, + description = + "Source text max count per locale sent to MT (this param is used to avoid " + + "sending too many strings to MT)") + int sourceTextMaxCount = 100; + + @Parameter( + names = {"--text-unit-ids"}, + arity = 1, + description = "The list of TmTextUnitIds to translate") + List textUnitIds; + + @Parameter( + names = {"--use-batch"}, + arity = 1, + description = "To use the batch API or not") + boolean useBatch = false; + + @Autowired CommandHelper commandHelper; + + @Autowired RepositoryAiTranslateClient repositoryAiTranslateClient; + + @Override + public boolean shouldShowInCommandList() { + return false; + } + + @Override + public void execute() throws CommandException { + + consoleWriter + .newLine() + .a("Ai translate repository: ") + .fg(Color.CYAN) + .a(repositoryParam) + .reset() + .a(" for locales: ") + .fg(Color.CYAN) + .a(locales == null ? "" : locales.stream().collect(Collectors.joining(", ", "[", "]"))) + .println(2); + + ProtoAiTranslateResponse protoAiTranslateResponse = + repositoryAiTranslateClient.translateRepository( + new RepositoryAiTranslateClient.ProtoAiTranslateRequest( + repositoryParam, locales, sourceTextMaxCount, textUnitIds, useBatch)); + + PollableTask pollableTask = protoAiTranslateResponse.pollableTask(); + commandHelper.waitForPollableTask(pollableTask.getId()); + } +} diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java new file mode 100644 index 0000000000..0ed54713b4 --- /dev/null +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryTmTranslateCommand.java @@ -0,0 +1,208 @@ +package com.box.l10n.mojito.cli.command; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.box.l10n.mojito.cli.command.param.Param; +import com.box.l10n.mojito.cli.console.ConsoleWriter; +import com.box.l10n.mojito.rest.client.RepositoryClient; +import com.box.l10n.mojito.rest.client.TextUnitClient; +import com.box.l10n.mojito.rest.entity.Locale; +import com.box.l10n.mojito.rest.entity.PollableTask; +import com.box.l10n.mojito.rest.entity.Repository; +import com.box.l10n.mojito.rest.entity.RepositoryLocale; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.fusesource.jansi.Ansi; +import org.fusesource.jansi.Ansi.Color; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +/** + * Command to populate untranslated string with translation from another repository translation + * memory. This meant to process small volumes but can be helpful to combine the with the import + * localized files command. + * + *

Matches the text units by the source. Use text unit name and comment match as priority (moved + * assets), else just take the newest translation match. Has an option to not import string that + * have same source and target to avoid copy translation that might be from importing partly + * translated files. The drawback is that it could skip string that should be imported but the + * benefits is that the localized files will be same regardless. + * + *

Limited to translating at most 1000 strings in a run + * + * @author jaurambault + */ +@Component +@Scope("prototype") +@Parameters( + commandNames = {"repository-translate"}, + commandDescription = + "Translate a repository using another repository TM. Only untranslated text units" + + " will be processed, no overriding. Priorities text unit that have same and comment, else just look" + + "at the newest match") +public class RepositoryTmTranslateCommand extends Command { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(RepositoryTmTranslateCommand.class); + + @Autowired ConsoleWriter consoleWriter; + + @Parameter( + names = {Param.SOURCE_REPOSITORY_LONG, Param.SOURCE_REPOSITORY_SHORT}, + arity = 1, + required = false, + description = Param.SOURCE_REPOSITORY_DESCRIPTION) + String sourceRepositoryParam; + + @Parameter( + names = {Param.TARGET_REPOSITORY_LONG, Param.TARGET_REPOSITORY_SHORT}, + arity = 1, + required = true, + description = Param.TARGET_REPOSITORY_DESCRIPTION) + String targetRepositoryParam; + + @Parameter( + names = {"--import-source-equals-target"}, + arity = 0, + description = "Import the translation if the source is equal to the target") + boolean importSourceEqualsTargetParam = false; + + @Parameter( + names = {Param.REPOSITORY_LOCALES_LONG, Param.REPOSITORY_LOCALES_SHORT}, + variableArity = true, + required = false, + description = "List of locales (bcp47 tags) to translate") + List localesParam = null; + + @Autowired CommandHelper commandHelper; + + @Autowired TextUnitClient textUnitClient; + + @Autowired RepositoryClient repositoryClient; + + @Override + public void execute() throws CommandException { + + if (sourceRepositoryParam == null) { + sourceRepositoryParam = targetRepositoryParam; + } + + consoleWriter + .newLine() + .a("Translate repository: ") + .fg(Color.CYAN) + .a(targetRepositoryParam) + .reset() + .a(" with text units from repository: ") + .fg(Color.CYAN) + .a(sourceRepositoryParam) + .println(2); + + Repository sourceRepository = commandHelper.findRepositoryByName(sourceRepositoryParam); + Repository targetRepository = commandHelper.findRepositoryByName(targetRepositoryParam); + + List locales = localesParam; + if (locales == null) { + locales = + targetRepository.getRepositoryLocales().stream() + .filter(rl -> rl.getParentLocale() != null) + .map(RepositoryLocale::getLocale) + .map(Locale::getBcp47Tag) + .toList(); + } + + for (String locale : locales) { + consoleWriter.a("Processing locale: ").fg(Color.MAGENTA).a(locale).println(); + List textUnitsToSave = new ArrayList<>(); + + List untranslatedTextUnitsForLocale = + getUntranslatedTextUnitsForLocale(targetRepository.getName(), locale); + + for (TextUnitClient.TextUnit textUnit : untranslatedTextUnitsForLocale) { + consoleWriter.a("Processing source: ").fg(Color.CYAN).a(textUnit.source()).println(); + List existingTranslationWithSameSource = + getTranslationsWithExactSource(sourceRepository.getName(), locale, textUnit.source()); + + Optional match = + getMatchByNameAndComment( + textUnit.source(), textUnit.comment(), existingTranslationWithSameSource); + if (match.isPresent()) { + match = getMatchByNewest(existingTranslationWithSameSource); + } + + if (match.isPresent()) { + var translation = match.get(); + consoleWriter.a("Found match: ").fg(Color.CYAN).a(translation.target()).println(); + + if (translation.source().equals(translation.target())) { + if (importSourceEqualsTargetParam) { + textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); + } else { + consoleWriter.a("Skip import, source = target").println(); + } + } else { + textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); + } + } else { + consoleWriter.a("No match found").println(); + } + consoleWriter.println(); + } + + consoleWriter + .a("Saving translations for locale: ") + .fg(Color.MAGENTA) + .a(locale) + .reset() + .a("Text unit count: ") + .fg(Color.CYAN) + .a(textUnitsToSave.size()) + .println(); + TextUnitClient.ImportTextUnitsBatch importTextUnitsBatch = + new TextUnitClient.ImportTextUnitsBatch(false, true, textUnitsToSave); + PollableTask pollableTask = textUnitClient.importTextUnitBatch(importTextUnitsBatch); + commandHelper.waitForPollableTask(pollableTask.getId()); + } + consoleWriter.fg(Ansi.Color.GREEN).newLine().a("Finished").println(2); + } + + private List getTranslationsWithExactSource( + String repositoryName, String locale, String source) { + TextUnitClient.TextUnitSearchBody textUnitSearchBody = new TextUnitClient.TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.TRANSLATED); + textUnitSearchBody.setSource(source); + textUnitSearchBody.setLimit(20); + return textUnitClient.searchTextUnits(textUnitSearchBody); + } + + private List getUntranslatedTextUnitsForLocale( + String repositoryName, String locale) { + TextUnitClient.TextUnitSearchBody textUnitSearchBody = new TextUnitClient.TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setUsedFilter(TextUnitClient.UsedFilter.USED); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.UNTRANSLATED); + textUnitSearchBody.setLimit(1000); + return textUnitClient.searchTextUnits(textUnitSearchBody); + } + + private Optional getMatchByNewest( + List candidates) { + return candidates.stream().max(Comparator.comparingLong(TextUnitClient.TextUnit::createdDate)); + } + + private static Optional getMatchByNameAndComment( + String name, String comment, List candidates) { + return candidates.stream() + .filter(m -> Objects.equals(name, m.name()) && Objects.equals(comment, m.comment())) + .max(Comparator.comparingLong(TextUnitClient.TextUnit::createdDate)); + } +} diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java index 921b9645fe..3306baff4e 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommand.java @@ -41,6 +41,8 @@ public class SimpleFileEditorCommand extends Command { /** logger */ static Logger logger = LoggerFactory.getLogger(SimpleFileEditorCommand.class); + static final String COMMENTS_PATTERN = "(?s)/\\*.*?\\*/"; + @Autowired ConsoleWriter consoleWriter; @Autowired CommandHelper commandHelper; @@ -77,6 +79,12 @@ public class SimpleFileEditorCommand extends Command { description = "To remove location from both Mac strings and string dict files") boolean removeUsagesInMacStrings = false; + @Parameter( + names = {"--macstrings-remove-comments"}, + required = false, + description = "To remove comments from both Mac strings") + boolean removeCommentsInMacStrings = false; + @Parameter( names = {"--json-indent"}, required = false, @@ -107,6 +115,10 @@ public void execute() throws CommandException { removeUsagesInMacStringsAndStringDict(); } + if (removeCommentsInMacStrings) { + removeCommentsInMacStrings(); + } + if (indentJson) { indentJsonFiles(); } @@ -184,6 +196,43 @@ void removeUsagesInMacStringsAndStringDict() throws CommandException { }); } + void removeCommentsInMacStrings() throws CommandException { + MacStringsFileType fileType = new MacStringsFileType(); + + commandDirectories + .listFilesWithExtensionInSourceDirectory( + fileType.getSourceFileExtension(), fileType.getTargetFileExtension()) + .stream() + .filter( + path -> + fileType + .getSourceFilePattern() + .getPattern() + .matcher(path.toAbsolutePath().toString()) + .matches() + || fileType + .getTargetFilePattern() + .getPattern() + .matcher(path.toAbsolutePath().toString()) + .matches()) + .filter(getInputFilterMatch()) + .forEach( + inputPath -> { + consoleWriter + .a(" - Remove comments: ") + .fg(Ansi.Color.MAGENTA) + .a(inputPath.toString()) + .print(); + String modifiedContent = + commandHelper + .getFileContent(inputPath) + .replaceAll(COMMENTS_PATTERN, "") + .replaceFirst("^\n+", "") + .replaceAll("\n{2,}", "\n\n"); + writeOutputFile(inputPath, modifiedContent); + }); + } + Predicate getInputFilterMatch() { return path -> inputFilterPattern == null diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java index 7fe5b758ac..1d6352ab80 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommand.java @@ -28,7 +28,7 @@ @Parameters( commandNames = {"thirdparty-sync", "tps"}, commandDescription = - "Third-party command to sychronize text units and screenshots with third party TMS") + "Third-party command to synchronize text units and screenshots with third party TMS") public class ThirdPartySyncCommand extends Command { /** logger */ @@ -156,7 +156,7 @@ public void execute() throws CommandException { .fg(CYAN) .a(skipAssetsWithPathPattern) .reset() - .a(" include-text-units-with-pattern") + .a(" include-text-units-with-pattern: ") .fg(CYAN) .a(includeTextUnitsWithPattern) .reset() diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FileTypes.java b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FileTypes.java index 59d53670eb..d3700ae6b1 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FileTypes.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FileTypes.java @@ -24,6 +24,7 @@ public enum FileTypes { JSON(JSONFileType.class), JSON_NOBASENAME(JSONNoBasenameFileType.class), CHROME_EXT_JSON(ChromeExtensionJSONFileType.class), + FORMATJS_JSON_NOBASENAME(FormatJSJSONNoBasenameFileType.class), I18NEXT_PARSER_JSON(I18NextFileType.class), TS(TSFileType.class), YAML(YamlFileType.class), diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FormatJSJSONNoBasenameFileType.java b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FormatJSJSONNoBasenameFileType.java new file mode 100644 index 0000000000..dd3ad2bee1 --- /dev/null +++ b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FormatJSJSONNoBasenameFileType.java @@ -0,0 +1,19 @@ +package com.box.l10n.mojito.cli.filefinder.file; + +import java.util.Arrays; + +/** + * @author jeanaurambault + */ +public class FormatJSJSONNoBasenameFileType extends LocaleAsFileNameType { + + public FormatJSJSONNoBasenameFileType() { + this.sourceFileExtension = "json"; + this.defaultFilterOptions = + Arrays.asList( + "noteKeyPattern=description", + "extractAllPairs=false", + "exceptions=defaultMessage", + "removeKeySuffix=/defaultMessage"); + } +} diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleType.java b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleType.java index b36e146bfe..2d741a30ab 100644 --- a/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleType.java +++ b/cli/src/main/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleType.java @@ -19,6 +19,10 @@ public String getTargetLocaleRepresentation(String targetLocale) { String androidLocale = forLanguageTag.getLanguage(); + if (targetLocale.startsWith("in")) { + androidLocale = "in"; + } + if (!Strings.isNullOrEmpty(forLanguageTag.getCountry())) { androidLocale += "-r" + forLanguageTag.getCountry(); } diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java index ed34a12b57..4e455abbf6 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest.java @@ -4,10 +4,12 @@ import com.box.l10n.mojito.entity.Locale; import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.service.locale.LocaleService; +import org.junit.Assume; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; /** * @author jeanaurambault @@ -19,6 +21,10 @@ public class ImportLocalizedAssetCommandTest extends CLITestBase { @Autowired LocaleService localeService; + // TODO(ja) move in its own class + @Value("${test.phrase-client.projectId:}") + String testProjectId; + @Test public void importAndroidStrings() throws Exception { @@ -92,6 +98,151 @@ public void importAndroidStringsPlural() throws Exception { checkExpectedGeneratedResources(); } + @Test + public void importAndroidStringsPostProcessing() throws Exception { + + Repository repository = createTestRepoUsingRepoService(); + repositoryService.addRepositoryLocale(repository, "ru-RU"); + + getL10nJCommander() + .run( + "push", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath()); + + getL10nJCommander() + .run( + "import", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getInputResourcesTestDir("translations").getAbsolutePath()); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("removeDescription").getAbsolutePath(), + "-fo", + "removeDescription=true"); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("removeUntranslated").getAbsolutePath(), + "--inheritance-mode", + "REMOVE_UNTRANSLATED"); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("removeUntranslatedAndDescription").getAbsolutePath(), + "--inheritance-mode", + "REMOVE_UNTRANSLATED", + "-fo", + "removeDescription=true"); + + checkExpectedGeneratedResources(); + } + + @Test + public void importAndroidStringsPluralWithThirdPartySync() throws Exception { + Assume.assumeNotNull(testProjectId); + + Repository repository = createTestRepoUsingRepoService(); + repositoryService.addRepositoryLocale(repository, "ru-RU"); + + getL10nJCommander() + .run( + "push", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath()); + + getL10nJCommander() + .run( + "push", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source2").getAbsolutePath(), + "-b", + "source2"); + + getL10nJCommander() + .run( + "import", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getInputResourcesTestDir("translations").getAbsolutePath()); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("before-sync").getAbsolutePath()); + + getL10nJCommander() + .run( + "thirdparty-sync", + "-r", + repository.getName(), + "-p", + testProjectId, + "-a", + "PUSH,MAP_TEXTUNIT,PUSH_TRANSLATION,PULL"); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir("after-sync").getAbsolutePath()); + + getL10nJCommander() + .run( + "thirdparty-sync", + "-r", + repository.getName(), + "-p", + testProjectId, + "-a", + "MAP_TEXTUNIT", + "-o", + "deleteCurrentMapping=true"); + + checkExpectedGeneratedResources(); + } + @Test public void importMacStrings() throws Exception { @@ -637,11 +788,7 @@ public void importJsonDefaultFormatJs() throws Exception { "-s", getInputResourcesTestDir("source").getAbsolutePath(), "-ft", - "JSON_NOBASENAME", - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage"); + "FORMATJS_JSON_NOBASENAME"); getL10nJCommander() .run( @@ -653,11 +800,49 @@ public void importJsonDefaultFormatJs() throws Exception { "-t", getInputResourcesTestDir("translations").getAbsolutePath(), "-ft", - "JSON_NOBASENAME", - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage"); + "FORMATJS_JSON_NOBASENAME"); + + getL10nJCommander() + .run( + "pull", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getTargetTestDir().getAbsolutePath(), + "-ft", + "FORMATJS_JSON_NOBASENAME"); + + checkExpectedGeneratedResources(); + } + + @Test + public void importJsonDefaultFormatJsCompiled() throws Exception { + + Repository repository = createTestRepoUsingRepoService(); + + getL10nJCommander() + .run( + "push", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-ft", + "FORMATJS_JSON_NOBASENAME"); + + getL10nJCommander() + .run( + "import", + "-r", + repository.getName(), + "-s", + getInputResourcesTestDir("source").getAbsolutePath(), + "-t", + getInputResourcesTestDir("translations").getAbsolutePath(), + "-ft", + "JSON_NOBASENAME"); getL10nJCommander() .run( @@ -673,7 +858,8 @@ public void importJsonDefaultFormatJs() throws Exception { "-fo", "noteKeyPattern=description", "extractAllPairs=false", - "exceptions=defaultMessage"); + "exceptions=defaultMessage", + "removeKeySuffix=/defaultMessage"); checkExpectedGeneratedResources(); } @@ -988,7 +1174,9 @@ public void importDifferentSourceLocale() throws Exception { "-t", getTargetTestDir("withMapping").getAbsolutePath(), "-lm", - "en-US:en-US,en:en,fr-FR:fr-FR"); + "en-US:en-US,en:en,fr-FR:fr-FR", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java index f66d48be26..a9a5de0d09 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java @@ -516,7 +516,9 @@ public void localeMapping() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-FR:fr-FR,ja:ja-JP"); + "fr:fr-FR,fr-FR:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -550,6 +552,8 @@ public void localeMappingParallel() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,fr-FR:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--parallel"); checkExpectedGeneratedResources(); @@ -773,7 +777,9 @@ public void pullResw() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP"); + "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -785,7 +791,9 @@ public void pullResw() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP"); + "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -1004,6 +1012,8 @@ public void pullXcodeXliff() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-ft", "XCODE_XLIFF"); @@ -1018,6 +1028,8 @@ public void pullXcodeXliff() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-ft", "XCODE_XLIFF"); @@ -1052,7 +1064,9 @@ public void pullPo() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP"); + "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -1064,7 +1078,9 @@ public void pullPo() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP"); + "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -1101,6 +1117,8 @@ public void pullHtml() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-ft", "HTML_ALPHA", "-fo", @@ -1117,6 +1135,8 @@ public void pullHtml() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-ft", "HTML_ALPHA", "-fo", @@ -1154,6 +1174,8 @@ public void removeUntranslated() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--inheritance-mode", "REMOVE_UNTRANSLATED"); @@ -1168,6 +1190,8 @@ public void removeUntranslated() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--inheritance-mode", "REMOVE_UNTRANSLATED"); @@ -1205,6 +1229,8 @@ public void onlyApproved() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--inheritance-mode", "REMOVE_UNTRANSLATED", "--status", @@ -1224,6 +1250,8 @@ public void onlyApproved() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP", + "-lmt", + "MAP_ONLY", "--inheritance-mode", "REMOVE_UNTRANSLATED", "--status", @@ -1397,7 +1425,9 @@ public void pullJS() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -1409,7 +1439,9 @@ public void pullJS() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -1441,7 +1473,9 @@ public void pullTS() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -1453,7 +1487,9 @@ public void pullTS() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -1620,12 +1656,8 @@ public void pullJsonDefaultFormatJs() throws Exception { repository.getName(), "-s", getInputResourcesTestDir("source").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME"); + "FORMATJS_JSON_NOBASENAME"); Asset asset = assetClient.getAssetByPathAndRepositoryId("en.json", repository.getId()); importTranslations(asset.getId(), "source-xliff_", "fr-FR"); @@ -1640,12 +1672,8 @@ public void pullJsonDefaultFormatJs() throws Exception { getInputResourcesTestDir("source").getAbsolutePath(), "-t", getTargetTestDir("target").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME"); + "FORMATJS_JSON_NOBASENAME"); getL10nJCommander() .run( @@ -1656,12 +1684,8 @@ public void pullJsonDefaultFormatJs() throws Exception { getInputResourcesTestDir("source_modified").getAbsolutePath(), "-t", getTargetTestDir("target_modified").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME"); + "FORMATJS_JSON_NOBASENAME"); checkExpectedGeneratedResources(); } @@ -1677,12 +1701,8 @@ public void pullJsonDefaultFormatJsRemoveUntranslated() throws Exception { repository.getName(), "-s", getInputResourcesTestDir("source").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME"); + "FORMATJS_JSON_NOBASENAME"); Asset asset = assetClient.getAssetByPathAndRepositoryId("en.json", repository.getId()); importTranslations(asset.getId(), "source-xliff_", "fr-FR"); @@ -1697,12 +1717,8 @@ public void pullJsonDefaultFormatJsRemoveUntranslated() throws Exception { getInputResourcesTestDir("source").getAbsolutePath(), "-t", getTargetTestDir("target").getAbsolutePath(), - "-fo", - "noteKeyPattern=description", - "extractAllPairs=false", - "exceptions=defaultMessage", "-ft", - "JSON_NOBASENAME", + "FORMATJS_JSON_NOBASENAME", "--inheritance-mode", "REMOVE_UNTRANSLATED"); @@ -1983,7 +1999,9 @@ public void recordPullPoPlural() throws Exception { getTargetTestDir("target_baseline").getAbsolutePath(), "-lm", "ru-RU:ru-RU", - "--record-pull-run"); + "--record-pull-run", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -1995,7 +2013,9 @@ public void recordPullPoPlural() throws Exception { "-t", getInputResourcesTestDir("translations").getAbsolutePath(), "-lm", - "ru-RU:ru-RU"); + "ru-RU:ru-RU", + "-lmt", + "MAP_ONLY"); logger.debug("Record a second pull run after translation import"); getL10nJCommander() @@ -2009,7 +2029,9 @@ public void recordPullPoPlural() throws Exception { getTargetTestDir("target_translated").getAbsolutePath(), "-lm", "ru-RU:ru-RU", - "--record-pull-run"); + "--record-pull-run", + "-lmt", + "MAP_ONLY"); logger.debug("Simulate commit and linked to pull-run"); String pullRunHash1 = "ddaa11"; @@ -2481,7 +2503,9 @@ public void pullYaml() throws Exception { "-t", getTargetTestDir("target").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); getL10nJCommander() .run( @@ -2493,7 +2517,9 @@ public void pullYaml() throws Exception { "-t", getTargetTestDir("target_modified").getAbsolutePath(), "-lm", - "fr:fr-FR,ja:ja-JP"); + "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY"); checkExpectedGeneratedResources(); } @@ -2529,6 +2555,8 @@ public void pullYamlWithFilterOptions() throws Exception { getTargetTestDir("target").getAbsolutePath(), "-lm", "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-fo", "extractAllPairs=false", "exceptions=1_day_duration|1_year_duration"); @@ -2544,6 +2572,8 @@ public void pullYamlWithFilterOptions() throws Exception { getTargetTestDir("target_modified").getAbsolutePath(), "-lm", "fr:fr-FR,ja:ja-JP", + "-lmt", + "MAP_ONLY", "-fo", "extractAllPairs=false", "exceptions=1_day_duration|1_year_duration"); diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest.java index e3f1f4facb..da6531ab4a 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest.java @@ -47,4 +47,19 @@ public void macStringsRemoveUsages() { checkExpectedGeneratedResources(); } + + @Test + public void macStringsRemoveComments() { + + getL10nJCommander() + .run( + "simple-file-editor", + "-i", + getInputResourcesTestDir().getAbsolutePath(), + "-o", + getTargetTestDir().getAbsolutePath(), + "--macstrings-remove-comments"); + + checkExpectedGeneratedResources(); + } } diff --git a/cli/src/test/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleTypeTest.java b/cli/src/test/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleTypeTest.java index b24319d214..01b7de4d93 100644 --- a/cli/src/test/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleTypeTest.java +++ b/cli/src/test/java/com/box/l10n/mojito/cli/filefinder/locale/AndroidLocaleTypeTest.java @@ -26,4 +26,31 @@ public void testGetTargetLocaleRepresentationWithRegion() { String result = instance.getTargetLocaleRepresentation(targetLocale); assertEquals(expResult, result); } + + @Test + public void testGetTargetLocaleIndonesianOld() { + String targetLocale = "in"; + AndroidLocaleType instance = new AndroidLocaleType(); + String expResult = "in"; + String result = instance.getTargetLocaleRepresentation(targetLocale); + assertEquals(expResult, result); + } + + @Test + public void testGetTargetLocaleIndonesianIndonesiaOld() { + String targetLocale = "in-ID"; + AndroidLocaleType instance = new AndroidLocaleType(); + String expResult = "in-rID"; + String result = instance.getTargetLocaleRepresentation(targetLocale); + assertEquals(expResult, result); + } + + @Test + public void testGetTargetLocaleIndonesianNew() { + String targetLocale = "id"; + AndroidLocaleType instance = new AndroidLocaleType(); + String expResult = "id"; + String result = instance.getTargetLocaleRepresentation(targetLocale); + assertEquals(expResult, result); + } } diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..bf62334b5b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rCA/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..bf62334b5b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..9091b03269 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ja-rJP/strings.xml @@ -0,0 +1,22 @@ + + + + + 所要時間：%1$,d 時間 + + + + ユーザー + + + ボード参加者：%1$,d 人 + + + + + %1$d 個の材料 + + + + 削除 + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..0df87a4dfc --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/after-sync/res/values-ru-rRU/strings.xml @@ -0,0 +1,31 @@ + + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + + + Люди + + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + + Удалить + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..bf62334b5b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rCA/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..bf62334b5b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..9091b03269 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ja-rJP/strings.xml @@ -0,0 +1,22 @@ + + + + + 所要時間：%1$,d 時間 + + + + ユーザー + + + ボード参加者：%1$,d 人 + + + + + %1$d 個の材料 + + + + 削除 + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..0df87a4dfc --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/expected/before-sync/res/values-ru-rRU/strings.xml @@ -0,0 +1,31 @@ + + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + + + Люди + + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + + Удалить + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source/res/values/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source/res/values/strings.xml new file mode 100644 index 0000000000..df1703cc80 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source/res/values/strings.xml @@ -0,0 +1,25 @@ + + + + + %,d hr to cook + %,d hrs to cook + + + + People + + + %1$,d collaborator + %1$,d collaborators + + + + + %d ingredient + %d ingredients + + + + Delete + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source2/res/values/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source2/res/values/strings.xml new file mode 100644 index 0000000000..8be79e5824 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/source2/res/values/strings.xml @@ -0,0 +1,7 @@ + + + + %1$,d collaborator in branch + %1$,d collaborators in branch + + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..0f85d2dd1e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rCA/strings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..0623031bdd --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..22a1c39bf1 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ja-rJP/strings.xml @@ -0,0 +1,22 @@ + + + + + 所要時間：%1$,d 時間 + + + + ユーザー + + + ボード参加者：%1$,d 人 + + + + + %1$d 個の材料 + + + + 削除 + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..0df87a4dfc --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPluralWithThirdPartySync/input/translations/res/values-ru-rRU/strings.xml @@ -0,0 +1,31 @@ + + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + + + Люди + + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + + Удалить + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..06d4ae0878 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml @@ -0,0 +1,30 @@ + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + Membres + + %1$,d collaborateur + %1$,d collaborateurs + + + + %1$d ingrédient + %1$d ingrédients + + + Supprimer + Untranslated + + plural_untranslated_one + plural_untranslated_other + + do_not_translate + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..06d4ae0878 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml @@ -0,0 +1,30 @@ + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + Membres + + %1$,d collaborateur + %1$,d collaborateurs + + + + %1$d ingrédient + %1$d ingrédients + + + Supprimer + Untranslated + + plural_untranslated_one + plural_untranslated_other + + do_not_translate + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..fdec8b186e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml @@ -0,0 +1,25 @@ + + + + 所要時間：%1$,d 時間 + + ユーザー + + ボード参加者：%1$,d 人 + + + + %1$d 個の材料 + + + 削除 + Untranslated + + plural_untranslated_other + + do_not_translate + + + plural_do_not_translate_other + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..db73c90d99 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml @@ -0,0 +1,40 @@ + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + Люди + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + Удалить + Untranslated + + plural_untranslated_one + plural_untranslated_other + plural_untranslated_other + plural_untranslated_other + + do_not_translate + + + plural_do_not_translate_one + plural_do_not_translate_other + plural_do_not_translate_other + plural_do_not_translate_other + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..57d77bf1f7 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml @@ -0,0 +1,7 @@ + + + + + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..dd671274a9 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + Membres + + %1$,d collaborateur + %1$,d collaborateurs + + + + %1$d ingrédient + %1$d ingrédients + + + Supprimer + do_not_translate + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..b6dc7ce899 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml @@ -0,0 +1,18 @@ + + + + 所要時間：%1$,d 時間 + + ユーザー + + ボード参加者：%1$,d 人 + + + + %1$d 個の材料 + + + 削除 + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..4c831f9553 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml @@ -0,0 +1,27 @@ + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + Люди + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + Удалить + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..57d77bf1f7 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml @@ -0,0 +1,7 @@ + + + + + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..aaddcd25e0 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml @@ -0,0 +1,25 @@ + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + Membres + + %1$,d collaborateur + %1$,d collaborateurs + + + + %1$d ingrédient + %1$d ingrédients + + + Supprimer + do_not_translate + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..e95e22548c --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml @@ -0,0 +1,18 @@ + + + + 所要時間：%1$,d 時間 + + ユーザー + + ボード参加者：%1$,d 人 + + + + %1$d 個の材料 + + + 削除 + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..a08ef21339 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml @@ -0,0 +1,27 @@ + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + Люди + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + Удалить + do_not_translate + + diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/source/res/values/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/source/res/values/strings.xml new file mode 100644 index 0000000000..75700cc771 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/source/res/values/strings.xml @@ -0,0 +1,38 @@ + + + + + %,d hr to cook + %,d hrs to cook + + + People + + + %1$,d collaborator + %1$,d collaborators + + + + + %d ingredient + %d ingredients + + + + Delete + + Untranslated + + plural_untranslated_one + plural_untranslated_other + + + do_not_translate + + + + plural_do_not_translate_one + plural_do_not_translate_other + + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rCA/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rCA/strings.xml new file mode 100644 index 0000000000..0f85d2dd1e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rCA/strings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rFR/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rFR/strings.xml new file mode 100644 index 0000000000..92f1912ff1 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-fr-rFR/strings.xml @@ -0,0 +1,31 @@ + + + + + %1$,d heure à cuisiner + %1$,d heures à cuisiner + + + Membres + + + %1$,d collaborateur + %1$,d collaborateurs + + + + + %1$d ingrédient + %1$d ingrédients + + + + Supprimer + + do_not_translate_fr + + + plural_do_not_translate_one_fr + plural_do_not_translate_other_fr + + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ja-rJP/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ja-rJP/strings.xml new file mode 100644 index 0000000000..b4d3cad434 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ja-rJP/strings.xml @@ -0,0 +1,20 @@ + + + + 所要時間：%1$,d 時間 + + + ユーザー + + + ボード参加者：%1$,d 人 + + + + + %1$d 個の材料 + + + + 削除 + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ru-rRU/strings.xml b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ru-rRU/strings.xml new file mode 100644 index 0000000000..7cb534e72b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importAndroidStringsPostProcessing/input/translations/res/values-ru-rRU/strings.xml @@ -0,0 +1,30 @@ + + + + + Приготовление: %1$,d час + Приготовление: %1$,d часа + Приготовление: %1$,d часов + Приготовление: %1$,d часа + + + Люди + + + %1$,d соавтор + %1$,d соавтора + %1$,d соавторов + %1$,d соавтора + + + + + %1$d ингредиент + %1$d ингредиента + %1$d ингредиентов + %1$d ингредиента + + + + Удалить + \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-CA.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-CA.json new file mode 100644 index 0000000000..bc67f94b3e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-CA.json @@ -0,0 +1,21 @@ +{ + "100_character_description_": { + "defaultMessage": "Description de 100 caractères :" + }, + "15_min_duration": { + "defaultMessage": "15 min", + "description": "File lock dialog duration" + }, + "1_day_duration": { + "defaultMessage": "1 jour", + "description": "File lock dialog duration" + }, + "1_hour_duration": { + "defaultMessage": "1 heure", + "description": "File lock dialog duration" + }, + "1_month_duration": { + "defaultMessage": "1 mois", + "description": "File lock dialog duration" + } +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-FR.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-FR.json new file mode 100644 index 0000000000..bc67f94b3e --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/fr-FR.json @@ -0,0 +1,21 @@ +{ + "100_character_description_": { + "defaultMessage": "Description de 100 caractères :" + }, + "15_min_duration": { + "defaultMessage": "15 min", + "description": "File lock dialog duration" + }, + "1_day_duration": { + "defaultMessage": "1 jour", + "description": "File lock dialog duration" + }, + "1_hour_duration": { + "defaultMessage": "1 heure", + "description": "File lock dialog duration" + }, + "1_month_duration": { + "defaultMessage": "1 mois", + "description": "File lock dialog duration" + } +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/ja-JP.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/ja-JP.json new file mode 100644 index 0000000000..f2a6578f59 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/expected/ja-JP.json @@ -0,0 +1,21 @@ +{ + "100_character_description_": { + "defaultMessage": "100文字の説明:" + }, + "15_min_duration": { + "defaultMessage": "15分", + "description": "File lock dialog duration" + }, + "1_day_duration": { + "defaultMessage": "1日", + "description": "File lock dialog duration" + }, + "1_hour_duration": { + "defaultMessage": "1時間", + "description": "File lock dialog duration" + }, + "1_month_duration": { + "defaultMessage": "1か月", + "description": "File lock dialog duration" + } +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/source/en.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/source/en.json new file mode 100644 index 0000000000..e5e0f7149b --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/source/en.json @@ -0,0 +1,21 @@ +{ + "100_character_description_": { + "defaultMessage": "100 character description:" + }, + "15_min_duration": { + "defaultMessage": "15 min", + "description": "File lock dialog duration" + }, + "1_day_duration": { + "defaultMessage": "1 day", + "description": "File lock dialog duration" + }, + "1_hour_duration": { + "defaultMessage": "1 hour", + "description": "File lock dialog duration" + }, + "1_month_duration": { + "defaultMessage": "1 month", + "description": "File lock dialog duration" + } +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-CA.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-CA.json new file mode 100644 index 0000000000..38918f96d2 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-CA.json @@ -0,0 +1,7 @@ +{ + "100_character_description_": "Description de 100 caractères :", + "15_min_duration": "15 min", + "1_day_duration": "1 jour", + "1_hour_duration": "1 heure", + "1_month_duration": "1 mois" +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-FR.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-FR.json new file mode 100644 index 0000000000..38918f96d2 --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/fr-FR.json @@ -0,0 +1,7 @@ +{ + "100_character_description_": "Description de 100 caractères :", + "15_min_duration": "15 min", + "1_day_duration": "1 jour", + "1_hour_duration": "1 heure", + "1_month_duration": "1 mois" +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/ja-JP.json b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/ja-JP.json new file mode 100644 index 0000000000..afa9c8925d --- /dev/null +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommandTest_IO/importJsonDefaultFormatJsCompiled/input/translations/ja-JP.json @@ -0,0 +1,7 @@ +{ + "100_character_description_": "100文字の説明:", + "15_min_duration": "15分", + "1_day_duration": "1日", + "1_hour_duration": "1時間", + "1_month_duration": "1か月" +} \ No newline at end of file diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_fr-FR.xliff b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_fr-FR.xliff index 270f4f848e..7364cce233 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_fr-FR.xliff +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_fr-FR.xliff @@ -2,26 +2,26 @@ - + 100 character description: Description de 100 caractères : - + 15 min 15 min File lock dialog duration - + 1 day 1 jour File lock dialog duration - + 1 hour 1 heure File lock dialog duration - + 1 month 1 mois File lock dialog duration diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_ja-JP.xliff b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_ja-JP.xliff index 7a27bc0549..0f9c2f5b5c 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_ja-JP.xliff +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJs/input/translations/source-xliff_ja-JP.xliff @@ -2,26 +2,26 @@ - + 100 character description: 100文字の説明: - + 15 min 15分 File lock dialog duration - + 1 day 1日 File lock dialog duration - + 1 hour 1時間 File lock dialog duration - + 1 month 1か月 File lock dialog duration diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_fr-FR.xliff b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_fr-FR.xliff index 2420b4e634..89602f4de7 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_fr-FR.xliff +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_fr-FR.xliff @@ -2,21 +2,21 @@ - + 100 character description: Description de 100 caractères : - + 15 min 15 min File lock dialog duration - + 1 hour 1 heure File lock dialog duration - + 1 month 1 mois File lock dialog duration diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_ja-JP.xliff b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_ja-JP.xliff index cf041d39be..f383f05e1d 100644 --- a/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_ja-JP.xliff +++ b/cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullJsonDefaultFormatJsRemoveUntranslated/input/translations/source-xliff_ja-JP.xliff @@ -2,16 +2,16 @@ - + 100 character description: 100文字の説明: - + 1 hour 1時間 File lock dialog duration - + 1 month 1か月 File lock dialog duration diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/expected/en.lproj/Localizable.strings b/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/expected/en.lproj/Localizable.strings new file mode 100644 index 0000000000..b096e6b6d9 Binary files /dev/null and b/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/expected/en.lproj/Localizable.strings differ diff --git a/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/input/en.lproj/Localizable.strings b/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/input/en.lproj/Localizable.strings new file mode 100644 index 0000000000..5fca63d2cf Binary files /dev/null and b/cli/src/test/resources/com/box/l10n/mojito/cli/command/SimpleFileEditorCommandTest_IO/macStringsRemoveComments/input/en.lproj/Localizable.strings differ diff --git a/common/pom.xml b/common/pom.xml index 94e7c73393..38ac5f7840 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -60,6 +60,11 @@ jackson-module-blackbird + + com.fasterxml.jackson.module + jackson-module-jsonSchema + + org.springframework.boot spring-boot-starter diff --git a/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java b/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java index 8601ea642c..3b78325577 100644 --- a/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java +++ b/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java @@ -9,10 +9,10 @@ public class LocaleMappingHelper { /** - * Gets the locale mapping given the locale mapping param + * Gets the locale mapping given the locale mapping param. * * @param localeMapppingParam locale mapping param coming from the CLI - * @return A map containing the locale mapping + * @return A map containing the locale mapping (key: output tag, value: the tag in the repository) */ public Map getLocaleMapping(String localeMapppingParam) { @@ -28,7 +28,8 @@ public Map getLocaleMapping(String localeMapppingParam) { * Gets the inverse locale mapping given the locale mapping param * * @param localeMapppingParam locale mapping param coming from the CLI - * @return A map containing the inverse locale mapping + * @return A map containing the inverse locale mapping (key: the tag in the repository, value: + * file output tag) */ public Map getInverseLocaleMapping(String localeMapppingParam) { diff --git a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java index de087a6e47..a9de13a22c 100644 --- a/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java +++ b/common/src/main/java/com/box/l10n/mojito/github/GithubClient.java @@ -1,9 +1,16 @@ package com.box.l10n.mojito.github; +import com.box.l10n.mojito.json.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.ImmutableMap; import io.jsonwebtoken.JwtBuilder; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; @@ -11,10 +18,14 @@ import java.security.spec.PKCS8EncodedKeySpec; import java.util.Date; import java.util.List; +import java.util.Map; import org.apache.commons.codec.binary.Base64; import org.kohsuke.github.GHAppInstallationToken; import org.kohsuke.github.GHCommitState; import org.kohsuke.github.GHIssueComment; +import org.kohsuke.github.GHIssueState; +import org.kohsuke.github.GHPullRequest; +import org.kohsuke.github.GHUser; import org.kohsuke.github.GitHub; import org.kohsuke.github.GitHubBuilder; import org.slf4j.Logger; @@ -69,12 +80,78 @@ public GithubClient(String appId, String key, String owner) { private PrivateKey createPrivateKey(String key) throws NoSuchAlgorithmException, InvalidKeySpecException { - byte[] encodedKey = Base64.decodeBase64(key); + byte[] encodedKey = keyToBytes(key); PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(encodedKey); KeyFactory keyFactory = KeyFactory.getInstance("RSA"); return keyFactory.generatePrivate(spec); } + /** + * Converts a PEM-formatted private key string to a byte array. + * + *

This method handles both PKCS#1 and PKCS#8 formatted private keys. It strips the PEM headers + * and footers, decodes the base64 content, and, if necessary, converts a PKCS#1 key to a PKCS#8 + * format. + */ + private static byte[] keyToBytes(String formattedKey) { + boolean pkcs1Header = formattedKey.startsWith("-----BEGIN RSA PRIVATE KEY-----"); + boolean pkcs8Header = formattedKey.startsWith("-----BEGIN PRIVATE KEY-----"); + + if (pkcs1Header || pkcs8Header) { + formattedKey = + formattedKey + .replaceAll("-----BEGIN.*?-----", "") + .replaceAll("-----END.*?-----", "") + .replaceAll("\\s+", ""); + } + + byte[] encodedKey = Base64.decodeBase64(formattedKey); + + if (pkcs1Header) { + encodedKey = convertPkcs1ToPkcs8(encodedKey); + } + + return encodedKey; + } + + private static byte[] convertPkcs1ToPkcs8(byte[] pkcs1Bytes) { + // PKCS#8 header for RSA encryption + byte[] pkcs8Header = { + 0x30, + (byte) 0x82, // SEQUENCE + LENGTH + (byte) ((pkcs1Bytes.length + 22) >> 8), + (byte) ((pkcs1Bytes.length + 22) & 0xff), // PKCS#8 LENGTH + 0x02, + 0x01, + 0x00, // INTEGER (0) + 0x30, + 0x0d, // SEQUENCE + 0x06, + 0x09, // OID + 0x2a, + (byte) 0x86, + 0x48, + (byte) 0x86, + (byte) 0xf7, + 0x0d, + 0x01, + 0x01, + 0x01, // rsaEncryption OID + 0x05, + 0x00, // NULL + 0x04, + (byte) 0x82, // OCTET STRING + LENGTH + (byte) (pkcs1Bytes.length >> 8), + (byte) (pkcs1Bytes.length & 0xff) // PKCS#1 LENGTH + }; + + byte[] pkcs8Bytes = new byte[pkcs8Header.length + pkcs1Bytes.length]; + System.arraycopy(pkcs8Header, 0, pkcs8Bytes, 0, pkcs8Header.length); + System.arraycopy(pkcs1Bytes, 0, pkcs8Bytes, pkcs8Header.length, pkcs1Bytes.length); + + return pkcs8Bytes; + } + public void addCommentToPR(String repository, int prNumber, String comment) { String repoFullPath = getRepositoryPath(repository); try { @@ -238,6 +315,137 @@ public List getPRComments(String repository, int prNumber) { } } + public List listPR(String repository, GHIssueState state) { + try { + String repoFullPath = getRepositoryPath(repository); + return getGithubClient(repository) + .getRepository(repoFullPath) + .getPullRequests(GHIssueState.OPEN); + } catch (IOException | NoSuchAlgorithmException | InvalidKeySpecException e) { + throw new RuntimeException(e); + } + } + + public GHPullRequest createPR( + String repository, + String title, + String head, + String base, + String body, + List reviewers) { + String repoFullPath = getRepositoryPath(repository); + try { + GHPullRequest pullRequest = + getGithubClient(repository) + .getRepository(repoFullPath) + .createPullRequest(title, head, base, body); + + if (reviewers != null) { + List reviewersGH = + reviewers.stream() + .map( + s -> { + try { + return gitHubClient.getUser(s); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .toList(); + + pullRequest.requestReviewers(reviewersGH); + } + return pullRequest; + } catch (Exception e) { + String message = + String.format("Error creating a PR in repository '%s': %s", repoFullPath, e.getMessage()); + logger.error(message, e); + throw new GithubException(message, e); + } + } + + public enum AutoMergeType { + SQUASH, + MERGE + } + + /** + * This is implemented using the GraphQL API since, the Java client does not support this method + * yet. + */ + public void enableAutoMerge(GHPullRequest pullRequest, AutoMergeType autoMergeType) { + + logger.debug( + "Enable Auto Merge on PR: {}, with type: {}", + pullRequest.getNumber(), + autoMergeType.name()); + + ObjectMapper objectMapper = new ObjectMapper(); + + HttpResponse mutationResponse; + + try (HttpClient client = HttpClient.newHttpClient()) { + + String mutation = + """ + mutation($pullRequestId: ID!, $mergeMethod: PullRequestMergeMethod!) { + enablePullRequestAutoMerge(input: { + pullRequestId: $pullRequestId, + mergeMethod: $mergeMethod, + }) { + clientMutationId + } + } + """; + + ImmutableMap variables = + ImmutableMap.of( + "pullRequestId", pullRequest.getNodeId(), "mergeMethod", autoMergeType.name()); + + ImmutableMap payload = + ImmutableMap.of("query", mutation, "variables", variables); + + String jsonPayload = objectMapper.writeValueAsStringUnchecked(payload); + + String authToken = + getGithubAppInstallationToken(pullRequest.getRepository().getName()).getToken(); + + HttpRequest mutationRequest = + HttpRequest.newBuilder() + .uri(URI.create("https://api.github.com/graphql")) + .header("Authorization", "Bearer " + authToken) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .build(); + try { + mutationResponse = client.send(mutationRequest, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + if (mutationResponse.statusCode() != 200) { + throw new GithubException("Error enabling auto-merge: " + mutationResponse.body()); + } + + Map mutationResponseMap = + objectMapper.readValueUnchecked(mutationResponse.body(), new TypeReference<>() {}); + + if (mutationResponseMap.containsKey("errors")) { + throw new GithubException("Error enabling auto-merge: " + mutationResponse.body()); + } + } + } + + public void addLabelsToPR(GHPullRequest pullRequest, List labels) { + if (labels != null) { + try { + pullRequest.addLabels(labels.toArray(new String[0])); + } catch (IOException e) { + throw new GithubException("Can't add labels to PR", e); + } + } + } + public String getOwner() { return owner; } @@ -250,22 +458,25 @@ public String getEndpoint() { return endpoint; } - public GHAppInstallationToken getGithubAppInstallationToken(String repository) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { - if (githubAppInstallationToken == null - || githubAppInstallationToken.getExpiresAt().getTime() - <= System.currentTimeMillis() - 30000) { - // Existing installation token has less than 30 seconds before expiry, get new token - GitHub gitHub = - new GitHubBuilder() - .withEndpoint(getEndpoint()) - .withJwtToken(getGithubJWT(tokenTTL).getToken()) - .build(); - githubAppInstallationToken = - gitHub.getApp().getInstallationByRepository(owner, repository).createToken().create(); + public GHAppInstallationToken getGithubAppInstallationToken(String repository) { + try { + if (githubAppInstallationToken == null + || githubAppInstallationToken.getExpiresAt().getTime() + <= System.currentTimeMillis() - 30000) { + // Existing installation token has less than 30 seconds before expiry, get new token + GitHub gitHub = + new GitHubBuilder() + .withEndpoint(getEndpoint()) + .withJwtToken(getGithubJWT(tokenTTL).getToken()) + .build(); + githubAppInstallationToken = + gitHub.getApp().getInstallationByRepository(owner, repository).createToken().create(); + } + + return githubAppInstallationToken; + } catch (Exception e) { + throw new RuntimeException("Can't get the App installation token", e); } - - return githubAppInstallationToken; } protected GitHub createGithubClient(String repository) @@ -276,7 +487,7 @@ protected GitHub createGithubClient(String repository) .build(); } - private String getRepositoryPath(String repository) { + String getRepositoryPath(String repository) { return owner != null && !owner.isEmpty() ? owner + "/" + repository : repository; } diff --git a/common/src/main/java/com/box/l10n/mojito/io/Files.java b/common/src/main/java/com/box/l10n/mojito/io/Files.java index ada5d8ed1e..3f4089398e 100644 --- a/common/src/main/java/com/box/l10n/mojito/io/Files.java +++ b/common/src/main/java/com/box/l10n/mojito/io/Files.java @@ -55,6 +55,14 @@ public static void write(Path path, String content) { } } + public static String readString(Path path) { + try { + return java.nio.file.Files.readString(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public static Stream lines(Path path) { try { return java.nio.file.Files.lines(path, StandardCharsets.UTF_8); @@ -75,4 +83,12 @@ public static void write( throw new UncheckedIOException(ioe); } } + + public static Path createTempDirectory(String prefix, FileAttribute... attrs) { + try { + return java.nio.file.Files.createTempDirectory(prefix, attrs); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java index f60b6ed9ff..06ff9c6d59 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java @@ -1,10 +1,24 @@ package com.box.l10n.mojito.okapi.filters; import com.box.l10n.mojito.okapi.TextUnitUtils; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation.OutputDocumentPostProcessorBase; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import net.sf.okapi.common.Event; import net.sf.okapi.common.IResource; import net.sf.okapi.common.LocaleId; @@ -19,6 +33,13 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; /** * @author jaurambault @@ -35,6 +56,13 @@ public class AndroidFilter extends XMLFilter { private static final String OPTION_OLD_ESCAPING = "oldEscaping"; + private static final String REMOVE_DESCRIPTION = "removeDescription"; + + private static final String POST_PROCESS_INDENT = "postProcessIndent"; + + private static final String POST_PROCESS_REMOVE_TRANSLATABLE_FALSE = + "postRemoveTranslatableFalse"; + private static final Pattern PATTERN_PLURAL_START = Pattern.compile(""); private static final Pattern PATTERN_XML_COMMENT = Pattern.compile(""); @@ -84,12 +112,35 @@ public List getConfigurations() { List eventQueue = new ArrayList<>(); + boolean removeDescription = false; + + boolean removeTranslatableFalse = false; + + int postProcessIndent = 2; + + /** + * Historically there was no processing if no option was passed. We want to keep this behavior to + * avoid output change. + */ + boolean shouldApplyPostProcessingRemoveUntranslatedExcluded = false; + @Override public void open(RawDocument input) { super.open(input); targetLocale = input.getTargetLocale(); hasAnnotation = input.getAnnotation(CopyFormsOnImport.class) != null; applyFilterOptions(input); + input.setAnnotation( + new RemoveUntranslatedStategyAnnotation( + RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); + input.setAnnotation( + new OutputDocumentPostProcessingAnnotation( + new AndroidFilePostProcessor( + false, + removeDescription, + postProcessIndent, + removeTranslatableFalse, + shouldApplyPostProcessingRemoveUntranslatedExcluded))); } void applyFilterOptions(RawDocument input) { @@ -104,9 +155,29 @@ void applyFilterOptions(RawDocument input) { androidXMLEncoder.oldEscaping = oldEscaping; } }); - } + logger.debug("filter option, old escaping: {}", oldEscaping); + + filterOptions.getBoolean( + REMOVE_DESCRIPTION, + b -> { + removeDescription = b; + shouldApplyPostProcessingRemoveUntranslatedExcluded = true; + }); + + filterOptions.getBoolean( + POST_PROCESS_REMOVE_TRANSLATABLE_FALSE, + b -> { + removeTranslatableFalse = b; + shouldApplyPostProcessingRemoveUntranslatedExcluded = true; + }); - logger.debug("filter option, old escaping: {}", oldEscaping); + filterOptions.getInteger( + POST_PROCESS_INDENT, + i -> { + postProcessIndent = i; + shouldApplyPostProcessingRemoveUntranslatedExcluded = true; + }); + } } @Override @@ -369,4 +440,166 @@ void updateFormInSkeleton(ITextUnit textUnit) { } } } + + static class AndroidFilePostProcessor extends OutputDocumentPostProcessorBase { + static final String DESCRIPTION_ATTRIBUTE = "description"; + boolean removeDescription; + boolean removeTranslatableFalse; + int indent; + boolean shouldApplyPostProcessingRemoveUntranslatedExcluded; + + AndroidFilePostProcessor( + boolean removeUntranslated, + boolean removeDescription, + int indent, + boolean removeTranslatableFalse, + boolean shouldApplyPostProcessingRemoveUntranslatedExcluded) { + this.setRemoveUntranslated(removeUntranslated); + this.removeDescription = removeDescription; + this.removeTranslatableFalse = removeTranslatableFalse; + this.indent = indent; + this.shouldApplyPostProcessingRemoveUntranslatedExcluded = + shouldApplyPostProcessingRemoveUntranslatedExcluded; + } + + public String execute(String xmlContent) { + + if (xmlContent == null + || xmlContent.isBlank() + || (!shouldApplyPostProcessingRemoveUntranslatedExcluded && !hasRemoveUntranslated())) { + return xmlContent; + } + + try { + DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); + Document document = documentBuilder.parse(new InputSource(new StringReader(xmlContent))); + document.getDocumentElement().normalize(); + + NodeList stringElements = document.getElementsByTagName("string"); + for (int i = 0; i < stringElements.getLength(); i++) { + Node node = stringElements.item(i); + if (node.getNodeType() == Node.ELEMENT_NODE) { + Element element = (Element) node; + if (hasRemoveUntranslated() + && element + .getTextContent() + .equals(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { + element.getParentNode().removeChild(element); + i--; + } + + if (element.hasAttribute(DESCRIPTION_ATTRIBUTE)) { + if (removeDescription) { + element.removeAttribute(DESCRIPTION_ATTRIBUTE); + } + } + } + } + + NodeList pluralsElements = document.getElementsByTagName("plurals"); + for (int i = 0; i < pluralsElements.getLength(); i++) { + Element plurals = (Element) pluralsElements.item(i); + NodeList items = plurals.getElementsByTagName("item"); + boolean hasTranslated = false; + boolean hasOther = false; + + for (int j = 0; j < items.getLength(); j++) { + Element item = (Element) items.item(j); + + if ("other".equals(item.getAttribute("quantity"))) { + hasOther = + !RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER.equals( + item.getTextContent()); + } + + if (hasRemoveUntranslated() + && item.getTextContent() + .equals(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { + item.getParentNode().removeChild(item); + j--; + } else { + hasTranslated = true; + } + } + + if (!hasOther || !hasTranslated) { + plurals.getParentNode().removeChild(plurals); + i--; + } + + if (plurals.hasAttribute(DESCRIPTION_ATTRIBUTE)) { + if (removeDescription) { + plurals.removeAttribute(DESCRIPTION_ATTRIBUTE); + } + } + } + + if (removeTranslatableFalse) { + removeTranslatableFalseElements(document); + } + removeWhitespaceNodes(document); + + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + Transformer transformer = transformerFactory.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty( + "{http://xml.apache.org/xslt}indent-amount", String.valueOf(indent)); + + boolean hasDeclaration = xmlContent.trim().startsWith("= 0; i--) { + Node childNode = childNodes.item(i); + if (childNode instanceof Text && childNode.getNodeValue().isBlank()) { + node.removeChild(childNode); + } else if (childNode instanceof Element) { + removeWhitespaceNodes(childNode); + } + } + } + + void removeTranslatableFalseElements(Node node) { + NodeList childNodes = node.getChildNodes(); + for (int i = childNodes.getLength() - 1; i >= 0; i--) { + Node childNode = childNodes.item(i); + if (childNode.getNodeType() == Node.ELEMENT_NODE) { + Element element = (Element) childNode; + if (element.hasAttribute("translatable") + && element.getAttribute("translatable").equals("false")) { + node.removeChild(element); + } else { + removeTranslatableFalseElements(element); + } + } + } + } + } } diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java index 5d55a3aabf..8451451281 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/FilterOptions.java @@ -1,36 +1,12 @@ package com.box.l10n.mojito.okapi.filters; -import com.google.common.base.Splitter; -import java.util.LinkedHashMap; +import com.box.l10n.mojito.utils.OptionsParser; import java.util.List; -import java.util.Map; -import java.util.function.Consumer; import net.sf.okapi.common.annotation.IAnnotation; -public class FilterOptions implements IAnnotation { - - Map options = new LinkedHashMap<>(); +public class FilterOptions extends OptionsParser implements IAnnotation { public FilterOptions(List options) { - if (options != null) { - for (String option : options) { - List optionKeyAndValue = Splitter.on("=").limit(2).splitToList(option); - if (optionKeyAndValue.size() == 2) { - this.options.put(optionKeyAndValue.get(0), optionKeyAndValue.get(1)); - } - } - } - } - - public void getString(String key, Consumer consumer) { - if (this.options.containsKey(key)) { - consumer.accept(this.options.get(key)); - } - } - - public void getBoolean(String key, Consumer consumer) { - if (this.options.containsKey(key)) { - consumer.accept(Boolean.valueOf(this.options.get(key))); - } + super(options); } } diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java index c3f33dc32b..0c9bec5d81 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java @@ -2,6 +2,7 @@ import com.box.l10n.mojito.json.JsonObjectRemoverByValue; import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation.OutputDocumentPostProcessorBase; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.HashSet; @@ -48,6 +49,18 @@ public class JSONFilter extends net.sf.okapi.filters.json.JSONFilter { */ boolean noteKeepOrReplace = false; + /** + * Remove the suffix from the key. + * + *

Typically useful for FormatJS: + * + *

"text-unit-name": { "defaultMessage": "example", "description": "example description" } + * + *

text unit name would be text-unit-name/defaultMessage. With removeKeySuffix set, the + * "/defaultMessage" can be removed. + */ + String removeKeySuffix = null; + NoteAnnotation noteAnnotation; UsagesAnnotation usagesAnnotation; String currentKeyName; @@ -79,10 +92,19 @@ public void open(RawDocument input) { input.setAnnotation( new RemoveUntranslatedStategyAnnotation( RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); - // Post processing is disable for now, it will be enabled by the TranslsateStep if there are - // actual text unit to remove input.setAnnotation( - new OutputDocumentPostProcessingAnnotation(JSONFilter::removeUntranslated, false)); + new OutputDocumentPostProcessingAnnotation( + new OutputDocumentPostProcessorBase() { + @Override + public String execute(String content) { + + if (hasRemoveUntranslated()) { + content = removeUntranslated(content); + } + + return content; + } + })); } static String removeUntranslated(String content) { @@ -110,6 +132,7 @@ void applyFilterOptions(RawDocument input) { filterOptions.getString("usagesKeyPattern", s -> usagesKeyPattern = Pattern.compile(s)); filterOptions.getBoolean("noteKeepOrReplace", b -> noteKeepOrReplace = b); filterOptions.getBoolean("usagesKeepOrReplace", b -> usagesKeepOrReplace = b); + filterOptions.getString("removeKeySuffix", s -> removeKeySuffix = s); filterOptions.getBoolean( "convertToHtmlCodes", b -> { @@ -121,6 +144,18 @@ void applyFilterOptions(RawDocument input) { } } + @Override + public Event next() { + Event next = super.next(); + + if (next.isTextUnit()) { + ITextUnit textUnit = next.getTextUnit(); + textUnit.setName(removeKeySuffixIfMatch(textUnit.getName())); + } + + return next; + } + @Override public void handleComment(String c) { comment = c; @@ -170,6 +205,15 @@ void addNote(String value) { noteAnnotation.add(note); } + String removeKeySuffixIfMatch(String key) { + if (removeKeySuffix != null) { + if (key.endsWith(removeKeySuffix)) { + key = key.substring(0, key.length() - removeKeySuffix.length()); + } + } + return key; + } + void extractNoteIfMatch(String value) { if (noteKeyPattern != null) { Matcher m = noteKeyPattern.matcher(currentKeyName); diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/MacStringsFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/MacStringsFilter.java index 5c86aa0adf..ed18693d61 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/MacStringsFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/MacStringsFilter.java @@ -1,11 +1,17 @@ package com.box.l10n.mojito.okapi.filters; import com.box.l10n.mojito.okapi.ExtractUsagesFromTextUnitComments; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation; +import com.box.l10n.mojito.okapi.steps.OutputDocumentPostProcessingAnnotation.OutputDocumentPostProcessorBase; +import com.google.common.base.Strings; import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import net.sf.okapi.common.Event; import net.sf.okapi.common.EventType; import net.sf.okapi.common.filters.FilterConfiguration; +import net.sf.okapi.common.resource.RawDocument; import net.sf.okapi.common.resource.TextUnit; import net.sf.okapi.filters.regex.RegexFilter; import org.springframework.beans.factory.annotation.Autowired; @@ -18,14 +24,41 @@ public class MacStringsFilter extends RegexEscapeDoubleQuoteFilter { public static final String FILTER_CONFIG_ID = "okf_regex@mojito"; + private static final String REMOVE_COMMENTS = "removeComment"; @Autowired ExtractUsagesFromTextUnitComments extractUsagesFromTextUnitComments; + boolean removeComment = false; + @Override public String getName() { return FILTER_CONFIG_ID; } + @Override + public void open(RawDocument input) { + super.open(input); + applyFilterOptions(input); + input.setAnnotation( + new RemoveUntranslatedStategyAnnotation( + RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); + input.setAnnotation( + new OutputDocumentPostProcessingAnnotation( + new MacStringsFilterPostProcessor(removeComment))); + } + + void applyFilterOptions(RawDocument input) { + FilterOptions filterOptions = input.getAnnotation(FilterOptions.class); + + if (filterOptions != null) { + filterOptions.getBoolean( + REMOVE_COMMENTS, + b -> { + removeComment = b; + }); + } + } + @Override public List getConfigurations() { List list = new ArrayList<>(); @@ -51,4 +84,94 @@ public Event next() { return event; } + + static class MacStringsFilterPostProcessor extends OutputDocumentPostProcessorBase { + + static final Pattern COMMENT_KEY_VALUE_PATTERN = + Pattern.compile("(?s)(/\\*.*?\\*/\\s*)?(\".*?\"\\s*=\\s*\".*?\";)"); + static final Pattern COMMENT_PATTERN = Pattern.compile("(?s)/\\*.*?\\*/"); + + boolean removeComment = false; + + public MacStringsFilterPostProcessor(boolean removeComment) { + this.removeComment = removeComment; + } + + public String execute(String fileContent) { + String result = fileContent; + + if (hasRemoveUntranslated()) { + result = removeUntranslated(fileContent); + } + + if (removeComment) { + result = removeComments(fileContent); + } + + return result; + } + + public static String removeUntranslated(String fileContent) { + + Matcher matcher = COMMENT_KEY_VALUE_PATTERN.matcher(fileContent); + StringBuilder stringBuilder = new StringBuilder(); + + while (matcher.find()) { + String entry = matcher.group(); + if (entry.contains(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)) { + matcher.appendReplacement(stringBuilder, ""); + } else { + matcher.appendReplacement(stringBuilder, Matcher.quoteReplacement(entry)); + } + } + matcher.appendTail(stringBuilder); + + String result = collapseBlankLines(stringBuilder.toString()); + result = ensureEndLineAsInInput(result, fileContent); + + return result; + } + + public static String removeComments(String fileContent) { + Matcher matcher = COMMENT_PATTERN.matcher(fileContent); + String result = matcher.replaceAll(""); + result = collapseBlankLines(result); + result = ensureEndLineAsInInput(result, fileContent); + return result; + } + + static String ensureEndLineAsInInput(String output, String input) { + String result = output.trim(); + + if (input.isBlank()) { + return "\n".repeat(input.length()) + result; + } else { + + int leadingNewlines = 0; + int trailingNewlines = 0; + int length = input.length(); + + int index = 0; + while (index < length && input.charAt(index) == '\n') { + leadingNewlines++; + index++; + } + + index = length - 1; + while (index >= 0 && input.charAt(index) == '\n') { + trailingNewlines++; + index--; + } + return "\n".repeat(leadingNewlines) + result + "\n".repeat(trailingNewlines); + } + } + + static String collapseBlankLines(String input) { + String result = input; + if (!Strings.isNullOrEmpty(input)) { + result = input.replaceAll("(?m)^(\\s*\\n){2,}", "\n"); + } + return result; + } + } } diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/POFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/POFilter.java index e65bf342a9..73f8670851 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/filters/POFilter.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/filters/POFilter.java @@ -106,10 +106,18 @@ public void open(RawDocument input) { input.setAnnotation( new RemoveUntranslatedStategyAnnotation( RemoveUntranslatedStrategy.PLACEHOLDER_AND_POST_PROCESSING)); - // Post processing is disable for now, it will be enabled by the TranslsateStep if there are - // actual text unit to remove + input.setAnnotation( - new OutputDocumentPostProcessingAnnotation(POFilter::removeUntranslated, false)); + new OutputDocumentPostProcessingAnnotation( + new OutputDocumentPostProcessingAnnotation.OutputDocumentPostProcessorBase() { + @Override + public String execute(String content) { + if (hasRemoveUntranslated()) { + content = removeUntranslated(content); + } + return content; + } + })); } @Override diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/steps/FilterEventsToInMemoryRawDocumentStep.java b/common/src/main/java/com/box/l10n/mojito/okapi/steps/FilterEventsToInMemoryRawDocumentStep.java index 5c3c93c401..7baa13477d 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/steps/FilterEventsToInMemoryRawDocumentStep.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/steps/FilterEventsToInMemoryRawDocumentStep.java @@ -126,9 +126,11 @@ String getOutputDocument() throws UnsupportedEncodingException { OutputDocumentPostProcessingAnnotation outputDocumentPostProcessingAnnotation = rawDocument.getAnnotation(OutputDocumentPostProcessingAnnotation.class); if (outputDocumentPostProcessingAnnotation != null - && outputDocumentPostProcessingAnnotation.isEnabled()) { + && outputDocumentPostProcessingAnnotation.getOutputDocumentPostProcessor() != null) { outputDocument = - outputDocumentPostProcessingAnnotation.getPostProcessing().apply(outputDocument); + outputDocumentPostProcessingAnnotation + .getOutputDocumentPostProcessor() + .execute(outputDocument); } return outputDocument; diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/steps/OutputDocumentPostProcessingAnnotation.java b/common/src/main/java/com/box/l10n/mojito/okapi/steps/OutputDocumentPostProcessingAnnotation.java index 3f8b59b906..02366aae38 100644 --- a/common/src/main/java/com/box/l10n/mojito/okapi/steps/OutputDocumentPostProcessingAnnotation.java +++ b/common/src/main/java/com/box/l10n/mojito/okapi/steps/OutputDocumentPostProcessingAnnotation.java @@ -1,31 +1,38 @@ package com.box.l10n.mojito.okapi.steps; -import java.util.function.Function; import net.sf.okapi.common.annotation.IAnnotation; public class OutputDocumentPostProcessingAnnotation implements IAnnotation { - Function postProcessing; - boolean enabled; + OutputDocumentPostProcessor outputDocumentPostProcessor; public OutputDocumentPostProcessingAnnotation( - Function postProcessing, boolean enabled) { - this.postProcessing = postProcessing; - this.enabled = enabled; + OutputDocumentPostProcessor outputDocumentPostProcessor) { + this.outputDocumentPostProcessor = outputDocumentPostProcessor; } - public Function getPostProcessing() { - return postProcessing; + public OutputDocumentPostProcessor getOutputDocumentPostProcessor() { + return outputDocumentPostProcessor; } - public void setPostProcessing(Function postProcessing) { - this.postProcessing = postProcessing; - } + public interface OutputDocumentPostProcessor { + String execute(String content); + + boolean hasRemoveUntranslated(); - public boolean isEnabled() { - return enabled; + void setRemoveUntranslated(boolean removeUntranslated); } - public void setEnabled(boolean enabled) { - this.enabled = enabled; + public abstract static class OutputDocumentPostProcessorBase + implements OutputDocumentPostProcessor { + boolean removeUntranslated = false; + + public boolean hasRemoveUntranslated() { + return removeUntranslated; + } + + @Override + public void setRemoveUntranslated(boolean removeUntranslated) { + this.removeUntranslated = removeUntranslated; + } } } diff --git a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java index fdcf1facf0..35290cd880 100644 --- a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java +++ b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClient.java @@ -5,17 +5,28 @@ import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator; +import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.ForkJoinPool; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -30,11 +41,19 @@ public class OpenAIClient { final HttpClient httpClient; - OpenAIClient(String apiKey, String host, ObjectMapper objectMapper, HttpClient httpClient) { + final Executor asyncExecutor; + + OpenAIClient( + String apiKey, + String host, + ObjectMapper objectMapper, + HttpClient httpClient, + Executor asyncExecutor) { this.apiKey = Objects.requireNonNull(apiKey); this.host = Objects.requireNonNull(host); this.objectMapper = Objects.requireNonNull(objectMapper); this.httpClient = Objects.requireNonNull(httpClient); + this.asyncExecutor = Objects.requireNonNull(asyncExecutor); } public static class Builder { @@ -47,6 +66,8 @@ public static class Builder { private HttpClient httpClient; + private Executor asyncExecutor; + public Builder() {} public Builder apiKey(String apiKey) { @@ -69,6 +90,11 @@ public Builder httpClient(HttpClient httpClient) { return this; } + public Builder asyncExecutor(Executor asyncExecutor) { + this.asyncExecutor = asyncExecutor; + return this; + } + public OpenAIClient build() { if (apiKey == null) { throw new IllegalStateException("API key must be provided"); @@ -80,11 +106,16 @@ public OpenAIClient build() { if (httpClient == null) { httpClient = createHttpClient(); } - return new OpenAIClient(apiKey, host, objectMapper, httpClient); + + if (asyncExecutor == null) { + asyncExecutor = ForkJoinPool.commonPool(); + } + + return new OpenAIClient(apiKey, host, objectMapper, httpClient, asyncExecutor); } private HttpClient createHttpClient() { - return HttpClient.newHttpClient(); + return HttpClient.newBuilder().build(); } private ObjectMapper createObjectMapper() { @@ -126,7 +157,7 @@ public CompletableFuture getChatCompletions( CompletableFuture chatCompletionsResponse = httpClient .sendAsync(request, HttpResponse.BodyHandlers.ofString()) - .thenApply( + .thenApplyAsync( httpResponse -> { if (httpResponse.statusCode() != 200) { throw new OpenAIClientResponseException("ChatCompletion failed", httpResponse); @@ -139,7 +170,8 @@ public CompletableFuture getChatCompletions( "Can't deserialize ChatCompletionsResponse", e, httpResponse); } } - }); + }, + asyncExecutor); return chatCompletionsResponse; } @@ -214,7 +246,8 @@ public record ChatCompletionsRequest( @JsonProperty("max_tokens") int maxTokens, @JsonProperty("top_p") float topP, @JsonProperty("frequency_penalty") float frequencyPenalty, - @JsonProperty("presence_penalty") float presencePenalty) { + @JsonProperty("presence_penalty") float presencePenalty, + @JsonProperty("response_format") ResponseFormat responseFormat) { static String ENDPOINT = "/v1/chat/completions"; @@ -228,6 +261,79 @@ public enum Models { } } + public interface ResponseFormat {} + + public record JsonFormat(String type, @JsonProperty("json_schema") JsonSchema jsonSchema) + implements ResponseFormat { + + public record JsonSchema(boolean strict, String name, Object schema) { + + public static ObjectNode createJsonSchema(Class type) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(objectMapper); + com.fasterxml.jackson.module.jsonSchema.JsonSchema baseSchema = null; + try { + baseSchema = schemaGen.generateSchema(type); + } catch (JsonMappingException e) { + throw new RuntimeException(e); + } + JsonNode schemaNode = objectMapper.valueToTree(baseSchema); + ObjectNode rootNode = (ObjectNode) schemaNode; + enhanceSchema(rootNode); + return rootNode; + } + + private static void enhanceSchema(ObjectNode objectNode) { + + if (!objectNode.has("type")) { + objectNode.put("type", "object"); + } + objectNode.put("additionalProperties", false); + + if (objectNode.has("properties")) { + ObjectNode propertiesNode = (ObjectNode) objectNode.get("properties"); + ArrayNode requiredFields = objectNode.putArray("required"); + + Iterator> fields = propertiesNode.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + String fieldName = field.getKey(); + requiredFields.add(fieldName); + + JsonNode fieldSchema = field.getValue(); + if (fieldSchema.isObject()) { + ObjectNode fieldObjectNode = (ObjectNode) fieldSchema; + + String fieldType = + fieldObjectNode.has("type") ? fieldObjectNode.get("type").asText() : null; + + if ("object".equals(fieldType) && fieldObjectNode.has("properties")) { + enhanceSchema(fieldObjectNode); + } else if ("array".equals(fieldType) && fieldObjectNode.has("items")) { + enhanceArrayItems(fieldObjectNode); + } + } + } + } + } + + private static void enhanceArrayItems(ObjectNode arrayNode) { + JsonNode itemsNode = arrayNode.get("items"); + if (itemsNode != null && itemsNode.isObject()) { + ObjectNode itemsObjectNode = (ObjectNode) itemsNode; + + if (itemsObjectNode.has("properties")) { + enhanceSchema(itemsObjectNode); + } + + if (!itemsObjectNode.has("additionalProperties")) { + itemsObjectNode.put("additionalProperties", false); + } + } + } + } + } + public interface Message { String role(); @@ -314,6 +420,7 @@ public static class Builder { private float topP; private float frequencyPenalty; private float presencePenalty; + private ResponseFormat responseFormat; public Builder model(Models model) { return model(model.name); @@ -364,6 +471,11 @@ public Builder presencePenalty(float presencePenalty) { return this; } + public Builder responseFormat(ResponseFormat responseFormat) { + this.responseFormat = responseFormat; + return this; + } + public ChatCompletionsRequest build() { return new ChatCompletionsRequest( model, @@ -374,7 +486,8 @@ public ChatCompletionsRequest build() { maxTokens, topP, frequencyPenalty, - presencePenalty); + presencePenalty, + responseFormat); } } @@ -523,19 +636,307 @@ public record Usage( @JsonProperty("total_tokens") int totalTokens) {} } + public UploadFileResponse uploadFile(UploadFileRequest uploadFileRequest) { + + final String boundary = UUID.randomUUID().toString(); + + String body = uploadFileRequest.getMultipartBody(boundary); + + HttpRequest request = + HttpRequest.newBuilder() + .uri(getUriForEndpoint(UploadFileRequest.ENDPOINT)) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(HttpRequest.BodyPublishers.ofString(body)) + .build(); + + HttpResponse response; + try { + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + if (response.statusCode() != 200) { + throw new OpenAIClientResponseException("Can't upload file", response); + } + + UploadFileResponse uploadFileResponse; + try { + uploadFileResponse = objectMapper.readValue(response.body(), UploadFileResponse.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + return uploadFileResponse; + } + + public record UploadFileRequest( + String purpose, String filename, String content, String contentType) { + + static final String ENDPOINT = "/v1/files"; + + public static UploadFileRequest forBatch(String filename, String content) { + return new UploadFileRequest(Purpose.BATCH.toString(), filename, content, "application/json"); + } + + enum Purpose { + BATCH("batch"), + ASSISTANTS("assistants"), + FINE_TUNE("fine-tune"), + VISION("vision"); + + private final String purposeCode; + + Purpose(String purposeCode) { + this.purposeCode = purposeCode; + } + + public String getPurposeCode() { + return purposeCode; + } + + @Override + public String toString() { + return purposeCode; + } + + public static Purpose fromCode(String purposeCode) { + for (Purpose purpose : Purpose.values()) { + if (purpose.purposeCode.equalsIgnoreCase(purposeCode)) { + return purpose; + } + } + throw new IllegalArgumentException("Unknown purpose: " + purposeCode); + } + } + + String getMultipartBody(String boundary) { + String body = + """ + --%1$s\r + Content-Disposition: form-data; name="purpose"\r + \r + %5$s\r + --%1$s\r + Content-Disposition: form-data; name="file"; filename="%2$s"\r + Content-Type: %3$s\r + \r + %4$s\r + --%1$s--\r + """ + .formatted(boundary, filename, contentType, content, purpose); + return body; + } + } + + public record UploadFileResponse( + String object, + String id, + String purpose, + String filename, + int bytes, + @JsonProperty("created_at") long createdAt, + String status, + @JsonProperty("status_details") String statusDetails) {} + + public record RequestBatchFileLine( + @JsonProperty("custom_id") String customId, String method, String url, Object body) { + + public static RequestBatchFileLine forChatCompletion( + String customId, ChatCompletionsRequest chatCompletionsRequest) { + return new RequestBatchFileLine( + customId, "POST", "/v1/chat/completions", chatCompletionsRequest); + } + } + + public record ChatCompletionResponseBatchFileLine( + String id, @JsonProperty("custom_id") String customId, Response response) { + public record Response( + @JsonProperty("status_code") int statusCode, + @JsonProperty("request_id") String requestId, + @JsonProperty("body") ChatCompletionsResponse chatCompletionsResponse) {} + } + + public DownloadFileContentResponse downloadFileContent( + DownloadFileContentRequest downloadFileContentRequest) { + HttpResponse response; + HttpRequest request = + HttpRequest.newBuilder() + .uri( + getUriForEndpoint( + DownloadFileContentRequest.ENDPOINT.formatted( + downloadFileContentRequest.fileId()))) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .GET() + .build(); + + try { + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException( + "Error while sending the request to download the file: " + + downloadFileContentRequest.fileId(), + e); + } + + if (response.statusCode() != 200) { + throw new OpenAIClientResponseException("Can't download file content", response); + } + + return new DownloadFileContentResponse(response.body()); + } + + public record DownloadFileContentRequest(String fileId) { + static final String ENDPOINT = "/v1/files/%s/content"; + } + + public record DownloadFileContentResponse(String content) {} + + public CreateBatchResponse createBatch(CreateBatchRequest createBatchRequest) { + + String jsonBody; + try { + jsonBody = objectMapper.writeValueAsString(createBatchRequest); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + HttpRequest request = + HttpRequest.newBuilder() + .uri(getUriForEndpoint(CreateBatchRequest.ENDPOINT)) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(jsonBody)) + .build(); + + HttpResponse response; + try { + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + if (response.statusCode() != 200) { + throw new OpenAIClientResponseException("Can't create batch", response); + } + + CreateBatchResponse createBatchResponse; + try { + createBatchResponse = objectMapper.readValue(response.body(), CreateBatchResponse.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + return createBatchResponse; + } + + public record CreateBatchRequest( + @JsonProperty("input_file_id") String inputFileId, + String endpoint, + @JsonProperty("completion_window") String completionWindow, + Map metadata) { + + public static final String ENDPOINT = "/v1/batches"; + + public static CreateBatchRequest forChatCompletion( + String fileId, Map metadata) { + return new CreateBatchRequest(fileId, "/v1/chat/completions", "24h", metadata); + } + } + + public record CreateBatchResponse( + String id, + String object, + String endpoint, + String errors, + @JsonProperty("input_file_id") String inputFileId, + @JsonProperty("completion_window") String completionWindow, + String status, + @JsonProperty("output_file_id") String outputFileId, + @JsonProperty("error_file_id") String errorFileId, + @JsonProperty("created_at") long createdAt, + @JsonProperty("in_progress_at") Long inProgressAt, + @JsonProperty("expires_at") long expiresAt, + @JsonProperty("completed_at") Long completedAt, + @JsonProperty("failed_at") Long failedAt, + @JsonProperty("expired_at") Long expiredAt, + @JsonProperty("request_counts") RequestCounts requestCounts, + Map metadata) { + record RequestCounts(int total, int completed, int failed) {} + } + + public RetrieveBatchResponse retrieveBatch(RetrieveBatchRequest retrieveBatchRequest) { + HttpRequest request = + HttpRequest.newBuilder() + .uri( + getUriForEndpoint( + RetrieveBatchRequest.ENDPOINT.formatted(retrieveBatchRequest.batchId()))) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .GET() + .build(); + + HttpResponse response; + try { + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + if (response.statusCode() != 200) { + throw new OpenAIClientResponseException("Can't retrieve batch", response); + } + + RetrieveBatchResponse retrieveBatchResponse; + try { + retrieveBatchResponse = objectMapper.readValue(response.body(), RetrieveBatchResponse.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + return retrieveBatchResponse; + } + + public record RetrieveBatchRequest(String batchId) { + public static final String ENDPOINT = "/v1/batches/%s"; + } + + public record RetrieveBatchResponse( + String id, + String object, + String endpoint, + String errors, + @JsonProperty("input_file_id") String inputFileId, + @JsonProperty("completion_window") String completionWindow, + String status, + @JsonProperty("output_file_id") String outputFileId, + @JsonProperty("error_file_id") String errorFileId, + @JsonProperty("created_at") long createdAt, + @JsonProperty("in_progress_at") Long inProgressAt, + @JsonProperty("expires_at") long expiresAt, + @JsonProperty("completed_at") Long completedAt, + @JsonProperty("failed_at") Long failedAt, + @JsonProperty("expired_at") Long expiredAt, + @JsonProperty("request_counts") RequestCounts requestCounts, + Map metadata) { + record RequestCounts(int total, int completed, int failed) {} + } + private URI getUriForEndpoint(String endpoint) { return URI.create(host).resolve(endpoint); } - public class OpenAIClientResponseException extends RuntimeException { - HttpResponse httpResponse; + public static class OpenAIClientResponseException extends RuntimeException { + HttpResponse httpResponse; - public OpenAIClientResponseException(String message, HttpResponse httpResponse) { + public OpenAIClientResponseException(String message, HttpResponse httpResponse) { super(message); this.httpResponse = Objects.requireNonNull(httpResponse); } - public OpenAIClientResponseException(String message, Exception e, HttpResponse httpResponse) { + public OpenAIClientResponseException( + String message, Exception e, HttpResponse httpResponse) { super(message, e); this.httpResponse = Objects.requireNonNull(httpResponse); } diff --git a/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClientPool.java b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClientPool.java new file mode 100644 index 0000000000..f18ed0c433 --- /dev/null +++ b/common/src/main/java/com/box/l10n/mojito/openai/OpenAIClientPool.java @@ -0,0 +1,77 @@ +package com.box.l10n.mojito.openai; + +import com.google.common.base.Function; +import java.net.http.HttpClient; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadLocalRandom; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OpenAIClientPool { + + static Logger logger = LoggerFactory.getLogger(OpenAIClientPool.class); + + int numberOfClients; + OpenAIClientWithSemaphore[] openAIClientWithSemaphores; + + /** + * Pool to parallelize slower requests (1s+) over HTTP/2 connections. + * + * @param numberOfClients Number of OpenAIClient instances with independent HttpClients. + * @param numberOfParallelRequestPerClient Maximum parallel requests per client, controlled by a + * semaphore to prevent overload. + * @param sizeOfAsyncProcessors Shared async processors across all HttpClients to limit threads, + * as request time is the main bottleneck. + * @param apiKey API key for authentication. + */ + public OpenAIClientPool( + int numberOfClients, + int numberOfParallelRequestPerClient, + int sizeOfAsyncProcessors, + String apiKey) { + ExecutorService asyncExecutor = Executors.newWorkStealingPool(sizeOfAsyncProcessors); + this.numberOfClients = numberOfClients; + this.openAIClientWithSemaphores = new OpenAIClientWithSemaphore[numberOfClients]; + for (int i = 0; i < numberOfClients; i++) { + this.openAIClientWithSemaphores[i] = + new OpenAIClientWithSemaphore( + OpenAIClient.builder() + .apiKey(apiKey) + .asyncExecutor(asyncExecutor) + .httpClient(HttpClient.newBuilder().executor(asyncExecutor).build()) + .build(), + new Semaphore(numberOfParallelRequestPerClient)); + } + } + + public CompletableFuture submit(Function> f) { + + while (true) { + for (OpenAIClientWithSemaphore openAIClientWithSemaphore : openAIClientWithSemaphores) { + if (openAIClientWithSemaphore.semaphore().tryAcquire()) { + return f.apply(openAIClientWithSemaphore.openAIClient()) + .whenComplete((o, e) -> openAIClientWithSemaphore.semaphore().release()); + } + } + + try { + logger.debug("can't directly acquire any semaphore, do blocking"); + int randomSemaphoreIndex = + ThreadLocalRandom.current().nextInt(openAIClientWithSemaphores.length); + OpenAIClientWithSemaphore randomClientWithSemaphore = + this.openAIClientWithSemaphores[randomSemaphoreIndex]; + randomClientWithSemaphore.semaphore().acquire(); + return f.apply(randomClientWithSemaphore.openAIClient()) + .whenComplete((o, e) -> randomClientWithSemaphore.semaphore().release()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Can't submit task to the OpenAIClientPool", e); + } + } + } + + record OpenAIClientWithSemaphore(OpenAIClient openAIClient, Semaphore semaphore) {} +} diff --git a/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java b/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java new file mode 100644 index 0000000000..f18489c6dd --- /dev/null +++ b/common/src/main/java/com/box/l10n/mojito/utils/OptionsParser.java @@ -0,0 +1,61 @@ +package com.box.l10n.mojito.utils; + +import com.google.common.base.Splitter; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +public class OptionsParser { + + Map options = new LinkedHashMap<>(); + + public OptionsParser(List options) { + if (options != null) { + for (String option : options) { + List optionKeyAndValue = Splitter.on("=").limit(2).splitToList(option); + if (optionKeyAndValue.size() == 2) { + this.options.put(optionKeyAndValue.get(0), optionKeyAndValue.get(1)); + } + } + } + } + + public void getString(String key, Consumer consumer) { + if (this.options.containsKey(key)) { + consumer.accept(this.options.get(key)); + } + } + + public void getBoolean(String key, Consumer consumer) { + if (this.options.containsKey(key)) { + consumer.accept(Boolean.valueOf(this.options.get(key))); + } + } + + public Boolean getBoolean(String key, Boolean defaultValue) { + Boolean value = defaultValue; + + if (this.options.containsKey(key)) { + value = Boolean.valueOf(this.options.get(key)); + } + + return value; + } + + public String getString(String key, String defaultValue) { + String value = defaultValue; + + if (this.options.containsKey(key)) { + value = this.options.get(key); + } + + return value; + } + + public void getInteger(String key, Consumer consumer) { + if (this.options.containsKey(key)) { + consumer.accept(Integer.valueOf(this.options.get(key))); + } + } +} diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java index 29a7187e91..90874d2bce 100644 --- a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java +++ b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java @@ -109,4 +109,285 @@ void testUnescaping(String input, String expected) { logger.debug("> Input:\n{}\n> Expected:\n{}\n> Actual:\n{}\n>>>", input, expected, s); assertEquals(expected, s); } + + @Test + public void testPostProcessingKeepDescription() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, false, 2, false, false); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + @#$untranslated$#@ + translated + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + somestring to keep + + + translated + + + pin fr + pins fr + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingRemoveDescription() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, false, false); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + @#$untranslated$#@ + translated + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + somestring to keep + + + translated + + + pin fr + pins fr + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingEmptyFile() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, false, false); + String input = ""; + String output = androidFilePostProcessor.execute(input); + String expected = ""; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingNoProlog() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, false, false); + String input = + """ + + somestring to keep + @#$untranslated$#@ + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + somestring to keep + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingRemoveTranslatableFalse() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + @#$untranslated$#@ + translated + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + + + translated + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingRemoveMissingOther() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); + String input = + """ + + + + pin fr + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingRemoveMissingOtherUntranslated() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); + String input = + """ + + + + pin fr + @#$untranslated$#@ + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingStandaloneNo() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + @#$untranslated$#@ + translated + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + + + translated + + + """; + assertEquals(expected, output); + } + + @Test + public void testPostProcessingStandaloneYes() { + AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor = + new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false); + String input = + """ + + + somestring to keep + @#$untranslated$#@ + + + @#$untranslated$#@ + @#$untranslated$#@ + + + @#$untranslated$#@ + translated + + + pin fr + pins fr + + + """; + String output = androidFilePostProcessor.execute(input); + String expected = + """ + + + + + translated + + + """; + assertEquals(expected, output); + } } diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/MacStringsFilterTest.java b/common/src/test/java/com/box/l10n/mojito/okapi/filters/MacStringsFilterTest.java new file mode 100644 index 0000000000..c74c1d5a48 --- /dev/null +++ b/common/src/test/java/com/box/l10n/mojito/okapi/filters/MacStringsFilterTest.java @@ -0,0 +1,306 @@ +package com.box.l10n.mojito.okapi.filters; + +import static com.box.l10n.mojito.okapi.filters.MacStringsFilter.MacStringsFilterPostProcessor.collapseBlankLines; +import static com.box.l10n.mojito.okapi.filters.MacStringsFilter.MacStringsFilterPostProcessor.ensureEndLineAsInInput; +import static com.box.l10n.mojito.okapi.filters.MacStringsFilter.MacStringsFilterPostProcessor.removeComments; +import static com.box.l10n.mojito.okapi.filters.MacStringsFilter.MacStringsFilterPostProcessor.removeUntranslated; +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class MacStringsFilterTest { + + @Test + public void testRemoveUntranslated_NoUntranslatedEntries() { + String input = + """ + /* Welcome message */ + "welcome_message" = "Welcome to our app!"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String output = + """ + /* Welcome message */ + "welcome_message" = "Welcome to our app!"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_WithUntranslatedEntry() { + String input = + """ + /* Welcome message */ + "welcome_message" = "@#$untranslated$#@"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String output = + """ + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_UntranslatedEntryWithoutComment() { + String input = + """ + "welcome_message" = "@#$untranslated$#@"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + + String output = + """ + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_MultipleUntranslatedEntries() { + String input = + """ + /* Welcome message */ + "welcome_message" = "@#$untranslated$#@"; + + /* Farewell message */ + "farewell_message" = "@#$untranslated$#@"; + + /* Info message */ + "info_message" = "Information available."; + """; + String output = + """ + /* Info message */ + "info_message" = "Information available."; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_EscapedCharacters() { + String input = + """ + /* Message with escaped characters */ + "escaped_message" = "Line1\\nLine2\\tTabbed"; + + /* Untranslated message */ + "untranslated_key" = "@#$untranslated$#@"; + """; + String output = + """ + /* Message with escaped characters */ + "escaped_message" = "Line1\\nLine2\\tTabbed"; + """; + String result = removeUntranslated(input); + System.out.println("[" + input + "]"); + System.out.println("[" + result + "]"); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_MultilineValue() { + String input = + """ + /* Multiline message */ + "multiline_message" = "This is a long message that spans multiple lines \\ + in the .strings file, but will be rendered as a single line \\ + when displayed in the application."; + + /* Untranslated message */ + "untranslated_key" = "@#$untranslated$#@"; + """; + String output = + """ + /* Multiline message */ + "multiline_message" = "This is a long message that spans multiple lines \\ + in the .strings file, but will be rendered as a single line \\ + when displayed in the application."; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_OnlyUntranslatedEntries() { + String input = + """ + /* Untranslated message */ + "untranslated_key1" = "@#$untranslated$#@"; + + /* Untranslated message */ + "untranslated_key2" = "@#$untranslated$#@"; + """; + String output = "\n"; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_EmptyFile() { + String input = ""; + String output = ""; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveUntranslated_UntranslatedAtEnd() { + String input = + """ + /* Welcome message */ + "welcome_message" = "Welcome!"; + + /* Untranslated message */ + "untranslated_key" = "@#$untranslated$#@"; + """; + String output = + """ + /* Welcome message */ + "welcome_message" = "Welcome!"; + """; + String result = removeUntranslated(input); + assertEquals(output, result); + } + + @Test + public void testRemoveComments_NoComments() { + String input = + """ + "welcome_message" = "Welcome to our app!"; + "farewell_message" = "Goodbye!"; + """; + String expected = + """ + "welcome_message" = "Welcome to our app!"; + "farewell_message" = "Goodbye!"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_WithBlockComments() { + String input = + """ + /* Welcome message */ + "welcome_message" = "Welcome to our app!"; + + /* Farewell message */ + "farewell_message" = "Goodbye!"; + """; + String expected = + """ + "welcome_message" = "Welcome to our app!"; + + "farewell_message" = "Goodbye!"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_UntranslatedEntryWithComments() { + String input = + """ + /* Untranslated message */ + "untranslated_key" = "@#$untranslated$#@"; + """; + String expected = + """ + "untranslated_key" = "@#$untranslated$#@"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_EscapedCharacters() { + String input = + """ + /* Message with escaped characters */ + "escaped_message" = "Line1\\nLine2\\tTabbed"; + """; + String expected = + """ + "escaped_message" = "Line1\\nLine2\\tTabbed"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_EmptyFile() { + String input = ""; + String expected = ""; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_MultilineBlockComment() { + String input = + """ + /* Multiline + block comment */ + "message" = "Hello!"; + """; + String expected = + """ + "message" = "Hello!"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testRemoveComments_NestedBlockComments() { + String input = + """ + /* Outer comment + /* Inner comment */ + */ + "message" = "Hello!"; + """; + String expected = + """ + */ + "message" = "Hello!"; + """; + String result = removeComments(input); + assertEquals(expected, result); + } + + @Test + public void testAddEndLineAsInInput() { + assertEquals("", ensureEndLineAsInInput("", "")); + assertEquals("\n", ensureEndLineAsInInput("", "\n")); + assertEquals("\n", ensureEndLineAsInInput("\n", "\n")); + assertEquals("", ensureEndLineAsInInput("\n", "")); + } + + @Test + public void testCollapseBlankLines() { + assertEquals("", collapseBlankLines("")); + assertEquals("\n", collapseBlankLines("\n")); + assertEquals("\n", collapseBlankLines("\n\n")); + assertEquals("\na\nb\n\nc\n\n", collapseBlankLines("\na\nb\n\nc\n\n")); + assertEquals("a\nb\n\nc\n\n", collapseBlankLines("a\nb\n\nc\n\n")); + assertEquals("\na\nb\n\nc", collapseBlankLines("\na\nb\n\nc")); + } +} diff --git a/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientPoolTest.java b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientPoolTest.java new file mode 100644 index 0000000000..3ddbbeb15b --- /dev/null +++ b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientPoolTest.java @@ -0,0 +1,134 @@ +package com.box.l10n.mojito.openai; + +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.SystemMessage.systemMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.UserMessage.userMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.chatCompletionsRequest; + +import com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsResponse; +import com.google.common.base.Stopwatch; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Disabled +public class OpenAIClientPoolTest { + + static Logger logger = LoggerFactory.getLogger(OpenAIClientPoolTest.class); + + static final String API_KEY; + + static { + try { + // API_KEY = + // + // Files.readString(Paths.get(System.getProperty("user.home")).resolve(".keys/openai")) + // .trim(); + API_KEY = "test-api-key"; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Test + public void test() { + int numberOfClients = 10; + int numberOfParallelRequestPerClient = 50; + int numberOfRequests = 10000; + int sizeOfAsyncProcessors = 10; + int totalExecutions = numberOfClients * numberOfParallelRequestPerClient; + + OpenAIClientPool openAIClientPool = + new OpenAIClientPool( + numberOfClients, numberOfParallelRequestPerClient, sizeOfAsyncProcessors, API_KEY); + + AtomicInteger responseCounter = new AtomicInteger(); + AtomicInteger submitted = new AtomicInteger(); + Stopwatch stopwatch = Stopwatch.createStarted(); + + ArrayList submissionTimes = new ArrayList<>(); + ArrayList responseTimes = new ArrayList<>(); + + List> responses = new ArrayList<>(); + for (int i = 0; i < numberOfRequests; i++) { + String message = "Is %d prime?".formatted(i); + Stopwatch requestStopwatch = Stopwatch.createStarted(); + OpenAIClient.ChatCompletionsRequest chatCompletionsRequest = + chatCompletionsRequest() + .model("gpt-4o-2024-08-06") + .messages( + List.of( + systemMessageBuilder() + .content("You're an engine designed to check prime numbers") + .build(), + userMessageBuilder().content(message).build())) + .build(); + + CompletableFuture response = + openAIClientPool.submit( + openAIClient -> { + CompletableFuture chatCompletions = + openAIClient.getChatCompletions(chatCompletionsRequest); + submissionTimes.add(requestStopwatch.elapsed(TimeUnit.SECONDS)); + if (submitted.incrementAndGet() % 100 == 0) { + logger.info( + "--> request per second: " + + submitted.get() / (stopwatch.elapsed(TimeUnit.SECONDS) + 0.00001) + + ", submission count: " + + submitted.get() + + ", future response count: " + + responses.size() + + ", last submissions took: " + + submissionTimes.subList( + Math.max(0, submissionTimes.size() - 100), submissionTimes.size())); + } + return chatCompletions; + }); + + response.thenApply( + chatCompletionsResponse -> { + responseTimes.add(requestStopwatch.elapsed(TimeUnit.MILLISECONDS)); + if (responseCounter.incrementAndGet() % 10 == 0) { + double avg = + responseTimes.stream().collect(Collectors.averagingLong(Long::longValue)); + logger.info( + "<-- response per second: " + + responseCounter.get() / stopwatch.elapsed(TimeUnit.SECONDS) + + ", average response time: " + + Math.round(avg) + + " (rps: " + + Math.round(totalExecutions / (avg / 1000.0)) + + "), response count from counter: " + + responseCounter.get() + + ", last elapsed times: " + + responseTimes.subList(responseTimes.size() - 20, responseTimes.size())); + } + return chatCompletionsResponse; + }); + + responses.add(response); + } + + Stopwatch started = Stopwatch.createStarted(); + CompletableFuture.allOf(responses.toArray(new CompletableFuture[responses.size()])).join(); + logger.info("Waiting for join: " + started.elapsed()); + + double avg = responseTimes.stream().collect(Collectors.averagingLong(Long::longValue)); + logger.info( + "Total time: " + + stopwatch.elapsed().toString() + + ", request per second: " + + Math.round((double) numberOfRequests / stopwatch.elapsed(TimeUnit.SECONDS)) + + ", average response time: " + + Math.round(avg) + + " (theory rps: " + + Math.round(totalExecutions / (avg / 1000.0)) + + ")"); + } +} diff --git a/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java index cceebc5056..d538cb45b3 100644 --- a/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java +++ b/common/src/test/java/com/box/l10n/mojito/openai/OpenAIClientTest.java @@ -14,15 +14,20 @@ import static org.mockito.Mockito.when; import com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsResponse; +import com.box.l10n.mojito.openai.OpenAIClient.OpenAIClientResponseException; +import com.box.l10n.mojito.openai.OpenAIClient.UploadFileRequest; +import com.box.l10n.mojito.openai.OpenAIClient.UploadFileResponse; +import java.io.IOException; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import org.junit.jupiter.api.Test; -class OpenAIClientTest { +public class OpenAIClientTest { static final String API_KEY; @@ -60,29 +65,29 @@ public void testGetChatCompletionsSuccess() throws Exception { String jsonResponse = """ - { - "id": "chatcmpl-9DNYjOkXJxILUK3NXFv9MCZV0P8jZ", - "object": "chat.completion", - "created": 1712975853, - "model": "gpt-3.5-turbo-0125", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Il s'agit d'un test unitaire" - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 24, - "completion_tokens": 9, - "total_tokens": 33 - }, - "system_fingerprint": "fp_c2295e73ad" - }"""; + { + "id": "chatcmpl-9DNYjOkXJxILUK3NXFv9MCZV0P8jZ", + "object": "chat.completion", + "created": 1712975853, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Il s'agit d'un test unitaire" + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 9, + "total_tokens": 33 + }, + "system_fingerprint": "fp_c2295e73ad" + }"""; HttpResponse mockResponse = mock(HttpResponse.class); when(mockResponse.statusCode()).thenReturn(200); @@ -126,14 +131,14 @@ public void testGetChatCompletionsRequestError() throws Exception { when(mockResponse.statusCode()).thenReturn(400); String errorMsg = """ - { - "error": { - "message": "The model `invalid-model` does not exist or you do not have access to it.", - "type": "invalid_request_error", - "param": null, - "code": "model_not_found" - } - }"""; + { + "error": { + "message": "The model `invalid-model` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": null, + "code": "model_not_found" + } + }"""; when(mockResponse.body()).thenReturn(errorMsg); HttpClient mockHttpClient = mock(HttpClient.class); @@ -153,12 +158,12 @@ public void testGetChatCompletionsRequestError() throws Exception { .getMessage() .contains( """ - "error": { - "message": "The model `invalid-model` does not exist or you do not have access to it.", - "type": "invalid_request_error", - "param": null, - "code": "model_not_found" - }""")); + "error": { + "message": "The model `invalid-model` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": null, + "code": "model_not_found" + }""")); } @Test @@ -180,29 +185,29 @@ public void testGetChatCompletionsDeserializationError() throws Exception { String jsonResponse = """ - { - "id": "chatcmpl-9DNYjOkXJxILUK3NXFv9MCZV0P8jZ", - "object": "chat.completion", - "created": "invalid date to break deserialization", - "model": "gpt-3.5-turbo-0125", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Il s'agit d'un test unitaire" - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 24, - "completion_tokens": 9, - "total_tokens": 33 - }, - "system_fingerprint": "fp_c2295e73ad" - }"""; + { + "id": "chatcmpl-9DNYjOkXJxILUK3NXFv9MCZV0P8jZ", + "object": "chat.completion", + "created": "invalid date to break deserialization", + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Il s'agit d'un test unitaire" + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 9, + "total_tokens": 33 + }, + "system_fingerprint": "fp_c2295e73ad" + }"""; HttpResponse mockResponse = mock(HttpResponse.class); when(mockResponse.statusCode()).thenReturn(200); @@ -222,4 +227,335 @@ public void testGetChatCompletionsDeserializationError() throws Exception { assertEquals( "Can't deserialize ChatCompletionsResponse", completionException.getCause().getMessage()); } + + @Test + public void testUploadFileSuccess() throws Exception { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(200); + when(mockResponse.body()) + .thenReturn( + """ + { + "id": "file-123", + "filename": "example.jsonl", + "status": "uploaded", + "created_at": 1690000000 + }"""); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + UploadFileRequest fileUploadRequest = UploadFileRequest.forBatch("example.jsonl", "{}\n{}\n"); + + UploadFileResponse uploadFileResponse = openAIClient.uploadFile(fileUploadRequest); + + assertNotNull(uploadFileResponse); + assertEquals("file-123", uploadFileResponse.id()); + assertEquals("example.jsonl", uploadFileResponse.filename()); + assertEquals("uploaded", uploadFileResponse.status()); + assertEquals(1690000000, uploadFileResponse.createdAt()); + } + + @Test + public void testUploadFileError() throws Exception { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(400); + String errorMessage = + """ + { + "error": { + "message": "Invalid file format for Batch API. Must be .jsonl", + "type": "invalid_request_error", + "param": null, + "code": null + } + } + """; + when(mockResponse.body()).thenReturn(errorMessage); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + UploadFileRequest fileUploadRequest = + UploadFileRequest.forBatch( + "example.jsonl", + """ + { + "a" : "b" + } + """); + + OpenAIClientResponseException openAIClientResponseException = + assertThrows( + OpenAIClientResponseException.class, () -> openAIClient.uploadFile(fileUploadRequest)); + assertEquals(openAIClientResponseException.httpResponse.statusCode(), 400); + assertEquals(openAIClientResponseException.httpResponse.body(), errorMessage); + } + + @Test + public void testFileUploadRequestMultiPartBody() { + UploadFileRequest uploadFileRequest = UploadFileRequest.forBatch("test.jsonl", "{}\n{}"); + String actual = uploadFileRequest.getMultipartBody("test-boundary"); + assertEquals( + """ + --test-boundary\r + Content-Disposition: form-data; name="purpose"\r + \r + batch\r + --test-boundary\r + Content-Disposition: form-data; name="file"; filename="test.jsonl"\r + Content-Type: application/json\r + \r + {} + {}\r + --test-boundary--\r + """, + actual); + } + + @Test + public void testDownloadFileContentSuccess() throws IOException, InterruptedException { + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(200); + String fileContent = + """ + {"a" : "b"} + {"c" : "d"} + """; + when(mockResponse.body()).thenReturn(fileContent); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClient.DownloadFileContentResponse downloadFileContentResponse = + openAIClient.downloadFileContent( + new OpenAIClient.DownloadFileContentRequest("id-for-test")); + + assertEquals(fileContent, downloadFileContentResponse.content()); + } + + @Test + public void testDownloadFileContentError() throws IOException, InterruptedException { + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(404); + String body = + """ + { + "error": { + "message": "No such File object: id-for-test", + "type": "invalid_request_error", + "param": "id", + "code": null + } + } + """; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClientResponseException openAIClientResponseException = + assertThrows( + OpenAIClientResponseException.class, + () -> + openAIClient.downloadFileContent( + new OpenAIClient.DownloadFileContentRequest("id-for-test"))); + assertEquals(body, openAIClientResponseException.httpResponse.body()); + assertEquals(404, openAIClientResponseException.httpResponse.statusCode()); + } + + @Test + public void testCreateBatchSuccess() throws IOException, InterruptedException { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(200); + String body = + """ + { + "id": "batch_67199315c20081909074e442115938a2", + "object": "batch", + "endpoint": "/v1/chat/completions", + "errors": null, + "input_file_id": "file-pp1I2zv79eAnm47wt6rCNL5a", + "completion_window": "24h", + "status": "validating", + "output_file_id": null, + "error_file_id": null, + "created_at": 1729729301, + "in_progress_at": null, + "expires_at": 1729815701, + "finalizing_at": null, + "completed_at": null, + "failed_at": null, + "expired_at": null, + "cancelling_at": null, + "cancelled_at": null, + "request_counts": { + "total": 0, + "completed": 0, + "failed": 0 + }, + "metadata": { + "k1": "v1", + "k2": "v2" + } + } + """; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClient.CreateBatchResponse batch = + openAIClient.createBatch( + OpenAIClient.CreateBatchRequest.forChatCompletion( + "file-pp1I2zv79eAnm47wt6rCNL5a", Map.of("k1", "v1", "k2", "v2"))); + assertEquals("batch_67199315c20081909074e442115938a2", batch.id()); + assertEquals("file-pp1I2zv79eAnm47wt6rCNL5a", batch.inputFileId()); + assertEquals("v1", batch.metadata().get("k1")); + } + + @Test + public void testCreateBatchError() throws IOException, InterruptedException { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(400); + String body = + """ + { + "error": { + "message": "Invalid 'input_file_id': 'wrong-id'. Expected an ID that begins with 'file'.", + "type": "invalid_request_error", + "param": "input_file_id", + "code": "invalid_value" + } + }"""; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClientResponseException openAIClientResponseException = + assertThrows( + OpenAIClientResponseException.class, + () -> + openAIClient.createBatch( + OpenAIClient.CreateBatchRequest.forChatCompletion("wrong-id", null))); + assertEquals(body, openAIClientResponseException.httpResponse.body()); + assertEquals(400, openAIClientResponseException.httpResponse.statusCode()); + } + + @Test + public void testRetrieveBatchSuccess() throws IOException, InterruptedException { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(200); + String body = + """ + { + "id": "batch_67199315c20081909074e442115938a2", + "object": "batch", + "endpoint": "/v1/chat/completions", + "errors": null, + "input_file_id": "file-pp1I2zv79eAnm47wt6rCNL5a", + "completion_window": "24h", + "status": "validating", + "output_file_id": null, + "error_file_id": null, + "created_at": 1729729301, + "in_progress_at": null, + "expires_at": 1729815701, + "finalizing_at": null, + "completed_at": null, + "failed_at": null, + "expired_at": null, + "cancelling_at": null, + "cancelled_at": null, + "request_counts": { + "total": 0, + "completed": 0, + "failed": 0 + }, + "metadata": { + "k1": "v1", + "k2": "v2" + } + } + """; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClient.RetrieveBatchResponse batch = + openAIClient.retrieveBatch( + new OpenAIClient.RetrieveBatchRequest("batch_67199315c20081909074e442115938a2")); + assertEquals("batch_67199315c20081909074e442115938a2", batch.id()); + assertEquals("file-pp1I2zv79eAnm47wt6rCNL5a", batch.inputFileId()); + assertEquals("v1", batch.metadata().get("k1")); + } + + @Test + public void testRetrieveBatchError() throws IOException, InterruptedException { + + HttpClient mockHttpClient = mock(HttpClient.class); + HttpResponse mockResponse = mock(HttpResponse.class); + + when(mockResponse.statusCode()).thenReturn(400); + String body = + """ + { + "error": { + "message": "Invalid 'input_file_id': 'wrong-id'. Expected an ID that begins with 'file'.", + "type": "invalid_request_error", + "param": "input_file_id", + "code": "invalid_value" + } + }"""; + when(mockResponse.body()).thenReturn(body); + when(mockHttpClient.send(any(HttpRequest.class), any(HttpResponse.BodyHandler.class))) + .thenReturn(mockResponse); + + OpenAIClient openAIClient = + OpenAIClient.builder().apiKey(API_KEY).httpClient(mockHttpClient).build(); + + OpenAIClientResponseException openAIClientResponseException = + assertThrows( + OpenAIClientResponseException.class, + () -> + openAIClient.createBatch( + OpenAIClient.CreateBatchRequest.forChatCompletion("wrong-id", null))); + assertEquals(body, openAIClientResponseException.httpResponse.body()); + assertEquals(400, openAIClientResponseException.httpResponse.statusCode()); + } } diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/FilterOptionsTest.java b/common/src/test/java/com/box/l10n/mojito/utils/OptionsParserTest.java similarity index 67% rename from common/src/test/java/com/box/l10n/mojito/okapi/filters/FilterOptionsTest.java rename to common/src/test/java/com/box/l10n/mojito/utils/OptionsParserTest.java index 31d293b6e7..cd9452c7b5 100644 --- a/common/src/test/java/com/box/l10n/mojito/okapi/filters/FilterOptionsTest.java +++ b/common/src/test/java/com/box/l10n/mojito/utils/OptionsParserTest.java @@ -1,4 +1,4 @@ -package com.box.l10n.mojito.okapi.filters; +package com.box.l10n.mojito.utils; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -8,26 +8,26 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Test; -public class FilterOptionsTest { +public class OptionsParserTest { @Test public void testNullGivesEmptyMap() { - FilterOptions filterOptions = new FilterOptions(null); - assertTrue(filterOptions.options.isEmpty()); + OptionsParser optionsParser = new OptionsParser(null); + assertTrue(optionsParser.options.isEmpty()); } @Test public void testSplit() { - FilterOptions filterOptions = - new FilterOptions(Arrays.asList("option1=value1", "option2=value2")); + OptionsParser optionsParser = + new OptionsParser(Arrays.asList("option1=value1", "option2=value2")); AtomicBoolean called = new AtomicBoolean(false); - filterOptions.getString( + optionsParser.getString( "option1", v -> { called.set(true); assertEquals("value1", v); }); - filterOptions.getString( + optionsParser.getString( "option2", v -> { called.set(true); @@ -38,16 +38,16 @@ public void testSplit() { @Test public void testGetBoolean() { - FilterOptions filterOptions = new FilterOptions(Arrays.asList("option1=true", "option2=false")); + OptionsParser optionsParser = new OptionsParser(Arrays.asList("option1=true", "option2=false")); AtomicBoolean called = new AtomicBoolean(false); - filterOptions.getBoolean( + optionsParser.getBoolean( "option1", v -> { called.set(true); assertTrue(v); }); - filterOptions.getBoolean( + optionsParser.getBoolean( "option2", v -> { called.set(true); @@ -60,8 +60,8 @@ public void testGetBoolean() { @Test public void testSetObjectVariable() { - FilterOptions filterOptions = new FilterOptions(Arrays.asList("option1=someobjectvalue")); - filterOptions.getString("option1", v -> forTestSetObjectVariable = v); + OptionsParser optionsParser = new OptionsParser(Arrays.asList("option1=someobjectvalue")); + optionsParser.getString("option1", v -> forTestSetObjectVariable = v); assertEquals("someobjectvalue", forTestSetObjectVariable); } } diff --git a/docker/Dockerfile-bk21 b/docker/Dockerfile-bk21 new file mode 100644 index 0000000000..270669c8c2 --- /dev/null +++ b/docker/Dockerfile-bk21 @@ -0,0 +1,43 @@ +# syntax=docker/dockerfile:experimental + +FROM maven:3.9.6-eclipse-temurin-21 as build +VOLUME ["/tmp"] + +WORKDIR /mnt/mojito + +# copy source and make sure node* are not present (Mac version may conflict with Linux) +COPY . /mnt/mojito + +ENV PATH="/mnt/mojito/webapp/node/:${PATH}" +RUN --mount=type=cache,target=/root/.m2 --mount=type=cache,target=/mnt/mojito/node --mount=type=cache,target=/mnt/mojito/node_module mvn clean install -DskipTests + + +FROM eclipse-temurin:21-jdk-jammy as service +VOLUME /tmp + +ENV MOJITO_BIN=/usr/local/mojito/bin +ENV PATH $PATH:${MOJITO_BIN} +ENV MOJITO_HOST=localhost +ENV MOJITO_SCHEME=http +# MOJITO_PORT can conflict with Kubernetes, changing to MOJITO_SERVICE_PORT +ENV MOJITO_SERVICE_PORT=8080 + +COPY --from=build /mnt/mojito/webapp/target/mojito-webapp-*-exec.jar ${MOJITO_BIN}/mojito-webapp.jar +COPY --from=build /mnt/mojito/cli/target/mojito-cli-*-exec.jar ${MOJITO_BIN}/mojito-cli.jar +RUN sh -c 'touch ${MOJITO_BIN}/mojito-webapp.jar' +RUN sh -c 'touch ${MOJITO_BIN}/mojito-cli.jar' + +RUN apt-get update && apt-get install -y \ + mysql-client \ + socat + +# Create the shell wrapper for the jar +RUN /bin/echo -e "#!/bin/sh \n\ +java -Dl10n.resttemplate.host=\${MOJITO_HOST} \\\\\n \ + -Dl10n.resttemplate.scheme=\${MOJITO_SCHEME} \\\\\n \ + -Dl10n.resttemplate.port=\${MOJITO_PORT} \\\\\n \ + -jar $MOJITO_BIN/mojito-cli.jar \"\${@}\"" \ + >> /usr/local/mojito/bin/mojito && chmod +x $MOJITO_BIN/mojito + +# starting with "exec doesn't seem to be needed with openjdk:8-alpine. As per docker documentation, it is required in general +ENTRYPOINT exec java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar $MOJITO_BIN/mojito-webapp.jar diff --git a/docker/readme.md b/docker/readme.md index 02db69876f..c91bef3f87 100644 --- a/docker/readme.md +++ b/docker/readme.md @@ -1,9 +1,10 @@ # Docker compose -Run `docker-compose up` from within `mojito/docker/` or `docker-compose -f docker/docker-compose.yml up` from the +Run `docker-compose up` from within `mojito/docker/` or `docker-compose -f docker/docker-compose.yml up` from the project directory. It will start Mysql and build/start the Webapp. -In detached mode `docker compose up -d`. And to remove everything including volumes: `docker compose rm -s -v` to remove volumes. +In detached mode `docker compose up -d`. And to remove everything including volumes: `docker compose rm -s -v` to remove +volumes. To re-use a pre-built image, uncomment the `image` configuration in `docker-compose.yml`. @@ -39,18 +40,20 @@ services: Incompatibility Mac/Linux for `node/` `node_modules/`, remove the directories before calling docker commands. Some `Dockerfile` remove the directories explicitly. -# One-off build and push +# One-off build and push ## CLI image -Build: `docker build -t aurambaj/mojito-cli -f docker/Dockerfile-cli-bk8 .` and push: `docker push aurambaj/mojito-cli:latest` +Build: `docker build -t aurambaj/mojito-cli -f docker/Dockerfile-cli-bk8 .` and +push: `docker push aurambaj/mojito-cli:latest` -Example to run a command: +Example to run a command: `docker run --rm --name mojito-cli -it -e MOJITO_HOST="mojito.org" -e MOJITO_PORT="443" -e MOJITO_SCHEME="https" aurambaj/mojito-cli repo-view -n demo1` ## Webapp image -Build: `docker build -t aurambaj/mojito-webapp -f docker/Dockerfile-bk8 .` and push: `docker push aurambaj/mojito-webapp:latest` +Build: `docker build -t aurambaj/mojito-webapp -f docker/Dockerfile-bk8 .` and +push: `docker push aurambaj/mojito-webapp:latest` Start the webapp: `docker run --rm --name mojito-webapp -it aurambaj/mojito-webapp` and get a shell to try some command `docker exec -it mojito-webapp bash` @@ -87,7 +90,8 @@ This is not needed with Docker Desktop for Mac but was needed before so for the ### Find IP on Mac -Depending on how you instlled docker on Mac, `localhost` may not work. To get the IP to reach the service: `docker-machine ip default`. +Depending on how you instlled docker on Mac, `localhost` may not work. To get the IP to reach the +service: `docker-machine ip default`. `export l10n_resttemplate_host="192.168.99.111" ` and then whatever CLI command you have to run, eg. `mojito demo-create -n dockertest` @@ -113,7 +117,24 @@ services: - --innodb_use_native_aio=0 # needed on Mac ``` -## Alpine version +## Alpine version `FROM adoptopenjdk:8-jre` --> `FROM adoptopenjdk/openjdk8:alpine-jre` and change to `echo -e` for script generation -and make sure to use `/bin/echo` for consistent behavior between alpine and ubuntu. \ No newline at end of file +and make sure to use `/bin/echo` for consistent behavior between alpine and ubuntu. + +## Kubernetes + +Be aware that Kubernetes injects environment variables in the container, and that some could conflict with the +ones defined in the `Dockerfile`. For example, `MOJITO_PORT` was used in the `Dockerfile` but it is also injected by +Kubernetes with a value like `tcp://{ip}:{port}` which was making the CLI command fail in the container. Replaced +`MOJITO_PORT` with `MOJITO_SERVER_PORT` in the `Dockerfile` and the CLI command. + +### Port forwarding for MySQL + +First forward the pod port to the localhost port, then in the container use `socat` to forward pod port to the remote MySQL +server. Finally connect the SQL client to the localhost port: `33306`. + +``` +kubectl port-forward svc/mojito-webapp 33306:3306 +socat TCP-LISTEN:3306,reuseaddr,fork TCP:mojito-mysql-staging.mysql.database.azure.com:3306 +``` diff --git a/docs/_docs/guides/002-install_springboot3.md b/docs/_docs/guides/002-install_springboot3.md index 3a0247b399..1e23124afb 100644 --- a/docs/_docs/guides/002-install_springboot3.md +++ b/docs/_docs/guides/002-install_springboot3.md @@ -9,13 +9,13 @@ permalink: /docs/guides/install-springboot3/ No binaries are available for the `Spring Boot 3` at the moment. It should be built from the `master` branch. -Assuming `Java 17` is installed, `./mvnw install -DskipTests` should be enough to build the `jar` files. +Assuming `Java 21` is installed, `./mvnw install -DskipTests` should be enough to build the `jar` files. For detail instructions on development environment setup, [see here]({{ site.url }}/docs/guides/open-source-contributors/). ### Using Executable Jars -`Java 17` is required. +`Java 21` is required. Run the Webapp with: @@ -78,8 +78,8 @@ The port can be changed with the `server.port` property. ### MySQL -[Install MySQL 5.7](http://dev.mysql.com/doc/refman/5.7/en/installing.html) and then create a database for {{ site.mojito_green }} -(with Brew: `brew install mysql@5.7`). +[Install MySQL 8](https://dev.mysql.com/doc/mysql-installation-excerpt/8.0/en/) and then create a database for {{ site.mojito_green }} +(with Brew: `brew install mysql@8`). Connect to MySQL DB as root user diff --git a/docs/_docs/guides/authentication_springboot3.md b/docs/_docs/guides/authentication_springboot3.md index 1d50be9860..772bf4099e 100644 --- a/docs/_docs/guides/authentication_springboot3.md +++ b/docs/_docs/guides/authentication_springboot3.md @@ -32,9 +32,10 @@ If the redirect is enabled, it is still possible to access {{ site.mojito_green #### Example with GitHub -Create a `GitHub OAuth app` with `Authorization callback URL`: `http://localhost:8080/login/oauth`. +Create a `GitHub OAuth app` with `Authorization callback URL`: `http://localhost:8080/login/oauth2/code/github`. This URI maps to the `redirect_uri` in OAuth and to `preEstablishedRedirectUri` in Spring settings. - The `clientId` and `clientSecret` are available once the app has been created. + The `clientId` and `clientSecret` are available once the app has been created. The homepage URL should not matter +but can be set to: `http://localhost:8080/` Settings to be added, substituting the client `id` and `secret`: diff --git a/docs/_docs/guides/open-source-contributors.md b/docs/_docs/guides/open-source-contributors.md index 772acee666..b80c05a92b 100644 --- a/docs/_docs/guides/open-source-contributors.md +++ b/docs/_docs/guides/open-source-contributors.md @@ -32,13 +32,13 @@ Ensure that you have added the `cask-versions` [tap](https://github.com/Homebrew brew tap homebrew/cask-versions ``` -Install `java 17` from `Temurin`: +Install `java 21` from `Temurin`: ```sh brew install --cask temurin17 ``` -Check that `java 17` is now in use with `java -version`. If you need multiple versions of `java` consider using +Check that `java 21` is now in use with `java -version`. If you need multiple versions of `java` consider using something like `jenv` (don't forget the plugins: `jenv enable-plugin maven` & `jenv enable-plugin export` ) or `sdkman`. Install `maven` (latest version should be fine): @@ -58,13 +58,13 @@ Note, to install the exact same `maven` version as the wrapper: `brew install ma ### Install on Ubuntu 18.4 LTS -Install `java 17` from `OpenJDK`: +Install `java 21` from `OpenJDK`: ```sh -sudo apt-get install openjdk-17-jdk +sudo apt-get install openjdk-21-jdk ``` -Check that `java 17` is now in use with `java -version`. If not you, can set it as default with -`sudo update-java-alternatives -s /usr/lib/jvm/java-1.17.0-openjdk-amd64` (To check the list +Check that `java 21` is now in use with `java -version`. If not you, can set it as default with +`sudo update-java-alternatives -s /usr/lib/jvm/java-1.21.0-openjdk-amd64` (To check the list `sudo update-java-alternatives -l`). If you need multiple versions of `java` consider using something like `jenv` or `sdkman`. @@ -140,8 +140,8 @@ Run the following commands to create the databases. Change the user name, databa CREATE USER 'mojito'@'localhost' IDENTIFIED BY 'mojito'; CREATE DATABASE IF NOT EXISTS mojito CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_bin'; CREATE DATABASE IF NOT EXISTS mojito_dev CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_bin'; -GRANT ALL ON mojito.* TO 'mojito'@'localhost' IDENTIFIED BY 'mojito'; -GRANT ALL ON mojito_dev.* TO 'mojito'@'localhost' IDENTIFIED BY 'mojito'; +GRANT ALL PRIVILEGES ON mojito.* TO 'mojito'@'localhost'; +GRANT ALL PRIVILEGES ON mojito_dev.* TO 'mojito'@'localhost'; FLUSH PRIVILEGES; ``` diff --git a/idea.md b/idea.md index 3580e105be..f2a715870f 100644 --- a/idea.md +++ b/idea.md @@ -7,6 +7,7 @@ mkdir local/ cd local/ ln -s ~/.l10n/config ln -s ~/code/mojito-deploy/ +ln -s ~/tmp/mojito ``` ## Simplest way to get everything to work quickly (but slower that intellij incremental update) @@ -27,6 +28,12 @@ Some useful command This can be configured in Maven UI too. +## Re-indent list of commits + +``` +git rebase -i HEAD~20 --exec 'mvn spotless:apply && git add -A && git commit --amend --no-edit' +``` + ## Debug commands with Maven ### Debug the CLI diff --git a/pom.xml b/pom.xml index 531224df72..629f491990 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ 2.8.2 1.22.0 0.10.5 - 1.313 + 1.325 64.2 true true diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java b/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java new file mode 100644 index 0000000000..27986eb8a4 --- /dev/null +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiTranslateClient.java @@ -0,0 +1,39 @@ +package com.box.l10n.mojito.rest.client; + +import com.box.l10n.mojito.rest.entity.PollableTask; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** + * @author jaurambault + */ +@Component +public class RepositoryAiTranslateClient extends BaseClient { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(RepositoryAiTranslateClient.class); + + @Override + public String getEntityName() { + return "proto-ai-translate"; + } + + /** Ai translate untranslated and rejected strings in a repository for a given list of locales */ + public ProtoAiTranslateResponse translateRepository( + ProtoAiTranslateRequest protoAiTranslateRequest) { + + return authenticatedRestTemplate.postForObject( + getBasePathForEntity(), protoAiTranslateRequest, ProtoAiTranslateResponse.class); + } + + public record ProtoAiTranslateRequest( + String repositoryName, + List targetBcp47tags, + int sourceTextMaxCountPerLocale, + List tmTextUnitIds, + boolean useBatch) {} + + public record ProtoAiTranslateResponse(PollableTask pollableTask) {} +} diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/client/TextUnitClient.java b/restclient/src/main/java/com/box/l10n/mojito/rest/client/TextUnitClient.java new file mode 100644 index 0000000000..4abebde773 --- /dev/null +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/client/TextUnitClient.java @@ -0,0 +1,313 @@ +package com.box.l10n.mojito.rest.client; + +import com.box.l10n.mojito.rest.entity.PollableTask; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +@Component +public class TextUnitClient extends BaseClient { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(TextUnitClient.class); + + @Override + public String getEntityName() { + return "textunits"; + } + + public List searchTextUnits(TextUnitSearchBody textUnitSearchBody) { + TextUnit[] search = + authenticatedRestTemplate.postForObject( + getBasePathForEntity() + "/search", textUnitSearchBody, TextUnit[].class); + return Arrays.asList(search); + } + + public PollableTask importTextUnitBatch(ImportTextUnitsBatch importTextUnitsBatch) { + return authenticatedRestTemplate.postForObject( + getBasePath() + "/textunitsBatch", importTextUnitsBatch, PollableTask.class); + } + + public record ImportTextUnitsBatch( + boolean integrityCheckSkipped, + boolean integrityCheckKeepStatusIfFailedAndSameTarget, + List textUnits) {} + + public enum UsedFilter { + USED, + UNUSED + } + + public enum SearchType { + EXACT, + CONTAINS, + ILIKE + } + + public enum StatusFilter { + ALL, + TRANSLATED, + UNTRANSLATED, + TRANSLATED_AND_NOT_REJECTED, + APPROVED_OR_NEEDS_REVIEW_AND_NOT_REJECTED, + APPROVED_AND_NOT_REJECTED, + FOR_TRANSLATION, + REVIEW_NEEDED, + REVIEW_NOT_NEEDED, + TRANSLATION_NEEDED, + REJECTED, + NOT_REJECTED, + } + + public record TextUnit( + @JsonProperty("tmTextUnitId") Long tmTextUnitId, + @JsonProperty("tmTextUnitVariantId") Long tmTextUnitVariantId, + @JsonProperty("localeId") Long localeId, + @JsonProperty("name") String name, + @JsonProperty("source") String source, + @JsonProperty("comment") String comment, + @JsonProperty("target") String target, + @JsonProperty("targetLocale") String targetLocale, + @JsonProperty("targetComment") String targetComment, + @JsonProperty("assetId") Long assetId, + @JsonProperty("lastSuccessfulAssetExtractionId") Long lastSuccessfulAssetExtractionId, + @JsonProperty("assetExtractionId") Long assetExtractionId, + @JsonProperty("tmTextUnitCurrentVariantId") Long tmTextUnitCurrentVariantId, + @JsonProperty("status") Status status, + @JsonProperty("includedInLocalizedFile") Boolean includedInLocalizedFile, + @JsonProperty("createdDate") Long createdDate, + @JsonProperty("assetDeleted") Boolean assetDeleted, + @JsonProperty("pluralForm") String pluralForm, + @JsonProperty("pluralFormOther") String pluralFormOther, + @JsonProperty("repositoryName") String repositoryName, + @JsonProperty("assetPath") String assetPath, + @JsonProperty("assetTextUnitId") Long assetTextUnitId, + @JsonProperty("tmTextUnitCreatedDate") Long tmTextUnitCreatedDate, + @JsonProperty("doNotTranslate") Boolean doNotTranslate, + @JsonProperty("translated") Boolean translated, + @JsonProperty("used") Boolean used) { + + public TextUnit withTarget(String target, Status status) { + return new TextUnit( + tmTextUnitId, + tmTextUnitVariantId, + localeId, + name, + source, + comment, + target, + targetLocale, + targetComment, + assetId, + lastSuccessfulAssetExtractionId, + assetExtractionId, + tmTextUnitCurrentVariantId, + status, + includedInLocalizedFile, + createdDate, + assetDeleted, + pluralForm, + pluralFormOther, + repositoryName, + assetPath, + assetTextUnitId, + tmTextUnitCreatedDate, + doNotTranslate, + translated, + used); + } + } + + public enum Status { + TRANSLATION_NEEDED, + REVIEW_NEEDED, + APPROVED + } + + public static class TextUnitSearchBody { + List repositoryIds; + List repositoryNames; + List tmTextUnitIds; + String name; + String source; + String target; + String assetPath; + String pluralFormOther; + boolean pluralFormFiltered = true; + boolean pluralFormExcluded = false; + SearchType searchType = SearchType.EXACT; + List localeTags; + UsedFilter usedFilter; + StatusFilter statusFilter; + Boolean doNotTranslateFilter; + ZonedDateTime tmTextUnitCreatedBefore; + ZonedDateTime tmTextUnitCreatedAfter; + Long branchId; + Integer limit = 10; + Integer offset = 0; + + public List getRepositoryIds() { + return repositoryIds; + } + + public void setRepositoryIds(List repositoryIds) { + this.repositoryIds = repositoryIds; + } + + public List getRepositoryNames() { + return repositoryNames; + } + + public void setRepositoryNames(List repositoryNames) { + this.repositoryNames = repositoryNames; + } + + public List getTmTextUnitIds() { + return tmTextUnitIds; + } + + public void setTmTextUnitIds(List tmTextUnitIds) { + this.tmTextUnitIds = tmTextUnitIds; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getTarget() { + return target; + } + + public void setTarget(String target) { + this.target = target; + } + + public String getAssetPath() { + return assetPath; + } + + public void setAssetPath(String assetPath) { + this.assetPath = assetPath; + } + + public String getPluralFormOther() { + return pluralFormOther; + } + + public void setPluralFormOther(String pluralFormOther) { + this.pluralFormOther = pluralFormOther; + } + + public boolean isPluralFormFiltered() { + return pluralFormFiltered; + } + + public void setPluralFormFiltered(boolean pluralFormFiltered) { + this.pluralFormFiltered = pluralFormFiltered; + } + + public boolean isPluralFormExcluded() { + return pluralFormExcluded; + } + + public void setPluralFormExcluded(boolean pluralFormExcluded) { + this.pluralFormExcluded = pluralFormExcluded; + } + + public SearchType getSearchType() { + return searchType; + } + + public void setSearchType(SearchType searchType) { + this.searchType = searchType; + } + + public List getLocaleTags() { + return localeTags; + } + + public void setLocaleTags(List localeTags) { + this.localeTags = localeTags; + } + + public UsedFilter getUsedFilter() { + return usedFilter; + } + + public void setUsedFilter(UsedFilter usedFilter) { + this.usedFilter = usedFilter; + } + + public StatusFilter getStatusFilter() { + return statusFilter; + } + + public void setStatusFilter(StatusFilter statusFilter) { + this.statusFilter = statusFilter; + } + + public Boolean getDoNotTranslateFilter() { + return doNotTranslateFilter; + } + + public void setDoNotTranslateFilter(Boolean doNotTranslateFilter) { + this.doNotTranslateFilter = doNotTranslateFilter; + } + + public ZonedDateTime getTmTextUnitCreatedBefore() { + return tmTextUnitCreatedBefore; + } + + public void setTmTextUnitCreatedBefore(ZonedDateTime tmTextUnitCreatedBefore) { + this.tmTextUnitCreatedBefore = tmTextUnitCreatedBefore; + } + + public ZonedDateTime getTmTextUnitCreatedAfter() { + return tmTextUnitCreatedAfter; + } + + public void setTmTextUnitCreatedAfter(ZonedDateTime tmTextUnitCreatedAfter) { + this.tmTextUnitCreatedAfter = tmTextUnitCreatedAfter; + } + + public Long getBranchId() { + return branchId; + } + + public void setBranchId(Long branchId) { + this.branchId = branchId; + } + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + public Integer getOffset() { + return offset; + } + + public void setOffset(Integer offset) { + this.offset = offset; + } + } +} diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java index ff6e3e33a2..73b4d8b7eb 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/entity/IntegrityCheckerType.java @@ -19,5 +19,9 @@ public enum IntegrityCheckerType { HTML_TAG, ELLIPSIS, BACKQUOTE, - EMPTY_TARGET_NOT_EMPTY_SOURCE; + EMPTY_TARGET_NOT_EMPTY_SOURCE, + MARKDOWN_LINKS, + PYTHON_FPRINT, + EMAIL, + URL; } diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java b/restclient/src/main/java/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java index cc70321514..91b74f3708 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java @@ -6,6 +6,7 @@ import java.net.URI; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.hc.client5.http.cookie.Cookie; @@ -43,7 +44,7 @@ public class FormLoginAuthenticationCsrfTokenInterceptor implements ClientHttpRe public static final String CSRF_PARAM_NAME = "_csrf"; public static final String CSRF_HEADER_NAME = "X-CSRF-TOKEN"; - public static final String COOKIE_SESSION_NAME = "JSESSIONID"; + public static final Set COOKIE_SESSION_NAMES = Set.of("SESSION", "JSESSIONID"); @Autowired FormLoginConfig formLoginConfig; @@ -223,12 +224,13 @@ protected boolean doesSessionIdInCookieStoreExistAndMatchLatestSessionId() { } /** - * @return null if no sesson id cookie is found + * @return null if no session id cookie is found */ protected String getAuthenticationSessionIdFromCookieStore() { List cookies = cookieStore.getCookies(); + for (Cookie cookie : cookies) { - if (cookie.getName().equals(COOKIE_SESSION_NAME)) { + if (COOKIE_SESSION_NAMES.contains(cookie.getName())) { String cookieValue = cookie.getValue(); logger.debug("Found session cookie: {}", cookieValue); return cookieValue; diff --git a/restclient/src/test/java/com/box/l10n/mojito/rest/script/LeverageScript.java b/restclient/src/test/java/com/box/l10n/mojito/rest/script/LeverageScript.java new file mode 100644 index 0000000000..aa3a11f96a --- /dev/null +++ b/restclient/src/test/java/com/box/l10n/mojito/rest/script/LeverageScript.java @@ -0,0 +1,138 @@ +package com.box.l10n.mojito.rest.script; + +import com.box.l10n.mojito.rest.client.PollableTaskClient; +import com.box.l10n.mojito.rest.client.RepositoryClient; +import com.box.l10n.mojito.rest.client.TextUnitClient; +import com.box.l10n.mojito.rest.client.TextUnitClient.TextUnit; +import com.box.l10n.mojito.rest.client.TextUnitClient.TextUnitSearchBody; +import com.box.l10n.mojito.rest.client.exception.RepositoryNotFoundException; +import com.box.l10n.mojito.rest.entity.PollableTask; +import com.box.l10n.mojito.rest.resttemplate.AuthenticatedRestTemplateTest; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@Configuration +@SpringBootTest( + classes = { + AuthenticatedRestTemplateTest.class, + RepositoryClient.class, + TextUnitClient.class, + PollableTaskClient.class + }) +@TestPropertySource(locations = "file:/Users/ja/.l10n/config/script/application.properties") +@EnableConfigurationProperties +// @Ignore +public class LeverageScript { + + static Logger logger = LoggerFactory.getLogger(LeverageScript.class); + + @Autowired TextUnitClient textUnitClient; + + @Autowired PollableTaskClient pollableTaskClient; + + @Autowired RepositoryClient repositoryClient; + + @Test + public void script() throws RepositoryNotFoundException { + + var targetRepository = "ios"; + var sourceRepository = "zzzzzzzzz_ios"; + var skipSourceEqTarget = true; + var locales = List.of("fr"); + + List textUnitsToSave = new ArrayList<>(); + + for (String locale : locales) { + logger.info("Processing locale: {}", locale); + List untranslatedTextUnitsForLocale = + getUntranslatedTextUnitsForLocale(targetRepository, locale); + + for (TextUnit textUnit : untranslatedTextUnitsForLocale) { + List existingTranslationWithSameSource = + getTranslationsWithExactSource(sourceRepository, locale, textUnit.source()); + Optional match = + getMatchByNameAndComment( + textUnit.source(), textUnit.comment(), existingTranslationWithSameSource); + + if (match.isPresent()) { + logger.info( + "Found a match (tuvid: {}) by name and comment for:\n {}\ntranslation is: {}", + match.get().tmTextUnitVariantId(), + textUnit.source(), + match.get().target()); + } else { + match = getMatchByNewest(existingTranslationWithSameSource); + if (match.isPresent()) { + logger.info( + "Found match (tuvid: {}) by newest date for:\n {}\ntranslation is: {}", + match.get().tmTextUnitVariantId(), + textUnit.source(), + match.get().target()); + } + } + + if (match.isPresent()) { + logger.info("found match for source: {}", textUnit.source()); + var translation = match.get(); + if (skipSourceEqTarget && translation.source().equals(translation.target())) { + logger.info("skipping because source and target are the same"); + } + textUnitsToSave.add(textUnit.withTarget(translation.target(), translation.status())); + } else { + logger.info("No match found for source: {}", textUnit.source()); + } + } + } + + TextUnitClient.ImportTextUnitsBatch importTextUnitsBatch = + new TextUnitClient.ImportTextUnitsBatch(false, true, textUnitsToSave); + PollableTask pollableTask = textUnitClient.importTextUnitBatch(importTextUnitsBatch); + pollableTaskClient.waitForPollableTask(pollableTask.getId()); + } + + private List getTranslationsWithExactSource( + String repositoryName, String locale, String source) { + TextUnitSearchBody textUnitSearchBody = new TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.TRANSLATED); + textUnitSearchBody.setSource(source); + textUnitSearchBody.setLimit(20); + return textUnitClient.searchTextUnits(textUnitSearchBody); + } + + private List getUntranslatedTextUnitsForLocale(String repositoryName, String locale) { + TextUnitSearchBody textUnitSearchBody = new TextUnitSearchBody(); + textUnitSearchBody.setRepositoryNames(List.of(repositoryName)); + textUnitSearchBody.setLocaleTags(List.of(locale)); + textUnitSearchBody.setUsedFilter(TextUnitClient.UsedFilter.USED); + textUnitSearchBody.setStatusFilter(TextUnitClient.StatusFilter.UNTRANSLATED); + textUnitSearchBody.setLimit(1000); + return textUnitClient.searchTextUnits(textUnitSearchBody); + } + + private Optional getMatchByNewest(List candidates) { + return candidates.stream().max(Comparator.comparingLong(TextUnit::createdDate)); + } + + private static Optional getMatchByNameAndComment( + String name, String comment, List candidates) { + return candidates.stream() + .filter(m -> Objects.equals(name, m.name()) && Objects.equals(comment, m.comment())) + .max(Comparator.comparingLong(TextUnit::createdDate)); + } +} diff --git a/test-common/src/main/java/com/box/l10n/mojito/test/IOTestBase.java b/test-common/src/main/java/com/box/l10n/mojito/test/IOTestBase.java index 77086d6ec9..9de797bb72 100644 --- a/test-common/src/main/java/com/box/l10n/mojito/test/IOTestBase.java +++ b/test-common/src/main/java/com/box/l10n/mojito/test/IOTestBase.java @@ -254,9 +254,9 @@ protected void checkDirectoriesContainSameContent(File dir1, File dir2) // If that's a file, check that the both files have the same content if (file2.isFile() && !Files.equal(file1, file2)) { throw new DifferentDirectoryContentException( - "File: " + "File1: " + file1.toString() - + " and file: " + + " and file2: " + file2.toString() + " have different content." + "\n\nfile1 content:\n\n" diff --git a/webapp/package-lock.json b/webapp/package-lock.json index 59d242bee5..b99b572bbd 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -150,13 +150,15 @@ "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", "dev": true, "requires": { - "kind-of": "^6.0.3", + "kind-of": "^3.0.2", "longest": "^1.0.1", "repeat-string": "^1.5.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -293,11 +295,13 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -308,11 +312,13 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -325,17 +331,21 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^6.0.3" + "kind-of": "^5.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -425,11 +435,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -440,11 +452,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -457,11 +471,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -472,11 +488,13 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -488,7 +506,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "micromatch": { "version": "3.1.10", @@ -503,7 +523,7 @@ "extend-shallow": "^3.0.2", "extglob": "^2.0.4", "fragment-cache": "^0.2.1", - "kind-of": "^6.0.3", + "kind-of": "^6.0.2", "nanomatch": "^1.2.9", "object.pick": "^1.3.0", "regex-not": "^1.0.0", @@ -512,7 +532,9 @@ }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1022,7 +1044,9 @@ }, "dependencies": { "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "mkdirp": { "version": "0.5.5", @@ -1030,11 +1054,13 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -1693,7 +1719,9 @@ "dev": true }, "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "mkdirp": { "version": "0.5.5", @@ -1701,11 +1729,13 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -1835,11 +1865,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1850,11 +1882,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1867,11 +1901,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -1883,7 +1919,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -2561,17 +2599,21 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "normalize-path": { "version": "3.0.0", @@ -2649,18 +2691,22 @@ }, "dependencies": { "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "mkdirp": { "version": "0.5.0", "resolved": "https://box.jfrog.io/box/api/npm/boxnpm/mkdirp/-/mkdirp-0.5.0.tgz", "integrity": "sha1-HXMHam35hs2TROFecfzAWkyavxI=", "requires": { - "minimist": "^0.2.1" + "minimist": "0.0.8" }, "dependencies": { "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" } } } @@ -2707,7 +2753,7 @@ "requires": { "for-own": "^1.0.0", "is-plain-object": "^2.0.4", - "kind-of": "^6.0.3", + "kind-of": "^6.0.0", "shallow-clone": "^1.0.0" }, "dependencies": { @@ -2721,7 +2767,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -2870,13 +2918,16 @@ "dev": true, "optional": true, "requires": { - "ini": "^1.3.8", + "ini": "^1.3.4", "proto-list": "~1.2.1" }, "dependencies": { "ini": { - "version": "^1.3.8", - "dev": true + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "optional": true } } }, @@ -3425,11 +3476,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -3440,11 +3493,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -3457,11 +3512,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -3473,7 +3530,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -4358,7 +4417,9 @@ "optional": true }, "ini": { - "version": "^1.3.8" + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "is-fullwidth-code-point": { "version": "1.0.0", @@ -4385,7 +4446,9 @@ } }, "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" }, "minipass": { "version": "2.9.0", @@ -4412,12 +4475,15 @@ "dev": true, "optional": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", - "dev": true + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "dev": true, + "optional": true } } }, @@ -4566,18 +4632,24 @@ "optional": true, "requires": { "deep-extend": "^0.6.0", - "ini": "^1.3.8", - "minimist": "^0.2.1", + "ini": "~1.3.0", + "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" }, "dependencies": { "ini": { - "version": "^1.3.8", - "dev": true + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "optional": true }, "minimist": { - "version": "^0.2.1", - "dev": true + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", + "dev": true, + "optional": true } } }, @@ -5082,7 +5154,7 @@ "dev": true, "requires": { "is-number": "^3.0.0", - "kind-of": "^6.0.3" + "kind-of": "^4.0.0" }, "dependencies": { "is-number": { @@ -5091,17 +5163,21 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5661,11 +5737,13 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5722,11 +5800,13 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5746,11 +5826,13 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^6.0.3" + "kind-of": "^5.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -5960,12 +6042,14 @@ "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz", "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=", "requires": { - "node-fetch": "^2.6.1", + "node-fetch": "^1.0.1", "whatwg-fetch": ">=0.10.0" }, "dependencies": { "node-fetch": { - "version": "^2.6.1", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "requires": { "whatwg-url": "^5.0.0" } @@ -6108,7 +6192,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true }, "lazy-cache": { @@ -6170,17 +6256,21 @@ "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.0" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } }, "minimist": { - "version": "^0.2.1" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==" } } }, @@ -6380,7 +6470,7 @@ "decamelize": "^1.1.2", "loud-rejection": "^1.0.0", "map-obj": "^1.0.1", - "minimist": "^0.2.1", + "minimist": "^1.1.3", "normalize-package-data": "^2.3.4", "object-assign": "^4.0.1", "read-pkg-up": "^1.0.1", @@ -6389,7 +6479,9 @@ }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -6467,7 +6559,9 @@ } }, "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true }, "mixin-deep": { @@ -6515,11 +6609,13 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "^0.2.1" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { - "version": "^0.2.1", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.4.tgz", + "integrity": "sha512-Pkrrm8NjyQ8yVt8Am9M+yUt74zE3iokhzbG1bFVNjLB92vwM71hf40RkEsryg98BujhVOncKm/C1xROxZ030LQ==", "dev": true } } @@ -6565,7 +6661,7 @@ "extend-shallow": "^3.0.2", "fragment-cache": "^0.2.1", "is-windows": "^1.0.2", - "kind-of": "^6.0.3", + "kind-of": "^6.0.2", "object.pick": "^1.3.0", "regex-not": "^1.0.0", "snapdragon": "^0.8.1", @@ -6604,7 +6700,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -7022,7 +7120,7 @@ "requires": { "copy-descriptor": "^0.1.0", "define-property": "^0.2.5", - "kind-of": "^6.0.3" + "kind-of": "^3.0.3" }, "dependencies": { "define-property": { @@ -7035,7 +7133,9 @@ } }, "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8072,11 +8172,13 @@ "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8087,11 +8189,13 @@ "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8104,17 +8208,21 @@ "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", - "kind-of": "^6.0.3" + "kind-of": "^5.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -8204,11 +8312,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8219,11 +8329,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8236,11 +8348,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8251,11 +8365,13 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8267,7 +8383,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "micromatch": { "version": "3.1.10", @@ -8282,7 +8400,7 @@ "extend-shallow": "^3.0.2", "extglob": "^2.0.4", "fragment-cache": "^0.2.1", - "kind-of": "^6.0.3", + "kind-of": "^6.0.2", "nanomatch": "^1.2.9", "object.pick": "^1.3.0", "regex-not": "^1.0.0", @@ -8291,7 +8409,9 @@ }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8796,12 +8916,14 @@ "dev": true, "requires": { "is-extendable": "^0.1.1", - "kind-of": "^6.0.3", + "kind-of": "^5.0.0", "mixin-object": "^2.0.1" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8888,11 +9010,13 @@ "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8903,11 +9027,13 @@ "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^6.0.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8920,11 +9046,13 @@ "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.3" + "kind-of": "^6.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -8936,7 +9064,9 @@ "dev": true }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -8946,11 +9076,13 @@ "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.2.0" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -9593,11 +9725,13 @@ "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } @@ -9651,17 +9785,21 @@ "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, "requires": { - "kind-of": "^6.0.3" + "kind-of": "^3.0.2" }, "dependencies": { "kind-of": { - "version": "^6.0.3", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true } } }, "kind-of": { - "version": "^6.0.3" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -10173,7 +10311,7 @@ "string-width": "^1.0.2", "which-module": "^1.0.0", "y18n": "^3.2.1", - "yargs-parser": "18.1.2" + "yargs-parser": "^4.2.0" }, "dependencies": { "yargs-parser": { @@ -10325,7 +10463,7 @@ "string-width": "^1.0.2", "which-module": "^1.0.0", "y18n": "^3.2.1", - "yargs-parser": "18.1.2" + "yargs-parser": "^5.0.0" }, "dependencies": { "camelcase": { @@ -10366,4 +10504,4 @@ } } } -} \ No newline at end of file +} diff --git a/webapp/package.json b/webapp/package.json index f6104db8f4..6c8fb88698 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -60,6 +60,7 @@ "start-server-nofe": "mvn spring-boot:run -P '!frontend' -Dspring-boot.run.jvmArguments=\"-Dspring.config.additional-location=optional:file://$HOME/.l10n/config/webapp/ -Dspring.profiles.active=$USER,npm -Duser.timezone=UTC\"", "start-server-nofe-debug": "mvn spring-boot:run -P '!frontend' -Dspring-boot.run.jvmArguments=\"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005 -Dspring.config.additional-location=optional:file://$HOME/.l10n/config/webapp/ -Dspring.profiles.active=$USER,npm -Duser.timezone=UTC\"", "start-server-nofe-debugn": "mvn spring-boot:run -P '!frontend' -Dspring-boot.run.jvmArguments=\"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -Dspring.config.additional-location=optional:file://$HOME/.l10n/config/webapp/ -Dspring.profiles.active=$USER,npm -Duser.timezone=UTC\"", + "start-server-nofe-dep": "cd .. && mvn install -DskipTests -P'!frontend' && cd webapp && mvn spring-boot:run -P '!frontend' -Dspring-boot.run.jvmArguments=\"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -Dspring.config.additional-location=optional:file://$HOME/.l10n/config/webapp/ -Dspring.profiles.active=$USER,npm -Duser.timezone=UTC\"", "watch": "webpack --config webpack.config.js --progress --colors --watch --env.inlineSourceMap", "watch-ict": "webpack --config webpack.config_ict.js --progress --colors --watch --env.inlineSourceMap", "build": "webpack --config webpack.config.js --progress --env.production", diff --git a/webapp/pom.xml b/webapp/pom.xml index 3a43625de3..0bc0b1cfe4 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -40,6 +40,11 @@ spring-boot-starter-web + + org.springframework.boot + spring-boot-starter-oauth2-client + + org.springframework.boot spring-boot-starter-mustache @@ -55,6 +60,12 @@ spring-boot-starter-actuator + + com.phrase + phrase-java + 2.0.2 + + org.springframework.boot spring-boot-starter-data-jpa diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java index 646527ebb4..ca846803cf 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidPlural.java @@ -46,7 +46,7 @@ private Map buildItemMap( throw new AndroidPluralDuplicateKeyException( "A duplicate was found when building an Android Plural. " + androidPluralItem - + " dupplicates: " + + " duplicates: " + androidPluralItem2); })); } @@ -69,6 +69,10 @@ public AndroidPluralBuilder addItem(AndroidPluralItem item) { return this; } + public String getName() { + return name; + } + public AndroidPluralBuilder setName(String name) { this.name = name; return this; @@ -79,6 +83,12 @@ public AndroidPluralBuilder setComment(String comment) { return this; } + public List getSortedItems() { + return items.stream() + .sorted(Comparator.comparing(item -> item.getQuantity().ordinal())) + .collect(Collectors.toList()); + } + public AndroidPlural build() { return new AndroidPlural(name, comment, items); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java index 5919a27bf5..4ae2ebb2a8 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java @@ -1,10 +1,7 @@ package com.box.l10n.mojito.android.strings; -import static com.google.common.base.Preconditions.checkArgument; - import java.util.ArrayList; import java.util.List; -import org.w3c.dom.Node; public class AndroidStringDocument { @@ -29,46 +26,8 @@ public void addSingular(AndroidSingular string) { strings.add(string); } - public void addSingular(AndroidStringElement element, Node comment) { - - checkArgument(element.isSingular(), "element should be singular"); - - addSingular( - new AndroidSingular( - element.getIdAttribute(), - element.getNameAttribute(), - element.getUnescapedContent(), - comment != null ? comment.getTextContent() : null)); - } - public void addPlural(AndroidPlural plural) { plurals.add(plural); strings.add(plural); } - - public void addPlural(AndroidStringElement element, Node comment) { - - checkArgument(element.isPlural(), "element should be plural"); - - AndroidPlural.AndroidPluralBuilder builder = AndroidPlural.builder(); - builder.setName(element.getNameAttribute()); - if (comment != null) { - builder.setComment(comment.getTextContent()); - } - - element.forEachPluralItem(builder::addItem); - - addPlural(builder.build()); - } - - public void addElement(Node node, Node comment) { - - AndroidStringElement element = new AndroidStringElement(node); - - if (element.isSingular()) { - addSingular(element, comment); - } else if (element.isPlural()) { - addPlural(element, comment); - } - } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java index f13abd183b..0cc53bbc0e 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java @@ -6,9 +6,12 @@ import com.box.l10n.mojito.service.tm.search.TextUnitDTO; import com.google.common.base.CharMatcher; import com.google.common.base.Strings; +import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -29,15 +32,29 @@ public class AndroidStringDocumentMapper { private final String locale; private final String repositoryName; private final PluralNameParser pluralNameParser; + private final boolean addTextUnitIdInName; + private final Map pluralFormToCommaId; public AndroidStringDocumentMapper( - String pluralSeparator, String assetDelimiter, String locale, String repositoryName) { + String pluralSeparator, + String assetDelimiter, + String locale, + String repositoryName, + boolean addTextUnitIdInName, + Map pluralFormToCommaId) { this.pluralSeparator = pluralSeparator; this.assetDelimiter = Optional.ofNullable(Strings.emptyToNull(assetDelimiter)).orElse(DEFAULT_ASSET_DELIMITER); this.locale = locale; this.repositoryName = repositoryName; this.pluralNameParser = new PluralNameParser(); + this.addTextUnitIdInName = addTextUnitIdInName; + this.pluralFormToCommaId = pluralFormToCommaId; + } + + public AndroidStringDocumentMapper( + String pluralSeparator, String assetDelimiter, String locale, String repositoryName) { + this(pluralSeparator, assetDelimiter, locale, repositoryName, false, null); } public AndroidStringDocumentMapper(String pluralSeparator, String assetDelimiter) { @@ -85,7 +102,23 @@ public AndroidStringDocument readFromTextUnits(List textUnits, bool } } - pluralByOther.forEach((pluralFormOther, builder) -> document.addPlural(builder.build())); + pluralByOther.forEach( + (pluralFormOther, builder) -> { + if (addTextUnitIdInName) { + String ids; + if (pluralFormToCommaId != null) { + ids = this.pluralFormToCommaId.get(pluralFormOther); + } else { + ids = + builder.getSortedItems().stream() + .map(AndroidPluralItem::getId) + .map(Objects::toString) + .collect(Collectors.joining(",")); + } + builder.setName(ids + "#@#" + builder.getName()); + } + document.addPlural(builder.build()); + }); return document; } @@ -117,11 +150,34 @@ TextUnitDTO addTextUnitDTOAttributes(TextUnitDTO textUnit) { } if (textUnit.getName().contains(assetDelimiter)) { - String[] nameParts = textUnit.getName().split(assetDelimiter, 2); - if (nameParts.length > 1) { - textUnit.setAssetPath(nameParts[0]); - textUnit.setName(nameParts[1]); + if (addTextUnitIdInName) { + String[] nameParts = textUnit.getName().split(assetDelimiter, 3); + if (nameParts.length > 2) { + + if (textUnit.getPluralForm() == null) { + textUnit.setTmTextUnitId(Long.valueOf(nameParts[0])); + } else { + String idsString = nameParts[0]; + List ids = Arrays.stream(idsString.split(",")).map(Long::valueOf).toList(); + if (ids.size() != 6) { + throw new RuntimeException( + "There must be 6 ids, check that all the required text units are provided (typically TextUnitSearcherParameters#setPluralFormsFiltered(false)"); + } + + AndroidPluralQuantity androidPluralQuantity = + AndroidPluralQuantity.valueOf(textUnit.getPluralForm().toUpperCase(Locale.ROOT)); + textUnit.setTmTextUnitId(ids.get(androidPluralQuantity.ordinal())); + } + textUnit.setAssetPath(nameParts[1]); + textUnit.setName(nameParts[2]); + } + } else { + String[] nameParts = textUnit.getName().split(assetDelimiter, 2); + if (nameParts.length > 1) { + textUnit.setAssetPath(nameParts[0]); + textUnit.setName(nameParts[1]); + } } } @@ -142,11 +198,10 @@ Stream stringToTextUnits(AbstractAndroidString androidString) { Stream singularToTextUnit(AndroidSingular singular) { TextUnitDTO textUnit = new TextUnitDTO(); - textUnit.setName(singular.getName()); textUnit.setComment(singular.getComment()); textUnit.setTmTextUnitId(singular.getId()); - textUnit.setTarget(unescape(singular.getContent())); + textUnit.setTarget(singular.getContent()); addTextUnitDTOAttributes(textUnit); return Stream.of(textUnit); @@ -170,7 +225,7 @@ Stream pluralToTextUnits(AndroidPlural plural) { textUnit.setTmTextUnitId(item.getId()); textUnit.setPluralForm(quantity); textUnit.setPluralFormOther(pluralFormOther); - textUnit.setTarget(unescape(item.getContent())); + textUnit.setTarget(item.getContent()); addTextUnitDTOAttributes(textUnit); return textUnit; @@ -181,18 +236,34 @@ boolean isSingularTextUnit(TextUnitDTO textUnit) { return Strings.isNullOrEmpty(textUnit.getPluralForm()); } - String getKeyToGroupByPluralOtherAndComment(TextUnitDTO textUnit) { + public static String getKeyToGroupByPluralOtherAndComment(TextUnitDTO textUnit) { return textUnit.getAssetPath() + DEFAULT_ASSET_DELIMITER + textUnit.getPluralFormOther() + "_" - + textUnit.getComment(); + + textUnit.getComment() + + "-" + // Support plural strings with the same name but different content across branches + // + // Under normal circumstances, there can only be one active text unit for a given name. + // However, with multiple branches, it's possible to have multiple text units with the + // same name. While this isn't an issue for singular strings, it becomes problematic for + // plural strings, as we group by name when building the AndroidStringDocument. This can + // lead to too many or duplicated entries for one or more plural forms. + // + // To resolve this, we now include the branch ID in the group by logic. This ensures + // that plural strings are properly grouped, allowing for correct document generation. + + textUnit.getBranchId(); } AndroidSingular textUnitToAndroidSingular(TextUnitDTO textUnit, boolean useSource) { + return new AndroidSingular( textUnit.getTmTextUnitId(), - textUnit.getAssetPath() + assetDelimiter + textUnit.getName(), + (addTextUnitIdInName ? textUnit.getTmTextUnitId() + assetDelimiter : "") + + textUnit.getAssetPath() + + assetDelimiter + + textUnit.getName(), removeInvalidControlCharacter( Strings.nullToEmpty(useSource ? textUnit.getSource() : textUnit.getTarget())), removeInvalidControlCharacter(textUnit.getComment())); @@ -214,12 +285,4 @@ static String removeInvalidControlCharacter(String str) { return withoutControlCharacters; } - - static String unescape(String str) { - return Strings.nullToEmpty(str) - .replaceAll("\\\\'", "'") - .replaceAll("\\\\\"", "\"") - .replaceAll("\\\\@", "@") - .replaceAll("\\\\n", "\n"); - } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java index b5ea19e6a6..ebfd371e2d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReader.java @@ -7,24 +7,30 @@ import java.io.IOException; import java.io.StringReader; import java.util.LinkedList; +import java.util.Optional; import java.util.Queue; -import javax.xml.parsers.ParserConfigurationException; +import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; import org.w3c.dom.Document; +import org.w3c.dom.Element; import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; public class AndroidStringDocumentReader { - public static AndroidStringDocument fromFile(String fileName) - throws ParserConfigurationException, IOException, SAXException { + public static AndroidStringDocument fromFile(String fileName) throws IOException, SAXException { return buildFromDocument(documentBuilder().parse(new File(fileName))); } - public static AndroidStringDocument fromText(String text) - throws ParserConfigurationException, IOException, SAXException { - return buildFromDocument( - documentBuilder().parse(new InputSource(new StringReader(Strings.nullToEmpty(text))))); + public static AndroidStringDocument fromText(String text) { + try { + return buildFromDocument( + documentBuilder().parse(new InputSource(new StringReader(Strings.nullToEmpty(text))))); + } catch (SAXException | IOException e) { + throw new AndroidStringDocumentReaderException(e); + } } private static AndroidStringDocument buildFromDocument(Document source) { @@ -41,10 +47,90 @@ private static AndroidStringDocument buildFromDocument(Document source) { commentNodes.offer(node); } else if (Node.ELEMENT_NODE == node.getNodeType()) { - document.addElement(node, commentNodes.poll()); + Element nodeAsElement = (Element) node; + Node comment = commentNodes.poll(); + + if (nodeAsElement.getTagName().equals(AndroidStringElement.SINGULAR_ELEMENT_NAME)) { + document.addSingular( + new AndroidSingular( + getAttributeAs( + nodeAsElement, AndroidStringElement.ID_ATTRIBUTE_NAME, Long::valueOf), + getNameAttribute(nodeAsElement), + unescape(nodeAsElement.getTextContent()), + comment != null ? comment.getTextContent() : null)); + } else if (nodeAsElement.getTagName().equals(AndroidStringElement.PLURAL_ELEMENT_NAME)) { + + AndroidPlural.AndroidPluralBuilder builder = AndroidPlural.builder(); + + builder.setName(nodeAsElement.getAttribute(AndroidStringElement.NAME_ATTRIBUTE_NAME)); + if (comment != null) { + builder.setComment(comment.getTextContent()); + } + + NodeList nodeList = + nodeAsElement.getElementsByTagName(AndroidStringElement.PLURAL_ITEM_ELEMENT_NAME); + + for (int j = 0; j < nodeList.getLength(); j++) { + Element item = (Element) nodeList.item(j); + + builder.addItem( + new AndroidPluralItem( + getAttribute(item, AndroidStringElement.QUANTITY_ATTRIBUTE_NAME), + getAttributeAs(item, AndroidStringElement.ID_ATTRIBUTE_NAME, Long::valueOf), + unescape(getTextContent(item)))); + } + + document.addPlural(builder.build()); + } } } return document; } + + public static String getAttribute(Element element, String name) { + // looks like ofNullable is useless and then the orElse + return Optional.ofNullable(element.getAttribute(name)).orElse(""); + } + + public static T getAttributeAs( + Element element, String attributeName, Function converter) { + + T result = null; + + if (element.hasAttribute(attributeName)) { + result = converter.apply(element.getAttribute(attributeName)); + } + + return result; + } + + public static String getTextContent(Element element) { + return Optional.ofNullable(element).map(Element::getTextContent).orElse(""); + } + + public static String getNameAttribute(Element element) { + return element.getAttribute(AndroidStringElement.NAME_ATTRIBUTE_NAME); + } + + /** should use {@link com.box.l10n.mojito.okapi.filters.AndroidFilter#unescape(String)} */ + static String unescape(String str) { + + String unescape = str; + + if (!Strings.isNullOrEmpty(str)) { + + if (StringUtils.startsWith(unescape, "\"") && StringUtils.endsWith(unescape, "\"")) { + unescape = unescape.substring(1, unescape.length() - 1); + } + + unescape = + Strings.nullToEmpty(unescape) + .replaceAll("\\\\'", "'") + .replaceAll("\\\\\"", "\"") + .replaceAll("\\\\@", "@") + .replaceAll("\\\\n", "\n"); + } + return unescape; + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReaderException.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReaderException.java new file mode 100644 index 0000000000..6d8b4bf787 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentReaderException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.android.strings; + +public class AndroidStringDocumentReaderException extends RuntimeException { + public AndroidStringDocumentReaderException(Exception e) { + super(e); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentUtils.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentUtils.java index cee751111e..1263050d3a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentUtils.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentUtils.java @@ -11,8 +11,12 @@ public final class AndroidStringDocumentUtils { DocumentBuilderFactory.newInstance(); static final TransformerFactory TRANSFORMER_FACTORY = TransformerFactory.newInstance(); - static DocumentBuilder documentBuilder() throws ParserConfigurationException { - return DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); + static DocumentBuilder documentBuilder() { + try { + return DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } } private AndroidStringDocumentUtils() { diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java index 0d02d862d4..ec23b2ebb7 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriter.java @@ -17,9 +17,9 @@ import java.io.IOException; import java.io.StringWriter; import java.io.Writer; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; @@ -33,14 +33,19 @@ public class AndroidStringDocumentWriter { private DOMSource domSource; private Document document; private Node root; + private EscapeType escapeType; - public AndroidStringDocumentWriter(final AndroidStringDocument source) - throws ParserConfigurationException { + public AndroidStringDocumentWriter(final AndroidStringDocument source) { + this(source, EscapeType.QUOTE_AND_NEW_LINE); + } + + public AndroidStringDocumentWriter(final AndroidStringDocument source, EscapeType escapeType) { this.source = requireNonNull(source); + this.escapeType = escapeType; buildDomSource(); } - public void buildDomSource() throws ParserConfigurationException { + public void buildDomSource() { document = documentBuilder().newDocument(); root = document.createElement(ROOT_ELEMENT_NAME); document.setXmlStandalone(true); @@ -49,20 +54,29 @@ public void buildDomSource() throws ParserConfigurationException { domSource = new DOMSource(document); } - private W buildWriter(W writer) throws TransformerException { + private W buildWriter(W writer) { - Transformer transformer = TRANSFORMER_FACTORY.newTransformer(); + Transformer transformer = null; + try { + transformer = TRANSFORMER_FACTORY.newTransformer(); + } catch (TransformerConfigurationException e) { + throw new RuntimeException(e); + } transformer.setOutputProperty(OutputKeys.INDENT, "yes"); transformer.setOutputProperty(OutputKeys.METHOD, "xml"); transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "0"); - transformer.transform(domSource, new StreamResult(writer)); + try { + transformer.transform(domSource, new StreamResult(writer)); + } catch (TransformerException e) { + throw new RuntimeException(e); + } return writer; } - public String toText() throws TransformerException { + public String toText() { return buildWriter(new StringWriter()).toString(); } @@ -123,7 +137,7 @@ private Element addChild(Node node, String name) { private void setContent(Element element, String content) { if (!Strings.isNullOrEmpty(content)) { - element.setTextContent(escapeQuotes(content)); + element.setTextContent(escapeQuotes(content, escapeType)); } } @@ -145,7 +159,22 @@ private void setAttribute(Element element, String name, String value) { } } - static String escapeQuotes(String str) { - return Strings.nullToEmpty(str).replaceAll("\"", "\\\\\"").replaceAll("\n", "\\\\n"); + static String escapeQuotes(String str, EscapeType escapeType) { + String escaped = str; + if (!Strings.isNullOrEmpty(str)) { + escaped = + switch (escapeType) { + case QUOTE_AND_NEW_LINE -> str.replaceAll("\"", "\\\\\"").replaceAll("\n", "\\\\n"); + case NEW_LINE -> str.replaceAll("\n", "\\\\n"); + case NONE -> str; + }; + } + return escaped; + } + + public enum EscapeType { + QUOTE_AND_NEW_LINE, // Legacy + NEW_LINE, + NONE } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringElement.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringElement.java index 4833b113f6..96fd248fdf 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringElement.java +++ b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringElement.java @@ -1,13 +1,5 @@ package com.box.l10n.mojito.android.strings; -import com.google.common.base.Strings; -import java.util.Optional; -import java.util.function.Consumer; -import java.util.function.Function; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - public class AndroidStringElement { static final String ROOT_ELEMENT_NAME = "resources"; @@ -17,88 +9,4 @@ public class AndroidStringElement { static final String NAME_ATTRIBUTE_NAME = "name"; static final String QUANTITY_ATTRIBUTE_NAME = "quantity"; static final String ID_ATTRIBUTE_NAME = "tmTextUnitId"; - - private final Element element; - - public AndroidStringElement(Element element) { - this.element = element; - } - - public AndroidStringElement(Node node) { - this((Element) node); - } - - public boolean isSingular() { - return element.getTagName().equals(SINGULAR_ELEMENT_NAME); - } - - public boolean isPlural() { - return element.getTagName().equals(PLURAL_ELEMENT_NAME); - } - - public boolean isPluralItem() { - return element.getTagName().equals(PLURAL_ITEM_ELEMENT_NAME); - } - - public String getTextContent() { - return Optional.ofNullable(element).map(Element::getTextContent).orElse(""); - } - - public Long getIdAttribute() { - return getLongAttribute(ID_ATTRIBUTE_NAME); - } - - public String getNameAttribute() { - return getAttribute(NAME_ATTRIBUTE_NAME); - } - - public String getAttribute(String name) { - return Optional.ofNullable(element.getAttribute(name)).orElse(""); - } - - public String getUnescapedContent() { - return removeEscape(element.getTextContent()); - } - - public Long getLongAttribute(String attributeName) { - return getAttributeAs(attributeName, Long::valueOf); - } - - public T getAttributeAs(String attributeName, Function converter) { - - T result = null; - - if (element.hasAttribute(attributeName)) { - result = converter.apply(element.getAttribute(attributeName)); - } - - return result; - } - - public void forEachPluralItem(Consumer consumer) { - - AndroidStringElement node; - NodeList nodeList = element.getElementsByTagName(PLURAL_ITEM_ELEMENT_NAME); - - for (int i = 0; i < nodeList.getLength(); i++) { - - node = new AndroidStringElement((Element) nodeList.item(i)); - - if (node.isPluralItem()) { - consumer.accept( - new AndroidPluralItem( - node.getAttribute(QUANTITY_ATTRIBUTE_NAME), - node.getLongAttribute(ID_ATTRIBUTE_NAME), - node.getTextContent())); - } - } - } - - private static String removeEscape(String str) { - return Strings.nullToEmpty(str) - .replaceAll("\\\\'", "'") - .replaceAll("\\\\\"", "\"") - .replace("\\\\\n", "\n") - .replaceAll("\\\\@", "@"); - } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java b/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java index 81dabc59aa..5a6c9297d2 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java +++ b/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java @@ -33,14 +33,21 @@ @NamedEntityGraph( name = "Drop.legacy", attributeNodes = { - @NamedAttributeNode("repository"), + @NamedAttributeNode(value = "repository", subgraph = "Drop.legacy.repository"), @NamedAttributeNode(value = "importPollableTask", subgraph = "Drop.legacy.subTask"), - @NamedAttributeNode(value = "exportPollableTask", subgraph = "Drop.legacy.subTask") + @NamedAttributeNode(value = "exportPollableTask", subgraph = "Drop.legacy.subTask"), + @NamedAttributeNode(value = "translationKits", subgraph = "Drop.legacy.translationKits") }, subgraphs = { @NamedSubgraph( name = "Drop.legacy.subTask", - attributeNodes = {@NamedAttributeNode("subTasks")}) + attributeNodes = {@NamedAttributeNode("subTasks")}), + @NamedSubgraph( + name = "Drop.legacy.repository", + attributeNodes = {@NamedAttributeNode("createdByUser")}), + @NamedSubgraph( + name = "Drop.legacy.translationKits", + attributeNodes = {@NamedAttributeNode("locale")}) }) public class Drop extends AuditableEntity { diff --git a/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java b/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java index 778c092262..c02ade6e2a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java +++ b/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java @@ -5,16 +5,25 @@ import com.box.l10n.mojito.entity.TMTextUnit; import com.box.l10n.mojito.entity.TMTextUnitCurrentVariant; import com.box.l10n.mojito.entity.TMTextUnitVariant; +import com.box.l10n.mojito.entity.TMTextUnitVariantComment; import com.box.l10n.mojito.service.NormalizationUtils; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckException; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckerFactory; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TMTextUnitVariantCommentAnnotation; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TMTextUnitVariantCommentAnnotations; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TextUnitIntegrityChecker; import com.box.l10n.mojito.service.tm.TranslatorWithInheritance; import com.box.l10n.mojito.service.tm.search.TextUnitDTO; import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; +import java.time.ZonedDateTime; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import net.sf.okapi.common.Event; import net.sf.okapi.common.resource.TextContainer; import org.slf4j.Logger; @@ -33,6 +42,8 @@ public class ImportTranslationsFromLocalizedAssetStep extends AbstractImportTran @Autowired TextUnitSearcher textUnitSearcher; + @Autowired IntegrityCheckerFactory integrityCheckerFactory; + Asset asset; RepositoryLocale repositoryLocale; StatusForEqualTarget statusForEqualTarget; @@ -43,6 +54,8 @@ public class ImportTranslationsFromLocalizedAssetStep extends AbstractImportTran TranslatorWithInheritance translatorWithInheritance; + private Set textUnitIntegrityCheckers = new HashSet<>(); + boolean hasTranslationWithoutInheritance; public enum StatusForEqualTarget { @@ -67,6 +80,17 @@ protected Event handleStartDocument(Event event) { translatorWithInheritance = new TranslatorWithInheritance(asset, repositoryLocale, InheritanceMode.USE_PARENT); hasTranslationWithoutInheritance = translatorWithInheritance.hasTranslationWithoutInheritance(); + + textUnitIntegrityCheckers = integrityCheckerFactory.getTextUnitCheckers(asset); + if (textUnitIntegrityCheckers.isEmpty()) { + logger.debug("There is no integrity checkers for asset id {}", asset.getId()); + } else { + logger.debug( + "Found {} integrity checker(s) for asset id {}", + textUnitIntegrityCheckers.size(), + asset.getId()); + } + return event; } @@ -117,6 +141,35 @@ void initTextUnitsMapByName() { } } + @Override + protected TMTextUnitVariant importTextUnit( + TMTextUnit tmTextUnit, + TextContainer target, + TMTextUnitVariant.Status status, + ZonedDateTime createdDate) { + + for (TextUnitIntegrityChecker textUnitIntegrityChecker : textUnitIntegrityCheckers) { + try { + textUnitIntegrityChecker.check(tmTextUnit.getContent(), target.toString()); + } catch (IntegrityCheckException integrityCheckException) { + TMTextUnitVariantCommentAnnotation tmTextUnitVariantCommentAnnotation = + new TMTextUnitVariantCommentAnnotation(); + tmTextUnitVariantCommentAnnotation.setCommentType( + TMTextUnitVariantComment.Type.INTEGRITY_CHECK); + + tmTextUnitVariantCommentAnnotation.setMessage(integrityCheckException.getMessage()); + + tmTextUnitVariantCommentAnnotation.setSeverity( + TMTextUnitVariantComment.Severity.ERROR); // TODO(ja) dial it down for plural strings? + + new TMTextUnitVariantCommentAnnotations(target) + .addAnnotation(tmTextUnitVariantCommentAnnotation); + } + } + + return super.importTextUnit(tmTextUnit, target, status, createdDate); + } + void initTextUnitsMapByMd5() { logger.debug("initTextUnitsMapByMd5"); List textUnits = tmTextUnitRepository.findByAsset(asset); diff --git a/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java b/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java index 8ade9e401d..cf9ec7afd4 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java +++ b/webapp/src/main/java/com/box/l10n/mojito/okapi/TranslateStep.java @@ -43,7 +43,6 @@ public class TranslateStep extends AbstractMd5ComputationStep { Status status; InheritanceMode inheritanceMode; RawDocument rawDocument; - boolean rawDocumentProcessingEnabled = false; boolean saveUsedTmTextUnitVariantIds = false; /** @@ -137,22 +136,20 @@ protected Event handleTextUnit(Event event) { if (translation == null && InheritanceMode.REMOVE_UNTRANSLATED.equals(inheritanceMode)) { logger.debug("Remove untranslated text unit"); - Event androidEvent = getEventForAndroidFirstTextUnit(textUnit); - - if (androidEvent == null) { - switch (getRemoveUntranslatedStrategyFromAnnotation()) { - case NOOP_EVENT: - event = Event.createNoopEvent(); - break; - case PLACEHOLDER_AND_POST_PROCESSING: - logger.debug("Set untranslated placeholder for text unit with name: {}", name); - textUnit.setTarget( - targetLocale, - new TextContainer(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)); - enableOutputDocumentPostProcessing(); - break; - } + + switch (getRemoveUntranslatedStrategyFromAnnotation()) { + case NOOP_EVENT: + event = Event.createNoopEvent(); + break; + case PLACEHOLDER_AND_POST_PROCESSING: + logger.debug("Set untranslated placeholder for text unit with name: {}", name); + textUnit.setTarget( + targetLocale, + new TextContainer(RemoveUntranslatedStrategy.UNTRANSLATED_PLACEHOLDER)); + setOutputDocumentPostProcessingRemoveUntranslated(); + break; } + } else { if (!shouldConvertToHtmlCodes) { @@ -197,12 +194,11 @@ protected Event handleDocumentPart(Event event) { return event; } - void enableOutputDocumentPostProcessing() { + void setOutputDocumentPostProcessingRemoveUntranslated() { OutputDocumentPostProcessingAnnotation annotation = rawDocument.getAnnotation(OutputDocumentPostProcessingAnnotation.class); - if (annotation != null && !rawDocumentProcessingEnabled) { - annotation.setEnabled(true); - rawDocumentProcessingEnabled = true; + if (annotation != null && annotation.getOutputDocumentPostProcessor() != null) { + annotation.getOutputDocumentPostProcessor().setRemoveUntranslated(true); } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/asset/AssetWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/asset/AssetWS.java index 04dff59e8d..c9d1c5fc9d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/rest/asset/AssetWS.java +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/asset/AssetWS.java @@ -124,7 +124,7 @@ public SourceAsset importSourceAsset(@RequestBody SourceAsset sourceAsset) throw Repository repository = repositoryRepository - .findById(sourceAsset.getRepositoryId()) + .findNoGraphById(sourceAsset.getRepositoryId()) .orElseThrow( () -> new RepositoryWithIdNotFoundException(sourceAsset.getRepositoryId())); diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewConfigurationProperties.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewConfigurationProperties.java new file mode 100644 index 0000000000..11584eae63 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewConfigurationProperties.java @@ -0,0 +1,18 @@ +package com.box.l10n.mojito.rest.textunit; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties("l10n.ai-review") +public class AiReviewConfigurationProperties { + String openaiClientToken; + + public String getOpenaiClientToken() { + return openaiClientToken; + } + + public void setOpenaiClientToken(String openaiClientToken) { + this.openaiClientToken = openaiClientToken; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewWS.java new file mode 100644 index 0000000000..28ccd9a962 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewWS.java @@ -0,0 +1,189 @@ +package com.box.l10n.mojito.rest.textunit; + +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.JsonFormat.JsonSchema.createJsonSchema; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.SystemMessage.systemMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.UserMessage.userMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.chatCompletionsRequest; + +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.openai.OpenAIClient; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class AiReviewWS { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(AiReviewWS.class); + + TextUnitSearcher textUnitSearcher; + + AiReviewConfigurationProperties aiReviewConfigurationProperties; + + public AiReviewWS( + TextUnitSearcher textUnitSearcher, + AiReviewConfigurationProperties aiReviewConfigurationProperties) { + this.textUnitSearcher = textUnitSearcher; + this.aiReviewConfigurationProperties = aiReviewConfigurationProperties; + } + + @RequestMapping(method = RequestMethod.GET, value = "/api/proto-ai-review") + @ResponseStatus(HttpStatus.OK) + public ProtoAiReviewResponse getTextUnitsWithGet(ProtoAiReviewRequest protoAiReviewRequest) { + + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setTmTextUnitVariantId(protoAiReviewRequest.tmTextUnitVariantId); + + List search = textUnitSearcher.search(textUnitSearcherParameters); + if (search.isEmpty()) { + throw new RuntimeException("Wrong tmTextUnitVariantId"); + } + + TextUnitDTO textUnit = search.getFirst(); + + AiReviewInput input = + new AiReviewInput( + textUnit.getTargetLocale(), + textUnit.getSource(), + textUnit.getComment(), + new AiReviewInput.ExistingTarget( + textUnit.getTarget(), !textUnit.isIncludedInLocalizedFile())); + + ObjectMapper objectMapper = ObjectMapper.withIndentedOutput(); + String inputAsJsonString = objectMapper.writeValueAsStringUnchecked(input); + + ObjectNode jsonSchema = createJsonSchema(AiReviewOutput.class); + + OpenAIClient.ChatCompletionsRequest chatCompletionsRequest = + chatCompletionsRequest() + .model("gpt-4o-2024-08-06") + .maxTokens(16384) + .messages( + List.of( + systemMessageBuilder().content(PROMPT).build(), + userMessageBuilder().content(inputAsJsonString).build())) + .responseFormat( + new OpenAIClient.ChatCompletionsRequest.JsonFormat( + "json_schema", + new OpenAIClient.ChatCompletionsRequest.JsonFormat.JsonSchema( + true, "request_json_format", jsonSchema))) + .build(); + + logger.info(objectMapper.writeValueAsStringUnchecked(chatCompletionsRequest)); + + OpenAIClient openAIClient = + OpenAIClient.builder() + .apiKey(aiReviewConfigurationProperties.getOpenaiClientToken()) + .build(); + + OpenAIClient.ChatCompletionsResponse chatCompletionsResponse = + openAIClient.getChatCompletions(chatCompletionsRequest).join(); + + logger.info(objectMapper.writeValueAsStringUnchecked(chatCompletionsResponse)); + + String jsonResponse = chatCompletionsResponse.choices().getFirst().message().content(); + AiReviewOutput review = objectMapper.readValueUnchecked(jsonResponse, AiReviewOutput.class); + + return new ProtoAiReviewResponse(textUnit, review); + } + + public record ProtoAiReviewRequest(long tmTextUnitVariantId) {} + + public record ProtoAiReviewResponse(TextUnitDTO textUnitDTO, AiReviewOutput aiReviewOutput) {} + + record AiReviewOutput( + String source, + Target target, + DescriptionRating descriptionRating, + AltTarget altTarget, + ExistingTargetRating existingTargetRating, + ReviewRequired reviewRequired) { + record Target(String content, String explanation, int confidenceLevel) {} + + record AltTarget(String content, String explanation, int confidenceLevel) {} + + record DescriptionRating(String explanation, int score) {} + + record ExistingTargetRating(String explanation, int score) {} + + record ReviewRequired(boolean required, String reason) {} + } + + record AiReviewInput( + String locale, String source, String sourceDescription, ExistingTarget existingTarget) { + record ExistingTarget(String content, boolean hasBrokenPlaceholders) {} + } + + static final String PROMPT = + """ + Your role is to act as a translator. + You are tasked with translating provided source strings while preserving both the tone and the technical structure of the string. This includes protecting any tags, placeholders, or code elements that should not be translated. + + The input will be provided in JSON format with the following fields: + + • "source": The source text to be translated. + • "locale": The target language locale, following the BCP47 standard (e.g., “fr”, “es-419”). + • "sourceDescription": A description providing context for the source text. + • "existingTarget" (optional): An existing translation to review. + + Instructions: + + • If the source is colloquial, keep the translation colloquial; if it’s formal, maintain formality in the translation. + • Pay attention to regional variations specified in the "locale" field (e.g., “es” vs. “es-419”, “fr” vs. “fr-CA”, “zh” vs. “zh-Hant”), and ensure the translation length remains similar to the source text. + • Aim to provide the best translation, while compromising on length to ensure it remains close to the original text length + + Handling Tags and Code: + + Some strings contain code elements such as tags (e.g., {atag}, ICU message format, or HTML tags). You are provided with a inputs of tags that need to be protected. Ensure that: + + • Tags like {atag} remain untouched. + • In cases of nested content (e.g., text that needs translation), only translate the inner text while preserving the outer structure. + • Complex structures like ICU message formats should have placeholders or variables left intact (e.g., {count, plural, one {# item} other {# items}}), but translate any inner translatable text. + + Ambiguity and Context: + + After translating, assess the usefulness of the "sourceDescription" field: + + • Rate its usefulness on a scale of 0 to 2: + • 0 – Not helpful at all; irrelevant or misleading. + • 1 – Somewhat helpful; provides partial or unclear context but is useful to some extent. + • 2 – Very helpful; provides clear and sufficient guidance for the translation. + + If the source is ambiguous—for example, if it could be interpreted as a noun or a verb—you must: + + • Indicate the ambiguity in your explanation. + • Provide translations for all possible interpretations. + • Set "reviewRequired" to true, and explain the need for review due to the ambiguity. + + You will provide an output in JSON format with the following fields: + + • "source": The original source text. + • "target": An object containing: + • "content": The best translation. + • "explanation": A brief explanation of your translation choices. + • "confidenceLevel": Your confidence level (0-100%) in the translation. + • "descriptionRating": An object containing: + • "explanation": An explanation of how the "sourceDescription" aided your translation. + • "score": The usefulness score (0-2). + • "altTarget": An object containing: + • "content": An alternative translation, if applicable. Focus on showcasing grammar differences, + • "explanation": Explanation for the alternative translation. + • "confidenceLevel": Your confidence level (0-100%) in the alternative translation. + • "existingTargetRating" (if "existingTarget" is provided): An object containing: + • "explanation": Feedback on the existing translation’s accuracy and quality. + • "score": A rating score (0-2). + • "reviewRequired": An object containing: + • "required": true or false, indicating if review is needed. + • "reason": A detailed explanation of why review is or isn’t needed. + """; +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java new file mode 100644 index 0000000000..6f600b0ed5 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiTranslateWS.java @@ -0,0 +1,61 @@ +package com.box.l10n.mojito.rest.textunit; + +import com.box.l10n.mojito.entity.PollableTask; +import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateConfigurationProperties; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateService; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateService.AiTranslateInput; +import com.box.l10n.mojito.service.pollableTask.PollableFuture; +import com.box.l10n.mojito.service.repository.RepositoryRepository; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class AiTranslateWS { + + /** logger */ + static Logger logger = LoggerFactory.getLogger(AiTranslateWS.class); + + @Autowired AiTranslateService aiTranslateService; + + @Autowired RepositoryRepository repositoryRepository; + + @Autowired QuartzPollableTaskScheduler quartzPollableTaskScheduler; + + @Autowired AiTranslateConfigurationProperties aiTranslateConfigurationProperties; + + @RequestMapping(method = RequestMethod.POST, value = "/api/proto-ai-translate") + @ResponseStatus(HttpStatus.OK) + public ProtoAiTranslateResponse aiTranslate( + @RequestBody ProtoAiTranslateRequest protoAiTranslateRequest) { + + PollableFuture pollableFuture = + aiTranslateService.aiTranslateAsync( + new AiTranslateInput( + protoAiTranslateRequest.repositoryName(), + protoAiTranslateRequest.targetBcp47tags(), + protoAiTranslateRequest.sourceTextMaxCountPerLocale(), + protoAiTranslateRequest.tmTextUnitIds(), + protoAiTranslateRequest.useBatch())); + + return new ProtoAiTranslateResponse(pollableFuture.getPollableTask()); + } + + public record ProtoAiTranslateRequest( + String repositoryName, + List targetBcp47tags, + int sourceTextMaxCountPerLocale, + boolean useBatch, + List tmTextUnitIds, + boolean allLocales) {} + + public record ProtoAiTranslateResponse(PollableTask pollableTask) {} +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/TextUnitWS.java b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/TextUnitWS.java index f39cab94d1..3740d4df2f 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/TextUnitWS.java +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/textunit/TextUnitWS.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.rest.textunit; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; + import com.box.l10n.mojito.entity.Asset; import com.box.l10n.mojito.entity.AssetTextUnit; import com.box.l10n.mojito.entity.BaseEntity; @@ -310,9 +312,9 @@ public PollableTask importTextUnitBatch(@RequestBody String string) { PollableFuture pollableFuture = textUnitBatchImporterService.asyncImportTextUnits( importTextUnitsBatch.getTextUnits(), - importTextUnitsBatch.isIntegrityCheckSkipped(), - importTextUnitsBatch.isIntegrityCheckKeepStatusIfFailedAndSameTarget()); - + fromLegacy( + importTextUnitsBatch.isIntegrityCheckSkipped(), + importTextUnitsBatch.isIntegrityCheckKeepStatusIfFailedAndSameTarget())); return pollableFuture.getPollableTask(); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/security/OidcUserDetailsImpl.java b/webapp/src/main/java/com/box/l10n/mojito/security/OidcUserDetailsImpl.java new file mode 100644 index 0000000000..87f14d247b --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/security/OidcUserDetailsImpl.java @@ -0,0 +1,42 @@ +package com.box.l10n.mojito.security; + +import com.box.l10n.mojito.entity.security.user.User; +import java.util.Map; +import org.springframework.security.oauth2.core.oidc.OidcIdToken; +import org.springframework.security.oauth2.core.oidc.OidcUserInfo; +import org.springframework.security.oauth2.core.oidc.user.OidcUser; + +public class OidcUserDetailsImpl extends UserDetailsImpl implements OidcUser { + + private final OidcUser oidcUser; + + public OidcUserDetailsImpl(User user, OidcUser oidcUser) { + super(user); + this.oidcUser = oidcUser; + } + + @Override + public Map getClaims() { + return oidcUser.getClaims(); + } + + @Override + public OidcUserInfo getUserInfo() { + return oidcUser.getUserInfo(); + } + + @Override + public OidcIdToken getIdToken() { + return oidcUser.getIdToken(); + } + + @Override + public Map getAttributes() { + return oidcUser.getAttributes(); + } + + @Override + public String getName() { + return oidcUser.getName(); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/security/SecurityConfig.java b/webapp/src/main/java/com/box/l10n/mojito/security/SecurityConfig.java index 8863f92e22..db3388374a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/security/SecurityConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/security/SecurityConfig.java @@ -72,6 +72,17 @@ public static class OAuth2 { String unwrapUserAttributes; + /** + * In case the principal ID form the auth provider is a UUID, it is of little use on the Mojito + * side for tasks such as sending notifications, associating with Git commit, PRs, etc. + * + *

With this option, the username is extracted from the email. This assumes a 1:1 mapping + * between the email and the Unix username + * + *

Currently only implement in {@link UserDetailImplOidcUserService} + */ + boolean usernameFromEmail = false; + String givenNameAttribute = "first_name"; String surnameAttribute = "last_name"; String commonNameAttribute = "name"; @@ -115,5 +126,13 @@ public String getUnwrapUserAttributes() { public void setUnwrapUserAttributes(String unwrapUserAttributes) { this.unwrapUserAttributes = unwrapUserAttributes; } + + public boolean isUsernameFromEmail() { + return usernameFromEmail; + } + + public void setUsernameFromEmail(boolean usernameFromEmail) { + this.usernameFromEmail = usernameFromEmail; + } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/security/UserDetailImplOidcUserService.java b/webapp/src/main/java/com/box/l10n/mojito/security/UserDetailImplOidcUserService.java new file mode 100644 index 0000000000..54665fcf77 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/security/UserDetailImplOidcUserService.java @@ -0,0 +1,50 @@ +package com.box.l10n.mojito.security; + +import com.box.l10n.mojito.entity.security.user.User; +import com.box.l10n.mojito.service.security.user.UserService; +import java.util.Objects; +import org.springframework.security.oauth2.client.oidc.userinfo.OidcUserRequest; +import org.springframework.security.oauth2.client.oidc.userinfo.OidcUserService; +import org.springframework.security.oauth2.core.OAuth2AuthenticationException; +import org.springframework.security.oauth2.core.oidc.user.OidcUser; + +class UserDetailImplOidcUserService extends OidcUserService { + + SecurityConfig securityConfig; + UserService userService; + + public UserDetailImplOidcUserService(SecurityConfig securityConfig, UserService userService) { + this.securityConfig = Objects.requireNonNull(securityConfig); + this.userService = Objects.requireNonNull(userService); + } + + @Override + public OidcUser loadUser(OidcUserRequest userRequest) throws OAuth2AuthenticationException { + OidcUser oidcUser = super.loadUser(userRequest); + + SecurityConfig.OAuth2 securityConfigOAuth2 = + securityConfig + .getoAuth2() + .getOrDefault( + userRequest.getClientRegistration().getRegistrationId(), + new SecurityConfig.OAuth2()); + + String username; + if (securityConfigOAuth2.usernameFromEmail) { + String email = oidcUser.getEmail(); + if (email == null) { + throw new RuntimeException( + "OidcUser's email must not be null since it used to extract username"); + } + username = email.split("@")[0]; + } else { + username = oidcUser.getName(); + } + + User user = + userService.getOrCreateOrUpdateBasicUser( + username, oidcUser.getGivenName(), oidcUser.getFamilyName(), oidcUser.getFullName()); + + return new OidcUserDetailsImpl(user, oidcUser); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/security/WebSecurityConfig.java b/webapp/src/main/java/com/box/l10n/mojito/security/WebSecurityConfig.java index 1c05bbf271..9f0b1c3f2f 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/security/WebSecurityConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/security/WebSecurityConfig.java @@ -268,6 +268,9 @@ public SecurityFilterChain configure(HttpSecurity http) throws Exception { userInfoEndpoint -> { userInfoEndpoint.userService( new UserDetailImplOAuth2UserService(securityConfig, userService)); + + userInfoEndpoint.oidcUserService( + new UserDetailImplOidcUserService(securityConfig, userService)); }); }); break; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJob.java b/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJob.java index 0d0b914d26..152605adf3 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJob.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJob.java @@ -3,6 +3,8 @@ import com.box.l10n.mojito.quartz.QuartzPollableJob; import com.box.l10n.mojito.service.pollableTask.PollableTaskService; import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -24,10 +26,9 @@ public class ImportTextUnitJob extends QuartzPollableJob textUnitDTOs = input.getTextUnitDTOs(); + + textUnitBatchImporterService.importTextUnits(textUnitDTOs, input.getIntegrityChecksType()); return null; } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJobInput.java b/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJobInput.java index 901c0aaf67..ea84d95fee 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJobInput.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/asset/ImportTextUnitJobInput.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.service.asset; +import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; import com.box.l10n.mojito.service.tm.search.TextUnitDTO; import java.util.List; @@ -8,26 +9,16 @@ */ public class ImportTextUnitJobInput { - boolean integrityCheckSkipped; - boolean integrityCheckKeepStatusIfFailedAndSameTarget; List textUnitDTOs; + TextUnitBatchImporterService.IntegrityChecksType integrityChecksType; - public boolean isIntegrityCheckSkipped() { - return integrityCheckSkipped; + public TextUnitBatchImporterService.IntegrityChecksType getIntegrityChecksType() { + return integrityChecksType; } - public void setIntegrityCheckSkipped(boolean integrityCheckSkipped) { - this.integrityCheckSkipped = integrityCheckSkipped; - } - - public boolean isIntegrityCheckKeepStatusIfFailedAndSameTarget() { - return integrityCheckKeepStatusIfFailedAndSameTarget; - } - - public void setIntegrityCheckKeepStatusIfFailedAndSameTarget( - boolean integrityCheckKeepStatusIfFailedAndSameTarget) { - this.integrityCheckKeepStatusIfFailedAndSameTarget = - integrityCheckKeepStatusIfFailedAndSameTarget; + public void setIntegrityChecksType( + TextUnitBatchImporterService.IntegrityChecksType integrityChecksType) { + this.integrityChecksType = integrityChecksType; } public List getTextUnitDTOs() { diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/asset/VirtualAssetService.java b/webapp/src/main/java/com/box/l10n/mojito/service/asset/VirtualAssetService.java index b939f6a967..37e27ddbd9 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/asset/VirtualAssetService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/asset/VirtualAssetService.java @@ -2,6 +2,7 @@ import static com.box.l10n.mojito.entity.TMTextUnitVariant.Status.APPROVED; import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.slf4j.LoggerFactory.getLogger; import com.box.l10n.mojito.entity.Asset; @@ -341,7 +342,8 @@ public PollableFuture importLocalizedTextUnits( textUnitDTOs.add(textUnitDTO); } - return textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOs, false, false); + return textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOs, fromLegacy(false, false)); } @Transactional diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/BackquoteIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/BackquoteIntegrityChecker.java index bccdd4dbbb..2cf281350a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/BackquoteIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/BackquoteIntegrityChecker.java @@ -25,7 +25,8 @@ public void check(String sourceContent, String targetContent) try { super.check(sourceContent, targetContent); } catch (RegexCheckerException rce) { - throw new BackquoteIntegrityCheckerException((rce.getMessage())); + throw new BackquoteIntegrityCheckerException( + "Backquoted stings are different in source and target"); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java index b8b5486d23..c3621b4935 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityChecker.java @@ -9,7 +9,7 @@ public class CompositeFormatIntegrityChecker extends RegexIntegrityChecker { @Override public String getRegex() { - return "\\{.*?\\}"; + return "(\\{){1,3}[^\\{\\}]*\\}+"; } @Override @@ -18,7 +18,8 @@ public void check(String sourceContent, String targetContent) try { super.check(sourceContent, targetContent); } catch (RegexCheckerException rce) { - throw new CompositeFormatIntegrityCheckerException(rce); + throw new CompositeFormatIntegrityCheckerException( + "Composite Format placeholders in source and target are different"); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerException.java index 0349800d67..a2e6b4abe1 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerException.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerException.java @@ -3,9 +3,9 @@ /** * @author jaurambault */ -public class CompositeFormatIntegrityCheckerException extends RegexCheckerException { +public class CompositeFormatIntegrityCheckerException extends IntegrityCheckException { - public CompositeFormatIntegrityCheckerException(RegexCheckerException rce) { - super(rce.getMessage()); + public CompositeFormatIntegrityCheckerException(String message) { + super(message); } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityChecker.java new file mode 100644 index 0000000000..81467203cb --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityChecker.java @@ -0,0 +1,20 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class EmailIntegrityChecker extends RegexIntegrityChecker { + + static final String EMAIL_REGEX = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; + + @Override + public String getRegex() { + return EMAIL_REGEX; + } + + @Override + public void check(String content, String target) { + try { + super.check(content, target); + } catch (RegexCheckerException ex) { + throw new EmailIntegrityCheckerException("Emails are changed."); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerException.java new file mode 100644 index 0000000000..07b78bcd2b --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class EmailIntegrityCheckerException extends IntegrityCheckException { + public EmailIntegrityCheckerException(String message) { + super(message); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java index 8f61b69441..58ef8249e9 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityChecker.java @@ -1,8 +1,14 @@ package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.regex.Matcher; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +26,7 @@ public class HtmlTagIntegrityChecker extends RegexIntegrityChecker { @Override public String getRegex() { - return "(<\\w+(\\s+\\w+(\\s*=\\s*('(\\\'|[^'])*?'|\"(\\\"|[^\"])*?\")))*?>|)"; + return "(<[a-zA-Z][\\w-]*(\\s+\\w+(\\s*=\\s*('([^']*?)'|\"([^\"]*?)\"))?)*\\s*/?>|<\\/[a-zA-Z][\\w-]*>)"; } @Override @@ -33,10 +39,52 @@ public void check(String sourceContent, String targetContent) throws IntegrityCh List sourceHtmlTags = getHtmlTags(sourceContent); logger.debug("Source Html tags: {}", sourceHtmlTags); + Map sourceTagCount = + sourceHtmlTags.stream() + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); + Map targetTagCount = + targetHtmlTags.stream() + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); + logger.debug("Make sure the target has the same Html tags as the source"); - if (!sourceHtmlTags.containsAll(targetHtmlTags) - || !targetHtmlTags.containsAll(sourceHtmlTags)) { - throw new HtmlTagIntegrityCheckerException("HTML tags in source and target are different"); + if (!sourceTagCount.equals(targetTagCount)) { + StringBuilder differences = new StringBuilder(); + + Set allTags = new HashSet<>(); + allTags.addAll(sourceTagCount.keySet()); + allTags.addAll(targetTagCount.keySet()); + + for (String tag : allTags) { + Long sourceCount = sourceTagCount.getOrDefault(tag, 0L); + Long targetCount = targetTagCount.getOrDefault(tag, 0L); + if (sourceCount > targetCount) { + differences.append( + String.format("Target is missing %d tag(s) '%s'%n", sourceCount - targetCount, tag)); + } else if (sourceCount < targetCount) { + differences.append( + String.format("Target has extra %d tag(s) '%s'%n", targetCount - sourceCount, tag)); + } + } + throw new HtmlTagIntegrityCheckerException( + "HTML tag counts in source and target are different:\n" + differences.toString().trim()); + } + + logger.debug("Make sure the target tags are in valid order"); + if (!isValidTagOrder(targetHtmlTags)) { + throw new HtmlTagIntegrityCheckerException("HTML tags in target are not in valid order"); + } + + logger.debug("Ad-hoc checks"); + checkDoubleAnnotationElements(sourceContent, targetContent); + } + + /** Adhoc check to unblock. Eventually needs a better solution. */ + void checkDoubleAnnotationElements(String sourceContent, String targetContent) { + String doubleAnnotationString = " getHtmlTags(String string) { return tags; } + + public static boolean isValidTagOrder(List tags) { + + boolean res = true; + + ArrayDeque stack = new ArrayDeque<>(); + + for (String tag : tags) { + if (!tag.startsWith(".+?)]\\((?.+?)\\)"; + } + + @Override + Set getPlaceholders(String string) { + Set placeholders = new LinkedHashSet<>(); + + if (string != null) { + Matcher matcher = getPattern().matcher(string); + while (matcher.find()) { + placeholders.add("[%s](%s)".formatted("--translatable--", matcher.group("url"))); + } + } + return placeholders; + } + + @Override + public void check(String content, String target) { + try { + super.check(content, target); + } catch (RegexCheckerException ex) { + throw new MarkdownLinkIntegrityCheckerException("Markdown Links do not match."); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerException.java new file mode 100644 index 0000000000..f6a5db5907 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class MarkdownLinkIntegrityCheckerException extends IntegrityCheckException { + public MarkdownLinkIntegrityCheckerException(String message) { + super(message); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxer.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxer.java new file mode 100644 index 0000000000..13d08b167c --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxer.java @@ -0,0 +1,41 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import java.util.Set; +import org.springframework.stereotype.Component; + +@Component +public class PluralIntegrityCheckerRelaxer { + + /** + * This is very ad hoc! + * + *

There are cases where the plural form doesn’t retain the number. Right now, those are + * wrongly marked as rejected. Ideally, we should clearly identify which placeholder might be + * missing from the string, but that’s another level of complexity. Also, use the class name to + * target certain integrity checks to keep this hack constrained. + */ + public boolean shouldRelaxIntegrityCheck( + String source, String target, String pluralForm, TextUnitIntegrityChecker textUnitChecker) { + + boolean shouldRelax = false; + + if (pluralForm != null && !"other".equals(pluralForm)) { + if (textUnitChecker instanceof PrintfLikeIntegrityChecker + || textUnitChecker instanceof PrintfLikeIgnorePercentageAfterBracketsIntegrityChecker + || textUnitChecker instanceof PrintfLikeVariableTypeIntegrityChecker + || textUnitChecker instanceof SimplePrintfLikeIntegrityChecker) { + + RegexIntegrityChecker regexIntegrityChecker = (RegexIntegrityChecker) textUnitChecker; + + Set sourcePlaceholders = regexIntegrityChecker.getPlaceholders(source); + Set targetPlaceholders = regexIntegrityChecker.getPlaceholders(target); + + if (sourcePlaceholders.size() - targetPlaceholders.size() <= 1) { + shouldRelax = true; + } + } + } + + return shouldRelax; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityChecker.java index 0031cc4ec2..9270cae383 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityChecker.java @@ -29,7 +29,8 @@ public void check(String sourceContent, String targetContent) try { super.check(sourceContent, targetContent); } catch (RegexCheckerException rce) { - throw new PrintfLikeIntegrityCheckerException((rce.getMessage())); + throw new PrintfLikeIntegrityCheckerException( + "PrintfLike placeholders are different in source and target"); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeVariableTypeIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeVariableTypeIntegrityChecker.java index afe8dbc687..5fd2ebcca6 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeVariableTypeIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeVariableTypeIntegrityChecker.java @@ -22,7 +22,8 @@ public void check(String content, String target) { try { super.check(content, target); } catch (RegexCheckerException ex) { - throw new PrintfLikeVariableTypeIntegrityCheckerException("Variable types do not match."); + throw new PrintfLikeVariableTypeIntegrityCheckerException( + "PrintfLikeVariableType placeholder are different in source and target."); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java new file mode 100644 index 0000000000..bbc216d483 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityChecker.java @@ -0,0 +1,19 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class PythonFStringIntegrityChecker extends RegexIntegrityChecker { + + @Override + public String getRegex() { + return "\\$\\{?[a-zA-Z_][a-zA-Z0-9_]*\\}?"; + } + + @Override + public void check(String content, String target) { + try { + super.check(content, target); + } catch (RegexCheckerException ex) { + throw new PythonFStringIntegrityCheckerException( + "PythonFString placeholders are different in source and target."); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerException.java new file mode 100644 index 0000000000..43724d91fa --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerException.java @@ -0,0 +1,7 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +public class PythonFStringIntegrityCheckerException extends IntegrityCheckException { + public PythonFStringIntegrityCheckerException(String message) { + super(message); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/RegexIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/RegexIntegrityChecker.java index e0ccc3552f..6a8d900882 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/RegexIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/RegexIntegrityChecker.java @@ -1,6 +1,6 @@ package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -68,7 +68,7 @@ Pattern getPattern() { */ Set getPlaceholders(String string) { - Set placeholders = new HashSet<>(); + Set placeholders = new LinkedHashSet<>(); if (string != null) { diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityChecker.java index df0f36e243..c6d19c264c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityChecker.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityChecker.java @@ -22,7 +22,8 @@ public void check(String sourceContent, String targetContent) try { super.check(sourceContent, targetContent); } catch (RegexCheckerException rce) { - throw new SimplePrintfLikeIntegrityCheckerException((rce.getMessage())); + throw new SimplePrintfLikeIntegrityCheckerException( + "SimplePrintfLike placeholders are different in source and target."); } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java new file mode 100644 index 0000000000..23f447ccaf --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityChecker.java @@ -0,0 +1,29 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +/** + * Checks for URLs in plain text. + * + *

Regex is simple and might under match but that's preferable too triggering false positive + * + *

Consider using another checker for format with links like Markdown: + * MarkdownLinkIntegrityChecker + * + * @author jaurambault + */ +public class URLIntegrityChecker extends RegexIntegrityChecker { + + @Override + public String getRegex() { + return "(https?|ftp)://(?:www\\.)?(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}(?:/[a-zA-Z0-9/_\\.\\-?#+%]*)*|mailto:[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; + } + + @Override + public void check(String sourceContent, String targetContent) + throws CompositeFormatIntegrityCheckerException { + try { + super.check(sourceContent, targetContent); + } catch (RegexCheckerException rce) { + throw new URLIntegrityCheckerException("URLs in source and target are different"); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java new file mode 100644 index 0000000000..aa1a13948e --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerException.java @@ -0,0 +1,11 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +/** + * @author jaurambault + */ +public class URLIntegrityCheckerException extends IntegrityCheckException { + + public URLIntegrityCheckerException(String message) { + super(message); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java index 32ef760e50..bfea907c06 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/StructuredBlobStorage.java @@ -48,5 +48,6 @@ public enum Prefix { MULTI_BRANCH_STATE, TEXT_UNIT_DTOS_CACHE, CLOB_STORAGE_WS, + AI_TRANSLATE_WS, } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/database/DatabaseBlobStorage.java b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/database/DatabaseBlobStorage.java index 313d397eea..cabda6db83 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/database/DatabaseBlobStorage.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/blobstorage/database/DatabaseBlobStorage.java @@ -11,6 +11,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.domain.PageRequest; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; /** * Implementation that use the database to store the blobs. @@ -60,6 +62,7 @@ public void put(String name, byte[] content, Retention retention) { }); } + @Transactional(propagation = Propagation.REQUIRES_NEW) void putBase(String name, byte[] content, Retention retention) { MBlob mBlob = mBlobRepository diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/commit/CommitService.java b/webapp/src/main/java/com/box/l10n/mojito/service/commit/CommitService.java index e9f5eafb72..a5ac52163c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/commit/CommitService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/commit/CommitService.java @@ -148,7 +148,7 @@ public Commit getOrCreateCommit( commit.setAuthorName(authorName); commit.setSourceCreationDate(sourceCreationDate); - System.out.printf("setSourceCreationDate: %s%n", commit.getSourceCreationDate()); + logger.debug("setSourceCreationDate: {}", commit.getSourceCreationDate()); commit = commitRepository.save(commit); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java b/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java index 4a366a3017..f0d60d3580 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java @@ -2,6 +2,9 @@ import com.box.l10n.mojito.entity.Drop; import java.util.Optional; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; import org.springframework.data.jpa.repository.EntityGraph; import org.springframework.data.jpa.repository.EntityGraph.EntityGraphType; import org.springframework.data.jpa.repository.JpaRepository; @@ -14,6 +17,10 @@ @RepositoryRestResource(exported = false) public interface DropRepository extends JpaRepository, JpaSpecificationExecutor { + @Override + @EntityGraph(value = "Drop.legacy", type = EntityGraphType.FETCH) + Page findAll(Specification spec, Pageable pageable); + @Override @EntityGraph(value = "Drop.legacy", type = EntityGraphType.FETCH) Optional findById(Long aLong); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/machinetranslation/RepositoryMachineTranslationService.java b/webapp/src/main/java/com/box/l10n/mojito/service/machinetranslation/RepositoryMachineTranslationService.java index 155c1d260a..dedb5ec65d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/machinetranslation/RepositoryMachineTranslationService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/machinetranslation/RepositoryMachineTranslationService.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.service.machinetranslation; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; + import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.service.pollableTask.Pollable; import com.box.l10n.mojito.service.pollableTask.PollableFuture; @@ -127,7 +129,7 @@ public PollableFuture translateRepository( .collect(Collectors.toList()); textUnitBatchImporterService.importTextUnits( - machineTranslatedTextUnitDTOs, false, true); + machineTranslatedTextUnitDTOs, fromLegacy(false, true)); }); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java new file mode 100644 index 0000000000..4ef75a3db5 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfig.java @@ -0,0 +1,56 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.openai.OpenAIClient; +import com.box.l10n.mojito.openai.OpenAIClientPool; +import java.time.Duration; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +@Configuration +public class AiTranslateConfig { + + AiTranslateConfigurationProperties aiTranslateConfigurationProperties; + + public AiTranslateConfig(AiTranslateConfigurationProperties aiTranslateConfigurationProperties) { + this.aiTranslateConfigurationProperties = aiTranslateConfigurationProperties; + } + + @Bean + @Qualifier("AiTranslate") + OpenAIClient openAIClient() { + String openaiClientToken = aiTranslateConfigurationProperties.getOpenaiClientToken(); + if (openaiClientToken == null) { + return null; + } + return new OpenAIClient.Builder().apiKey(openaiClientToken).build(); + } + + @Bean + @Qualifier("AiTranslate") + OpenAIClientPool openAIClientPool() { + String openaiClientToken = aiTranslateConfigurationProperties.getOpenaiClientToken(); + if (openaiClientToken == null) { + return null; + } + return new OpenAIClientPool( + 10, 50, 5, aiTranslateConfigurationProperties.getOpenaiClientToken()); + } + + @Bean + @Qualifier("AiTranslate") + ObjectMapper objectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + AiTranslateService.configureObjectMapper(objectMapper); + return objectMapper; + } + + @Bean + @Qualifier("AiTranslate") + RetryBackoffSpec retryBackoffSpec() { + return Retry.backoff(5, Duration.ofMillis(500)).maxBackoff(Duration.ofSeconds(5)); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfigurationProperties.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfigurationProperties.java new file mode 100644 index 0000000000..7810883643 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateConfigurationProperties.java @@ -0,0 +1,28 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import com.box.l10n.mojito.quartz.QuartzSchedulerManager; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties("l10n.ai-translate") +public class AiTranslateConfigurationProperties { + String openaiClientToken; + String schedulerName = QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; + + public String getOpenaiClientToken() { + return openaiClientToken; + } + + public void setOpenaiClientToken(String openaiClientToken) { + this.openaiClientToken = openaiClientToken; + } + + public String getSchedulerName() { + return schedulerName; + } + + public void setSchedulerName(String schedulerName) { + this.schedulerName = schedulerName; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateJob.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateJob.java new file mode 100644 index 0000000000..30d08a8155 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateJob.java @@ -0,0 +1,27 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import com.box.l10n.mojito.quartz.QuartzPollableJob; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateService.AiTranslateInput; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Class to process a batch of strings for machine translation against a set of target languages. + * + * @author garion + */ +@Component +public class AiTranslateJob extends QuartzPollableJob { + + static Logger logger = LoggerFactory.getLogger(AiTranslateJob.class); + + @Autowired AiTranslateService aiTranslateService; + + @Override + public Void call(AiTranslateInput aiTranslateJobInput) throws Exception { + aiTranslateService.aiTranslate(aiTranslateJobInput); + return null; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java new file mode 100644 index 0000000000..d36483330a --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateService.java @@ -0,0 +1,703 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionResponseBatchFileLine; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.JsonFormat.JsonSchema.createJsonSchema; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.SystemMessage.systemMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.UserMessage.userMessageBuilder; +import static com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsRequest.chatCompletionsRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.CreateBatchRequest.forChatCompletion; +import static com.box.l10n.mojito.openai.OpenAIClient.DownloadFileContentRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.DownloadFileContentResponse; +import static com.box.l10n.mojito.openai.OpenAIClient.RetrieveBatchRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.RetrieveBatchResponse; +import static com.box.l10n.mojito.openai.OpenAIClient.UploadFileRequest; +import static com.box.l10n.mojito.openai.OpenAIClient.UploadFileResponse; +import static com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage.Prefix.AI_TRANSLATE_WS; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toMap; + +import com.box.l10n.mojito.entity.Repository; +import com.box.l10n.mojito.entity.RepositoryLocale; +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.openai.OpenAIClient; +import com.box.l10n.mojito.openai.OpenAIClient.ChatCompletionsResponse; +import com.box.l10n.mojito.openai.OpenAIClient.CreateBatchResponse; +import com.box.l10n.mojito.openai.OpenAIClient.RequestBatchFileLine; +import com.box.l10n.mojito.openai.OpenAIClientPool; +import com.box.l10n.mojito.quartz.QuartzJobInfo; +import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; +import com.box.l10n.mojito.service.blobstorage.Retention; +import com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage; +import com.box.l10n.mojito.service.oaitranslate.AiTranslateService.CompletionInput.ExistingTarget; +import com.box.l10n.mojito.service.pollableTask.PollableFuture; +import com.box.l10n.mojito.service.repository.RepositoryNameNotFoundException; +import com.box.l10n.mojito.service.repository.RepositoryRepository; +import com.box.l10n.mojito.service.repository.RepositoryService; +import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; +import com.box.l10n.mojito.service.tm.search.StatusFilter; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.box.l10n.mojito.service.tm.search.UsedFilter; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +@Service +public class AiTranslateService { + + static final String METADATA__TEXT_UNIT_DTOS__BLOB_ID = "textUnitDTOs"; + + /** logger */ + static Logger logger = LoggerFactory.getLogger(AiTranslateService.class); + + TextUnitSearcher textUnitSearcher; + + RepositoryRepository repositoryRepository; + + RepositoryService repositoryService; + + AiTranslateConfigurationProperties aiTranslateConfigurationProperties; + + OpenAIClient openAIClient; + + OpenAIClientPool openAIClientPool; + + TextUnitBatchImporterService textUnitBatchImporterService; + + StructuredBlobStorage structuredBlobStorage; + + ObjectMapper objectMapper; + + RetryBackoffSpec retryBackoffSpec; + + QuartzPollableTaskScheduler quartzPollableTaskScheduler; + + public AiTranslateService( + TextUnitSearcher textUnitSearcher, + RepositoryRepository repositoryRepository, + RepositoryService repositoryService, + TextUnitBatchImporterService textUnitBatchImporterService, + StructuredBlobStorage structuredBlobStorage, + AiTranslateConfigurationProperties aiTranslateConfigurationProperties, + @Qualifier("AiTranslate") @Autowired(required = false) OpenAIClient openAIClient, + @Qualifier("AiTranslate") @Autowired(required = false) OpenAIClientPool openAIClientPool, + @Qualifier("AiTranslate") ObjectMapper objectMapper, + @Qualifier("AiTranslate") RetryBackoffSpec retryBackoffSpec, + QuartzPollableTaskScheduler quartzPollableTaskScheduler) { + this.textUnitSearcher = textUnitSearcher; + this.repositoryRepository = repositoryRepository; + this.repositoryService = repositoryService; + this.textUnitBatchImporterService = textUnitBatchImporterService; + this.structuredBlobStorage = structuredBlobStorage; + this.aiTranslateConfigurationProperties = aiTranslateConfigurationProperties; + this.objectMapper = objectMapper; + this.openAIClient = openAIClient; + this.openAIClientPool = openAIClientPool; + this.retryBackoffSpec = retryBackoffSpec; + this.quartzPollableTaskScheduler = quartzPollableTaskScheduler; + } + + public record AiTranslateInput( + String repositoryName, + List targetBcp47tags, + int sourceTextMaxCountPerLocale, + List tmTextUnitIds, + boolean useBatch) {} + + public PollableFuture aiTranslateAsync(AiTranslateInput aiTranslateInput) { + + QuartzJobInfo quartzJobInfo = + QuartzJobInfo.newBuilder(AiTranslateJob.class) + .withInlineInput(false) + .withInput(aiTranslateInput) + .withScheduler(aiTranslateConfigurationProperties.getSchedulerName()) + .build(); + + return quartzPollableTaskScheduler.scheduleJob(quartzJobInfo); + } + + public void aiTranslate(AiTranslateInput aiTranslateInput) throws AiTranslateException { + if (aiTranslateInput.useBatch()) { + aiTranslateBatch(aiTranslateInput); + } else { + aiTranslateNoBatch(aiTranslateInput); + } + } + + public void aiTranslateNoBatch(AiTranslateInput aiTranslateInput) { + + Repository repository = getRepository(aiTranslateInput); + + logger.info("Start AI Translation (no batch) for repository: {}", repository.getName()); + + Set filteredRepositoryLocales = + getFilteredRepositoryLocales(aiTranslateInput, repository); + + Flux.fromIterable(filteredRepositoryLocales) + .flatMap( + rl -> + asyncProcessLocale( + rl, + aiTranslateInput.sourceTextMaxCountPerLocale(), + aiTranslateInput.tmTextUnitIds(), + openAIClientPool), + 10) + .then() + .doOnTerminate( + () -> + logger.info( + "Done with AI Translation (no batch) for repository: {}", repository.getName())) + .block(); + } + + Mono asyncProcessLocale( + RepositoryLocale repositoryLocale, + int sourceTextMaxCountPerLocale, + List tmTextUnitIds, + OpenAIClientPool openAIClientPool) { + + Repository repository = repositoryLocale.getRepository(); + + logger.info( + "Get untranslated strings for locale: '{}' in repository: '{}'", + repositoryLocale.getLocale().getBcp47Tag(), + repository.getName()); + + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(repository.getId()); + textUnitSearcherParameters.setStatusFilter(StatusFilter.FOR_TRANSLATION); + textUnitSearcherParameters.setLocaleId(repositoryLocale.getLocale().getId()); + textUnitSearcherParameters.setUsedFilter(UsedFilter.USED); + if (tmTextUnitIds != null) { + logger.debug( + "Using tmTextUnitIds: {} for ai translate repository: {}", + tmTextUnitIds, + repository.getName()); + textUnitSearcherParameters.setTmTextUnitIds(tmTextUnitIds); + } else { + textUnitSearcherParameters.setLimit(sourceTextMaxCountPerLocale); + } + + List textUnitDTOS = textUnitSearcher.search(textUnitSearcherParameters); + + if (textUnitDTOS.isEmpty()) { + logger.debug( + "Nothing to translate for locale: {}", repositoryLocale.getLocale().getBcp47Tag()); + return Mono.empty(); + } + + logger.info( + "Starting parallel processing for each string in locale: {}, count: {}", + repositoryLocale.getLocale().getBcp47Tag(), + textUnitDTOS.size()); + + return Flux.fromIterable(textUnitDTOS) + .buffer(500) + .concatMap( + batch -> + Flux.fromIterable(batch) + .flatMap( + textUnitDTO -> + getChatCompletionForTextUnitDTO(textUnitDTO, openAIClientPool) + .retryWhen( + Retry.backoff(5, Duration.ofSeconds(1)) + .filter(this::isRetriableException) + .doBeforeRetry( + retrySignal -> { + logger.warn( + "Retrying request for TextUnitDTO {} due to exception of type {}", + textUnitDTO.getTmTextUnitId(), + retrySignal.failure().getMessage()); + })) + .onErrorResume( + error -> { + logger.error( + "Request for TextUnitDTO {} failed after retries: {}", + textUnitDTO.getTmTextUnitId(), + error.getMessage()); + return Mono.empty(); + })) + .collectList() + .flatMap(this::submitForImport) + .doOnTerminate(() -> logger.info("Done submitting for processing"))) + .then(); + } + + record MyRecord(TextUnitDTO textUnitDTO, ChatCompletionsResponse chatCompletionsResponse) {} + + private Mono submitForImport(List results) { + logger.info("Submit for import for locale {}", results.get(0).textUnitDTO().getTargetLocale()); + List forImport = + results.stream() + .map( + myRecord -> { + TextUnitDTO textUnitDTO = myRecord.textUnitDTO(); + ChatCompletionsResponse chatCompletionsResponse = + myRecord.chatCompletionsResponse(); + + String completionOutputAsJson = + chatCompletionsResponse.choices().getFirst().message().content(); + + CompletionOutput completionOutput = + objectMapper.readValueUnchecked( + completionOutputAsJson, CompletionOutput.class); + + textUnitDTO.setTarget(completionOutput.target().content()); + textUnitDTO.setTargetComment("ai-translate"); + return textUnitDTO; + }) + .collect(Collectors.toList()); + + textUnitBatchImporterService.importTextUnits( + forImport, + TextUnitBatchImporterService.IntegrityChecksType.ALWAYS_USE_INTEGRITY_CHECKER_STATUS); + + return Mono.empty(); + } + + private Mono getChatCompletionForTextUnitDTO( + TextUnitDTO textUnitDTO, OpenAIClientPool openAIClientPool) { + + CompletionInput completionInput = + new CompletionInput( + textUnitDTO.getTargetLocale(), + textUnitDTO.getSource(), + textUnitDTO.getComment(), + textUnitDTO.getTarget() == null + ? null + : new ExistingTarget( + textUnitDTO.getTarget(), !textUnitDTO.isIncludedInLocalizedFile())); + + String inputAsJsonString = objectMapper.writeValueAsStringUnchecked(completionInput); + ObjectNode jsonSchema = createJsonSchema(CompletionOutput.class); + + ChatCompletionsRequest chatCompletionsRequest = + chatCompletionsRequest() + .model("gpt-4o-2024-08-06") + .maxTokens(16384) + .messages( + List.of( + systemMessageBuilder().content(PROMPT).build(), + userMessageBuilder().content(inputAsJsonString).build())) + .responseFormat( + new ChatCompletionsRequest.JsonFormat( + "json_schema", + new ChatCompletionsRequest.JsonFormat.JsonSchema( + true, "request_json_format", jsonSchema))) + .build(); + + CompletableFuture futureResult = + openAIClientPool.submit( + (openAIClient) -> openAIClient.getChatCompletions(chatCompletionsRequest)); + return Mono.fromFuture(futureResult) + .map(chatCompletionsResponse -> new MyRecord(textUnitDTO, chatCompletionsResponse)); + } + + private boolean isRetriableException(Throwable throwable) { + Throwable cause = throwable instanceof CompletionException ? throwable.getCause() : throwable; + return cause instanceof IOException || cause instanceof TimeoutException; + } + + public void aiTranslateBatch(AiTranslateInput aiTranslateInput) throws AiTranslateException { + + Repository repository = getRepository(aiTranslateInput); + + logger.debug("Start AI Translation for repository: {}", repository.getName()); + + try { + Set repositoryLocalesWithoutRootLocale = + getFilteredRepositoryLocales(aiTranslateInput, repository); + + logger.debug("Create batches for repository: {}", repository.getName()); + ArrayDeque batches = + repositoryLocalesWithoutRootLocale.stream() + .map( + createBatchForRepositoryLocale( + repository, aiTranslateInput.sourceTextMaxCountPerLocale())) + .filter(Objects::nonNull) + .collect(Collectors.toCollection(ArrayDeque::new)); + + logger.debug("Import batches for repository: {}", repository.getName()); + while (!batches.isEmpty()) { + RetrieveBatchResponse retrieveBatchResponse = getNextFinishedBatch(batches); + importBatch(retrieveBatchResponse); + } + } catch (OpenAIClient.OpenAIClientResponseException openAIClientResponseException) { + logger.error( + "Failed to ai translate: %s".formatted(openAIClientResponseException), + openAIClientResponseException); + throw new AiTranslateException(openAIClientResponseException); + } + } + + private Set getFilteredRepositoryLocales( + AiTranslateInput aiTranslateInput, Repository repository) { + return repositoryService.getRepositoryLocalesWithoutRootLocale(repository).stream() + .filter( + rl -> + aiTranslateInput.targetBcp47tags == null + || aiTranslateInput.targetBcp47tags.contains(rl.getLocale().getBcp47Tag())) + .collect(Collectors.toSet()); + } + + private Repository getRepository(AiTranslateInput aiTranslateInput) { + Repository repository = repositoryRepository.findByName(aiTranslateInput.repositoryName()); + + if (repository == null) { + throw new RepositoryNameNotFoundException( + String.format( + "Repository with name '%s' can not be found!", aiTranslateInput.repositoryName())); + } + return repository; + } + + void importBatch(RetrieveBatchResponse retrieveBatchResponse) { + + logger.info("Importing batch: {}", retrieveBatchResponse.id()); + + String textUnitDTOsBlobId = + retrieveBatchResponse.metadata().get(METADATA__TEXT_UNIT_DTOS__BLOB_ID); + + logger.info("Trying to load textUnitDTOs from blob: {}", textUnitDTOsBlobId); + AiTranslateBlobStorage aiTranslateBlobStorage = + structuredBlobStorage + .getString(AI_TRANSLATE_WS, textUnitDTOsBlobId) + .map(s -> objectMapper.readValueUnchecked(s, AiTranslateBlobStorage.class)) + .orElseThrow( + () -> + new RuntimeException( + "There must be an entry for textUnitDTOsBlobId: " + textUnitDTOsBlobId)); + + Map tmTextUnitIdToTextUnitDTOs = + aiTranslateBlobStorage.textUnitDTOS().stream() + .collect(toMap(TextUnitDTO::getTmTextUnitId, Function.identity())); + + DownloadFileContentResponse downloadFileContentResponse = + getOpenAIClient() + .downloadFileContent( + new DownloadFileContentRequest(retrieveBatchResponse.outputFileId())); + + List forImport = + downloadFileContentResponse + .content() + .lines() + .map( + line -> { + ChatCompletionResponseBatchFileLine chatCompletionResponseBatchFileLine = + objectMapper.readValueUnchecked( + line, ChatCompletionResponseBatchFileLine.class); + + if (chatCompletionResponseBatchFileLine.response().statusCode() != 200) { + throw new RuntimeException( + "Response batch file line failed: " + chatCompletionResponseBatchFileLine); + } + + String completionOutputAsJson = + chatCompletionResponseBatchFileLine + .response() + .chatCompletionsResponse() + .choices() + .getFirst() + .message() + .content(); + + CompletionOutput completionOutput = + objectMapper.readValueUnchecked( + completionOutputAsJson, CompletionOutput.class); + + TextUnitDTO textUnitDTO = + tmTextUnitIdToTextUnitDTOs.get( + Long.valueOf(chatCompletionResponseBatchFileLine.customId())); + textUnitDTO.setTarget(completionOutput.target().content()); + textUnitDTO.setTargetComment("ai-translate"); + return textUnitDTO; + }) + .toList(); + + textUnitBatchImporterService.importTextUnits( + forImport, + TextUnitBatchImporterService.IntegrityChecksType.ALWAYS_USE_INTEGRITY_CHECKER_STATUS); + } + + Function createBatchForRepositoryLocale( + Repository repository, int sourceTextMaxCountPerLocale) { + + return repositoryLocale -> { + logger.debug( + "Get untranslated string for locale: '{}' in repository: '{}'", + repositoryLocale.getLocale().getBcp47Tag(), + repository.getName()); + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(repository.getId()); + textUnitSearcherParameters.setStatusFilter(StatusFilter.UNTRANSLATED); + textUnitSearcherParameters.setLocaleId(repositoryLocale.getLocale().getId()); + textUnitSearcherParameters.setLimit(sourceTextMaxCountPerLocale); + textUnitSearcherParameters.setUsedFilter(UsedFilter.USED); + List textUnitDTOS = textUnitSearcher.search(textUnitSearcherParameters); + + CreateBatchResponse createBatchResponse = null; + if (textUnitDTOS.isEmpty()) { + logger.debug("Nothing to translate, don't create a batch"); + } else { + logger.debug("Save the TextUnitDTOs in blob storage for later batch import"); + String batchId = + "%s_%s".formatted(repositoryLocale.getLocale().getBcp47Tag(), UUID.randomUUID()); + structuredBlobStorage.put( + AI_TRANSLATE_WS, + batchId, + objectMapper.writeValueAsStringUnchecked(new AiTranslateBlobStorage(textUnitDTOS)), + Retention.MIN_1_DAY); + + logger.debug("Generate the batch file content"); + String batchFileContent = generateBatchFileContent(textUnitDTOS); + + UploadFileResponse uploadFileResponse = + getOpenAIClient() + .uploadFile( + UploadFileRequest.forBatch("%s.jsonl".formatted(batchId), batchFileContent)); + + logger.debug("Create the batch using file: {}", uploadFileResponse); + createBatchResponse = + getOpenAIClient() + .createBatch( + forChatCompletion( + uploadFileResponse.id(), + Map.of(METADATA__TEXT_UNIT_DTOS__BLOB_ID, batchId))); + } + + logger.info( + "Created batch for locale: {} with {} text units", + repositoryLocale.getLocale().getBcp47Tag(), + textUnitDTOS.size()); + return createBatchResponse; + }; + } + + String generateBatchFileContent(List textUnitDTOS) { + return textUnitDTOS.stream() + .map( + textUnitDTO -> { + CompletionInput completionInput = + new CompletionInput( + textUnitDTO.getTargetLocale(), + textUnitDTO.getSource(), + textUnitDTO.getComment(), + new ExistingTarget( + textUnitDTO.getTarget(), !textUnitDTO.isIncludedInLocalizedFile())); + + String inputAsJsonString = objectMapper.writeValueAsStringUnchecked(completionInput); + + ObjectNode jsonSchema = createJsonSchema(CompletionOutput.class); + + ChatCompletionsRequest chatCompletionsRequest = + chatCompletionsRequest() + .model("gpt-4o-2024-08-06") + .maxTokens(16384) + .messages( + List.of( + systemMessageBuilder().content(PROMPT).build(), + userMessageBuilder().content(inputAsJsonString).build())) + .responseFormat( + new ChatCompletionsRequest.JsonFormat( + "json_schema", + new ChatCompletionsRequest.JsonFormat.JsonSchema( + true, "request_json_format", jsonSchema))) + .build(); + + return RequestBatchFileLine.forChatCompletion( + textUnitDTO.getTmTextUnitId().toString(), chatCompletionsRequest); + }) + .map(objectMapper::writeValueAsStringUnchecked) + .collect(joining("\n")); + } + + /** + * Use a queue to not stay stuck on a slow job, and try to import faster. Batch are imported + * sequentially. + * + *

Note: This is an active blocking pooling which blocks the thread but is isolated in a thread + * pool. + */ + RetrieveBatchResponse getNextFinishedBatch(ArrayDeque batches) { + while (true) { + int size = batches.size(); + + for (int i = 0; i < size; i++) { + CreateBatchResponse batch = batches.removeFirst(); + + logger.debug("Retrieve current status of batch: {}", batch.id()); + RetrieveBatchResponse retrieveBatchResponse = retrieveBatchWithRetry(batch); + + if ("completed".equals(retrieveBatchResponse.status())) { + logger.info("Next completed batch is: {}", retrieveBatchResponse.id()); + return retrieveBatchResponse; + } else if ("failed".equals(retrieveBatchResponse.status())) { + logger.error("Batch failed, skipping it: {}", retrieveBatchResponse); + } else { + logger.debug( + "Batch is still processing append to the end of the queue: {}", + retrieveBatchResponse); + batches.offerLast(batch); + } + } + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + RetrieveBatchResponse retrieveBatchWithRetry(CreateBatchResponse batch) { + + return Mono.fromCallable( + () -> getOpenAIClient().retrieveBatch(new RetrieveBatchRequest(batch.id()))) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> { + logger.info("Retrying retrieving batch: {}", batch.id()); + })) + .doOnError( + throwable -> new RuntimeException("Failed to retrieve batch: " + batch.id(), throwable)) + .block(); + } + + record CompletionInput( + String locale, String source, String sourceDescription, ExistingTarget existingTarget) { + record ExistingTarget(String content, boolean hasBrokenPlaceholders) {} + } + + record CompletionOutput( + String source, + Target target, + DescriptionRating descriptionRating, + AltTarget altTarget, + ExistingTargetRating existingTargetRating, + ReviewRequired reviewRequired) { + record Target(String content, String explanation, int confidenceLevel) {} + + record AltTarget(String content, String explanation, int confidenceLevel) {} + + record DescriptionRating(String explanation, int score) {} + + record ExistingTargetRating(String explanation, int score) {} + + record ReviewRequired(boolean required, String reason) {} + } + + record AiTranslateBlobStorage(List textUnitDTOS) {} + + static final String PROMPT = + """ + Your role is to act as a translator. + You are tasked with translating provided source strings while preserving both the tone and the technical structure of the string. This includes protecting any tags, placeholders, or code elements that should not be translated. + + The input will be provided in JSON format with the following fields: + + • "source": The source text to be translated. + • "locale": The target language locale, following the BCP47 standard (e.g., “fr”, “es-419”). + • "sourceDescription": A description providing context for the source text. + • "existingTarget" (optional): An existing translation to review. Indicates if it has broken placeholders. + + Instructions: + + • If the source is colloquial, keep the translation colloquial; if it’s formal, maintain formality in the translation. + • Pay attention to regional variations specified in the "locale" field (e.g., “es” vs. “es-419”, “fr” vs. “fr-CA”, “zh” vs. “zh-Hant”), and ensure the translation length remains similar to the source text. + + Handling Tags and Code: + + Some strings contain code elements such as tags (e.g., {atag}, ICU message format, or HTML tags). You are provided with a inputs of tags that need to be protected. Ensure that: + + • Tags like {atag} remain untouched. + • In cases of nested content (e.g., text that needs translation), only translate the inner text while preserving the outer structure. + • Complex structures like ICU message formats should have placeholders or variables left intact (e.g., {count, plural, one {# item} other {# items}}), but translate any inner translatable text. + • If an existing translation is provided and has broken placeholder, make sure to fix them in the new translation. + + Ambiguity and Context: + + After translating, assess the usefulness of the "sourceDescription" field: + + • Rate its usefulness on a scale of 0 to 2: + • 0 – Not helpful at all; irrelevant or misleading. + • 1 – Somewhat helpful; provides partial or unclear context but is useful to some extent. + • 2 – Very helpful; provides clear and sufficient guidance for the translation. + + If the source is ambiguous—for example, if it could be interpreted as a noun or a verb—you must: + + • Indicate the ambiguity in your explanation. + • Provide translations for all possible interpretations. + • Set "reviewRequired" to true, and explain the need for review due to the ambiguity. + + You will provide an output in JSON format with the following fields: + + • "source": The original source text. + • "target": An object containing: + • "content": The best translation. + • "explanation": A brief explanation of your translation choices. + • "confidenceLevel": Your confidence level (0-100%) in the translation. + • "descriptionRating": An object containing: + • "explanation": An explanation of how the "sourceDescription" aided your translation. + • "score": The usefulness score (0-2). + • "altTarget": An object containing: + • "content": An alternative translation, if applicable. Focus on showcasing grammar differences, + • "explanation": Explanation for the alternative translation. + • "confidenceLevel": Your confidence level (0-100%) in the alternative translation. + • "reviewRequired": An object containing: + • "required": true or false, indicating if review is needed. + • "reason": A detailed explanation of why review is or isn’t needed. + """; + + /** + * Typical configuration for the ObjectMapper needed by this class. + * + *

The ObjectMapper must not use indentation else Jsonl serialization will fail. + */ + public static void configureObjectMapper(ObjectMapper objectMapper) { + objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + objectMapper.disable(SerializationFeature.INDENT_OUTPUT); + objectMapper.registerModule(new JavaTimeModule()); + } + + OpenAIClient getOpenAIClient() { + if (openAIClient == null) { + String msg = + "OpenAI client is not configured for AiTranslateService. Ensure that the OpenAI API key is provided in the configuration (qualifier='aiTranslate')."; + logger.error(msg); + throw new RuntimeException(msg); + } + return openAIClient; + } + + public class AiTranslateException extends Exception { + public AiTranslateException(Throwable cause) { + super(cause); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/repository/RepositoryRepository.java b/webapp/src/main/java/com/box/l10n/mojito/service/repository/RepositoryRepository.java index c28d7969b8..c014d74621 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/repository/RepositoryRepository.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/repository/RepositoryRepository.java @@ -27,6 +27,8 @@ public interface RepositoryRepository @Override Optional findById(Long aLong); + Optional findNoGraphById(Long aLong); + @Override List findAll(Specification s, Sort sort); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java index 281cdbb1d3..0e8c07dc9c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java @@ -24,6 +24,7 @@ import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; import com.box.l10n.mojito.service.tm.textunitdtocache.TextUnitDTOsCacheService; import com.box.l10n.mojito.service.tm.textunitdtocache.UpdateType; +import com.box.l10n.mojito.utils.OptionsParser; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -268,6 +269,9 @@ void mapMojitoAndThirdPartyTextUnits( repository.getName(), projectId); + OptionsParser optionsParser = new OptionsParser(options); + Boolean deleteCurrentMapping = optionsParser.getBoolean("deleteCurrentMapping", false); + logger.debug("Get the text units of the third party TMS"); List thirdPartyTextUnits = thirdPartyTMS.getThirdPartyTextUnits(repository, projectId, options); @@ -278,7 +282,14 @@ void mapMojitoAndThirdPartyTextUnits( thirdPartyTextUnits.stream() .collect( groupingBy( - o -> assetCache.getUnchecked(o.getAssetPath()).orElse(null), + o -> + assetCache + .getUnchecked(o.getAssetPath()) + .orElseThrow( + () -> + new RuntimeException( + "Trying to map a third party text unit for an asset (%s) that does not exist in the repository" + .formatted(o.getAssetPath()))), LinkedHashMap::new, toList())); @@ -287,16 +298,31 @@ void mapMojitoAndThirdPartyTextUnits( thirdPartyTextUnitsByAsset.entrySet().stream() .filter(e -> e.getKey() != null) .forEach( - e -> mapThirdPartyTextUnitsToTextUnitDTOs(e.getKey(), e.getValue(), pluralSeparator)); + e -> + mapThirdPartyTextUnitsToTextUnitDTOs( + e.getKey(), e.getValue(), pluralSeparator, deleteCurrentMapping)); } void mapThirdPartyTextUnitsToTextUnitDTOs( - Asset asset, List thirdPartyTextUnitsToMap, String pluralSeparator) { + Asset asset, + List thirdPartyTextUnitsToMap, + String pluralSeparator, + boolean deleteCurrentMapping) { logger.debug("Map third party text units to text unit DTOs for asset: {}", asset.getId()); - Set alreadyMappedTmTextUnitId = - thirdPartyTextUnitRepository.findTmTextUnitIdsByAsset(asset); - Boolean allWithTmTextUnitId = + Set alreadyMappedTmTextUnitId; + + if (deleteCurrentMapping) { + logger.info("Delete existing ThirdPartyTextUnit mapping for asset id: {}", asset.getId()); + int deletedCount = thirdPartyTextUnitRepository.deleteByAssetId(asset.getId()); + logger.info( + "Deleted {} ThirdPartyTextUnit mappings for asset id: {}", deletedCount, asset.getId()); + alreadyMappedTmTextUnitId = Collections.emptySet(); + } else { + alreadyMappedTmTextUnitId = thirdPartyTextUnitRepository.findTmTextUnitIdsByAsset(asset); + } + + boolean allWithTmTextUnitId = thirdPartyTextUnitsToMap.stream() .map(ThirdPartyTextUnit::getTmTextUnitId) .allMatch(Objects::nonNull); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java new file mode 100644 index 0000000000..c95db2112c --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -0,0 +1,697 @@ +package com.box.l10n.mojito.service.thirdparty; + +import com.box.l10n.mojito.JSR310Migration; +import com.box.l10n.mojito.android.strings.AndroidStringDocument; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentReader; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter.EscapeType; +import com.box.l10n.mojito.entity.PollableTask; +import com.box.l10n.mojito.entity.Repository; +import com.box.l10n.mojito.entity.RepositoryLocale; +import com.box.l10n.mojito.service.pollableTask.PollableFuture; +import com.box.l10n.mojito.service.repository.RepositoryService; +import com.box.l10n.mojito.service.thirdparty.phrase.PhraseClient; +import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; +import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType; +import com.box.l10n.mojito.service.tm.search.SearchType; +import com.box.l10n.mojito.service.tm.search.StatusFilter; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.box.l10n.mojito.service.tm.search.UsedFilter; +import com.box.l10n.mojito.utils.OptionsParser; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; +import com.phrase.client.model.Tag; +import com.phrase.client.model.TranslationKey; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@ConditionalOnProperty(value = "l10n.ThirdPartyTMS.impl", havingValue = "ThirdPartyTMSPhrase") +@Component +public class ThirdPartyTMSPhrase implements ThirdPartyTMS { + + static final String TAG_PREFIX = "push_"; + static final String TAG_PREFIX_WITH_REPOSITORY = "push_%s"; + static final String TAG_DATE_FORMAT = "yyyy_MM_dd_HH_mm_ss_SSS"; + static final boolean NATIVE_CLIENT_DEFAULT_VALUE = false; + + static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhrase.class); + + TextUnitSearcher textUnitSearcher = new TextUnitSearcher(); + + TextUnitBatchImporterService textUnitBatchImporterService; + + PhraseClient phraseClient; + + RepositoryService repositoryService; + + MeterRegistry meterRegistry; + + public ThirdPartyTMSPhrase( + TextUnitSearcher textUnitSearcher, + TextUnitBatchImporterService textUnitBatchImporterService, + PhraseClient phraseClient, + RepositoryService repositoryService, + MeterRegistry meterRegistry) { + + this.textUnitSearcher = textUnitSearcher; + this.textUnitBatchImporterService = textUnitBatchImporterService; + this.phraseClient = phraseClient; + this.repositoryService = repositoryService; + this.meterRegistry = meterRegistry; + } + + @Override + public void removeImage(String projectId, String imageId) { + throw new UnsupportedOperationException("Remove image is not supported"); + } + + @Override + public ThirdPartyTMSImage uploadImage(String projectId, String name, byte[] content) { + throw new UnsupportedOperationException("Upload image is not supported"); + } + + @Override + public List getThirdPartyTextUnits( + Repository repository, String projectId, List optionList) { + + List thirdPartyTextUnits = new ArrayList<>(); + + String currentTagsForRepository = getCurrentTagsForRepository(repository, projectId); + + List phraseTranslationKeys = + phraseClient.getKeys(projectId, currentTagsForRepository); + + for (TranslationKey translationKey : phraseTranslationKeys) { + + String[] nameParts = translationKey.getName().split("#@#", 3); + + if (nameParts.length != 3) { + logger.error( + "Skipping entry. Name: {} should have 3 parts. Missing part could happen in old project", + translationKey.getName()); + continue; + } + + String idSection = nameParts[0]; + if (!idSection.contains(",")) { + ThirdPartyTextUnit thirdPartyTextUnit = new ThirdPartyTextUnit(); + thirdPartyTextUnit.setAssetPath(nameParts[1]); + thirdPartyTextUnit.setName(nameParts[2]); + thirdPartyTextUnit.setId(translationKey.getId()); + thirdPartyTextUnit.setTmTextUnitId(Long.valueOf(idSection)); + thirdPartyTextUnits.add(thirdPartyTextUnit); + } else { + List ids = Arrays.stream(idSection.split(",")).map(Long::valueOf).toList(); + for (Long id : ids) { + ThirdPartyTextUnit thirdPartyTextUnit = new ThirdPartyTextUnit(); + thirdPartyTextUnit.setAssetPath(nameParts[1]); + thirdPartyTextUnit.setName(nameParts[2]); + thirdPartyTextUnit.setId(translationKey.getId()); + thirdPartyTextUnit.setTmTextUnitId(id); + thirdPartyTextUnits.add(thirdPartyTextUnit); + } + } + } + + return thirdPartyTextUnits; + } + + @Override + public void createImageToTextUnitMappings( + String projectId, List thirdPartyImageToTextUnits) { + throw new UnsupportedOperationException("Create image to text units is not supported"); + } + + @Override + public void push( + Repository repository, + String projectId, + String pluralSeparator, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + List options) { + + OptionsParser optionsParser = new OptionsParser(options); + Boolean nativeClient = optionsParser.getBoolean("nativeClient", NATIVE_CLIENT_DEFAULT_VALUE); + Map formatOptions = getFormatOptions(optionsParser); + final AtomicReference escapeType = + getEscapeTypeAtomicReference(nativeClient, optionsParser); + + Stopwatch stopwatchGetTextUnitDTO = Stopwatch.createStarted(); + List search = + getSourceTextUnitDTOs(repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); + logger.info("Get Source TextUnitDTO for push took: {}", stopwatchGetTextUnitDTO.elapsed()); + String text = getFileContent(pluralSeparator, search, true, null, escapeType.get()); + + String tagForUpload = getTagForUpload(repository.getName()); + + if (nativeClient) { + logger.debug("Pushing with native and options: {}", formatOptions); + Stopwatch stopWatchPhaseNativePush = Stopwatch.createStarted(); + phraseClient.nativeUploadAndWait( + projectId, + repository.getSourceLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + text, + ImmutableList.of(tagForUpload), + formatOptions.isEmpty() ? null : formatOptions); + logger.info( + "Pushing with native and options: {}, took: {}", + formatOptions, + stopWatchPhaseNativePush.elapsed()); + } else { + phraseClient.uploadAndWait( + projectId, + repository.getSourceLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + text, + ImmutableList.of(tagForUpload), + null); + } + + boolean skipRemoveUnusedKeysAndTags = + optionsParser.getBoolean("skipRemoveUnusedKeysAndTags", false); + + if (!skipRemoveUnusedKeysAndTags) { + logger.info("Skipping removing unused keys and tags"); + removeUnusedKeysAndTags(projectId, repository.getName(), tagForUpload); + } + } + + /** + * Remove unused keys and tags + * + *

    + *
  • Remove Unused Keys: + *
      + *
    • Unused keys in a Phrase project are defined as keys that are not tagged with the + * latest push tag from any Mojito repository. + *
    • To identify unused keys, fetch the current push tags of all repositories, excluding + * the push tag related to the Mojito repository being processed, and include the + * current upload tag. + *
    + *
  • Manage Tags: + *
      + *
    • Ensure that there is only one active "push" tag per repository. + *
    • Remove old tags that are prefixed with "push" but not active + *
    + *
+ * + *

Explanation: + * + *

The N:1 relationship between Mojito and Phrase is maintained through this logic, allowing + * for efficient management of keys and tags. + */ + public void removeUnusedKeysAndTags( + String projectId, String repositoryName, String tagForUpload) { + + Stopwatch stopwatchRemoveUnusedKeysAndTags = Stopwatch.createStarted(); + + List tagsForOtherRepositories = + phraseClient.listTags(projectId).stream() + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> tagName.startsWith(TAG_PREFIX)) + .filter(tagName -> !tagName.startsWith(getTagNamePrefixForRepository(repositoryName))) + .toList(); + + List allActiveTags = new ArrayList<>(tagsForOtherRepositories); + allActiveTags.add(tagForUpload); + + logger.debug("All active tags: {}", allActiveTags); + phraseClient.removeKeysNotTaggedWith(projectId, allActiveTags); + + List pushTagsToDelete = + phraseClient.listTags(projectId).stream() + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> tagName.startsWith(TAG_PREFIX)) + .filter(tagName -> !allActiveTags.contains(tagName)) + // That's to handle concurrent sync and make sure a tag that was just pushed is not + // deleted before the pull from the same sync is finished. + // We don't prevent syncs to run concurrently + .filter(tagName -> !areTagsWithin5Minutes(tagForUpload, tagName)) + .toList(); + + logger.info("Tags to delete: {}", pushTagsToDelete); + phraseClient.deleteTags(projectId, pushTagsToDelete); + logger.info("RemoveUnusedKeysAndTags took: {}", stopwatchRemoveUnusedKeysAndTags); + } + + static boolean areTagsWithin5Minutes(String tagName1, String tagName2) { + return Duration.between(uploadTagToLocalDateTime(tagName2), uploadTagToLocalDateTime(tagName1)) + .abs() + .getSeconds() + < (60 * 5); + } + + private List getSourceTextUnitDTOs( + Repository repository, String skipTextUnitsWithPattern, String skipAssetsWithPathPattern) { + TextUnitSearcherParameters parameters = new TextUnitSearcherParameters(); + + parameters.setRepositoryIds(repository.getId()); + parameters.setForRootLocale(true); + parameters.setDoNotTranslateFilter(false); + parameters.setUsedFilter(UsedFilter.USED); + parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); + parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + parameters.setPluralFormsFiltered(false); + parameters.setOrderByTextUnitID(true); + + return textUnitSearcher.search(parameters); + } + + private List getSourceTextUnitDTOsPluralOnly( + Repository repository, String skipTextUnitsWithPattern, String skipAssetsWithPathPattern) { + + TextUnitSearcherParameters parameters = new TextUnitSearcherParameters(); + + parameters.setRepositoryIds(repository.getId()); + parameters.setForRootLocale(true); + parameters.setDoNotTranslateFilter(false); + parameters.setUsedFilter(UsedFilter.USED); + parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); + parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + parameters.setSearchType(SearchType.ILIKE); + parameters.setPluralFormsFiltered(false); + parameters.setPluralFormOther("%"); + + return textUnitSearcher.search(parameters); + } + + public static String getTagForUpload(String repositoryName) { + ZonedDateTime zonedDateTime = JSR310Migration.dateTimeNowInUTC(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(TAG_DATE_FORMAT); + return normalizeTagName( + "%s%s_%s_%s" + .formatted( + TAG_PREFIX, + repositoryName, + formatter.format(zonedDateTime), + Math.abs(UUID.randomUUID().getLeastSignificantBits() % 1000))); + } + + public static LocalDateTime uploadTagToLocalDateTime(String tag) { + + if (tag == null || !tag.contains("_")) { + throw new IllegalArgumentException("Invalid tag format: " + tag); + } + + int dateEndIndex = tag.lastIndexOf('_'); // last part is a random number + int dateStartIndex = dateEndIndex; + + for (int i = 0; i < 7; i++) { + dateStartIndex = tag.lastIndexOf('_', dateStartIndex - 1); + if (dateStartIndex == -1) { + throw new IllegalArgumentException("Invalid tag format: " + tag); + } + } + + String dateTimePart = tag.substring(dateStartIndex + 1, dateEndIndex); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(TAG_DATE_FORMAT); + return LocalDateTime.parse(dateTimePart, formatter); + } + + private static String getTagNamePrefixForRepository(String repositoryName) { + return normalizeTagName(TAG_PREFIX_WITH_REPOSITORY.formatted(repositoryName)); + } + + /** + * At least "/" are getting converted by phrase into "_", apply that logic on our side to have + * consistent filtering + */ + private static String normalizeTagName(String repositoryName) { + return repositoryName.replace("/", "_"); + } + + @Override + public PollableFuture pull( + Repository repository, + String projectId, + String pluralSeparator, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + List optionList, + String schedulerName, + PollableTask currentTask) { + + Set repositoryLocalesWithoutRootLocale = + repositoryService.getRepositoryLocalesWithoutRootLocale(repository); + + String currentTags = getCurrentTagsForRepository(repository, projectId); + + OptionsParser optionsParser = new OptionsParser(optionList); + Boolean integrityCheckKeepStatusIfFailedAndSameTarget = + optionsParser.getBoolean("integrityCheckKeepStatusIfFailedAndSameTarget", true); + + AtomicReference integrityChecksType = + new AtomicReference<>(IntegrityChecksType.KEEP_STATUS_IF_SAME_TARGET); + optionsParser.getString( + "integrityChecksType", s -> integrityChecksType.set(IntegrityChecksType.valueOf(s))); + + // may already hit rate limit, according it is 4 qps ... there is a retry in the locale client + // though. + // but 4 qps is very low to download 64 locales. + repositoryLocalesWithoutRootLocale.parallelStream() + .forEach( + locale -> + pullLocaleTimed( + repository, + projectId, + locale, + pluralSeparator, + currentTags, + integrityChecksType.get(), + optionList)); + + return null; + } + + private void pullLocaleTimed( + Repository repository, + String projectId, + RepositoryLocale repositoryLocale, + String pluralSeparator, + String currentTags, + IntegrityChecksType integrityChecksType, + List optionList) { + try (var timer = + Timer.resource(meterRegistry, "ThirdPartyTMSPhrase.pullLocale") + .tag("repository", repository.getName())) { + + pullLocale( + repository, + projectId, + repositoryLocale, + pluralSeparator, + currentTags, + integrityChecksType, + optionList); + } + } + + private void pullLocale( + Repository repository, + String projectId, + RepositoryLocale repositoryLocale, + String pluralSeparator, + String currentTags, + IntegrityChecksType integrityChecksType, + List optionList) { + + String localeTag = repositoryLocale.getLocale().getBcp47Tag(); + logger.info("Downloading locale: {} from Phrase with tags: {}", localeTag, currentTags); + + OptionsParser optionsParser = new OptionsParser(optionList); + Boolean nativeClient = optionsParser.getBoolean("nativeClient", NATIVE_CLIENT_DEFAULT_VALUE); + Boolean escapeTags = optionsParser.getBoolean("escapeTags", true); + Boolean escapeLineBreaks = optionsParser.getBoolean("escapeLineBreaks", false); + Map formatOptions = new HashMap<>(); + if (escapeTags) { + formatOptions.put("escape_tags", "true"); + } + if (escapeLineBreaks) { + formatOptions.put("escape_linebreaks", "true"); + } + final AtomicReference escapeTypeAtomicReference = + getEscapeTypeAtomicReference(nativeClient, optionsParser); + + Stopwatch localeDownloadStopWatch = Stopwatch.createStarted(); + String fileContent; + + if (nativeClient) { + logger.info("Pulling locale with native and options: {}", formatOptions); + + fileContent = + phraseClient.nativeLocaleDownload( + projectId, + localeTag, + "xml", + currentTags, + formatOptions, + () -> getCurrentTagsForRepository(repository, projectId)); + } else { + fileContent = + phraseClient.localeDownload( + projectId, + localeTag, + "xml", + currentTags, + () -> getCurrentTagsForRepository(repository, projectId)); + } + + logger.info("Phrase locale download took: {}", localeDownloadStopWatch.elapsed()); + + logger.debug("file content from pull: {}", fileContent); + + AndroidStringDocumentMapper mapper = + new AndroidStringDocumentMapper( + pluralSeparator, null, localeTag, repository.getName(), true, null); + + List textUnitDTOS = + mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); + + // TODO(ja) that should not be necessary since the TextUnitBatchImporterService was fixed? + // make sure that the source comment does not get replicated to all translations. Eventually + // we may want to + // communicate some comments, but anyway it should not be the source comment + textUnitDTOS.forEach(t -> t.setComment(null)); + + Stopwatch importStopWatch = Stopwatch.createStarted(); + + textUnitBatchImporterService.importTextUnits(textUnitDTOS, integrityChecksType); + logger.info("Time importing text units: {}", importStopWatch.elapsed()); + } + + /** + * It should typically return a single tag, but if concurrent syncs, it could return more. That + * should not be a problem when downloading or keys for mapping or pulling strings + */ + private String getCurrentTagsForRepository(Repository repository, String projectId) { + String tagNamePrefixForRepository = getTagNamePrefixForRepository(repository.getName()); + String tags = + phraseClient.listTags(projectId).stream() + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> tagName.startsWith(tagNamePrefixForRepository)) + .collect(Collectors.joining(",")); + + if (tags.isBlank()) { + throw new RuntimeException( + "There are no current tags for the repository, make sure push was run first or that the repository name does not contain special character (which will need to get normalized)"); + } + + return tags; + } + + @Override + public void pushTranslations( + Repository repository, + String projectId, + String pluralSeparator, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + List optionList) { + + OptionsParser optionsParser = new OptionsParser(optionList); + Boolean nativeClient = optionsParser.getBoolean("nativeClient", NATIVE_CLIENT_DEFAULT_VALUE); + Map formatOptions = getFormatOptions(optionsParser); + + final AtomicReference escapeType = + getEscapeTypeAtomicReference(nativeClient, optionsParser); + + List search = + getSourceTextUnitDTOs(repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); + + List pluralTextUnitDTOs = + getSourceTextUnitDTOsPluralOnly( + repository, skipTextUnitsWithPattern, skipAssetsWithPathPattern); + + Map> pluralFormOtherToTextUnitDTO = + pluralTextUnitDTOs.stream() + .collect( + Collectors.groupingBy( + AndroidStringDocumentMapper::getKeyToGroupByPluralOtherAndComment)); + + pluralFormOtherToTextUnitDTO.forEach( + (key, value) -> { + if (value.size() != 6) { + throw new RuntimeException("there must be only 6 text units per PluralFormOther value"); + } + }); + + Map pluralFormToCommaId = + pluralFormOtherToTextUnitDTO.entrySet().stream() + .map( + e -> + new SimpleEntry<>( + e.getKey(), + e.getValue().stream() + .sorted(new ByPluralFormComparator()) + .map(TextUnitDTO::getTmTextUnitId) + .map(String::valueOf) + .collect(Collectors.joining(",")))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Set repositoryLocalesWithoutRootLocale = + repositoryService.getRepositoryLocalesWithoutRootLocale(repository); + + for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale) { + List textUnitDTOS = + getTextUnitDTOSForLocale( + repository, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern, + repositoryLocale); + + if (textUnitDTOS.isEmpty()) { + logger.info("Not translation, skip upload"); + } else { + String fileContent = + getFileContent( + pluralSeparator, textUnitDTOS, false, pluralFormToCommaId, escapeType.get()); + logger.info("Push translation to phrase:\n{}", fileContent); + + if (nativeClient) { + logger.info("Pushing translations with native and options: {}", formatOptions); + phraseClient.nativeUploadAndWait( + projectId, + repositoryLocale.getLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + fileContent, + null, + formatOptions.isEmpty() ? null : formatOptions); + } else { + phraseClient.uploadAndWait( + projectId, + repositoryLocale.getLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + fileContent, + null, + null); + } + } + } + } + + private List getTextUnitDTOSForLocale( + Repository repository, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + RepositoryLocale repositoryLocale) { + TextUnitSearcherParameters parameters = new TextUnitSearcherParameters(); + parameters.setRepositoryIds(repository.getId()); + parameters.setLocaleId(repositoryLocale.getLocale().getId()); + parameters.setDoNotTranslateFilter(false); + parameters.setStatusFilter(StatusFilter.TRANSLATED); + parameters.setUsedFilter(UsedFilter.USED); + parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); + parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + parameters.setIncludeTextUnitsWithPattern(includeTextUnitsWithPattern); + parameters.setPluralFormsFiltered(true); + return textUnitSearcher.search(parameters); + } + + private static String getFileContent( + String pluralSeparator, + List textUnitDTOS, + boolean useSource, + Map pluralFormToCommaId, + EscapeType escapeType) { + + AndroidStringDocumentMapper androidStringDocumentMapper = + new AndroidStringDocumentMapper( + pluralSeparator, null, null, null, true, pluralFormToCommaId); + + AndroidStringDocument androidStringDocument = + androidStringDocumentMapper.readFromTextUnits(textUnitDTOS, useSource); + + return new AndroidStringDocumentWriter(androidStringDocument, escapeType).toText(); + } + + @Override + public void pullSource( + Repository repository, + String projectId, + List optionList, + Map localeMapping) { + throw new UnsupportedOperationException("Pull source is not supported"); + } + + private static Map getFormatOptions(OptionsParser optionsParser) { + Boolean unescapeTags = optionsParser.getBoolean("unescapeTags", true); + Boolean unescapeLineBreaks = optionsParser.getBoolean("unescapeLineBreaks", false); + Map formatOptions = new HashMap<>(); + if (unescapeTags) { + formatOptions.put("unescape_tags", "true"); + } + + if (unescapeLineBreaks) { + formatOptions.put("unescape_linebreaks", "true"); + } + return formatOptions; + } + + private static AtomicReference getEscapeTypeAtomicReference( + Boolean nativeClient, OptionsParser optionsParser) { + final AtomicReference escapeType = + new AtomicReference<>(EscapeType.QUOTE_AND_NEW_LINE); + if (nativeClient) { + escapeType.set(EscapeType.NEW_LINE); + } + optionsParser.getString("escapeType", s -> escapeType.set(EscapeType.valueOf(s))); + return escapeType; + } + + static class ByPluralFormComparator implements Comparator { + + private final Map orderMap; + + public ByPluralFormComparator() { + this.orderMap = new HashMap<>(); + int i = 0; + for (String v : Arrays.asList("zero", "one", "two", "few", "many", "other")) { + this.orderMap.put(v, i++); + } + } + + @Override + public int compare(TextUnitDTO o1, TextUnitDTO o2) { + return Integer.compare( + orderMap.getOrDefault(o1.getPluralForm(), -1), + orderMap.getOrDefault(o2.getPluralForm(), -1)); + } + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java index b2f0c951df..8457edc967 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java @@ -50,8 +50,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.TransformerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -472,17 +470,10 @@ private SmartlingFile uploadTextUnitsToSmartling( SmartlingFile file = new SmartlingFile(); file.setFileName(getOutputSourceFile(batchNumber, repository.getName(), filePrefix.getType())); - try { - - logger.debug("Save source file to: {}", file.getFileName()); - AndroidStringDocumentWriter writer = - new AndroidStringDocumentWriter(mapper.readFromSourceTextUnits(result)); - file.setFileContent(writer.toText()); - - } catch (ParserConfigurationException | TransformerException e) { - logger.error("An error occurred when processing a push batch", e); - throw new RuntimeException(e); - } + logger.debug("Save source file to: {}", file.getFileName()); + AndroidStringDocumentWriter writer = + new AndroidStringDocumentWriter(mapper.readFromSourceTextUnits(result)); + file.setFileContent(writer.toText()); if (!options.isDryRun()) { Mono.fromCallable( @@ -753,27 +744,19 @@ private SmartlingFile processTranslationBatch( SmartlingFile file = new SmartlingFile(); file.setFileName(targetFilename); - try { - - logger.debug("Save target file to: {}", file.getFileName()); + logger.debug("Save target file to: {}", file.getFileName()); - if (filterTmTextUnitIds != null) { - fileBatch = - fileBatch.stream() - .filter( - textUnitDTO -> filterTmTextUnitIds.contains(textUnitDTO.getTmTextUnitId())) - .collect(Collectors.toList()); - } - - AndroidStringDocumentWriter writer = - new AndroidStringDocumentWriter(mapper.readFromTargetTextUnits(batch)); - file.setFileContent(writer.toText()); - - } catch (ParserConfigurationException | TransformerException e) { - logger.error("An error occurred when processing a push_translations batch", e); - throw new RuntimeException(e); + if (filterTmTextUnitIds != null) { + fileBatch = + fileBatch.stream() + .filter(textUnitDTO -> filterTmTextUnitIds.contains(textUnitDTO.getTmTextUnitId())) + .collect(Collectors.toList()); } + AndroidStringDocumentWriter writer = + new AndroidStringDocumentWriter(mapper.readFromTargetTextUnits(batch)); + file.setFileContent(writer.toText()); + if (!options.isDryRun()) { logger.debug( "Push Android file to Smartling project: {} and locale: {}", projectId, localeTag); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java index 4343368218..682479d36f 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java @@ -2,6 +2,7 @@ import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSUtils.isFileEqualToPreviousRun; import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.isPluralFile; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.entity.RepositoryLocale; @@ -250,7 +251,9 @@ && isFileEqualToPreviousRun( t.setRepositoryName(repository.getName()); t.setTargetLocale(repositoryLocale.getLocale().getBcp47Tag()); }); - textUnitBatchImporterService.importTextUnits(textUnitDTOS, false, true); + + textUnitBatchImporterService.importTextUnits( + textUnitDTOS, fromLegacy(false, true)); }); }); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnitRepository.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnitRepository.java index 7bad9752ec..68e798a00c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnitRepository.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnitRepository.java @@ -12,7 +12,9 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.data.rest.core.annotation.RepositoryRestResource; +import org.springframework.transaction.annotation.Transactional; /** * @author jeanaurambault @@ -32,4 +34,7 @@ public interface ThirdPartyTextUnitRepository @Override @EntityGraph(value = "ThirdPartyTextUnit.legacy", type = EntityGraphType.FETCH) List findAll(); + + @Transactional + int deleteByAssetId(@Param("assetId") long assetId); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java new file mode 100644 index 0000000000..1756f5b2a6 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThridPartyTMSPhraseException.java @@ -0,0 +1,11 @@ +package com.box.l10n.mojito.service.thirdparty; + +public class ThridPartyTMSPhraseException extends RuntimeException { + public ThridPartyTMSPhraseException(String msg, Throwable e) { + super(msg, e); + } + + public ThridPartyTMSPhraseException(String message) { + super(message); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java new file mode 100644 index 0000000000..71d5259543 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -0,0 +1,726 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import static com.box.l10n.mojito.io.Files.createDirectories; +import static com.box.l10n.mojito.io.Files.createTempDirectory; +import static com.box.l10n.mojito.io.Files.write; + +import com.box.l10n.mojito.json.ObjectMapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableSet; +import com.phrase.client.ApiClient; +import com.phrase.client.ApiException; +import com.phrase.client.api.KeysApi; +import com.phrase.client.api.LocalesApi; +import com.phrase.client.api.TagsApi; +import com.phrase.client.api.UploadsApi; +import com.phrase.client.auth.ApiKeyAuth; +import com.phrase.client.model.Tag; +import com.phrase.client.model.TranslationKey; +import com.phrase.client.model.Upload; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Mono; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +public class PhraseClient { + + static Logger logger = LoggerFactory.getLogger(PhraseClient.class); + + static final int BATCH_SIZE = 100; + + final ApiClient apiClient; + + final RetryBackoffSpec retryBackoffSpec; + + public PhraseClient(ApiClient apiClient) { + this.apiClient = apiClient; + this.retryBackoffSpec = + Retry.backoff(5, Duration.ofMillis(500)).maxBackoff(Duration.ofSeconds(5)); + } + + public Upload nativeUploadAndWait( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + Map formatOptions) { + + String uploadId = + nativeUploadCreateFileWithRetry( + projectId, localeId, fileFormat, fileName, fileContent, tags, formatOptions); + return waitForUploadToFinish(projectId, uploadId); + } + + public Upload uploadAndWait( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + String formatOptions) { + + String uploadId = + uploadCreateFile( + projectId, localeId, fileFormat, fileName, fileContent, tags, formatOptions); + return waitForUploadToFinish(projectId, uploadId); + } + + Upload waitForUploadToFinish(String projectId, String uploadId) { + UploadsApi uploadsApi = new UploadsApi(apiClient); + try { + logger.debug("Waiting for upload to finish: {}", uploadId); + + Stopwatch stopwatch = Stopwatch.createStarted(); + + Upload upload = uploadsApi.uploadShow(projectId, uploadId, null, null); + logger.debug( + "Upload info, first fetch: {}", new ObjectMapper().writeValueAsStringUnchecked(upload)); + + while (!ImmutableSet.of("success", "error").contains(upload.getState())) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + upload = uploadsApi.uploadShow(projectId, uploadId, null, null); + logger.debug( + "upload info after polling for success or error: {}", + new ObjectMapper().writeValueAsStringUnchecked(upload)); + } + + if ("error".equals(upload.getState())) { + throw new PhraseClientException( + "Upload failed: %s".formatted(new ObjectMapper().writeValueAsStringUnchecked(upload))); + } + + logger.info("Waited: {} for upload: {} to finish", stopwatch.elapsed(), uploadId); + + return upload; + } catch (ApiException e) { + logger.error("Error calling Phrase for waitForUploadToFinish: {}", e.getResponseBody()); + throw new PhraseClientException(e); + } + } + + String uploadCreateFile( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + String formatOptions) { + + Path tmpWorkingDirectory = null; + + logger.info( + "uploadCreateFile: projectId: {}, localeId: {}, fileName: {}, tags: {}", + projectId, + localeId, + fileName, + tags); + + try { + tmpWorkingDirectory = createTempDirectory("phrase-integration"); + + if (tmpWorkingDirectory.toFile().exists()) { + logger.debug("Created temporary working directory: {}", tmpWorkingDirectory); + } + + Path fileToUpload = tmpWorkingDirectory.resolve(fileName); + + logger.debug("Create file: {}", fileToUpload); + createDirectories(fileToUpload.getParent()); + write(fileToUpload, fileContent); + + Upload upload = + uploadsApiUploadCreateWithRetry( + projectId, localeId, fileFormat, tags, fileToUpload, formatOptions); + + return upload.getId(); + } finally { + if (tmpWorkingDirectory != null) { + com.box.l10n.mojito.io.Files.deleteRecursivelyIfExists(tmpWorkingDirectory); + } + } + } + + public String nativeUploadCreateFileWithRetry( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + Map formatOptions) { + + logger.info( + "nativeUploadCreateFile: projectId: {}, localeId: {}, fileName: {}, tags: {}", + projectId, + localeId, + fileName, + tags); + + return Mono.fromCallable( + () -> + nativeUploadCreateFile( + projectId, localeId, fileFormat, fileName, fileContent, tags, formatOptions)) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to uploadCreate to Phrase, file: %s, project id: %s" + .formatted(fileName, projectId)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error in UploadCreate from Phrase, file: %s, project id: %s" + .formatted(fileName, projectId))) + .block(); + } + + /** + * The official SDK does not support format_options properly, so adding a replacement method base + * on pure Java client + */ + public String nativeUploadCreateFile( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + Map formatOptions) { + + Stopwatch stopwatch = Stopwatch.createStarted(); + + String urlString = String.format("%s/projects/%s/uploads", apiClient.getBasePath(), projectId); + String boundary = UUID.randomUUID().toString(); + final String LINE_FEED = "\r\n"; + + StringBuilder multipartBody = new StringBuilder(); + + multipartBody.append("--").append(boundary).append(LINE_FEED); + multipartBody + .append("Content-Disposition: form-data; name=\"file\"; filename=\"") + .append(fileName) + .append("\"") + .append(LINE_FEED); + multipartBody.append("Content-Type: application/xml").append(LINE_FEED); + multipartBody.append(LINE_FEED); + multipartBody.append(fileContent).append(LINE_FEED); + + addFormField(multipartBody, boundary, "locale_id", localeId); + addFormField(multipartBody, boundary, "file_format", fileFormat); + addFormField(multipartBody, boundary, "update_translations", "true"); + addFormField(multipartBody, boundary, "update_descriptions", "true"); + + if (tags != null) { + String tagsString = String.join(",", tags); + addFormField(multipartBody, boundary, "tags", tagsString); + } + + if (formatOptions != null) { + for (Map.Entry e : formatOptions.entrySet()) { + addFormField( + multipartBody, boundary, "format_options[%s]".formatted(e.getKey()), e.getValue()); + } + } + multipartBody.append("--").append(boundary).append("--").append(LINE_FEED); + + String token = ((ApiKeyAuth) apiClient.getAuthentication("Token")).getApiKey(); + + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(urlString)) + .header("Authorization", "token " + token) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST( + HttpRequest.BodyPublishers.ofString( + multipartBody.toString(), StandardCharsets.UTF_8)) + .build(); + + HttpResponse response; + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + logger.info("nativeUploadCreateFile took: {}", stopwatch.elapsed()); + + int statusCode = response.statusCode(); + String responseBody = response.body(); + + if (statusCode == 201) { + JsonNode rootNode = new ObjectMapper().readTreeUnchecked(responseBody); + return rootNode.path("id").asText(); + } else { + throw new RuntimeException("Server returned status code " + statusCode + ": " + responseBody); + } + } + + /** + * Helper method to add a form field to the multipart body. + * + * @param builder The StringBuilder for the multipart body. + * @param boundary The boundary string. + * @param name The name of the form field. + * @param value The value of the form field. + */ + private static void addFormField( + StringBuilder builder, String boundary, String name, String value) { + String LINE_FEED = "\r\n"; + builder.append("--").append(boundary).append(LINE_FEED); + builder + .append("Content-Disposition: form-data; name=\"") + .append(name) + .append("\"") + .append(LINE_FEED); + builder.append(LINE_FEED); + builder.append(value).append(LINE_FEED); + } + + Upload uploadsApiUploadCreateWithRetry( + String projectId, + String localeId, + String fileFormat, + List tags, + Path fileToUpload, + String formatOptions) { + + return Mono.fromCallable( + () -> + new UploadsApi(apiClient) + .uploadCreate( + projectId, + fileToUpload.toFile(), + fileFormat, + localeId, + null, + null, + tags == null ? null : String.join(",", tags), + true, + true, + null, + null, + null, + null, + null, + formatOptions, + null, + null, + null)) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to uploadCreate to Phrase, file: %s, project id: %s" + .formatted(fileToUpload.toAbsolutePath(), projectId)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error in UploadCreate from Phrase, file: %s, project id: %s" + .formatted(fileToUpload.toAbsolutePath(), projectId))) + .block(); + } + + /** + * Conducted tests on keysDeleteCollection using the -tags:tag1,tag2 option, and it + * operates as a filter for keys "not in any of the tags". + * + *

It removes keys that are not tagged with any of the provided tags. + * + *

For example: + * + *

    + *
  • If key ka has tag tag1, + *
  • If key kb has tag tag2, + *
  • If key kc has tag tag3, + *
+ * + *

Calling keysDeleteCollection with -tags:tag1,tag3 will remove + * kb. + */ + public void removeKeysNotTaggedWith(String projectId, List anyOfTheseTags) { + logger.info("Removing keys not tagged with any of the following tags: {}", anyOfTheseTags); + + Mono.fromCallable( + () -> { + KeysApi keysApi = new KeysApi(apiClient); + keysApi.keysDeleteCollection( + projectId, + null, + null, + "-tags:%s".formatted(String.join(",", anyOfTheseTags)), + null); + return null; + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to removeKeysNotTaggedWith from Phrase, project id: %s" + .formatted(projectId)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to removeKeysNotTaggedWith from Phrase, project id: %s" + .formatted(projectId))) + .block(); + } + + /** + * @param onTagErrorRefreshCallback with concurrent update, the tags could be updated during + * download. We don't want to retry the whole logic, so we provide a callback to refresh the + * tags + */ + public String localeDownload( + String projectId, + String locale, + String fileFormat, + String tags, + Supplier onTagErrorRefreshCallback) { + AtomicReference refTags = new AtomicReference<>(tags); + return Mono.fromCallable( + () -> { + LocalesApi localesApi = new LocalesApi(apiClient); + logger.info( + "Downloading locale: {} from project id: {} in file format: {}", + locale, + projectId, + fileFormat); + File file = + localesApi.localeDownload( + projectId, + locale, + null, + null, + null, + null, + fileFormat, + refTags.get(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null); + + String localeDownloadContent = Files.readString(file.toPath()); + logger.debug("File: {}, Content: {}", file.toPath(), localeDownloadContent); + + return localeDownloadContent; + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> { + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale)); + + if (onTagErrorRefreshCallback != null + && getErrorMessageFromOptionalApiException(doBeforeRetry.failure()) + .contains("Invalid Download Options. Parameter tags ")) { + String newTags = onTagErrorRefreshCallback.get(); + logger.warn( + "Replacing old tags: {} with new tags: {} for download locale", + refTags.get(), + newTags); + refTags.set(newTags); + } + })) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale))) + .block(); + } + + public String nativeLocaleDownload( + String projectId, + String locale, + String fileFormat, + String tags, + Map formatOptions, + Supplier onTagErrorRefreshCallback) { + AtomicReference refTags = new AtomicReference<>(tags); + return Mono.fromCallable( + () -> { + logger.info( + "Native Downloading locale: {} from project id: {} in file format: {}", + locale, + projectId, + fileFormat); + + Map parameters = new HashMap<>(); + parameters.put("file_format", fileFormat); + parameters.put("tags", refTags.get()); + for (Map.Entry e : formatOptions.entrySet()) { + parameters.put("format_options[%s]".formatted(e.getKey()), e.getValue()); + } + + StringJoiner queryJoiner = new StringJoiner("&"); + for (Map.Entry entry : parameters.entrySet()) { + queryJoiner.add( + URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8) + + "=" + + URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8)); + } + + String url = + String.format( + "%s/projects/%s/locales/%s/download?%s", + apiClient.getBasePath(), + URLEncoder.encode(projectId, StandardCharsets.UTF_8), + URLEncoder.encode(locale, StandardCharsets.UTF_8), + queryJoiner); + + String token = ((ApiKeyAuth) apiClient.getAuthentication("Token")).getApiKey(); + + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Authorization", "token " + token) + .header("Accept", "*") + .GET() + .build(); + + HttpResponse response; + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + int statusCode = response.statusCode(); + String responseBody = response.body(); + + if (statusCode == 200) { + return responseBody; + } else { + throw new RuntimeException( + "Can't download locale. status code " + statusCode + ": " + responseBody); + } + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> { + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale)); + + if (onTagErrorRefreshCallback != null + && getErrorMessageFromOptionalApiException(doBeforeRetry.failure()) + .contains("Invalid Download Options. Parameter tags ")) { + String newTags = onTagErrorRefreshCallback.get(); + logger.warn( + "Replacing old tags: {} with new tags: {} for download locale", + refTags.get(), + newTags); + refTags.set(newTags); + } + })) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to localeDownload from Phrase, project id: %s, locale: %s" + .formatted(projectId, locale))) + .block(); + } + + public List getKeys(String projectId, String tags) { + KeysApi keysApi = new KeysApi(apiClient); + AtomicInteger page = new AtomicInteger(0); + int batchSize = BATCH_SIZE; + List translationKeys = new ArrayList<>(); + while (true) { + List translationKeysInPage = + Mono.fromCallable( + () -> { + logger.info("Fetching keys for project: {}, page: {}", projectId, page); + return keysApi.keysList( + projectId, + null, + page.get(), + batchSize, + null, + null, + null, + "tags:%s".formatted(tags), + null); + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to fetch keys for project: %s, page: %s" + .formatted(projectId, page.get())))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to fetch keys for project: %s, page: %s" + .formatted(projectId, page))) + .block(); + + translationKeys.addAll(translationKeysInPage); + + if (translationKeysInPage.size() < batchSize) { + break; + } else { + page.incrementAndGet(); + } + } + return translationKeys; + } + + public List listTags(String projectId) { + TagsApi tagsApi = new TagsApi(apiClient); + final AtomicInteger page = new AtomicInteger(0); + List tags = new ArrayList<>(); + while (true) { + List tagsInPage = + Mono.fromCallable( + () -> { + logger.info("Fetching tags for project: {}", projectId); + return tagsApi.tagsList(projectId, null, page.get(), BATCH_SIZE, null); + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to fetch tags for project: %s, page: %d" + .formatted(projectId, page.get())))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error to fetch tags for project: %s, page: %s" + .formatted(projectId, page))) + .block(); + + tags.addAll(tagsInPage); + + if (tagsInPage.size() < BATCH_SIZE) { + break; + } else { + page.incrementAndGet(); + } + } + + return tags; + } + + public void deleteTags(String projectId, List tagNames) { + + logger.debug("Delete tags: {}", tagNames); + + TagsApi tagsApi = new TagsApi(apiClient); + Map exceptions = new LinkedHashMap<>(); + for (String tagName : tagNames) { + Mono.fromCallable( + () -> { + logger.debug("Deleting tag: %s in project id: %s".formatted(tagName, projectId)); + tagsApi.tagDelete(projectId, tagName, null, null); + return null; + }) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> { + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to delete tag: %s in project id: %s" + .formatted(tagName, projectId)); + })) + .doOnError( + throwable -> { + exceptions.put(tagName, throwable); + rethrowExceptionWithLog( + throwable, + "Final error to delete tag: %s in project id: %s" + .formatted(tagName, projectId)); + }) + .block(); + } + + if (!exceptions.isEmpty()) { + List tagsWithErrors = exceptions.keySet().stream().limit(10).toList(); + String andMore = (tagsWithErrors.size() < exceptions.size()) ? " and more." : ""; + throw new PhraseClientException( + String.format("Can't delete tagNames: %s%s", tagsWithErrors, andMore)); + } + } + + private void logAttempt(Throwable throwable, String message) { + String errorMessage = getErrorMessageFromOptionalApiException(throwable); + logger.info("%s, error: %s".formatted(message, errorMessage), throwable); + } + + private void rethrowExceptionWithLog(Throwable throwable, String message) { + String errorMessage = getErrorMessageFromOptionalApiException(throwable); + logger.error("%s, error: %s".formatted(message, errorMessage)); + if (throwable.getCause() instanceof ApiException) { + throw new PhraseClientException(errorMessage, (ApiException) throwable.getCause()); + } else { + throw new RuntimeException(errorMessage, throwable); + } + } + + private String getErrorMessageFromOptionalApiException(Throwable t) { + String errorMessage; + if (t instanceof ApiException) { + errorMessage = ((ApiException) t).getResponseBody(); + } else { + errorMessage = t.getMessage(); + } + return errorMessage; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java new file mode 100644 index 0000000000..65e69823ab --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientConfig.java @@ -0,0 +1,23 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import com.phrase.client.ApiClient; +import com.phrase.client.auth.ApiKeyAuth; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class PhraseClientConfig { + + @ConditionalOnProperty("l10n.phrase.client.token") + @Bean + public PhraseClient getPhraseClient(PhraseClientPropertiesConfig phraseClientPropertiesConfig) { + + ApiClient apiClient = new ApiClient(); + ApiKeyAuth authentication = (ApiKeyAuth) apiClient.getAuthentication("Token"); + authentication.setApiKey(phraseClientPropertiesConfig.getToken()); + authentication.setApiKeyPrefix("token"); + + return new PhraseClient(apiClient); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java new file mode 100644 index 0000000000..65f0ffc6c4 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientException.java @@ -0,0 +1,22 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import com.phrase.client.ApiException; + +public class PhraseClientException extends RuntimeException { + + ApiException apiException; + + public PhraseClientException(String message) { + super(message); + } + + public PhraseClientException(ApiException e) { + super(e); + apiException = e; + } + + public PhraseClientException(String message, ApiException apiException) { + super(message); + this.apiException = apiException; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java new file mode 100644 index 0000000000..9baf70b304 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientPropertiesConfig.java @@ -0,0 +1,19 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties("l10n.phrase.client") +public class PhraseClientPropertiesConfig { + + String token; + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java index 8f02159cb4..024bb802f0 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJob.java @@ -1,6 +1,7 @@ package com.box.l10n.mojito.service.thirdparty.smartling.quartz; import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSUtils.isFileEqualToPreviousRun; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import com.box.l10n.mojito.LocaleMappingHelper; import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper; @@ -16,13 +17,10 @@ import com.box.l10n.mojito.smartling.SmartlingClientException; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; -import java.io.IOException; import java.util.List; -import javax.xml.parsers.ParserConfigurationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.xml.sax.SAXException; import reactor.core.publisher.Mono; public class SmartlingPullLocaleFileJob @@ -103,15 +101,8 @@ && matchesChecksumFromPreviousSync(input, localeTag, fileName, fileContent)) { return null; } - List textUnits; - - try { - textUnits = mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); - } catch (ParserConfigurationException | IOException | SAXException e) { - String msg = "An error occurred when processing a pull batch"; - logger.error(msg, e); - throw new RuntimeException(msg, e); - } + List textUnits = + mapper.mapToTextUnits(AndroidStringDocumentReader.fromText(fileContent)); if (!textUnits.isEmpty() && input.getSmartlingFilePrefix().equalsIgnoreCase("plural") @@ -121,7 +112,8 @@ && matchesChecksumFromPreviousSync(input, localeTag, fileName, fileContent)) { if (!input.isDryRun()) { logger.debug("Importing text units for locale: {}", smartlingLocale); - textUnitBatchImporterService.importTextUnits(textUnits, false, true); + + textUnitBatchImporterService.importTextUnits(textUnits, fromLegacy(false, true)); } } return null; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java index ad45b4c186..531e29876a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMTextUnitIntegrityCheckService.java @@ -7,6 +7,7 @@ import com.box.l10n.mojito.service.asset.AssetRepository; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckException; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckerFactory; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.PluralIntegrityCheckerRelaxer; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TextUnitIntegrityChecker; import java.util.Set; import org.slf4j.Logger; @@ -27,6 +28,8 @@ public class TMTextUnitIntegrityCheckService { @Autowired TMTextUnitRepository tmTextUnitRepository; + @Autowired PluralIntegrityCheckerRelaxer pluralIntegrityCheckerRelaxer; + /** * Checks the integrity of the content given the {@link com.box.l10n.mojito.entity.TMTextUnit#id} * @@ -46,7 +49,22 @@ public void checkTMTextUnitIntegrity(Long tmTextUnitId, String contentToCheck) logger.debug("No designated checker for this asset. Nothing to do"); } else { for (TextUnitIntegrityChecker textUnitChecker : textUnitCheckers) { - textUnitChecker.check(tmTextUnit.getContent(), contentToCheck); + try { + textUnitChecker.check(tmTextUnit.getContent(), contentToCheck); + } catch (IntegrityCheckException e) { + if (tmTextUnit.getPluralForm() != null + && pluralIntegrityCheckerRelaxer.shouldRelaxIntegrityCheck( + tmTextUnit.getContent(), + contentToCheck, + tmTextUnit.getPluralForm().getName(), + textUnitChecker)) { + logger.debug( + "Relaxing the check for plural string with form: {}", + tmTextUnit.getPluralForm().getName()); + } else { + throw e; + } + } } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java index 5c7e3c3b0e..3402a3ca4b 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterService.java @@ -5,6 +5,7 @@ import static com.box.l10n.mojito.utils.Predicates.logIfFalse; import com.box.l10n.mojito.JSR310Migration; +import com.box.l10n.mojito.aspect.StopWatch; import com.box.l10n.mojito.entity.Asset; import com.box.l10n.mojito.entity.Locale; import com.box.l10n.mojito.entity.Repository; @@ -21,6 +22,7 @@ import com.box.l10n.mojito.service.asset.ImportTextUnitJobInput; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckException; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckerFactory; +import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.PluralIntegrityCheckerRelaxer; import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TextUnitIntegrityChecker; import com.box.l10n.mojito.service.locale.LocaleService; import com.box.l10n.mojito.service.pollableTask.PollableFuture; @@ -36,6 +38,7 @@ import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; import com.box.l10n.mojito.service.tm.textunitdtocache.TextUnitDTOsCacheService; import com.box.l10n.mojito.service.tm.textunitdtocache.UpdateType; +import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -63,6 +66,8 @@ @Component public class TextUnitBatchImporterService { + static final String FALSE_POSITIVE_TAG_FOR_STATUS = "false positive"; + /** logger */ static Logger logger = LoggerFactory.getLogger(TextUnitBatchImporterService.class); @@ -92,9 +97,53 @@ public class TextUnitBatchImporterService { @Autowired MeterRegistry meterRegistry; + @Autowired PluralIntegrityCheckerRelaxer pluralIntegrityCheckerRelaxer; + @Value("${l10n.textUnitBatchImporterService.quartz.schedulerName:" + DEFAULT_SCHEDULER_NAME + "}") String schedulerName; + public enum IntegrityChecksType { + /** Don't run integrity checks */ + SKIP, + /** Always use the status from the integrity checker (legacy behavior 1) */ + ALWAYS_USE_INTEGRITY_CHECKER_STATUS, + + /** + * Use the status from the integrity check unless the current translation has a tag in the + * comment saying this was a false positive. + * + *

In case of false positive, we don't want to change the status back + */ + UNLESS_TAGGED_USE_INTEGRITY_CHECKER_STATUS, + /** + * Run integrity checks. If it fails and the target is the same, keep the current status; + * otherwise, reject (legacy behavior 2). + */ + KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET, + /** + * Run integrity checks. If the target is the same, keep the current status. + * + *

This is an extension of the legacy behavior that allows marking a translation as invalid + * when the integrity check did not catch the issue, eventually causing a build failure. + */ + KEEP_STATUS_IF_SAME_TARGET; + + public static IntegrityChecksType fromLegacy( + boolean integrityCheckSkipped, boolean integrityCheckKeepStatusIfFailedAndSameTarget) { + IntegrityChecksType legacy = IntegrityChecksType.SKIP; + + if (!integrityCheckSkipped) { + if (!integrityCheckKeepStatusIfFailedAndSameTarget) { + legacy = ALWAYS_USE_INTEGRITY_CHECKER_STATUS; + } else { + legacy = KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET; + } + } + + return legacy; + } + } + /** * Imports a batch of text units. * @@ -114,24 +163,19 @@ public class TextUnitBatchImporterService { * @return */ public PollableFuture asyncImportTextUnits( - List textUnitDTOs, - boolean integrityCheckSkipped, - boolean integrityCheckKeepStatusIfFailedAndSameTarget) { + List textUnitDTOs, IntegrityChecksType integrityChecksType) { ImportTextUnitJobInput importTextUnitJobInput = new ImportTextUnitJobInput(); importTextUnitJobInput.setTextUnitDTOs(textUnitDTOs); - importTextUnitJobInput.setIntegrityCheckSkipped(integrityCheckSkipped); - importTextUnitJobInput.setIntegrityCheckKeepStatusIfFailedAndSameTarget( - integrityCheckKeepStatusIfFailedAndSameTarget); + importTextUnitJobInput.setIntegrityChecksType(integrityChecksType); return quartzPollableTaskScheduler.scheduleJob( ImportTextUnitJob.class, importTextUnitJobInput, schedulerName); } + @StopWatch public PollableFuture importTextUnits( - List textUnitDTOs, - boolean integrityCheckSkipped, - boolean integrityCheckKeepStatusIfFailedAndSameTarget) { + List textUnitDTOs, IntegrityChecksType integrityChecksType) { return meterRegistry .timer("TextUnitBatchImporterService.importTextUnits") @@ -162,7 +206,7 @@ public PollableFuture importTextUnits( mapTextUnitsToImportWithExistingTextUnits( locale, asset, textUnitsForBatchImport); - if (!integrityCheckSkipped) { + if (!IntegrityChecksType.SKIP.equals(integrityChecksType)) { try (var timer2 = Timer.resource( meterRegistry, @@ -171,9 +215,7 @@ public PollableFuture importTextUnits( .tag("asset", asset.getPath())) { applyIntegrityChecks( - asset, - textUnitsForBatchImport, - integrityCheckKeepStatusIfFailedAndSameTarget); + asset, textUnitsForBatchImport, integrityChecksType); } } importTextUnitsOfLocaleAndAsset(locale, asset, textUnitsForBatchImport); @@ -194,6 +236,7 @@ public PollableFuture importTextUnits( * @param asset * @param textUnitsToImport text units to which the current text units must be added */ + @StopWatch void mapTextUnitsToImportWithExistingTextUnits( Locale locale, Asset asset, List textUnitsToImport) { logger.debug( @@ -205,14 +248,16 @@ void mapTextUnitsToImportWithExistingTextUnits( textUnitsToImport.forEach(tu -> match.apply(tu).ifPresent(m -> tu.setCurrentTextUnit(m))); } + @StopWatch @Transactional void importTextUnitsOfLocaleAndAsset( Locale locale, Asset asset, List textUnitsToImport) { ZonedDateTime importTime = JSR310Migration.newDateTimeEmptyCtor(); - logger.debug( - "Start import text units for asset: {} and locale: {}", + logger.info( + "Start import text units for asset: {}, locale: {}, count: {}", asset.getPath(), - locale.getBcp47Tag()); + locale.getBcp47Tag(), + textUnitsToImport.size()); textUnitsToImport.stream() .filter( @@ -244,7 +289,7 @@ void importTextUnitsOfLocaleAndAsset( TMTextUnitCurrentVariant tmTextUnitCurrentVariant = null; if (currentTextUnit.getTmTextUnitCurrentVariantId() != null) { - logger.debug("Looking up current variant"); + // this is making many calls! tmTextUnitCurrentVariant = tmTextUnitCurrentVariantRepository.findByLocale_IdAndTmTextUnit_Id( currentTextUnit.getLocaleId(), currentTextUnit.getTmTextUnitId()); @@ -259,7 +304,7 @@ void importTextUnitsOfLocaleAndAsset( currentTextUnit.getTmTextUnitId(), locale.getId(), textUnitForBatchImport.getContent(), - textUnitForBatchImport.getComment(), + textUnitForBatchImport.getTargetComment(), textUnitForBatchImport.getStatus(), textUnitForBatchImport.isIncludedInLocalizedFile(), importTime, @@ -294,11 +339,11 @@ boolean isUpdateNeeded(TextUnitForBatchMatcherImport textUnitForBatchImport) { currentTextUnit.getStatus(), DigestUtils.md5Hex(currentTextUnit.getTarget()), currentTextUnit.isIncludedInLocalizedFile(), - currentTextUnit.getComment(), + currentTextUnit.getTargetComment(), textUnitForBatchImport.getStatus(), DigestUtils.md5Hex(textUnitForBatchImport.getContent()), textUnitForBatchImport.isIncludedInLocalizedFile(), - textUnitForBatchImport.getComment()); + textUnitForBatchImport.getTargetComment()); } List getTextUnitTDOsForLocaleAndAsset(Locale locale, Asset asset) { @@ -309,7 +354,7 @@ List getTextUnitTDOsForLocaleAndAsset(Locale locale, Asset asset) { void applyIntegrityChecks( Asset asset, List textUnitsForBatchImport, - boolean keepStatusIfCheckFailedAndSameTarget) { + IntegrityChecksType integrityChecksType) { Set textUnitCheckers = integrityCheckerFactory.getTextUnitCheckers(asset); @@ -325,34 +370,60 @@ void applyIntegrityChecks( textUnitForBatchImport.setIncludedInLocalizedFile(true); textUnitForBatchImport.setStatus(APPROVED); + boolean hasSameTarget = + textUnitForBatchImport.getContent().equals(currentTextUnit.getTarget()); + for (TextUnitIntegrityChecker textUnitChecker : textUnitCheckers) { try { textUnitChecker.check(currentTextUnit.getSource(), textUnitForBatchImport.getContent()); - } catch (IntegrityCheckException ice) { - boolean hasSameTarget = - textUnitForBatchImport.getContent().equals(currentTextUnit.getTarget()); - - if (hasSameTarget && keepStatusIfCheckFailedAndSameTarget) { + if (hasSameTarget + && !IntegrityChecksType.KEEP_STATUS_IF_SAME_TARGET.equals(integrityChecksType)) { textUnitForBatchImport.setIncludedInLocalizedFile( currentTextUnit.isIncludedInLocalizedFile()); textUnitForBatchImport.setStatus(currentTextUnit.getStatus()); + } + } catch (IntegrityCheckException ice) { + logger.info( + "Integrity check failed for string with source:\n{}\n\nand content:\n{}", + currentTextUnit.getSource(), + textUnitForBatchImport.getContent(), + ice); + + if (pluralIntegrityCheckerRelaxer.shouldRelaxIntegrityCheck( + currentTextUnit.getSource(), + textUnitForBatchImport.getContent(), + textUnitForBatchImport.getCurrentTextUnit().getPluralForm(), + textUnitChecker)) { + logger.debug( + "Relaxing the check for plural string with form: {}", + textUnitForBatchImport.getCurrentTextUnit().getPluralForm()); } else { - logger.info( - "Integrity check failed for string with source:\n{}\n\nand content:\n{}", - currentTextUnit.getSource(), - textUnitForBatchImport.getContent(), - ice); textUnitForBatchImport.setIncludedInLocalizedFile(false); textUnitForBatchImport.setStatus(Status.TRANSLATION_NEEDED); + if (hasSameTarget) { + switch (integrityChecksType) { + case UNLESS_TAGGED_USE_INTEGRITY_CHECKER_STATUS: + if (!Strings.nullToEmpty(currentTextUnit.getTargetComment()) + .toLowerCase() + .contains(FALSE_POSITIVE_TAG_FOR_STATUS)) { + break; + } + case KEEP_STATUS_IF_SAME_TARGET: + case KEEP_STATUS_IF_REJECTED_AND_SAME_TARGET: + textUnitForBatchImport.setIncludedInLocalizedFile( + currentTextUnit.isIncludedInLocalizedFile()); + textUnitForBatchImport.setStatus(currentTextUnit.getStatus()); + break; + } + } + TMTextUnitVariantComment tmTextUnitVariantComment = new TMTextUnitVariantComment(); tmTextUnitVariantComment.setSeverity(Severity.ERROR); tmTextUnitVariantComment.setContent(ice.getMessage()); textUnitForBatchImport.getTmTextUnitVariantComments().add(tmTextUnitVariantComment); } - - break; } } } @@ -425,6 +496,7 @@ List skipInvalidAndConvertToTextUnitForBatchImpor textUnitForBatchImport.setLocale(localeService.findByBcp47Tag(t.getTargetLocale())); textUnitForBatchImport.setContent(NormalizationUtils.normalize(t.getTarget())); textUnitForBatchImport.setComment(t.getComment()); + textUnitForBatchImport.setTargetComment(t.getTargetComment()); textUnitForBatchImport.setIncludedInLocalizedFile(t.isIncludedInLocalizedFile()); textUnitForBatchImport.setStatus(t.getStatus() == null ? APPROVED : t.getStatus()); return textUnitForBatchImport; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitForBatchMatcherImport.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitForBatchMatcherImport.java index e5f81a23ff..32a7dc55fe 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitForBatchMatcherImport.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/importer/TextUnitForBatchMatcherImport.java @@ -21,6 +21,7 @@ public class TextUnitForBatchMatcherImport implements TextUnitForBatchMatcher { String content; String name; String comment; + String targetComment; Long tmTextUnitId; TextUnitDTO currentTextUnit; boolean includedInLocalizedFile; @@ -78,6 +79,14 @@ public void setComment(String comment) { this.comment = comment; } + public String getTargetComment() { + return targetComment; + } + + public void setTargetComment(String targetComment) { + this.targetComment = targetComment; + } + @Override public Long getTmTextUnitId() { return tmTextUnitId; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java index 025ad75599..0be8393b04 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java @@ -34,6 +34,7 @@ public class TextUnitDTO { private Long assetTextUnitId; private ZonedDateTime tmTextUnitCreatedDate; private boolean doNotTranslate; + private Long branchId; public Long getTmTextUnitId() { return tmTextUnitId; @@ -242,4 +243,12 @@ public boolean isDoNotTranslate() { public void setDoNotTranslate(boolean doNotTranslate) { this.doNotTranslate = doNotTranslate; } + + public Long getBranchId() { + return branchId; + } + + public void setBranchId(Long branchId) { + this.branchId = branchId; + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java index dd2c35bd25..75bcfcfd02 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java @@ -51,11 +51,12 @@ public TextUnitDTO mapObject(CriteriaResult cr) { t.setAssetPath(cr.getString(idx++)); t.setAssetTextUnitId(cr.getLong(idx++)); t.setTmTextUnitCreatedDate(JSR310Migration.newDateTimeCtorWithDate(cr.getDate(idx++))); - t.setDoNotTranslate(Boolean.valueOf(includedInLocalizedFile)); String doNotTranslate = cr.getString(idx++); t.setDoNotTranslate(Boolean.valueOf(doNotTranslate)); + t.setBranchId(cr.getLong(idx++)); + return t; } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java index e5e737debe..216d6f8e95 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java @@ -219,7 +219,8 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) .addProjection("a.path", "assetPath") .addProjection("atu.id", "assetTextUnitId") .addProjection("tu.created_date", "tmTextUnitCreatedDate") - .addProjection("atu.do_not_translate", "doNotTranslate")); + .addProjection("atu.do_not_translate", "doNotTranslate") + .addProjection("atu.branch_id", "branch_id")); logger.debug("Add search filters"); NativeJunctionExp conjunction = NativeExps.conjunction(); @@ -429,6 +430,10 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) new NativeDateGteExp("tu.created_date", searchParameters.getTmTextUnitCreatedAfter())); } + if (searchParameters.getTmTextUnitVariantId() != null) { + conjunction.add(new NativeEqExpFix("tuv.id", searchParameters.getTmTextUnitVariantId())); + } + if (!conjunction.toSQL().isEmpty()) { c.add(conjunction); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java index cc062c6afc..9792093ff0 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java @@ -50,6 +50,8 @@ public class TextUnitSearcherParameters { String includeTextUnitsWithPattern; String skipAssetPathWithPattern; + Long tmTextUnitVariantId; + public String getName() { return name; } @@ -287,6 +289,14 @@ public void setTmTextUnitIds(Long... tmTextUnitIds) { } } + public Long getTmTextUnitVariantId() { + return tmTextUnitVariantId; + } + + public void setTmTextUnitVariantId(Long tmTextUnitVariantId) { + this.tmTextUnitVariantId = tmTextUnitVariantId; + } + public String getSkipTextUnitWithPattern() { return skipTextUnitWithPattern; } diff --git a/webapp/src/main/resources/config/application.properties b/webapp/src/main/resources/config/application.properties index 8d8f46baba..6db15e2df0 100644 --- a/webapp/src/main/resources/config/application.properties +++ b/webapp/src/main/resources/config/application.properties @@ -12,7 +12,8 @@ info.build.version=@project.version@ #logging.file.path=/var/logs #logging.file.name=mojito.log #logging.config= # location of config file (default classpath:logback.xml for logback) -#logging.level.*=ERROR +#logging.level.root=ERROR +#logging.level.com.box.l10n.*=ERROR #logging.level.org.apache.http.wire=DEBUG logging.level.net.sf.okapi.common.pipelinedriver.PipelineDriver=ERROR @@ -112,6 +113,55 @@ l10n.org.quartz.scheduler.skipUpdateCheck=true #l10n.org.quartz.dataSource.myDS.maxConnections=27 #l10n.org.quartz.dataSource.myDS.validationQuery=select 1 +### for multiple Quartz schedulers +#l10n.org.multi-quartz.enabled=true +### Copy/paste the following section as many time as needed, renaming "default" to scheduler names eg. "ai-translate". +### By default, different services use the default scheduler (search for DEFAULT_SCHEDULER_NAME usages) +### configure services as needed to use a different scheduler eg. + +#l10n.ai-translate.scheduler-name=ai-translate + +#l10n.assetWS.quartz.schedulerName= +#l10n.assetService.quartz.schedulerName= +#l10n.assetExtraction.quartz.schedulerName= +#l10n.branchService.quartz.schedulerName= +#l10n.branchStatistic.quartz.schedulerName= +#l10n.branchNotification.quartz.schedulerName= +#l10n.repositoryStatistics.quartz.schedulerName= +#l10n.thirdPartyService.quartz.schedulerName= +#l10n.tmService.quartz.schedulerName= +#l10n.textUnitBatchImporterService.quartz.schedulerName +#l10n.machineTranslation.quartz.schedulerName= + +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.useProperties=true +#l10n.org.multi-quartz.schedulers.default.quartz.scheduler.instanceId=AUTO +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.isClustered=true +#l10n.org.multi-quartz.schedulers.default.quartz.threadPool.threadCount=10 +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.class=org.quartz.impl.jdbcjobstore.JobStoreTX +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate +#l10n.org.multi-quartz.schedulers.default.quartz.jobStore.dataSource=myDS +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.provider=hikaricp +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.driver=com.mysql.jdbc.Driver +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.URL=jdbc:mysql://localhost:3306/mojito?characterEncoding=UTF-8&useUnicode=true +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.user=mojito +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.password=mojito +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.maxConnections=12 +#l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.validationQuery=select 1 +# +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.useProperties=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.useProperties} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.scheduler.instanceId=${l10n.org.multi-quartz.schedulers.default.quartz.scheduler.instanceId} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.isClustered=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.isClustered} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.threadPool.threadCount=10 +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.class=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.class} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.driverDelegateClass=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.driverDelegateClass} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.jobStore.dataSource=${l10n.org.multi-quartz.schedulers.default.quartz.jobStore.dataSource} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.provider=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.provider} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.driver=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.driver} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.URL=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.URL} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.user=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.user} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.password=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.password} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.maxConnections=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.maxConnections} +#l10n.org.multi-quartz.schedulers.ai-translate.quartz.dataSource.myDS.validationQuery=${l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.validationQuery} ### Settings to enable/disable upgrade jobs. "true" by default to make upgrades easy. # Once upgrades are done it can be set to false to stop running the logic. diff --git a/webapp/src/main/resources/db/hsql/data.sql b/webapp/src/main/resources/db/hsql/data.sql index 8fe1f5221c..b48f9d22e6 100644 --- a/webapp/src/main/resources/db/hsql/data.sql +++ b/webapp/src/main/resources/db/hsql/data.sql @@ -989,6 +989,7 @@ insert into locale (id, bcp47_tag) values (1637, 'zh-Hant-alt-long'); insert into locale (id, bcp47_tag) values (1638, 'zun'); insert into locale (id, bcp47_tag) values (1639, 'zxx'); insert into locale (id, bcp47_tag) values (1640, 'zza'); +insert into locale (id, bcp47_tag) values (1641, 'bn-BD'); insert into plural_form (id, name) values (1, 'one'); diff --git a/webapp/src/main/resources/db/migration/V66__Add_bn_BD.sql b/webapp/src/main/resources/db/migration/V66__Add_bn_BD.sql new file mode 100644 index 0000000000..6fd1882600 --- /dev/null +++ b/webapp/src/main/resources/db/migration/V66__Add_bn_BD.sql @@ -0,0 +1,4 @@ +insert into locale (id, bcp47_tag) values (1641, 'bn-BD'); + +insert into plural_form_for_locale (locale_id, plural_form_id) values (1641, 1); +insert into plural_form_for_locale (locale_id, plural_form_id) values (1641, 5); \ No newline at end of file diff --git a/webapp/src/main/resources/properties/en.properties b/webapp/src/main/resources/properties/en.properties index 58bca97d29..26555aa4d3 100644 --- a/webapp/src/main/resources/properties/en.properties +++ b/webapp/src/main/resources/properties/en.properties @@ -758,6 +758,9 @@ textUnit.gitBlameModal.screenshotModalOpen=View # Tooltip for button to open the translation history modal workbench.translationHistoryModal.info=Text unit history +# Tooltip for button to open the translation history modal +workbench.aiReviewModal.info=AI Review + # Title of the translation history modal dialog workbench.translationHistoryModal.title=Translation History @@ -851,3 +854,6 @@ userErrorModal.delete=The user could not be deleted userErrorModal.forbidden=You do not have the necessary permissions for this operation. users.userInformationAlert=Administrators and Project Managers have write access to the user management as well as creating and deleting repositories. Translators can only translate. Users do not have any write access. + + +aiReviewModal.title=AI Review diff --git a/webapp/src/main/resources/public/js/actions/workbench/AiReviewActions.js b/webapp/src/main/resources/public/js/actions/workbench/AiReviewActions.js new file mode 100644 index 0000000000..9c6e5bbbe1 --- /dev/null +++ b/webapp/src/main/resources/public/js/actions/workbench/AiReviewActions.js @@ -0,0 +1,15 @@ +import alt from "../../alt"; + +class AiReviewActions { + + constructor() { + this.generateActions( + "openWithTextUnit", + "getAiReviewSuccess", + "getAiReviewError", + "close" + ); + } +} + +export default alt.createActions(AiReviewActions); diff --git a/webapp/src/main/resources/public/js/actions/workbench/TextUnitDataSource.js b/webapp/src/main/resources/public/js/actions/workbench/TextUnitDataSource.js index 77d6cbb513..30c0d7ef0d 100644 --- a/webapp/src/main/resources/public/js/actions/workbench/TextUnitDataSource.js +++ b/webapp/src/main/resources/public/js/actions/workbench/TextUnitDataSource.js @@ -4,6 +4,7 @@ import TextUnitClient from "../../sdk/TextUnitClient"; import WorkbenchActions from "./WorkbenchActions"; import GitBlameActions from "./GitBlameActions"; import TranslationHistoryActions from "./TranslationHistoryActions"; +import AiReviewActions from "./AiReviewActions"; const TextUnitDataSource = { performSaveTextUnit: { @@ -69,6 +70,14 @@ const TextUnitDataSource = { }, success: TranslationHistoryActions.getTranslationHistorySuccess, error: TranslationHistoryActions.getTranslationHistoryError + }, + + getAiReview: { + remote(aiReviewStoreState, textUnit) { + return TextUnitClient.getAiReview(textUnit); + }, + success: AiReviewActions.getAiReviewSuccess, + error: AiReviewActions.getAiReviewError } }; diff --git a/webapp/src/main/resources/public/js/components/workbench/AIReviewModal.js b/webapp/src/main/resources/public/js/components/workbench/AIReviewModal.js new file mode 100644 index 0000000000..51980b0f89 --- /dev/null +++ b/webapp/src/main/resources/public/js/components/workbench/AIReviewModal.js @@ -0,0 +1,291 @@ +import PropTypes from 'prop-types'; +import React from "react"; +import {FormattedMessage, injectIntl} from "react-intl"; +import {Button, Glyphicon, Modal} from "react-bootstrap"; +import {withAppConfig} from "../../utils/AppConfig"; +import TextUnitSDK from "../../sdk/TextUnit"; + +class AIReviewModal extends React.Component { + static propTypes() { + return { + "show": PropTypes.bool.isRequired, + "onCloseModal": PropTypes.func.isRequired, + }; + } + + closeModal = () => { + this.props.onCloseModal(); + }; + + getTitle = () => { + return this.props.intl.formatMessage({id: "aiReviewModal.title"}); + }; + + renderRating = (rating) => { + let label; + switch (rating) { + case 0: + label = bad; + break; + case 2: + label = good; + break; + default: + label = average; + } + + return label; + }; + + renderHeader() { + return + {this.props.textUnit.getTargetLocale()} + {this.props.textUnit.getName()} + + } + + renderHeaderRight() { + return + asset: {this.props.textUnit.getAssetPath()} + repo: {this.props.textUnit.getRepositoryName()} + + } + + + renderSource() { + return {this.props.textUnit.getSource()} + } + + renderTarget() { + return {this.props.textUnit.getTarget()} + } + + renderTargetStatus() { + return

{this.renderReviewGlyph()}
+ } + + renderComment() { + return {this.props.textUnit.getComment()} + } + + renderCommentRating() { + return this.props.review && + +
{this.renderRating(this.props.review.descriptionRating.score)}
+
{this.props.review.descriptionRating.explanation}
+ + } + + renderTargetRating() { + return this.props.review && + +
{this.renderRating(this.props.review.existingTargetRating.score)}
+
{this.props.review.existingTargetRating.explanation}
+ + } + + renderSuggestion1() { + return this.props.review && +
{this.props.review.target.content} {this.props.review.target.confidenceLevel}
+ + + } + + renderSuggestion2() { + return this.props.review && +
{this.props.review.altTarget.content} {this.props.review.altTarget.confidenceLevel}
+ + } + + /* TODO(ja) make a component instead of copy/paste */ + renderReviewGlyph() { + + let ui = ""; + if (this.props.textUnit.isTranslated()) { + + let glyphType = "ok"; + let glyphTitle = this.props.intl.formatMessage({id: "textUnit.reviewModal.accepted"}); + + if (!this.props.textUnit.isIncludedInLocalizedFile()) { + + glyphType = "alert"; + glyphTitle = this.props.intl.formatMessage({id: "textUnit.reviewModal.rejected"}); + + } else if (this.props.textUnit.getStatus() === TextUnitSDK.STATUS.REVIEW_NEEDED) { + + glyphType = "eye-open"; + glyphTitle = this.props.intl.formatMessage({id: "textUnit.reviewModal.needsReview"}); + + } else if (this.props.textUnit.getStatus() === TextUnitSDK.STATUS.TRANSLATION_NEEDED) { + + glyphType = "edit"; + glyphTitle = this.props.intl.formatMessage({id: "textUnit.reviewModal.translationNeeded"}); + } + ui = ( + + ); + } + + return ui; + } + + render() { + if (!this.props.show) return null; + + return ( + + + {this.getTitle()} + + + +
+
Text Unit
+
+
+ {this.props.textUnit.getTargetLocale()} + {this.props.textUnit.getName()} + {this.props.textUnit.getAssetPath()} + {this.props.textUnit.getRepositoryName()} +
+
+
{this.props.textUnit.getSource()}
+
{this.props.textUnit.getComment()}
+
+
+
{this.renderTarget()}
+
{this.renderReviewGlyph()}
+
+
+ + {this.props.review == null ? +
+ {this.props.loading && + } +
+ : +
Target Analysis
+
+
+ {this.renderRating(this.props.review.existingTargetRating.score)} +
+
+ {this.props.review.existingTargetRating.explanation} +
+
+ +
Target Suggestions
+
+
+ {this.props.review.target.confidenceLevel} +
+
+ {this.props.review.target.content} +
+
+
+
+ {this.props.review.altTarget.confidenceLevel} +
+
+ {this.props.review.altTarget.content} +
+
+ +
Comment Analysis
+
+
+ {this.renderRating(this.props.review.descriptionRating.score)} +
+
+ {this.props.review.descriptionRating.explanation} +
+
+ + } +
+ {/*
*/} + {/*
{this.renderHeader()}
*/} + {/*
{this.renderHeaderRight()}
*/} + {/*
{this.renderSource()}
*/} + {/*
{this.renderTarget()}
*/} + {/* /!*
{this.renderTargetStatus()}
*!/*/} + {/*
{this.renderComment()}
*/} + {/*
Comment and Target Ratings
*/} + {/*
{this.renderCommentRating()}
*/} + {/*
{this.renderTargetRating()}
*/} + {/*
Target Suggestions
*/} + {/*
{this.renderSuggestion1()}
*/} + {/*
{this.renderSuggestion2()}
*/} + {/*
*/} + + + {/*
*/} + {/*
*/} + {/*
{this.props.textUnit.getSource()}
*/} + {/*
{this.props.textUnit.getComment()}
*/} + {/*
*/} + {/*
*/} + {/*
{this.props.textUnit.getTarget()}
*/} + + {/* {this.props.loading ? :*/} + {/*
*/} + {/*
{this.renderRating(this.getRating())}*/} + {/*
*/} + {/*{this.props.review.existingTargetRating.explanation}
*/} + + {/*
*/} + {/*
*/} + {/*
*/} + + + {/*
Suggestions
*/} + {/*
{this.props.review.target.content}
*/} + {/*{this.props.review.target.explanation}*/} + {/*
{this.props.review.target.confidenceLevel}*/} + {/*
*/} + + {/*
*/} + {/*
{this.props.review.altTarget.content}
*/} + {/*{this.props.review.altTarget.explanation}*/} + {/*
{this.props.review.altTarget.confidenceLevel}
*/} + + {/*
*/} + {/* {this.props.review.descriptionRating.explanation}
*/} + {/**/} + + {/*{this.renderRating(this.props.review.descriptionRating.score)}*/} + {/* */} + {/* }*/} + {/**/} + + + + + + + + ); + } + + getRating() { + return this.props.review.existingTargetRating.score; + } +} + +export default withAppConfig(injectIntl(AIReviewModal)); diff --git a/webapp/src/main/resources/public/js/components/workbench/TextUnit.js b/webapp/src/main/resources/public/js/components/workbench/TextUnit.js index deeae25400..89c7a734f8 100644 --- a/webapp/src/main/resources/public/js/components/workbench/TextUnit.js +++ b/webapp/src/main/resources/public/js/components/workbench/TextUnit.js @@ -34,6 +34,7 @@ import { Tooltip } from "react-bootstrap"; import ViewModeStore from "../../stores/workbench/ViewModeStore"; +import AiReviewActions from "../../actions/workbench/AiReviewActions"; let TextUnit = createReactClass({ @@ -658,6 +659,13 @@ let TextUnit = createReactClass({ TranslationHistoryActions.openWithTextUnit(this.props.textUnit); }, + onAiReviewClick(e){ + + e.stopPropagation(); + + AiReviewActions.openWithTextUnit(this.props.textUnit); + }, + /** * Handle click on the asset path icon: stop event propagation (no need to bubble * up as we're reloading the workbench with new data) and update the search @@ -859,6 +867,10 @@ let TextUnit = createReactClass({ let assetPathTranslationHistoryTooltip = {this.props.intl.formatMessage( {id: 'workbench.translationHistoryModal.info'} )}; + let aiReviewTooltip = + {this.props.intl.formatMessage( {id: 'workbench.aiReviewModal.info'} )}; + + // Only show the overlay trigger for the translation history if there is a current text unit variant (ie. there is // at least one translation) If not, we have no history to show anyways! return ( + + + + ); }, diff --git a/webapp/src/main/resources/public/js/components/workbench/Workbench.js b/webapp/src/main/resources/public/js/components/workbench/Workbench.js index d7a9616deb..47d90d9295 100644 --- a/webapp/src/main/resources/public/js/components/workbench/Workbench.js +++ b/webapp/src/main/resources/public/js/components/workbench/Workbench.js @@ -23,6 +23,9 @@ import ShareSearchParamsModal from "./ShareSearchParamsModal"; import ShareSearchParamsModalActions from "../../actions/workbench/ShareSearchParamsModalActions"; import ShareSearchParamsButton from "./ShareSearchParamsButton"; import AuthorityService from "../../utils/AuthorityService"; +import AIReviewModal from "./AIReviewModal"; +import AiReviewActions from "../../actions/workbench/AiReviewActions"; +import AIReviewStore from "../../stores/workbench/AiReviewStore"; let Workbench = createReactClass({ displayName: 'Workbench', @@ -49,6 +52,11 @@ let Workbench = createReactClass({ GitBlameScreenshotViewerActions.openScreenshotsViewer(branchScreenshots); }}/> + + + { diff --git a/webapp/src/main/resources/public/js/sdk/TextUnitClient.js b/webapp/src/main/resources/public/js/sdk/TextUnitClient.js index 57ef104ea6..eaea836a59 100644 --- a/webapp/src/main/resources/public/js/sdk/TextUnitClient.js +++ b/webapp/src/main/resources/public/js/sdk/TextUnitClient.js @@ -110,6 +110,12 @@ class TextUnitClient extends BaseClient { }); } + getAiReview(textUnit) { + return this.get(this.baseUrl + "proto-ai-review", { + "tmTextUnitVariantId": textUnit.getTmTextUnitVariantId() + }); + } + getEntityName() { return 'textunits'; } diff --git a/webapp/src/main/resources/public/js/stores/workbench/AiReviewStore.js b/webapp/src/main/resources/public/js/stores/workbench/AiReviewStore.js new file mode 100644 index 0000000000..fb212fe0c9 --- /dev/null +++ b/webapp/src/main/resources/public/js/stores/workbench/AiReviewStore.js @@ -0,0 +1,46 @@ +import alt from "../../alt"; +import TextUnitDataSource from "../../actions/workbench/TextUnitDataSource"; +import AIReviewActions from "../../actions/workbench/AiReviewActions"; + +class AIReviewStore { + + constructor() { + this.setDefaultState(); + this.bindActions(AIReviewActions); + + this.registerAsync(TextUnitDataSource); + } + + setDefaultState() { + this.show = false; + this.textUnit = null; + this.review = null; + this.loading = false; + } + + close() { + this.show = false; + } + + openWithTextUnit(textUnit) { + this.show = true; + this.textUnit = textUnit; + this.review = null; + this.loading = true; + this.getInstance().getAiReview(textUnit); + } + + onGetAiReviewSuccess(protoAiReviewResponse) { + console.log("AIReviewStore::onGetAiReviewInfoSuccess"); + this.review = protoAiReviewResponse.aiReviewOutput; + console.log("\n\n\n\n" + this.review + "\n\n\n\n") + this.loading = false; + } + + onGetAiReviewError(errorResponse) { + console.log("AIReviewStore::onGetAiReviewInfoError"); + this.loading = false; + } +} + +export default alt.createStore(AIReviewStore, 'AIReviewStore'); diff --git a/webapp/src/main/resources/sass/mojito.scss b/webapp/src/main/resources/sass/mojito.scss index f2e197e201..f54082b5a6 100644 --- a/webapp/src/main/resources/sass/mojito.scss +++ b/webapp/src/main/resources/sass/mojito.scss @@ -442,6 +442,15 @@ div.textunit label { color: $brand-success; } +.textunit:hover .textunit-assetpath, .textunit:hover .textunit-ai-review { + visibility: visible; +} + +.textunit-name .textunit-assetpath, .textunit-name .textunit-ai-review { + visibility: hidden; + color: $brand-success; +} + .textunit-returnline { color: $brand-primary; font-size: 10px; @@ -466,6 +475,164 @@ div.textunit label { white-space: pre-wrap; } +.ai-review-modal { +} + +.ai-review-modal .modal-dialog { + max-width: 100%; + width: 90%; +} + + +.ai-review-root2 { + +} + +.ai-review-root2 .section-title { + font-weight: bold; + margin-top: 10px; + margin-bottom: 10px; +} + +.ai-review-root2 .text-unit { + display: grid; + grid-template-columns: 1fr 1fr; + grid-template-areas: + "header header" + "left right"; +} + +.ai-review-root2 .text-unit .right { + align-self: center; + display: grid; + grid-template-columns: 1fr min-content 10px; + grid-template-areas: + "target status ." +} + +.ai-review-root2 .grid-2c { + display: grid; + grid-template-columns: 1fr 11fr; + grid-template-areas: "left right"; +} + +.ai-review-root2 .grid-2c .left { + grid-area: left; + text-align: center; +} + +.ai-review-root2 .grid-2c .right { + grid-area: right; +} + + + +////// + + +.ai-review-root { + display: grid; + grid-gap: 20px; + grid-template-columns: min-content 2fr 1fr; + grid-template: + "header header-right" + "source target" + "comment target" + "ratings ratings" + "comment-rating target-rating" + "suggestions suggestions" + "suggestion1 suggestion2" + +} + +.ai-review-root .locale { + grid-area: locale; +} + +.ai-review-root .header { + grid-area: header; + display: flex; + gap: 10px; +} + +.ai-review-root .header-right { + grid-area: header-right; + display: flex; + gap: 5px; + flex-wrap: wrap; + align-content: space-around; +} + +.ai-review-root .source { + grid-area: source; +} + +.ai-review-root .target { + grid-area: target; + align-content: center; +} + +.ai-review-root .target-status { + grid-area: target-status; + text-align: right; + align-content: center; +} + + +.ai-review-root .comment { + grid-area: comment; +} + +.ai-review-root .comment-rating { + grid-area: comment-rating; + align-content: center; +} + +.ai-review-root .target-rating { + grid-area: target-rating; + align-content: center; +} + +.ai-review-root .ratings { + grid-area: ratings; + align-content: center; + font-weight: bold; + text-align: center; + margin-top: 10px; +} + +.ai-review-root .rating { + display: grid; + grid-template-columns: 1fr 11fr; + grid-gap: 5px; +} + + +.ai-review-root .suggestions { + display: grid; + grid-area: suggestions; + margin-top: 10px; + text-align: center; + font-weight: bold; + margin-bottom: 10px; +} + +.ai-review-root .suggestion1 { + grid-area: suggestion1; +} + +.ai-review-root .suggestion2 { + grid-area: suggestion2; +} + + +.ai-review-modal .loading { + display: flex; + align-items: center; + justify-content: center; + min-height: 75px; +} + .text-unit-root { display: grid; align-items: center; diff --git a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java index 25b35caf64..aff676de04 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapperTest.java @@ -58,6 +58,29 @@ public void testReadFromSourceTextUnitsWithoutPluralForms() { assertThat(singular2.getComment()).isEqualTo("comment1"); } + @Test + public void testReadFromSourceTextUnitsWithoutPluralFormsAndWithTmTextUnitIdInName() { + mapper = new AndroidStringDocumentMapper("_", assetDelimiter, null, null, true, null); + textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); + textUnits.add(sourceTextUnitDTO(124L, "name1", "content1", "comment1", "my/path1", null, null)); + + document = mapper.readFromSourceTextUnits(textUnits); + + assertThat(document).isNotNull(); + assertThat(document.getPlurals()).isEmpty(); + assertThat(document.getSingulars()).hasSize(2); + AndroidSingular singular1 = document.getSingulars().get(0); + assertThat(singular1.getId()).isEqualTo(123L); + assertThat(singular1.getName()).isEqualTo("123#@#my/path0#@#name0"); + assertThat(singular1.getContent()).isEqualTo("content0"); + assertThat(singular1.getComment()).isEqualTo("comment0"); + AndroidSingular singular2 = document.getSingulars().get(1); + assertThat(singular2.getId()).isEqualTo(124L); + assertThat(singular2.getName()).isEqualTo("124#@#my/path1#@#name1"); + assertThat(singular2.getContent()).isEqualTo("content1"); + assertThat(singular2.getComment()).isEqualTo("comment1"); + } + @Test public void testReadFromSourceTextUnitsWithoutPluralFormsRemovesBadCharacters() { mapper = new AndroidStringDocumentMapper("_", assetDelimiter); @@ -135,7 +158,66 @@ public void testReadFromSourceTextUnitsWithPlurals() { } @Test - public void testReadFromSourceTextUnitsWithDuplicatePlurals() { + public void testReadFromSourceTextUnitsWithPluralsAndWithTmTextUnitIdInName() { + mapper = new AndroidStringDocumentMapper(" _", assetDelimiter, null, null, true, null); + textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); + + textUnits.add( + sourceTextUnitDTO( + 100L, "name1 _other", "content1_zero", "comment1", "my/path1", "zero", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 101L, "name1 _other", "content1_one", "comment1", "my/path1", "one", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 102L, "name1 _other", "content1_two", "comment1", "my/path1", "two", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 103L, "name1 _other", "content1_few", "comment1", "my/path1", "few", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 104L, "name1 _other", "content1_many", "comment1", "my/path1", "many", "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 105L, + "name1 _other", + "content1_other", + "comment1", + "my/path1", + "other", + "name1_other")); + + document = mapper.readFromSourceTextUnits(textUnits); + + assertThat(document).isNotNull(); + assertThat(document.getSingulars()).hasSize(1); + AndroidSingular singular = document.getSingulars().get(0); + assertThat(singular.getId()).isEqualTo(123L); + assertThat(singular.getName()).isEqualTo("123#@#my/path0#@#name0"); + assertThat(singular.getContent()).isEqualTo("content0"); + assertThat(singular.getComment()).isEqualTo("comment0"); + + assertThat(document.getPlurals()).hasSize(1); + AndroidPlural plural = document.getPlurals().get(0); + assertThat(plural.getName()).isEqualTo("100,101,102,103,104,105#@#my/path1#@#name1"); + assertThat(plural.getComment()).isEqualTo("comment1"); + assertThat(plural.getItems()).hasSize(6); + assertThat(plural.getItems().get(ZERO).getId()).isEqualTo(100L); + assertThat(plural.getItems().get(ZERO).getContent()).isEqualTo("content1_zero"); + assertThat(plural.getItems().get(ONE).getId()).isEqualTo(101L); + assertThat(plural.getItems().get(ONE).getContent()).isEqualTo("content1_one"); + assertThat(plural.getItems().get(TWO).getId()).isEqualTo(102L); + assertThat(plural.getItems().get(TWO).getContent()).isEqualTo("content1_two"); + assertThat(plural.getItems().get(FEW).getId()).isEqualTo(103L); + assertThat(plural.getItems().get(FEW).getContent()).isEqualTo("content1_few"); + assertThat(plural.getItems().get(MANY).getId()).isEqualTo(104L); + assertThat(plural.getItems().get(MANY).getContent()).isEqualTo("content1_many"); + assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(105L); + assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); + } + + @Test + public void testReadFromSourceTextUnitsWithDuplicatePluralsDifferentAsset() { mapper = new AndroidStringDocumentMapper(" _", assetDelimiter); textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); @@ -260,7 +342,7 @@ public void testReadFromSourceTextUnitsWithDuplicatePlurals() { assertThat(singular.getComment()).isEqualTo("comment0"); assertThat(document.getPlurals()).hasSize(2); - AndroidPlural plural = document.getPlurals().get(1); + AndroidPlural plural = document.getPlurals().get(0); assertThat(plural.getName()).isEqualTo("my/path0.xml#@#name1"); assertThat(plural.getComment()).isEqualTo("comment1"); assertThat(plural.getItems()).hasSize(6); @@ -277,7 +359,7 @@ public void testReadFromSourceTextUnitsWithDuplicatePlurals() { assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(105L); assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); - plural = document.getPlurals().get(0); + plural = document.getPlurals().get(1); assertThat(plural.getName()).isEqualTo("my/path1.xml#@#name1"); assertThat(plural.getComment()).isEqualTo("comment1"); assertThat(plural.getItems()).hasSize(6); @@ -295,6 +377,175 @@ public void testReadFromSourceTextUnitsWithDuplicatePlurals() { assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); } + @Test + public void testReadFromSourceTextUnitsWithDuplicatePluralsDifferentBranch() { + mapper = new AndroidStringDocumentMapper(" _", assetDelimiter); + textUnits.add(sourceTextUnitDTO(123L, "name0", "content0", "comment0", "my/path0", null, null)); + + // first group + textUnits.add( + sourceTextUnitDTO( + 100L, + "name1 _other", + "content1_zero", + "comment1", + "my/path0.xml", + "zero", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 101L, + "name1 _other", + "content1_one", + "comment1", + "my/path0.xml", + "one", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 102L, + "name1 _other", + "content1_two", + "comment1", + "my/path0.xml", + "two", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 103L, + "name1 _other", + "content1_few", + "comment1", + "my/path0.xml", + "few", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 104L, + "name1 _other", + "content1_many", + "comment1", + "my/path0.xml", + "many", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 105L, + "name1 _other", + "content1_other", + "comment1", + "my/path0.xml", + "other", + "name1_other")); + + // second group + textUnits.add( + sourceTextUnitDTO( + 200L, + "name1 _other", + "content1_zero_b", + "comment1", + "my/path0.xml", + "zero", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 201L, + "name1 _other", + "content1_one_b", + "comment1", + "my/path0.xml", + "one", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 202L, + "name1 _other", + "content1_two_b", + "comment1", + "my/path0.xml", + "two", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 203L, + "name1 _other", + "content1_few_b", + "comment1", + "my/path0.xml", + "few", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 204L, + "name1 _other", + "content1_many_b", + "comment1", + "my/path0.xml", + "many", + "name1_other")); + textUnits.add( + sourceTextUnitDTO( + 205L, + "name1 _other", + "content1_other_b", + "comment1", + "my/path0.xml", + "other", + "name1_other")); + + // Set different asset extraction id ~= branch to the text units. Keep null on first one + int group2StartIdx = 7; + for (int i = group2StartIdx; i < group2StartIdx + 6; i++) { + textUnits.get(i).setBranchId(123L); + } + + document = mapper.readFromSourceTextUnits(textUnits); + + assertThat(document).isNotNull(); + assertThat(document.getSingulars()).hasSize(1); + AndroidSingular singular = document.getSingulars().get(0); + assertThat(singular.getId()).isEqualTo(123L); + assertThat(singular.getName()).isEqualTo("my/path0#@#name0"); + assertThat(singular.getContent()).isEqualTo("content0"); + assertThat(singular.getComment()).isEqualTo("comment0"); + + assertThat(document.getPlurals()).hasSize(2); + AndroidPlural plural = document.getPlurals().get(0); + assertThat(plural.getName()).isEqualTo("my/path0.xml#@#name1"); + assertThat(plural.getComment()).isEqualTo("comment1"); + assertThat(plural.getItems()).hasSize(6); + assertThat(plural.getItems().get(ZERO).getId()).isEqualTo(100L); + assertThat(plural.getItems().get(ZERO).getContent()).isEqualTo("content1_zero"); + assertThat(plural.getItems().get(ONE).getId()).isEqualTo(101L); + assertThat(plural.getItems().get(ONE).getContent()).isEqualTo("content1_one"); + assertThat(plural.getItems().get(TWO).getId()).isEqualTo(102L); + assertThat(plural.getItems().get(TWO).getContent()).isEqualTo("content1_two"); + assertThat(plural.getItems().get(FEW).getId()).isEqualTo(103L); + assertThat(plural.getItems().get(FEW).getContent()).isEqualTo("content1_few"); + assertThat(plural.getItems().get(MANY).getId()).isEqualTo(104L); + assertThat(plural.getItems().get(MANY).getContent()).isEqualTo("content1_many"); + assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(105L); + assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); + + plural = document.getPlurals().get(1); + assertThat(plural.getName()).isEqualTo("my/path0.xml#@#name1"); + assertThat(plural.getComment()).isEqualTo("comment1"); + assertThat(plural.getItems()).hasSize(6); + assertThat(plural.getItems().get(ZERO).getId()).isEqualTo(200L); + assertThat(plural.getItems().get(ZERO).getContent()).isEqualTo("content1_zero_b"); + assertThat(plural.getItems().get(ONE).getId()).isEqualTo(201L); + assertThat(plural.getItems().get(ONE).getContent()).isEqualTo("content1_one_b"); + assertThat(plural.getItems().get(TWO).getId()).isEqualTo(202L); + assertThat(plural.getItems().get(TWO).getContent()).isEqualTo("content1_two_b"); + assertThat(plural.getItems().get(FEW).getId()).isEqualTo(203L); + assertThat(plural.getItems().get(FEW).getContent()).isEqualTo("content1_few_b"); + assertThat(plural.getItems().get(MANY).getId()).isEqualTo(204L); + assertThat(plural.getItems().get(MANY).getContent()).isEqualTo("content1_many_b"); + assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(205L); + assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other_b"); + } + @Test public void testReadFromTargetTextUnitsForEmptyList() { mapper = new AndroidStringDocumentMapper("", assetDelimiter); @@ -531,7 +782,7 @@ public void testReadFromTargetTextUnitsWithPluralsDuplicated() { assertThat(singular.getComment()).isEqualTo("comment0"); assertThat(document.getPlurals()).hasSize(2); - AndroidPlural plural = document.getPlurals().get(1); + AndroidPlural plural = document.getPlurals().get(0); assertThat(plural.getName()).isEqualTo("my/path0.xml#@#name1"); assertThat(plural.getComment()).isEqualTo("comment1"); assertThat(plural.getItems()).hasSize(6); @@ -548,7 +799,7 @@ public void testReadFromTargetTextUnitsWithPluralsDuplicated() { assertThat(plural.getItems().get(OTHER).getId()).isEqualTo(105L); assertThat(plural.getItems().get(OTHER).getContent()).isEqualTo("content1_other"); - plural = document.getPlurals().get(0); + plural = document.getPlurals().get(1); assertThat(plural.getName()).isEqualTo("my/path1.xml#@#name1"); assertThat(plural.getComment()).isEqualTo("comment1"); assertThat(plural.getItems()).hasSize(6); @@ -667,19 +918,29 @@ public void testReadingTextUnitsFromFile() throws Exception { assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("some_string")) - .extracting("target", "comment") - .containsOnly(tuple("Dela...", "Some string")); + .extracting("name", "target", "comment") + .containsOnly(tuple("some_string", "Dela...", "Some string")); assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("show_options")) - .extracting("target", "comment") - .containsOnly(tuple("Mer \" dela", "Options")); + .extracting("name", "target", "comment") + .containsOnly(tuple("show_options", "Mer \" dela", null)); assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("without_comment")) .extracting("target", "comment") .containsOnly(tuple("ei ' kommentteja", null)); + assertThat(textUnits) + .filteredOn(tu -> tu.getName().equalsIgnoreCase("show_options2")) + .extracting("name", "target", "comment") + .containsOnly(tuple("show_options2", "Mer \\\" dela", "Options")); + + assertThat(textUnits) + .filteredOn(tu -> tu.getName().equalsIgnoreCase("without_comment2")) + .extracting("target", "comment") + .containsOnly(tuple("ei \\' kommentteja", null)); + assertThat(textUnits) .filteredOn(tu -> tu.getName().equalsIgnoreCase("line_break")) .extracting("target", "comment") @@ -734,6 +995,22 @@ public void testAddTextUnitDTOAttributesAssetPathAndName() { .containsExactly("asset_path", "name_part1#@#name_part2"); } + @Test + public void testAddTextUnitDTOAttributesTextUnitIdAndAssetPathAndName() { + mapper = new AndroidStringDocumentMapper("_", null, null, null, true, null); + TextUnitDTO textUnitDTO = new TextUnitDTO(); + + textUnitDTO.setName("156151#@#asset_path#@#name_part1"); + assertThat(mapper.addTextUnitDTOAttributes(textUnitDTO)) + .extracting(TextUnitDTO::getTmTextUnitId, TextUnitDTO::getAssetPath, TextUnitDTO::getName) + .containsExactly(156151L, "asset_path", "name_part1"); + + textUnitDTO.setName("156152#@#asset_path#@#name_part1#@#name_part2"); + assertThat(mapper.addTextUnitDTOAttributes(textUnitDTO)) + .extracting(TextUnitDTO::getTmTextUnitId, TextUnitDTO::getAssetPath, TextUnitDTO::getName) + .containsExactly(156152L, "asset_path", "name_part1#@#name_part2"); + } + private TextUnitDTO sourceTextUnitDTO( Long id, String name, diff --git a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java index fc215103d6..214e3660dc 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentWriterTest.java @@ -140,6 +140,44 @@ public void testWriteChars() throws Exception { assertThat(getTempFileContent()).isEqualTo(result); } + @Test + public void testEscaping() throws Exception { + + // Not sure why the current implementation uses escapeQuotes(). Intuitively, for + // machine-to-machine communication, pure XML + // seems to be the best option. However, it appears that escapeQuotes() tries to mimic the + // Android format + // overloads. I'm wondering if this was required by Smartling. + result = + """ + + with a <a href=\\"http://test.org\\"> link</a>. + <i>i</i> could be unescape, but currently is escaped. + a string with\\n return line + a string with \\" quote. + <annotation url=\\"http://test.com\\">test</annotation> + + """; + + source = new AndroidStringDocument(); + source.addSingular( + new AndroidSingular(120L, "a_href", "with a link.", "")); + source.addSingular( + new AndroidSingular( + 121L, "i", "i could be unescape, but currently is escaped.", "")); + source.addSingular(new AndroidSingular(122L, "return_line", "a string with\n return line", "")); + source.addSingular(new AndroidSingular(123L, "with_quotes", "a string with \" quote.", "")); + source.addSingular( + new AndroidSingular( + 124L, "with_annotation", "test", "")); + + writer = new AndroidStringDocumentWriter(source); + assertThat(writer.toText()).isEqualTo(result); + + writer.toFile(tmpFile.getPath()); + assertThat(getTempFileContent()).isEqualTo(result); + } + @Test public void testWritePlurals() throws Exception { @@ -226,14 +264,40 @@ public void testWritePreservesOrdering() throws Exception { assertThat(getTempFileContent()).isEqualTo(result); } + @Test + public void testGenerateWithHTML() { + source = new AndroidStringDocument(); + source.addSingular( + new AndroidSingular(10L, "string1", "some link to a page", "test comment1")); + source.addSingular( + new AndroidSingular( + 11L, + "string2", + "some link to a page", + "test comment2")); + source.addSingular( + new AndroidSingular( + 12L, + "string3", + "If that is your IP address click here to unblock it.", + "test comment2")); + + writer = new AndroidStringDocumentWriter(source); + // TODO(ja) must finsih + System.out.println(writer.toText()); + // assertThat(writer.toText()).isEqualTo(result); + } + @Test public void testEscapeQuotes() { - assertThat(escapeQuotes(null)).isEqualTo(""); - assertThat(escapeQuotes("")).isEqualTo(""); - assertThat(escapeQuotes("String")).isEqualTo("String"); - assertThat(escapeQuotes("second\\\"String")).isEqualTo("second\\\\\"String"); - assertThat(escapeQuotes("third\nString")).isEqualTo("third\\nString"); - assertThat(escapeQuotes("fourth\ntest\\\"String\\\"")) + AndroidStringDocumentWriter.EscapeType escapeType = + AndroidStringDocumentWriter.EscapeType.QUOTE_AND_NEW_LINE; + assertThat(escapeQuotes(null, escapeType)).isEqualTo(null); + assertThat(escapeQuotes("", escapeType)).isEqualTo(""); + assertThat(escapeQuotes("String", escapeType)).isEqualTo("String"); + assertThat(escapeQuotes("second\\\"String", escapeType)).isEqualTo("second\\\\\"String"); + assertThat(escapeQuotes("third\nString", escapeType)).isEqualTo("third\\nString"); + assertThat(escapeQuotes("fourth\ntest\\\"String\\\"", escapeType)) .isEqualTo("fourth\\ntest\\\\\"String\\\\\""); } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java index 6c8c97ea4a..1503212bcc 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/CompositeFormatIntegrityCheckerTest.java @@ -17,7 +17,7 @@ public class CompositeFormatIntegrityCheckerTest { public void testGetPlaceholder() throws CompositeFormatIntegrityCheckerException { CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); - String string = "{0} '{2}' {0:0.00}% \"{1}\""; + String string = "{0} '{2}' {0:0.00}% \"{1}\" {{3}} {{{4}}} {{{5}} {{6}}}"; Set placeholders = checker.getPlaceholders(string); @@ -26,6 +26,10 @@ public void testGetPlaceholder() throws CompositeFormatIntegrityCheckerException expected.add("{2}"); expected.add("{0:0.00}"); expected.add("{1}"); + expected.add("{{3}}"); + expected.add("{{{4}}}"); + expected.add("{{{5}}"); + expected.add("{{6}}}"); assertEquals(expected, placeholders); } @@ -62,6 +66,16 @@ public void testMustache() { checker.check(source, target); } + @Test(expected = CompositeFormatIntegrityCheckerException.class) + public void testMustacheUnmatchedBrace() { + CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); + + String source = "A {{ mustache }} template."; + String target = "Un modėle {{ mustache }."; + + checker.check(source, target); + } + @Test(expected = CompositeFormatIntegrityCheckerException.class) public void testMustacheInvalidMissingCurlyBraces() { CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); @@ -72,6 +86,26 @@ public void testMustacheInvalidMissingCurlyBraces() { checker.check(source, target); } + @Test(expected = CompositeFormatIntegrityCheckerException.class) + public void testMustacheInvalidMissingCurlyBraces2() { + CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); + + String source = "A {{mustache} template"; + String target = "Un modėle {{{mustache}"; + + checker.check(source, target); + } + + @Test(expected = CompositeFormatIntegrityCheckerException.class) + public void testMustacheInvalidMissingCurlyBraces3() { + CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); + + String source = "A {{mustache}} template"; + String target = "Un modėle {{{mustache}}"; + + checker.check(source, target); + } + @Test(expected = CompositeFormatIntegrityCheckerException.class) public void testMustacheInvalidMissingPlaceholder() { CompositeFormatIntegrityChecker checker = new CompositeFormatIntegrityChecker(); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerTest.java new file mode 100644 index 0000000000..c924428e7f --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/EmailIntegrityCheckerTest.java @@ -0,0 +1,38 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import org.junit.Ignore; +import org.junit.Test; + +public class EmailIntegrityCheckerTest { + + EmailIntegrityChecker checker = new EmailIntegrityChecker(); + + @Test + public void testNoEmail() { + String source = "There is no email"; + String target = "Il n'y a pas d'email"; + checker.check(source, target); + } + + @Test(expected = EmailIntegrityCheckerException.class) + public void testMissingInTarget() { + String source = "There is an ja@test.com"; + String target = "Il n'y a pas d'email"; + checker.check(source, target); + } + + @Test(expected = EmailIntegrityCheckerException.class) + public void testAddedInTarget() { + String source = "There is no email"; + String target = "Il n'y a un email ja@test.com"; + checker.check(source, target); + } + + @Ignore // not supported yet + @Test(expected = EmailIntegrityCheckerException.class) + public void testDuplicates() { + String source = "There is an ja@test.com"; + String target = "Il n'y a un email ja@test.com ja@test.com"; + checker.check(source, target); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java index 32c24087fa..3b68f8644f 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/HtmlTagIntegrityCheckerTest.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; +import java.util.Set; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,6 +48,34 @@ public void testHtmlTagCheckWorksWhenMissingATag() { checker.check(source, target); } + @Test(expected = HtmlTagIntegrityCheckerException.class) + public void testHtmlTagCheckWorksWhenEndingFirst() { + String source = "There are %1 files"; + String target = "Il y a %1 fichiers et %2 dossiers"; + checker.check(source, target); + } + + @Test(expected = HtmlTagIntegrityCheckerException.class) + public void testHtmlTagCheckWorksWhenCrossing() { + String source = "There are %1 files and %2 folders"; + String target = "Il y a %1 fichiers et %2 dossiers"; + checker.check(source, target); + } + + @Test(expected = HtmlTagIntegrityCheckerException.class) + public void testHtmlTagCheckFailsWithDuplicatedOpening() { + String source = "There are %1 files"; + String target = "Il y a %1 fichiers"; + checker.check(source, target); + } + + @Test(expected = HtmlTagIntegrityCheckerException.class) + public void testHtmlTagCheckFailsWithDuplicatedButSameCount() { + String source = "

some text

some other text

"; + String target = "

some text

some other text

"; + checker.check(source, target); + } + @Test(expected = HtmlTagIntegrityCheckerException.class) public void testHtmlTagCheckWorksWhenTagIsModified() { String source = "There are %1 files and %2 folders"; @@ -163,6 +193,14 @@ public void testHtmlTagCheckCloseTagsSame() { checker.check(source, target); } + @Test(expected = HtmlTagIntegrityCheckerException.class) + public void testHtmlTagCheckWithDuplicatedAnnotation() { + String source = "text"; + String target = "text"; + + checker.check(source, target); + } + @Test public void testHtmlTagCheckNonTagLessThanDoesntConfuseThings() { String source = "Upload is <10% complete."; @@ -170,4 +208,12 @@ public void testHtmlTagCheckNonTagLessThanDoesntConfuseThings() { checker.check(source, target); } + + @Test + public void testHtmlTagCheckWithDash() { + String source = "Mag-sign up o upang "; + Set placeholders = checker.getPlaceholders(source); + Assertions.assertThat(placeholders) + .containsExactly("", "", "", ""); + } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerTest.java new file mode 100644 index 0000000000..8ec6a5614e --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityCheckerTest.java @@ -0,0 +1,54 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; + +import java.util.Set; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public class MarkdownLinkIntegrityCheckerTest { + + MarkdownLinkIntegrityChecker checker = new MarkdownLinkIntegrityChecker(); + + @Test + public void testLink() { + String source = "This is [a link](http://localhost)"; + String target = "c'est [un lien](http://localhost)"; + checker.check(source, target); + } + + @Test + public void testBrokenLinkExtraSpace() { + String source = "This is [a link](http://localhost)"; + String target = "c'est [un lien] (http://localhost)"; + assertThrowsExactly( + MarkdownLinkIntegrityCheckerException.class, () -> checker.check(source, target)); + } + + @Test + public void testBrokenLinkTranslatedUrl() { + String source = "This is [a link](http://localhost)"; + String target = "c'est [un lien] (http://translated)"; + assertThrowsExactly( + MarkdownLinkIntegrityCheckerException.class, () -> checker.check(source, target)); + } + + @Test + public void testReorder() { + String source = + "This is the [first link](http://localhost/1) and this is the [second link](http://localhost/2)"; + String target = + "C'est le [dieuxieme lien](http://localhost/2) et c'est le [premier lien](http://localhost/1)"; + checker.check(source, target); + } + + @Test + public void getPlaceholder() { + String source = + "This is [a link](http://localhost/1) and another link [2nd link](http://localhost/2)"; + Set placeholders = checker.getPlaceholders(source); + Assertions.assertThat(placeholders) + .containsExactly( + "[--translatable--](http://localhost/1)", "[--translatable--](http://localhost/2)"); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java index 12d0d51124..faf288f852 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/MessageFormatIntegrityCheckerTest.java @@ -3,6 +3,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import com.google.common.collect.ImmutableMap; +import com.ibm.icu.text.MessageFormat; import org.junit.Test; public class MessageFormatIntegrityCheckerTest { @@ -128,4 +130,17 @@ public void testNamedParametersChangedButWithDuplicates() // assertEquals("Different placeholder name in source and target", e.getMessage()); } } + + @Test(expected = IntegrityCheckException.class) + public void testQuoteCurlyEscaping() throws MessageFormatIntegrityCheckerException { + // ' with character are rendered by if it is a special character like {, it will escape it .... + MessageFormatIntegrityChecker checker = new MessageFormatIntegrityChecker(); + String source = "This is a \"{placeholder}\""; + String target = "C'est un '{placeholder}'"; + checker.check(source, target); + + MessageFormat messageFormat = new MessageFormat(target); + String format = messageFormat.format(ImmutableMap.of("placeholder", "stuff")); + assertEquals("C'est un {placeholder}", format); + } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxerTest.java new file mode 100644 index 0000000000..7afeb4961d --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PluralIntegrityCheckerRelaxerTest.java @@ -0,0 +1,64 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.Test; + +public class PluralIntegrityCheckerRelaxerTest { + + @Test + public void testShouldRelaxIntegrityCheck_whenPluralFormIsOther() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + boolean result = + relaxer.shouldRelaxIntegrityCheck( + "source %d", "target", "other", new PrintfLikeIntegrityChecker()); + + assertFalse(result, "Integrity check should not be relaxed for plural form 'other'."); + } + + @Test + public void testShouldRelaxIntegrityCheck_whenPlaceholdersDifferByOne_One() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + boolean result = + relaxer.shouldRelaxIntegrityCheck( + "source %d", "target", "one", new PrintfLikeIntegrityChecker()); + + assertTrue( + result, "Integrity check should be relaxed when placeholders differ by one for 'one' form"); + } + + @Test + public void testShouldRelaxIntegrityCheck_whenPlaceholdersDifferByOne_One_DuplicatePlaceholder() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + boolean result = + relaxer.shouldRelaxIntegrityCheck( + "source %d %d", "target", "one", new PrintfLikeIntegrityChecker()); + + assertTrue( + result, + "Integrity check should be relaxed when placeholders differ by one for 'one' form (as placeholders are stored in a set, ignoring duplicates)"); + } + + @Test + public void testShouldRelaxIntegrityCheck_whenPlaceholdersDifferByTwo_One() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + boolean result = + relaxer.shouldRelaxIntegrityCheck( + "source %1d %2d", "target", "one", new PrintfLikeIntegrityChecker()); + + assertFalse( + result, + "Integrity check should not be relaxed when placeholders differ by two for 'one' form"); + } + + @Test + public void testShouldRelaxIntegrityCheck_withUnsupportedChecker() { + PluralIntegrityCheckerRelaxer relaxer = new PluralIntegrityCheckerRelaxer(); + + boolean result = + relaxer.shouldRelaxIntegrityCheck("source %d", "target", "one", new URLIntegrityChecker()); + + assertFalse(result, "Integrity check should not be relaxed for unsupported checker types."); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintLikeVariableTypeIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintLikeVariableTypeIntegrityCheckerTest.java index 2f47fe968d..33a8f69fac 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintLikeVariableTypeIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintLikeVariableTypeIntegrityCheckerTest.java @@ -27,7 +27,8 @@ public void testMissingVariableTypeCausesIntegrityViolation() checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } } @@ -42,7 +43,8 @@ public void testModifiedVariableTypeCausesIntegrityViolation() checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } } @@ -56,7 +58,8 @@ public void testVariableWithFormattingFlagChecked() { checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } source = "%(count) s view"; @@ -66,7 +69,8 @@ public void testVariableWithFormattingFlagChecked() { checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } source = "%(count).1f view"; @@ -76,7 +80,8 @@ public void testVariableWithFormattingFlagChecked() { checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } } @@ -90,7 +95,8 @@ public void testCurlyBracketsAreChecked() { checker.check(source, target); fail("PrintfLikeVariableTypeIntegrityCheckerException should have been thrown."); } catch (PrintfLikeVariableTypeIntegrityCheckerException e) { - assertEquals("Variable types do not match.", e.getMessage()); + assertEquals( + "PrintfLikeVariableType placeholder are different in source and target.", e.getMessage()); } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeAddParameterSpecifierIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeAddParameterSpecifierIntegrityCheckerTest.java index 82ebce5fad..1417f0cfaa 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeAddParameterSpecifierIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeAddParameterSpecifierIntegrityCheckerTest.java @@ -133,7 +133,7 @@ public void testMacPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -150,7 +150,7 @@ public void testMacPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeatedOr checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -167,7 +167,7 @@ public void testMacPlaceholderCheckFailsIfSamePlaceholdersCountButSpecifierModif checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -243,7 +243,7 @@ public void testAndroidPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -260,7 +260,7 @@ public void testAndroidPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeat checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -277,7 +277,7 @@ public void testAndroidPlaceholderCheckFailsIfSamePlaceholdersCountButSpecifierM checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -304,7 +304,7 @@ public void testIncorrectlyModifiedSpecifier() throws PrintfLikeIntegrityChecker checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityCheckerTest.java index 8bba1b6b16..48ab2a28e3 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PrintfLikeIntegrityCheckerTest.java @@ -123,7 +123,7 @@ public void testMacPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -139,7 +139,7 @@ public void testMacPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeatedOr checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -155,7 +155,7 @@ public void testMacPlaceholderCheckFailsIfSamePlaceholdersCountButSpecifierModif checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -225,7 +225,7 @@ public void testAndroidPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -241,7 +241,7 @@ public void testAndroidPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeat checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -257,7 +257,7 @@ public void testAndroidPlaceholderCheckFailsIfSamePlaceholdersCountButSpecifierM checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } @@ -282,7 +282,7 @@ public void testIncorrectlyModifiedSpecifier() throws PrintfLikeIntegrityChecker checker.check(source, target); fail("PrintfLikeIntegrityCheckerException must be thrown"); } catch (PrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals(e.getMessage(), "PrintfLike placeholders are different in source and target"); } } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerTest.java new file mode 100644 index 0000000000..7a8c223800 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/PythonFStringIntegrityCheckerTest.java @@ -0,0 +1,40 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.Test; + +public class PythonFStringIntegrityCheckerTest { + + PythonFStringIntegrityChecker checker = new PythonFStringIntegrityChecker(); + + @Test + public void testPlaceholderOK() { + String source = "This is a $placeholder"; + String target = "C'est un $placeholder"; + checker.check(source, target); + } + + @Test + public void testPlaceholderMissing() { + String source = "This is a $placeholder"; + String target = "C'est un $placehor"; + assertThrowsExactly( + PythonFStringIntegrityCheckerException.class, () -> checker.check(source, target)); + } + + @Test + public void testPlaceholderCurlyOK() { + String source = "This is a ${placeholder}"; + String target = "C'est un ${placeholder}"; + checker.check(source, target); + } + + @Test + public void testPlaceholderCurlyMissing() { + String source = "This is a ${placeholder}"; + String target = "C'est un ${placehor"; + assertThrowsExactly( + PythonFStringIntegrityCheckerException.class, () -> checker.check(source, target)); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityCheckerTest.java index ab514561ef..11cb8ed38d 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityCheckerTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/SimplePrintfLikeIntegrityCheckerTest.java @@ -67,7 +67,8 @@ public void testPlaceholderCheckFailsIfDifferentPlaceholdersCount() checker.check(source, target); fail("SimplePrintfLikeIntegrityCheckerException must be thrown"); } catch (SimplePrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals( + e.getMessage(), "SimplePrintfLike placeholders are different in source and target."); } } @@ -82,7 +83,8 @@ public void testPlaceholderCheckFailsIfSamePlaceholdersCountButSomeRepeatedOrMis checker.check(source, target); fail("SimplePrintfLikeIntegrityCheckerException must be thrown"); } catch (SimplePrintfLikeIntegrityCheckerException e) { - assertEquals(e.getMessage(), "Placeholders in source and target are different"); + assertEquals( + e.getMessage(), "SimplePrintfLike placeholders are different in source and target."); } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java new file mode 100644 index 0000000000..aa0687a685 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/URLIntegrityCheckerTest.java @@ -0,0 +1,99 @@ +package com.box.l10n.mojito.service.assetintegritychecker.integritychecker; + +import org.junit.Test; + +public class URLIntegrityCheckerTest { + + URLIntegrityChecker checker = new URLIntegrityChecker(); + + @Test + public void noURL() { + String source = "No URL"; + String target = "Pas d'URL"; + checker.check(source, target); + } + + @Test + public void validUrl() { + String source = "Visit our site at https://example.org"; + String target = "Visitez notre site à https://example.org"; + checker.check(source, target); + } + + @Test + public void validUrlCJK() { + String source = "Please visit our website at http://example.org for more information."; + String target = "詳細については、こちらのウェブサイトhttp://example.orgをご覧ください。"; + checker.check(source, target); + } + + @Test + public void invalidUrlCJK() { + String source = "Please visit our website at http://example.org for more information."; + String target = "詳細については、こちらのウェブサイトhttp://example.org-をご覧ください。"; + checker.check(source, target); + } + + @Test + public void validUrlDifferentQuotes() { + String source = "Visit our site at \"https://example.org\""; + String target = "Visitez notre site à 'https://example.org'"; + checker.check(source, target); + } + + @Test + public void validUrlParam() { + String source = "Visit our site at \"https://example.org?test=value1\""; + String target = "Visitez notre site à 'https://example.org?test=value1'"; + checker.check(source, target); + } + + @Test(expected = URLIntegrityCheckerException.class) + public void missingUrl() { + String source = "Visit our site at https://example.org"; + String target = "Visitez notre site à"; + checker.check(source, target); + } + + @Test(expected = URLIntegrityCheckerException.class) + public void changedUrl() { + String source = "The website is https://example.org"; + String target = "Le site web est https://bad.org"; + checker.check(source, target); + } + + @Test + public void validHttpUrl() { + String source = "Visit our site at http://example.org"; + String target = "Visitez notre site à http://example.org"; + checker.check(source, target); + } + + @Test + public void validEmail() { + String source = "Send message to mailto:test@test.org"; + String target = "Envoyer un message a mailto:test@test.org"; + checker.check(source, target); + } + + @Test(expected = URLIntegrityCheckerException.class) + public void missingEmail() { + String source = "Send message to mailto:test@test.org"; + String target = "Envoyer un message a"; + checker.check(source, target); + } + + @Test(expected = URLIntegrityCheckerException.class) + public void chanagedEmail() { + String source = "There is no mailto:test@test.org"; + String target = "Il n'y a pas d'email mailto:tedst@test.org"; + checker.check(source, target); + } + + @Test(expected = URLIntegrityCheckerException.class) + public void urlWithDash() { + String source = "A url https://test.more.com/account/manage."; + String target = "Une url https://test.more.com/account/manage-."; + checker.check(source, target); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/branch/notification/BranchNotificationServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/branch/notification/BranchNotificationServiceTest.java index 98dd0c1e62..feec95d81b 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/branch/notification/BranchNotificationServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/branch/notification/BranchNotificationServiceTest.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.service.branch.notification; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; + import com.box.l10n.mojito.entity.AssetContent; import com.box.l10n.mojito.entity.Branch; import com.box.l10n.mojito.entity.BranchNotification; @@ -152,8 +154,9 @@ void translateBranch(Branch branch) throws ExecutionException, InterruptedExcept textUnitBatchImporterService .asyncImportTextUnits( importTextUnitsBatch.getTextUnits(), - importTextUnitsBatch.isIntegrityCheckSkipped(), - importTextUnitsBatch.isIntegrityCheckKeepStatusIfFailedAndSameTarget()) + fromLegacy( + importTextUnitsBatch.isIntegrityCheckSkipped(), + importTextUnitsBatch.isIntegrityCheckKeepStatusIfFailedAndSameTarget())) .get(); // make sure the stats are ready for whatever is done next in notificaiton diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java new file mode 100644 index 0000000000..afad21f8c3 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/oaitranslate/AiTranslateServiceTest.java @@ -0,0 +1,53 @@ +package com.box.l10n.mojito.service.oaitranslate; + +import com.box.l10n.mojito.service.assetExtraction.ServiceTestBase; +import com.box.l10n.mojito.service.repository.RepositoryNameAlreadyUsedException; +import com.box.l10n.mojito.service.repository.RepositoryService; +import com.box.l10n.mojito.service.tm.TMTestData; +import com.box.l10n.mojito.test.TestIdWatcher; +import java.util.concurrent.ExecutionException; +import org.junit.Assume; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +public class AiTranslateServiceTest extends ServiceTestBase { + + static Logger logger = LoggerFactory.getLogger(AiTranslateServiceTest.class); + + @Rule public TestIdWatcher testIdWatcher = new TestIdWatcher(); + + @Autowired AiTranslateService aiTranslateService; + + @Autowired AiTranslateConfigurationProperties aiTranslateConfigurationProperties; + + @Autowired RepositoryService repositoryService; + + @Test + public void aiTranslateBatch() throws ExecutionException, InterruptedException { + Assume.assumeNotNull(aiTranslateConfigurationProperties.getOpenaiClientToken()); + + TMTestData tmTestData = new TMTestData(testIdWatcher); + aiTranslateService + .aiTranslateAsync( + new AiTranslateService.AiTranslateInput( + tmTestData.repository.getName(), null, 100, null, true)) + .get(); + } + + @Test + public void aiTranslateNoBatch() + throws ExecutionException, InterruptedException, RepositoryNameAlreadyUsedException { + Assume.assumeNotNull(aiTranslateConfigurationProperties.getOpenaiClientToken()); + + TMTestData tmTestData = new TMTestData(testIdWatcher); + + aiTranslateService + .aiTranslateAsync( + new AiTranslateService.AiTranslateInput( + tmTestData.repository.getName(), null, 100, null, false)) + .get(); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java index 1b081589f3..642bf18ca9 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyServiceTestData.java @@ -177,4 +177,8 @@ public ThirdPartyServiceTestData init() throws Exception { logger.debug("Finished init of ThirdPartyServiceTestData"); return this; } + + public String getPluralSeparator() { + return "_"; + } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java new file mode 100644 index 0000000000..77af92b666 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhraseTest.java @@ -0,0 +1,140 @@ +package com.box.l10n.mojito.service.thirdparty; + +import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSPhrase.areTagsWithin5Minutes; +import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSPhrase.uploadTagToLocalDateTime; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.box.l10n.mojito.entity.Repository; +import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.service.assetExtraction.ServiceTestBase; +import com.box.l10n.mojito.service.repository.RepositoryLocaleCreationException; +import com.box.l10n.mojito.service.repository.RepositoryService; +import com.box.l10n.mojito.service.tm.search.StatusFilter; +import com.box.l10n.mojito.service.tm.search.TextUnitDTO; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcher; +import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters; +import com.box.l10n.mojito.test.TestIdWatcher; +import com.google.common.collect.ImmutableMap; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import org.junit.Assume; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +public class ThirdPartyTMSPhraseTest extends ServiceTestBase { + + static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhraseTest.class); + + @Autowired(required = false) + ThirdPartyTMSPhrase thirdPartyTMSPhrase; + + @Autowired RepositoryService repositoryService; + + @Autowired TextUnitSearcher textUnitSearcher; + + @Value("${test.phrase-client.projectId:}") + String testProjectId; + + @Rule public TestIdWatcher testIdWatcher = new TestIdWatcher(); + + @Test + public void testBasics() throws RepositoryLocaleCreationException { + Assume.assumeNotNull(thirdPartyTMSPhrase); + Assume.assumeNotNull(testProjectId); + + ThirdPartyServiceTestData thirdPartyServiceTestData = + new ThirdPartyServiceTestData(testIdWatcher); + + Repository repository = thirdPartyServiceTestData.repository; + repositoryService.addRepositoryLocale(repository, "fr"); + + repository + .getRepositoryLocales() + .forEach(rl -> logger.info("repository locale: {}", rl.getLocale().getBcp47Tag())); + + thirdPartyTMSPhrase.push( + repository, + testProjectId, + thirdPartyServiceTestData.getPluralSeparator(), + null, + null, + null); + + thirdPartyTMSPhrase.push( + repository, + testProjectId, + thirdPartyServiceTestData.getPluralSeparator(), + null, + null, + null); + + thirdPartyTMSPhrase.pull( + repository, + testProjectId, + thirdPartyServiceTestData.getPluralSeparator(), + ImmutableMap.of("fr-FR", "fr"), + null, + null, + null, + null, + null); + + TextUnitSearcherParameters params = new TextUnitSearcherParameters(); + params.setRepositoryIds(repository.getId()); + params.setRootLocaleExcluded(false); + params.setStatusFilter(StatusFilter.TRANSLATED); + List search = textUnitSearcher.search(params); + + if (logger.isInfoEnabled()) { + ObjectMapper objectMapper = new ObjectMapper(); + search.stream().forEach(t -> logger.info(objectMapper.writeValueAsStringUnchecked(t))); + } + + // List thirdPartyTextUnits = + // thirdPartyTMSPhrase.getThirdPartyTextUnits(repository, testProjectId, null); + // logger.info("Get") + // thirdPartyTextUnits.stream().forEach(t -> logger.info("third party text unit: {}", + // t)); + + } + + @Test + public void testUploadTagToLocalDateTimeValid() { + LocalDateTime localDateTime = uploadTagToLocalDateTime("push_test_2024_11_21_18_55_38_004_502"); + assertEquals( + "Parsed LocalDateTime does not match the expected value", + "2024-11-21T18:55:38.004", + localDateTime.format(DateTimeFormatter.ISO_DATE_TIME)); + } + + @Test + public void testUploadTagToLocalDateTimeInvalid() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> uploadTagToLocalDateTime("invalid_tag_2024_11_21_18_55_004")); + assertTrue(exception.getMessage().contains("Invalid tag format")); + } + + @Test + public void testTagsWithin5Minutes() { + String tag1 = "push_test_2024_11_21_18_55_38_004_502"; + String tag2 = "push_test_2024_11_21_18_53_30_123_456"; + assertTrue(areTagsWithin5Minutes(tag1, tag2)); + } + + @Test + public void testTagsOutside5Minutes() { + String tag1 = "push_test_2024_11_21_18_55_38_004_502"; + String tag2 = "push_test_2024_11_21_18_49_30_123_456"; + assertFalse(areTagsWithin5Minutes(tag1, tag2)); + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java index e79e185914..eeaa00ef98 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java @@ -2,6 +2,7 @@ import static com.box.l10n.mojito.android.strings.AndroidStringDocumentReader.fromText; import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -66,7 +67,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Ints; import io.micrometer.core.instrument.MeterRegistry; -import java.io.IOException; import java.time.Duration; import java.time.ZonedDateTime; import java.util.ArrayList; @@ -77,7 +77,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import javax.xml.parsers.ParserConfigurationException; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -96,7 +95,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.HttpStatus; import org.springframework.web.client.HttpServerErrorException; -import org.xml.sax.SAXException; import reactor.util.retry.Retry; import reactor.util.retry.RetryBackoffSpec; @@ -202,8 +200,7 @@ public void setUp() throws SchedulerException { anyString()); doReturn(null) .when(mockTextUnitBatchImporterService) - .importTextUnits(any(), eq(false), eq(true)); - + .importTextUnits(any(), eq(fromLegacy(false, true))); resultProcessor = new StubSmartlingResultProcessor(); tmsSmartling = new ThirdPartyTMSSmartling( @@ -1805,14 +1802,7 @@ private List readTextUnits(SmartlingFile file, String pluralSeparat AndroidStringDocumentMapper mapper = new AndroidStringDocumentMapper(pluralSeparator, null, null, null); List result; - - try { - result = mapper.mapToTextUnits(fromText(file.getFileContent())); - } catch (ParserConfigurationException | IOException | SAXException e) { - result = new ArrayList<>(); - e.printStackTrace(); - } - return result; + return mapper.mapToTextUnits(fromText(file.getFileContent())); } private void prepareAssetAndTextUnits(AssetExtraction assetExtraction, Asset asset, TM tm) { diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java index 636f45b2bb..f32b77e2e2 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.service.thirdparty; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; import static org.mockito.ArgumentMatchers.*; @@ -147,7 +148,9 @@ public void testJsonWithICUMessagFormats() throws Exception { return textUnitDTO; }) .collect(ImmutableList.toImmutableList()); - textUnitBatchImporterService.importTextUnits(localizedTextUnitDTOs, false, false); + + textUnitBatchImporterService.importTextUnits( + localizedTextUnitDTOs, fromLegacy(false, false)); }); SmartlingOptions smartlingOptions = SmartlingOptions.parseList(ImmutableList.of()); @@ -386,7 +389,7 @@ public void testPullWithUntranslatedUnits() throws Exception { ArgumentCaptor> dtoListCaptor = ArgumentCaptor.forClass(ImmutableList.class); Mockito.verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(dtoListCaptor.capture(), anyBoolean(), anyBoolean()); + .importTextUnits(dtoListCaptor.capture(), any()); ImmutableList translatedUnits = dtoListCaptor.getValue(); // Expecting two fully translated units @@ -409,7 +412,7 @@ public void testPullWithUntranslatedUnits() throws Exception { dtoListCaptor = ArgumentCaptor.forClass(ImmutableList.class); Mockito.verify(textUnitBatchImporterServiceMock, times(2)) - .importTextUnits(dtoListCaptor.capture(), anyBoolean(), anyBoolean()); + .importTextUnits(dtoListCaptor.capture(), any()); translatedUnits = dtoListCaptor.getValue(); // Expecting two fully translated units diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java new file mode 100644 index 0000000000..5f4ae75a73 --- /dev/null +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -0,0 +1,282 @@ +package com.box.l10n.mojito.service.thirdparty.phrase; + +import static com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter.EscapeType.NONE; + +import com.box.l10n.mojito.JSR310Migration; +import com.box.l10n.mojito.android.strings.AndroidSingular; +import com.box.l10n.mojito.android.strings.AndroidStringDocument; +import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.phrase.client.model.Tag; +import com.phrase.client.model.Upload; +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Objects; +import org.junit.Assume; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest( + classes = { + PhraseClientTest.class, + PhraseClientConfig.class, + PhraseClientPropertiesConfig.class + }) +@EnableConfigurationProperties +@Ignore +public class PhraseClientTest { + + static Logger logger = LoggerFactory.getLogger(PhraseClientTest.class); + + @Autowired(required = false) + PhraseClient phraseClient; + + @Value("${test.phrase-client.projectId:}") + String testProjectId; + + @Autowired private PhraseClientConfig phraseClientConfig; + + @Test + public void testRemoveTag() { + String tagForUpload = "push_2024_06_10_07_17_00_089_122"; + List tagsToDelete = + phraseClient.listTags(testProjectId).stream() + .peek(tag -> logger.info("tag: {}", tag)) + .map(Tag::getName) + .filter(Objects::nonNull) + .filter(tagName -> !tagName.equals(tagForUpload)) + .toList(); + + phraseClient.deleteTags(testProjectId, tagsToDelete); + } + + @Test + public void testParallelDownload() { + + Assume.assumeNotNull(testProjectId); + // measure time of following call + Stopwatch total = Stopwatch.createStarted(); + + List locales = + List.of( + "bg", "bn", "bs", "ca", "cs", "da", "de", "el", "es-419", "es", "et", "fi", "fr-CA", + "fr", "gu", "hi", "hr", "hu", "hy", "id", "is", "it", "ja", "ka", "kn", "ko", "lv", + "mk", "mr", "ms", "my", "nb", "nl", "pl", "pt", "pt-PT", "ro", "ru", "sk", "sl", "so", + "sq", "sr", "sv", "sw", "ta", "te", "th", "tr", "uk", "vi", "zh", "zh-HK", "zh-Hant"); + + locales.parallelStream() + .forEach( + l -> { + Stopwatch started = Stopwatch.createStarted(); + String s = + phraseClient.localeDownload( + "9b6f4e167397549fe7ec1fe82497159b", + l, + "xml", + "push_chatgpt-web_2024_06_27_05_14_58_356_208", + null); + // logger.info(s); + logger.info("Download: {} in {}", l, started.elapsed()); + }); + + logger.info("total time: {}", total.elapsed()); + } + + @Test + public void test() throws IOException, InterruptedException { + Assume.assumeNotNull(testProjectId); + + for (int i = 1; i < 2; i++) { + AndroidStringDocument source = new AndroidStringDocument(); + source.addSingular( + new AndroidSingular(11L, i + "-string1", "some link to a page", "test comment1")); + source.addSingular( + new AndroidSingular( + 12L, + i + "-string2", + "some link to a page", + "test comment2")); + source.addSingular( + new AndroidSingular( + 13L, + i + "-string3", + "If that is your IP address click here to unblock it.", + "test comment2")); + + source.addSingular( + new AndroidSingular( + 14L, + i + "-string4", + "If that is your IP address click here to unblock it.", + "test comment2")); + + source.addSingular( + new AndroidSingular( + 15L, + i + "-string5", + "If that is your IP address click here to unblock it.", + "test comment2")); + + source.addSingular( + new AndroidSingular( + 16L, + i + "-string6", + "visit your settings", + "test comment2")); + + source.addSingular( + new AndroidSingular(17L, i + "-string7", "a string\nwith return line", "test comment2")); + + source.addSingular( + new AndroidSingular( + 18L, i + "-string9", "a string\n\nwith two return line", "test comment2")); + + source.addSingular( + new AndroidSingular( + 19L, i + "-string10", "a string & and the & escape", "test comment2")); + + source.addSingular( + new AndroidSingular(20L, i + "-string11", "simple tag", "test comment2")); + + String androidFile = new AndroidStringDocumentWriter(source, NONE).toText(); + System.out.println(androidFile); + + Upload upload = + phraseClient.nativeUploadAndWait( + testProjectId, + "en", + "xml", + "strings.xml", + androidFile, + ImmutableList.of(i % 2 == 0 ? "test-escaping" : "test-no-escaping"), + ImmutableMap.of("unescape_tags", "true")); + + // Upload upload = + // phraseClient.uploadAndWait( + // testProjectId, + // "en", + // "xml", + // "strings.xml", + // androidFile, + // ImmutableList.of(i % 2 == 0 ? "test-escaping" : "test-no-escaping"), + // null); + + System.out.println(upload); + + String s = + phraseClient.localeDownload( + testProjectId, + "en", + "xml", + i % 2 == 0 ? "test-escaping" : "test-no-escaping", + () -> null); + System.out.println(s); + + String ns = + phraseClient.nativeLocaleDownload( + testProjectId, + "en", + "xml", + i % 2 == 0 ? "test-escaping" : "test-no-escaping", + ImmutableMap.of("escape_tags", "true"), + () -> null); + + System.out.println(ns); + + // Assert.assertEquals(androidFile, ns); + } + + // for (int i = 0; i < 3; i++) { + // String repoName = "repo_%d".formatted(i); + // String tagForUpload = ThirdPartyTMSPhrase.getTagForUpload(repoName); + // + // logger.info("tagForUpload: {}", tagForUpload); + // + // String fileContentAndroid = generateFileContent(repoName).toString(); + // phraseClient.uploadAndWait( + // testProjectId, + // "en", + // "xml", + // "strings.xml", + // fileContentAndroid, + // ImmutableList.of(tagForUpload)); + // + // new ThirdPartyTMSPhrase(phraseClient) + // .removeUnusedKeysAndTags(testProjectId, repoName, tagForUpload); + // } + // + // List translationKeys = phraseClient.getKeys(testProjectId); + // for (TranslationKey translationKey : translationKeys) { + // logger.info("{}", translationKey); + // } + + // + // String fileContentAndroid2 = + // """ + // + // + // Locale Tester - fr + // Settings - fr + // Hello + // + // One thing - fr + // Multiple things - fr + // + // + // """; + // phraseClient.uploadCreateFile( + // testProjectId, "fr", "xml", "strings.xml", fileContentAndroid2, null); + + // String s2 = + // phraseClient.localeDownload( + // testProjectId, + // "en", + // "xml", + // "startWithABadTag", + // () -> + // phraseClient.listTags(testProjectId).stream() + // .map(Tag::getName) + // .filter(Objects::nonNull) + // .filter(tagName -> tagName.startsWith("push_repo_2")) + // .collect(Collectors.joining(","))); + // + // logger.info(s2); + } + + static StringBuilder generateFileContent(String repositoryName) { + StringBuilder fileContentAndroidBuilder = new StringBuilder(); + + fileContentAndroidBuilder.append( + """ + + + Locale Tester + """ + .formatted(repositoryName)); + + ZonedDateTime now = JSR310Migration.dateTimeNowInUTC(); + for (int i = 0; i < 1; i++) { + fileContentAndroidBuilder.append( + String.format("Settings\n", i)); + fileContentAndroidBuilder.append( + String.format( + "Settings\n", + repositoryName, i, now.toString())); + } + + fileContentAndroidBuilder.append(""); + return fileContentAndroidBuilder; + } +} diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJobTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJobTest.java index 95b99bfafd..99b7524108 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJobTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/smartling/quartz/SmartlingPullLocaleFileJobTest.java @@ -1,6 +1,7 @@ package com.box.l10n.mojito.service.thirdparty.smartling.quartz; import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.groups.Tuple.tuple; import static org.mockito.ArgumentMatchers.any; @@ -85,7 +86,7 @@ public void setup() { anyString()); doReturn(null) .when(textUnitBatchImporterServiceMock) - .importTextUnits(any(), eq(false), eq(true)); + .importTextUnits(any(), eq(fromLegacy(false, true))); RetryBackoffSpec retryConfiguration = Retry.backoff(10, Duration.ofMillis(1)).maxBackoff(Duration.ofMillis(10)); @@ -132,7 +133,7 @@ public void testPullSingular() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); List captured = textUnitListCaptor.getValue(); @@ -186,7 +187,7 @@ public void testPullPlural() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); List captured = textUnitListCaptor.getValue(); @@ -251,7 +252,7 @@ public void testPullPluralsFix() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); List captured = textUnitListCaptor.getValue(); @@ -319,7 +320,7 @@ public void testPullDryRun() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(0)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); } @Test(expected = SmartlingClientException.class) @@ -370,7 +371,7 @@ public void testPullShortCircuitIfNoChanges() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(0)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); verify(thirdPartyFileChecksumRepositoryMock, times(0)).save(any(ThirdPartyFileChecksum.class)); } @@ -412,7 +413,7 @@ public void testPullShortCircuitIfChanges() throws Exception { smartlingPullLocaleFileJob.call(smartlingPullLocaleFileJobInput); verify(textUnitBatchImporterServiceMock, times(1)) - .importTextUnits(textUnitListCaptor.capture(), eq(false), eq(true)); + .importTextUnits(textUnitListCaptor.capture(), eq(fromLegacy(false, true))); verify(thirdPartyFileChecksumRepositoryMock, times(1)).save(any(ThirdPartyFileChecksum.class)); } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java index 15c6f32a67..5ee946df5c 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/TMServiceTest.java @@ -1702,6 +1702,52 @@ public void testLocalizeAndroidCommentWithTranslatableFalse() throws Exception { assertEquals(assetContent, localizedAsset); } + @Test + public void testLocalizeAndroidTranslatableFalse() throws Exception { + + Repository repo = repositoryService.createRepository(testIdWatcher.getEntityName("repository")); + RepositoryLocale repoLocale = repositoryService.addRepositoryLocale(repo, "en-GB"); + + String assetContent = + """ + + + do_not_translate + + + plural_do_not_translate_one + plural_do_not_translate_other + + """; + + asset = + assetService.createAssetWithContent(repo.getId(), "res/values/strings.xml", assetContent); + asset = assetRepository.findById(asset.getId()).orElse(null); + assetId = asset.getId(); + tmId = repo.getTm().getId(); + + PollableFuture assetResult = + assetService.addOrUpdateAssetAndProcessIfNeeded( + repo.getId(), asset.getPath(), assetContent, false, null, null, null, null, null, null); + try { + pollableTaskService.waitForPollableTask(assetResult.getPollableTask().getId()); + } catch (PollableTaskException | InterruptedException e) { + throw new RuntimeException(e); + } + assetResult.get(); + + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(repo.getId()); + textUnitSearcherParameters.setStatusFilter(StatusFilter.FOR_TRANSLATION); + List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); + for (TextUnitDTO textUnitDTO : textUnitDTOs) { + logger.info("name=[{}]", textUnitDTO.getName()); + } + assertEquals(2, textUnitDTOs.size()); + // TODO translatable on plural strings don't work + + } + @Test public void testLocalizeAndroidUnicodeEscape() throws Exception { @@ -2034,7 +2080,7 @@ public void testLocalizeAndroidStringsRemoveUntranslatedOldEsaping() throws Exce assetResult.get(); String forImport = - "\n" + " Le test\n" + ""; + "\n" + " Le test\n" + "\n"; tmService .importLocalizedAssetAsync( @@ -2053,7 +2099,7 @@ public void testLocalizeAndroidStringsRemoveUntranslatedOldEsaping() throws Exce repoLocale, "en-GB", null, - null, + List.of("postProcessIndent=4"), Status.ALL, InheritanceMode.REMOVE_UNTRANSLATED, null); @@ -2099,8 +2145,7 @@ public void testLocalizeAndroidStringsRemoveUntranslatedSingleItem() throws Exce String expectedLocalized = "\n" + "" - + "\n" - + ""; + + "\n"; String localizedAsset = tmService.generateLocalized( @@ -3601,6 +3646,93 @@ public void testLocalizePoPluralAr() throws Exception { assertEquals(forImport, localizedAsset); } + @Test + public void testLocalizeMacStringsRemoveUntranslated() throws Exception { + + Repository repo = repositoryService.createRepository(testIdWatcher.getEntityName("repository")); + RepositoryLocale repoLocale; + try { + repoLocale = repositoryService.addRepositoryLocale(repo, "ja-JP"); + } catch (RepositoryLocaleCreationException e) { + throw new RuntimeException(e); + } + + String assetContent = + """ + /* comment 1 */ + "key1" = "value1"; + + /* comment 2 */ + "key2" = "value2"; + """; + + String expectedLocalizedAsset = "\n"; + + asset = assetService.createAssetWithContent(repo.getId(), "Localizable.strings", assetContent); + asset = assetRepository.findById(asset.getId()).orElse(null); + assetId = asset.getId(); + tmId = repo.getTm().getId(); + + PollableFuture assetResult = + assetService.addOrUpdateAssetAndProcessIfNeeded( + repo.getId(), asset.getPath(), assetContent, false, null, null, null, null, null, null); + try { + pollableTaskService.waitForPollableTask(assetResult.getPollableTask().getId()); + } catch (PollableTaskException | InterruptedException e) { + throw new RuntimeException(e); + } + assetResult.get(); + + String localizedAsset = + tmService.generateLocalized( + asset, + assetContent, + repoLocale, + "ja-JP", + null, + null, + Status.ALL, + InheritanceMode.REMOVE_UNTRANSLATED, + null); + logger.debug("localized=\n{}\nEOL", localizedAsset); + assertEquals(expectedLocalizedAsset, localizedAsset); + + String forImport = + """ + /* comment 1 */ + "key1" = "value1-jp"; + + /* comment 2 */ + "key2" = "value2-jp"; + """; + + logger.debug("formimport=\n{}", forImport); + + tmService + .importLocalizedAssetAsync( + assetId, + forImport, + repoLocale.getLocale().getId(), + StatusForEqualTarget.TRANSLATION_NEEDED, + null, + null) + .get(); + + localizedAsset = + tmService.generateLocalized( + asset, + assetContent, + repoLocale, + "ja-JP", + null, + null, + Status.ALL, + InheritanceMode.REMOVE_UNTRANSLATED, + null); + logger.info("localized after import=\n{}", localizedAsset); + assertEquals(forImport, localizedAsset); + } + @Test public void testLocalizeMacStringsNamessNotEnclosedInDoubleQuotes() throws Exception { @@ -3902,6 +4034,86 @@ public void testLocalizeJson() throws Exception { logger.debug("name=[{}], source=[{}]", textUnitDTO.getName(), textUnitDTO.getSource()); } + assertEquals("hello_world/string", textUnitDTOs.get(0).getName()); + + String localizedAsset = + tmService.generateLocalized( + asset, + assetContent, + repoLocale, + "en-GB", + null, + jsonFilterOptions, + Status.ALL, + InheritanceMode.USE_PARENT, + null); + logger.debug("localized=\n{}", localizedAsset); + assertEquals(expectedContent, localizedAsset); + } + + @Test + public void testLocalizeJsonRemoveKeySuffix() throws Exception { + Repository repo = repositoryService.createRepository(testIdWatcher.getEntityName("repository")); + RepositoryLocale repoLocale = repositoryService.addRepositoryLocale(repo, "en-GB"); + + List jsonFilterOptions = + Arrays.asList( + "useFullKeyPath=true", + "extractAllPairs=false", + "exceptions=.*/string", + "removeKeySuffix=/string"); + + String assetContent = + "{\n" + + " \"this to ignore\": {\n" + + " \"k1\": \"v1\"\n" + + " },\n" + + " \"hello_world\": {\n" + + " \"string\": \"Hello World\",\n" + + " \"note\": \"The start of every language book.\"\n" + + " },\n" + + " \"num_photos\": {\n" + + " \"string\": \"You have {numPhotos, plural, =0 {no photos.} =1 {one photo.} other {# photos.}}\",\n" + + " \"note\": \"A description that shows the number of photos a user has.\"\n" + + " }\n" + + "}"; + String expectedContent = assetContent; + + asset = assetService.createAssetWithContent(repo.getId(), "strings.json", assetContent); + asset = assetRepository.findById(asset.getId()).orElse(null); + assetId = asset.getId(); + tmId = repo.getTm().getId(); + + PollableFuture assetResult = + assetService.addOrUpdateAssetAndProcessIfNeeded( + repo.getId(), + asset.getPath(), + assetContent, + false, + null, + null, + null, + null, + null, + jsonFilterOptions); + try { + pollableTaskService.waitForPollableTask(assetResult.getPollableTask().getId()); + } catch (PollableTaskException | InterruptedException e) { + throw new RuntimeException(e); + } + assetResult.get(); + + TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); + textUnitSearcherParameters.setRepositoryIds(repo.getId()); + textUnitSearcherParameters.setStatusFilter(StatusFilter.FOR_TRANSLATION); + List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); + assertEquals(2, textUnitDTOs.size()); + for (TextUnitDTO textUnitDTO : textUnitDTOs) { + logger.debug("name=[{}], source=[{}]", textUnitDTO.getName(), textUnitDTO.getSource()); + } + + assertEquals("hello_world", textUnitDTOs.get(0).getName()); + String localizedAsset = tmService.generateLocalized( asset, diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java index 970fe254fe..3566f5b61d 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/tm/importer/TextUnitBatchImporterServiceTest.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.service.tm.importer; +import static com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService.IntegrityChecksType.fromLegacy; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -82,6 +83,7 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException { textUnitDTO.setAssetPath(tmTestData.asset.getPath()); textUnitDTO.setName("TEST2"); textUnitDTO.setTarget("TEST2 translation for fr"); + textUnitDTO.setComment("Comment2"); TextUnitDTO textUnitDTO2 = new TextUnitDTO(); textUnitDTO2.setRepositoryName(tmTestData.repository.getName()); @@ -89,6 +91,7 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException { textUnitDTO2.setAssetPath(tmTestData.asset.getPath()); textUnitDTO2.setName("TEST3"); textUnitDTO2.setTarget("TEST3 translation for fr"); + textUnitDTO2.setTargetComment("TEST3 target comment"); TextUnitDTO textUnitDTO3 = new TextUnitDTO(); textUnitDTO3.setRepositoryName(tmTestData.repository.getName()); @@ -101,7 +104,8 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException { Arrays.asList(textUnitDTO, textUnitDTO2, textUnitDTO3); PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false); + textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOsForImport, fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); @@ -123,9 +127,11 @@ public void testAsyncImportTextUnitsNameOnly() throws InterruptedException { i++; assertEquals("TEST2", textUnitDTOsFromSearch.get(i).getName()); assertEquals("TEST2 translation for fr", textUnitDTOsFromSearch.get(i).getTarget()); + assertNull(textUnitDTOsFromSearch.get(i).getTargetComment()); i++; assertEquals("TEST3", textUnitDTOsFromSearch.get(i).getName()); assertEquals("TEST3 translation for fr", textUnitDTOsFromSearch.get(i).getTarget()); + assertEquals("TEST3 target comment", textUnitDTOsFromSearch.get(i).getTargetComment()); i++; } @@ -146,7 +152,8 @@ public void testAsyncImportTextUnitsFromSearch() throws InterruptedException { } PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false); + textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOsForImport, fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -196,7 +203,8 @@ public void testAsyncImportTextUnitsDuplicatedNames() throws InterruptedExceptio } PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false); + textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOsForImport, fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -250,7 +258,8 @@ public void testAsyncImportTextUnitsDuplicatedEntries() throws InterruptedExcept textUnitDTOsForImport.add(duplicatedEntry); PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false); + textUnitBatchImporterService.asyncImportTextUnits( + textUnitDTOsForImport, fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -322,7 +331,9 @@ public void testImportMulipleRepositoryAssetAndLocale() throws Exception { + textUnitDTO.getName()); } - textUnitBatchImporterService.asyncImportTextUnits(textUnitDTOsForImport, false, false).get(); + textUnitBatchImporterService + .asyncImportTextUnits(textUnitDTOsForImport, fromLegacy(false, false)) + .get(); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); assertFalse(textUnitDTOs.isEmpty()); @@ -381,7 +392,7 @@ public void testUnused() textUnitDTO.setTarget("v1"); textUnitBatchImporterService - .asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false) + .asyncImportTextUnits(Arrays.asList(textUnitDTO), fromLegacy(false, false)) .get(); List textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -413,7 +424,7 @@ public void testUnused() textUnitDTO.setTarget("v2"); textUnitBatchImporterService - .asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false) + .asyncImportTextUnits(Arrays.asList(textUnitDTO), fromLegacy(false, false)) .get(); textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -428,7 +439,7 @@ public void testUnused() .addTextUnits(virtualAsset1.getId(), Arrays.asList(virtualAssetTextUnit)) .get(); textUnitBatchImporterService - .asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false) + .asyncImportTextUnits(Arrays.asList(textUnitDTO), fromLegacy(false, false)) .get(); textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); @@ -473,7 +484,8 @@ public void testIntegirtyChecker() throws Exception { textUnitDTO.setTarget("with some broken {placeholder"); PollableFuture asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false); + textUnitBatchImporterService.asyncImportTextUnits( + Arrays.asList(textUnitDTO), fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); TextUnitSearcherParameters textUnitSearcherParameters = @@ -491,7 +503,8 @@ public void testIntegirtyChecker() throws Exception { textUnitDTO.setTarget("with fixed {placeholder}"); asyncImportTextUnits = - textUnitBatchImporterService.asyncImportTextUnits(Arrays.asList(textUnitDTO), false, false); + textUnitBatchImporterService.asyncImportTextUnits( + Arrays.asList(textUnitDTO), fromLegacy(false, false)); pollableTaskService.waitForPollableTask(asyncImportTextUnits.getPollableTask().getId()); textUnitDTOs = textUnitSearcher.search(textUnitSearcherParameters); diff --git a/webapp/src/test/resources/com/box/l10n/mojito/android/strings/test_resources_file.xml b/webapp/src/test/resources/com/box/l10n/mojito/android/strings/test_resources_file.xml index 6f877f2698..1556c0f2c2 100644 --- a/webapp/src/test/resources/com/box/l10n/mojito/android/strings/test_resources_file.xml +++ b/webapp/src/test/resources/com/box/l10n/mojito/android/strings/test_resources_file.xml @@ -3,8 +3,10 @@ Dela... - Mer \\" dela - ei \\' kommentteja + Mer \\" dela + ei \\' kommentteja + Mer \" dela + ei \' kommentteja salto de \nlinea