Skip to content

Commit

Permalink
Phrase Connector support N Mojito repositories to 1 project mapping
Browse files Browse the repository at this point in the history
  • Loading branch information
@aurambaj
aurambaj committed Jun 24, 2024
1 parent 1cb59e6 commit d2c97ab
Show file tree
Hide file tree
Showing 3 changed files with 205 additions and 62 deletions.
93 changes: 82 additions & 11 deletions webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
Expand All @@ -45,6 +46,7 @@
public class ThirdPartyTMSPhrase implements ThirdPartyTMS {

static final String TAG_PREFIX = "push_";
static final String TAG_PREFIX_WITH_REPOSITORY = "push_%s";

static Logger logger = LoggerFactory.getLogger(ThirdPartyTMSPhrase.class);

Expand All @@ -56,6 +58,12 @@ public class ThirdPartyTMSPhrase implements ThirdPartyTMS {

@Autowired RepositoryService repositoryService;

public ThirdPartyTMSPhrase() {}

public ThirdPartyTMSPhrase(PhraseClient phraseClient) {
this.phraseClient = phraseClient;
}

@Override
public void removeImage(String projectId, String imageId) {
throw new UnsupportedOperationException("Remove image is not supported");
Expand Down Expand Up @@ -129,7 +137,7 @@ public void push(

String text = getFileContent(pluralSeparator, search, true, null);

String tagForUpload = getTagForUpload();
String tagForUpload = getTagForUpload(repository.getName());
phraseClient.uploadAndWait(
projectId,
repository.getSourceLocale().getBcp47Tag(),
Expand All @@ -138,17 +146,62 @@ public void push(
text,
ImmutableList.of(tagForUpload));

phraseClient.removeKeysNotTaggedWith(projectId, tagForUpload);
removeUnusedKeysAndTags(projectId, repository.getName(), tagForUpload);
}

List<Tag> tagsToDelete =
/**
* Remove unused keys and tags
*
* <ul>
* <li><b>Remove Unused Keys:</b>
* <ul>
* <li>Unused keys in a Phrase project are defined as keys that are not tagged with the
* latest push tag from any Mojito repository.
* <li>To identify unused keys, fetch the current push tags of all repositories, excluding
* the push tag related to the Mojito repository being processed, and include the
* current upload tag.
* </ul>
* <li><b>Manage Tags:</b>
* <ul>
* <li>Ensure that there is only one active "push" tag per repository.
* <li>Remove old tags that are prefixed with "push" but not active
* </ul>
* </ul>
*
* <p><b>Explanation:</b>
*
* <p>The N:1 relationship between Mojito and Phrase is maintained through this logic, allowing
* for efficient management of keys and tags.
*/
public void removeUnusedKeysAndTags(
String projectId, String repositoryName, String tagForUpload) {

List<String> tagsForOtherRepositories =
phraseClient.listTags(projectId).stream()
.map(Tag::getName)
.filter(Objects::nonNull)
.filter(tagName -> tagName.startsWith(TAG_PREFIX))
.filter(
tag ->
tag.getName() != null
&& !tag.getName().equals(tagForUpload)
&& tag.getName().startsWith(TAG_PREFIX))
tagName ->
!tagName.startsWith(TAG_PREFIX_WITH_REPOSITORY.formatted(repositoryName)))
.toList();

List<String> allActiveTags = new ArrayList<>(tagsForOtherRepositories);
allActiveTags.add(tagForUpload);

logger.info("All active tags: {}", allActiveTags);
phraseClient.removeKeysNotTaggedWith(projectId, allActiveTags);

List<String> pushTagsToDelete =
phraseClient.listTags(projectId).stream()
.map(Tag::getName)
.filter(Objects::nonNull)
.filter(tagName -> tagName.startsWith(TAG_PREFIX))
.filter(tagName -> !allActiveTags.contains(tagName))
.toList();
phraseClient.deleteTags(projectId, tagsToDelete);

logger.info("Tags to delete: {}", pushTagsToDelete);
phraseClient.deleteTags(projectId, pushTagsToDelete);
}

private List<TextUnitDTO> getSourceTextUnitDTOs(
Expand Down Expand Up @@ -185,12 +238,13 @@ private List<TextUnitDTO> getSourceTextUnitDTOsPluralOnly(
return textUnitSearcher.search(parameters);
}

public static String getTagForUpload() {
public static String getTagForUpload(String repositoryName) {
ZonedDateTime zonedDateTime = JSR310Migration.dateTimeNowInUTC();
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy_MM_dd_HH_mm_ss_SSS");
return ("%s%s_%s")
return ("%s%s_%s_%s")
.formatted(
TAG_PREFIX,
repositoryName,
formatter.format(zonedDateTime),
Math.abs(UUID.randomUUID().getLeastSignificantBits() % 1000));
}
Expand All @@ -210,10 +264,19 @@ public PollableFuture<Void> pull(
Set<RepositoryLocale> repositoryLocalesWithoutRootLocale =
repositoryService.getRepositoryLocalesWithoutRootLocale(repository);

String currentTags = getCurrentTagsForRepository(repository, projectId);

for (RepositoryLocale repositoryLocale : repositoryLocalesWithoutRootLocale) {
String localeTag = repositoryLocale.getLocale().getBcp47Tag();
logger.info("Downloading locale: {} from Phrase", localeTag);
String fileContent = phraseClient.localeDownload(projectId, localeTag, "xml");

String fileContent =
phraseClient.localeDownload(
projectId,
localeTag,
"xml",
currentTags,
() -> getCurrentTagsForRepository(repository, projectId));

logger.info("file content from pull: {}", fileContent);

Expand All @@ -230,6 +293,14 @@ public PollableFuture<Void> pull(
return null;
}

private String getCurrentTagsForRepository(Repository repository, String projectId) {
return phraseClient.listTags(projectId).stream()
.map(Tag::getName)
.filter(Objects::nonNull)
.filter(tagName -> tagName.startsWith(repository.getName()))
.collect(Collectors.joining(","));
}

@Override
public void pushTranslations(
Repository repository,
Expand Down
86 changes: 67 additions & 19 deletions webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.publisher.Mono;
Expand Down Expand Up @@ -168,13 +170,35 @@ Upload uploadsApiUploadCreateWithRetry(
.block();
}

public void removeKeysNotTaggedWith(String projectId, String tag) {
logger.info("Removing keys not tagged with: {}", tag);
/**
* Conducted tests on keysDeleteCollection using the <code>-tags:tag1,tag2</code> option, and it
* operates as a filter for keys "not in any of the tags".
*
* <p>It removes keys that are not tagged with any of the provided tags.
*
* <p>For example:
*
* <ul>
* <li>If key <code>ka</code> has tag <code>tag1</code>,
* <li>If key <code>kb</code> has tag <code>tag2</code>,
* <li>If key <code>kc</code> has tag <code>tag3</code>,
* </ul>
*
* <p>Calling <code>keysDeleteCollection</code> with <code>-tags:tag1,tag3</code> will remove
* <code>kb</code>.
*/
public void removeKeysNotTaggedWith(String projectId, List<String> anyOfTheseTags) {
logger.info("Removing keys not tagged with any of the following tags: {}", anyOfTheseTags);

Mono.fromCallable(
() -> {
KeysApi keysApi = new KeysApi(apiClient);
keysApi.keysDeleteCollection(projectId, null, null, "-tags:%s".formatted(tag), null);
keysApi.keysDeleteCollection(
projectId,
null,
null,
"-tags:%s".formatted(String.join(",", anyOfTheseTags)),
null);
return null;
})
.retryWhen(
Expand All @@ -193,7 +217,18 @@ public void removeKeysNotTaggedWith(String projectId, String tag) {
.block();
}

public String localeDownload(String projectId, String locale, String fileFormat) {
/**
* @param onTagErrorRefreshCallback with concurrent update, the tags could be updated during
* download. We don't want to retry the whole logic, so we provide a callback to refresh the
* tags
*/
public String localeDownload(
String projectId,
String locale,
String fileFormat,
String tags,
Supplier<String> onTagErrorRefreshCallback) {
AtomicReference<String> refTags = new AtomicReference<>(tags);
return Mono.fromCallable(
() -> {
LocalesApi localesApi = new LocalesApi(apiClient);
Expand All @@ -211,7 +246,7 @@ public String localeDownload(String projectId, String locale, String fileFormat)
null,
null,
fileFormat,
null,
refTags.get(),
null,
null,
null,
Expand All @@ -234,11 +269,22 @@ public String localeDownload(String projectId, String locale, String fileFormat)
})
.retryWhen(
retryBackoffSpec.doBeforeRetry(
doBeforeRetry ->
logAttempt(
doBeforeRetry.failure(),
"Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s"
.formatted(projectId, locale))))
doBeforeRetry -> {
logAttempt(
doBeforeRetry.failure(),
"Retrying failed attempt to localeDownload from Phrase, project id: %s, locale: %s"
.formatted(projectId, locale));

if (getErrorMessageFromOptionalApiException(doBeforeRetry.failure())
.contains("Invalid Download Options. Parameter tags ")) {
String newTags = onTagErrorRefreshCallback.get();
logger.warn(
"Replacing old tags: {} with new tags: {} for download locale",
refTags.get(),
newTags);
refTags.set(newTags);
}
}))
.doOnError(
throwable ->
rethrowExceptionWithLog(
Expand Down Expand Up @@ -325,15 +371,17 @@ public List<Tag> listTags(String projectId) {
return tags;
}

public void deleteTags(String projectId, List<Tag> tags) {
public void deleteTags(String projectId, List<String> tagNames) {

logger.debug("Delete tags: {}", tagNames);

TagsApi tagsApi = new TagsApi(apiClient);
Map<String, Throwable> exceptions = new LinkedHashMap<>();
for (Tag tag : tags) {
for (String tagName : tagNames) {
Mono.fromCallable(
() -> {
logger.debug(
"Deleting tag: %s in project id: %s".formatted(tag.getName(), projectId));
tagsApi.tagDelete(projectId, tag.getName(), null, null);
logger.debug("Deleting tag: %s in project id: %s".formatted(tagName, projectId));
tagsApi.tagDelete(projectId, tagName, null, null);
return null;
})
.retryWhen(
Expand All @@ -342,15 +390,15 @@ public void deleteTags(String projectId, List<Tag> tags) {
logAttempt(
doBeforeRetry.failure(),
"Retrying failed attempt to delete tag: %s in project id: %s"
.formatted(tag.getName(), projectId));
.formatted(tagName, projectId));
}))
.doOnError(
throwable -> {
exceptions.put(tag.getName(), throwable);
exceptions.put(tagName, throwable);
rethrowExceptionWithLog(
throwable,
"Final error to delete tag: %s in project id: %s"
.formatted(tag.getName(), projectId));
.formatted(tagName, projectId));
})
.block();
}
Expand All @@ -359,7 +407,7 @@ public void deleteTags(String projectId, List<Tag> tags) {
List<String> tagsWithErrors = exceptions.keySet().stream().limit(10).toList();
String andMore = (tagsWithErrors.size() < exceptions.size()) ? " and more." : "";
throw new PhraseClientException(
String.format("Can't delete tags: %s%s", tagsWithErrors, andMore));
String.format("Can't delete tagNames: %s%s", tagsWithErrors, andMore));
}
}

Expand Down
Loading

0 comments on commit d2c97ab

Please sign in to comment.