diff --git a/.github/workflows/sync-translations.yml b/.github/workflows/sync-translations.yml deleted file mode 100644 index 93bd524c1b3..00000000000 --- a/.github/workflows/sync-translations.yml +++ /dev/null @@ -1,189 +0,0 @@ -name: Migrate translations from the old Transifex project - -on: - workflow_dispatch: - - -jobs: - migrate-translations: - runs-on: ubuntu-latest - strategy: - fail-fast: false - max-parallel: 1 - matrix: - resource: - - new_slug: ecommerce - old_slug: ecommerce - old_project_slug: edx-platform - - - new_slug: ecommerce-js - old_slug: ecommerce-js - old_project_slug: edx-platform - - - new_slug: edx-ora2 - old_slug: openassessment - old_project_slug: edx-platform - - - new_slug: edx-ora2-js - old_slug: openassessment-js - old_project_slug: edx-platform - - - new_slug: edx-proctoring - old_slug: edx-proctoring - old_project_slug: edx-platform - - - new_slug: studio-frontend - old_slug: studio-frontend - old_project_slug: edx-platform - - - new_slug: donexblock - old_slug: xblock-done - old_project_slug: xblocks - - - new_slug: xblock-drag-and-drop-v2 - old_slug: drag-and-drop-v2 - old_project_slug: xblocks - - - new_slug: xblock-free-text-response - old_slug: xblock-free-text-response - old_project_slug: xblocks - - - new_slug: course-discovery - old_slug: course_discovery - old_project_slug: edx-platform - - - new_slug: course-discovery-js - old_slug: course_discovery-js - old_project_slug: edx-platform - - - new_slug: credentials-js - old_slug: credentials-js - old_project_slug: edx-platform - - - new_slug: credentials - old_slug: credentials - old_project_slug: edx-platform - - - new_slug: frontend-app-account - old_slug: frontend-app-account - old_project_slug: edx-platform - - - new_slug: frontend-app-authn - old_slug: frontend-app-authn - old_project_slug: edx-platform - - - new_slug: frontend-app-course-authoring - old_slug: frontend-app-course-authoring - old_project_slug: edx-platform - - - new_slug: frontend-app-discussions - old_slug: frontend-app-discussions - old_project_slug: edx-platform - - - new_slug: frontend-app-ecommerce - old_slug: frontend-app-ecommerce - old_project_slug: edx-platform - - - new_slug: frontend-app-gradebook - old_slug: frontend-app-gradebook - old_project_slug: edx-platform - - - new_slug: frontend-app-learner-dashboard - old_slug: frontend-app-learner-dashboard - old_project_slug: edx-platform - - - new_slug: frontend-app-learner-record - old_slug: frontend-app-learner-record - old_project_slug: edx-platform - - - new_slug: frontend-app-learning - old_slug: frontend-app-learning - old_project_slug: edx-platform - - - new_slug: frontend-app-profile - old_slug: frontend-app-profile - old_project_slug: edx-platform - - - new_slug: frontend-app-program-console - old_slug: frontend-app-program-manager - old_project_slug: edx-platform - - - new_slug: frontend-component-footer - old_slug: frontend-component-footer-edx - old_project_slug: edx-platform - - - new_slug: frontend-component-header - old_slug: frontend-component-header - old_project_slug: edx-platform - - - new_slug: paragon - old_slug: paragon - old_project_slug: edx-platform - - # Start: edx-platform repo resources - # The edx-platform repo resources has been consolidated into a two resources - # - https://github.com/openedx/edx-platform/blob/master/docs/decisions/0018-standarize-django-po-files.rst - - - new_slug: edx-platform - old_slug: django-partial - old_project_slug: edx-platform - - - new_slug: edx-platform - old_slug: django-studio - old_project_slug: edx-platform - - - new_slug: edx-platform - old_slug: edx_proctoring_proctortrack - old_project_slug: edx-platform - - - new_slug: edx-platform - old_slug: mako - old_project_slug: edx-platform - - - new_slug: edx-platform - old_slug: mako-studio - old_project_slug: edx-platform - - - new_slug: edx-platform - old_slug: wiki - old_project_slug: edx-platform - - - new_slug: edx-platform-js - old_slug: underscore - old_project_slug: edx-platform - - - new_slug: edx-platform-js - old_slug: djangojs-studio - old_project_slug: edx-platform - - - new_slug: edx-platform-js - old_slug: underscore-studio - old_project_slug: edx-platform - - - new_slug: edx-platform-js - old_slug: djangojs-account-settings-view - old_project_slug: edx-platform - - - new_slug: edx-platform-js - old_slug: djangojs-partial - old_project_slug: edx-platform - - # End: edx-platform repo resources - - steps: - - uses: actions/checkout@v3 - - name: setup python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Install Python dependencies - run: make translations_scripts_requirements - - - name: Sync - env: - # `TX_LANGUAGES` list of languages is set in the `Makefile` - TX_NEW_SLUG: ${{ matrix.resource.new_slug }} - TX_OLD_SLUG: ${{ matrix.resource.old_slug }} - TX_OLD_PROJECT_SLUG: ${{ matrix.resource.old_project_slug }} - TX_API_TOKEN: ${{ secrets.TRANSIFEX_API_TOKEN }} - run: make sync_translations diff --git a/Makefile b/Makefile index 45c93d64a06..825450389e9 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,9 @@ -.PHONY: piptools upgrade fix_transifex_resource_names translations_scripts_requirements validate_translation_files \ -sync_translations sync_translations_github_workflow rerun_tests_for_transifex_bot_pull_requests - - -# Default languages for the sync_translations.py file -# This list represents the supported languages by the Open edX community as stated by the Translators Working Group: -# - https://openedx.atlassian.net/wiki/spaces/COMM/pages/3157524644/Translation+Working+Group#The-following-is-a-table-of-the-latest-list-of-languages-supported--by-the-Translation-Working-Group -export TX_LANGUAGES := ar,da,de_DE,el,es_419,es_ES,fr_CA,hi,he,id,it_IT,pt_BR,pt_PT,ru,th,tr_TR,uk,zh_CN +.PHONY: piptools upgrade fix_transifex_resource_names translations_scripts_requirements \ +validate_translation_files test_requirements test fix_transifex_resource_names_dry_run \ +retry_merge_transifex_bot_pull_requests +# Default project to work on. Override to release project e.g. `openedx-translations-redwood` when cutting a release. +export TRANSIFEX_PROJECT_SLUG := openedx-translations piptools: pip install -q -r requirements/pip_tools.txt @@ -24,10 +21,10 @@ upgrade: piptools ## update the requirements/*.txt files with the latest packag translations_scripts_requirements: ## Installs the requirements file pip install -q -r requirements/translations.txt -fix_transifex_resource_names: ## Runs the script +fix_transifex_resource_names: ## Runs the script on the TRANSIFEX_PROJECT_SLUG project python scripts/fix_transifex_resource_names.py -fix_transifex_resource_names_dry_run: ## Runs the script in --dry-run mode +fix_transifex_resource_names_dry_run: ## Runs the script in --dry-run mode on the TRANSIFEX_PROJECT_SLUG project python scripts/fix_transifex_resource_names.py --dry-run @@ -40,12 +37,6 @@ test: ## Run scripts tests validate_translation_files: ## Run basic validation to ensure files are compilable python scripts/validate_translation_files.py -sync_translations: ## Syncs from the old projects to the new openedx-translations project - python scripts/sync_translations.py $(SYNC_ARGS) - -sync_translations_github_workflow: ## Run with parameters from .github/workflows/sync-translations.yml - make SYNC_ARGS="--simulate-github-workflow $(SYNC_ARGS)" sync_translations - export MAX_PULL_REQUESTS_TO_RESTART := 1000 retry_merge_transifex_bot_pull_requests: ## Fix Transifex bot stuck and unmerged pull requests. bash scripts/retry_merge_transifex_bot_pull_requests.sh diff --git a/README.rst b/README.rst index bd3cb901193..2e2a2fb210d 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ openedx-translations #################### -The openedx-translations repository contains translation files from Open edX repositories +This openedx-translations repository contains translation files from Open edX repositories to be kept in sync with Transifex. To accomplish this task, a GitHub Action in ``.github/workflows/`` named ``extract-translation-source-files.yml`` regularly extracts English translation source files form Open edX repositories containing code and adds them @@ -11,20 +11,58 @@ into this repository. The translation files in this repository can then be acces using the `openedx-atlas`_ CLI tool to download specific directories of translation files from openedx-translations. -Current State -************* +This repository implements the `OEP-58`_ proposal. -This repository is currently under active development with limited use. -The GitHub Action -``extract-translation-source-files.yml`` generates English translation source -files for the configured repositories. These translation source files are the only -ones uploaded to the Transifex project openedx-translations. The English translation -source files have only been translated into one language: French Canadian (fr_CA). The -`openedx-atlas`_ CLI tool can only be used to pull translation files from the credentials -directory in openedx-translations. +Main and Release branches +************************* + +This repository has a main branch in addition to a dedicated branch for every +release. As of May 10th, 2024 the following are the release branches: + +``main`` branch +=============== + +This branch is used for the latest version of Open edX such as +`Tutor nightly`_, `edx-platform "master" branch`_ and others. + +To translate the latest versions the `open-edx/openedx-translations`_ Transifex +project should be used. + + +``open-release/.master`` branch +============================================= + +This branch is used for the latest version of the Open edX Release, which will +be a version of Tutor and corresponding branches in tagged repos. For example, +for the Redwood release (June 2024), the branches were: +`Tutor Redwood v18`_, `edx-platform "open-release/redwood.master" branch`_ +and others. + +To update translations for a named release, find the corresponding named release project in the `Open edX Transifex project `_ by searching for the release name (for example, Redwood) in the search box. + +Tools for repository maintainers +******************************** + +This repository contains both `GitHub Actions workflows`_ and +`Makefile programs`_ to automate and assist maintainers chores including: + +Fix resource names in Transifex +=============================== + +The GitHub Transifex App integeration puts an inconvenient names for resources like ``translations..frontend-app-something..src-i18n-transifex-input--main`` +instead of ``frontend-app-something``. + +Running this command should be safe and can be ran multiple times on +both the main ``openedx-translations`` project or on release projects +by setting the ``TRANSIFEX_PROJECT_SLUG`` make variable as shown below:: + + # Dry run the name fix + make TRANSIFEX_PROJECT_SLUG='openedx-translations-zebrawood' fix_transifex_resource_names_dry_run + # If runs without errors, run the actual command: + make TRANSIFEX_PROJECT_SLUG='openedx-translations-zebrawood' fix_transifex_resource_names Translation validation -********************** +====================== This repository validates translations with the GNU gettext ``msgfmt`` tool. @@ -38,6 +76,9 @@ The validation can be run locally with the following command: The validation errors is also posted as a comment on the update translation pull requests. +Retry merging Transifex pull requests +===================================== + If GitHub Actions has an outage or any other issues there will be a backlog of stale unmerged Transifex bot pull requests. To re-run tests and merge the pull requests, run the following command: @@ -46,31 +87,19 @@ pull requests, run the following command: make retry_merge_transifex_bot_pull_requests - -Translations sync from old Transifex projects -********************************************* - -This repository allows for syncing translations old -`open-edx/edx-platform`_ and `open-edx/xblocks`_ Transifex projects into -the new `open-edx/openedx-translations`_ Transifex project. This is done by -trigger the `sync_translations.yml workflow on GitHub`_. - -Alternatively, you can run the following command to trigger the workflow: - -.. code-block:: bash - - # Run with parameters from .github/workflows/sync-translations.yml - make sync_translations_github_workflow - - -For more information, please see the pull request for `OEP-58`_. - - .. _OEP-58: https://github.com/openedx/open-edx-proposals/pull/367 .. _openedx-atlas: https://github.com/openedx/openedx-atlas .. _sync_translations.yml workflow on GitHub: https://github.com/openedx/openedx-translations/actions/workflows/sync-translations.yml -.. _open-edx/edx-platform: https://app.transifex.com/open-edx/edx-platform/dashboard/ -.. _open-edx/xblocks: https://app.transifex.com/open-edx/xblocks/dashboard/ .. _open-edx/openedx-translations: https://app.transifex.com/open-edx/openedx-translations/dashboard/ +.. _open-edx/openedx-translations-redwood: https://app.transifex.com/open-edx/openedx-translations-redwood/dashboard/ + + +.. _Tutor nightly: https://docs.tutor.edly.io/tutorials/nightly.html +.. _edx-platform "master" branch: https://github.com/openedx/edx-platform +.. _Tutor Redwood v18: https://docs.tutor.edly.io/ +.. _edx-platform "open-release/redwood.master" branch: https://github.com/openedx/edx-platform/tree/open-release/redwood.master + +.. _GitHub Actions workflows: https://github.com/openedx/openedx-translations/tree/main/.github/workflows +.. _Makefile programs: https://github.com/openedx/openedx-translations/blob/main/Makefile diff --git a/scripts/fix_transifex_resource_names.py b/scripts/fix_transifex_resource_names.py index fe2049cb408..df92e6f8894 100644 --- a/scripts/fix_transifex_resource_names.py +++ b/scripts/fix_transifex_resource_names.py @@ -39,9 +39,25 @@ def is_dry_run(): return '--dry-run' in sys.argv +def get_transifex_project_slug(): + """ + Get Transifex project slug e.g. openedx-translations or openedx-translations-. + """ + slug = getenv('TRANSIFEX_PROJECT_SLUG') + if not slug: + raise RuntimeError( + 'Error: Cannot determine Transifex project slug. Set `TRANSIFEX_PROJECT_SLUG` environment variable to ' + '"openedx-translations" or "openedx-translations-". ' + 'List of Open edX releases are available in the following page: ' + 'https://openedx.atlassian.net/wiki/spaces/OEPM/pages/4191191044/Open+edX+Releases+Homepage' + ) + + return slug + + def get_transifex_project(): """ - Get openedx-translations project from Transifex. + Get the translations project object from Transifex. """ transifex_api_token = getenv('TRANSIFEX_API_TOKEN') if not transifex_api_token: @@ -58,7 +74,7 @@ def get_transifex_project(): transifex_api.setup(auth=transifex_api_token) openedx_org = transifex_api.Organization.get(slug='open-edx') - return openedx_org.fetch('projects').get(slug='openedx-translations') + return openedx_org.fetch('projects').get(slug=get_transifex_project_slug()) def get_repo_slug_from_resource(resource): @@ -101,7 +117,7 @@ def main(argv): print(__doc__) return - print('Updating openedx-translations project resource and slug names:') + print(f'Updating "{get_transifex_project_slug()}" project resource and slug names:') openedx_translations_proj = get_transifex_project() for resource in openedx_translations_proj.fetch('resources'): diff --git a/scripts/sync_translations.py b/scripts/sync_translations.py deleted file mode 100644 index 1bf06bc3166..00000000000 --- a/scripts/sync_translations.py +++ /dev/null @@ -1,302 +0,0 @@ -""" -Sync translations from the deprecated Transifex projects into the new openedx-translations project. - - - Old projects links: - * edX Platform Core: https://app.transifex.com/open-edx/edx-platform/ - * XBlocks: https://app.transifex.com/open-edx/xblocks/ - - - New project link: - * https://app.transifex.com/open-edx/openedx-translations/ - - -Variable names meaning: - - - current_translation: translation in the new "open-edx/openedx-translations" project - - translation_from_old_project: translation in the old "open-edx/edx-platform" or "open-edx/xblocks" projects - -""" - -import argparse -import configparser -from datetime import datetime -import os -from os.path import expanduser -import yaml - -from transifex.api import transifex_api -from transifex.api.jsonapi import exceptions - -NEW_PROJECT_SLUG = 'openedx-translations' -ORGANIZATION_SLUG = 'open-edx' - - -def parse_tx_date(date_str): - """ - Parse a date string coming from Transifex into a datetime object. - """ - if date_str: - if date_str.endswith('Z'): - date_str = date_str.replace('Z', '+00:00') - - return datetime.fromisoformat(date_str) - - return None - - -class Command: - - workflow_file_path = '.github/workflows/sync-translations.yml' - - def __init__(self, tx_api, dry_run, simulate_github_workflow, environ): - self.dry_run = dry_run - self.simulate_github_workflow = simulate_github_workflow - self.tx_api = tx_api - self.environ = environ - - def is_dry_run(self): - """ - Check if the script is running in dry-run mode. - """ - return self.dry_run - - def is_simulated_github_actions(self): - """ - Check if the script is running in simulated GitHub Actions mode. - """ - return self.simulate_github_workflow - - def get_resource_url(self, resource, project_slug): - return f'https://www.transifex.com/{ORGANIZATION_SLUG}/{project_slug}/{resource.slug}' - - def get_transifex_organization_projects(self): - """ - Get openedx-translations project from Transifex. - """ - tx_api_token = self.environ.get('TX_API_TOKEN') - if not tx_api_token: - config = configparser.ConfigParser() - config.read(expanduser('~/.transifexrc')) - tx_api_token = config['https://www.transifex.com']['password'] - - if not tx_api_token: - raise Exception( - 'Error: No auth token found. ' - 'Set transifex API token via TX_API_TOKEN environment variable or via the ~/.transifexrc file.' - ) - - self.tx_api.setup(auth=tx_api_token) - return self.tx_api.Organization.get(slug=ORGANIZATION_SLUG).fetch('projects') - - def get_resources_pair(self, new_slug, old_slug, old_project_slug): - """ - Load the old and new Transifex resources pair. - """ - projects = self.get_transifex_organization_projects() - new_project = projects.get(slug=NEW_PROJECT_SLUG) - - new_resource_id = f'o:{ORGANIZATION_SLUG}:p:{new_project.slug}:r:{new_slug}' - print(f'new resource id: {new_resource_id}') - try: - new_resource = self.tx_api.Resource.get(id=new_resource_id) - except exceptions.JsonApiException as error: - print(f'Error: New resource error: {new_resource_id}. Error: {error}') - raise - - old_resource_id = f'o:{ORGANIZATION_SLUG}:p:{old_project_slug}:r:{old_slug}' - print(f'old resource id: {old_resource_id}') - try: - old_resource = self.tx_api.Resource.get(id=old_resource_id) - except exceptions.JsonApiException as error: - print(f'Error: Old resource error: {new_resource_id}. Error: {error}') - raise - - return { - 'old_resource': old_resource, - 'new_resource': new_resource, - } - - def get_translations(self, language_code, resource): - """ - Get a list of translations for a given language and resource. - """ - language = self.tx_api.Language.get(code=language_code) - translations = self.tx_api.ResourceTranslation. \ - filter(resource=resource, language=language). \ - include('resource_string') - - return translations.all() - - def sync_translations(self, language_code, old_resource, new_resource): - """ - Sync specific language translations into the new Transifex resource. - """ - print(' syncing', language_code, '...') - translations_from_old_project = { - self.get_translation_id(translation): translation - for translation in self.get_translations(language_code=language_code, resource=old_resource) - } - - for current_translation in self.get_translations(language_code=language_code, resource=new_resource): - translation_id = self.get_translation_id(current_translation) - if translation_from_old_project := translations_from_old_project.get(translation_id): - self.sync_translation_entry( - translation_from_old_project=translation_from_old_project, - current_translation=current_translation, - ) - - def sync_translation_entry(self, translation_from_old_project, current_translation): - """ - Sync a single translation entry from the old project to the new one. - - Return: - str: status code - - updated: if the entry was updated - - skipped: if the entry was skipped - - updated-dry-run: if the entry was updated in dry-run mode - """ - translation_id = self.get_translation_id(current_translation) - - updates = {} - for attr in ['reviewed', 'proofread', 'strings']: - if old_attr_value := getattr(translation_from_old_project, attr, None): - if old_attr_value != getattr(current_translation, attr, None): - updates[attr] = old_attr_value - - # Avoid overwriting more recent translations in the open-edx/openedx-translations project - newer_translation_found = False - old_project_translation_time = parse_tx_date(translation_from_old_project.datetime_translated) - current_translation_time = parse_tx_date(current_translation.datetime_translated) - - if old_project_translation_time and current_translation_time: - newer_translation_found = current_translation_time > old_project_translation_time - - if updates: - if newer_translation_found: - print(translation_id, updates, - ( - f'[Skipped: current translation "{current_translation_time}" ' - f'is more recent than "{old_project_translation_time}"]' - ) - ) - return 'skipped' - else: - print(translation_id, updates, '[Dry run]' if self.is_dry_run() else '') - if self.is_dry_run(): - return 'updated-dry-run' - else: - current_translation.save(**updates) - return 'updated' - - def sync_tags(self, old_resource, new_resource): - """ - Sync tags from the old Transifex resource into the new Transifex resource. This process is language independent. - """ - old_resource_str = self.tx_api.ResourceString.filter(resource=old_resource) - new_resource_str = self.tx_api.ResourceString.filter(resource=new_resource) - - old_quick_lookup = {} - for item in old_resource_str.all(): - dict_item = item.to_dict() - old_quick_lookup[dict_item['attributes']['string_hash']] = dict_item['attributes']['tags'] - - for new_info in new_resource_str.all(): - old_tags = old_quick_lookup.get(new_info.string_hash) - new_tags = new_info.tags - - if old_tags is None: # in case of new changes are not synced yet - continue - if len(new_tags) == 0 and len(old_tags) == 0: # nothing to compare - continue - - if len(new_tags) != len(old_tags) or set(new_tags) != set(old_tags): - print(f' - found tag difference for {new_info.string_hash}. overwriting: {new_tags} with {old_tags}') - - if not self.is_dry_run(): - new_info.save(tags=old_tags) - - def get_translation_id(self, translation): - """ - Build a unique identifier for a translation entry. - """ - return f'context:{translation.resource_string.context}:key:{translation.resource_string.key}' - - def get_languages(self): - """ - Get a list of languages to sync translations for. - """ - return self.environ['TX_LANGUAGES'].split(',') - - def sync_pair_into_new_resource(self, new_slug, old_slug, old_project_slug): - """ - Sync translations from both the edx-platform and XBlock projects into the new openedx-translations project. - """ - languages = self.get_languages() - resource_pair = self.get_resources_pair(new_slug, old_slug, old_project_slug) - - print(f'Syncing {resource_pair["new_resource"].name} from {resource_pair["old_resource"].name}...') - print(f'Syncing: {languages}') - print(f' - from: {self.get_resource_url(resource_pair["old_resource"], old_project_slug)}') - print(f' - to: {self.get_resource_url(resource_pair["new_resource"], NEW_PROJECT_SLUG)}') - - for lang_code in languages: - self.sync_translations(language_code=lang_code, **resource_pair) - - print('Syncing tags...') - self.sync_tags(**resource_pair) - - print('-' * 80, '\n') - - def run_from_workflow_yaml_file(self, workflow_configs): - """ - Run the script from a GitHub Actions migrate-from-transifex-old-project.yml workflow file. - """ - pairs_list = workflow_configs['jobs']['migrate-translations']['strategy']['matrix']['resource'] - - print('Verifying existence of resource pairs...') - for pair in pairs_list: - self.get_resources_pair( - new_slug=pair['new_slug'], - old_slug=pair['old_slug'], - old_project_slug=pair['old_project_slug'], - ) - print('\n', '-' * 80, '\n') - - for pair in pairs_list: - self.sync_pair_into_new_resource( - new_slug=pair['new_slug'], - old_slug=pair['old_slug'], - old_project_slug=pair['old_project_slug'], - ) - - def run(self): - if self.is_simulated_github_actions(): - with open(self.workflow_file_path) as workflow_file: - self.run_from_workflow_yaml_file( - workflow_configs=yaml.safe_load(workflow_file.read()), - ) - else: - self.sync_pair_into_new_resource( - new_slug=self.environ['TX_NEW_SLUG'], - old_slug=self.environ['TX_OLD_SLUG'], - old_project_slug=self.environ['TX_OLD_PROJECT_SLUG'], - ) - - -def main(): # pragma: no cover - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--simulate-github-workflow', action='store_true', - dest='simulate_github_workflow') - parser.add_argument('--dry-run', action='store_true', dest='dry_run') - argparse_args = parser.parse_args() - - command = Command( - tx_api=transifex_api, - environ=os.environ, - dry_run=argparse_args.dry_run, - simulate_github_workflow=argparse_args.simulate_github_workflow, - ) - command.run() - - -if __name__ == '__main__': - main() # pragma: no cover diff --git a/scripts/tests/test_sync_translations.py b/scripts/tests/test_sync_translations.py deleted file mode 100644 index 199c0e3fd2c..00000000000 --- a/scripts/tests/test_sync_translations.py +++ /dev/null @@ -1,316 +0,0 @@ -""" -Tests for sync_translations.py -""" -from dataclasses import dataclass -from datetime import datetime, timezone -import types -from typing import Union - -import pytest -import responses - -from transifex.api import transifex_api, Project -from transifex.api.jsonapi import Resource -from transifex.api.jsonapi.auth import BearerAuthentication - -from . import response_data -from ..sync_translations import ( - Command, - ORGANIZATION_SLUG, - parse_tx_date, -) - -HOST = transifex_api.HOST - - -def sync_command(**kwargs): - command_args = { - 'tx_api': transifex_api, - 'dry_run': True, - 'simulate_github_workflow': False, - 'environ': { - 'TX_API_TOKEN': 'dummy-token' - } - } - command_args.update(kwargs) - result = Command(**command_args) - result.tx_api.make_auth_headers = BearerAuthentication('dummy-token') - return result - - -@dataclass -class ResourceStringMock: - """ - String entry in Transifex. - - Mock class for the transifex.api.ResourceString class. - """ - key: str - context: str = '' - - -@dataclass -class ResourceTranslationMock: - """ - Translation for an entry in Transifex. - - Mock class for the transifex.api.ResourceTranslation class. - """ - resource_string: ResourceStringMock - strings: dict - reviewed: bool - proofread: bool - datetime_translated: str - - _updates: dict = None # Last updates applied via `save()` - - def save(self, **updates): - """ - Mock ResourceTranslation.save() method. - """ - self._updates = updates - - @property - def updates(self): - """ - Return the last updates applied via `save()`. - """ - return self._updates - - @classmethod - def factory( - cls, - key='key', - context='', - translation: Union[str, None] = 'dummy translation', - **kwargs - ): - mock_kwargs = dict( - resource_string=ResourceStringMock( - key=key, - context=context - ), - strings={ - key: translation, - }, - reviewed=False, - proofread=False, - datetime_translated='2021-01-01T00:00:00Z', - ) - - mock_kwargs.update(kwargs) - return cls(**mock_kwargs) - - -@responses.activate -def test_get_transifex_organization_projects(): - """ - Verify that the get_transifex_organization_projects() method returns the correct data. - """ - command = sync_command() - - # Mocking responses - responses.add( - responses.GET, - HOST + f'/organizations?filter[slug]={ORGANIZATION_SLUG}', - json=response_data.RESPONSE_GET_ORGANIZATION, - status=200 - ) - responses.add( - responses.GET, - HOST + f'/projects?filter[organization]={response_data.RESPONSE_GET_ORGANIZATION["data"][0]["id"]}', - json=response_data.RESPONSE_GET_PROJECTS, - status=200 - ) - - # Remove the make_auth_headers to verify later that transifex setup is called - delattr(command.tx_api, 'make_auth_headers') - - data = command.get_transifex_organization_projects() - assert hasattr(command.tx_api, 'make_auth_headers') - assert isinstance(command.tx_api.make_auth_headers, BearerAuthentication) - assert len(data) == 1 - assert isinstance(data[0], Project) - assert data[0].id == response_data.RESPONSE_GET_PROJECTS['data'][0]['id'] - - -@responses.activate -def test_get_translations(): - """ - Verify that the get_translations() method returns the correct data. - """ - command = sync_command() - resource_id = f'{response_data.RESPONSE_GET_PROJECTS["data"][0]["id"]}:r:ar' - - # Mocking responses - responses.add( - responses.GET, - HOST + f'/languages?filter[code]=ar', - json=response_data.RESPONSE_GET_LANGUAGE, - status=200 - ) - responses.add( - responses.GET, - HOST + f'/resource_translations?filter[resource]={resource_id}&filter[language]=l:ar&include=resource_string', - json=response_data.RESPONSE_GET_LANGUAGE, - status=200 - ) - - data = command.get_translations( - language_code='ar', - resource=Resource(id=resource_id) - ) - assert isinstance(data, types.GeneratorType) - items = list(data) - assert len(items) == 1 - assert items[0].id == response_data.RESPONSE_GET_LANGUAGE['data'][0]['id'] - - -def test_translations_entry_update_empty_translation(): - """ - Test updating an entry from old project where `current_translation` is empty. - """ - command = sync_command(dry_run=False) - - translation_from_old_project = ResourceTranslationMock.factory( - key='test_key', - translation='old translation', - reviewed=True, - datetime_translated='2023-01-01T00:00:00Z', - ) - - # Current translation is empty - current_translation = ResourceTranslationMock.factory( - key='test_key', - translation=None, - datetime_translated=None, - ) - - status = command.sync_translation_entry( - translation_from_old_project, current_translation - ) - - assert status == 'updated' - assert current_translation.updates == { - 'strings': { - 'test_key': 'old translation' - }, - 'reviewed': True, - } - - -@pytest.mark.parametrize( - 'old_project_date, current_translation_date, new_translation_str', - [ - # As long as the current translation is _not_ more recent, it should be updated - (None, '2023-01-01T00:00:00Z', None), - (None, '2023-01-01T00:00:00Z', 'some translation'), - ('2023-01-01T00:00:00Z', '2023-01-01T00:00:00Z', 'some translation'), # Same date - ('2023-01-01T00:00:00Z', '2021-01-01T00:00:00Z', 'some translation'), # New project has newer date - ('2023-01-01T00:00:00Z', None, 'some translation'), - ('2023-01-01T00:00:00Z', None, None), - ] -) -def test_translations_entry_update_translation(old_project_date, current_translation_date, new_translation_str): - """ - Test updating an entry from old project where `current_translation` is has outdated translation. - """ - command = sync_command(dry_run=False) - - translation_from_old_project = ResourceTranslationMock.factory( - key='test_key', - translation='old translation', - reviewed=True, - datetime_translated=old_project_date, - ) - - current_translation = ResourceTranslationMock.factory( - key='test_key', - translation=new_translation_str, - datetime_translated=current_translation_date, - ) - - status = command.sync_translation_entry( - translation_from_old_project, current_translation - ) - - assert status == 'updated' - assert current_translation.updates == { - 'strings': { - 'test_key': 'old translation' - }, - 'reviewed': True, - } - - -def test_translations_entry_more_recent_translation(): - """ - Verify that the more recent translations in the open-edx/openedx-translations project are not overridden. - """ - command = sync_command(dry_run=False) - - translation_from_old_project = ResourceTranslationMock.factory( - key='test_key', - translation='one translation', - reviewed=True, - datetime_translated='2019-01-01T00:00:00Z', - ) - - # Current translation is empty - current_translation = ResourceTranslationMock.factory( - key='test_key', - translation='more recent translation', - datetime_translated='2023-01-01T00:00:00Z', - ) - - status = command.sync_translation_entry( - translation_from_old_project, current_translation - ) - - assert status == 'skipped' - assert not current_translation.updates, 'save() should not be called' - - -def test_translations_entry_dry_run(): - """ - Verify that --dry-run option skips the save() call. - """ - command = sync_command(dry_run=True) - - translation_from_old_project = ResourceTranslationMock.factory( - key='test_key', - translation='old translation', - reviewed=True, - datetime_translated='2023-01-01T00:00:00Z', - ) - - current_translation = ResourceTranslationMock.factory( - key='test_key', - translation=None, - datetime_translated=None, - ) - - status = command.sync_translation_entry( - translation_from_old_project, current_translation - ) - - assert status == 'updated-dry-run' - assert not current_translation.updates, 'save() should not be called in --dry-run mode' - - -@pytest.mark.parametrize( - "date_str, parse_result, test_message", - [ - (None, None, 'None cannot be parsed'), - ('2023-01-01T00:00:00Z', datetime(2023, 1, 1, 0, 0, tzinfo=timezone.utc), - 'Z suffix is replaced with the explict "+00:00" timezone'), - ('2023-01-01T00:00:00', datetime(2023, 1, 1, 0, 0), - 'If there is no Z suffix, no timezone is added'), - ] -) -def test_parse_tx_date(date_str, parse_result, test_message): - """ - Tests for parse_tx_date() helper function. - """ - assert parse_tx_date(date_str) == parse_result, test_message -