Skip to content

Commit

Permalink
Better translation manager
Browse files Browse the repository at this point in the history
Allow previewing the JSON with color coded values
Sort the output
Allow moving keys around
wbazant committed Jan 21, 2025
1 parent 899ee2f commit fb55d0a
Showing 3 changed files with 407 additions and 114 deletions.
498 changes: 391 additions & 107 deletions scripts/translation_manager.py
100644 → 100755
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
#!/usr/bin/env python
import argparse
import json
import os
import sys
import re
from pathlib import Path

import yaml
from colored import Fore, Style


class Component:
@@ -29,36 +32,134 @@ def extract_translation_keys(self):
self.key_locations[key] = str(self.file_path)
return self.keys

def rename_key_in_file(self, old_key, new_key):
"""Rename a translation key in the source file"""
with open(self.file_path, 'r') as file:
content = file.read()

# Replace the key in t() calls, being careful with quote types
for quote in ['"', "'", '`']:
content = content.replace(f't({quote}{old_key}{quote})',
f't({quote}{new_key}{quote})')

with open(self.file_path, 'w') as file:
file.write(content)

class Translation:
def __init__(self):
self.entries = {} # flat dictionary of dot-notation keys to values

def set(self, key: str, value: str):
"""Set a value using dot notation key"""
self.entries[key] = value

def get(self, key: str) -> str:
"""Get a value using dot notation key"""
return self.entries.get(key)


def to_nested_dict(self) -> dict:
"""Convert flat dot-notation keys into nested dictionary structure"""
result = {}
for key, value in self.entries.items():
keys = key.split('.')
current = result
for k in keys[:-1]:
current = current.setdefault(k, {})
current[keys[-1]] = value
return result

def save_as_json(self, file_path, encoder):
"""Save translation data as JSON file"""
path = Path(file_path)
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, 'w') as file:
json_str = encoder.encode(self.to_nested_dict())
file.write(json_str)

class LocaleFile:
def __init__(self, file_path):
self.file_path = Path(file_path)
self.lang_code = self.file_path.stem
self.data = self.load_data()

def load_data(self):
def clean_value(self, value):
raise NotImplementedError

def save_data(self):
def load_data(self):
raise NotImplementedError

def to_translation(self) -> Translation:
"""Convert nested dictionary to Translation object"""
translation = Translation()

def flatten_dict(d, prefix=''):
for key, value in d.items():
full_key = f"{prefix}.{key}" if prefix else key
if isinstance(value, dict):
flatten_dict(value, full_key)
else:
renamed_key = self.rename_key_if_needed(full_key)
translation.set(renamed_key, self.clean_value(value))

flatten_dict(self.data)
return translation

class YamlLocaleFile(LocaleFile):
def load_data(self):
with open(self.file_path, 'r') as file:
return yaml.safe_load(file)

def save_data(self):
with open(self.file_path, 'w') as file:
yaml.dump(self.data, file, allow_unicode=True)

def remove_key(self, keys):
"""Remove a nested key from the YAML data"""
current = self.data
for key in keys[:-1]:
if key not in current:
return
current = current[key]
if keys[-1] in current:
del current[keys[-1]]
data = yaml.safe_load(file)
# Get the language code subdict directly
return data.get(self.lang_code, {})

def clean_value(self, value):
"""Clean up YAML value for JSON format"""
if isinstance(value, str):
# Remove locale parameters
value = re.sub(r'\?locale=[a-z]{2}', '', value)
# Replace %{...} with {{...}}
value = re.sub(r'%\{(\w+)\}', r'{{\1}}', value)
return value

def rename_key_if_needed(self, key):
"""Apply any configured key renames"""
if hasattr(self, 'manager') and self.manager.key_renames:
return self.manager.key_renames.get(key, key)
return key

def print_colored_dict(d, indent=0):
"""Print a nested dictionary with proper indentation"""
for key, value in sorted(d.items()):
indent_str = " " * indent
if isinstance(value, dict):
print(f'{indent_str}"{key}": {{')
print_colored_dict(value, indent + 1)
print(f'{indent_str}}}{"," if indent > 0 else ""}')
else:
is_last = key == list(sorted(d.keys()))[-1] and indent > 0
print(f'{indent_str}"{key}": "{value}"{"" if is_last else ","}')

class SortedJsonEncoder(json.JSONEncoder):
def default(self, obj):
return super().default(obj)

def encode(self, obj, level=0):
indent = " " * level
next_indent = " " * (level + 1)

if isinstance(obj, dict):
# Sort dictionary keys
items = sorted(obj.items(), key=lambda x: x[0])
if not items:
return "{}"
parts = [f"\n{next_indent}{json.dumps(k)}: {self.encode(v, level + 1)}" for k, v in items]
return "{" + ",".join(parts) + f"\n{indent}}}"
elif isinstance(obj, (list, tuple)):
items = [self.encode(item, level + 1) for item in obj]
if not items:
return "[]"
return "[\n" + next_indent + f",\n{next_indent}".join(items) + f"\n{indent}]"
return json.dumps(obj, ensure_ascii=False)

class JsonLocaleFile(LocaleFile):
def load_data(self):
@@ -67,22 +168,45 @@ def load_data(self):
with open(self.file_path, 'r') as file:
return json.load(file)

def save_data(self):
self.file_path.parent.mkdir(parents=True, exist_ok=True) # Ensure the directory exists
with open(self.file_path, 'w') as file:
json.dump(self.data, file, indent=2, ensure_ascii=False)
def clean_value(self, value):
return value

def rename_key_if_needed(self, key):
return key


class TranslationManager:
def __init__(self, source_path, yaml_folder, json_folder):
def __init__(self, source_path, yaml_folder_path, json_folder_path, yaml_key_renames_path=None):
self.source_path = Path(source_path) if source_path else None
self.components = []
if self.source_path:
if self.source_path.is_file():
self.components.append(Component(self.source_path))
else:
self.scan_source_files()
self.yaml_folder = Path(yaml_folder) if yaml_folder else None
self.json_folder = Path(json_folder) if json_folder else None
self.yaml_folder_path = Path(yaml_folder_path) if yaml_folder_path else None
self.json_folder_path = Path(json_folder_path) if json_folder_path else None
self.yaml_key_renames_path = Path(yaml_key_renames_path) if yaml_key_renames_path else None
self.key_renames = self.load_key_renames()

def load_key_renames(self):
"""Load key renames from TSV file"""
renames = {}
if self.yaml_key_renames_path and self.yaml_key_renames_path.exists():
with open(self.yaml_key_renames_path, 'r') as f:
for line in f:
old_key, new_key = line.strip().split('\t')
renames[old_key] = new_key
return renames

def append_key_rename(self, old_key: str, new_key: str):
"""Append a key rename to the TSV file"""
if not self.yaml_key_renames_path:
return

with open(self.yaml_key_renames_path, 'a') as f:
f.write(f"{old_key}\t{new_key}\n")
self.key_renames[old_key] = new_key

def scan_source_files(self):
for file_path in self.source_path.rglob('*'):
@@ -91,90 +215,70 @@ def scan_source_files(self):
if component.keys: # Only add components that have translation keys
self.components.append(component)

def clean_yaml_translations(self):
if not self.components:
print("Error: No source files with translations found. Skipping cleanup.")
return
if not self.yaml_folder.exists():
print(f"Error: YAML folder '{self.yaml_folder}' does not exist.")
return

yaml_files = list(self.yaml_folder.glob('*.yml')) + list(self.yaml_folder.glob('*.yaml'))
if not yaml_files:
print(f"No YAML files found in {self.yaml_folder}")
return

for yaml_file in yaml_files:
yaml_locale = YamlLocaleFile(yaml_file)
removed_keys = 0

all_keys = set()
for component in self.components:
all_keys.update(component.keys)

for key in all_keys:
if self.get_nested_value(yaml_locale.data.get(yaml_locale.lang_code, {}), key.split('.')):
yaml_locale.remove_key([yaml_locale.lang_code] + key.split('.'))
removed_keys += 1

if removed_keys > 0:
yaml_locale.save_data()
print(f"Removed {removed_keys} keys from {yaml_file}")
else:
print(f"No keys to remove from {yaml_file}")

def migrate_translations(self):
if not self.components:
print("Error: No source files with translations found. Skipping migration.")
return
if not self.yaml_folder.exists():
print(f"Error: YAML folder '{self.yaml_folder}' does not exist.")
if not self.yaml_folder_path.exists():
print(f"Error: YAML folder '{self.yaml_folder_path}' does not exist.")
return
if not self.json_folder.exists():
print(f"Creating JSON folder: {self.json_folder}")
self.json_folder.mkdir(parents=True, exist_ok=True)
if not self.json_folder_path.exists():
print(f"Creating JSON folder: {self.json_folder_path}")
self.json_folder_path.mkdir(parents=True, exist_ok=True)

yaml_files = list(self.yaml_folder.glob('*.yml')) + list(self.yaml_folder.glob('*.yaml'))
yaml_files = list(self.yaml_folder_path.glob('*.yml')) + list(self.yaml_folder_path.glob('*.yaml'))
if not yaml_files:
print(f"No YAML files found in {self.yaml_folder}")
print(f"No YAML files found in {self.yaml_folder_path}")
return

for yaml_file in yaml_files:
yaml_locale = YamlLocaleFile(yaml_file)
json_file = self.json_folder / f"{yaml_locale.lang_code}.json"
json_file = self.json_folder_path / f"{yaml_locale.lang_code}.json"
if json_file.exists():
json_locale = JsonLocaleFile(json_file)

added_keys = 0
edited_keys = 0
missing_keys = 0

all_keys = set()
for component in self.components:
all_keys.update(component.keys)

# Convert YAML to Translation
yaml_translation = yaml_locale.to_translation()
json_translation = json_locale.to_translation()

for key in all_keys:
value = self.get_nested_value(yaml_locale.data.get(yaml_locale.lang_code, {}), key.split('.'))
value = yaml_translation.get(key)
if value is not None:
# Clean up the value before setting
if isinstance(value, str):
# Remove locale parameters
value = re.sub(r'\?locale=[a-z]{2}', '', value)
# Replace %{...} with {{...}}
value = re.sub(r'%\{(\w+)\}', r'{{\1}}', value)

existing_value = self.get_nested_value(json_locale.data, key.split('.'))
existing_value = json_translation.get(key)
if existing_value is None:
added_keys += 1
elif existing_value != value:
edited_keys += 1
self.set_nested_value(json_locale.data, key.split('.'), value)

json_locale.save_data()
print(f"{yaml_file} -> {json_file} added {added_keys} keys, edited {edited_keys} keys")
json_translation.set(key, value)
elif json_translation.get(key) is None:
# decide later:
# we could instead seed the translation with key as value
# json_translation.set(key, key)
missing_keys += 1

json_translation.save_as_json(json_file, SortedJsonEncoder(indent=2))
print(f"{yaml_file} -> {json_file} added {added_keys} keys, edited {edited_keys} keys, missing {missing_keys} keys. Checked: {len(all_keys)} keys")
else:
print(f"{yaml_file} skipped: no {json_file}")


def components_to_translation(self) -> Translation:
"""Convert components' keys into a Translation object where values equal keys"""
translation = Translation()
for component in self.components:
for key in component.keys:
translation.set(key, key)
return translation

def list_component_keys(self):
if not self.components:
print("Error: No source files with translations found.")
@@ -193,60 +297,240 @@ def list_component_keys(self):
for file_path, key in sorted(all_keys):
print(f"{file_path}\t{key}")

@staticmethod
def get_nested_value(dictionary, keys):
for key in keys:
if key in dictionary:
dictionary = dictionary[key]
def preview_migration(self, language):
"""Preview migration for a specific language without writing files"""
if not self.components:
print("Error: No source files with translations found.", file=sys.stderr)
return
if not self.yaml_folder_path.exists():
print(f"Error: YAML folder '{self.yaml_folder_path}' does not exist.", file=sys.stderr)
return

yaml_file = next((f for f in self.yaml_folder_path.glob(f'{language}.*')
if f.suffix in ['.yml', '.yaml']), None)
if not yaml_file:
print(f"Error: No YAML file found for language {language}", file=sys.stderr)
return

yaml_locale = YamlLocaleFile(yaml_file)
yaml_translation = yaml_locale.to_translation()

json_file = self.json_folder_path / f"{yaml_locale.lang_code}.json"
json_translation = JsonLocaleFile(json_file).to_translation() if json_file.exists() else Translation()

# Get all keys from components and existing JSON
component_keys = set()
for component in self.components:
component_keys.update(component.keys)

# First handle all component keys
for key in component_keys:
value = yaml_translation.get(key)
existing_value = json_translation.get(key)
if value is not None:
# Color existing translations green
colored_value = f"{Fore.dark_blue}{value}{Style.reset}"
json_translation.set(key, colored_value)
elif existing_value is not None:
colored_value = f"{Fore.green}{existing_value}{Style.reset}"
json_translation.set(key, colored_value)
else:
return None
return dictionary
# Color seeded translations red
colored_value = f"{Fore.red}{key}{Style.reset}"
json_translation.set(key, colored_value) # Seed missing keys

# Then handle any keys in JSON that aren't in components
for key, value in json_translation.entries.items():
if key not in component_keys:
# Color orphaned keys magenta
colored_value = f"{Fore.magenta}{value}{Style.reset}"
print(key, colored_value)
json_translation.set(key, colored_value)

# Print the colored nested dictionary
print("{")
print_colored_dict(json_translation.to_nested_dict(), 1)
print("}")

def rename_key(self, old_key: str, new_key: str):
"""Rename a translation key in source files and JSON files"""
# First verify the old key exists in both source AND English JSON
key_in_source = False
for component in self.components:
if old_key in component.keys:
key_in_source = True
break

key_in_english = False
en_json_path = self.json_folder_path / "en.json"
if en_json_path.exists():
en_locale = JsonLocaleFile(en_json_path)
if en_locale.to_translation().get(old_key) is not None:
key_in_english = True

if not key_in_source and not key_in_english:
print(f"Error: Key '{old_key}' not found in source files or English JSON")
return False
if not key_in_source:
print(f"Error: Key '{old_key}' not found in source files")
return False
if not key_in_english:
print(f"Error: Key '{old_key}' not found in English JSON")
return False

# If key exists in YAML, add to renames file
yaml_has_key = False
if self.yaml_folder_path and self.yaml_folder_path.exists():
yaml_files = list(self.yaml_folder_path.glob('*.yml')) + list(self.yaml_folder_path.glob('*.yaml'))
for yaml_file in yaml_files:
yaml_locale = YamlLocaleFile(yaml_file)
yaml_locale.manager = self # Allow access to renames
if yaml_locale.to_translation().get(old_key) is not None:
yaml_has_key = True
break

if yaml_has_key:
if self.yaml_key_renames_path:
self.append_key_rename(old_key, new_key)
print(f"Info: '{old_key}' exists in YAML file {yaml_file}")
else:
print(f"Error: '{old_key}' exists in YAML file {yaml_file} but no renames file. Aborting")
return False


# Check if the key exists in YAML files (which we don't want to modify)
if self.yaml_folder_path and self.yaml_folder_path.exists():
yaml_files = list(self.yaml_folder_path.glob('*.yml')) + list(self.yaml_folder_path.glob('*.yaml'))
for yaml_file in yaml_files:
yaml_locale = YamlLocaleFile(yaml_file)
if yaml_locale.to_translation().get(old_key) is not None:
print(f"Info: '{old_key}' exists in YAML file {yaml_file}")

# Rename in source files
source_files_changed = 0
for component in self.components:
if old_key in component.keys:
print(f"Renaming key in source file: {component.file_path}")
component.rename_key_in_file(old_key, new_key)
source_files_changed += 1
print(f"Updated key in {source_files_changed} source files")

# Rename in JSON files
json_files_changed = 0
if self.json_folder_path and self.json_folder_path.exists():
for json_file in self.json_folder_path.glob('*.json'):
json_locale = JsonLocaleFile(json_file)
translation = json_locale.to_translation()
value = translation.get(old_key)
if value is not None:
print(f"Renaming key in JSON file: {json_file}")
print(f" Old value: {value}")
translation.set(new_key, value)
# Remove old key
translation.entries.pop(old_key, None)
# Save using Translation methods
translation.save_as_json(json_file, SortedJsonEncoder(indent=2))
json_files_changed += 1
print(f"Updated key in {json_files_changed} JSON files")

return True

def list_json_keys(self):
if not self.json_folder_path or not self.json_folder_path.exists():
print("Error: JSON folder does not exist.")
return

json_files = list(self.json_folder_path.glob('*.json'))
if not json_files:
print(f"No JSON files found in {self.json_folder_path}")
return

def traverse_dict(d, prefix=""):
items = []
for k, v in sorted(d.items()):
current_key = f"{prefix}.{k}" if prefix else k
if isinstance(v, dict):
items.extend(traverse_dict(v, current_key))
else:
items.append((current_key, str(v)))
return items

for json_file in sorted(json_files):
json_locale = JsonLocaleFile(json_file)
all_keys = traverse_dict(json_locale.data)
for key, value in all_keys:
# Replace newlines with spaces in the value for TSV compatibility
value = value.replace('\n', ' ').replace('\r', '')
print(f"{json_file}\t{key}\t{value}")

@staticmethod
def set_nested_value(dictionary, keys, value):
for key in keys[:-1]:
dictionary = dictionary.setdefault(key, {})
dictionary[keys[-1]] = value

def main():
parser = argparse.ArgumentParser(description="Manage translations")
parser.add_argument("--source_path", help="Path to source file or directory to scan for translations")
parser.add_argument("--component_path", help="Alias for --source_path (deprecated)", dest="source_path")
parser.add_argument("--yaml_folder", help="Path to the folder containing YAML locale files")
parser.add_argument("--json_folder", help="Path to the folder for output JSON locale files")
parser.add_argument("--yaml_folder_path", help="Path to the folder containing YAML locale files")
parser.add_argument("--json_folder_path", help="Path to the folder for output JSON locale files")
parser.add_argument("--yaml-key-renames-path", help="Path to TSV file mapping original YAML keys to new keys")
parser.add_argument("--migrate", action="store_true", help="Migrate translations from YAML to JSON")
parser.add_argument("--list-in-component", action="store_true", help="List all translation keys in the component file")
parser.add_argument("--clean-yaml", action="store_true", help="Remove keys found in components from YAML files")
parser.add_argument("--list-in-source", action="store_true", help="List all translation keys in the component file")
parser.add_argument("--list-in-json", action="store_true", help="List all translation keys and values from JSON files")
parser.add_argument("--source-as-json", action="store_true", help="Output source translation keys as JSON to stdout")
parser.add_argument("--preview-migrate", metavar="LANG", help="Preview migration for a language code without writing files")
parser.add_argument("--rename-key", nargs=2, metavar=('OLD_KEY', 'NEW_KEY'),
help="Rename a translation key in source and JSON files")

args = parser.parse_args()

if args.migrate and (not args.yaml_folder or not args.json_folder or not args.source_path):
print("Error: --source_path, --yaml_folder, and --json_folder must be specified for migration.")
if args.migrate and (not args.yaml_folder_path or not args.json_folder_path or not args.source_path):
print("Error: --source_path, --yaml_folder_path, and --json_folder_path must be specified for migration.")
parser.print_help()
return


if args.list_in_component and not args.source_path:
print("Error: --source_path must be specified for --list-in-component")
if args.list_in_source and not args.source_path:
print("Error: --source_path must be specified for --list-in-source")
parser.print_help()
return

manager = TranslationManager(args.source_path, args.yaml_folder, args.json_folder)
if args.list_in_json and not args.json_folder_path:
print("Error: --json_folder_path must be specified for --list-in-json")
parser.print_help()
return

manager = TranslationManager(args.source_path, args.yaml_folder_path, args.json_folder_path, args.yaml_key_renames_path)


if args.clean_yaml:
if not args.yaml_folder or not args.source_path:
print("Error: --source_path and --yaml_folder must be specified for cleaning YAML files.")
if args.rename_key:
if not args.source_path or not args.json_folder_path:
print("Error: --source_path and --json_folder_path must be specified for --rename-key")
parser.print_help()
return
manager.clean_yaml_translations()

if args.migrate:
old_key, new_key = args.rename_key
manager.rename_key(old_key, new_key)
elif args.preview_migrate:
if not args.yaml_folder_path or not args.source_path:
print("Error: --source_path and --yaml_folder_path must be specified for --preview-migrate")
parser.print_help()
return
manager.preview_migration(args.preview_migrate)
elif args.migrate:
manager.migrate_translations()

if args.list_in_component:
if args.list_in_source:
manager.list_component_keys()
elif not args.migrate:
print("No action specified. Use --migrate or --list-in-component.")

if args.list_in_json:
manager.list_json_keys()

if args.source_as_json:
if not args.source_path:
print("Error: --source_path must be specified for --source-as-json")
parser.print_help()
return
translation = manager.components_to_translation()
print(SortedJsonEncoder(indent=2).encode(translation.to_nested_dict()))

if not (args.rename_key or args.preview_migrate or args.migrate or args.list_in_source or args.list_in_json or args.source_as_json):
print("No action specified. Use --rename_key {old_key} {new_key} --preview-migrate {language}, --migrate, --list-in-source, --list-in-json, --source-as-json.")
parser.print_help()

if __name__ == "__main__":
15 changes: 14 additions & 1 deletion src/components/activity/ChangesPeriod.jsx
Original file line number Diff line number Diff line change
@@ -10,6 +10,19 @@ import { setAnchorElementId } from '../../redux/activitySlice'
import { viewToString } from '../../utils/appUrl'
import { useIsDesktop } from '../../utils/useBreakpoint'

const formatChangeType = (type, t) => {
switch (type) {
case 'added':
return t('changes.type.added')
case 'edited':
return t('changes.type.edited')
case 'visited':
return t('changes.type.visited')
default:
return `changes.type.${type}`
}
}

const AuthorLink = styled(Link)`
color: ${({ theme }) => theme.blue} !important;
`
@@ -105,7 +118,7 @@ const ActivityTextComponent = ({
return (
<ActivityText>
{t('changes.change_in_city', {
type: t(`changes.type.${interactionType}`),
type: formatChangeType(interactionType, t),
city: '',
})}
{hasLocationInfo ? (
8 changes: 2 additions & 6 deletions src/components/ui/LoadingIndicator.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { transparentize } from 'polished'
import { useTranslation } from 'react-i18next'
import styled from 'styled-components/macro'

import Spinner from './Spinner'
@@ -26,14 +25,11 @@ const LoadingIndicatorWrapper = styled.div`
* - cover {boolean} - covers container with loading state when true
* - vertical {boolean} - arranged vertically when true
*/
const LoadingIndicator = (props) => {
const { t } = useTranslation()
return (
const LoadingIndicator = (props) => (
<LoadingIndicatorWrapper {...props}>
<Spinner /> {t('Loading...')}
<Spinner /> Loading...
</LoadingIndicatorWrapper>
)
}

/**
* LoadingOverlay - masked loading indicator

0 comments on commit fb55d0a

Please sign in to comment.