Move UI translations from content packs to repository

kalite/distributed/management/commands/contentpackchecker.py

@@ -10,7 +10,7 @@
 
 from kalite.topic_tools.content_models import Item, set_database, parse_data, AssessmentItem
 
-from kalite.i18n.base import get_locale_path, get_subtitle_file_path as get_subtitle_path, \
+from kalite.i18n.base import get_metadata_path, get_subtitle_file_path as get_subtitle_path, \
     get_installed_language_packs
 
 logging = django_settings.LOG
@@ -94,7 +94,7 @@ def count_assessment_items(self, **kwargs):
         return AssessmentItem.select().count()
 
     def get_content_pack_metadata(self, language):
-        metadata_path = os.path.join(get_locale_path(language), "{language}_metadata.json".format(language=language))
+        metadata_path = os.path.join(get_metadata_path(), "{language}_metadata.json".format(language=language))
 
         with open(metadata_path) as f:
             metadata = json.load(f)

kalite/distributed/management/commands/retrievecontentpack.py

@@ -13,8 +13,8 @@
 from fle_utils.general import ensure_dir
 
 from kalite.contentload import settings as content_settings
-from kalite.i18n.base import lcode_to_django_lang, get_po_filepath, get_locale_path, \
-    download_content_pack, update_jsi18n_file, get_subtitle_file_path as get_subtitle_path, \
+from kalite.i18n.base import lcode_to_django_lang, get_po_filepath, get_metadata_path, \
+    download_content_pack, get_subtitle_file_path as get_subtitle_path, \
     extract_content_db
 from kalite.topic_tools import settings as topic_settings
 from kalite.updates.management.utils import UpdatesStaticCommand
@@ -161,8 +161,6 @@ def local(self, *args, **options):
     def process_content_pack(self, zf, lang):
 
         self.next_stage(_("Moving content files to the right place."))
-        extract_catalog_files(zf, lang)
-        update_jsi18n_file(lang)
         extract_content_db(zf, lang, is_template=self.is_template)
         extract_subtitles(zf, lang)
         extract_content_pack_metadata(zf, lang)  # always extract to the en lang
@@ -176,27 +174,17 @@ def process_content_pack(self, zf, lang):
 
 
 def extract_content_pack_metadata(zf, lang):
-    metadata_path = os.path.join(get_locale_path(lang), "{lang}_metadata.json".format(lang=lang))
-    pack_metadata_name = "metadata.json"
-
-    with open(metadata_path, "wb") as f, zf.open(pack_metadata_name) as mf:
-        shutil.copyfileobj(mf, f)
+    lang = lcode_to_django_lang(lang)        
+    metadata_path = get_metadata_path(lang)
 
+    if not os.path.exists(metadata_path):
+        os.makedirs(metadata_path)
 
-def extract_catalog_files(zf, lang):
-    lang = lcode_to_django_lang(lang)
-    modir = get_po_filepath(lang)
-    ensure_dir(modir)
-
-    filename_mapping = {"frontend.mo": "djangojs.mo",
-                        "backend.mo": "django.mo"}
+    metadata_file = os.path.join(metadata_path, "{lang}_metadata.json".format(lang=lang))
+    pack_metadata_name = "metadata.json"
 
-    for zipmo, djangomo in filename_mapping.items():
-        zipmof = zf.open(zipmo)
-        mopath = os.path.join(modir, djangomo)
-        logging.debug("writing to %s" % mopath)
-        with open(mopath, "wb") as djangomof:
-            shutil.copyfileobj(zipmof, djangomof)
+    with open(metadata_file, "wb") as f, zf.open(pack_metadata_name) as mf:
+        shutil.copyfileobj(mf, f)
 
 
 def extract_subtitles(zf, lang):

kalite/i18n/base.py

@@ -41,15 +41,20 @@ def get_locale_path(lang_code=None):
     if none is provided."""
 
     if not lang_code:
-        return settings.USER_WRITABLE_LOCALE_DIR
+        return settings.KALITE_LOCALE_DIR
     else:
-        return os.path.join(settings.USER_WRITABLE_LOCALE_DIR, lcode_to_django_dir(lang_code))
+        return os.path.join(settings.KALITE_LOCALE_DIR, lcode_to_django_dir(lang_code))
 
 def get_po_filepath(lang_code, filename=None):
     """Return the LC_MESSAGES directory for the language code, with an optional filename appended."""
     base_dirpath = os.path.join(get_locale_path(lang_code=lang_code), "LC_MESSAGES")
     return (filename and os.path.join(base_dirpath, filename)) or base_dirpath
 
+def get_metadata_path(lang_code=None):
+    if not lang_code:
+        return settings.CONTENT_PACK_METADATA_DIR
+
+    return os.path.join(settings.CONTENT_PACK_METADATA_DIR, lcode_to_django_dir(lang_code))
 
 LANG2CODE_MAP = None
 CACHE_VARS.append("LANG2CODE_MAP")
@@ -208,36 +213,30 @@ def _get_installed_language_packs():
         'native_name': 'English',
     }]
 
-    # Loop through locale folders
-    for locale_dir in settings.LOCALE_PATHS:
-        if not os.path.exists(locale_dir):
-            continue
 
-        # Loop through folders in each locale dir
-        # This is idiotic, it just assumes that every directory / file is
-        # a valid language code
-        for django_disk_code in os.listdir(locale_dir):
-
-            # Skip if it's a file
-            if not os.path.isdir(os.path.join(locale_dir, django_disk_code)):
-                continue
-
-            # Inside each folder, read from the JSON file - language name, % UI trans, version number
-            try:
-                # Get the metadata
-                metadata_filepath = os.path.join(locale_dir, django_disk_code, "%s_metadata.json" % lcode_to_ietf(django_disk_code))
-                lang_meta = softload_json(metadata_filepath, raises=True)
-
-                logging.debug("Found language pack %s" % (django_disk_code))
-            except IOError as e:
-                if e.errno == errno.ENOENT:
-                    logging.info("Ignoring non-language pack %s in %s" % (django_disk_code, locale_dir))
-                else:
-                    logging.error("Error reading %s metadata (%s): %s" % (django_disk_code, metadata_filepath, e))
-                continue
+    # Loop through folders in each locale dir
+    # This is idiotic, it just assumes that every directory / file is
+    # a valid language code
+    for django_disk_code in os.listdir(settings.CONTENT_PACK_METADATA_DIR):
+        if not os.path.isdir(os.path.join(get_metadata_path(django_disk_code))):
+            continue
 
-            installed_language_packs.append(lang_meta)
+        # Inside each folder, read from the JSON file - language name, % UI trans, version number
+        try:
+            # Get the metadata
+            metadata_filepath = os.path.join(get_metadata_path(django_disk_code), "%s_metadata.json" % lcode_to_ietf(django_disk_code))
+            lang_meta = softload_json(metadata_filepath, raises=True)
+
+            logging.debug("Found language pack %s" % (django_disk_code))
+        except IOError as e:
+            if e.errno == errno.ENOENT:
+                logging.info("Ignoring non-language pack %s in %s" % (django_disk_code, locale_dir))
+            else:
+                logging.error("Error reading %s metadata (%s): %s" % (django_disk_code, metadata_filepath, e))
+            continue
 
+        installed_language_packs.append(lang_meta)
+
     sorted_list = sorted(installed_language_packs, key=lambda m: m['name'].lower())
     return OrderedDict([(lcode_to_ietf(val["code"]), val) for val in sorted_list])
 

kalite/i18n/management/commands/makemessages.py

@@ -35,6 +35,7 @@ def handle_noargs(self, *args, **options):
         domain = options.get('domain', "django")
         django_po = os.path.join(
             os.getcwd(),
+            "project",
             "locale",
             "en",
             "LC_MESSAGES",

kalite/i18n/management/commands/setup_in_context_translations.py

@@ -47,7 +47,7 @@ def handle_noargs(self, **kwargs):
         zf_po_file_name = "versioned/{version}-django.po".format(version=SHORTVERSION)
         zf_js_po_file_name = "versioned/{version}-djangojs.po".format(version=SHORTVERSION)
 
-        po_file_dir = os.path.join(settings.USER_WRITABLE_LOCALE_DIR, "en", "LC_MESSAGES")
+        po_file_dir = os.path.join(settings.KALITE_LOCALE_DIR, "en", "LC_MESSAGES")
         ensure_dir(po_file_dir)
 
         zf_po_file = zf.read(zf_po_file_name)

kalite/i18n/tests/makemessages_tests.py

@@ -21,14 +21,14 @@ def test_makemessages(self):
         kalite_dir = os.path.join(d, "kalite")
         os.mkdir(kalite_dir)
         os.chdir(kalite_dir)
-        os.mkdir(os.path.join(kalite_dir, "locale"))
         os.mkdir(os.path.join(kalite_dir, "templates"))
+        os.makedirs(os.path.join(kalite_dir, "project", "locale"))
         f = open(os.path.join(kalite_dir, "templates", "test.html"), "w")
         f.write("""{% trans "Something for translation" %}""")
         f.close()
         try:
             call_command("makemessages", locale="en")
-            f = open(os.path.join(kalite_dir, "locale", "en", "LC_MESSAGES", "django.po"), "r")
+            f = open(os.path.join(kalite_dir, "project", "locale", "en", "LC_MESSAGES", "django.po"), "r")
             self.assertIn("Something", f.read())
         finally:
             os.chdir(cwd)

kalite/legacy/i18n_settings.py

@@ -50,14 +50,14 @@ def allow_all_languages_alist(langlookupfile):
 # Use resource_filename instead of get_data because it does not try to open
 # a file and does not complain that its a directory
 
-from kalite.settings.base import USER_WRITABLE_LOCALE_DIR
+from kalite.settings.base import CONTENT_PACK_METADATA_DIR
 
 from pkg_resources import resource_filename
 I18N_DATA_PATH = resource_filename("kalite", "i18n/data")
 LANG_LOOKUP_FILEPATH = os.path.join(I18N_DATA_PATH, "languagelookup.json")
 
 __dubbed_video_mapping_source = os.path.join(I18N_DATA_PATH, "dubbed_video_mappings.json")
-DUBBED_VIDEOS_MAPPING_FILEPATH = os.path.join(USER_WRITABLE_LOCALE_DIR, "dubbed_video_mappings.json")
+DUBBED_VIDEOS_MAPPING_FILEPATH = os.path.join(CONTENT_PACK_METADATA_DIR, "dubbed_video_mappings.json")
 
 # If user does not have this file in their own directory, create a new one as a
 # copy of the distributed one since user processes may write to it

kalite/packages/bundled/django/core/management/commands/makemessages.py

@@ -295,8 +295,8 @@ def make_messages(locale=None, domain='django', verbosity=1, all=False,
         invoked_for_django = True
         # Ignoring all contrib apps
         ignore_patterns += ['contrib/*']
-    elif os.path.isdir('locale'):
-        localedir = os.path.abspath('locale')
+    elif os.path.isdir('project'):
+        localedir = os.path.join(os.path.abspath('project'), 'locale')
     else:
         raise CommandError("This script should be run from the Django Git "
                 "tree or your project or app tree. If you did indeed run it "

kalite/settings/base.py

@@ -188,14 +188,17 @@
 # the user running kalite and should be in a user-data
 # storage place.
 
-USER_WRITABLE_LOCALE_DIR = os.path.join(USER_DATA_ROOT, 'locale')
-KALITE_APP_LOCALE_DIR = os.path.join(USER_DATA_ROOT, 'locale')
+KALITE_LOCALE_DIR = os.path.join(PACKAGE_PATH, "project", "locale")
+if not os.path.exists(KALITE_LOCALE_DIR):
+    os.mkdir(KALITE_LOCALE_DIR)
 
-LOCALE_PATHS = (USER_WRITABLE_LOCALE_DIR, KALITE_APP_LOCALE_DIR)
-if not os.path.exists(USER_WRITABLE_LOCALE_DIR):
-    os.mkdir(USER_WRITABLE_LOCALE_DIR)
+CONTENT_PACK_METADATA_DIR = os.path.join(USER_DATA_ROOT, "locale")
+if not os.path.exists(CONTENT_PACK_METADATA_DIR):
+    os.mkdir(CONTENT_PACK_METADATA_DIR)
 
+LOCALE_PATHS = (KALITE_LOCALE_DIR,)
 DEFAULT_DATABASE_DIR = os.path.join(USER_DATA_ROOT, "database",)
+
 if not os.path.exists(DEFAULT_DATABASE_DIR):
     os.mkdir(DEFAULT_DATABASE_DIR)