Merge pull request #2147 from weather-gov/sspj/1579-chinese-dynamic-l…

…ocalization

Dynamic Chinese translations

scripts/post-deploy.sh

@@ -23,6 +23,7 @@ echo  "Updating drupal ... "
 drush state:set system.maintenance_mode 1 -y
 drush deploy -y
 drush state:set system.maintenance_mode 0 -y
+drush locale:import-all /home/vcap/app/web/modules/weather_i18n/translations
 drush locale:clear-status
 drush locale:update
 drush cache:rebuild

tests/translations/config.js

@@ -34,4 +34,15 @@ module.exports = {
     include: ["../../web/modules/weather_i18n/translations/*.po"],
     exclude: [],
   },
+
+  suppress: {
+    missing: {
+      en: [],
+      "zh-hans": [],
+    },
+    stale: {
+      en: [],
+      "zh-hans": [],
+    }
+  },
 };

tests/translations/gettextExtraction.js

@@ -63,6 +63,8 @@ const parseGettextSource = (str) => {
  * For a given list of translation file paths,
  * respond with a dictionary mapping filenames
  * to match information for the gettext values
+ *
+ * @type (sourcePaths: string) => Object<string, Object<string, {comments: string, msgid: string, msgstr: string, msgidString: string, msgstrString: string}>>
  */
 const getTranslationMatchInfo = (sourcePaths) => {
   const lookup = {};

tests/translations/main.js

@@ -4,9 +4,13 @@ const { getFileMatchInfo } = require("./translationExtraction");
 const { getTranslationMatchInfo } = require("./gettextExtraction");
 const config = require("./config.js");
 
-/**
- * Get all of the template and php paths as flat arrays
- */
+const RED_ERROR = "\x1b[31mError:\x1b[0m";
+const YELLOW_WARNING = "\x1b[33mWarning:\x1b[0m";
+const GREEN_SUCCESS = "\x1b[0;32mGood:\x1b[0m";
+
+// Parse config file to get all of the template and php paths as flat arrays
+
+/** Twig template locations. Defined in config.js. @type string[] */
 const templatePaths = config.templates.include
   .reduce((prev, current) => {
     const relativeGlob = path.resolve(__dirname, current);
@@ -17,6 +21,7 @@ const templatePaths = config.templates.include
     const fileName = path.basename(filePath);
     return !config.templates.exclude.includes(fileName);
   });
+/** Translatable PHP file locations. Defined in config.js. @type string[] */
 const phpPaths = config.php.include
   .reduce((prev, current) => {
     const relativeGlob = path.resolve(__dirname, current);
@@ -27,6 +32,7 @@ const phpPaths = config.php.include
     const fileName = path.basename(filePath);
     return !config.php.exclude.includes(fileName);
   });
+/** Translation file (eg. `fr.po`) locations. Defined in config.js. @type string[] */
 const translationPaths = config.translations.include
   .reduce((prev, current) => {
     const relativeGlob = path.resolve(__dirname, current);
@@ -38,52 +44,97 @@ const translationPaths = config.translations.include
     return !config.translations.exclude.includes(fileName);
   });
 
+/** Map of translation keys (message ids) to info about their occurrence in the code. */
 const templateLookup = getFileMatchInfo(templatePaths, phpPaths);
+/** Map of language codes to dictionaries of their translation keys and strings. */
 const translationLookup = getTranslationMatchInfo(translationPaths);
+/** Array of language filenames, like `fr.po`. */
 const languages = Object.keys(translationLookup);
-const errorsSummary = [];
 
-languages.forEach((langCode) => {
-  console.log(`Checking translation integrity for: ${langCode}`);
-  // First get any translations defined in templates
-  // that are missing from the translation file for this
-  // language
-  const translations = translationLookup[langCode];
-  const translationTerms = Object.keys(translations);
-  const templateTerms = Object.keys(templateLookup);
-
-  const fileNames = new Set();
-  templateTerms.forEach((key) => {
-    templateLookup[key].forEach((phrase) => {
-      fileNames.add(phrase.filename);
-    });
+/** Any problems worth stopping a CI run? */
+let hasErrors = false;
+
+// Check the English translation file for missing or stale translation keys
+
+/** Map of English translation keys (msgid) to msgstr, comments, etc. */
+const english = translationLookup['en'];
+/** Array of English translation keys. */
+const translationKeys = Object.keys(english);
+/** Array of strings marked for translation in Twig and PHP files. */
+const translatable = Object.keys(templateLookup);
+
+const ignore = new Set(config.suppress.missing.en ?? []);
+const missing = translatable.filter((key) => !translationKeys.includes(key)).filter((key) => !ignore.has(key));
+
+if (missing.length) {
+  hasErrors = true;
+  console.error(
+    `${RED_ERROR} ${missing.length} strings are marked for translation but have no msgid in en.po`
+  );
+  missing.forEach((key) => {
+    const entryList = templateLookup[key];
+    if (entryList) {
+      console.error(`\t"${entryList[0].extracted}"`);
+      entryList.map(
+        (entry) => console.error(`\t\t${entry.filename}:${entry.lineNumber}`)
+      );
+    }
   });
+} else {
+  console.log(`${GREEN_SUCCESS} No English translations are missing from en.po.`);
+  console.log("Caution: This test won't help if you forget to mark a string for translation.");
+}
 
-  console.log("Checking for missing translations");
-  const missing = templateTerms.filter((key) => !translationTerms.includes(key));
+const ignored = new Set(config.suppress.stale.en ?? []);
+const stale = translationKeys.filter((key) => !translatable.includes(key)).filter((key) => !ignored.has(key));
 
+if (stale.length) {
+  console.warn(
+    `${YELLOW_WARNING} ${stale.length} msgids are in en.po but do not seem to be used anywhere`
+  );
+  stale.forEach((key) => {
+    console.warn(`\t"${key}"`);
+  });
+} else {
+  console.log(`${GREEN_SUCCESS} All English translations in en.po are used in Twig or PHP`);
+}
+
+// Check each language for parity with the English translation file
+
+languages.forEach((langCode) => {
+  if (langCode === 'en') return;  // we already did this one
+
+  const keyLookup = translationLookup[langCode];
+  const comparisonKeys = Object.keys(keyLookup);
+  const ignore = new Set(config.suppress.missing[langCode] ?? []);
+  /** Translation keys in English which are missing or not translated in `langCode`. */
+  const missing = translationKeys.filter(
+    (key) => (!comparisonKeys.includes(key) || !keyLookup[key]?.msgstr)
+  ).filter((key) => !ignore.has(key));
   if (missing.length) {
-    const errString = `Missing [${missing.length}] translations in the ${langCode} translations file`;
-    errorsSummary.push(errString);
-    console.error(`${errString  }:`);
-    missing.forEach((key) => {
-      const entryList = templateLookup[key];
-      if (entryList) {
-        const fileLocations = entryList.map((entry) => `${entry.filename}:${entry.lineNumber}`);
-        const serialized = JSON.stringify(entryList, null, 2);
-        console.error(`${fileLocations.join("\n")}\n${serialized}`);
-      }
-    });
-    process.exit(-1);
+    hasErrors = true;
+    console.error(
+      `${RED_ERROR} ${missing.length} strings from en.po are not in ${langCode}.po`
+    );
+    missing.forEach((key) => console.error(`\t"${key}"`));
+  } else {
+    console.log(`${GREEN_SUCCESS} All strings from en.po are in ${langCode}.po.`);
   }
 
-  console.log("Checking for stale translations");
-  const stale = translationTerms.filter((key) => !templateTerms.includes(key));
-
+  const ignored = new Set(config.suppress.stale[langCode] ?? []);
+  /** Translation keys in `langCode` which are missing in English. */
+  const stale = comparisonKeys.filter((key) => !translationKeys.includes(key)).filter((key) => !ignored.has(key));
   if (stale.length) {
     console.warn(
-      `Found ${stale.length} potentially stale translations in the ${langCode} file`,
+      `${YELLOW_WARNING} ${missing.length} strings from ${langCode}.po are not in en.po`
     );
-    console.log(stale);
+    missing.forEach((key) => console.warn(`\t"${key}"`));
+  } else {
+    console.log(`${GREEN_SUCCESS} All strings from ${langCode}.po are in en.po.`);
   }
 });
+
+if (hasErrors) {
+  console.warn("Hint: You can add keys to config.suppress.missing or config.suppress.stale to suppress test failures.");
+  process.exit(-1);
+}

tests/translations/tests/fixtures/t.filter.html.twig

@@ -1,11 +1,11 @@
             {% if period.probabilityOfPrecipitation and period.probabilityOfPrecipitation > 1 %}
             <p class="text-gray-50 font-body-xs margin-top-05 margin-bottom-0">
-            {{period.probabilityOfPrecipitation}}% {{ "chance of precipitation" | t }}
+            {{period.probabilityOfPrecipitation}}% {{ "daily-forecast.text.chance-precip.01" | t }}
             </p>
             {% endif %}
 
 <div class="margin-top-05 position-relative">
-  <div class="font-mono-2xs font-family-mono text-base text-uppercase">{{ "Feels like" | t }}</div>
+  <div class="font-mono-2xs font-family-mono text-base text-uppercase">{{ "forecast.current.feels-like.01" | t }}</div>
   <div class="text-primary-dark">
     <p class="margin-top-2px font-body-md">
       {{ content.feels_like }}<span class="font-body-3xs position-absolute margin-top">&deg;F</span>

tests/translations/tests/fixtures/t.variable.twig

@@ -1,9 +1,9 @@
 {% if content.error %}
-    {% set message = "There was an error loading the current conditions." | t %}
+    {% set message = "forecast.errors.current-conditions.01" | t %}
     {% include '@new_weather_theme/partials/uswds-alert.html.twig' with { 'level': "error", body: message } %}
 {% endif %}
 {%  if period.isOvernight %}
-    {% set label = "Overnight" | t %}
+    {% set label = "daily-forecast.labels.overnight.01" | t %}
     {% set lineColor = "midnight-indigo" %}
     {% set tabletCol = 12 %}
     {% set tabletMarginUnit = 3 %}

tests/translations/translationExtraction.js

@@ -48,9 +48,10 @@ const matchTranslationFilters = (source) => {
 };
 
 /**
- * For a given PHP file, extract all of the translation matches
- * and return information about the match line number
- * and matched / extracted strings
+ * For a given PHP file, extract all of the translatable strings.
+ *
+ * @type (filePath: string) => {filename: string, matchedString: string,
+ * extracted: string, extractedArgs: ?string, lineNumber: number, index: number}[]
  */
 const extractPHPTranslations = (filePath) => {
   const source = fs.readFileSync(filePath).toString();
@@ -65,6 +66,7 @@ const extractPHPTranslations = (filePath) => {
           extracted: match[1],
           extractedArgs: match[3] | null,
           lineNumber: getLineNumberForPosition(source, match.index),
+          index: match.index,
         })),
     );
   }
@@ -73,9 +75,9 @@ const extractPHPTranslations = (filePath) => {
 };
 
 /**
- * For a given template file, extract all of the
- * translation matches and return information about
- * the match line number and string
+ * For a given template file, extract all of the translatable strings.
+ *
+ * @type (filePath: string) => {filename: string, matchedString: string, extracted: string, extractedArgs: ?string, lineNumber: number, index: number}[]
  */
 const extractTemplateTranslations = (filePath) => {
   const source = fs.readFileSync(filePath).toString();
@@ -135,10 +137,10 @@ const getLineNumberForPosition = (source, position) => {
 };
 
 /**
- * Appends the value to the lookup dictionary's
- * key. Because keys map to arrays, if there is not
- * yet an entry for the key, it creates the initial array
- * value and sets the passed-in value as the first element.
+ * Maps an occurrence of a translation to its translation key.
+ *
+ * Keys map to arrays because a translation can occur in multiple places,
+ * so if there is not yet an entry for the key, it creates the initial array.
  */
 const appendToLookup = (lookup, key, val) => {
   if (!Object.keys(lookup).includes(key)) {
@@ -149,9 +151,9 @@ const appendToLookup = (lookup, key, val) => {
 };
 
 /**
- * Given a source path of templates, return a lookup
- * dictionary that maps string to be translated to
- * arrays of match information.
+ * Extract translation keys (`msgid`s) from files.
+ *
+ * @type (templatePaths: string[], phpPaths: string[]) => Object<string, {filename: string, matchedString: string, extracted: string, extractedArgs: ?string, lineNumber: number, index: number}[]>
  */
 const getFileMatchInfo = (templatePaths, phpPaths) => {
   const lookupByTerm = {};