From 8b38955a2d69d46726f0f310e1a6e2ceb4a490cb Mon Sep 17 00:00:00 2001
From: past-due <30942300+past-due@users.noreply.github.com>
Date: Thu, 27 Jun 2024 14:19:06 -0400
Subject: [PATCH] Update scripts

---
 .ci/sync_translations_to_master.sh            | 36 ++++++++++++-------
 .../workflows/sync_translations_to_master.yml |  2 ++
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/.ci/sync_translations_to_master.sh b/.ci/sync_translations_to_master.sh
index e594a0813..78628ffd8 100755
--- a/.ci/sync_translations_to_master.sh
+++ b/.ci/sync_translations_to_master.sh
@@ -37,6 +37,12 @@ fi
 
 cd "${working_dir}"
 
+split_script="${master_branch_dir}/.ci/split_front_matter_from_markdown.py"
+if [ ! -f "${split_script}" ]; then
+  echo "Missing expected script: ${split_script}"
+  exit 1
+fi
+
 ###################
 # Create list of content markdown files
 if [ ! -d "${master_branch_dir}/content/${source_lang}" ]; then
@@ -46,15 +52,16 @@ fi
 cd "${master_branch_dir}/content/${source_lang}"
 find . -name '*.md' > "${working_dir}/mdcontent.dat"
 # Extract base content front-matter
-mkdir -p "${working_dir}/base_frontmatter"
+mkdir -p "${working_dir}/baselang_split"
 while read file
 do
   if [ -f "${file}" ]; then
     echo "- Extracting base front-matter: ${file}"
-    output_file="${working_dir}/base_frontmatter/${file}.base_frontmatter"
+    output_file="${working_dir}/baselang_split/${file}.frontmatter"
     destination_path=$(dirname "${output_file}")
     mkdir -p "${destination_path}"
-    cat "${file}" | sed '/^---/,/^---/!d;//d' > "${output_file}"
+    # Split original file into front matter and content
+    python3 "${split_script}" "${file}" "${destination_path}"
   fi
 done < "${working_dir}/mdcontent.dat"
 cd "${working_dir}"
@@ -80,11 +87,7 @@ cd "${working_dir}"
 
 ###################
 # Convert translated markdown files
-split_script="${master_branch_dir}/.ci/split_front_matter_from_markdown.py"
-if [ ! -f "${split_script}" ]; then
-  echo "Missing expected script: ${split_script}"
-  exit 1
-fi
+
 # in each language content directory
 base_content_dir="${translations_branch_dir}/content"
 cd "${base_content_dir}"
@@ -111,13 +114,21 @@ find * -prune -type d | while IFS= read -r d; do
         if [ -s "${file}.frontmatter" ]; then
           # Use yq to merge translated front-matter back into full base file front-matter
           printf "\t%s\n" "+ Merging front-matter"
-          yq eval-all 'select(fileIndex == 0) *? select(fileIndex == 1)' "${working_dir}/base_frontmatter/${file}.base_frontmatter" "${file}.frontmatter" > "${file}.frontmatter.merged"
+          yq eval-all 'select(fileIndex == 0) *? select(fileIndex == 1)' "${working_dir}/baselang_split/${file}.frontmatter" "${file}.frontmatter" > "${file}.frontmatter.merged"
         else
           # Just use base front matter
           printf "\t%s\n" "+ Using base front-matter"
-          cp "${working_dir}/base_frontmatter/${file}.base_frontmatter" "${file}.frontmatter.merged"
+          cp "${working_dir}/baselang_split/${file}.frontmatter" "${file}.frontmatter.merged"
         fi
 
+        # Diff the frontmatter
+        dyff between --omit-header --set-exit-code --ignore-order-changes "${working_dir}/baselang_split/${file}.frontmatter" "${file}.frontmatter.merged"
+        FRONTMATTER_DIFF_RESULT=$?
+
+        # Diff the markdown content
+        diff --ignore-blank-lines --ignore-space-change -q "${file}.markdown" "${working_dir}/baselang_split/${file}.markdown" >/dev/null
+        MARKDOWN_DIFF_RESULT=$?
+
         # Concatenate the merged front-matter with the translated content
         echo "---" > "${file}.merged"
         cat "${file}.frontmatter.merged" >> "${file}.merged"
@@ -132,8 +143,7 @@ find * -prune -type d | while IFS= read -r d; do
         # Move the merged translated file to the master branch - if it's different from the base file
         destination_file="${master_branch_lang_folder}/${file}"
         destination_path=$(dirname "${destination_file}")
-        diff --ignore-blank-lines --ignore-space-change -q "${file}.merged" "${master_branch_dir}/content/${source_lang}/${file}" >/dev/null
-        if [ $? -ne 0 ]; then
+        if [ $FRONTMATTER_DIFF_RESULT -ne 0 ] || [ $MARKDOWN_DIFF_RESULT -ne 0 ]; then
           mkdir -p "${destination_path}"
           printf "\t%s\n" "+ [Saved] translated file"
           mv "${file}.merged" "${destination_file}"
@@ -168,7 +178,7 @@ cd "${working_dir}"
 
 ###################
 # Cleanup base front-matter / tmp files
-rm -rf "${working_dir}/base_frontmatter"
+rm -rf "${working_dir}/baselang_split"
 rm "${working_dir}/mdcontent.dat"
 
 ###################
diff --git a/.github/workflows/sync_translations_to_master.yml b/.github/workflows/sync_translations_to_master.yml
index 2487e4eb0..c77a10899 100644
--- a/.github/workflows/sync_translations_to_master.yml
+++ b/.github/workflows/sync_translations_to_master.yml
@@ -24,6 +24,8 @@ jobs:
         run: |
           sudo snap install yq --channel=v4/stable
           yq --version
+      - name: Install Dyff
+        run: . ./master/.ci/install_dyff.sh
       - name: 'Sync translation changes to master branch'
         run: ./translations/.ci/sync_translations_to_master.sh "${GITHUB_WORKSPACE}/translations" "${GITHUB_WORKSPACE}/master"
       - name: Commit changes to master branch