Chore/merge release

.github/workflows/CI.yml

@@ -34,7 +34,7 @@ jobs:
         run: NDP461-DevPack-KB3105179-ENU.exe /q
 
       - name: Setup dotnet
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: | 
             2.1.x

.github/workflows/CommitMessage.yml

@@ -0,0 +1,52 @@
+name: Commit messages check
+on:
+  pull_request:
+  workflow_call:
+
+jobs:
+  gitlint:
+    name: Check commit messages
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Install dependencies
+      run: |
+        pip install --upgrade gitlint
+    - name: Lint git commit messages
+      shell: bash
+      # run the linter and tee the output to a file, this will make the check fail but allow us to use the results in summary
+      run: gitlint --ignore body-is-missing --commits origin/$GITHUB_BASE_REF..  2>&1 | tee check_results.log
+    - name: Propegate Error Summary
+      if: always()
+      shell: bash
+      # put the output of the commit message linting into the summary for the job and in an environment variable
+      run: |
+        # Change the commit part of the log into a markdown link to the commit
+        commitsUrl="https:\/\/github.com\/${{ github.repository_owner }}\/${{ github.event.repository.name }}\/commit\/"
+        sed -i "s/Commit \([0-9a-f]\{7,40\}\)/[commit \1]($commitsUrl\1)/g" check_results.log
+        # Put the results into the job summary
+        cat check_results.log >> "$GITHUB_STEP_SUMMARY"
+        # Put the results into a multi-line environment variable to use in the next step
+        echo "check_results<<###LINT_DELIMITER###" >> "$GITHUB_ENV"
+        echo "$(cat check_results.log)" >> "$GITHUB_ENV"
+        echo "###LINT_DELIMITER###" >> "$GITHUB_ENV"
+    # add a comment on the PR if the commit message linting failed
+    - name: Comment on PR
+      if: failure()
+      uses: marocchino/sticky-pull-request-comment@v2
+      with:
+        header: Commit Comment
+        message: |
+          ⚠️ Commit Message Format Issues ⚠️
+          ${{ env.check_results }}
+    - name: Clear PR Comment
+      if: success()
+      uses: marocchino/sticky-pull-request-comment@v2
+      with:
+        header: Commit Comment
+        hide: true
+        hide_classify: "RESOLVED"
+

.github/workflows/check-whitespace.yml

@@ -0,0 +1,64 @@
+name: check-whitespace
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+# Avoid unnecessary builds
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-whitespace:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: git log --check
+      id: check_out
+      run: |
+        echo "Starting the script."
+        baseSha=${{ github.event.pull_request.base.sha }}
+        git log --check --pretty=format:"---% h% s" ${baseSha}.. | tee check-results.log
+        problems=()
+        commit=
+        commitText=
+        commitTextmd=
+        # Use git log --check to look for whitespace errors in each commit of this PR
+        log_output=$(cat check-results.log)
+        echo "${log_output}"
+        # Use a for loop to iterate over lines of log_output
+        IFS=$'\n'
+        for line in $log_output; do
+          echo "Line: ${line}"
+          case "${line}" in
+            "--- "*)
+              IFS=' ' read -r _ commit commitText <<< "$line"
+              commitTextmd="[${commit}](https://github.com/${{ github.repository }}/commit/${commit}) ${commitText}"
+              ;;
+            "")
+              ;;
+            *:[1-9]*:*) # contains file and line number information - This indicates that a whitespace error was found
+              file="${line%%:*}"
+              afterFile="${line#*:}" # Remove the first colon and everything before it
+              lineNumber="${afterFile%%:*}"  # Remove anything after and including the first remaining colon to get only the line number
+              problems+=("[${commitTextmd}]")
+              problems+=("[${line}](https://github.com/${{ github.repository }}/blob/${{github.event.pull_request.head.ref}}/${file}#L${lineNumber})")
+              problems+=""
+              ;;
+          esac
+        done
+        if test ${#problems[*]} -gt 0; then
+          echo "⚠️ Please review the Summary output for further information." >> $GITHUB_STEP_SUMMARY
+          echo "### A whitespace issue was found in one or more of the commits." >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Errors:" >> $GITHUB_STEP_SUMMARY
+          for i in "${problems[@]}"; do
+            echo "${i}" >> $GITHUB_STEP_SUMMARY
+          done
+          exit 1
+        fi
+        echo "No problems found"

.gitignore

@@ -29,6 +29,7 @@ Bin/_setLatestBuildConfig.bat
 Bin/_setroot.bat
 Lib/debug/DebugProcs.lib
 Lib/debug/Generic.lib
+Lib/debug/System.ValueTuple.dll
 Lib/debug/unit++.*
 Lib/release/DebugProcs.lib
 Lib/release/Generic.lib

Build/Installer.targets

@@ -101,10 +101,10 @@
 		A base build should warn if we have 'RemovedSinceLastBase' items to help us remember to clear these out.
 	-->
 	<Target Name="RescuePatching">
-		<!-- <ItemGroup>
-			<RemovedSinceLastBase Include="$(dir-outputBase)/Example.dll" />
+		<ItemGroup>
+			<RemovedSinceLastBase Include="$(dir-outputBase)/Helps/WW-ConceptualIntro/ConceptualIntroduction.htm" />
 		</ItemGroup>
-		<WriteLinesToFile File="%(RemovedSinceLastBase.FullPath)" Lines="" /> -->
+		<WriteLinesToFile File="%(RemovedSinceLastBase.FullPath)" Lines="" />
 	</Target>
 	<!-- ########################################################################################################## -->
 	<!-- ### Compile Targets                                                                                    ### -->
@@ -210,6 +210,7 @@
 			<L10nFiles Include="$(OutputDirForConfig)\ur\**\*"/>
 			<L10nFiles Include="$(OutputDirForConfig)\vi\**\*"/>
 			<L10nFiles Include="$(OutputDirForConfig)\zh-CN\**\*"/>
+			<L10nFiles Include="$(OutputDirForConfig)\zlm\**\*"/>
 			<!-- no need to install the following; most are installed by merge modules -->
 			<MergeModules Include="$(OutputDirForConfig)\asserts.log"/>
 			<MergeModules Include="$(OutputDirForConfig)\basicTest.xml"/>
@@ -303,7 +304,8 @@
 		<PropertyGroup>
 			<!-- Substring keeps Chinese the same length as everyone else (and '-' is an illegal character in some places) -->
 			<!-- "The nail that sticks up gets hammered down" (Japanese proverb) -->
-			<SafeTargetLocale>$(TargetLocale.Substring(0,2))</SafeTargetLocale>
+			<SafeTargetLocale Condition="'$(TargetLocale)'=='zh-CN'">$(TargetLocale.Substring(0,2))</SafeTargetLocale>
+			<SafeTargetLocale Condition="'$(TargetLocale)'!='zh-CN'">$(TargetLocale)</SafeTargetLocale>
 			<!-- KeyPathFix changes KeyPath="yes" to KeyPath="no" so we can overwrite later versions of files, if necessary. -->
 			<!-- See https://github.com/sillsdev/genericinstaller's README for details. -->
 			<KeyPathFixArg Condition="'$(FixKeyPath)'=='true'">-t $(InstallerDir)/BaseInstallerBuild/KeyPathFix.xsl</KeyPathFixArg>
@@ -396,7 +398,8 @@
 			Condition="!Exists('$(WixLibsDir)/vcredist_2015-19_x64.exe') And $(Platform)=='x64'"
 			DownloadsDir="$(WixLibsDir)"/>	<!-- VisualC++ 14.1 runtime -->
 		<!-- FLEx Bridge -->
-		<DownloadFile Address="https://build.palaso.org/guestAuth/repository/download/FLExBridgeDevelopWin32InstallerSansPublish/fw-9.1.22.tcbuildtag/FLExBridge_Offline.exe"
+		<DownloadFile Address="https://software.sil.org/downloads/r/fieldworks/FlexBridge_Offline_4.2.0.exe"
+			LocalFilename="FLExBridge_Offline.exe"
 			Condition="!Exists('$(WixLibsDir)/FLExBridge_Offline.exe')"
 			DownloadsDir="$(WixLibsDir)"/> <!-- TODO: always download to get the latest version -->
 	</Target>
@@ -439,12 +442,12 @@
 		</MSBuild.ExtensionPack.Framework.Guid>
 		<Message Text="Generated GUID '$(ProductIdGuid)'"/>
 		<PropertyGroup>
-			<MsiFile>$(SafeApplicationName)_$(Revision).msi</MsiFile>
+			<MsiFile>$(SafeApplicationName)_$(PatchVersionSegment).msi</MsiFile>
 			<BaseBuildDir>$(InstallerDir)/BaseInstallerBuild</BaseBuildDir>
 			<BaseBuildArgs>"$(ApplicationName)" $(SafeApplicationName) $(BuildVersion) $(ProductIdGuid) $(UpgradeCodeGuid) "$(AppBuildDir)/$(BinDirSuffix)" "$(AppBuildDir)/$(DataDirSuffix)" $(CopyrightYear) "$(Manufacturer)" $(SafeManufacturer) $(Arch)</BaseBuildArgs>
 		</PropertyGroup>
 		<!-- Suppress ICE30 because we and Encoding Converters MM both install Geckofx, resulting in an ICE warning for double ref counting -->
-		<Exec WorkingDirectory="$(BaseBuildDir)" Command="(set SuppressICE=-sice:ICE30) &amp; buildBaseInstaller.bat $(BaseBuildArgs)" />
+		<Exec WorkingDirectory="$(BaseBuildDir)" Command="buildBaseInstaller.bat $(BaseBuildArgs)" EnvironmentVariables="SuppressICE=-sice:ICE30" />
 		<ItemGroup>
 			<InstallerFiles Include="$(BaseBuildDir)/**/$(SafeApplicationName)_*.exe"/>
 			<InstallerFiles Include="$(BaseBuildDir)/**/$(SafeApplicationName)_*.msi"/>

Build/Localize.targets

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="Current">
 
+	<UsingTask TaskName="CopyLocale" AssemblyFile="FwBuildTasks.dll"/>
 	<UsingTask TaskName="GoldEticToXliff" AssemblyFile="FwBuildTasks.dll"/>
 	<UsingTask TaskName="ListsToXliff" AssemblyFile="FwBuildTasks.dll"/>
 	<UsingTask TaskName="LocalizeFieldWorks" AssemblyFile="FwBuildTasks.dll"/>
@@ -13,7 +14,7 @@
 
 	<!-- Localize strings-en.xml, lists, and all resx files -->
 	<Target Name="localize" DependsOnTargets="localize-source;localize-binaries"/>
-	<Target Name="localize-source" DependsOnTargets="Initialize;downloadTranslatedFiles;copyLibL10ns;combineGOLDEtic;zipLocalizedLists;processLanguages-source" Condition="'$(DisableLocalization)'!='true'"/>
+	<Target Name="localize-source" DependsOnTargets="Initialize;downloadTranslatedFiles;copyLibL10ns;combineGOLDEtic;zipLocalizedLists;copyLocales;processLanguages-source" Condition="'$(DisableLocalization)'!='true'"/>
 	<Target Name="localize-binaries" DependsOnTargets="Initialize;processLanguages-binaries" Condition="'$(DisableLocalization)'!='true'"/>
 
 	<PropertyGroup>
@@ -34,6 +35,9 @@
 		<!-- Install overcrowdin, or update if already installed. -->
 		<Exec WorkingDirectory="$(fwrt)" Command="dotnet tool update -g overcrowdin || dotnet tool install -g overcrowdin"/>
 	</Target>
+	<Target Name="copyLocales"> <!-- Copy locales to alternate ws identifiers -->
+		<CopyLocale SourceL10n="$(L10nsDirectory)/ms" DestL10n="$(L10nsDirectory)/zlm" LcmDir="$(LcmSrcDir)" />
+	</Target>
 
 	<!-- Update localizable files in Crowdin -->
 	<Target Name="uploadUpdatesForTranslation" DependsOnTargets="CopyLcmResxFiles;InstallOvercrowdin">
@@ -69,7 +73,7 @@
 	<Target Name="downloadTranslatedFiles" Condition="'$(disableDownloads)'!='true'" DependsOnTargets="InstallOvercrowdin">
 		<ForceDelete Files="$(L10nsDirectory)"/>
 		<MakeDir Directories="$(DownloadsDir)"/>
-		<Exec WorkingDirectory="$(fwrt)" Command="overcrowdin download -v -r2 -f $(CrowdinZip)" ContinueOnError="$(OnFailedCrowdinDownload)"/>
+		<Exec WorkingDirectory="$(fwrt)" Command="overcrowdin download -v -f $(CrowdinZip)" ContinueOnError="$(OnFailedCrowdinDownload)"/>
 		<Unzip ZipFilename="$(CrowdinZip)" ToDir="$(L10nsDirectory)"/>
 		<NormalizeLocales L10nsDirectory="$(L10nsDirectory)"/>
 		<ItemGroup>

Build/Src/FwBuildTasks/FwBuildTasks.csproj

@@ -3,14 +3,15 @@
     <RootNamespace>SIL.FieldWorks.Build.Tasks</RootNamespace>
     <Description>Additional msbuild tasks for FieldWorks</Description>
     <Product>FwBuildTasks</Product>
-    <TargetFramework>net461</TargetFramework>
+    <TargetFramework>net462</TargetFramework>
     <OutputPath>../..</OutputPath>
     <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
   </PropertyGroup>
-
   <ItemGroup>
     <PackageReference Include="NUnit" Version="3.13.3" PrivateAssets="All" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.6.0" />
     <PackageReference Include="SIL.TestUtilities" Version="12.0.0-*" PrivateAssets="All" />
+    <Reference Include="netstandard" />
     <Reference Include="Microsoft.Build.Framework" />
     <Reference Include="Microsoft.Build.Utilities.v4.0" />
     <Reference Include="System.IO.Compression" />

Build/Src/FwBuildTasks/FwBuildTasksTests/NormalizeLocalesTests.cs

@@ -1,4 +1,4 @@
-// Copyright (c) 2020 SIL International
+// Copyright (c) 2020 SIL International
 // This software is licensed under the LGPL, version 2.1 or later
 // (http://www.gnu.org/licenses/lgpl-2.1.html)
 
@@ -60,6 +60,19 @@ public void Works()
 			VerifyLocale("zh-CN", "zh");
 		}
 
+		[Test]
+		public void CopyMalay()
+		{
+			FileSystemSetup(new[] { "ms" });
+
+			VerifyLocale("ms", "zlm");
+
+			_task.Execute();
+
+			VerifyLocale("ms", "zzz");
+			VerifyLocale("zlm", "zzz");
+		}
+
 		private void FileSystemSetup(string[] locales)
 		{
 			foreach (var locale in locales)

Build/Src/FwBuildTasks/Localization/CopyLocale.cs

@@ -0,0 +1,105 @@
+// Copyright (c) 2024 SIL International
+// This software is licensed under the LGPL, version 2.1 or later
+// (http://www.gnu.org/licenses/lgpl-2.1.html)
+
+using System;
+using System.IO;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+
+namespace SIL.FieldWorks.Build.Tasks.Localization
+{
+	public class CopyLocale : Task
+	{
+		[Required]
+		public string SourceL10n { get; set; }
+
+		[Required]
+		public string DestL10n { get; set; }
+
+		[Required]
+		public string LcmDir { get; set; }
+
+		public override bool Execute()
+		{
+			var srcLangCode = Path.GetFileName(SourceL10n);
+			var destLangCode = Path.GetFileName(DestL10n);
+			if (!Directory.Exists(SourceL10n))
+			{
+				Log.LogError($"Source directory '{SourceL10n}' does not exist.");
+				return false;
+			}
+			if (Directory.Exists(DestL10n))
+			{
+				Log.LogError($"Destination directory '{DestL10n}' already exists.");
+				return false;
+			}
+			// Create the destination directory
+			Directory.CreateDirectory(DestL10n);
+
+			// Get the files in the source directory and copy to the destination directory
+			CopyDirectory(SourceL10n, DestL10n, true);
+
+			NormalizeLocales.RenameLocaleFiles(DestL10n, srcLangCode, destLangCode);
+			// Get the files in the source directory and copy to the destination directory
+			foreach (var file in Directory.GetFiles(LcmDir, "*.resx", SearchOption.AllDirectories))
+			{
+				var relativePath = GetRelativePath(LcmDir, file);
+				Log.LogMessage(MessageImportance.Normal, "CopyLocale: relpath - " + relativePath);
+				var newFileName = Path.GetFileNameWithoutExtension(file) + $".{destLangCode}.resx";
+				var newFilePath = Path.Combine(DestL10n, Path.Combine("Src", Path.GetDirectoryName(relativePath)));
+
+				// Create the directory for the new file if it doesn't exist
+				Directory.CreateDirectory(newFilePath);
+
+				Log.LogMessage(MessageImportance.Normal, $"CopyLocale: {newFilePath}, {newFileName}");
+				// Copy the file to the new location
+				File.Move(file, Path.Combine(newFilePath, newFileName));
+			}
+
+			return true;
+		}
+
+		static void CopyDirectory(string sourceDir, string destinationDir, bool recursive)
+		{
+			// From: https://learn.microsoft.com/en-us/dotnet/standard/io/how-to-copy-directories
+			// Get information about the source directory
+			var dir = new DirectoryInfo(sourceDir);
+
+			// Check if the source directory exists
+			if (!dir.Exists)
+				throw new DirectoryNotFoundException($"Source directory not found: {dir.FullName}");
+
+			// Cache directories before we start copying
+			DirectoryInfo[] dirs = dir.GetDirectories();
+
+			// Create the destination directory
+			Directory.CreateDirectory(destinationDir);
+
+			// Get the files in the source directory and copy to the destination directory
+			foreach (FileInfo file in dir.GetFiles())
+			{
+				string targetFilePath = Path.Combine(destinationDir, file.Name);
+				file.CopyTo(targetFilePath);
+			}
+
+			// If recursive and copying subdirectories, recursively call this method
+			if (recursive)
+			{
+				foreach (DirectoryInfo subDir in dirs)
+				{
+					string newDestinationDir = Path.Combine(destinationDir, subDir.Name);
+					CopyDirectory(subDir.FullName, newDestinationDir, true);
+				}
+			}
+		}
+
+		static string GetRelativePath(string baseDir, string filePath)
+		{
+			Uri baseUri = new Uri(baseDir);
+			Uri fileUri = new Uri(filePath);
+			return Uri.UnescapeDataString(baseUri.MakeRelativeUri(fileUri).ToString().Replace('/', Path.DirectorySeparatorChar));
+		}
+	}
+}