Workflow per binary for better status info

.github/generate_workflows.csx

@@ -0,0 +1,16 @@
+var packages = File.ReadAllLines("packages.txt");
+var orig_cont = File.ReadAllText("tool_build_template.yml.tmpl");
+foreach (var package in packages){
+    var arr = package.Trim().Split(new []{":"},StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+    var pkg_name = arr[0];
+    if (String.IsNullOrWhiteSpace(pkg_name))
+    	continue;
+    var deps ="";
+    if (arr.Length > 1){
+    	var dep_arr = arr[1].Split(new []{" "},StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+        var tabs="        ";
+		deps = $"RequiredDeps: |\n{tabs}" + string.Join("\n" + tabs,dep_arr);
+	}
+    var cont = orig_cont.Replace("[NAME]",pkg_name).Replace("[DEPS]",deps);
+    File.WriteAllText($"workflows/tool_{pkg_name}_build.yml", cont);
+}

.github/move_dl_artifacts.ps1

@@ -3,6 +3,16 @@ $ErrorActionPreference = "Stop";
 $VerbosePreference="Continue";
 
 
+$files = (Get-ChildItem -Path d:/artifacts/*.zip -Force -ErrorAction SilentlyContinue)
+foreach ($file in $files){
+	$folder = $file.BaseName.replace("WLB-","")
+	$srcPath = "d:/artifacts/$($folder)"
+	if ( -not (Test-Path -Path $srcPath)  ){
+		Expand-Archive -Path $file.FullName -DestinationPath $srcPath
+		Write-Host "Extracting $folder"
+	}
+}
+
 $folders = (Get-ChildItem -Path d:/artifacts/* -Directory -Force -ErrorAction SilentlyContinue)
 
 foreach ($folder in $folders) {

.github/packages.txt

@@ -0,0 +1,25 @@
+automake
+awk
+coreutils
+diffutils
+findutils
+gawk
+grep: pcre2
+gzip
+highlight
+libhsts
+libpsl
+make
+openssl
+patch
+pcre2
+pdcurses
+sed
+symlinks
+tar
+wget: pcre2 libpsl zlib
+wget2: pcre2 libpsl zlib libhsts wolfcrypt
+which
+wolfcrypt
+zlib
+zstd

.github/set_dep_vars.ps1

@@ -0,0 +1,12 @@
+Set-StrictMode -version latest;
+$ErrorActionPreference = "Stop";
+$VerbosePreference="Continue";
+
+$arr=($env:DEPS).split()
+$cnt=1
+foreach ($dep in $arr) {
+	if ($dep){
+		echo "Dep$($cnt)Name=$dep" >> $env:GITHUB_OUTPUT
+		$cnt++		
+	}
+}

.github/tool_build_template.yml.tmpl

@@ -0,0 +1,14 @@
+name: [NAME] Tool Build
+on:
+  push:
+    branches-ignore:
+      - trash
+  schedule:
+    - cron: '30 5 * * *'
+
+jobs:
+  build:
+    uses: ./.github/workflows/tool_build.yml
+    with:
+      ToolName: [NAME]
+      [DEPS]

.github/workflows/do_build.yml

@@ -1,4 +1,4 @@
-name: do_build
+name: __incl_primary_build_script
 on:
   workflow_call:
     inputs:
@@ -80,18 +80,25 @@ jobs:
 
 
 
+#previously we built all in one workflow now with them as individual workflows we download differently
 
-
-      - name: Download Required Artifacts
-        uses: mitchcapper/action-download-artifact@c026c9be4097d153332e6cd0974d140ba22cd6cc
+      # - name: Download Required Artifacts
+      #   uses: mitchcapper/action-download-artifact@c026c9be4097d153332e6cd0974d140ba22cd6cc
+      #   if: ${{ inputs.RequiredDeps != '' && inputs.BuildPkg != 'script' }}
+      #   with:
+      #     name: ${{ inputs.RequiredDeps }}
+      #     path: d:/artifacts
+      #     name_prefix: WLB-
+      #     run_id: ${{ github.run_id }}
+      #     workflow_conclusion: "success"
+
+      - name: Restore Existing Dep Cache
         if: ${{ inputs.RequiredDeps != '' && inputs.BuildPkg != 'script' }}
+        uses: actions/cache/restore@v3
         with:
-          name: ${{ inputs.RequiredDeps }}
-          path: d:/artifacts
-          name_prefix: WLB-
-          run_id: ${{ github.run_id }}
-          workflow_conclusion: ""
-
+          path: |
+            d:/artifacts
+          key: ${{ inputs.BuildPkg }}
 
 
       - name: MSBuild Setup
@@ -124,7 +131,7 @@ jobs:
 #          find /c/ > d:/c_all_files.txt
 #      need to do vs pwoershell so cant use msys powershell
 
-      - name: Unpack Artifacts
+      - name: Unpack / Move Artifacts
         if: ${{ inputs.RequiredDeps != '' && inputs.BuildPkg != 'script' }}
         run: ${{env.WLB_SCRIPT_FOLDER}}/.github/move_dl_artifacts.ps1
 

.github/workflows/restore_deps_to_cache.yml

@@ -0,0 +1,135 @@
+name: __incl_restore_deps
+# Deps are across workflows we need to really run a singular step multiple times in one job which isn't really doable. while jobs can be generated dynamically you can't have multiple jobs contribute to the same cache.  We could store the deps into the artifacts for the job but that would just polute artifacts. Instead this script will download up to N deps in one job and save to a specific cache name.
+
+on:
+  workflow_call:
+    inputs:
+      #if buildpkg is "script" then only the script is executed no deps or other items handled
+      BuildPkg:
+        required: true
+        type: string
+      RequiredDeps:
+        required: false
+        type: string
+        description: 'multi-line scalar for any required artifacts or empty string for none'
+        default: ""
+
+jobs:
+  build:
+    name:  DEP Download
+    runs-on: windows-2022
+    environment: main
+    defaults:
+      run:
+        shell: pwsh
+    steps:
+    - name: Check Existing Cache
+      id: cachecheck
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          d:/artifacts
+        key: ${{ inputs.BuildPkg }}
+        lookup-only: true
+
+    - uses: actions/[email protected]
+      if: steps.cachecheck.outputs.cache-hit != 'true'
+      with:
+        submodules: true
+
+    - name: Set Variables
+      shell: powershell
+      id: vars
+      if: steps.cachecheck.outputs.cache-hit != 'true'
+      env:
+        DEPS: "${{ inputs.RequiredDeps }}"
+      run: ${{ github.workspace }}\.github\set_dep_vars.ps1
+
+    - name: Download Artifact ${{ steps.vars.outputs.Dep1Name }}
+      uses: mitchcapper/action-download-artifact@ce1b9a1572bf43f4e5e89568e74cb1f2299e6362
+      if: ${{ steps.vars.outputs.Dep1Name && steps.cachecheck.outputs.cache-hit != 'true' }}
+      with:
+        name: ${{ steps.vars.outputs.Dep1Name }}
+        path: d:/artifacts
+        name_prefix: WLB-
+        workflow_conclusion: "success"
+        skip_unpack: true
+        workflow: tool_${{ steps.vars.outputs.Dep1Name }}_build.yml
+
+    - name: Download Artifact ${{ steps.vars.outputs.Dep2Name }}
+      uses: mitchcapper/action-download-artifact@ce1b9a1572bf43f4e5e89568e74cb1f2299e6362
+      if: ${{ steps.vars.outputs.Dep2Name && steps.cachecheck.outputs.cache-hit != 'true' }}
+      with:
+        name: ${{ steps.vars.outputs.Dep2Name }}
+        path: d:/artifacts
+        name_prefix: WLB-
+        workflow_conclusion: "success"
+        skip_unpack: true
+        workflow: tool_${{ steps.vars.outputs.Dep2Name }}_build.yml
+
+    - name: Download Artifact ${{ steps.vars.outputs.Dep3Name }}
+      uses: mitchcapper/action-download-artifact@ce1b9a1572bf43f4e5e89568e74cb1f2299e6362
+      if: ${{ steps.vars.outputs.Dep3Name && steps.cachecheck.outputs.cache-hit != 'true' }}
+      with:
+        name: ${{ steps.vars.outputs.Dep3Name }}
+        path: d:/artifacts
+        name_prefix: WLB-
+        workflow_conclusion: "success"
+        skip_unpack: true
+        workflow: tool_${{ steps.vars.outputs.Dep3Name }}_build.yml
+
+    - name: Download Artifact ${{ steps.vars.outputs.Dep4Name }}
+      uses: mitchcapper/action-download-artifact@ce1b9a1572bf43f4e5e89568e74cb1f2299e6362
+      if: ${{ steps.vars.outputs.Dep4Name && steps.cachecheck.outputs.cache-hit != 'true' }}
+      with:
+        name: ${{ steps.vars.outputs.Dep4Name }}
+        path: d:/artifacts
+        name_prefix: WLB-
+        workflow_conclusion: "success"
+        skip_unpack: true
+        workflow: tool_${{ steps.vars.outputs.Dep4Name }}_build.yml
+
+    - name: Download Artifact ${{ steps.vars.outputs.Dep5Name }}
+      uses: mitchcapper/action-download-artifact@ce1b9a1572bf43f4e5e89568e74cb1f2299e6362
+      if: ${{ steps.vars.outputs.Dep5Name && steps.cachecheck.outputs.cache-hit != 'true' }}
+      with:
+        name: ${{ steps.vars.outputs.Dep5Name }}
+        path: d:/artifacts
+        name_prefix: WLB-
+        workflow_conclusion: "success"
+        skip_unpack: true
+        workflow: tool_${{ steps.vars.outputs.Dep5Name }}_build.yml
+
+    - name: Download Artifact ${{ steps.vars.outputs.Dep6Name }}
+      uses: mitchcapper/action-download-artifact@ce1b9a1572bf43f4e5e89568e74cb1f2299e6362
+      if: ${{ steps.vars.outputs.Dep6Name && steps.cachecheck.outputs.cache-hit != 'true' }}
+      with:
+        name: ${{ steps.vars.outputs.Dep6Name }}
+        path: d:/artifacts
+        name_prefix: WLB-
+        workflow_conclusion: "success"
+        skip_unpack: true
+        workflow: tool_${{ steps.vars.outputs.Dep6Name }}_build.yml
+
+    - name: Download Artifact ${{ steps.vars.outputs.Dep7Name }}
+      uses: mitchcapper/action-download-artifact@ce1b9a1572bf43f4e5e89568e74cb1f2299e6362
+      if: ${{ steps.vars.outputs.Dep7Name && steps.cachecheck.outputs.cache-hit != 'true' }}
+      with:
+        name: ${{ steps.vars.outputs.Dep7Name }}
+        path: d:/artifacts
+        name_prefix: WLB-
+        workflow_conclusion: "success"
+        skip_unpack: true
+        workflow: tool_${{ steps.vars.outputs.Dep7Name }}_build.yml
+
+    - name: Save Cache
+      if: steps.cachecheck.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v3
+      with:
+        path: |
+          d:/artifacts
+        key: ${{ inputs.BuildPkg }}
+    # - name: Debug Session
+    #   if: ${{ failure() }}
+    #   run: D:/a/docs/docs/.github/debug_ssh_start.ps1
+    #   #run: pwd && dir

.github/workflows/tool_automake_build.yml

@@ -0,0 +1,14 @@
+name: automake Tool Build
+on:
+  push:
+    branches-ignore:
+      - trash
+  schedule:
+    - cron: '30 5 * * *'
+
+jobs:
+  build:
+    uses: ./.github/workflows/tool_build.yml
+    with:
+      ToolName: automake
+

.github/workflows/tool_awk_build.yml

@@ -0,0 +1,14 @@
+name: awk Tool Build
+on:
+  push:
+    branches-ignore:
+      - trash
+  schedule:
+    - cron: '30 5 * * *'
+
+jobs:
+  build:
+    uses: ./.github/workflows/tool_build.yml
+    with:
+      ToolName: awk
+

.github/workflows/tool_build.yml

@@ -0,0 +1,37 @@
+name: __incl_tool_build
+
+on:
+  workflow_call:
+    inputs:
+      ToolName:
+        required: true
+        type: string
+      RequiredDeps:
+        required: false
+        type: string
+        description: 'multi-line scalar for any required artifacts or empty string for none'
+        default: ""
+
+env:
+  CI_REQ_DOTNET_SDK_VER: 6.0.401
+  NUKE_TELEMETRY_OPTOUT: 1
+  MSYS: "winsymlinks:native wincmdln"
+  MSYS2_ARG_CONV_EXCL: "*"
+  MSYS2_PATH_TYPE: inherit
+  WLB_BASE_FOLDER: "d:/WLB"
+  MSYSTEM: UCRT64
+  MSYS_PATH: d:/msys64
+
+jobs:
+  deps:
+    uses: ./.github/workflows/restore_deps_to_cache.yml
+    with:
+      BuildPkg: ${{ inputs.ToolName }}
+      RequiredDeps: ${{ inputs.RequiredDeps }}
+
+  build:
+    uses: ./.github/workflows/do_build.yml
+    needs: deps
+    with:
+      BuildPkg: ${{ inputs.ToolName }}
+      RequiredDeps: ${{ inputs.RequiredDeps }}