Initial script commit

mitchcapper · May 26, 2023 · 2d381dc · 2d381dc
commit 2d381dc
Show file tree

Hide file tree

Showing 69 changed files with 14,610 additions and 0 deletions.
diff --git a/.github/debug_ssh_start.ps1 b/.github/debug_ssh_start.ps1
@@ -0,0 +1,59 @@
+Set-StrictMode -version latest;
+$ErrorActionPreference = "Stop";
+$VerbosePreference="Continue";
+
+
+$publicKey=(curl https://github.com/$($env:GITHUB_REPOSITORY_OWNER).keys) + " gh"
+Function InstallSshServer(){
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory=$true)]
+        [String] $publicKey
+    )
+    Add-WindowsCapability -Online -Name OpenSSH.Server
+    echo "PubkeyAuthentication yes`nPasswordAuthentication no`nSubsystem sftp sftp-server.exe`nMatch Group administrators`n`tAuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys`n" | out-file -Encoding ASCII $env:programData/ssh/sshd_config
+    ssh-keygen -A
+    echo "$publicKey`n" | out-file -Encoding ASCII $env:programData/ssh/administrators_authorized_keys
+    icacls.exe "$env:programData\ssh\administrators_authorized_keys" /inheritance:r /grant "Administrators:F" /grant "SYSTEM:F"
+    cat $env:programData/ssh/administrators_authorized_keys
+}
+Function DownloadStartCloudflareServer(){
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory=$true)]
+        [String] $LocalHostnameAndPort, #ie 127.0.0.1:22
+        [Parameter(Mandatory=$false)]
+        [String] $SaveToFilename="cloudflared.exe" #can include path
+    )
+    if (([System.IO.File]::Exists($SaveToFilename)) -eq $false) {
+        Invoke-WebRequest -URI https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-amd64.exe -OutFile $SaveToFilename
+    }
+    $myargs = "tunnel --no-autoupdate --url tcp://$LocalHostnameAndPort --logfile cfd.log"
+    #$scriptBlock = [Scriptblock]::Create("Start-Process -NoNewWindow -Wait `"$SaveToFilename`" $myargs ")
+    $myjob = Start-Process -PassThru -NoNewWindow `"$SaveToFilename`" -ArgumentList $myargs
+
+    #Start-Job -Name CFD -ScriptBlock $scriptBlock
+    #$myjob= Receive-Job -Name CFD
+    return $myjob
+}
+Function InstallSSHStartCF(){
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory=$true)]
+        [String] $publicKey,
+        [Parameter(Mandatory=$false)]
+        [String] $SaveToFilename="cloudflared.exe" #can include path
+    )
+    InstallSshServer $publicKey
+    $server = DownloadStartCloudflareServer("127.0.0.1:22")
+    $scriptBlock = [Scriptblock]::Create("Start-Process -NoNewWindow -Wait `"sshd.exe`" ")
+
+    Start-Job -Name SSHD -ScriptBlock $scriptBlock
+    return $server
+}
+InstallSSHStartCF $publicKey
+while ($true) {
+    Start-Sleep -Seconds 30
+    cat cfd.log
+}
+#Wait-Job SSHD
diff --git a/.github/gnu_patchtest.sh b/.github/gnu_patchtest.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+OUR_PATH="$(readlink -f "$0")";
+CALL_CMD="$1" # should be branch or patch or all_patch
+PATCH_NAME="$2"
+
+SCRIPT_FOLDER="$(dirname "${OUR_PATH}")"
+if [[ ! -z "$WLB_SCRIPT_FOLDER" ]]; then
+	SCRIPT_FOLDER="${WLB_SCRIPT_FOLDER}"
+fi
+. "$SCRIPT_FOLDER/helpers.sh" "${CALL_CMD}" "${OUR_PATH}"
+
+PreInitialize;
+
+BLD_CONFIG_BUILD_NAME="gnulib";
+BLD_CONFIG_LOG_FILE_AUTOTAIL=0;
+function ourmain() {
+	startcommon;
+	set -e
+	git config --global user.email "[email protected]"
+  	git config --global user.name "Your Name"
+	git clone --quiet https://github.com/mitchcapper/gnulib .
+	git remote add upstream https://github.com/coreutils/gnulib.git
+	git fetch upstream --quiet
+	git checkout master --quiet
+	git branch master -u upstream/master
+	git pull
+	TEST_WHAT="$CALL_CMD"
+	BRANCH_NAME="ours_${PATCH_NAME,,}"
+
+	if [[ "$TEST_WHAT" == "patch" ]]; then
+		echo "Running patch test"
+		gnulib_apply_patch "$PATCH_NAME" "skip_fixes"
+	elif [[ "$TEST_WHAT" == "branch" ]]; then
+		echo "Running branch test"
+		git checkout "$BRANCH_NAME"
+		git merge master -m done
+
+		exit_out $? "Merge Done"
+	else
+		echo "Running all patches test"
+		gnulib_patches;
+	fi
+	finalcommon;
+}
+ourmain;
+
diff --git a/.github/move_dl_artifacts.ps1 b/.github/move_dl_artifacts.ps1
@@ -0,0 +1,15 @@
+Set-StrictMode -version latest;
+$ErrorActionPreference = "Stop";
+$VerbosePreference="Continue";
+
+
+$folders = (Get-ChildItem -Path d:/artifacts/* -Directory -Force -ErrorAction SilentlyContinue)
+
+foreach ($folder in $folders) {
+	$srcPath = "d:/artifacts/$($folder.Name)"
+	$dstPath = "$($env:WLB_BASE_FOLDER)/$($folder.Name)/final"
+	New-Item -ItemType Directory -Force -Path "$($env:WLB_BASE_FOLDER)/$($folder.Name)"
+
+	Move-Item -Path $srcPath -Destination $dstPath
+	Write-Output "$srcPath => $dstPath"
+}
diff --git a/.github/workflows/do_build.yml b/.github/workflows/do_build.yml
@@ -0,0 +1,153 @@
+name: do_build
+on:
+  workflow_call:
+    inputs:
+      #if buildpkg is "script" then only the script is executed no deps or other items handled
+      BuildPkg:
+        required: true
+        type: string
+      Script: #should be relative to the repo root
+        required: false
+        type: string
+      RequiredDeps:
+        required: false
+        type: string
+        description: 'multi-line scalar for any required artifacts or empty string for none'
+        default: ""
+    outputs:
+      ScriptRes:
+        description: "Result of script execution (optional)"
+        value: ${{ jobs.build.outputs.ScriptRes }}
+env:
+  CI_REQ_DOTNET_SDK_VER: 6.0.401
+  NUKE_TELEMETRY_OPTOUT: 1
+  MSYS: "winsymlinks:native wincmdln"
+  MSYS2_ARG_CONV_EXCL: "/a;/b;/c;/d;/e;/f;/g;/h;/i;/j;/k;/l;/m;/n;/o;/p;/q;/r;/s;/u;/v;/w;/x;/y;/z;/0;/1;/2;/3;/4;/5;/6;/7;/8;/9;/A;/B;/C;/D;/E;/F;/G;/H;/I;/J;/K;/L;/M;/N;/O;/P;/Q;/R;/S;/T;/U;/V;/W;/X;/Y;/Z"
+  MSYS2_PATH_TYPE: inherit
+  WLB_BASE_FOLDER: "d:/WLB"
+  MSYSTEM: UCRT64
+  MSYS_PATH: d:/msys64
+
+jobs:
+  build:
+    name:  ${{ inputs.BuildPkg == 'script' && 'Run Script' || 'Build Package' }}
+    runs-on: windows-2022
+    environment: main
+    outputs:
+      ScriptRes: ${{ steps.script_step.outputs.ScriptRes }}
+    defaults:
+        run:
+          shell: pwsh
+    strategy:
+      fail-fast: false
+      matrix:
+        #BuildPkg: [patch]
+        #BuildPkg: [grep]
+#        BuildPkg: [which,patch,zstd,pcre2,gzip,tar,zlib,findutils,libpsl,coreutils,grep]
+        BaseImage: [windows-latest]
+        Configuration: [Debug]
+        Arch: [x64]
+        #BaseImage: [windows-latest, ubuntu-latest]
+        #Configuration: [Debug, Release]
+        #Arch: [x86, x64]
+        exclude:
+          - Arch: x86
+            BaseImage: ubuntu-latest
+        include:
+          - BaseImage: windows-latest
+            platform: windows
+          #- BuildPkg: grep
+            #needs: [pcre2]
+#          - BaseImage: ubuntu-latest
+#            platform: linux
+    env:
+      BUILD_PKG: ${{inputs.BuildPkg}}
+      SCRIPT: ${{inputs.Script}}
+    steps:
+      - uses: actions/[email protected]
+        with:
+          submodules: true
+      - name: ENV Set
+        run: |
+          $dir=$env:GITHUB_WORKSPACE
+          $dir=$dir.replace("\","/")
+          echo "WLB_SCRIPT_FOLDER=$dir" >> $env:GITHUB_ENV
+          mkdir $env:WLB_BASE_FOLDER
+
+#      - uses: actions/[email protected]
+#        with:
+#          dotnet-version: ${{env.CI_REQ_DOTNET_SDK_VER}}
+
+
+
+
+
+      - name: Download Required Artifacts
+        uses: mitchcapper/action-download-artifact@b2e806fc9fc820918660db898ad04630eb99a22c
+        if: ${{ inputs.RequiredDeps != '' && inputs.BuildPkg != 'script' }}
+        with:
+          name: ${{ inputs.RequiredDeps }}
+          path: d:/artifacts
+          name_prefix: WLB-
+          run_id: ${{ github.run_id }}
+          workflow_conclusion: ""
+
+
+
+      - name: MSBuild Setup
+        uses: microsoft/[email protected]
+        if: matrix.platform == 'windows'
+
+      - name: Move Bad DEFAULT MSYS
+        shell: powershell
+        run: Rename-Item c:/msys64 c:/trashmsys
+
+      - name: MSYS Setup
+        uses: msys2/setup-msys2@v2
+        if: matrix.platform == 'windows'
+        with:
+          msystem: ucrt64
+          location: d:\
+          install: pkg-config make gperf rsync autoconf wget gettext-devel automake autogen texinfo git bison python autoconf-archive libtool flex
+
+      - name: Save ENV
+        run: "Get-ChildItem env: | Select-Object -Property Name, Value | ConvertTo-Json | Out-File -FilePath d:/env.json -Encoding ASCII"
+# Get-Content -Raw d:/env.json | ConvertFrom-Json | % { Set-Item "env:$($_.Name)" $_.Value }
+#      - name: MSYS Package Install
+#        shell: msys2 {0}
+#        run: pacman -S --noconfirm pkg-config make gperf rsync autoconf wget gettext-devel automake autogen texinfo git
+#      - name: File Dump
+#        shell: msys2 {0}
+#        working-directory: ${{env.WLB_BASE_FOLDER}}
+#        run: |
+#          find /d/ > d:/d_all_files.txt
+#          find /c/ > d:/c_all_files.txt
+#      need to do vs pwoershell so cant use msys powershell
+
+      - name: Unpack Artifacts
+        if: ${{ inputs.RequiredDeps != '' && inputs.BuildPkg != 'script' }}
+        run: ${{env.WLB_SCRIPT_FOLDER}}/.github/move_dl_artifacts.ps1
+
+      - name: Build Package
+        if: ${{ inputs.BuildPkg != 'script' }}
+        shell: powershell
+        working-directory: ${{env.WLB_BASE_FOLDER}}
+        run: ${{env.WLB_SCRIPT_FOLDER}}/vs_msys_shell_launch.ps1 "${{env.WLB_SCRIPT_FOLDER}}/build/f_$($env:BUILD_PKG)_build.sh"
+
+      - name: Run Script
+        id: script_step
+        if: ${{ inputs.BuildPkg == 'script' }}
+        shell: powershell
+        working-directory: ${{env.WLB_BASE_FOLDER}}
+        run: ${{env.WLB_SCRIPT_FOLDER}}/vs_msys_shell_launch.ps1 "${{env.WLB_SCRIPT_FOLDER}}/${{env.SCRIPT}}"
+
+      - name: Debug Session
+        if: ${{ failure() && (vars.DEBUG_FAIL == '1') }}
+        run: ${{env.WLB_SCRIPT_FOLDER}}/.github/debug_ssh_start.ps1
+
+      - uses: actions/[email protected]
+        if: ${{ inputs.BuildPkg != 'script' }}
+        with:
+          name: WLB-${{env.BUILD_PKG}}
+          path: ${{env.WLB_BASE_FOLDER}}/${{env.BUILD_PKG}}/final
+          if-no-files-found: error
diff --git a/.github/workflows/gnulib_tests.yml b/.github/workflows/gnulib_tests.yml
@@ -0,0 +1,43 @@
+name: GNULIB Patch Tests
+on:
+  push:
+    branches-ignore:
+      - trash
+  schedule:
+    - cron: '30 5 * * *'
+
+env:
+  CI_REQ_DOTNET_SDK_VER: 6.0.401
+  NUKE_TELEMETRY_OPTOUT: 1
+  MSYS: "winsymlinks:native wincmdln"
+  MSYS2_ARG_CONV_EXCL: "/a;/b;/c;/d;/e;/f;/g;/h;/i;/j;/k;/l;/m;/n;/o;/p;/q;/r;/s;/u;/v;/w;/x;/y;/z;/0;/1;/2;/3;/4;/5;/6;/7;/8;/9;/A;/B;/C;/D;/E;/F;/G;/H;/I;/J;/K;/L;/M;/N;/O;/P;/Q;/R;/S;/T;/U;/V;/W;/X;/Y;/Z"
+  MSYS2_PATH_TYPE: inherit
+  WLB_BASE_FOLDER: "d:/WLB"
+  MSYSTEM: UCRT64
+  MSYS_PATH: d:/msys64
+
+jobs:
+  get_patch_matrix:
+    uses: ./.github/workflows/do_build.yml
+    with:
+      BuildPkg: script
+      Script: f_template_build_file.sh gnulib_dump_patches
+  gnulib_test_all_patches:
+    name: Test All Patches Together
+    uses: ./.github/workflows/do_build.yml
+    with:
+      BuildPkg: script
+      Script: .github/gnu_patchtest.sh all_patch
+
+  gnulib_test_patch:
+    name: Test Single
+    needs: get_patch_matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        PatchName: ${{ fromjson(needs.get_patch_matrix.outputs.ScriptRes) }}
+        Mode: [branch, patch]
+    uses: ./.github/workflows/do_build.yml
+    with:
+      BuildPkg: script
+      Script: .github/gnu_patchtest.sh ${{ matrix.Mode }} '${{ matrix.PatchName }}'
diff --git a/.github/workflows/tool_builds.yml b/.github/workflows/tool_builds.yml
@@ -0,0 +1,49 @@
+name: Tool Builds
+on:
+  push:
+    branches-ignore:
+      - trash
+  schedule:
+    - cron: '30 5 * * *'
+
+env:
+  CI_REQ_DOTNET_SDK_VER: 6.0.401
+  NUKE_TELEMETRY_OPTOUT: 1
+  MSYS: "winsymlinks:native wincmdln"
+  MSYS2_ARG_CONV_EXCL: "/a;/b;/c;/d;/e;/f;/g;/h;/i;/j;/k;/l;/m;/n;/o;/p;/q;/r;/s;/u;/v;/w;/x;/y;/z;/0;/1;/2;/3;/4;/5;/6;/7;/8;/9;/A;/B;/C;/D;/E;/F;/G;/H;/I;/J;/K;/L;/M;/N;/O;/P;/Q;/R;/S;/T;/U;/V;/W;/X;/Y;/Z"
+  MSYS2_PATH_TYPE: inherit
+  WLB_BASE_FOLDER: "d:/WLB"
+  MSYSTEM: UCRT64
+  MSYS_PATH: d:/msys64
+
+jobs:
+  build_nodeps:
+    name: No Dep
+    strategy:
+      fail-fast: false
+      matrix:
+        BuildPkg: [which,patch,zstd,pcre2,gzip,tar,zlib,findutils,libpsl,coreutils,highlight]
+    uses: ./.github/workflows/do_build.yml
+    with:
+      BuildPkg: ${{ matrix.BuildPkg }}
+  #we always run these as they will fail at the artifact fetch step if the build package(s) they need did not complete successfully
+  build_grep:
+    if: success() || failure()
+    needs: build_nodeps
+    name: Build grep
+    uses: ./.github/workflows/do_build.yml
+    with:
+      BuildPkg: "grep"
+      RequiredDeps: |
+        pcre2
+  build_wget:
+    if: success() || failure()
+    needs: build_nodeps
+    name: Build wget
+    uses: ./.github/workflows/do_build.yml
+    with:
+      BuildPkg: "wget"
+      RequiredDeps: |
+        pcre2
+        libpsl
+        zlib
diff --git a/AUTORECONF_prewrapper.sh.template b/AUTORECONF_prewrapper.sh.template
@@ -0,0 +1,5 @@
+#!/bin/bash
+pushd .
+SCRIPT_PATH gnulib_tool_py_remove_nmd_makefiles
+popd
+exec autoreconf "$@"