publish_and_test_pykotor.yml

name: Compile and Test PyKotor

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  - push
  - workflow_dispatch

permissions:
  contents: write

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false  # Disable automatic cancellation of other jobs
      matrix:
        os: [windows-2019, ubuntu-20.04, macos-12]
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
        architecture: ['x86', 'x64']
        include:
          - arch: x86
            vc_redist2015: "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x86.exe"
            vc_redist-latest: "https://aka.ms/vs/17/release/vc_redist.x86.exe"
            vc_redist2019: "https://aka.ms/vs/17/release/vc_redist.x86.exe"
          - arch: x64
            vc_redist2015: "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x64.exe"
            vc_redist-latest: "https://aka.ms/vs/17/release/vc_redist.x64.exe"
            vc_redist2019: "https://aka.ms/vs/17/release/vc_redist.x64.exe"
        exclude:
          # unix x86 is definitely not supported.
          - os: ubuntu-20.04
            architecture: x86
          - os: macos-12
            architecture: x86
    outputs:
      matrix-os: ${{ toJson(matrix.os) }}
      matrix-python-version: ${{ toJson(matrix.python-version) }}

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      if: ${{ runner.os != 'macOS' }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        architecture: ${{ matrix.architecture }}

    - name: Set up Python macOS
      if: ${{ runner.os == 'macOS' }}
      run: |  # warning: `brew link --overwrite python@ver` is unsafe on any non-virtualized macos.
        echo "NONINTERACTIVE DEFAULT: $NONINTERACTIVE"
        export NONINTERACTIVE=1
        echo "NONINTERACTIVE NEW: $NONINTERACTIVE"
        brew analytics on
        brew update
        brew install python@${{ matrix.python-version }} || brew link --overwrite python@${{ matrix.python-version }}
    
    - name: Reset APT sources to default
      if: ${{ runner.os == 'Linux' }}
      run: |
        echo "Resetting APT sources to default Ubuntu repositories"
        sudo rm /etc/apt/sources.list
        echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
        echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
        echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-backports main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
        echo "deb http://security.ubuntu.com/ubuntu $(lsb_release -cs)-security main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
        sudo apt-get update -y

    - name: Setup python venvs
      run: |  # create the venv early to work around an issue with the matrix runners' concurrency
        $pythonExeName = "python"
        if ("${{ runner.os }}" -ne "Windows")
        {
          $pythonExeName = "python${{ matrix.python-version }}"
        }
        & $pythonExeName -m venv .venv_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExeName -m venv .venv_holopatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExeName -m venv .venv_toolset_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExeName -m venv .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExeName -m venv .venv_kotordiff_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        & $pythonExeName -m venv .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
      shell: pwsh

    - name: Set UPX download URL
      # upx docs express that crashes are happening on ventura and above with upx, don't use on mac.
      if: runner.os != 'macOS'
      id: upx_setup
      run: |
        $build = "no"
        $archiveName = ""
        if ("${{ runner.os }}" -eq "Windows") {
          if ("${{ matrix.architecture }}" -eq "x86") {
            $archiveName = "upx-4.2.2-win32.zip"
          } else {
            $archiveName = "upx-4.2.2-win64.zip"
          }
        } elseif ("${{ runner.os }}" -eq "Linux") {
          $archiveName = "upx-4.2.2-amd64_linux.tar.xz"
        } elseif ("${{ runner.os }}" -eq "macOS") {
          $build = "yes"
          $archiveName = "upx-4.2.2-src.tar.xz"
        }
        $url = "https://github.com/upx/upx/releases/download/v4.2.2/$archiveName"
    
        # Write to the GITHUB_OUTPUT environment file
        Add-Content -Path $env:GITHUB_OUTPUT -Value "build=$build"
        Add-Content -Path $env:GITHUB_OUTPUT -Value "url=$url"
        Add-Content -Path $env:GITHUB_OUTPUT -Value "archiveName=$archiveName"
      shell: pwsh

    - name: Download and prepare UPX
      if: runner.os != 'macOS'
      run: |        
        $ext = "${{ runner.os }}" -eq "Windows" ? "zip" : "tar.xz"
        $url = "${{ steps.upx_setup.outputs.url }}"
        $archiveName = "${{ steps.upx_setup.outputs.archiveName }}"
        $outputPath = "upx-dir"

        # Use Invoke-WebRequest or curl depending on the OS
        if ("${{ runner.os }}" -eq "Windows") {
          Invoke-WebRequest -Uri $url -OutFile $archiveName
        } elseif ("${{ runner.os }}" -eq "Linux") {
          curl -L $url -o $archiveName
        }
        
        New-Item -ItemType Directory -Force -Path "upx-dir" -ErrorAction SilentlyContinue
        if ("${{ runner.os }}" -ne "macOS") {
          if ($ext -eq "zip") {
            $fileNameWithoutExtension = [System.IO.Path]::GetFileNameWithoutExtension($archiveName)
            Expand-Archive -Path $archiveName -DestinationPath "temp_folder_upx"
            # Ensure upx-dir exists; create it if it doesn't
            if (-not (Test-Path -Path "upx-dir")) {
                New-Item -ItemType Directory -Path "upx-dir"
            }
            Get-ChildItem -Path "temp_folder_upx/$fileNameWithoutExtension" -Recurse | Move-Item -Destination "upx-dir"
            Remove-Item "temp_folder_upx" -Recurse -Force -ErrorAction SilentlyContinue
          } else {
            tar -xvf $archiveName --strip-components=1 -C "upx-dir"
          }
    
          Remove-Item $archiveName # Clean up downloaded archive
        }
      shell: pwsh

    - name: Set UPX directory path
      if: runner.os != 'macOS'
      id: upx_dir
      run: |
        $upx_dir = "./upx-dir"
        $upx_dir = $([System.IO.Path]::GetFullPath('./upx-dir'))
        Dir -Recurse $upx_dir | Get-Childitem
        echo "UPX_DIR=$upx_dir" | Out-File -FilePath $env:GITHUB_ENV -Append
        Write-Output "UPX_DIR set to '$upx_dir'"
      shell: pwsh

    - name: Install Visual Studio 2015 C++ Redistributable
      if: runner.os == 'Windows'
      run: |
        $url = "${{ matrix.vc_redist2015 }}"
        $output = "vc_redist.exe"
        Invoke-WebRequest -Uri $url -OutFile $output
        Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait
        Remove-Item -Path $output
        #choco install vcredist2015 -y
      shell: pwsh

    - name: Install Visual Studio 2019 C++ Redistributable
      if: runner.os == 'Windows'
      run: |
        $url = "${{ matrix.vc_redist2019 }}"
        $output = "vc_redist.exe"
        Invoke-WebRequest -Uri $url -OutFile $output
        Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait
        Remove-Item -Path $output
        #choco install vcredist2019 -y
      shell: pwsh

    - name: Install Visual Studio latest C++ Redistributable
      if: runner.os == 'Windows'
      run: |
        $url = "${{ matrix.vc_redist-latest }}"
        $output = "vc_redist.exe"
        Invoke-WebRequest -Uri $url -OutFile $output
        Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait
        Remove-Item -Path $output
      shell: pwsh

    - name: Install Holocron Toolset dependencies
      if: ${{ success() || failure() }}
      id: toolset_deps
      run: |  # known pyinstaller versions that work with upx compressions, not all are available on all python versions.
        try {
          ./install_python_venv.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
          python -m pip install --upgrade pip
          if ("${{ runner.os }}" -eq "Windows") {
            if ("${{ matrix.python-version }}" -eq "3.12") {
              pip install "pyinstaller==5.13" --prefer-binary
            } elseif ("${{ matrix.python-version }}" -eq "3.11") {
              pip install "pyinstaller==5.6.1" --prefer-binary
              pip install "pyinstaller==5.5" --prefer-binary
            } else {
              pip install "pyinstaller==5.4" --prefer-binary
            }
          } else {
            pip install pyinstaller -U --prefer-binary
          }
          $output = ""
          $errorLines = @()
          ./compile/deps_toolset.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} 2>&1 | ForEach-Object {
            Write-Output $_.ToString()
            $output += $_.ToString() + "`n"
            if($_ -match 'ERROR:') {
              $errorLines += $_.ToString()
            }
          }
          if ($errorLines.Count -gt 0) {
            $errorLines | ForEach-Object { Write-Error $_ }
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
            exit 1
          } else {
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
          }
        } catch {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        }
      shell: pwsh

    - name: Compile Holocron Toolset
      if: ${{ (success() || failure()) && steps.toolset_deps.outputs.success == 'true' }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        $output = ""
        ./compile/compile_toolset.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          $output += $_.ToString() + "`n"
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh
      env:
        UPX_DIR: ${{ env.UPX_DIR }}

    - name: Install HoloPatcher dependencies
      if: ${{ success() || failure() }}
      id: holopatcher_deps
      run: |  # known pyinstaller versions that work with upx compressions, not all are available on all python versions.
        try {
          ./install_python_venv.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
          python -m pip install --upgrade pip
          if ("${{ runner.os }}" -eq "Windows") {
            if ("${{ matrix.python-version }}" -eq "3.12") {
              pip install "pyinstaller==5.13" --prefer-binary
            } elseif ("${{ matrix.python-version }}" -eq "3.11") {
              pip install "pyinstaller==5.6.1" --prefer-binary
              pip install "pyinstaller==5.5" --prefer-binary
            } else {
              pip install "pyinstaller==5.4" --prefer-binary
            }
          } else {
            pip install pyinstaller -U --prefer-binary
          }
          $output = ""
          $errorLines = @()
          ./compile/deps_holopatcher.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} 2>&1 | ForEach-Object {
            Write-Output $_.ToString()
            $output += $_.ToString() + "`n"
            if($_ -match 'ERROR:') {
              $errorLines += $_.ToString()
            }
          }
          if ($errorLines.Count -gt 0) {
            $errorLines | ForEach-Object { Write-Error $_ }
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
            exit 1
          } else {
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
          }
        } catch {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        }
      shell: pwsh
    
    - name: Compile HoloPatcher
      if: ${{ (success() || failure()) && steps.holopatcher_deps.outputs.success == 'true' }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        $output = ""
        ./compile/compile_holopatcher.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          $output += $_.ToString() + "`n"
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh

    - name: Install BatchPatcher dependencies
      if: ${{ success() || failure() }}
      id: batchpatcher_deps
      run: |  # known pyinstaller versions that work with upx compressions, not all are available on all python versions.
        try {
          ./install_python_venv.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
          python -m pip install --upgrade pip
          if ("${{ runner.os }}" -eq "Windows") {
            if ("${{ matrix.python-version }}" -eq "3.12") {
              pip install "pyinstaller==5.13" --prefer-binary
            } elseif ("${{ matrix.python-version }}" -eq "3.11") {
              pip install "pyinstaller==5.6.1" --prefer-binary
              pip install "pyinstaller==5.5" --prefer-binary
            } else {
              pip install "pyinstaller==5.4" --prefer-binary
            }
          } else {
            pip install pyinstaller -U --prefer-binary
          }
          $output = ""
          $errorLines = @()
          ./compile/deps_batchpatcher.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} 2>&1 | ForEach-Object {
            Write-Output $_.ToString()
            $output += $_.ToString() + "`n"
            if($_ -match 'ERROR:') {
              $errorLines += $_.ToString()
            }
          }
          if ($errorLines.Count -gt 0) {
            $errorLines | ForEach-Object { Write-Error $_ }
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
            exit 1
          } else {
            Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
          }
        } catch {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        }
      shell: pwsh

    - name: Compile BatchPatcher
      if: ${{ (success() || failure()) && steps.batchpatcher_deps.outputs.success == 'true' }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        $output = ""
        ./compile/compile_batchpatcher.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          $output += $_.ToString() + "`n"
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh

    - name: Compile KotorDiff
      if: ${{ success() || failure() }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        ./install_python_venv.ps1 -noprompt -venv_name .venv_kotordiff_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        python -m pip install --upgrade pip
        if ("${{ runner.os }}" -eq "Windows") {
          if ("${{ matrix.python-version }}" -eq "3.12") {
            pip install "pyinstaller==5.13" --prefer-binary
          } elseif ("${{ matrix.python-version }}" -eq "3.11") {
            pip install "pyinstaller==5.6.1" --prefer-binary
            pip install "pyinstaller==5.5" --prefer-binary
          } else {
            pip install "pyinstaller==5.4" --prefer-binary
          }
        } else {
          pip install pyinstaller -U --prefer-binary
        }
        pip install -r Tools/KotorDiff/requirements.txt --prefer-binary
        $output = ""
        ./compile/compile_kotordiff.ps1 -noprompt -venv_name .venv_kotordiff_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          $output += $_.ToString() + "`n"
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh

    - name: Compile GUIDuplicator
      if: ${{ success() || failure() }}
      run: |
        $upxDir = $env:UPX_DIR
        Write-Host "Using UPX directory at '$upxDir'"
        ./install_python_venv.ps1 -noprompt -venv_name .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        python -m pip install --upgrade pip
        if ("${{ runner.os }}" -eq "Windows") {
          if ("${{ matrix.python-version }}" -eq "3.12") {
            pip install "pyinstaller==5.13" --prefer-binary
          } elseif ("${{ matrix.python-version }}" -eq "3.11") {
            pip install "pyinstaller==5.6.1" --prefer-binary
            pip install "pyinstaller==5.5" --prefer-binary
          } else {
            pip install "pyinstaller==5.4" --prefer-binary
          }
        } else {
          pip install pyinstaller -U --prefer-binary
        }
        pip install -r Tools/GuiDuplicator/requirements.txt --prefer-binary
        $output = ""
        ./compile/compile_gui_duplicator.ps1 -noprompt -venv_name .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object {
          Write-Output $_.ToString()
          if($_ -match 'ERROR:') {
            $errorLines += $_.ToString()
          }
        }
        $warningCount = 0
        $output -split "`n" | ForEach-Object {
            if ($_ -match 'WARNING: Library not found: could not resolve' -or
                $_ -match 'WARNING: Cannot find ' -or
                $_ -match 'WARNING: lib not found:' -or
                $_ -match 'WARNING: Tcl modules directory') {
                $warningCount++
            }
        }
        if ($errorLines.Count -gt 0) {
          $errorLines | ForEach-Object { Write-Error $_ }
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } elseif ($warningCount -ge 3) {
          Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful."
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false"
          exit 1
        } else {
          Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true"
        }
      shell: pwsh

    - name: Upload compiled binaries
      if: ${{ success() || failure() }}
      uses: actions/upload-artifact@v4
      with:
        name: publish_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        path: ./dist/**
        retention-days: 90

    - name: Install MinGW with GCC on Windows
      if: runner.os == 'Windows' && matrix.python-version == '3.13.0-alpha.4'
      run: |
        choco install mingw -y
        $mingwPath = Get-ChildItem -Path C:\, D:\, B:\ -Filter mingw64 -Recurse -ErrorAction SilentlyContinue -Directory | Select-Object -First 1 -ExpandProperty FullName
        if (-not $mingwPath) { throw "MinGW installation not found" }
        $binPath = Join-Path -Path $mingwPath -ChildPath "bin"
        echo "$binPath" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
      shell: pwsh

    - name: Install development packages
      if: ${{ success() || failure() }}
      run: |
        ./install_python_venv.ps1 -noprompt -venv_name .venv_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        python -m pip install --upgrade pip
        if ($env:python_version -eq "3.7") {
          pip install -r requirements-dev-py37.txt --prefer-binary
        }
        elseif ($env:python_version -ne "3.13.0-alpha.4" -and "${{ runner.os }}" -ne "Linux" -and "${{ runner.os }}" -ne "macOS") {
          pip install -r requirements-dev.txt --prefer-binary
        }
        else {
          pip install -r requirements-dev-py313.txt --prefer-binary
        }
      shell: pwsh

    - name: Run all unittests/pytests
      if: ${{ success() || failure() }}
      run: |
        ./install_python_venv.ps1 -noprompt -venv_name .venv_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        python -m pytest tests -v -ra -o log_cli=true --capture=no --junitxml=pytest_report.xml --html=pytest_report.html --self-contained-html --tb=no --continue-on-collection-errors
      shell: pwsh
      continue-on-error: true

    - name: Upload Pytest Reports
      if: ${{ success() || failure() }}
      uses: actions/upload-artifact@v4
      with:
        name: pytest_report_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }}
        path: |
          pytest_report.html
          pytest_report.xml
          retention-days: 90

  package:  # The goal of this job is to repackage by toolname, rather than all tools by os_pyversion_arch
    needs: build  # do not start this job until all 'build' jobs complete
    if: ${{ success() || failure() }}
    runs-on: ubuntu-latest
    outputs:
      filesToArchive: ${{ steps.set-matrix.outputs.matrixJson }}
    steps:
      - name: Download all artifacts
        uses: actions/download-artifact@v4
        with:
          path: published_workflow_builds/
          pattern: publish_*

      - name: Re-package artifacts by Python version and architecture
        shell: pwsh
        run: |
          $here_dirpath = (Get-Location)
          Write-Output "here: '$here_dirpath'"
          $packagedArtifactsPath = "$here_dirpath/packaged_artifacts"
          New-Item -ItemType Directory -Force -Path $packagedArtifactsPath
          $artifactNames = @()
          
          # Navigate to the directory with downloaded artifacts
          Set-Location -Path "published_workflow_builds"
          Get-ChildItem -Recurse -Path "." -Directory | ForEach-Object { Write-Output $_.FullName }
          
          # Extract each published artifact and repackage
          Get-ChildItem -Filter "publish_*" | ForEach-Object {
            $matrixPackagePath = $_.FullName
            Write-Output "Found matrix package '$matrixPackagePath'"
            Get-ChildItem -Recurse -Path "." -Directory | ForEach-Object { Write-Output $_.FullName }
            $os, $pythonVersion, $architecture = $_.BaseName -replace '^publish_', '' -split '_'
          
            # Dynamic identification and repackaging
            Get-ChildItem -Path $matrixPackagePath | ForEach-Object {
              $fullPath = $_.FullName
              $fileBaseName = [IO.Path]::GetFileNameWithoutExtension($_.Name) + "`_$architecture"
              $fileExtension = $_.Extension
              $outerArchiveName = "$fileBaseName`_$pythonVersion"
              Write-Output "outerArchiveName '$outerArchiveName'"
              $outerZipPath = $packagedArtifactsPath + "/" + $outerArchiveName
              Write-Output "outerZipPath '$outerZipPath'"
              New-Item -ItemType Directory -Force -Path $outerZipPath
              $osSpecificZipName = "$fileBaseName`_$os"
              Write-Output "osSpecificZipName '$osSpecificZipName'"
              $osSpecificZipPath = $outerZipPath + "/" + $osSpecificZipName
              Write-Output "osSpecificZipPath '$osSpecificZipPath'"
          
              # Archive the tool by os identifier.
              chmod 777 -R $fullPath
              if ((-not $fileExtension) -or (-not $fileExtension.Trim()) -or ($fileExtension.ToLower().Trim() -eq ".app")) {
                $outerArchiveName = "$osSpecificZipPath.tar.gz"
                # Check if the path is a directory or a file
                if (Test-Path $fullPath -PathType Container) {
                    # It's a directory, archive the entire directory
                    tar -czf "$outerArchiveName" -C "$fullPath" .
                } else {
                    # It's a file, archive the file specifically
                    # Determine the parent directory and the file name
                    $parentDir = Split-Path -Parent $fullPath
                    $fileName = Split-Path -Leaf $fullPath
                    tar -czf "$outerArchiveName" -C "$parentDir" "$fileName"
                }
              } else {
                $outerArchiveName = "$osSpecificZipPath.zip"
                zip -r -9 "$outerArchiveName" "$fullPath"
              }
              Write-Output "Compressed '$fullPath' and saved to '$outerArchiveName'"
              chmod 777 -R "$outerArchiveName"
              
              $item = Get-Item -Path $outerZipPath -ErrorAction SilentlyContinue
              if ($item -ne $null) {
                if (-not $artifactNames -contains $item.BaseName) {
                  $artifactNames += $item.BaseName
                }
              }
            }
          }
          
          # Save artifact names to a file
          $artifactNames | Out-File -FilePath "../artifact-names.txt" -Encoding UTF8
          
          # Navigate back to the root of the workspace
          Set-Location -Path "../"

      - name: Generate matrix for uploading
        if: ${{ success() || failure() }}
        id: set-matrix
        shell: pwsh
        run: |
          $artifactNames = Get-Content 'artifact-names.txt' -ReadCount 0
      
          # Initialize an array to hold the artifact names directly
          $matrixArray = @()
      
          foreach ($name in $artifactNames) {
            if (-not [string]::IsNullOrEmpty($name)) {
              Write-Host "Processing artifact name: $name"
              # Trim the name and add directly to the array
              $matrixArray += $name.trim()
            }
          }
      
          # Convert the array directly to JSON
          $jsonMatrix = $matrixArray | ConvertTo-Json -Depth 5 -Compress

          # Use a single line of JSON for the matrix to avoid issues
          $singleLineJsonMatrix = $jsonMatrix -replace "`r", ""
          $singleLineJsonMatrix = $singleLineJsonMatrix -replace "`n", ""
          Write-Host "Matrix JSON:"
          Write-Host $singleLineJsonMatrix
      
          echo "matrixJson<<EOF" >> $env:GITHUB_OUTPUT
          echo $singleLineJsonMatrix >> $env:GITHUB_OUTPUT
          echo "EOF" >> $env:GITHUB_OUTPUT

      - name: Upload all repackages for next job
        if: ${{ success() || failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: all_tool_dists_onearchive
          path: packaged_artifacts/**
          retention-days: 1  # only used for the next job.
          compression-level: 0

  upload:
    needs: package
    if: ${{ success() || failure() }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        artifactb: ${{ fromJson(needs.package.outputs.filesToArchive) }}
    steps:
      - name: Download repackages from package job
        if: ${{ success() || failure() }}
        uses: actions/download-artifact@v4
        with:
          path: all_tool_dists_onearchive
          pattern: all_tool_dists*

      - name: Upload re-packaged artifact
        if: ${{ success() || failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.artifactb }}
          path: all_tool_dists_onearchive/all_tool_dists_onearchive/${{ matrix.artifactb }}
          compression-level: 9
          if-no-files-found: error

  add-test-result-status-badges:
    needs: build  # do not start this job until all 'build' jobs complete
    if: ${{ success() || failure() }}
    runs-on: ubuntu-latest
    concurrency: 
      group: add-test-status-badges-${{ github.ref }}
      cancel-in-progress: true
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download all artifacts
        uses: actions/download-artifact@v4
        with:
          path: all_pytest_reports
          pattern: pytest_report_*

      - name: Extract and update README with custom test status badges
        shell: pwsh
        run: |
          # Git configuration and commit
          git config --global user.name "GitHub Action"
          git config --global user.email "action@github.com"

          # Determine the branch that triggered the workflow
          $branchName = "${{ github.ref_name }}"
          $repository_owner = "${{ github.repository_owner }}"
          $repository = "${{ github.repository }}"
          $commitSHA = "${{ github.sha }}"

          $testsResultsPath = "tests/results"
          $testsResultsCommitPath = $testsResultsPath + "/$commitSHA"
          Remove-Item -Path $testsResultsPath -Recurse -Force -ErrorAction SilentlyContinue
          New-Item -ItemType Directory -Force -Path $testsResultsCommitPath -ErrorAction SilentlyContinue

          $OS_NAMES = @("windows-2019", "ubuntu-20.04", "macos-12")
          $PYTHON_VERSIONS = @('3.8', '3.9', '3.10', '3.11', '3.12')
          $ARCHITECTURES = @('x86', 'x64')

          $artifact_reports_dir = "./all_pytest_reports"
          New-Item -ItemType Directory -Force -Path $artifact_reports_dir -ErrorAction SilentlyContinue
          #Dir -Recurse $artifact_reports_dir | Get-Childitem
          Get-ChildItem $artifact_reports_dir | ForEach-Object {
            Write-Output "Moving $($_.FullName) to $testsResultsCommitPath..."
            Move-Item -LiteralPath $_.FullName -Destination $testsResultsCommitPath
          }
          Remove-Item -Path $artifact_reports_dir -Recurse -Force -ErrorAction SilentlyContinue

          git fetch origin $branchName
          git add $testsResultsPath --force
          # Checking if there are newer commits on the remote branch than the commit that triggered the workflow
          if (git log "${{ github.sha }}"..origin/$branchName --oneline) {
              Write-Error "Newer commits are present on the remote branch, cannot update readme"
              exit 1
          }
          git commit -m "Add test results"
          git push --force-with-lease origin HEAD:${{ github.ref_name }}
          $commitSHA = git rev-parse HEAD

          $testResults = @{}

          Get-ChildItem $testsResultsCommitPath -Recurse -Filter pytest_report.xml | ForEach-Object {
            [xml]$TestResultsXml = Get-Content $_.FullName
            $totalTests = [int]$TestResultsXml.testsuites.testsuite.tests
            $failedTests = [int]$TestResultsXml.testsuites.testsuite.failures
            $errors = [int]$TestResultsXml.testsuites.testsuite.errors
            $passedTests = $totalTests - $failedTests - $errors

            $resultFilePathHtml = $_.FullName -replace '\.xml$', '.html'
            $relHtmlFilePath = Resolve-Path -Path $resultFilePathHtml -Relative
            if ($relHtmlFilePath.StartsWith(".\") -or $relHtmlFilePath.StartsWith("./")) {
              $cleanRelHtmlFilePath = $relHtmlFilePath.Substring(2)
            } else {
              $cleanRelHtmlFilePath = $relHtmlFilePath
            }
            $DetailsURL = "https://github.com/$repository/blob/$commitSHA/$cleanRelHtmlFilePath"
            $key = $_.Directory.Name.Replace('pytest_report_', '').Replace('_', '-')
            Write-Host "KEY FOUND: '$key'"

            $testResults[$key] = @{
              Passed = $passedTests
              Failed = $failedTests + $errors
              Total  = $totalTests
              DetailsURL = $DetailsURL
            }
          }

          $ReadmePath = "./README.md"
          $ReadmeContent = Get-Content $ReadmePath -Raw

          $WindowsBadgeContent = ""
          $LinuxBadgeContent = ""
          $MacOSBadgeContent = ""

          $windowsBadgesStartPlaceholder = "<!-- WINDOWS-BADGES-START -->"
          $windowsBadgesEndPlaceholder = "<!-- WINDOWS-BADGES-END -->"
          $linuxBadgesStartPlaceholder = "<!-- LINUX-BADGES-START -->"
          $linuxBadgesEndPlaceholder = "<!-- LINUX-BADGES-END -->"
          $macosBadgesStartPlaceholder = "<!-- MACOS-BADGES-START -->"
          $macosBadgesEndPlaceholder = "<!-- MACOS-BADGES-END -->"

          function Replace-BadgeContent {
            param (
              [string]$readmeContent,
              [string]$badgeContent,
              [string]$startPlaceholder,
              [string]$endPlaceholder
            )

            $pattern = [regex]::Escape($startPlaceholder) + "(.|\n)*?" + [regex]::Escape($endPlaceholder)
            $replacement = $startPlaceholder + "`n" + $badgeContent + "`n" + $endPlaceholder
            return $readmeContent -replace $pattern, $replacement
          }

          foreach ($OS in $OS_NAMES) {
            foreach ($PYTHON_VERSION in $PYTHON_VERSIONS) {
              foreach ($ARCH in $ARCHITECTURES) {
                if ($OS -ne "windows-2019" -and $ARCH -eq "x86") {
                  continue  # no x86 support for unix.
                }
                $key = "$OS-$PYTHON_VERSION-$ARCH"
                $shortKey = "$PYTHON_VERSION-$ARCH"
                if ($testResults.ContainsKey($key)) {
                    $passedTests = $testResults[$key]['Passed']
                    $failedTests = $testResults[$key]['Failed']
                    $DetailsURL = $testResults[$key]['DetailsURL']
                    # Encode the label to replace spaces with underscores and URI-encode other special characters
                    $encodedKey = [System.Web.HttpUtility]::UrlEncode($shortKey.Replace(' ', '_').Replace('-', '--'))
                    $BadgeURLPassed = "https://img.shields.io/badge/${encodedKey}_Passed-${passedTests}-brightgreen"
                    $BadgeMarkdown = '[![' + $key + '](https://img.shields.io/badge/build-' + $encodedKey + '_Passing_' + $passedTests + '-brightgreen?style=plastic&logo=simple-icons&logoColor=%23FF5e34&label=' + $failedTests + '&labelColor=%23c71818&color=%232f991a)](' + $DetailsURL + ')'
                } else {
                    Write-Host "No test results for '$key', must have failed, generating 'Build Failed' badge..."
                    $encodedKey = [System.Web.HttpUtility]::UrlEncode($shortKey.Replace(' ', '_').Replace('-', '--'))
                    $BadgeURLBuildFailed = "https://img.shields.io/badge/${encodedKey}_Build_Failed-lightgrey"
                    $DetailsURL = "https://github.com/$repository/actions/runs/${{ github.run_id }}"
                    $BadgeMarkdown = "[![$shortKey-Build_Failed]($BadgeURLBuildFailed)]($DetailsURL)"
                }

                switch ($OS) {
                    "windows-2019" { $WindowsBadgeContent += $BadgeMarkdown + "`n" }
                    "ubuntu-20.04" { $LinuxBadgeContent += $BadgeMarkdown + "`n" }
                    "macos-12" { $MacOSBadgeContent += $BadgeMarkdown + "`n" }
                }
              }
            }
          }

          $ReadmeContent = Replace-BadgeContent -readmeContent $ReadmeContent -badgeContent $WindowsBadgeContent.TrimEnd() -startPlaceholder $windowsBadgesStartPlaceholder -endPlaceholder $windowsBadgesEndPlaceholder
          $ReadmeContent = Replace-BadgeContent -readmeContent $ReadmeContent -badgeContent $LinuxBadgeContent.TrimEnd() -startPlaceholder $linuxBadgesStartPlaceholder -endPlaceholder $linuxBadgesEndPlaceholder
          $ReadmeContent = Replace-BadgeContent -readmeContent $ReadmeContent -badgeContent $MacOSBadgeContent.TrimEnd() -startPlaceholder $macosBadgesStartPlaceholder -endPlaceholder $macosBadgesEndPlaceholder

          Set-Content -Path $ReadmePath -Value $ReadmeContent
          git fetch origin $branchName
          git add $ReadmePath
          # Checking if there are newer commits on the remote branch than the commit that triggered the workflow
          if (git log "${{ github.sha }}"..origin/$branchName --oneline) {
              Write-Error "Newer commits are present on the remote branch, cannot update readme"
              exit 1
          }
          git commit -m "Update README.md status badges."
          git push --force-with-lease origin HEAD:${{ github.ref_name }}