This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compile and Test PyKotor | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| - push | |
| - workflow_dispatch | |
| permissions: | |
| contents: write | |
| jobs: | |
| build: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false # Disable automatic cancellation of other jobs | |
| matrix: | |
| os: [windows-2019, ubuntu-20.04, macos-12] | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] | |
| architecture: ['x86', 'x64'] | |
| include: | |
| - arch: x86 | |
| vc_redist2015: "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x86.exe" | |
| vc_redist-latest: "https://aka.ms/vs/17/release/vc_redist.x86.exe" | |
| vc_redist2019: "https://aka.ms/vs/17/release/vc_redist.x86.exe" | |
| - arch: x64 | |
| vc_redist2015: "https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x64.exe" | |
| vc_redist-latest: "https://aka.ms/vs/17/release/vc_redist.x64.exe" | |
| vc_redist2019: "https://aka.ms/vs/17/release/vc_redist.x64.exe" | |
| exclude: | |
| # unix x86 is definitely not supported. | |
| - os: ubuntu-20.04 | |
| architecture: x86 | |
| - os: macos-12 | |
| architecture: x86 | |
| outputs: | |
| matrix-os: ${{ toJson(matrix.os) }} | |
| matrix-python-version: ${{ toJson(matrix.python-version) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| if: ${{ runner.os != 'macOS' }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| architecture: ${{ matrix.architecture }} | |
| - name: Set up Python macOS | |
| if: ${{ runner.os == 'macOS' }} | |
| run: | # warning: `brew link --overwrite python@ver` is unsafe on any non-virtualized macos. | |
| echo "NONINTERACTIVE DEFAULT: $NONINTERACTIVE" | |
| export NONINTERACTIVE=1 | |
| echo "NONINTERACTIVE NEW: $NONINTERACTIVE" | |
| brew analytics on | |
| brew update | |
| brew install python@${{ matrix.python-version }} || brew link --overwrite python@${{ matrix.python-version }} | |
| - name: Reset APT sources to default | |
| if: ${{ runner.os == 'Linux' }} | |
| run: | | |
| echo "Resetting APT sources to default Ubuntu repositories" | |
| sudo rm /etc/apt/sources.list | |
| echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list | |
| echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list | |
| echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-backports main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list | |
| echo "deb http://security.ubuntu.com/ubuntu $(lsb_release -cs)-security main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list | |
| sudo apt-get update -y | |
| - name: Setup python venvs | |
| run: | # create the venv early to work around an issue with the matrix runners' concurrency | |
| $pythonExeName = "python" | |
| if ("${{ runner.os }}" -ne "Windows") | |
| { | |
| $pythonExeName = "python${{ matrix.python-version }}" | |
| } | |
| & $pythonExeName -m venv .venv_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| & $pythonExeName -m venv .venv_holopatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| & $pythonExeName -m venv .venv_toolset_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| & $pythonExeName -m venv .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| & $pythonExeName -m venv .venv_kotordiff_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| & $pythonExeName -m venv .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| shell: pwsh | |
| - name: Set UPX download URL | |
| # upx docs express that crashes are happening on ventura and above with upx, don't use on mac. | |
| if: runner.os != 'macOS' | |
| id: upx_setup | |
| run: | | |
| $build = "no" | |
| $archiveName = "" | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| if ("${{ matrix.architecture }}" -eq "x86") { | |
| $archiveName = "upx-4.2.2-win32.zip" | |
| } else { | |
| $archiveName = "upx-4.2.2-win64.zip" | |
| } | |
| } elseif ("${{ runner.os }}" -eq "Linux") { | |
| $archiveName = "upx-4.2.2-amd64_linux.tar.xz" | |
| } elseif ("${{ runner.os }}" -eq "macOS") { | |
| $build = "yes" | |
| $archiveName = "upx-4.2.2-src.tar.xz" | |
| } | |
| $url = "https://github.com/upx/upx/releases/download/v4.2.2/$archiveName" | |
| # Write to the GITHUB_OUTPUT environment file | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "build=$build" | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "url=$url" | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "archiveName=$archiveName" | |
| shell: pwsh | |
| - name: Download and prepare UPX | |
| if: runner.os != 'macOS' | |
| run: | | |
| $ext = "${{ runner.os }}" -eq "Windows" ? "zip" : "tar.xz" | |
| $url = "${{ steps.upx_setup.outputs.url }}" | |
| $archiveName = "${{ steps.upx_setup.outputs.archiveName }}" | |
| $outputPath = "upx-dir" | |
| # Use Invoke-WebRequest or curl depending on the OS | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| Invoke-WebRequest -Uri $url -OutFile $archiveName | |
| } elseif ("${{ runner.os }}" -eq "Linux") { | |
| curl -L $url -o $archiveName | |
| } | |
| New-Item -ItemType Directory -Force -Path "upx-dir" -ErrorAction SilentlyContinue | |
| if ("${{ runner.os }}" -ne "macOS") { | |
| if ($ext -eq "zip") { | |
| $fileNameWithoutExtension = [System.IO.Path]::GetFileNameWithoutExtension($archiveName) | |
| Expand-Archive -Path $archiveName -DestinationPath "temp_folder_upx" | |
| # Ensure upx-dir exists; create it if it doesn't | |
| if (-not (Test-Path -Path "upx-dir")) { | |
| New-Item -ItemType Directory -Path "upx-dir" | |
| } | |
| Get-ChildItem -Path "temp_folder_upx/$fileNameWithoutExtension" -Recurse | Move-Item -Destination "upx-dir" | |
| Remove-Item "temp_folder_upx" -Recurse -Force -ErrorAction SilentlyContinue | |
| } else { | |
| tar -xvf $archiveName --strip-components=1 -C "upx-dir" | |
| } | |
| Remove-Item $archiveName # Clean up downloaded archive | |
| } | |
| shell: pwsh | |
| - name: Set UPX directory path | |
| if: runner.os != 'macOS' | |
| id: upx_dir | |
| run: | | |
| $upx_dir = "./upx-dir" | |
| $upx_dir = $([System.IO.Path]::GetFullPath('./upx-dir')) | |
| Dir -Recurse $upx_dir | Get-Childitem | |
| echo "UPX_DIR=$upx_dir" | Out-File -FilePath $env:GITHUB_ENV -Append | |
| Write-Output "UPX_DIR set to '$upx_dir'" | |
| shell: pwsh | |
| - name: Install Visual Studio 2015 C++ Redistributable | |
| if: runner.os == 'Windows' | |
| run: | | |
| $url = "${{ matrix.vc_redist2015 }}" | |
| $output = "vc_redist.exe" | |
| Invoke-WebRequest -Uri $url -OutFile $output | |
| Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait | |
| Remove-Item -Path $output | |
| #choco install vcredist2015 -y | |
| shell: pwsh | |
| - name: Install Visual Studio 2019 C++ Redistributable | |
| if: runner.os == 'Windows' | |
| run: | | |
| $url = "${{ matrix.vc_redist2019 }}" | |
| $output = "vc_redist.exe" | |
| Invoke-WebRequest -Uri $url -OutFile $output | |
| Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait | |
| Remove-Item -Path $output | |
| #choco install vcredist2019 -y | |
| shell: pwsh | |
| - name: Install Visual Studio latest C++ Redistributable | |
| if: runner.os == 'Windows' | |
| run: | | |
| $url = "${{ matrix.vc_redist-latest }}" | |
| $output = "vc_redist.exe" | |
| Invoke-WebRequest -Uri $url -OutFile $output | |
| Start-Process $output -ArgumentList '/install', '/quiet', '/norestart' -Wait | |
| Remove-Item -Path $output | |
| shell: pwsh | |
| - name: Install Holocron Toolset dependencies | |
| if: ${{ success() || failure() }} | |
| id: toolset_deps | |
| run: | # known pyinstaller versions that work with upx compressions, not all are available on all python versions. | |
| try { | |
| ./install_python_venv.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| python -m pip install --upgrade pip | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| if ("${{ matrix.python-version }}" -eq "3.12") { | |
| pip install "pyinstaller==5.13" --prefer-binary | |
| } elseif ("${{ matrix.python-version }}" -eq "3.11") { | |
| pip install "pyinstaller==5.6.1" --prefer-binary | |
| pip install "pyinstaller==5.5" --prefer-binary | |
| } else { | |
| pip install "pyinstaller==5.4" --prefer-binary | |
| } | |
| } else { | |
| pip install pyinstaller -U --prefer-binary | |
| } | |
| $output = "" | |
| $errorLines = @() | |
| ./compile/deps_toolset.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} 2>&1 | ForEach-Object { | |
| Write-Output $_.ToString() | |
| $output += $_.ToString() + "`n" | |
| if($_ -match 'ERROR:') { | |
| $errorLines += $_.ToString() | |
| } | |
| } | |
| if ($errorLines.Count -gt 0) { | |
| $errorLines | ForEach-Object { Write-Error $_ } | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } else { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true" | |
| } | |
| } catch { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } | |
| shell: pwsh | |
| - name: Compile Holocron Toolset | |
| if: ${{ (success() || failure()) && steps.toolset_deps.outputs.success == 'true' }} | |
| run: | | |
| $upxDir = $env:UPX_DIR | |
| Write-Host "Using UPX directory at '$upxDir'" | |
| $output = "" | |
| ./compile/compile_toolset.ps1 -noprompt -venv_name .venv_toolset_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object { | |
| Write-Output $_.ToString() | |
| $output += $_.ToString() + "`n" | |
| if($_ -match 'ERROR:') { | |
| $errorLines += $_.ToString() | |
| } | |
| } | |
| $warningCount = 0 | |
| $output -split "`n" | ForEach-Object { | |
| if ($_ -match 'WARNING: Library not found: could not resolve' -or | |
| $_ -match 'WARNING: Cannot find ' -or | |
| $_ -match 'WARNING: lib not found:' -or | |
| $_ -match 'WARNING: Tcl modules directory') { | |
| $warningCount++ | |
| } | |
| } | |
| if ($errorLines.Count -gt 0) { | |
| $errorLines | ForEach-Object { Write-Error $_ } | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } elseif ($warningCount -ge 3) { | |
| Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful." | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } else { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true" | |
| } | |
| shell: pwsh | |
| env: | |
| UPX_DIR: ${{ env.UPX_DIR }} | |
| - name: Install HoloPatcher dependencies | |
| if: ${{ success() || failure() }} | |
| id: holopatcher_deps | |
| run: | # known pyinstaller versions that work with upx compressions, not all are available on all python versions. | |
| try { | |
| ./install_python_venv.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| python -m pip install --upgrade pip | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| if ("${{ matrix.python-version }}" -eq "3.12") { | |
| pip install "pyinstaller==5.13" --prefer-binary | |
| } elseif ("${{ matrix.python-version }}" -eq "3.11") { | |
| pip install "pyinstaller==5.6.1" --prefer-binary | |
| pip install "pyinstaller==5.5" --prefer-binary | |
| } else { | |
| pip install "pyinstaller==5.4" --prefer-binary | |
| } | |
| } else { | |
| pip install pyinstaller -U --prefer-binary | |
| } | |
| $output = "" | |
| $errorLines = @() | |
| ./compile/deps_holopatcher.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} 2>&1 | ForEach-Object { | |
| Write-Output $_.ToString() | |
| $output += $_.ToString() + "`n" | |
| if($_ -match 'ERROR:') { | |
| $errorLines += $_.ToString() | |
| } | |
| } | |
| if ($errorLines.Count -gt 0) { | |
| $errorLines | ForEach-Object { Write-Error $_ } | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } else { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true" | |
| } | |
| } catch { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } | |
| shell: pwsh | |
| - name: Compile HoloPatcher | |
| if: ${{ (success() || failure()) && steps.holopatcher_deps.outputs.success == 'true' }} | |
| run: | | |
| $upxDir = $env:UPX_DIR | |
| Write-Host "Using UPX directory at '$upxDir'" | |
| $output = "" | |
| ./compile/compile_holopatcher.ps1 -noprompt -venv_name .venv_holopatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object { | |
| Write-Output $_.ToString() | |
| $output += $_.ToString() + "`n" | |
| if($_ -match 'ERROR:') { | |
| $errorLines += $_.ToString() | |
| } | |
| } | |
| $warningCount = 0 | |
| $output -split "`n" | ForEach-Object { | |
| if ($_ -match 'WARNING: Library not found: could not resolve' -or | |
| $_ -match 'WARNING: Cannot find ' -or | |
| $_ -match 'WARNING: lib not found:' -or | |
| $_ -match 'WARNING: Tcl modules directory') { | |
| $warningCount++ | |
| } | |
| } | |
| if ($errorLines.Count -gt 0) { | |
| $errorLines | ForEach-Object { Write-Error $_ } | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } elseif ($warningCount -ge 3) { | |
| Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful." | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } else { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true" | |
| } | |
| shell: pwsh | |
| - name: Install BatchPatcher dependencies | |
| if: ${{ success() || failure() }} | |
| id: batchpatcher_deps | |
| run: | # known pyinstaller versions that work with upx compressions, not all are available on all python versions. | |
| try { | |
| ./install_python_venv.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| python -m pip install --upgrade pip | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| if ("${{ matrix.python-version }}" -eq "3.12") { | |
| pip install "pyinstaller==5.13" --prefer-binary | |
| } elseif ("${{ matrix.python-version }}" -eq "3.11") { | |
| pip install "pyinstaller==5.6.1" --prefer-binary | |
| pip install "pyinstaller==5.5" --prefer-binary | |
| } else { | |
| pip install "pyinstaller==5.4" --prefer-binary | |
| } | |
| } else { | |
| pip install pyinstaller -U --prefer-binary | |
| } | |
| $output = "" | |
| $errorLines = @() | |
| ./compile/deps_batchpatcher.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} 2>&1 | ForEach-Object { | |
| Write-Output $_.ToString() | |
| $output += $_.ToString() + "`n" | |
| if($_ -match 'ERROR:') { | |
| $errorLines += $_.ToString() | |
| } | |
| } | |
| if ($errorLines.Count -gt 0) { | |
| $errorLines | ForEach-Object { Write-Error $_ } | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } else { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true" | |
| } | |
| } catch { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } | |
| shell: pwsh | |
| - name: Compile BatchPatcher | |
| if: ${{ (success() || failure()) && steps.batchpatcher_deps.outputs.success == 'true' }} | |
| run: | | |
| $upxDir = $env:UPX_DIR | |
| Write-Host "Using UPX directory at '$upxDir'" | |
| $output = "" | |
| ./compile/compile_batchpatcher.ps1 -noprompt -venv_name .venv_batchpatcher_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object { | |
| Write-Output $_.ToString() | |
| $output += $_.ToString() + "`n" | |
| if($_ -match 'ERROR:') { | |
| $errorLines += $_.ToString() | |
| } | |
| } | |
| $warningCount = 0 | |
| $output -split "`n" | ForEach-Object { | |
| if ($_ -match 'WARNING: Library not found: could not resolve' -or | |
| $_ -match 'WARNING: Cannot find ' -or | |
| $_ -match 'WARNING: lib not found:' -or | |
| $_ -match 'WARNING: Tcl modules directory') { | |
| $warningCount++ | |
| } | |
| } | |
| if ($errorLines.Count -gt 0) { | |
| $errorLines | ForEach-Object { Write-Error $_ } | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } elseif ($warningCount -ge 3) { | |
| Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful." | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } else { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true" | |
| } | |
| shell: pwsh | |
| - name: Compile KotorDiff | |
| if: ${{ success() || failure() }} | |
| run: | | |
| $upxDir = $env:UPX_DIR | |
| Write-Host "Using UPX directory at '$upxDir'" | |
| ./install_python_venv.ps1 -noprompt -venv_name .venv_kotordiff_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| python -m pip install --upgrade pip | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| if ("${{ matrix.python-version }}" -eq "3.12") { | |
| pip install "pyinstaller==5.13" --prefer-binary | |
| } elseif ("${{ matrix.python-version }}" -eq "3.11") { | |
| pip install "pyinstaller==5.6.1" --prefer-binary | |
| pip install "pyinstaller==5.5" --prefer-binary | |
| } else { | |
| pip install "pyinstaller==5.4" --prefer-binary | |
| } | |
| } else { | |
| pip install pyinstaller -U --prefer-binary | |
| } | |
| pip install -r Tools/KotorDiff/requirements.txt --prefer-binary | |
| $output = "" | |
| ./compile/compile_kotordiff.ps1 -noprompt -venv_name .venv_kotordiff_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object { | |
| Write-Output $_.ToString() | |
| $output += $_.ToString() + "`n" | |
| if($_ -match 'ERROR:') { | |
| $errorLines += $_.ToString() | |
| } | |
| } | |
| $warningCount = 0 | |
| $output -split "`n" | ForEach-Object { | |
| if ($_ -match 'WARNING: Library not found: could not resolve' -or | |
| $_ -match 'WARNING: Cannot find ' -or | |
| $_ -match 'WARNING: lib not found:' -or | |
| $_ -match 'WARNING: Tcl modules directory') { | |
| $warningCount++ | |
| } | |
| } | |
| if ($errorLines.Count -gt 0) { | |
| $errorLines | ForEach-Object { Write-Error $_ } | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } elseif ($warningCount -ge 3) { | |
| Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful." | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } else { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true" | |
| } | |
| shell: pwsh | |
| - name: Compile GUIDuplicator | |
| if: ${{ success() || failure() }} | |
| run: | | |
| $upxDir = $env:UPX_DIR | |
| Write-Host "Using UPX directory at '$upxDir'" | |
| ./install_python_venv.ps1 -noprompt -venv_name .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| python -m pip install --upgrade pip | |
| if ("${{ runner.os }}" -eq "Windows") { | |
| if ("${{ matrix.python-version }}" -eq "3.12") { | |
| pip install "pyinstaller==5.13" --prefer-binary | |
| } elseif ("${{ matrix.python-version }}" -eq "3.11") { | |
| pip install "pyinstaller==5.6.1" --prefer-binary | |
| pip install "pyinstaller==5.5" --prefer-binary | |
| } else { | |
| pip install "pyinstaller==5.4" --prefer-binary | |
| } | |
| } else { | |
| pip install pyinstaller -U --prefer-binary | |
| } | |
| pip install -r Tools/GuiDuplicator/requirements.txt --prefer-binary | |
| $output = "" | |
| ./compile/compile_gui_duplicator.ps1 -noprompt -venv_name .venv_guiduplicator_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} -upx_dir $upxDir 2>&1 | ForEach-Object { | |
| Write-Output $_.ToString() | |
| if($_ -match 'ERROR:') { | |
| $errorLines += $_.ToString() | |
| } | |
| } | |
| $warningCount = 0 | |
| $output -split "`n" | ForEach-Object { | |
| if ($_ -match 'WARNING: Library not found: could not resolve' -or | |
| $_ -match 'WARNING: Cannot find ' -or | |
| $_ -match 'WARNING: lib not found:' -or | |
| $_ -match 'WARNING: Tcl modules directory') { | |
| $warningCount++ | |
| } | |
| } | |
| if ($errorLines.Count -gt 0) { | |
| $errorLines | ForEach-Object { Write-Error $_ } | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } elseif ($warningCount -ge 3) { | |
| Write-Output "Many 'library not found' warnings raised, pyinstaller was probably unsuccessful." | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=false" | |
| exit 1 | |
| } else { | |
| Add-Content -Path $env:GITHUB_OUTPUT -Value "success=true" | |
| } | |
| shell: pwsh | |
| - name: Upload compiled binaries | |
| if: ${{ success() || failure() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: publish_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| path: ./dist/** | |
| retention-days: 90 | |
| - name: Install MinGW with GCC on Windows | |
| if: runner.os == 'Windows' && matrix.python-version == '3.13.0-alpha.4' | |
| run: | | |
| choco install mingw -y | |
| $mingwPath = Get-ChildItem -Path C:\, D:\, B:\ -Filter mingw64 -Recurse -ErrorAction SilentlyContinue -Directory | Select-Object -First 1 -ExpandProperty FullName | |
| if (-not $mingwPath) { throw "MinGW installation not found" } | |
| $binPath = Join-Path -Path $mingwPath -ChildPath "bin" | |
| echo "$binPath" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| shell: pwsh | |
| - name: Install development packages | |
| if: ${{ success() || failure() }} | |
| run: | | |
| ./install_python_venv.ps1 -noprompt -venv_name .venv_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| python -m pip install --upgrade pip | |
| if ($env:python_version -eq "3.7") { | |
| pip install -r requirements-dev-py37.txt --prefer-binary | |
| } | |
| elseif ($env:python_version -ne "3.13.0-alpha.4" -and "${{ runner.os }}" -ne "Linux" -and "${{ runner.os }}" -ne "macOS") { | |
| pip install -r requirements-dev.txt --prefer-binary | |
| } | |
| else { | |
| pip install -r requirements-dev-py313.txt --prefer-binary | |
| } | |
| shell: pwsh | |
| - name: Run all unittests/pytests | |
| if: ${{ success() || failure() }} | |
| run: | | |
| ./install_python_venv.ps1 -noprompt -venv_name .venv_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| python -m pytest tests -v -ra -o log_cli=true --capture=no --junitxml=pytest_report.xml --html=pytest_report.html --self-contained-html --tb=no --continue-on-collection-errors | |
| shell: pwsh | |
| continue-on-error: true | |
| - name: Upload Pytest Reports | |
| if: ${{ success() || failure() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pytest_report_${{ matrix.os }}_${{ matrix.python-version }}_${{ matrix.architecture }} | |
| path: | | |
| pytest_report.html | |
| pytest_report.xml | |
| retention-days: 90 | |
| package: # The goal of this job is to repackage by toolname, rather than all tools by os_pyversion_arch | |
| needs: build # do not start this job until all 'build' jobs complete | |
| if: ${{ success() || failure() }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| filesToArchive: ${{ steps.set-matrix.outputs.matrixJson }} | |
| steps: | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: published_workflow_builds/ | |
| pattern: publish_* | |
| - name: Re-package artifacts by Python version and architecture | |
| shell: pwsh | |
| run: | | |
| $here_dirpath = (Get-Location) | |
| Write-Output "here: '$here_dirpath'" | |
| $packagedArtifactsPath = "$here_dirpath/packaged_artifacts" | |
| New-Item -ItemType Directory -Force -Path $packagedArtifactsPath | |
| $artifactNames = @() | |
| # Navigate to the directory with downloaded artifacts | |
| Set-Location -Path "published_workflow_builds" | |
| Get-ChildItem -Recurse -Path "." -Directory | ForEach-Object { Write-Output $_.FullName } | |
| # Extract each published artifact and repackage | |
| Get-ChildItem -Filter "publish_*" | ForEach-Object { | |
| $matrixPackagePath = $_.FullName | |
| Write-Output "Found matrix package '$matrixPackagePath'" | |
| Get-ChildItem -Recurse -Path "." -Directory | ForEach-Object { Write-Output $_.FullName } | |
| $os, $pythonVersion, $architecture = $_.BaseName -replace '^publish_', '' -split '_' | |
| # Dynamic identification and repackaging | |
| Get-ChildItem -Path $matrixPackagePath | ForEach-Object { | |
| $fullPath = $_.FullName | |
| $fileBaseName = [IO.Path]::GetFileNameWithoutExtension($_.Name) + "`_$architecture" | |
| $fileExtension = $_.Extension | |
| $outerArchiveName = "$fileBaseName`_$pythonVersion" | |
| Write-Output "outerArchiveName '$outerArchiveName'" | |
| $outerZipPath = $packagedArtifactsPath + "/" + $outerArchiveName | |
| Write-Output "outerZipPath '$outerZipPath'" | |
| New-Item -ItemType Directory -Force -Path $outerZipPath | |
| $osSpecificZipName = "$fileBaseName`_$os" | |
| Write-Output "osSpecificZipName '$osSpecificZipName'" | |
| $osSpecificZipPath = $outerZipPath + "/" + $osSpecificZipName | |
| Write-Output "osSpecificZipPath '$osSpecificZipPath'" | |
| # Archive the tool by os identifier. | |
| chmod 777 -R $fullPath | |
| if ((-not $fileExtension) -or (-not $fileExtension.Trim()) -or ($fileExtension.ToLower().Trim() -eq ".app")) { | |
| $outerArchiveName = "$osSpecificZipPath.tar.gz" | |
| # Check if the path is a directory or a file | |
| if (Test-Path $fullPath -PathType Container) { | |
| # It's a directory, archive the entire directory | |
| tar -czf "$outerArchiveName" -C "$fullPath" . | |
| } else { | |
| # It's a file, archive the file specifically | |
| # Determine the parent directory and the file name | |
| $parentDir = Split-Path -Parent $fullPath | |
| $fileName = Split-Path -Leaf $fullPath | |
| tar -czf "$outerArchiveName" -C "$parentDir" "$fileName" | |
| } | |
| } else { | |
| $outerArchiveName = "$osSpecificZipPath.zip" | |
| zip -r -9 "$outerArchiveName" "$fullPath" | |
| } | |
| Write-Output "Compressed '$fullPath' and saved to '$outerArchiveName'" | |
| chmod 777 -R "$outerArchiveName" | |
| $item = Get-Item -Path $outerZipPath -ErrorAction SilentlyContinue | |
| if ($item -ne $null) { | |
| if (-not $artifactNames -contains $item.BaseName) { | |
| $artifactNames += $item.BaseName | |
| } | |
| } | |
| } | |
| } | |
| # Save artifact names to a file | |
| $artifactNames | Out-File -FilePath "../artifact-names.txt" -Encoding UTF8 | |
| # Navigate back to the root of the workspace | |
| Set-Location -Path "../" | |
| - name: Generate matrix for uploading | |
| if: ${{ success() || failure() }} | |
| id: set-matrix | |
| shell: pwsh | |
| run: | | |
| $artifactNames = Get-Content 'artifact-names.txt' -ReadCount 0 | |
| # Initialize an array to hold the artifact names directly | |
| $matrixArray = @() | |
| foreach ($name in $artifactNames) { | |
| if (-not [string]::IsNullOrEmpty($name)) { | |
| Write-Host "Processing artifact name: $name" | |
| # Trim the name and add directly to the array | |
| $matrixArray += $name.trim() | |
| } | |
| } | |
| # Convert the array directly to JSON | |
| $jsonMatrix = $matrixArray | ConvertTo-Json -Depth 5 -Compress | |
| # Use a single line of JSON for the matrix to avoid issues | |
| $singleLineJsonMatrix = $jsonMatrix -replace "`r", "" | |
| $singleLineJsonMatrix = $singleLineJsonMatrix -replace "`n", "" | |
| Write-Host "Matrix JSON:" | |
| Write-Host $singleLineJsonMatrix | |
| echo "matrixJson<<EOF" >> $env:GITHUB_OUTPUT | |
| echo $singleLineJsonMatrix >> $env:GITHUB_OUTPUT | |
| echo "EOF" >> $env:GITHUB_OUTPUT | |
| - name: Upload all repackages for next job | |
| if: ${{ success() || failure() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: all_tool_dists_onearchive | |
| path: packaged_artifacts/** | |
| retention-days: 1 # only used for the next job. | |
| compression-level: 0 | |
| upload: | |
| needs: package | |
| if: ${{ success() || failure() }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| artifactb: ${{ fromJson(needs.package.outputs.filesToArchive) }} | |
| steps: | |
| - name: Download repackages from package job | |
| if: ${{ success() || failure() }} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: all_tool_dists_onearchive | |
| pattern: all_tool_dists* | |
| - name: Upload re-packaged artifact | |
| if: ${{ success() || failure() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.artifactb }} | |
| path: all_tool_dists_onearchive/all_tool_dists_onearchive/${{ matrix.artifactb }} | |
| compression-level: 9 | |
| if-no-files-found: error | |
| add-test-result-status-badges: | |
| needs: build # do not start this job until all 'build' jobs complete | |
| if: ${{ success() || failure() }} | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: add-test-status-badges-${{ github.ref }} | |
| cancel-in-progress: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: all_pytest_reports | |
| pattern: pytest_report_* | |
| - name: Extract and update README with custom test status badges | |
| shell: pwsh | |
| run: | | |
| # Git configuration and commit | |
| git config --global user.name "GitHub Action" | |
| git config --global user.email "[email protected]" | |
| # Determine the branch that triggered the workflow | |
| $branchName = "${{ github.ref_name }}" | |
| $repository_owner = "${{ github.repository_owner }}" | |
| $repository = "${{ github.repository }}" | |
| $commitSHA = "${{ github.sha }}" | |
| $testsResultsPath = "tests/results" | |
| $testsResultsCommitPath = $testsResultsPath + "/$commitSHA" | |
| Remove-Item -Path $testsResultsPath -Recurse -Force -ErrorAction SilentlyContinue | |
| New-Item -ItemType Directory -Force -Path $testsResultsCommitPath -ErrorAction SilentlyContinue | |
| $OS_NAMES = @("windows-2019", "ubuntu-20.04", "macos-12") | |
| $PYTHON_VERSIONS = @('3.8', '3.9', '3.10', '3.11', '3.12') | |
| $ARCHITECTURES = @('x86', 'x64') | |
| $artifact_reports_dir = "./all_pytest_reports" | |
| New-Item -ItemType Directory -Force -Path $artifact_reports_dir -ErrorAction SilentlyContinue | |
| #Dir -Recurse $artifact_reports_dir | Get-Childitem | |
| Get-ChildItem $artifact_reports_dir | ForEach-Object { | |
| Write-Output "Moving $($_.FullName) to $testsResultsCommitPath..." | |
| Move-Item -LiteralPath $_.FullName -Destination $testsResultsCommitPath | |
| } | |
| Remove-Item -Path $artifact_reports_dir -Recurse -Force -ErrorAction SilentlyContinue | |
| git fetch origin $branchName | |
| git add $testsResultsPath --force | |
| # Checking if there are newer commits on the remote branch than the commit that triggered the workflow | |
| if (git log "${{ github.sha }}"..origin/$branchName --oneline) { | |
| Write-Error "Newer commits are present on the remote branch, cannot update readme" | |
| exit 1 | |
| } | |
| git commit -m "Add test results" | |
| git push --force-with-lease origin HEAD:${{ github.ref_name }} | |
| $commitSHA = git rev-parse HEAD | |
| $testResults = @{} | |
| Get-ChildItem $testsResultsCommitPath -Recurse -Filter pytest_report.xml | ForEach-Object { | |
| [xml]$TestResultsXml = Get-Content $_.FullName | |
| $totalTests = [int]$TestResultsXml.testsuites.testsuite.tests | |
| $failedTests = [int]$TestResultsXml.testsuites.testsuite.failures | |
| $errors = [int]$TestResultsXml.testsuites.testsuite.errors | |
| $passedTests = $totalTests - $failedTests - $errors | |
| $resultFilePathHtml = $_.FullName -replace '\.xml$', '.html' | |
| $relHtmlFilePath = Resolve-Path -Path $resultFilePathHtml -Relative | |
| if ($relHtmlFilePath.StartsWith(".\") -or $relHtmlFilePath.StartsWith("./")) { | |
| $cleanRelHtmlFilePath = $relHtmlFilePath.Substring(2) | |
| } else { | |
| $cleanRelHtmlFilePath = $relHtmlFilePath | |
| } | |
| $DetailsURL = "https://github.com/$repository/blob/$commitSHA/$cleanRelHtmlFilePath" | |
| $key = $_.Directory.Name.Replace('pytest_report_', '').Replace('_', '-') | |
| Write-Host "KEY FOUND: '$key'" | |
| $testResults[$key] = @{ | |
| Passed = $passedTests | |
| Failed = $failedTests + $errors | |
| Total = $totalTests | |
| DetailsURL = $DetailsURL | |
| } | |
| } | |
| $ReadmePath = "./README.md" | |
| $ReadmeContent = Get-Content $ReadmePath -Raw | |
| $WindowsBadgeContent = "" | |
| $LinuxBadgeContent = "" | |
| $MacOSBadgeContent = "" | |
| $windowsBadgesStartPlaceholder = "<!-- WINDOWS-BADGES-START -->" | |
| $windowsBadgesEndPlaceholder = "<!-- WINDOWS-BADGES-END -->" | |
| $linuxBadgesStartPlaceholder = "<!-- LINUX-BADGES-START -->" | |
| $linuxBadgesEndPlaceholder = "<!-- LINUX-BADGES-END -->" | |
| $macosBadgesStartPlaceholder = "<!-- MACOS-BADGES-START -->" | |
| $macosBadgesEndPlaceholder = "<!-- MACOS-BADGES-END -->" | |
| function Replace-BadgeContent { | |
| param ( | |
| [string]$readmeContent, | |
| [string]$badgeContent, | |
| [string]$startPlaceholder, | |
| [string]$endPlaceholder | |
| ) | |
| $pattern = [regex]::Escape($startPlaceholder) + "(.|\n)*?" + [regex]::Escape($endPlaceholder) | |
| $replacement = $startPlaceholder + "`n" + $badgeContent + "`n" + $endPlaceholder | |
| return $readmeContent -replace $pattern, $replacement | |
| } | |
| foreach ($OS in $OS_NAMES) { | |
| foreach ($PYTHON_VERSION in $PYTHON_VERSIONS) { | |
| foreach ($ARCH in $ARCHITECTURES) { | |
| if ($OS -ne "windows-2019" -and $ARCH -eq "x86") { | |
| continue # no x86 support for unix. | |
| } | |
| $key = "$OS-$PYTHON_VERSION-$ARCH" | |
| $shortKey = "$PYTHON_VERSION-$ARCH" | |
| if ($testResults.ContainsKey($key)) { | |
| $passedTests = $testResults[$key]['Passed'] | |
| $failedTests = $testResults[$key]['Failed'] | |
| $DetailsURL = $testResults[$key]['DetailsURL'] | |
| # Encode the label to replace spaces with underscores and URI-encode other special characters | |
| $encodedKey = [System.Web.HttpUtility]::UrlEncode($shortKey.Replace(' ', '_').Replace('-', '--')) | |
| $BadgeURLPassed = "https://img.shields.io/badge/${encodedKey}_Passed-${passedTests}-brightgreen" | |
| $BadgeMarkdown = '[](' + $DetailsURL + ')' | |
| } else { | |
| Write-Host "No test results for '$key', must have failed, generating 'Build Failed' badge..." | |
| $encodedKey = [System.Web.HttpUtility]::UrlEncode($shortKey.Replace(' ', '_').Replace('-', '--')) | |
| $BadgeURLBuildFailed = "https://img.shields.io/badge/${encodedKey}_Build_Failed-lightgrey" | |
| $DetailsURL = "https://github.com/$repository/actions/runs/${{ github.run_id }}" | |
| $BadgeMarkdown = "[]($DetailsURL)" | |
| } | |
| switch ($OS) { | |
| "windows-2019" { $WindowsBadgeContent += $BadgeMarkdown + "`n" } | |
| "ubuntu-20.04" { $LinuxBadgeContent += $BadgeMarkdown + "`n" } | |
| "macos-12" { $MacOSBadgeContent += $BadgeMarkdown + "`n" } | |
| } | |
| } | |
| } | |
| } | |
| $ReadmeContent = Replace-BadgeContent -readmeContent $ReadmeContent -badgeContent $WindowsBadgeContent.TrimEnd() -startPlaceholder $windowsBadgesStartPlaceholder -endPlaceholder $windowsBadgesEndPlaceholder | |
| $ReadmeContent = Replace-BadgeContent -readmeContent $ReadmeContent -badgeContent $LinuxBadgeContent.TrimEnd() -startPlaceholder $linuxBadgesStartPlaceholder -endPlaceholder $linuxBadgesEndPlaceholder | |
| $ReadmeContent = Replace-BadgeContent -readmeContent $ReadmeContent -badgeContent $MacOSBadgeContent.TrimEnd() -startPlaceholder $macosBadgesStartPlaceholder -endPlaceholder $macosBadgesEndPlaceholder | |
| Set-Content -Path $ReadmePath -Value $ReadmeContent | |
| git fetch origin $branchName | |
| git add $ReadmePath | |
| # Checking if there are newer commits on the remote branch than the commit that triggered the workflow | |
| if (git log "${{ github.sha }}"..origin/$branchName --oneline) { | |
| Write-Error "Newer commits are present on the remote branch, cannot update readme" | |
| exit 1 | |
| } | |
| git commit -m "Update README.md status badges." | |
| git push --force-with-lease origin HEAD:${{ github.ref_name }} |