From b6fbc85d2d8791da754e91335f827cbf24bc552c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 3 Feb 2025 23:26:43 +0000 Subject: [PATCH] Revise and improve build --- windows-release/azure-pipelines.yml | 215 ++++++++------------ windows-release/layout-command.yml | 12 +- windows-release/stage-layout-embed.yml | 14 +- windows-release/stage-layout-full.yml | 13 +- windows-release/stage-layout-msix.yml | 12 +- windows-release/stage-layout-nuget.yml | 3 +- windows-release/stage-layout-pymanager.yml | 29 ++- windows-release/stage-publish-nugetorg.yml | 28 +-- windows-release/stage-publish-pymanager.yml | 25 +-- windows-release/stage-publish-pythonorg.yml | 128 ++++-------- windows-release/stage-publish-store.yml | 39 ---- windows-release/stage-test-pymanager.yml | 62 ++++++ windows-release/uploadrelease.ps1 | 4 +- 13 files changed, 240 insertions(+), 344 deletions(-) delete mode 100644 windows-release/stage-publish-store.yml create mode 100644 windows-release/stage-test-pymanager.yml diff --git a/windows-release/azure-pipelines.yml b/windows-release/azure-pipelines.yml index 078226c2..e3a8ca24 100644 --- a/windows-release/azure-pipelines.yml +++ b/windows-release/azure-pipelines.yml @@ -11,6 +11,7 @@ parameters: - 'Yhg1s' - 'pablogsal' - 'ambv' + - 'zooba' - '(Other)' - name: GitRemote_Other displayName: "If Other, specify Git remote" @@ -52,14 +53,6 @@ parameters: displayName: "Publish ARM64 build (3.11 and later)" type: boolean default: true -# Because there is no ARM64 Tcl/Tk pre-3.11, we need a separate option -# to keep those builds working when the files are going to be absent. -# Eventually when we stop releasing anything that old, we can drop this -# argument (and make it implicitly always 'true') -- name: ARM64TclTk - displayName: "Use Tcl/Tk for ARM64 (3.11 and later)" - type: boolean - default: true - name: DoPGO displayName: "Run PGO" type: boolean @@ -96,6 +89,10 @@ parameters: displayName: "Build number to publish (0 to skip)" type: number default: '0' +- name: BuildToRepackage + displayName: "Build number to repackage and publish (0 to skip)" + type: number + default: '0' - name: SignNuget displayName: "Enable Nuget signing (not recommended right now)" type: boolean @@ -109,39 +106,44 @@ variables: SourceTag: ${{ parameters.SourceTag }} ${{ if ne(parameters.SourceCommit, 'empty') }}: SourceCommit: ${{ parameters.SourceCommit }} - ${{ else }}: - SourceCommit: '' ${{ if ne(parameters.SigningCertificate, 'Unsigned') }}: SigningCertificate: ${{ parameters.SigningCertificate }} SigningDescription: ${{ parameters.SigningDescription }} - ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: - IsRealSigned: true - ${{ else }}: - IsRealSigned: false - DoFreethreaded: ${{ parameters.DoFreethreaded }} - DoLayout: ${{ parameters.DoLayout }} - DoMSIX: ${{ parameters.DoMSIX }} - DoNuget: ${{ parameters.DoNuget }} - DoEmbed: ${{ parameters.DoEmbed }} - DoMSI: ${{ parameters.DoMSI }} - DoPyManager: ${{ parameters.DoPyManager }} - DoPublish: ${{ parameters.DoPublish }} + IsRealSigned: ${{ eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }} PublishARM64: ${{ parameters.DoARM64 }} + ${{ if ne(parameters.BuildToPublish, '0') }}: + BuildToPublish: build_to_publish + ${{ else }}: + BuildToPublish: current + ${{ if ne(parameters.BuildToRepackage, '0') }}: + BuildToPackage: build_to_package + ${{ else }}: + BuildToPackage: current # QUEUE TIME VARIABLES # PyDotOrgUsername: '' # PyDotOrgServer: '' +resources: + pipelines: + - ${{ if ne(parameters.BuildToPublish, '0') }}: + - pipeline: build_to_publish + source: $(Build.DefinitionName) + version: ${{ parameters.BuildToPublish }} + - ${{ if ne(parameters.BuildToRepackage, '0') }}: + - pipeline: build_to_package + source: $(Build.DefinitionName) + version: ${{ parameters.BuildToRepackage }} + trigger: none pr: none stages: -- ${{ if eq(parameters.BuildToPublish, '0') }}: +- ${{ if and(eq(parameters.BuildToPublish, '0'), eq(parameters.BuildToRepackage, '0')) }}: - stage: Build displayName: Build binaries jobs: - template: stage-build.yml parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} DoFreethreaded: ${{ parameters.DoFreethreaded }} DoPGO: ${{ parameters.DoPGO }} DoPGOARM64: ${{ parameters.DoPGOARM64 }} @@ -152,142 +154,97 @@ stages: jobs: - template: stage-sign.yml parameters: - ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}: - SigningCertificate: ${{ parameters.SigningCertificate }} + SigningCertificate: ${{ parameters.SigningCertificate }} DoFreethreaded: ${{ parameters.DoFreethreaded }} +- ${{ if eq(parameters.BuildToPublish, '0') }}: - stage: Layout - displayName: Generate layouts - dependsOn: Sign + ${{ if eq(parameters.BuildToRepackage, '0') }}: + displayName: Generate layouts + dependsOn: Sign + ${{ else }}: + displayName: Generate layouts from build ${{ parameters.BuildToRepackage }} + dependsOn: [] jobs: - template: stage-layout-full.yml parameters: ARM64TclTk: ${{ parameters.ARM64TclTk }} DoFreethreaded: ${{ parameters.DoFreethreaded }} - - template: stage-layout-embed.yml - - template: stage-layout-nuget.yml - parameters: - DoFreethreaded: ${{ parameters.DoFreethreaded }} - - template: stage-layout-pymanager.yml - parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} - DoFreethreaded: ${{ parameters.DoFreethreaded }} - DoEmbed: ${{ parameters.DoEmbed }} + - ${{ if eq(parameters.DoEmbed, 'true') }}: + - template: stage-layout-embed.yml + - ${{ if eq(parameters.DoNuget, 'true') }}: + - template: stage-layout-nuget.yml + parameters: + DoFreethreaded: ${{ parameters.DoFreethreaded }} + - ${{ if eq(parameters.DoMSIX, 'true') }}: + - template: stage-layout-msix.yml + - ${{ if eq(parameters.DoPyManager, 'true') }}: + - template: stage-layout-pymanager.yml + parameters: + DoFreethreaded: ${{ parameters.DoFreethreaded }} + DoEmbed: ${{ parameters.DoEmbed }} + ${{ if ne(parameters.BuildToRepackage, '0') }}: + UseLayoutScriptFromBranch: main - stage: Pack dependsOn: Layout + displayName: Pack jobs: - - template: stage-pack-nuget.yml - parameters: - ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}: - ${{ if eq(parameters.SignNuget, 'true') }}: - SigningCertificate: ${{ parameters.SigningCertificate }} - DoFreethreaded: ${{ parameters.DoFreethreaded }} - - - stage: Test - dependsOn: Pack - jobs: - - template: stage-test-embed.yml - - template: stage-test-nuget.yml - parameters: - DoFreethreaded: ${{ parameters.DoFreethreaded }} - - - ${{ if eq(parameters.DoMSIX, 'true') }}: - - stage: Layout_MSIX - displayName: Generate MSIX layouts - dependsOn: Sign - jobs: - - template: stage-layout-msix.yml + - ${{ if eq(parameters.DoEmbed, 'true') }}: + - template: stage-pack-embed.yml + - ${{ if eq(parameters.DoMSI, 'true') }}: + - template: stage-msi.yml parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} - - - stage: Pack_MSIX - displayName: Package MSIX - dependsOn: Layout_MSIX - jobs: + DoARM64: ${{ parameters.DoARM64}} + DoFreethreaded: ${{ parameters.DoFreethreaded }} + - ${{ if eq(parameters.DoMSIX, 'true') }}: - template: stage-pack-msix.yml parameters: ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}: SigningCertificate: ${{ parameters.SigningCertificate }} - - - ${{ if eq(parameters.DoMSI, 'true') }}: - - stage: Build_MSI - displayName: Build MSI installer - dependsOn: Sign - jobs: - - template: stage-msi.yml + - ${{ if eq(parameters.DoNuget, 'true') }}: + - template: stage-pack-nuget.yml parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} - ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}: - SigningCertificate: ${{ parameters.SigningCertificate }} - DoARM64: ${{ parameters.DoARM64}} + ${{ if eq(parameters.SignNuget, 'true') }}: + ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}: + SigningCertificate: ${{ parameters.SigningCertificate }} DoFreethreaded: ${{ parameters.DoFreethreaded }} - - stage: Test_MSI - displayName: Test MSI installer - dependsOn: Build_MSI - jobs: + - stage: Test + dependsOn: Pack + jobs: + - ${{ if eq(parameters.DoEmbed, 'true') }}: + - template: stage-test-embed.yml + - ${{ if eq(parameters.DoMSI, 'true') }}: - template: stage-test-msi.yml parameters: DoFreethreaded: ${{ parameters.DoFreethreaded }} - - - ${{ if eq(parameters.DoPublish, 'true') }}: - - ${{ if eq(parameters.DoPyManager, 'true') }}: - - stage: PublishPyManager - displayName: Publish for PyManager - dependsOn: ['Test_MSI', 'Test'] - jobs: - - template: stage-publish-pymanager.yml - - - ${{ if eq(parameters.DoMSI, 'true') }}: - - stage: PublishPyDotOrg - displayName: Publish to python.org - dependsOn: ['Test_MSI', 'Test'] - jobs: - - template: stage-publish-pythonorg.yml - parameters: - IncludeGPG: ${{ parameters.DoGPG }} - - ${{ if eq(parameters.DoNuget, 'true') }}: - - stage: PublishNuget - displayName: Publish to nuget.org - ${{ if eq(parameters.DoMSI, 'true') }}: - dependsOn: ['Test_MSI', 'Test'] - ${{ else }}: - dependsOn: 'Test' - jobs: - - template: stage-publish-nugetorg.yml - - - ${{ if eq(parameters.DoMSIX, 'true') }}: - - stage: PublishStore - displayName: Publish to Store - ${{ if eq(parameters.DoMSI, 'true') }}: - dependsOn: ['Test_MSI', 'Pack_MSIX'] - ${{ else }}: - dependsOn: 'Pack_MSIX' - jobs: - - template: stage-publish-store.yml + - template: stage-test-nuget.yml + parameters: + DoFreethreaded: ${{ parameters.DoFreethreaded }} + - ${{ if eq(parameters.DoPyManager, 'true') }}: + - template: stage-test-pymanager.yml + parameters: + DoEmbed: ${{ parameters.DoEmbed }} + DoFreethreaded: ${{ parameters.DoFreethreaded }} -- ${{ else }}: - - stage: PublishExisting - displayName: Publish existing build - dependsOn: [] +- ${{ if eq(parameters.DoPublish, 'true') }}: + - stage: Publish + displayName: Publish + dependsOn: + - ${{ if eq(parameters.BuildToPublish, '0') }}: + - Test jobs: - ${{ if eq(parameters.DoPyManager, 'true') }}: - template: stage-publish-pymanager.yml - + parameters: + DoEmbed: ${{ parameters.DoEmbed }} + DoFreethreaded: ${{ parameters.DoFreethreaded }} - ${{ if eq(parameters.DoMSI, 'true') }}: - template: stage-publish-pythonorg.yml parameters: - BuildToPublish: ${{ parameters.BuildToPublish }} + DoEmbed: ${{ parameters.DoEmbed }} IncludeGPG: ${{ parameters.DoGPG }} - - ${{ if eq(parameters.DoNuget, 'true') }}: - template: stage-publish-nugetorg.yml - parameters: - BuildToPublish: ${{ parameters.BuildToPublish }} - - - ${{ if eq(parameters.DoMSIX, 'true') }}: - - template: stage-publish-store.yml - parameters: - BuildToPublish: ${{ parameters.BuildToPublish }} diff --git a/windows-release/layout-command.yml b/windows-release/layout-command.yml index 1524bca2..1e70411e 100644 --- a/windows-release/layout-command.yml +++ b/windows-release/layout-command.yml @@ -3,9 +3,10 @@ parameters: Sources: $(Build.SourcesDirectory) Temp: $(Build.BinariesDirectory)\layout-temp Docs: $(Build.BinariesDirectory)\doc + LayoutSources: steps: -- download: current +- download: $(BuildToPackage) artifact: bin_$(HostArch) displayName: 'Download artifact: bin_$(HostArch)' condition: and(succeeded(), variables['HostArch']) @@ -23,7 +24,7 @@ steps: - powershell: > $layout_cmd = '& "$(Python)" - "{1}\PC\layout" + "{4}\PC\layout" -vv --source "{1}" --build "{0}" @@ -31,7 +32,12 @@ steps: --temp "{2}" --include-cat "{0}\python.cat" --doc-build "{3}"' - -f ("${{ parameters.Binaries }}", "${{ parameters.Sources }}", "${{ parameters.Temp }}", "${{ parameters.Docs}}"); + -f ( + "${{ parameters.Binaries }}", + "${{ parameters.Sources }}", + "${{ parameters.Temp }}", + "${{ parameters.Docs}}", + "${{ coalesce(parameters.LayoutSources, parameters.Sources) }}"); Write-Host "##vso[task.setvariable variable=LayoutCmd]$layout_cmd"; Write-Host "Setting LayoutCmd=$layout_cmd" displayName: 'Set LayoutCmd' diff --git a/windows-release/stage-layout-embed.yml b/windows-release/stage-layout-embed.yml index a2888555..28efacd3 100644 --- a/windows-release/stage-layout-embed.yml +++ b/windows-release/stage-layout-embed.yml @@ -28,7 +28,7 @@ jobs: steps: - template: ./checkout.yml - - download: current + - download: $(BuildToPackage) artifact: bin_$(Name) displayName: 'Download artifact: bin_$(Name)' @@ -72,14 +72,10 @@ jobs: artifact: layout_embed_$(Name) displayName: 'Publish Artifact: layout_embed_$(Name)' - - task: PublishBuildArtifacts@1 + - publishBuild: $(Build.ArtifactStagingDirectory)\embed + artifact: embed displayName: 'Publish Artifact: embed' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\embed' - ArtifactName: embed - - task: PublishBuildArtifacts@1 + - publishBuild: $(Build.ArtifactStagingDirectory)\sbom + artifact: sbom displayName: 'Publish Artifact: sbom' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\sbom' - ArtifactName: sbom diff --git a/windows-release/stage-layout-full.yml b/windows-release/stage-layout-full.yml index 07030dc1..de19d510 100644 --- a/windows-release/stage-layout-full.yml +++ b/windows-release/stage-layout-full.yml @@ -1,11 +1,9 @@ parameters: - ARM64TclTk: true DoFreethreaded: false jobs: - job: Make_Layouts displayName: Make layouts - condition: and(succeeded(), eq(variables['DoLayout'], 'true')) pool: vmImage: windows-2022 @@ -35,9 +33,8 @@ jobs: DebugName: arm64_d Arch: arm64 HostArch: amd64 + TclLibrary: tcltk_lib_arm64 ExtraOptions: '' - ${{ if eq(parameters.ARM64TclTk, 'true') }}: - TclLibrary: tcltk_lib_arm64 ${{ if eq(parameters.DoFreethreaded, 'true') }}: win32_t: Name: win32_t @@ -64,11 +61,11 @@ jobs: steps: - template: ./checkout.yml - - download: current + - download: $(BuildToPackage) artifact: bin_$(Name) displayName: 'Download artifact: bin_$(Name)' - - download: current + - download: $(BuildToPackage) artifact: bin_$(DebugName) displayName: 'Download artifact: bin_$(DebugName)' @@ -79,11 +76,11 @@ jobs: %{ copy $_.FullName $dest } displayName: 'Copy debug binaries' - - download: current + - download: $(BuildToPackage) artifact: doc displayName: 'Download artifact: doc' - - download: current + - download: $(BuildToPackage) artifact: $(TclLibrary) displayName: 'Download artifact: $(TclLibrary)' condition: and(succeeded(), variables['TclLibrary']) diff --git a/windows-release/stage-layout-msix.yml b/windows-release/stage-layout-msix.yml index e45053fa..4f394a85 100644 --- a/windows-release/stage-layout-msix.yml +++ b/windows-release/stage-layout-msix.yml @@ -1,6 +1,3 @@ -parameters: - ARM64TclTk: true - jobs: - job: Make_MSIX_Layout displayName: Make MSIX layout @@ -29,18 +26,17 @@ jobs: Name: arm64 Arch: arm64 HostArch: amd64 + TclLibrary: tcltk_lib_arm64 ExtraOptions: --precompile - ${{ if eq(parameters.ARM64TclTk, 'true') }}: - TclLibrary: tcltk_lib_arm64 steps: - template: ./checkout.yml - - download: current + - download: $(BuildToPackage) artifact: bin_$(Name) displayName: 'Download artifact: bin_$(Name)' - - download: current + - download: $(BuildToPackage) artifact: $(TclLibrary) displayName: 'Download artifact: $(TclLibrary)' condition: and(succeeded(), variables['TclLibrary']) @@ -50,7 +46,7 @@ jobs: displayName: 'Update TCL_LIBRARY' condition: and(succeeded(), variables['TclLibrary']) - - download: current + - download: $(BuildToPackage) artifact: cert displayName: 'Download artifact: cert' condition: and(succeeded(), variables['SigningCertificate']) diff --git a/windows-release/stage-layout-nuget.yml b/windows-release/stage-layout-nuget.yml index bf69900b..b124b4ed 100644 --- a/windows-release/stage-layout-nuget.yml +++ b/windows-release/stage-layout-nuget.yml @@ -4,7 +4,6 @@ parameters: jobs: - job: Make_Nuget_Layout displayName: Make Nuget layout - condition: and(succeeded(), eq(variables['DoNuget'], 'true')) pool: vmImage: windows-2022 @@ -50,7 +49,7 @@ jobs: steps: - template: ./checkout.yml - - download: current + - download: $(BuildToPackage) artifact: bin_$(Name) displayName: 'Download artifact: bin_$(Name)' diff --git a/windows-release/stage-layout-pymanager.yml b/windows-release/stage-layout-pymanager.yml index 638fa1c7..9d627ffb 100644 --- a/windows-release/stage-layout-pymanager.yml +++ b/windows-release/stage-layout-pymanager.yml @@ -1,12 +1,11 @@ parameters: - ARM64TclTk: true DoFreethreaded: false DoEmbed: false + UseLayoutScriptFromBranch: jobs: - job: Make_PyManager_Layouts displayName: Make PyManager layouts - condition: and(succeeded(), eq(variables['DoPyManager'], 'true')) pool: vmImage: windows-2022 @@ -35,10 +34,9 @@ jobs: Name: arm64 Arch: arm64 HostArch: amd64 + TclLibrary: tcltk_lib_arm64\tcl8 LayoutOptions: '--preset-pymanager' IncludeDoc: true - ${{ if eq(parameters.ARM64TclTk, 'true') }}: - TclLibrary: tcltk_lib_arm64\tcl8 win32_test: Name: win32_test Arch: win32 @@ -55,10 +53,9 @@ jobs: Name: arm64_test Arch: arm64 HostArch: amd64 + TclLibrary: tcltk_lib_arm64\tcl8 LayoutOptions: '--preset-pymanager-test' IncludeDoc: true - ${{ if eq(parameters.ARM64TclTk, 'true') }}: - TclLibrary: tcltk_lib_arm64\tcl8 ${{ if eq(parameters.DoFreethreaded, 'true') }}: win32_t: Name: win32_t @@ -104,16 +101,28 @@ jobs: IncludeSelf: true Path: $(Build.SourcesDirectory)\cpython - - download: current + - ${{ if parameters.UseLayoutScriptFromBranch }}: + - powershell: > + git clone + --progress + -v + --depth 1 + --branch ${{ parameters.UseLayoutScriptFromBranch }} + --single-branch + https://github.com/python/cpython.git + "$(Build.SourcesDirectory)\layout-script" + displayName: 'Clone PC/layout script from ${{ parameters.UseLayoutScriptFromBranch }}' + + - download: $(BuildToPackage) artifact: bin_$(Name) displayName: 'Download artifact: bin_$(Name)' - - download: current + - download: $(BuildToPackage) artifact: doc displayName: 'Download artifact: doc' condition: and(succeeded(), variables['IncludeDoc']) - - download: current + - download: $(BuildToPackage) artifact: $(TclLibrary) displayName: 'Download artifact: $(TclLibrary)' condition: and(succeeded(), variables['TclLibrary']) @@ -132,6 +141,8 @@ jobs: - template: ./layout-command.yml parameters: Sources: $(Build.SourcesDirectory)\cpython + ${{ if parameters.UseLayoutScriptFromBranch }}: + LayoutSources: $(Build.SourcesDirectory)\layout-script - powershell: > $(LayoutCmd) diff --git a/windows-release/stage-publish-nugetorg.yml b/windows-release/stage-publish-nugetorg.yml index 34702350..869b3652 100644 --- a/windows-release/stage-publish-nugetorg.yml +++ b/windows-release/stage-publish-nugetorg.yml @@ -1,10 +1,7 @@ -parameters: - BuildToPublish: '' - jobs: - job: Publish_Nuget displayName: Publish Nuget packages - condition: and(succeeded(), eq(variables['DoNuget'], 'true'), ne(variables['SkipNugetPublish'], 'true')) + condition: and(succeeded(), ne(variables['SkipNugetPublish'], 'true')) pool: vmImage: windows-2022 @@ -15,25 +12,10 @@ jobs: steps: - checkout: none - - ${{ if parameters.BuildToPublish }}: - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact from ${{ parameters.BuildToPublish }}' - inputs: - artifactName: nuget - downloadPath: $(Build.BinariesDirectory) - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - ${{ else }}: - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: nuget' - inputs: - artifactName: nuget - downloadPath: $(Build.BinariesDirectory) - + - downloadBuild: $(BuildToPublish) + artifact: nuget + path: $(Build.BinariesDirectory) + displayName: 'Download artifact: nuget' - powershell: 'gci pythonarm*.nupkg | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' displayName: 'Prevent publishing ARM64 packages' diff --git a/windows-release/stage-publish-pymanager.yml b/windows-release/stage-publish-pymanager.yml index cc3892f4..d5b56306 100644 --- a/windows-release/stage-publish-pymanager.yml +++ b/windows-release/stage-publish-pymanager.yml @@ -22,25 +22,10 @@ jobs: inputs: versionSpec: '>=3.10' - - ${{ if parameters.BuildToPublish }}: - - ${{ each Name in variables.artifacts }}: - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact from ${{ parameters.BuildToPublish}}: pymanager_${{ Name }}' - inputs: - artifactName: pymanager_${{ Name }} - targetPath: $(Pipeline.Workspace) - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - ${{ else }}: - - ${{ each Name in variables.artifacts }}: - - download: current - artifact: pymanager_${{ Name }} - displayName: 'Download artifact: pymanager_${{ Name }}' - + - ${{ each Name in variables.artifacts }}: + - download: $(BuildToPublish) + artifact: pymanager_${{ Name }} + displayName: 'Download artifact: pymanager_${{ Name }}' - task: DownloadSecureFile@1 name: sshkey @@ -84,5 +69,5 @@ jobs: artifact: pymanager_index - publish: '$(Build.ArtifactStagingDirectory)\hashes' - artifact: hashes + artifact: pymanager_hashes displayName: 'Publish Artifact: hashes' diff --git a/windows-release/stage-publish-pythonorg.yml b/windows-release/stage-publish-pythonorg.yml index dbe1a388..ef293382 100644 --- a/windows-release/stage-publish-pythonorg.yml +++ b/windows-release/stage-publish-pythonorg.yml @@ -1,12 +1,12 @@ parameters: - BuildToPublish: '' - IncludeGPG: true + DoEmbed: true + IncludeGPG: false HashAlgorithms: ['SHA256', 'MD5'] jobs: - job: Publish_Python displayName: Publish python.org packages - condition: and(succeeded(), eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'), ne(variables['SkipPythonOrgPublish'], 'true')) + condition: and(succeeded(), ne(variables['SkipPythonOrgPublish'], 'true')) pool: vmImage: windows-2022 @@ -19,88 +19,34 @@ jobs: steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.6 or later' + displayName: 'Use Python 3.10 or later' inputs: - versionSpec: '>=3.6' + versionSpec: '>=3.10' - - ${{ if parameters.BuildToPublish }}: - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact from ${{ parameters.BuildToPublish }}: Doc' - inputs: - artifactName: Doc - targetPath: $(Build.BinariesDirectory)\Doc - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact from ${{ parameters.BuildToPublish }}: msi' - inputs: - artifactName: msi - targetPath: $(Build.BinariesDirectory)\msi - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - # Note that embed is a 'build' artifact, not a 'pipeline' artifact - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact from ${{ parameters.BuildToPublish }}: embed' - inputs: - artifactName: embed - downloadPath: $(Build.BinariesDirectory) - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact from ${{ parameters.BuildToPublish }}: sbom' - inputs: - artifactName: sbom - downloadPath: $(Build.BinariesDirectory)\sbom - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - ${{ else }}: - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: Doc' - inputs: - artifactName: Doc - targetPath: $(Build.BinariesDirectory)\Doc + - download: $(BuildToPublish) + artifact: Doc + displayName: 'Download artifact: Doc' - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: msi' - inputs: - artifactName: msi - targetPath: $(Build.BinariesDirectory)\msi + - download: $(BuildToPublish) + artifact: msi + displayName: 'Download artifact: msi' - # Note that embed is a 'build' artifact, not a 'pipeline' artifact - - task: DownloadBuildArtifacts@0 + - ${{ if eq(parameters.DoEmbed, 'true') }}: + - downloadBuild: $(BuildToPublish) + artifact: embed displayName: 'Download artifact: embed' - inputs: - artifactName: embed - downloadPath: $(Build.BinariesDirectory) - # Note that sbom is a 'build' artifact, not a 'pipeline' artifact - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: sbom' - inputs: - artifactName: sbom - downloadPath: $(Build.BinariesDirectory)\sbom + - downloadBuild: $(BuildToPublish) + artifact: sbom + displayName: 'Download artifact: sbom' - # Note that ARM64 MSIs are skipped at build when this option is specified - - powershell: 'gci *embed-arm*.zip | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' - displayName: 'Prevent publishing ARM64 packages' - workingDirectory: '$(Build.BinariesDirectory)\embed' - condition: and(succeeded(), ne(variables['PublishARM64'], 'true')) + + - ${{ if eq(parameters.DoEmbed, 'true') }}: + # Note that ARM64 MSIs are skipped at build when this option is specified + - powershell: 'gci *embed-arm*.zip | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' + displayName: 'Prevent publishing ARM64 packages' + workingDirectory: '$(Pipeline.Workspace)\embed' + condition: and(succeeded(), ne(variables['PublishARM64'], 'true')) - ${{ if eq(parameters.IncludeGPG, 'true') }}: @@ -113,13 +59,13 @@ jobs: - powershell: | git clone https://github.com/python/cpython-bin-deps --branch gpg --single-branch --depth 1 --progress -v "gpg" gpg/gpg2.exe --import "$(gpgkey.secureFilePath)" - $files = gci -File "msi\*\*", "embed\*.zip" + $files = gci -File "msi\*\*", "embed\*.zip" -EA SilentlyContinue $files.FullName | %{ gpg/gpg2.exe -ba --batch --passphrase $(GPGPassphrase) $_ "Made signature for $_" } displayName: 'Generate GPG signatures' - workingDirectory: $(Build.BinariesDirectory) + workingDirectory: $(Pipeline.Workspace) - powershell: | $p = gps "gpg-agent" -EA 0 @@ -137,7 +83,7 @@ jobs: - powershell: | git clone https://github.com/python/cpython-bin-deps --branch putty --single-branch --depth 1 --progress -v "putty" "##vso[task.prependpath]$(gi putty)" - workingDirectory: $(Build.BinariesDirectory) + workingDirectory: $(Pipeline.Workspace) displayName: 'Download PuTTY binaries' - powershell: > @@ -149,7 +95,7 @@ jobs: -keyfile "$(sshkey.secureFilePath)" -embed embed -sbom sbom - workingDirectory: $(Build.BinariesDirectory) + workingDirectory: $(Pipeline.Workspace) condition: and(succeeded(), eq(variables['IsRealSigned'], 'true')) displayName: 'Upload files to python.org' @@ -157,7 +103,7 @@ jobs: python "$(Build.SourcesDirectory)\windows-release\purge.py" (gci msi\*\python-*.exe | %{ $_.Name -replace 'python-(.+?)(-|\.exe).+', '$1' } | select -First 1) - workingDirectory: $(Build.BinariesDirectory) + workingDirectory: $(Pipeline.Workspace) condition: and(succeeded(), eq(variables['IsRealSigned'], 'true')) displayName: 'Purge CDN' @@ -178,13 +124,13 @@ jobs: Write-Error "Failed to validate $failures installers" exit 1 } - workingDirectory: $(Build.BinariesDirectory) + workingDirectory: $(Pipeline.Workspace) condition: and(succeeded(), eq(variables['IsRealSigned'], 'true')) displayName: 'Test layouts' - ${{ each alg in parameters.HashAlgorithms }}: - powershell: | - $files = gci -File "msi\*\*.exe", "embed\*.zip" + $files = gci -File "msi\*\*.exe", "embed\*.zip" -EA SilentlyContinue $hashes = $files | ` Sort-Object Name | ` Format-Table Name, @{ @@ -195,22 +141,20 @@ jobs: $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force $hashes | Out-File "$d\hashes.txt" -Encoding ascii -Append $hashes - workingDirectory: $(Build.BinariesDirectory) + workingDirectory: $(Pipeline.Workspace) displayName: 'Generate hashes (${{ alg }})' - ${{ if eq(parameters.IncludeGPG, 'true') }}: - powershell: | "Copying:" - $files = gci -File "msi\*\python*.asc", "embed\*.asc" + $files = gci -File "msi\*\python*.asc", "embed\*.asc" -EA SilentlyContinue $files.FullName $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force move $files $d -Force gci msi -Directory | %{ move "msi\$_\*.asc" (mkdir "$d\$_" -Force) } - workingDirectory: $(Build.BinariesDirectory) + workingDirectory: $(Pipeline.Workspace) displayName: 'Copy GPG signatures for build' - - task: PublishPipelineArtifact@0 + - publish: '$(Build.ArtifactStagingDirectory)\hashes' + artifact: hashes displayName: 'Publish Artifact: hashes' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\hashes' - artifactName: hashes diff --git a/windows-release/stage-publish-store.yml b/windows-release/stage-publish-store.yml deleted file mode 100644 index e342a18a..00000000 --- a/windows-release/stage-publish-store.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: - BuildToPublish: '' - -jobs: -- job: Publish_Store - displayName: Publish Store packages - condition: and(succeeded(), eq(variables['DoMSIX'], 'true'), ne(variables['SkipMSIXPublish'], 'true')) - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - steps: - - checkout: none - - - ${{ if parameters.BuildToPublish }}: - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: msixupload' - inputs: - artifactName: msixupload - downloadPath: $(Build.BinariesDirectory) - buildType: specific - project: cpython - pipeline: Windows-Release - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - ${{ else }}: - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: msixupload' - inputs: - artifactName: msixupload - downloadPath: $(Build.BinariesDirectory) - - # TODO: do publish step - # If we are *not* real-signed, DO NOT PUBLISH - # condition: and(succeeded(), eq(variables['IsRealSigned'], 'true')) diff --git a/windows-release/stage-test-pymanager.yml b/windows-release/stage-test-pymanager.yml new file mode 100644 index 00000000..b699548a --- /dev/null +++ b/windows-release/stage-test-pymanager.yml @@ -0,0 +1,62 @@ +parameters: + DoEmbed: false + DoFreethreaded: false + +jobs: +- job: Test_PyManager + displayName: Test PyManager + + pool: + vmImage: windows-2022 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + win32_test: + Name: win32_test + amd64_test: + Name: amd64_test + ${{ if eq(parameters.DoEmbed, 'true') }}: + win32_embed: + Name: win32_embed + amd64_embed: + Name: amd64_embed + ${{ if eq(parameters.DoFreethreaded, 'true') }}: + win32_t: + Name: win32_t + amd64_t: + Name: amd64_t + + steps: + - checkout: none + + - download: current + artifact: layout_pymanager_$(Name) + displayName: 'Download artifact: layout_pymanager_$(Name)' + + - powershell: | + $p = gi "$(Pipeline.Workspace)\layout_pymanager_$(Name)\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + condition: and(succeeded(), not(variables['SkipTests'])) + + - script: | + python -m site + displayName: 'Collect site' + condition: and(succeeded(), not(variables['SkipTests'])) + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' + condition: and(succeeded(), not(variables['SkipTests'])) diff --git a/windows-release/uploadrelease.ps1 b/windows-release/uploadrelease.ps1 index b974688c..cdc5c0f2 100644 --- a/windows-release/uploadrelease.ps1 +++ b/windows-release/uploadrelease.ps1 @@ -72,10 +72,10 @@ $d = "$target/$($p[0])/" & $plink -batch -hostkey $hostkey -noagent -i $keyfile "$user@$server" chmod "a+rx" $d $dirs = gci "$build" -Directory -if ($embed) { +if ($embed -and (Test-Path $embed)) { $dirs = ($dirs, (gi $embed)) | %{ $_ } } -if ($sbom) { +if ($sbom -and (Test-Path $sbom)) { $dirs = ($dirs, $sbom) | %{ $_ } }