Update the combined package build to release the daily builds (PowerS…

…hell#10449)

.vsts-ci/install-ps.yml

@@ -46,7 +46,7 @@ phases:
   parameters:
     scriptName: sudo ./tools/install-powershell.sh
     jobName: InstallPowerShellUbuntu
-    pool: Hosted Ubuntu 1604
+    pool: ubuntu-latest
     verification: |
       if ([Version]"$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor).$($PSVersionTable.PSVersion.Patch)" -lt [version]"6.2.0")
       {
@@ -57,7 +57,7 @@ phases:
   parameters:
     scriptName: sudo ./tools/install-powershell.sh
     jobName: InstallPowerShellAmazonLinux
-    pool: Hosted Ubuntu 1604
+    pool: ubuntu-latest
     container: pshorg/powershellcommunity-test-deps:amazonlinux-2.0
     verification: |
       if ([Version]"$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor).$($PSVersionTable.PSVersion.Patch)" -lt [version]"6.2.0")
@@ -69,7 +69,7 @@ phases:
   parameters:
     scriptName: sudo ./tools/installpsh-amazonlinux.sh
     jobName: InstallPSHAmazonLinux
-    pool: Hosted Ubuntu 1604
+    pool: ubuntu-latest
     container: pshorg/powershellcommunity-test-deps:amazonlinux-2.0
     verification: |
       if ([Version]"$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor).$($PSVersionTable.PSVersion.Patch)" -lt [version]"6.2.0")
@@ -83,14 +83,14 @@ phases:
   parameters:
     scriptName: ./tools/install-powershell.sh
     jobName: InstallPowerShellCentOS
-    pool: Hosted Ubuntu 1604
+    pool: ubuntu-latest
     container: mcr.microsoft.com/powershell/test-deps:centos-7
-    
+
 - template: templates/install-ps-phase.yml
   parameters:
     scriptName: ./tools/install-powershell.sh
     jobName: InstallPowerShellDebian9
-    pool: Hosted Ubuntu 1604
+    pool: ubuntu-latest
     container: mcr.microsoft.com/powershell/test-deps:debian-9
 
 
@@ -103,7 +103,7 @@ phases:
   parameters:
     scriptName: ./tools/install-powershell.sh
     jobName: InstallPowerShellMacOS
-    pool: Hosted macOS
+    pool: macOS-latest
     verification: |
       if ([Version]"$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor).$($PSVersionTable.PSVersion.Patch)" -lt [version]"6.2.0")
       {
@@ -115,4 +115,44 @@ phases:
   parameters:
     scriptName: pwsh -c ./tools/install-powershell.ps1 -AddToPath
     jobName: InstallPowerShellPS1Ubuntu
-    pool: Hosted Ubuntu 1604
+    pool: ubuntu-latest
+
+- template: templates/install-ps-phase.yml
+  parameters:
+    scriptName: pwsh -c ./tools/install-powershell.ps1 -AddToPath -Daily
+    jobName: InstallPowerShellPS1UbuntuDaily
+    pool: ubuntu-latest
+    verification: |
+      Write-Verbose $PSVersionTable.PSVersion -verbose
+      if ([Version]"$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor).$($PSVersionTable.PSVersion.Patch)" -lt [version]"7.0.0")
+      {
+        throw "powershell was not upgraded: $($PSVersionTable.PSVersion)"
+      }
+
+- template: templates/install-ps-phase.yml
+  parameters:
+    scriptName: pwsh -c ./tools/install-powershell.ps1 -AddToPath -Daily
+    jobName: InstallPowerShellMacOSDaily
+    pool: macOS-latest
+    verification: |
+      Write-Verbose $PSVersionTable.PSVersion -verbose
+      if ([Version]"$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor).$($PSVersionTable.PSVersion.Patch)" -lt [version]"7.0.0")
+      {
+        throw "powershell was not upgraded: $($PSVersionTable.PSVersion)"
+      }
+
+- template: templates/install-ps-phase.yml
+  parameters:
+    scriptName: |
+      pwsh -c ./tools/install-powershell.ps1 -AddToPath -Daily
+    jobName: InstallPowerShellWindowsDaily
+    pool: windows-latest
+    verification: |
+      $newVersion = &$env:LOCALAPPDATA\Microsoft\powershell-daily\pwsh -v
+      $newVersion -match '^PowerShell ((\d*\.\d*\.\d*)(-\w*(\.\d*)?)?){1}'
+      $versionOnly = $Matches[2]
+      Write-verbose "$newVersion; versionOnly: $versionOnly" -verbose
+      if ([Version]$versionOnly -lt [version]"7.0.0")
+      {
+        throw "powershell was not upgraded: $newVersion"
+      }

.vsts-ci/templates/install-ps-phase.yml

@@ -1,5 +1,5 @@
 parameters:
-  pool: 'Hosted Ubuntu 1604'
+  pool: 'ubuntu-latest'
   jobName: 'none'
   scriptName: ''
   container: ''
@@ -16,12 +16,12 @@ jobs:
     container: ${{ parameters.container }}
 
   pool:
-    name: ${{ parameters.pool }}
+    vmImage: ${{ parameters.pool }}
 
   displayName: ${{ parameters.jobName }}
 
   steps:
-  - powershell: |
+  - pwsh: |
       Get-ChildItem -Path env:
     displayName: Capture environment
     condition: succeededOrFailed()
@@ -37,6 +37,6 @@ jobs:
     continueOnError: ${{ parameters.continueOnError }}
 
   - ${{ if ne(parameters.verification, '') }}:
-    - powershell: ${{ parameters.verification }}
+    - pwsh: ${{ parameters.verification }}
       displayName: Verification
       continueOnError: ${{ parameters.continueOnError }}

tools/install-powershell.ps1

@@ -236,42 +236,71 @@ if (-not $IsWinEnv) {
     }
 }
 $tempDir = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName())
-New-Item -ItemType Directory -Path $tempDir -Force -ErrorAction SilentlyContinue
+$null = New-Item -ItemType Directory -Path $tempDir -Force -ErrorAction SilentlyContinue
 try {
     # Setting Tls to 12 to prevent the Invoke-WebRequest : The request was
     # aborted: Could not create SSL/TLS secure channel. error.
     $originalValue = [Net.ServicePointManager]::SecurityProtocol
     [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
 
     if ($Daily) {
-        if (-not (Get-Module -Name PackageManagement -ListAvailable)) {
-            throw "PackageManagement module is required to install daily PowerShell."
+        $metadata = Invoke-RestMethod https://pscoretestdata.blob.core.windows.net/buildinfo/daily.json
+        $release = $metadata.ReleaseTag -replace '^v'
+        $blobName = $metadata.BlobName
+
+        if ($IsWinEnv) {
+            if ($UseMSI) {
+                $packageName = "PowerShell-${release}-win-${architecture}.msi"
+            } else {
+                $packageName = "PowerShell-${release}-win-${architecture}.zip"
+            }
+        } elseif ($IsLinuxEnv) {
+            $packageName = "powershell-${release}-linux-${architecture}.tar.gz"
+        } elseif ($IsMacOSEnv) {
+            $packageName = "powershell-${release}-osx-${architecture}.tar.gz"
         }
 
         if ($architecture -ne "x64") {
             throw "The OS architecture is '$architecture'. However, we currently only support daily package for x64."
         }
 
-        ## Register source if not yet
-        if (-not (Get-PackageSource -Name powershell-core-daily -ErrorAction SilentlyContinue)) {
-            $packageSource = "https://powershell.myget.org/F/powershell-core-daily"
-            Write-Verbose "Register powershell-core-daily package source '$packageSource' with PackageManagement" -Verbose
-            Register-PackageSource -Name powershell-core-daily -Location $packageSource -ProviderName nuget -Trusted -ErrorAction SilentlyContinue
+
+        $downloadURL = "https://pscoretestdata.blob.core.windows.net/${blobName}/${packageName}"
+        Write-Verbose "About to download package from '$downloadURL'" -Verbose
+
+        $packagePath = Join-Path -Path $tempDir -ChildPath $packageName
+        if (!$PSVersionTable.ContainsKey('PSEdition') -or $PSVersionTable.PSEdition -eq "Desktop") {
+            # On Windows PowerShell, progress can make the download significantly slower
+            $oldProgressPreference = $ProgressPreference
+            $ProgressPreference = "SilentlyContinue"
         }
 
-        if ($IsWinEnv) {
-            $packageName = "powershell-win-x64-win7-x64"
-        } elseif ($IsLinuxEnv) {
-            $packageName = "powershell-linux-x64"
-        } elseif ($IsMacOSEnv) {
-            $packageName = "powershell-osx-x64"
+        try {
+            Invoke-WebRequest -Uri $downloadURL -OutFile $packagePath
+        } finally {
+            if (!$PSVersionTable.ContainsKey('PSEdition') -or $PSVersionTable.PSEdition -eq "Desktop") {
+                $ProgressPreference = $oldProgressPreference
+            }
         }
 
-        $package = Find-Package -Source powershell-core-daily -AllowPrereleaseVersions -Name $packageName
-        Write-Verbose "Daily package found. Name: $packageName; Version: $($package.Version)" -Verbose
+        $contentPath = Join-Path -Path $tempDir -ChildPath "new"
 
-        Install-Package -InputObject $package -Destination $tempDir -ExcludeVersion -ErrorAction SilentlyContinue
-        $contentPath = [System.IO.Path]::Combine($tempDir, $packageName, "content")
+        $null = New-Item -ItemType Directory -Path $contentPath -ErrorAction SilentlyContinue
+        if ($IsWinEnv) {
+            if ($UseMSI -and $Quiet) {
+                Write-Verbose "Performing quiet install"
+                $process = Start-Process msiexec -ArgumentList "/i", $packagePath, "/quiet" -Wait -PassThru
+                if ($process.exitcode -ne 0) {
+                    throw "Quiet install failed, please rerun install without -Quiet switch or ensure you have administrator rights"
+                }
+            } elseif ($UseMSI) {
+                Start-Process $packagePath -Wait
+            } else {
+                Expand-ArchiveInternal -Path $packagePath -DestinationPath $contentPath
+            }
+        } else {
+            tar zxf $packagePath -C $contentPath
+        }
     } else {
         $metadata = Invoke-RestMethod https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/metadata.json
         if ($Preview) {
@@ -312,7 +341,7 @@ try {
 
         $contentPath = Join-Path -Path $tempDir -ChildPath "new"
 
-        New-Item -ItemType Directory -Path $contentPath -ErrorAction SilentlyContinue
+        $null = New-Item -ItemType Directory -Path $contentPath -ErrorAction SilentlyContinue
         if ($IsWinEnv) {
             if ($UseMSI -and $Quiet) {
                 Write-Verbose "Performing quiet install"
@@ -350,7 +379,7 @@ try {
         if (-not (Test-Path "~/.rcedit/rcedit-x64.exe")) {
             Write-Verbose "Install RCEdit for modifying exe resources" -Verbose
             $rceditUrl = "https://github.com/electron/rcedit/releases/download/v1.0.0/rcedit-x64.exe"
-            New-Item -Path "~/.rcedit" -Type Directory -Force -ErrorAction SilentlyContinue
+            $null = New-Item -Path "~/.rcedit" -Type Directory -Force -ErrorAction SilentlyContinue
             Invoke-WebRequest -OutFile "~/.rcedit/rcedit-x64.exe" -Uri $rceditUrl
         }
 

tools/packaging/packaging.psm1

@@ -3129,6 +3129,12 @@ function Get-PackageVersionAsMajorMinorBuildRevision
 
         if ($packageBuildTokens)
         {
+            if($packageBuildTokens.length -gt 4)
+            {
+                # MSIX will fail if it is more characters
+                $packageBuildTokens = $packageBuildTokens.Substring(0,4)
+            }
+
             $packageVersion = $packageVersion + '.' + $packageBuildTokens
         }
         else

tools/releaseBuild/azureDevOps/releaseBuild.yml

@@ -95,6 +95,15 @@ jobs:
       - upload_alpine
       - build_macOS
 
+- template: templates/json.yml
+  parameters:
+    parentJobs:
+      - WinPackageSigningJob
+      - upload_deb
+      - upload_rpm
+      - upload_alpine
+      - MacPackageSigningJob
+
 - template: templates/testartifacts.yml
 
 - job: release_json

tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml

@@ -1,18 +1,20 @@
 parameters:
   ReleaseTagVar: v6.2.0
   ReleaseTagVarName: ReleaseTagVar
+  CreateJson: 'no'
 
 steps:
 - powershell: |
-    $releaseTag = tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}"
+    $createJson = ("${{ parameters.ReleaseTagVarName }}" -ne "no")
+    $releaseTag = tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson
     $version = $releaseTag.Substring(1)
     $vstsCommandString = "vso[task.setvariable variable=Version]$version"
-    Write-Host "sending " + $vstsCommandString
+    Write-Host ("sending " + $vstsCommandString)
     Write-Host "##$vstsCommandString"
 
     $azureVersion = $releaseTag.ToLowerInvariant() -replace '\.', '-'
     $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion"
-    Write-Host "sending " + $vstsCommandString
+    Write-Host ("sending " + $vstsCommandString)
     Write-Host "##$vstsCommandString"
   displayName: 'Set ${{ parameters.ReleaseTagVarName }} and other version Variables'
 

tools/releaseBuild/azureDevOps/templates/json.yml

@@ -0,0 +1,45 @@
+parameters:
+  parentJobs: []
+
+jobs:
+- job: json
+  displayName: Create Json for Blob
+  dependsOn:
+    ${{ parameters.parentJobs }}
+  condition: succeeded()
+  pool:
+    vmImage: windows-latest
+
+  steps:
+  #- task: <task type name>@<version>
+  #  inputs:
+  #    <task specific inputs>
+  #  displayName: '<display name of task>'
+  - template: SetVersionVariables.yml
+    parameters:
+      ReleaseTagVar: $(ReleaseTagVar)
+      CreateJson: yes
+
+  - task: AzureFileCopy@1
+    displayName: 'upload signed msi to Azure - ${{ parameters.architecture }}'
+    inputs:
+      SourcePath: '$(BuildInfoPath)'
+      azureSubscription: '$(AzureFileCopySubscription)'
+      Destination: AzureBlob
+      storage: '$(StorageAccount)'
+      ContainerName: 'BuildInfo'
+    condition: and(succeeded(), eq(variables['IS_DAILY'], 'true'))
+
+  - task: AzureCLI@1
+    displayName: 'Make blob public'
+    inputs:
+      azureSubscription: '$(AzureFileCopySubscription)'
+      scriptLocation: inlineScript
+      inlineScript: 'az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion) --public-access blob'
+    condition: and(succeeded(), eq(variables['IS_DAILY'], 'true'))
+
+  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+    displayName: 'Component Detection'
+    inputs:
+      sourceScanPath: '$(Build.SourcesDirectory)'
+      snapshotForceEnabled: true

tools/releaseBuild/azureDevOps/templates/linux.yml

@@ -51,6 +51,8 @@ jobs:
     buildName: ${{ parameters.buildName }}
   steps:
 
+  - template: shouldSign.yml
+
   - template: SetVersionVariables.yml
     parameters:
       ReleaseTagVar: $(ReleaseTagVar)
@@ -86,12 +88,12 @@ jobs:
       Get-ChildItem -Path '$(System.ArtifactsDirectory)\rpm\*.rpm' -recurse | ForEach-Object { $authenticodefiles += $_.FullName}
       tools/releaseBuild/generatePackgeSigning.ps1 -LinuxFiles $authenticodeFiles -path "$(System.ArtifactsDirectory)\package.xml"
     displayName: 'Generate RPM Signing Xml'
-    condition: and(and(succeeded(), eq(variables['Build.Reason'], 'Manual')),eq(variables['buildName'], 'RPM'))
+    condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM'))
 
   - powershell: |
       Get-Content "$(System.ArtifactsDirectory)\package.xml"
     displayName: 'Capture RPM signing xml'
-    condition: and(and(succeeded(), eq(variables['Build.Reason'], 'Manual')),eq(variables['buildName'], 'RPM'))
+    condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM'))
 
   - task: PkgESCodeSign@10
     displayName: 'CodeSign RPM $(System.ArtifactsDirectory)\package.xml'
@@ -102,7 +104,7 @@ jobs:
       outPathRoot: '$(Build.StagingDirectory)\signedPackages'
       binVersion: $(SigingVersion)
       binVersionOverride: $(SigningVersionOverride)
-    condition: and(and(succeeded(), eq(variables['Build.Reason'], 'Manual')),eq(variables['buildName'], 'RPM'))
+    condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM'))
 
   - task: AzureFileCopy@1
     displayName: 'Upload to Azure - DEB and tar.gz'
@@ -125,7 +127,7 @@ jobs:
       Destination: AzureBlob
       storage: '$(StorageAccount)'
       ContainerName: '$(AzureVersion)'
-    condition: and(and(succeeded(), ne(variables['Build.Reason'], 'Manual')),eq(variables['buildName'], 'RPM'))
+    condition: and(and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM'))
 
   - task: AzureFileCopy@1
     displayName: 'Upload to Azure - RPM - Signed'
@@ -135,17 +137,17 @@ jobs:
       Destination: AzureBlob
       storage: '$(StorageAccount)'
       ContainerName: '$(AzureVersion)'
-    condition: and(and(succeeded(), eq(variables['Build.Reason'], 'Manual')),eq(variables['buildName'], 'RPM'))
+    condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM'))
 
   - template: upload-final-results.yml
     parameters:
       artifactPath: $(System.ArtifactsDirectory)\rpm\release
-      condition: and(and(succeeded(), ne(variables['Build.Reason'], 'Manual')),eq(variables['buildName'], 'RPM'))
+      condition: and(and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM'))
 
   - template: upload-final-results.yml
     parameters:
       artifactPath: '$(Build.StagingDirectory)\signedPackages'
-      condition: and(and(succeeded(), eq(variables['Build.Reason'], 'Manual')),eq(variables['buildName'], 'RPM'))
+      condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM'))
 
   - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
     displayName: 'Component Detection'