From ec9a3f1e871eee69c1291cfa0d10154bb4aedebf Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Tue, 28 May 2024 10:44:02 +0200 Subject: [PATCH 1/7] source-build: bundle NativeAOT libraries with the SDK. This includes the libraries that are needed for source-built NativeAOT with the source-built SDK: - runtime..Microsoft.DotNet.ILCompiler extracted under 'packs'. - Microsoft.DotNet.ILCompiler and Microsoft.NET.ILLink.Tasks as NuGet packages under 'library-packs'. --- .../targets/GenerateLayout.targets | 16 ++++++++++++++++ .../ProcessFrameworkReferences.cs | 15 ++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/Installer/redist-installer/targets/GenerateLayout.targets b/src/Installer/redist-installer/targets/GenerateLayout.targets index 7db3fa0ace2c..2aca602dd42d 100644 --- a/src/Installer/redist-installer/targets/GenerateLayout.targets +++ b/src/Installer/redist-installer/targets/GenerateLayout.targets @@ -202,6 +202,17 @@ packs/%(PackageName)/%(PackageVersion) + + runtime.$(SharedFrameworkRid).Microsoft.DotNet.ILCompiler + $(MicrosoftNETCoreAppRuntimePackageVersion) + $(TargetFramework) + packs/%(PackageName)/%(PackageVersion) + + + + + + $(NetRuntimeRootUrl) @@ -460,6 +471,11 @@ SkipUnchangedFiles="true" /> + + dotnet-toolset-internal diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/ProcessFrameworkReferences.cs b/src/Tasks/Microsoft.NET.Build.Tasks/ProcessFrameworkReferences.cs index b491a450b531..81278881ebd2 100644 --- a/src/Tasks/Microsoft.NET.Build.Tasks/ProcessFrameworkReferences.cs +++ b/src/Tasks/Microsoft.NET.Build.Tasks/ProcessFrameworkReferences.cs @@ -788,17 +788,22 @@ private ToolPackSupport AddToolPack( var runtimePackName = packNamePattern.Replace("**RID**", hostRuntimeIdentifier); - if (EnableRuntimePackDownload) + var runtimePackItem = new TaskItem(runtimePackName); + runtimePackItem.SetMetadata(MetadataKeys.NuGetPackageId, runtimePackName); + runtimePackItem.SetMetadata(MetadataKeys.NuGetPackageVersion, packVersion); + + string runtimePackPath = GetPackPath(runtimePackName, packVersion); + if (runtimePackPath != null) + { + runtimePackItem.SetMetadata(MetadataKeys.PackageDirectory, runtimePackPath); + } + else if (EnableRuntimePackDownload) { // We need to download the runtime pack runtimePackToDownload = new TaskItem(runtimePackName); runtimePackToDownload.SetMetadata(MetadataKeys.Version, packVersion); } - var runtimePackItem = new TaskItem(runtimePackName); - runtimePackItem.SetMetadata(MetadataKeys.NuGetPackageId, runtimePackName); - runtimePackItem.SetMetadata(MetadataKeys.NuGetPackageVersion, packVersion); - switch (toolPackType) { case ToolPackType.Crossgen2: From 3d1c2246ab7e08a30696abb5609cde0c0d41d3af Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Tue, 18 Jun 2024 15:15:54 +0200 Subject: [PATCH 2/7] Add DotNetBuildOrchestrator condition for setting BundleNativeAotCompiler. --- src/Installer/redist-installer/redist-installer.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Installer/redist-installer/redist-installer.csproj b/src/Installer/redist-installer/redist-installer.csproj index 270f119db641..311563a2d562 100644 --- a/src/Installer/redist-installer/redist-installer.csproj +++ b/src/Installer/redist-installer/redist-installer.csproj @@ -8,7 +8,8 @@ true none true - true + + true From 713a6ff146257fee16502a506a9ab89cb6ee9643 Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Wed, 19 Jun 2024 11:12:10 +0200 Subject: [PATCH 3/7] Find out what happens when doubling the test timeout. --- eng/pipelines/templates/jobs/vmr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 873bf943123d..80643cbb7d7f 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -123,7 +123,7 @@ jobs: - name: additionalBuildArgs value: '' - name: runTestsTimeout - value: 30 + value: 60 - ${{ if parameters.isBuiltFromVmr }}: - name: vmrPath From cd188f3d29bd122fd409fcba3e22934226f66ee7 Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Wed, 19 Jun 2024 13:17:04 +0200 Subject: [PATCH 4/7] Try to get some more output from the CI test run. --- eng/pipelines/templates/jobs/vmr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 80643cbb7d7f..9ed9f6e9a0d0 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -406,7 +406,7 @@ jobs: # Only use Docker when a container is specified if [[ -n "${{ parameters.container }}" ]]; then - docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs) + docker run --rm $dockerVolumeArgs -e MSBUILDENSURESTDOUTFORTASKPROCESSES=1 -w /vmr ${{ parameters.container }} ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs) else cd $(sourcesPath) ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs) From 71f5d98dc4b03e89db47a45ae87d66e7c7a0aa0c Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Wed, 19 Jun 2024 16:14:50 +0200 Subject: [PATCH 5/7] Skip 'ilc' in SourceBuiltSdkContainsNativeDebugSymbols. --- .../DebugTests.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DebugTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DebugTests.cs index c556b0e907eb..8ed35c24c6c2 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DebugTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DebugTests.cs @@ -30,7 +30,7 @@ public void SourceBuiltSdkContainsNativeDebugSymbols() StringBuilder issueDetails = new(); foreach (var fileName in fileNames) { - if (!IsElfFile(fileName)) + if (!IsElfFile(fileName) || SkipFile(fileName)) { continue; } @@ -70,6 +70,14 @@ private bool IsElfFile(string fileName) return Regex.IsMatch(fileStdOut, @"ELF 64-bit [LM]SB (?:pie )?(?:executable|shared object)"); } + private static bool SkipFile(string path) + { + string fileName = Path.GetFileName(path); + + // 'ilc' is a NativeAOT-built application which doesn't meet the expectations set by the test. + return fileName == "ilc"; + } + private ScanResult ScanFile(string fileName) { string readelfSStdOut = ExecuteHelper.ExecuteProcessValidateExitCode("eu-readelf", $"-S {fileName}", OutputHelper); From 7be0c034723beb5d81652184cf4904c341384ef8 Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Wed, 19 Jun 2024 16:15:49 +0200 Subject: [PATCH 6/7] Revert runTestsTimeout change. --- eng/pipelines/templates/jobs/vmr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 9ed9f6e9a0d0..47badef43c5f 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -123,7 +123,7 @@ jobs: - name: additionalBuildArgs value: '' - name: runTestsTimeout - value: 60 + value: 30 - ${{ if parameters.isBuiltFromVmr }}: - name: vmrPath From 22a593b531c79fe3854bc1963ebdfbd23364f0cb Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Wed, 19 Jun 2024 16:57:14 +0200 Subject: [PATCH 7/7] Set MSBUILDENSURESTDOUTFORTASKPROCESSES in scripts when running tests. --- eng/pipelines/templates/jobs/vmr-build.yml | 2 +- src/SourceBuild/content/build.sh | 2 ++ src/SourceBuild/content/eng/build.ps1 | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 47badef43c5f..873bf943123d 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -406,7 +406,7 @@ jobs: # Only use Docker when a container is specified if [[ -n "${{ parameters.container }}" ]]; then - docker run --rm $dockerVolumeArgs -e MSBUILDENSURESTDOUTFORTASKPROCESSES=1 -w /vmr ${{ parameters.container }} ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs) + docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs) else cd $(sourcesPath) ./build.sh --test --excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog $customBuildArgs $extraBuildProperties $(additionalBuildArgs) diff --git a/src/SourceBuild/content/build.sh b/src/SourceBuild/content/build.sh index 53acf372e6cf..4bcc24876b31 100755 --- a/src/SourceBuild/content/build.sh +++ b/src/SourceBuild/content/build.sh @@ -219,6 +219,8 @@ targets="/t:Build" if [[ "$test" == true ]]; then project="$scriptroot/test/tests.proj" targets="$targets;VSTest" + # Workaround for vstest hangs (https://github.com/microsoft/vstest/issues/5091) [TODO] + export MSBUILDENSURESTDOUTFORTASKPROCESSES=1 fi function Build { diff --git a/src/SourceBuild/content/eng/build.ps1 b/src/SourceBuild/content/eng/build.ps1 index 128315cab9b3..d1a6924e10e5 100644 --- a/src/SourceBuild/content/eng/build.ps1 +++ b/src/SourceBuild/content/eng/build.ps1 @@ -60,6 +60,8 @@ $targets = "/t:Build" if ($test) { $project = Join-Path (Join-Path $RepoRoot "test") "tests.proj" $targets += ";VSTest" + # Workaround for vstest hangs (https://github.com/microsoft/vstest/issues/5091) [TODO] + $env:MSBUILDENSURESTDOUTFORTASKPROCESSES="1" } if ($buildRepoTests) {