Packaging cleanup (#6939)

* Packaging cleanup Originally I was just trying to remove mentions of snupkg, but then things got a bit carried away. :) This is trying to remove as much duplication and dead code related to packaging that I can. * Apply code review feedback * Suppress copying indirect references * Remove unwanted bundled files from AutoML * Remove leading slash * Refactor model download * Correct the packaging path of native symbols * Rename NoTargets projects from csproj to proj * Fix build issues around model download and respond to feedback * Remove NoTargets file extension enforcement * Rename proj to CSProj, include in SLN I'd like to ensure all our projects are included in the SLN and don't rely on separate build steps. VS prefers *.csproj in the sln so I renamed things back to csproj. * Respond to PR feedback
dotnet · Feb 28, 2024 · f22b60a · f22b60a
1 parent 3855dca
commit f22b60a
Show file tree

Hide file tree

Showing 76 changed files with 334 additions and 686 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -35,8 +35,17 @@
     </PropertyGroup>
 
   <PropertyGroup>
+    <ModelDownloadPath>$(ArtifactsDir)models/</ModelDownloadPath>
     <PackageAssetsPath>$(ArtifactsDir)pkgassets/</PackageAssetsPath>
     <PkgDir>$(RepoRoot)pkg/</PkgDir>
+
+    <Authors>Microsoft</Authors>
+    <PackageLicenseExpression>MIT</PackageLicenseExpression>
+    <PackageProjectUrl>https://dot.net/ml</PackageProjectUrl>
+    <PackageIcon>mlnetlogo.png</PackageIcon>
+    <PackageReleaseNotes>https://aka.ms/mlnetreleasenotes</PackageReleaseNotes>
+    <!-- space separated -->
+    <PackageTags>ML.NET ML Machine Learning</PackageTags>
   </PropertyGroup>
 
   <!-- Need to explicitly set these properties for the -net6_0 or -netfx configurations becuase they are typically based off 'Debug' or 'Release' configs -->
@@ -60,4 +69,5 @@
     <!-- Optional: Build symbol packages (.symbols.nupkg) to distribute the PDB containing Source Link -->
     <IncludeSymbols>true</IncludeSymbols>
   </PropertyGroup>
+
 </Project>
diff --git a/Directory.Build.targets b/Directory.Build.targets
@@ -1,6 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project>
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+
+  <Import Condition="'$(IsPackable)' == 'true'" Project="$(RepositoryEngineeringDir)Packaging.targets"  />
 
   <ItemGroup Condition="'$(IsUnitTestProject)' == 'true'">
     <PackageReference Include="coverlet.msbuild" Version="$(CoverletMsbuildVersion)" PrivateAssets="all" />

diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln
@@ -96,6 +96,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Samples.OneDal
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Recommender", "src\Microsoft.ML.Recommender\Microsoft.ML.Recommender.csproj", "{C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnImageFeaturizer.ModelRedist", "src\Microsoft.ML.DnnImageFeaturizer.ModelRedist\Microsoft.ML.DnnImageFeaturizer.ModelRedist.csproj", "{39E89702-1A46-4D5B-BA50-530D11309B5E}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnImageFeaturizer.ResNet18", "src\Microsoft.ML.DnnImageFeaturizer.ResNet18\Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj", "{9222FC9D-599A-49A5-B685-08CC9A5C81D7}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnImageFeaturizer.AlexNet", "src\Microsoft.ML.DnnImageFeaturizer.AlexNet\Microsoft.ML.DnnImageFeaturizer.AlexNet.csproj", "{6C29AA9B-054B-4762-BEA5-D305B932AA80}"
@@ -818,6 +820,14 @@ Global
 		{763FF013-8309-4680-A769-B54E7BB99612}.Release|Any CPU.Build.0 = Release|Any CPU
 		{763FF013-8309-4680-A769-B54E7BB99612}.Release|x64.ActiveCfg = Release|Any CPU
 		{763FF013-8309-4680-A769-B54E7BB99612}.Release|x64.Build.0 = Release|Any CPU
+		{39E89702-1A46-4D5B-BA50-530D11309B5E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{39E89702-1A46-4D5B-BA50-530D11309B5E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{39E89702-1A46-4D5B-BA50-530D11309B5E}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{39E89702-1A46-4D5B-BA50-530D11309B5E}.Debug|x64.Build.0 = Debug|Any CPU
+		{39E89702-1A46-4D5B-BA50-530D11309B5E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{39E89702-1A46-4D5B-BA50-530D11309B5E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{39E89702-1A46-4D5B-BA50-530D11309B5E}.Release|x64.ActiveCfg = Release|Any CPU
+		{39E89702-1A46-4D5B-BA50-530D11309B5E}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -903,6 +913,7 @@ Global
 		{FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{AB8D68F1-6C3E-41FD-B0EC-A093E009341D} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{763FF013-8309-4680-A769-B54E7BB99612} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{39E89702-1A46-4D5B-BA50-530D11309B5E} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

diff --git a/build/publish.proj b/build/publish.proj
diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj
@@ -23,4 +23,5 @@
   <ItemGroup>
     <PackageReference Include="SciSharp.TensorFlow.Redist" Version="$(TensorFlowVersion)" />
   </ItemGroup>
+
 </Project>
diff --git a/docs/samples/Microsoft.ML.Samples.GPU/Microsoft.ML.Samples.GPU.csproj b/docs/samples/Microsoft.ML.Samples.GPU/Microsoft.ML.Samples.GPU.csproj
@@ -23,6 +23,7 @@
     <ProjectReference Include="..\..\..\src\Microsoft.ML.Vision\Microsoft.ML.Vision.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.DnnImageFeaturizer.ModelRedist\Microsoft.ML.DnnImageFeaturizer.ModelRedist.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.LightGbm\Microsoft.ML.LightGbm.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.TensorFlow\Microsoft.ML.TensorFlow.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.SamplesUtils\Microsoft.ML.SamplesUtils.csproj" />
@@ -54,14 +55,14 @@
   </ItemGroup>
 
     <ItemGroup>
-    <Content Include="$(ArtifactsObjDir)DnnImageModels\ResNet18Onnx\ResNet18.onnx">
+    <Content Include="$(ModelDownloadPath)\ResNet18Onnx\ResNet18.onnx">
       <Link>DnnImageModels\ResNet18Onnx\ResNet18.onnx</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
 
     <ItemGroup>
-    <Content Include="$(ArtifactsObjDir)DnnImageModels\ResNetPrepOnnx\ResNetPreprocess.onnx">
+    <Content Include="$(ModelDownloadPath)\ResNetPrepOnnx\ResNetPreprocess.onnx">
       <Link>DnnImageModels\ResNetPrepOnnx\ResNetPreprocess.onnx</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

diff --git a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj
@@ -981,14 +981,14 @@
   </ItemGroup>
 
     <ItemGroup>
-    <Content Include="$(ArtifactsObjDir)DnnImageModels\ResNet18Onnx\ResNet18.onnx">
+    <Content Include="$(ModelDownloadPath)\ResNet18Onnx\ResNet18.onnx">
       <Link>DnnImageModels\ResNet18Onnx\ResNet18.onnx</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
 
     <ItemGroup>
-    <Content Include="$(ArtifactsObjDir)DnnImageModels\ResNetPrepOnnx\ResNetPreprocess.onnx">
+    <Content Include="$(ModelDownloadPath)\ResNetPrepOnnx\ResNetPreprocess.onnx">
       <Link>DnnImageModels\ResNetPrepOnnx\ResNetPreprocess.onnx</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

diff --git a/eng/Build.props b/eng/Build.props
@@ -9,7 +9,6 @@
   </ItemDefinitionGroup>
 
   <ItemGroup>
-    <ProjectToBuild Include="$(RepoRoot)src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj" />
     <ProjectToBuild Include="$(RepoRoot)src/Native/Native.proj" />
     <ProjectToBuild Include="$(RepoRoot)Microsoft.ML.sln" />
   </ItemGroup>

diff --git a/eng/Packaging.targets b/eng/Packaging.targets
@@ -0,0 +1,47 @@
+<Project>
+
+  <ItemGroup>
+    <Content Include="$(RepositoryEngineeringDir)pkg\mlnetlogo.png" Pack="true" PackagePath="" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(IncludeMLNetNotices)' != 'false'">
+    <Content Include="$(RepoRoot)THIRD-PARTY-NOTICES.TXT" Pack="true" PackagePath="" />
+    <Content Include="$(RepoRoot)LICENSE" Pack="true" PackagePath=""/>
+  </ItemGroup>
+
+  <PropertyGroup>    
+    <TargetsForTfmSpecificContentInPackage>$(TargetsForTfmSpecificContentInPackage);IncludeAdditionalFilesInPackage;</TargetsForTfmSpecificContentInPackage>
+  </PropertyGroup>
+
+  <Target Name="_IncludeProjectReferencesWithPackAttributeInPackage"
+          Condition="'@(ProjectReference)' != '' and @(ProjectReference->AnyHaveMetadataValue('Pack', 'true'))"
+          DependsOnTargets="BuildOnlySettings;ResolveReferences">
+    <ItemGroup>
+      <!-- Implements workaround for https://github.com/NuGet/Home/issues/3891 -->
+      <!-- Add ReferenceCopyLocalPaths for ProjectReferences which are flagged as Pack="true" into the package. -->
+      <_projectReferenceCopyLocalPaths Include="@(ReferenceCopyLocalPaths->WithMetadataValue('ReferenceSourceTarget', 'ProjectReference')->WithMetadataValue('Pack', 'true'))" />
+      <TfmSpecificPackageFile Include="@(_projectReferenceCopyLocalPaths)"
+                              PackagePath="$([MSBuild]::ValueOrDefault('%(ReferenceCopyLocalPaths.PackagePath)', '$(BuildOutputTargetFolder)\$(TargetFramework)\'))" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="_IncludeRuntimeSpecificAssetsInPackage">
+    <ItemGroup>    
+      <!-- Add runtime specific assets to the package. -->
+      <_runtimeFiles Include="$(PackageAssetsPath)$(PackageId)\runtimes\**\*%(NativeAssemblyReference.Identity)*" />
+      <TfmSpecificPackageFile Include="%(_runtimeFiles.Identity)" PackagePath="runtimes/%(_runtimeFiles.RecursiveDir)" TargetFramework="" />
+    </ItemGroup>    
+  </Target>
+
+  <Target Name="IncludeAdditionalFilesInPackage" DependsOnTargets="_IncludeProjectReferencesWithPackAttributeInPackage;_IncludeRuntimeSpecificAssetsInPackage">
+    <ItemGroup>
+      <!-- PDBs must be specified as TfmSpecificDebugSymbolsFile with a *rooted* TargetPath instead of PackagePath. -->
+      <TfmSpecificDebugSymbolsFile Include="@(TfmSpecificPackageFile->WithMetadataValue('Extension', '.pdb'))"
+                                   TargetPath="/%(TfmSpecificPackageFile.PackagePath)%(Filename)%(Extension)"
+                                   TargetFramework="$(TargetFramework)" />
+      <!-- Remove symbols from the file list. -->
+      <TfmSpecificPackageFile Remove="@(TfmSpecificPackageFile->WithMetadataValue('Extension', '.pdb'))" />
+    </ItemGroup>    
+  </Target>
+
+</Project>
diff --git a/eng/Signing.props b/eng/Signing.props
@@ -1,9 +1,4 @@
 <Project>
-  <ItemGroup>
-    <ItemsToSign Include="$(ArtifactsPackagesDir)**\*.snupkg" />
-    <FileExtensionSignInfo Include="*.snupkg" CertificateName="NuGet" />
-  </ItemGroup>
-
   <PropertyGroup>
     <UseDotNetCertificate>true</UseDotNetCertificate>
   </PropertyGroup>

diff --git a/eng/pkg/Pack.props b/eng/pkg/Pack.props
diff --git a/global.json b/global.json
@@ -15,6 +15,7 @@
   "msbuild-sdks": {
     "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24102.4",
     "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24102.4",
+    "Microsoft.Build.NoTargets": "3.7.0",
     "Microsoft.Build.Traversal": "3.2.0",
     "Microsoft.SourceLink.GitHub": "1.1.0-beta-20206-02",
     "Microsoft.SourceLink.Common": "1.1.0-beta-20206-02"

diff --git a/src/Directory.Build.props b/src/Directory.Build.props
@@ -3,7 +3,7 @@
   <Import Project="..\Directory.Build.props" />
 
   <PropertyGroup>
-    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+    <GenerateDocumentationFile Condition="'$(UsingMicrosoftNoTargetsSdk)' != 'true'">true</GenerateDocumentationFile>
     <!-- Not all types and members have XML docs yet.
          When you are ready to tackle adding them, remove <NoWarn>
          below, and when you get clean, also remove <WarningsNotAsErrors>
@@ -24,7 +24,7 @@
 
   <ItemGroup>
     <ProjectReference
-      Condition="'$(UseMLCodeAnalyzer)' != 'false' and '$(MSBuildProjectExtension)' == '.csproj'"
+      Condition="'$(UseMLCodeAnalyzer)' != 'false' and '$(MSBuildProjectExtension)' == '.csproj' and '$(UsingMicrosoftNoTargetsSdk)' != 'true'"
       Include="$(MSBuildThisFileDirectory)\..\tools-local\Microsoft.ML.InternalCodeAnalyzer\Microsoft.ML.InternalCodeAnalyzer.csproj">
       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
       <OutputItemType>Analyzer</OutputItemType>

diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets
@@ -2,30 +2,6 @@
 
   <Import Project="..\Directory.Build.targets" />
 
-  <Target Name="PreparePackageAssets" Condition="'$(CopyPackageAssets)' == 'true'"
-          AfterTargets="Build">
-
-    <ItemGroup Condition="'$(IncludeInPackage)' != ''">
-      <PackageAsset Include="$(TargetPath)"
-                    RelativePath="$(IncludeInPackage)\lib\$(TargetFramework)" />
-      <PackageAsset Include="@(DebugSymbolsProjectOutputGroupOutput)"
-                    RelativePath="$(IncludeInPackage)\lib\$(TargetFramework)" />
-      <PackageAsset Include="@(DocumentationProjectOutputGroupOutput)"
-                    RelativePath="$(IncludeInPackage)\lib\$(TargetFramework)" />
-    </ItemGroup>
-
-    <ItemGroup Condition="'$(IncludeAnalyzerInPackage)' != ''">
-      <PackageAsset Include="$(TargetPath)"
-                    RelativePath="$(IncludeAnalyzerInPackage)\analyzers\dotnet\cs" />
-      <PackageAsset Include="@(DebugSymbolsProjectOutputGroupOutput)"
-                    RelativePath="$(IncludeAnalyzerInPackage)\analyzers\dotnet\cs" />
-    </ItemGroup>
-
-    <Copy SourceFiles="@(PackageAsset)"
-          DestinationFolder="$(PackageAssetsPath)%(PackageAsset.RelativePath)" />
-
-  </Target>
-
   <!-- API Compat -->
   <PropertyGroup Condition="'$(IsPackable)' == 'true'">
     <!-- Enable API compat for any packages unless they've set EnablePackageValidation -->