Skip to content

Commit 6fe50e4

Browse files
Fix publishing symbols (#5556)
* Disable Portable PDB conversion * Push packages to artifacts * Fix symbols issues * Added note about Microsoft.ML.dll * try out just packing * Return Build=false, but actually use configuration * Added missing TargetArchitecture * add back tests * Added missing flags
1 parent 70ecfff commit 6fe50e4

File tree

9 files changed

+112
-43
lines changed

9 files changed

+112
-43
lines changed

build/ci/job-template.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,5 +197,5 @@ jobs:
197197
artifactName: ${{ parameters.name }} $(_config_short)
198198
artifactType: container
199199
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
200-
- script: ${{ parameters.buildScript }} /p:Build=false -pack -ci /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
201-
displayName: Build Packages
200+
- script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
201+
displayName: Build Packages

build/publish.proj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
<Target Name="SetupPublishSymbols">
2424
<PropertyGroup>
2525
<SymbolExpirationInDays Condition="'$(SymbolExpirationInDays)'=='' and '$(SymbolExpirationDate)'==''">180</SymbolExpirationInDays>
26-
<ConvertPortablePdbsToWindowsPdbs>true</ConvertPortablePdbsToWindowsPdbs>
26+
<ConvertPortablePdbsToWindowsPdbs>false</ConvertPortablePdbsToWindowsPdbs>
2727
</PropertyGroup>
2828

2929
<ItemGroup>

build/vsts-ci.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,16 @@ phases:
243243
displayName: sign packages
244244
continueOnError: false
245245

246+
# The generated .nupkgs and .snupkgs packages are published to Azure artifacts,
247+
# in case we need to debug them. They're found under Release/Shipping
248+
- task: PublishBuildArtifacts@1
249+
displayName: Push packages and symbol packages to Azure Artifacts
250+
inputs:
251+
pathToPublish: $(Build.SourcesDirectory)/artifacts/packages
252+
artifactName: PackageAssets
253+
artifactType: container
254+
continueOnError: true
255+
246256
- task: NuGetAuthenticate@0
247257
inputs:
248258
nuGetServiceConnections: machinelearning-dnceng-public-feed # To allow publishing to a feed of another organization
Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,41 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22
<Import Project="$(RepoRoot)eng/pkg/Pack.props"/>
3-
43
<PropertyGroup>
54
<TargetFrameworks>netstandard2.0;netcoreapp3.1</TargetFrameworks>
65
<IncludeInPackage>Microsoft.ML.CpuMath</IncludeInPackage>
76
<PackageDescription>Microsoft.ML.CpuMath contains optimized math routines for ML.NET.</PackageDescription>
87
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
98
<DefineConstants>$(DefineConstants);CPUMATH_INFRASTRUCTURE</DefineConstants>
9+
<TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
1010
</PropertyGroup>
11-
1211
<ItemGroup>
1312
<Compile Include="..\Microsoft.ML.Core\Utilities\Contracts.cs" />
1413
<Compile Include="..\Microsoft.ML.Core\BestFriendAttribute.cs" />
1514
<Compile Include="..\Microsoft.ML.Core\PublicKey.cs" />
16-
1715
<!-- Workaround https://github.com/dotnet/project-system/issues/935 -->
1816
<None Include="**/*.cs" />
1917
</ItemGroup>
20-
2118
<ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp3.1'">
2219
<Compile Remove="CpuMathUtils.netstandard.cs" />
2320
<Compile Remove="FactorizationMachine/FactorizationMachineInterface.netstandard.cs" />
2421
</ItemGroup>
25-
2622
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
2723
<Compile Remove="CpuMathUtils.netcoreapp.cs" />
2824
<Compile Remove="SseIntrinsics.cs" />
2925
<Compile Remove="AvxIntrinsics.cs" />
3026
<Compile Remove="FactorizationMachine/FactorizationMachineInterface.netcoreapp.cs" />
3127
<Compile Remove="FactorizationMachine/AvxIntrinsics.cs" />
32-
3328
<PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
3429
</ItemGroup>
35-
3630
<ItemGroup>
3731
<Content Include="build\**\*" Pack="true" PackagePath="build" />
3832
</ItemGroup>
39-
40-
</Project>
33+
<Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
34+
<ItemGroup Condition="'$(TargetFramework)' != 'netcoreapp3.1'">
35+
<!--Include native PDBs-->
36+
<!--The path needed to be hardcoded for this to work on our publishing CI-->
37+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\nativeassets\netstandard2.0\CpuMathNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\nativeassets\netstandard2.0\CpuMathNative.pdb" TargetPath="..\..\runtimes\win-x86\nativeassets\netstandard2.0" />
38+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\nativeassets\netstandard2.0\CpuMathNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\nativeassets\netstandard2.0\CpuMathNative.pdb" TargetPath="..\..\runtimes\win-x64\nativeassets\netstandard2.0" />
39+
</ItemGroup>
40+
</Target>
41+
</Project>
Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,15 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22
<Import Project="$(RepoRoot)eng/pkg/Pack.props" />
3-
43
<PropertyGroup>
54
<TargetFramework>netstandard2.0</TargetFramework>
65
<IncludeInPackage>Microsoft.ML.FastTree</IncludeInPackage>
76
<PackageDescription>ML.NET component for FastTree</PackageDescription>
87
<DefineConstants>$(DefineConstants);USE_FASTTREENATIVE;NO_STORE;CORECLR</DefineConstants>
98
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
9+
<TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
1010
</PropertyGroup>
11-
1211
<ItemGroup>
1312
<ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
14-
1513
<ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" >
1614
<PrivateAssets>all</PrivateAssets>
1715
</ProjectReference>
@@ -22,9 +20,15 @@
2220
<PrivateAssets>all</PrivateAssets>
2321
</ProjectReference>
2422
</ItemGroup>
25-
2623
<ItemGroup>
2724
<Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
2825
</ItemGroup>
29-
30-
</Project>
26+
<Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
27+
<ItemGroup>
28+
<!--Include native PDBs-->
29+
<!--The path needed to be hardcoded for this to work on our publishing CI-->
30+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\FastTreeNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\FastTreeNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
31+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\FastTreeNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\FastTreeNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
32+
</ItemGroup>
33+
</Target>
34+
</Project>
Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,15 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22
<Import Project="$(RepoRoot)eng/pkg/Pack.props" />
3-
43
<PropertyGroup>
54
<TargetFramework>netstandard2.0</TargetFramework>
65
<IncludeInPackage>Microsoft.ML.Mkl.Components</IncludeInPackage>
76
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
87
<PackageDescription>ML.NET additional learners making use of Intel Mkl.</PackageDescription>
8+
<TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
99
</PropertyGroup>
10-
1110
<ItemGroup>
1211
<ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
1312
<ProjectReference Include="..\Microsoft.ML.Mkl.Redist\Microsoft.ML.Mkl.Redist.csproj" />
14-
1513
<ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" >
1614
<PrivateAssets>all</PrivateAssets>
1715
</ProjectReference>
@@ -22,11 +20,9 @@
2220
<PrivateAssets>all</PrivateAssets>
2321
</ProjectReference>
2422
</ItemGroup>
25-
2623
<ItemGroup>
2724
<Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
2825
</ItemGroup>
29-
3026
<ItemGroup>
3127
<!-- The package reference is in there, so that the build can download the MlNetMklDeps package at build time.
3228
The binaries of this package, are extraceted, and repackaged in the Microsoft.ML.Mkl.Redist; that is the one that has a true
@@ -35,5 +31,12 @@
3531
<PrivateAssets>all</PrivateAssets>
3632
</PackageReference>
3733
</ItemGroup>
38-
39-
</Project>
34+
<Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
35+
<ItemGroup>
36+
<!--Include native PDBs-->
37+
<!--The path needed to be hardcoded for this to work on our publishing CI-->
38+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\SymSgdNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\SymSgdNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
39+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\SymSgdNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\SymSgdNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
40+
</ItemGroup>
41+
</Target>
42+
</Project>
Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,28 @@
11
<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Pack">
22
<Import Project="$(RepoRoot)eng/pkg/Pack.props" />
3-
43
<PropertyGroup>
54
<Authors>Intel</Authors>
6-
<IncludeSymbols>false</IncludeSymbols>
7-
<IncludeBuildOutput>false</IncludeBuildOutput>
5+
<IncludeSymbols>true</IncludeSymbols>
6+
<IncludeBuildOutput>true</IncludeBuildOutput>
87
<TargetFramework>netstandard2.0</TargetFramework>
98
<PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
109
<PackageDescription>$(MSBuildProjectName) contains the MKL library redistributed as a NuGet package.</PackageDescription>
1110
<PackageTags>$(PackageTags) MLNET MKL</PackageTags>
11+
<TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
1212
</PropertyGroup>
13-
1413
<PropertyGroup>
1514
<IncludeMLNetNotices>false</IncludeMLNetNotices>
1615
</PropertyGroup>
17-
1816
<ItemGroup>
1917
<Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
2018
<Content Include="$(RepoRoot)eng\pkg\_._" Pack="true" PackagePath="lib\netstandard2.0\" />
2119
<Content Include="$(NUGET_PACKAGES)\mlnetmkldeps\$(MlNetMklDepsPackageVersion)\LICENSE.txt" Pack="true" PackagePath="" />
2220
</ItemGroup>
23-
</Project>
21+
<Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
22+
<ItemGroup>
23+
<!--Include native PDBs-->
24+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\MklProxyNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\MklProxyNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
25+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\MklProxyNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\MklProxyNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
26+
</ItemGroup>
27+
</Target>
28+
</Project>
Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,31 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22
<Import Project="$(RepoRoot)eng/pkg/Pack.props" />
3-
43
<PropertyGroup>
54
<TargetFramework>netstandard2.0</TargetFramework>
65
<IncludeInPackage>Microsoft.ML.Recommender</IncludeInPackage>
76
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
87
<PackageDescription>LIBMF, the core computation library for matrix factorization in ML.NET</PackageDescription>
8+
<TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
99
</PropertyGroup>
10-
1110
<ItemGroup>
1211
<ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
13-
1412
<ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" >
1513
<PrivateAssets>all</PrivateAssets>
1614
</ProjectReference>
1715
<ProjectReference Include="..\Microsoft.ML.Data\Microsoft.ML.Data.csproj" >
1816
<PrivateAssets>all</PrivateAssets>
1917
</ProjectReference>
2018
</ItemGroup>
21-
2219
<ItemGroup>
2320
<Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
2421
<Content Include="$(RepoRoot)src\Native\MatrixFactorizationNative\libmf\COPYRIGHT" Pack="true" PackagePath="" />
2522
</ItemGroup>
26-
27-
</Project>
23+
<Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
24+
<ItemGroup>
25+
<!--Include native PDBs-->
26+
<!--The path needed to be hardcoded for this to work on our publishing CI-->
27+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\MatrixFactorizationNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\MatrixFactorizationNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
28+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\MatrixFactorizationNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\MatrixFactorizationNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
29+
</ItemGroup>
30+
</Target>
31+
</Project>

src/Microsoft.ML/Microsoft.ML.csproj

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,69 @@
11
<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Pack">
22
<Import Project="$(RepoRoot)eng/pkg/Pack.props"/>
3-
43
<PropertyGroup>
54
<TargetFramework>netstandard2.0</TargetFramework>
6-
<IncludeBuildOutput>false</IncludeBuildOutput>
5+
<!-- To include the DLL and PDBs of the underlying projects
6+
We need to remove the IncludeBuildOutput=false. This also generates a
7+
Microsoft.ML.dll in the nuget package, and a Microsoft.ML.pdb in the .snupkg
8+
But these are to be ignored, as they don't contain anything meaningful-->
9+
<!-- <IncludeBuildOutput>false</IncludeBuildOutput> -->
710
<NoWarn>$(NoWarn);NU5127;NU5128</NoWarn>
811
<PackageDescription>ML.NET is a cross-platform open-source machine learning framework which makes machine learning accessible to .NET developers.</PackageDescription>
12+
<TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
913
</PropertyGroup>
10-
1114
<ItemGroup>
1215
<ProjectReference Include="../Microsoft.ML.DataView/Microsoft.ML.DataView.csproj" />
1316
<ProjectReference Include="../Microsoft.ML.CpuMath/Microsoft.ML.CpuMath.csproj" />
14-
17+
<ProjectReference Include="../Microsoft.ML.Core/Microsoft.ML.Core.csproj">
18+
<IncludeInNuget>true</IncludeInNuget>
19+
<PrivateAssets>all</PrivateAssets>
20+
</ProjectReference>
21+
<ProjectReference Include="../Microsoft.ML.Data/Microsoft.ML.Data.csproj">
22+
<IncludeInNuget>true</IncludeInNuget>
23+
<PrivateAssets>all</PrivateAssets>
24+
</ProjectReference>
25+
<ProjectReference Include="../Microsoft.ML.KMeansClustering/Microsoft.ML.KMeansClustering.csproj">
26+
<IncludeInNuget>true</IncludeInNuget>
27+
<PrivateAssets>all</PrivateAssets>
28+
</ProjectReference>
29+
<ProjectReference Include="../Microsoft.ML.PCA/Microsoft.ML.PCA.csproj">
30+
<IncludeInNuget>true</IncludeInNuget>
31+
<PrivateAssets>all</PrivateAssets>
32+
</ProjectReference>
33+
<ProjectReference Include="../Microsoft.ML.StandardTrainers/Microsoft.ML.StandardTrainers.csproj">
34+
<IncludeInNuget>true</IncludeInNuget>
35+
<PrivateAssets>all</PrivateAssets>
36+
</ProjectReference>
37+
<ProjectReference Include="../Microsoft.ML.Transforms/Microsoft.ML.Transforms.csproj">
38+
<IncludeInNuget>true</IncludeInNuget>
39+
<PrivateAssets>all</PrivateAssets>
40+
</ProjectReference>
41+
<NativeAssemblyReference Include="LdaNative" />
1542
<PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonPackageVersion)" />
1643
<PackageReference Include="System.Reflection.Emit.Lightweight" Version="$(SystemReflectionEmitLightweightPackageVersion)" />
1744
<PackageReference Include="System.CodeDom" Version="$(SystemCodeDomPackageVersion)" />
1845
<PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
1946
<PackageReference Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />
2047
<PackageReference Include="System.Threading.Channels" Version="$(SystemThreadingChannelsPackageVersion)" />
2148
</ItemGroup>
22-
2349
<ItemGroup>
2450
<Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
2551
<Content Include="build\**\*" Pack="true" PackagePath="build" />
2652
</ItemGroup>
27-
</Project>
53+
<PropertyGroup>
54+
<!--Include Project References output-->
55+
<TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
56+
</PropertyGroup>
57+
<Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
58+
<ItemGroup>
59+
<!--Include DLLs of Project References-->
60+
<BuildOutputInPackage Include="@(ReferenceCopyLocalPaths->WithMetadataValue('ReferenceSourceTarget', 'ProjectReference')->WithMetadataValue('IncludeInNuget','true'))"/>
61+
<!--Include PDBs of Project References-->
62+
<BuildOutputInPackage Include="@(ReferenceCopyLocalPaths->WithMetadataValue('ReferenceSourceTarget', 'ProjectReference')->WithMetadataValue('IncludeInNuget','true')->Replace('.dll', '.pdb'))"/>
63+
<!--Include PDBs for Native binaries-->
64+
<!--The path needed to be hardcoded for this to work on our publishing CI-->
65+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\LdaNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\LdaNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
66+
<BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\LdaNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\LdaNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
67+
</ItemGroup>
68+
</Target>
69+
</Project>

0 commit comments

Comments
 (0)