Merge tools into Nuke (#1605)

* Merge GeneratePackageVersions into Nuke

* Move UpdateVendors tool to be a Nuke target

Embed the template as a resource for simplicity

* minor: move Nuke helper files into folder for easier navigation

* Move EnvironmentTools.cs to Datadog.Trace.TestHelpers

It's only used in test projects anyway

* Move TcpPortProvider to TestHelpers, as again, it's only used in tests

Some of the sample projects use this type, so rather than reference TestHelpers (and hence transitively Datadog.Trace), added the helper class a link file instead.

* Remove remaining references to Datadog.Core.Tools

* Move PrepareRelease to Nuke

This required quite a lot of work in the GenerateIntegrationDefinitions code to use reflection instead of direct references.

Additionally, I purposefully iterate all the built versions of the assembly, and de-dupe the duplicates, to avoid cases where some integrations are not available on some platforms. This required a slightly annoying amount of boilerplate and faffing with AppLoadContexts

* Move CIAppTracking project to Nuke

* Don't override everything in Nuke's editorconfig

* Update build/_build/Build.Utilities.cs

Co-authored-by: Zach Montoya <[email protected]>

* Remove DistributedTracer tool

It's for a very specific use-case, and as it's the last remaining tool, I decided to remove it

* Apply suggestions from code review

Co-authored-by: Lucas Pimentel-Ordyna <[email protected]>

* Update some target names

Co-authored-by: Zach Montoya <[email protected]>
Co-authored-by: Lucas Pimentel-Ordyna <[email protected]>

Datadog.Trace.proj

@@ -15,7 +15,6 @@
     <ExcludeEF6DemoProject Remove="test/test-applications/regression/**/EntityFramework6x*.csproj" />
     <ExcludeLegacyRedisProject Remove="test/test-applications/regression/**/StackExchange.Redis.AssemblyConflict.LegacyProject.csproj" />
     <CsharpTestHelperProject Include="test\**\*.TestHelpers.csproj"/>
-    <CsharpToolsProject Include="tools\Datadog.Core.Tools\Datadog.Core.Tools.csproj"/>
     <CppProject Include="src\**\*.vcxproj"/>
     <CppTestProject Include="test\**\*.vcxproj"/>
     <VbSampleLibProject Include="test\test-applications\integrations\**\*.vbproj" />
@@ -37,14 +36,14 @@
 
   <!--  Used by CompileManagedTestHelpers-->
   <Target Name="BuildCsharpTestHelpers">
-    <MSBuild Targets="Build" Projects="@(CsharpToolsProject);@(CsharpTestHelperProject)">
+    <MSBuild Targets="Build" Projects="@(CsharpTestHelperProject)">
       <Output TaskParameter="TargetOutputs" ItemName="CollectedBuildOutput"/>
     </MSBuild>
   </Target>
 
   <!--  Used by CompileManagedUnitTests-->
   <Target Name="BuildCsharpUnitTests">
-    <MSBuild Targets="Build" Projects="@(CsharpToolsProject);@(CsharpTestHelperProject);@(CsharpUnitTestProject)">
+    <MSBuild Targets="Build" Projects="@(CsharpTestHelperProject);@(CsharpUnitTestProject)">
       <Output TaskParameter="TargetOutputs" ItemName="CollectedBuildOutput"/>
     </MSBuild>
   </Target>

build/_build/.editorconfig

@@ -1,4 +1,4 @@
-[*.cs]
+[{Build.cs, Build.*.cs}]
 dotnet_style_qualification_for_field = false:warning
 dotnet_style_qualification_for_property = false:warning
 dotnet_style_qualification_for_method = false:warning

build/_build/Build.Utilities.cs

@@ -2,13 +2,18 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Reflection;
+using GeneratePackageVersions;
 using Nuke.Common;
 using Nuke.Common.IO;
 using Nuke.Common.Tooling;
 using Nuke.Common.Tools.Docker;
 using Nuke.Common.Tools.DotNet;
 using Nuke.Common.Tools.MSBuild;
+using PrepareRelease;
+using UpdateVendors;
 using static Nuke.Common.EnvironmentInfo;
+using static Nuke.Common.IO.FileSystemTasks;
 using static Nuke.Common.Tools.DotNet.DotNetTasks;
 using static Nuke.Common.Tools.MSBuild.MSBuildTasks;
 
@@ -168,4 +173,91 @@ partial class Build
                 .SetProcessEnvironmentVariables(envVars));
 
         });
+
+    Target GeneratePackageVersions => _ => _
+       .Description("Regenerate the PackageVersions props and .cs files")
+       .Executes(async () =>
+       {
+           var testDir = Solution.GetProject(Projects.ClrProfilerIntegrationTests).Directory;
+
+           var versionGenerator = new PackageVersionGenerator(RootDirectory, testDir);
+           await versionGenerator.GenerateVersions();
+       });
+
+    Target UpdateVendoredCode => _ => _
+       .Description("Updates the vendored dependency code and dependabot template")
+       .Executes(() =>
+       {
+            var honeypotProject = RootDirectory / "honeypot"  /  "Datadog.Dependabot.Honeypot.csproj";
+            UpdateVendorsTool.UpdateHoneypotProject(honeypotProject);
+
+            var vendorDirectory = Solution.GetProject(Projects.DatadogTrace).Directory / "Vendors";
+            var downloadDirectory = TemporaryDirectory / "Downloads";
+            EnsureCleanDirectory(downloadDirectory);
+            UpdateVendorsTool.UpdateVendors(downloadDirectory, vendorDirectory);
+       });
+
+    Target UpdateIntegrationsJson => _ => _
+       .Description("Update the integrations.json file")
+       .DependsOn(Clean, Restore, CreateRequiredDirectories, CompileManagedSrc, PublishManagedProfiler) // We load the dlls from the output, so need to do a clean build
+       .Before(CopyIntegrationsJson)
+       .Executes(() =>
+        {
+            var assemblies = TracerHomeDirectory
+                            .GlobFiles("**/Datadog.Trace.ClrProfiler.Managed.dll")
+                            .Select(x => x.ToString())
+                            .ToList();
+
+            GenerateIntegrationDefinitions.Run(assemblies, RootDirectory);
+        });
+
+    Target UpdateVersion => _ => _
+       .Description("Update the version number for the tracer")
+       .Before(Clean, BuildTracerHome)
+       .Requires(() => Version)
+       .Executes(() =>
+        {
+            new SetAllVersions(RootDirectory, Version, IsPrerelease).Run();
+        });
+
+    Target UpdateMsiContents => _ => _
+       .Description("Update the contents of the MSI")
+       .DependsOn(Clean, BuildTracerHome)
+       .Executes(() =>
+        {
+            SyncMsiContent.Run(RootDirectory, TracerHomeDirectory);
+        });
+
+    Target CiAppFeatureTracking => _ => _
+       .Description("Generate the CIApp FeatureTracking JSON")
+       .DependsOn(Clean, Restore, CreateRequiredDirectories, CompileManagedSrc, PublishManagedProfiler) // We load the dlls from the output, so need to do a clean build
+       .Executes(() =>
+        {
+            // Just grab the first one for now
+            var assemblyPath = TracerHomeDirectory
+                            .GlobFiles("**/netcoreapp*/Datadog.Trace.ClrProfiler.Managed.dll")
+                            .Select(x => x.ToString())
+                            .First();
+
+            var assembly = Assembly.LoadFrom(assemblyPath);
+            var featureTrackingAttribute = assembly.GetType("Datadog.Trace.Ci.FeatureTrackingAttribute");
+            var types = new[] { "Datadog.Trace.Ci.CommonTags", "Datadog.Trace.Ci.TestTags" }
+                       .Select(type => assembly.GetType(type))
+                       .ToArray();
+
+            var values = GetFeatureTrackingValueFromType(types);
+
+            var json = Newtonsoft.Json.JsonConvert.SerializeObject(values);
+            Console.WriteLine(json);
+
+            IEnumerable<string> GetFeatureTrackingValueFromType(params Type[] types)
+            {
+                return types
+                      .SelectMany(type => type.GetFields())
+                      .Where(f => f.GetCustomAttributes(featureTrackingAttribute, true).Length > 0)
+                      .Select(f => f.GetValue(null).ToString())
+                      .OrderBy(v => v);
+            }
+        });
+
 }

build/_build/Build.cs

@@ -50,9 +50,12 @@ partial class Build : NukeBuild
     [Parameter("Is the build running on Alpine linux? Default is 'false'")]
     readonly bool IsAlpine = false;
 
-    [Parameter("The build version (for packaging purposes). Default is latest")]
+    [Parameter("The build version. Default is latest")]
     readonly string Version = "1.28.1";
 
+    [Parameter("Whether the build version is a prerelease(for packaging purposes). Default is latest")]
+    readonly bool IsPrerelease = true;
+
     [Parameter("Prints the available drive space before executing each target. Defaults to false")]
     readonly bool PrintDriveSpace = false;
 

build/tools/GeneratePackageVersions/Program.cs → build/_build/GeneratePackageVersions/Program.cs

@@ -9,53 +9,41 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
+using Nuke.Common.IO;
 
 namespace GeneratePackageVersions
 {
-    public class Program
+    public class PackageVersionGenerator
     {
-        private static string _solutionDirectory;
-        private static string _baseXunitPath;
-        private static PackageGroup _latestMinors;
-        private static PackageGroup _comprehensive;
-        private static XunitStrategyFileGenerator _strategyGenerator;
-
-        public static async Task Main(string[] args)
+        private readonly AbsolutePath _definitionsFilePath;
+        private readonly PackageGroup _latestMinors;
+        private readonly PackageGroup _comprehensive;
+        private readonly XunitStrategyFileGenerator _strategyGenerator;
+
+        public PackageVersionGenerator(
+            AbsolutePath solutionDirectory,
+            AbsolutePath testProjectDirectory)
         {
-            if (args.Length < 1 || string.IsNullOrWhiteSpace(args[0]))
-            {
-                Console.Error.WriteLine("error: You must specify the solution directory. Exiting.");
-                return;
-            }
-
-            _solutionDirectory = args[0];
-
-            _baseXunitPath = Path.Combine(
-                _solutionDirectory,
-                "test",
-                "Datadog.Trace.ClrProfiler.IntegrationTests");
-
-            _latestMinors = new PackageGroup("LatestMinors");
-            _comprehensive = new PackageGroup("Comprehensive");
+            var propsDirectory = solutionDirectory / "build";
+            _definitionsFilePath = solutionDirectory / "build" / "PackageVersionsGeneratorDefinitions.json";
+            _latestMinors = new PackageGroup(propsDirectory, testProjectDirectory, "LatestMinors");
+            _comprehensive = new PackageGroup(propsDirectory, testProjectDirectory, "Comprehensive");
+            _strategyGenerator = new XunitStrategyFileGenerator(testProjectDirectory / "PackageVersions.g.cs");
 
-            _strategyGenerator = new XunitStrategyFileGenerator(
-                Path.Combine(
-                    _baseXunitPath,
-                    "PackageVersions.g.cs"));
-
-            var definitionsFilename = Path.Combine(args[0], "build", "PackageVersionsGeneratorDefinitions.json");
-
-            if (!File.Exists(definitionsFilename))
+            if (!File.Exists(_definitionsFilePath))
             {
-                Console.Error.WriteLine($"error: Definitions file {definitionsFilename} does not exist. Exiting.");
-                return;
+                throw new Exception($"Definitions file {_definitionsFilePath} does not exist. Exiting.");
             }
+        }
 
-            var entries = JsonConvert.DeserializeObject<PackageVersionEntry[]>(File.ReadAllText(definitionsFilename));
+        public async Task GenerateVersions()
+        {
+            var definitions = File.ReadAllText(_definitionsFilePath);
+            var entries = JsonConvert.DeserializeObject<PackageVersionEntry[]>(definitions);
             await RunFileGeneratorWithPackageEntries(entries);
         }
 
-        private static async Task RunFileGeneratorWithPackageEntries(IEnumerable<PackageVersionEntry> entries)
+        private async Task RunFileGeneratorWithPackageEntries(IEnumerable<PackageVersionEntry> entries)
         {
             _latestMinors.Start();
             _comprehensive.Start();
@@ -114,18 +102,15 @@ private static async Task RunFileGeneratorWithPackageEntries(IEnumerable<Package
         private class PackageGroup
         {
             private readonly MSBuildPropsFileGenerator _msBuildPropsFileGenerator;
-
             private readonly XUnitFileGenerator _xUnitFileGenerator;
 
-            public PackageGroup(string postfix)
+            public PackageGroup(string propsDirectory, string testDirectoryPath, string postfix)
             {
                 var className = $"PackageVersions{postfix}";
 
-                var outputPackageVersionsPropsFilename = Path.Combine(_solutionDirectory, "build", $"PackageVersions{postfix}.g.props");
+                var outputPackageVersionsPropsFilename = Path.Combine(propsDirectory, $"PackageVersions{postfix}.g.props");
 
-                var outputPackageVersionsXunitFilename = Path.Combine(
-                    _baseXunitPath,
-                    $"PackageVersions{postfix}.g.cs");
+                var outputPackageVersionsXunitFilename = Path.Combine(testDirectoryPath, $"PackageVersions{postfix}.g.cs");
 
                 _msBuildPropsFileGenerator = new MSBuildPropsFileGenerator(outputPackageVersionsPropsFilename);