Build the FleetInstaller executable and include in Windows OCI image (#…

…6644)

## Summary of changes

- Build the fleet installer exe in GitLab
- Include the fleet installer exe in Windows OCI
- Produce Windows OCI images in Staging

## Reason for change

We need to build the fleet installer executable, and include it in a
Windows OI image

## Implementation details

- Add a stage `BuildFleetInstaller` to explicitly publish the fleet
installer
- Update the `download-single-step-artifacts` to grab the
`windows-tracer-home.zip` and fleet installer artifact from the `Build`
stage
- Update the OCI image

For tagged releases this flow is a bit more convoluted. We need to grab
the fleet-instller.exe artifact from the s3 upload that's done in the
`publish` step, because we don't store it on the GitHub release. It's a
bit ugly, but it works.

## Test coverage

This is the test - if it builds, and the OCI image contains the expected
files, we're good

## Other details

Note that _currently_, the final OCI images will _not_ contain Windows
layers, until we unblock that both in our gitlab YAML, and in the
libdatadog-build one pipeline.


Part of a stack of PRs:
 
- #6643
- #6644 👈
- #6645

.gitlab-ci.yml

@@ -1,6 +1,7 @@
 
 stages:
   - build
+  - publish
   - package
   - shared-pipeline
   - publish
@@ -36,7 +37,7 @@ build:
         -e SIGN_WINDOWS=true `
         -e NUGET_CERT_REVOCATION_MODE=offline `
         registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:latest `
-        Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PackageTracerHome ZipSymbols SignDlls SignMsi
+        Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PublishFleetInstaller PackageTracerHome ZipSymbols SignDlls SignMsi
     - mkdir artifacts-out
     - xcopy /e/s build-out\${CI_JOB_ID}\*.* artifacts-out
     - remove-item -recurse -force build-out\${CI_JOB_ID}
@@ -96,18 +97,22 @@ download-single-step-artifacts:
   image: registry.ddbuild.io/docker:20.10.13-gbi-focal
   timeout: 45m
   tags: [ "arch:amd64" ]
-  needs: []
+  needs:
+    - job: build
+      optional: true
+      artifacts: true
   rules:
     - if: $DOTNET_PACKAGE_VERSION
       when: on_success
     - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(-prerelease)?$/' # Manually triggered as artifacts are from the Github release
       when: manual
       allow_failure: false
-    - when: delayed # Artifacts come from Azure pipeline, wait a reasonable time before polling
-      start_in: 15 minutes
+    - when: on_success # Artifacts come from Azure pipeline, but as we already depend on build, we already have a delayed start
   script:
     - .gitlab/download-single-step-artifacts.sh
     - cp tracer/build/artifacts/requirements.json artifacts/requirements.json
+    # We currently only copy the windows artifacts if we're _not_ building a tag, because we don't want to push these images to prod yet
+    - if [ -z "$CI_COMMIT_TAG" ]; then cp -r artifacts-out artifacts/windows; fi
   artifacts:
     expire_in: 2 weeks
     paths:

.gitlab/download-single-step-artifacts.sh

@@ -16,6 +16,18 @@ if [ -n "$CI_COMMIT_TAG" ] || [ -n "$DOTNET_PACKAGE_VERSION" ]; then
       "https://github.com/DataDog/dd-trace-dotnet/releases/download/v${VERSION}/datadog-dotnet-apm-${VERSION}${SUFFIX}.tar.gz"
   done
 
+  if [ -n "$CI_COMMIT_SHA" ]; then
+    echo "Downloading Windows fleet-installer from S3"
+
+    # Put this in the same place the "build" stage does
+    win_target_dir=artifacts-out
+    mkdir -p $win_target_dir
+
+    curl --location --fail \
+        --output $win_target_dir/serverless-artifacts.zip \
+        "https://dd-windowsfilter.s3.amazonaws.com/builds/tracer/${CI_COMMIT_SHA}/fleet-installer.zip"
+  fi
+
   echo -n $VERSION > $target_dir/version.txt
   exit 0
 fi

.gitlab/prepare-oci-package.sh

@@ -16,32 +16,46 @@ if [ -z "$ARCH" ]; then
   ARCH=amd64
 fi
 
-if [ "$OS" != "linux" ]; then
-  echo "Only linux packages are supported. Exiting"
-  exit 0
-fi
+if [ "$OS" == "linux" ]; then
+  if [ "$ARCH" == "amd64" ]; then
+    SUFFIX=""
+  elif [ "$ARCH" == "arm64" ]; then
+    SUFFIX=".arm64"
+  else
+    echo "Unsupported architecture: $ARCH"
+    exit 1
+  fi
 
-if [ "$ARCH" == "amd64" ]; then
-  SUFFIX=""
-elif [ "$ARCH" == "arm64" ]; then
-  SUFFIX=".arm64"
-else
-  echo "Unsupported architecture: $ARCH"
-  exit 1
-fi
+  SRC_TAR="../artifacts/datadog-dotnet-apm-$PACKAGE_VERSION${SUFFIX}.tar.gz"
 
-SRC_TAR="../artifacts/datadog-dotnet-apm-$PACKAGE_VERSION${SUFFIX}.tar.gz"
+  if [ ! -f $SRC_TAR ]; then
+     echo "$SRC_TAR was not found!"
+     exit 1
+  fi
 
-if [ ! -f $SRC_TAR ]; then
-   echo "$SRC_TAR was not found!"
-   exit 1
-fi
+  mkdir -p sources
 
-mkdir -p sources
+  # extract the tarball, making sure to preserve the owner and permissions
+  tar --same-owner -pxvzf $SRC_TAR -C sources
 
-# extract the tarball, making sure to preserve the owner and permissions
-tar --same-owner -pxvzf $SRC_TAR -C sources
+  cp ../artifacts/requirements.json sources/requirements.json
 
-echo -n $VERSION > sources/version
+elif [ "$OS" == "windows" ]; then
+
+  if [ "$ARCH" != "amd64" ]; then
+    echo "Unsupported architecture: win-$ARCH"
+    exit 0
+  fi
 
-cp ../artifacts/requirements.json sources/requirements.json
+  # unzip the tracer home directory, and remove the xml files
+  mkdir -p sources/library
+  unzip ../artifacts/windows/windows-tracer-home.zip -d sources/library
+  find sources/library -type f -name "*.xml" -delete
+
+  # Unzip the fleet installer to the sub-directory
+  mkdir -p sources/installer
+  unzip ../artifacts/windows/fleet-installer.zip -d sources/installer/
+
+fi
+
+echo -n $VERSION > sources/version

tracer/build/_build/Build.GitHub.cs

@@ -1441,7 +1441,7 @@ static async Task DownloadAzureArtifact(AbsolutePath outputDirectory, BuildArtif
         Console.WriteLine($"Artifact download complete");
     }
 
-    static async Task DownloadGitlabArtifacts(AbsolutePath outputDirectory, string commitSha, string version)
+    async Task DownloadGitlabArtifacts(AbsolutePath outputDirectory, string commitSha, string version)
     {
         var awsUri = $"https://dd-windowsfilter.s3.amazonaws.com/builds/tracer/{commitSha}/";
         var artifactsFiles= new []
@@ -1455,13 +1455,28 @@ static async Task DownloadGitlabArtifacts(AbsolutePath outputDirectory, string c
         EnsureExistingDirectory(destination);
 
         using var client = new HttpClient();
+
+        // download all the required artifacts that are included in the release
         foreach (var fileToDownload in artifactsFiles)
         {
-            var fileName = Path.GetFileName(fileToDownload);
-            var destinationFile = destination / fileName;
+            await DownloadArtifact(client, destination, fileToDownload);
+        }
+
+        // Ensure that the fleet-installer artifact is available for download, for later in the release
+        // We don't actually need the file now, we just need to make sure it's available, so that we can
+        // use it in GitLab later to build the OCI image.
+        var tempDir = TempDirectory / Path.GetRandomFileName();
+        await DownloadArtifact(client, tempDir, $"{awsUri}fleet-installer.zip");
+
+        return;
+
+        static async Task DownloadArtifact(HttpClient client, AbsolutePath outDir, string fileUrl)
+        {
+            var fileName = Path.GetFileName(fileUrl);
+            var destinationFile = outDir / fileName;
 
-            Console.WriteLine($"Downloading {fileToDownload} to {destinationFile}...");
-            var response = await client.GetAsync(fileToDownload);
+            Console.WriteLine($"Downloading {fileUrl} to {destinationFile}...");
+            var response = await client.GetAsync(fileUrl);
 
             if (!response.IsSuccessStatusCode)
             {

tracer/build/_build/Build.Steps.cs

@@ -743,6 +743,33 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo
         .DependsOn(PublishNativeTracerUnix)
         .DependsOn(PublishNativeTracerOsx);
 
+    Target PublishFleetInstaller => _ => _
+        .Unlisted()
+        .Description("Builds and publishes the fleet installer binary files as a zip")
+        .After(Clean, Restore, CompileManagedSrc)
+        .Before(SignDlls)
+        .OnlyWhenStatic(() => IsWin)
+        .Executes(() =>
+        {
+            // Build the fleet installer project
+            var project = SourceDirectory / "Datadog.FleetInstaller" / "Datadog.FleetInstaller.csproj";
+            var tfms = Solution.GetProject(project).GetTargetFrameworks();
+            // we should only have a single tfm for fleet installer
+            if (tfms.Count != 1)
+            {
+                throw new ApplicationException("Fleet installer should only have a single target framework but found: " + string.Join(", ", tfms));
+            }
+
+            var publishFolder = ArtifactsDirectory / "Datadog.FleetInstaller";
+            DotNetPublish(s => s
+                              .SetProject(project)
+                              .SetConfiguration(BuildConfiguration)
+                              .SetOutput(publishFolder)
+                              .CombineWith(tfms, (p, tfm) => p.SetFramework(tfm)));
+
+            CompressZip(publishFolder, ArtifactsDirectory / "fleet-installer.zip", fileMode: FileMode.Create);
+        });
+
     Target BuildMsi => _ => _
         .Unlisted()
         .Description("Builds the .msi files from the repo")