diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9b0fa188a40..faf3c3847592 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ stages: - build + - publish - package - shared-pipeline - publish @@ -36,7 +37,7 @@ build: -e SIGN_WINDOWS=true ` -e NUGET_CERT_REVOCATION_MODE=offline ` registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:latest ` - Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PackageTracerHome ZipSymbols SignDlls SignMsi + Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PublishFleetInstaller PackageTracerHome ZipSymbols SignDlls SignMsi - mkdir artifacts-out - xcopy /e/s build-out\${CI_JOB_ID}\*.* artifacts-out - remove-item -recurse -force build-out\${CI_JOB_ID} @@ -96,18 +97,22 @@ download-single-step-artifacts: image: registry.ddbuild.io/docker:20.10.13-gbi-focal timeout: 45m tags: [ "arch:amd64" ] - needs: [] + needs: + - job: build + optional: true + artifacts: true rules: - if: $DOTNET_PACKAGE_VERSION when: on_success - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(-prerelease)?$/' # Manually triggered as artifacts are from the Github release when: manual allow_failure: false - - when: delayed # Artifacts come from Azure pipeline, wait a reasonable time before polling - start_in: 15 minutes + - when: on_success # Artifacts come from Azure pipeline, but as we already depend on build, we already have a delayed start script: - .gitlab/download-single-step-artifacts.sh - cp tracer/build/artifacts/requirements.json artifacts/requirements.json + # We currently only copy the windows artifacts if we're _not_ building a tag, because we don't want to push these images to prod yet + - if [ -z "$CI_COMMIT_TAG" ]; then cp -r artifacts-out artifacts/windows; fi artifacts: expire_in: 2 weeks paths: diff --git a/.gitlab/download-single-step-artifacts.sh b/.gitlab/download-single-step-artifacts.sh index cbd93826992d..7a257e16e760 100755 --- a/.gitlab/download-single-step-artifacts.sh +++ b/.gitlab/download-single-step-artifacts.sh @@ -16,6 +16,18 @@ if [ -n "$CI_COMMIT_TAG" ] || [ -n "$DOTNET_PACKAGE_VERSION" ]; then "https://github.com/DataDog/dd-trace-dotnet/releases/download/v${VERSION}/datadog-dotnet-apm-${VERSION}${SUFFIX}.tar.gz" done + if [ -n "$CI_COMMIT_SHA" ]; then + echo "Downloading Windows fleet-installer from S3" + + # Put this in the same place the "build" stage does + win_target_dir=artifacts-out + mkdir -p $win_target_dir + + curl --location --fail \ + --output $win_target_dir/serverless-artifacts.zip \ + "https://dd-windowsfilter.s3.amazonaws.com/builds/tracer/${CI_COMMIT_SHA}/fleet-installer.zip" + fi + echo -n $VERSION > $target_dir/version.txt exit 0 fi diff --git a/.gitlab/prepare-oci-package.sh b/.gitlab/prepare-oci-package.sh index 7c62d6821338..29287b1ce0b8 100755 --- a/.gitlab/prepare-oci-package.sh +++ b/.gitlab/prepare-oci-package.sh @@ -16,32 +16,46 @@ if [ -z "$ARCH" ]; then ARCH=amd64 fi -if [ "$OS" != "linux" ]; then - echo "Only linux packages are supported. Exiting" - exit 0 -fi +if [ "$OS" == "linux" ]; then + if [ "$ARCH" == "amd64" ]; then + SUFFIX="" + elif [ "$ARCH" == "arm64" ]; then + SUFFIX=".arm64" + else + echo "Unsupported architecture: $ARCH" + exit 1 + fi -if [ "$ARCH" == "amd64" ]; then - SUFFIX="" -elif [ "$ARCH" == "arm64" ]; then - SUFFIX=".arm64" -else - echo "Unsupported architecture: $ARCH" - exit 1 -fi + SRC_TAR="../artifacts/datadog-dotnet-apm-$PACKAGE_VERSION${SUFFIX}.tar.gz" -SRC_TAR="../artifacts/datadog-dotnet-apm-$PACKAGE_VERSION${SUFFIX}.tar.gz" + if [ ! -f $SRC_TAR ]; then + echo "$SRC_TAR was not found!" + exit 1 + fi -if [ ! -f $SRC_TAR ]; then - echo "$SRC_TAR was not found!" - exit 1 -fi + mkdir -p sources -mkdir -p sources + # extract the tarball, making sure to preserve the owner and permissions + tar --same-owner -pxvzf $SRC_TAR -C sources -# extract the tarball, making sure to preserve the owner and permissions -tar --same-owner -pxvzf $SRC_TAR -C sources + cp ../artifacts/requirements.json sources/requirements.json -echo -n $VERSION > sources/version +elif [ "$OS" == "windows" ]; then + + if [ "$ARCH" != "amd64" ]; then + echo "Unsupported architecture: win-$ARCH" + exit 0 + fi -cp ../artifacts/requirements.json sources/requirements.json + # unzip the tracer home directory, and remove the xml files + mkdir -p sources/library + unzip ../artifacts/windows/windows-tracer-home.zip -d sources/library + find sources/library -type f -name "*.xml" -delete + + # Unzip the fleet installer to the sub-directory + mkdir -p sources/installer + unzip ../artifacts/windows/fleet-installer.zip -d sources/installer/ + +fi + +echo -n $VERSION > sources/version diff --git a/tracer/build/_build/Build.GitHub.cs b/tracer/build/_build/Build.GitHub.cs index 9b4dcfdfbe58..bea2da30c0c8 100644 --- a/tracer/build/_build/Build.GitHub.cs +++ b/tracer/build/_build/Build.GitHub.cs @@ -1441,7 +1441,7 @@ static async Task DownloadAzureArtifact(AbsolutePath outputDirectory, BuildArtif Console.WriteLine($"Artifact download complete"); } - static async Task DownloadGitlabArtifacts(AbsolutePath outputDirectory, string commitSha, string version) + async Task DownloadGitlabArtifacts(AbsolutePath outputDirectory, string commitSha, string version) { var awsUri = $"https://dd-windowsfilter.s3.amazonaws.com/builds/tracer/{commitSha}/"; var artifactsFiles= new [] @@ -1455,13 +1455,28 @@ static async Task DownloadGitlabArtifacts(AbsolutePath outputDirectory, string c EnsureExistingDirectory(destination); using var client = new HttpClient(); + + // download all the required artifacts that are included in the release foreach (var fileToDownload in artifactsFiles) { - var fileName = Path.GetFileName(fileToDownload); - var destinationFile = destination / fileName; + await DownloadArtifact(client, destination, fileToDownload); + } + + // Ensure that the fleet-installer artifact is available for download, for later in the release + // We don't actually need the file now, we just need to make sure it's available, so that we can + // use it in GitLab later to build the OCI image. + var tempDir = TempDirectory / Path.GetRandomFileName(); + await DownloadArtifact(client, tempDir, $"{awsUri}fleet-installer.zip"); + + return; + + static async Task DownloadArtifact(HttpClient client, AbsolutePath outDir, string fileUrl) + { + var fileName = Path.GetFileName(fileUrl); + var destinationFile = outDir / fileName; - Console.WriteLine($"Downloading {fileToDownload} to {destinationFile}..."); - var response = await client.GetAsync(fileToDownload); + Console.WriteLine($"Downloading {fileUrl} to {destinationFile}..."); + var response = await client.GetAsync(fileUrl); if (!response.IsSuccessStatusCode) { diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index 642d82dbc56e..0aab82bcf408 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -743,6 +743,33 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo .DependsOn(PublishNativeTracerUnix) .DependsOn(PublishNativeTracerOsx); + Target PublishFleetInstaller => _ => _ + .Unlisted() + .Description("Builds and publishes the fleet installer binary files as a zip") + .After(Clean, Restore, CompileManagedSrc) + .Before(SignDlls) + .OnlyWhenStatic(() => IsWin) + .Executes(() => + { + // Build the fleet installer project + var project = SourceDirectory / "Datadog.FleetInstaller" / "Datadog.FleetInstaller.csproj"; + var tfms = Solution.GetProject(project).GetTargetFrameworks(); + // we should only have a single tfm for fleet installer + if (tfms.Count != 1) + { + throw new ApplicationException("Fleet installer should only have a single target framework but found: " + string.Join(", ", tfms)); + } + + var publishFolder = ArtifactsDirectory / "Datadog.FleetInstaller"; + DotNetPublish(s => s + .SetProject(project) + .SetConfiguration(BuildConfiguration) + .SetOutput(publishFolder) + .CombineWith(tfms, (p, tfm) => p.SetFramework(tfm))); + + CompressZip(publishFolder, ArtifactsDirectory / "fleet-installer.zip", fileMode: FileMode.Create); + }); + Target BuildMsi => _ => _ .Unlisted() .Description("Builds the .msi files from the repo")