Add smoke tests for fleet installer

.azure-pipelines/steps/run-snapshot-test.yml

@@ -124,7 +124,7 @@ steps:
 - ${{ else }}:
   - bash: |
       echo "Verifying snapshot session (fail on mis-match)"
-      ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail "http://localhost:8126$(VERIFY_ENDPOINT)"
+      ${{ parameters.dockerComposePath }} -f $(COMPOSE_PATH) -p ddtrace_$(Build.BuildNumber) exec -T $(TEST_AGENT_TARGET) $(CURL_COMMAND) --fail-with-body "http://localhost:8126$(VERIFY_ENDPOINT)"
     displayName: check snapshots
     env:
       DD_LOGGER_DD_API_KEY: ${{ parameters.apiKey }}

.azure-pipelines/ultimate-pipeline.yml

@@ -1090,7 +1090,7 @@ stages:
           artifact: dd-dotnet-win-x64
           path: $(monitoringHome)/win-x64
 
-      - script: tracer\build.cmd PackageTracerHome
+      - script: tracer\build.cmd PackageTracerHome PublishFleetInstaller
         displayName: Build MSI and Tracer home
         retryCountOnTaskFailure: 1
 
@@ -1106,6 +1106,9 @@ stages:
         displayName: Publish NuGet packages
         artifact: nuget-packages
 
+      - publish: $(artifacts)/Datadog.FleetInstaller
+        displayName: Publish fleet installer
+        artifact: fleet-installer
 
 - stage: build_dd_dotnet_windows
   condition: and(succeeded(), eq(variables['isBenchmarksOnlyBuild'], 'False'))
@@ -6646,6 +6649,85 @@ stages:
     - script: tracer\build.cmd CheckSmokeTestsForErrors
       displayName: CheckSmokeTestsForErrors
 
+- stage: fleet_installer_smoke_tests
+  condition: and(succeeded(), eq(variables['isBenchmarksOnlyBuild'], 'False'))
+  dependsOn: [package_windows, generate_variables, merge_commit_id]
+  variables:
+    targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
+    targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+    dockerComposePath: 'C:/docker-compose/docker-compose.exe'
+  jobs:
+    - template: steps/update-github-status-jobs.yml
+      parameters:
+        jobs: [windows]
+
+    - job: windows
+      timeoutInMinutes: 45 # should take ~15 mins as large Windows docker files
+      strategy:
+        matrix: $[ stageDependencies.generate_variables.generate_variables_job.outputs['generate_variables_step.fleet_installer_windows_smoke_tests_matrix'] ]
+      variables:
+        smokeTestAppDir: "$(System.DefaultWorkingDirectory)/tracer/test/test-applications/regression/AspNetCoreSmokeTest"
+      pool:
+        vmImage: windows-2022
+
+      steps:
+        - template: steps/install-docker-compose-v1.yml
+          parameters:
+            dockerComposePath: $(dockerComposePath)
+
+        - template: steps/clone-repo.yml
+          parameters:
+            targetShaId: $(targetShaId)
+            targetBranch: $(targetBranch)
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download tracer home zip
+          inputs:
+            artifact: windows-tracer-home.zip
+            path: $(smokeTestAppDir)/artifacts
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download fleet installer to temp directory
+          inputs:
+            artifact: fleet-installer
+            path: $(smokeTestAppDir)/artifacts/installer
+
+        - powershell: |
+            ls $(smokeTestAppDir)/artifacts
+            ls $(smokeTestAppDir)/artifacts/installer
+            mkdir -p artifacts/build_data/snapshots
+            mkdir -p artifacts/build_data/logs
+          displayName: Create test data directories
+
+        - bash: |
+            $(dockerComposePath) -f docker-compose.windows.yml -p $(DockerComposeProjectName) build \
+              --build-arg DOTNETSDK_VERSION=$(dotnetCoreSdkLatestVersionShort) \
+              --build-arg RUNTIME_IMAGE=$(runtimeImage) \
+              --build-arg PUBLISH_FRAMEWORK=$(publishFramework) \
+              --build-arg CHANNEL="$(channel)" \
+              --build-arg TARGET_PLATFORM="$(targetPlatform)" \
+              fleet-installer-smoke-tests.windows
+          env:
+            dockerTag: $(dockerTag)
+          displayName: docker-compose build smoke-tests
+          retryCountOnTaskFailure: 3
+
+        - template: steps/run-snapshot-test.yml
+          parameters:
+            target: 'fleet-installer-smoke-tests.windows'
+            snapshotPrefix: "smoke_test_iis"
+            isLinux: false
+            dockerComposePath: $(dockerComposePath)
+
+        - publish: artifacts/build_data
+          artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
+          condition: always()
+          continueOnError: true
+
+        - template: steps/install-latest-dotnet-sdk.yml
+        - script: tracer\build.cmd CheckSmokeTestsForErrors
+          displayName: CheckSmokeTestsForErrors
+
 - stage: dotnet_tool_nuget_smoke_tests_macos
   condition: and(succeeded(), eq(variables['isBenchmarksOnlyBuild'], 'False'), or(eq(variables.isMainOrReleaseBranch, 'true'), eq(variables.run_all_installer_tests, 'true')))
   dependsOn: [dotnet_tool, generate_variables, merge_commit_id]

docker-compose.windows.yml

@@ -20,7 +20,7 @@ services:
     - ./tracer/build/smoke_test_snapshots:c:/snapshots
     - ./artifacts/build_data/snapshots:c:/debug_snapshots
     ports:
-    - "8126:8126"
+    - "8126"
     environment:
     - SNAPSHOT_CI=1
     - SNAPSHOT_IGNORED_ATTRS=span_id,trace_id,parent_id,duration,start,metrics.system.pid,meta.runtime-id,metrics.process_id,meta.http.client_ip,meta.network.client.ip,meta._dd.p.dm,meta._dd.p.tid,meta._dd.parent_id,meta._dd.appsec.s.req.params,meta._dd.appsec.s.res.body,meta._dd.appsec.s.req.headers,meta._dd.appsec.s.res.headers #api-security attrs are unfortunately ignored because gzip compression generates different bytes per platform windows/linux
@@ -157,6 +157,27 @@ services:
     depends_on:
     - test-agent.windows
 
+  fleet-installer-smoke-tests.windows:
+    build:
+      context: ./tracer/ # have to use this as the context, as Dockercompose requires dockerfile to be inside context dir
+      dockerfile: build/_build/docker/smoke.windows.fleet-installer.dockerfile
+        # args:
+        # Note that the following build arguments must be provided
+        # - DOTNETSDK_VERSION=
+        # - RUNTIME_IMAGE=
+        # - PUBLISH_FRAMEWORK=
+        # - CHANNEL=
+        # - TARGET_PLATFORM=
+    image: dd-trace-dotnet/${dockerTag:-not-set}-windows-fleet-installer-tester
+    volumes:
+    - ./:c:/project
+    - ./artifacts/build_data/logs:c:/logs
+    environment:
+    - dockerTag=${dockerTag:-unset}
+    - DD_TRACE_AGENT_URL=http://test-agent.windows:8126
+    depends_on:
+    - test-agent.windows
+
   dotnet-tool-smoke-tests.windows:
     build:
       context: ./tracer/ # have to use this as the context, as Dockercompose requires dockerfile to be inside context dir

tracer/build/_build/Build.VariableGenerations.cs

@@ -453,8 +453,9 @@ void GenerateSmokeTestsMatrices()
                 // msi smoke tests
                 GenerateWindowsMsiSmokeTestsMatrix();
 
-                // tracer home smoke tests
+                // tracer home / fleet installer smoke tests
                 GenerateWindowsTracerHomeSmokeTestsMatrix();
+                GenerateWindowsFleetInstalerSmokeTestsMatrix();
 
                 // macos smoke tests
                 GenerateMacosDotnetToolNugetSmokeTestsMatrix();
@@ -1248,6 +1249,36 @@ from image in runtimeImages
                     AzurePipelines.Instance.SetOutputVariable("tracer_home_installer_windows_smoke_tests_matrix", JsonConvert.SerializeObject(matrix, Formatting.None));
                 }
 
+                void GenerateWindowsFleetInstalerSmokeTestsMatrix()
+                {
+                    var dockerName = "mcr.microsoft.com/dotnet/framework/aspnet";
+
+                    var platforms = new[] { MSBuildTargetPlatform.x64, MSBuildTargetPlatform.x86, };
+                    var runtimeImages = new SmokeTestImage[]
+                    {
+                        // We can only test Windows 2022 images currently, due to VM + docker image support
+                        new (publishFramework: TargetFramework.NET9_0, "4.8-windowsservercore-ltsc2022"),
+                        new (publishFramework: TargetFramework.NET8_0, "4.8-windowsservercore-ltsc2022"),
+                    };
+
+                    var matrix = (
+                                     from platform in platforms
+                                     from image in runtimeImages
+                                     let dockerTag = $"{image.PublishFramework}_{platform}_{image.RuntimeTag}".Replace('.', '_')
+                                     select new
+                                     {
+                                         dockerTag = dockerTag,
+                                         publishFramework = image.PublishFramework,
+                                         runtimeImage = $"{dockerName}:{image.RuntimeTag}",
+                                         targetPlatform = platform,
+                                         channel = GetInstallerChannel(image.PublishFramework),
+                                     }).ToDictionary(x=>x.dockerTag, x => x);
+
+                    Logger.Information($"Installer smoke tests fleet-installer matrix Windows");
+                    Logger.Information(JsonConvert.SerializeObject(matrix, Formatting.Indented));
+                    AzurePipelines.Instance.SetOutputVariable("fleet_installer_windows_smoke_tests_matrix", JsonConvert.SerializeObject(matrix, Formatting.None));
+                }
+
                 void GenerateWindowsNuGetSmokeTestsMatrix()
                 {
                     var dockerName = "mcr.microsoft.com/dotnet/aspnet";

tracer/build/_build/docker/smoke.windows.fleet-installer.dockerfile

@@ -0,0 +1,73 @@
+ARG DOTNETSDK_VERSION
+ARG RUNTIME_IMAGE
+
+# Build the ASP.NET Core app using the latest SDK
+FROM mcr.microsoft.com/dotnet/sdk:$DOTNETSDK_VERSION-windowsservercore-ltsc2022 as builder
+
+# Build the smoke test app
+WORKDIR /src
+COPY ./test/test-applications/regression/AspNetCoreSmokeTest/ .
+
+ARG PUBLISH_FRAMEWORK
+RUN dotnet publish "AspNetCoreSmokeTest.csproj" -c Release --framework %PUBLISH_FRAMEWORK% -o /src/publish
+
+FROM $RUNTIME_IMAGE AS publish
+SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+WORKDIR /app
+
+ARG CHANNEL
+ARG TARGET_PLATFORM
+
+# Install the hosting bundle
+RUN  $url='https://builds.dotnet.microsoft.com/dotnet/aspnetcore/Runtime/' + $env:CHANNEL + '.0/dotnet-hosting-' + $env:CHANNEL + '.0-win.exe'; \
+    echo "Fetching " + $url; \
+    Invoke-WebRequest $url -OutFile c:/hosting.exe; \
+    Start-Process -Wait -PassThru -FilePath "c:/hosting.exe" -ArgumentList @('/install', '/q', '/norestart'); \
+    rm c:/hosting.exe;
+
+# Copy the tracer home file from tracer/test/test-applications/regression/AspNetCoreSmokeTest/artifacts
+COPY --from=builder /src/artifacts /install
+
+RUN mkdir /logs; \
+    mkdir /monitoring-home; \
+    cd /install; \
+    Expand-Archive 'c:\install\windows-tracer-home.zip' -DestinationPath 'c:\monitoring-home\';  \
+    c:\install\installer\Datadog.FleetInstaller.exe install --home-path c:\monitoring-home; \
+    cd /app;
+
+# Set the additional env vars
+ENV DD_PROFILING_ENABLED=1 \
+    DD_TRACE_DEBUG=1 \
+    DD_APPSEC_ENABLED=1 \
+    DD_TRACE_LOG_DIRECTORY="C:\logs"
+
+# Set a random env var we should ignore
+ENV SUPER_SECRET_CANARY=MySuperSecretCanary
+
+# see https://github.com/DataDog/dd-trace-dotnet/pull/3579
+ENV DD_INTERNAL_WORKAROUND_77973_ENABLED=1
+
+# Copy the app across
+COPY --from=builder /src/publish /app/.
+
+# Create new website we control, but keep auto-start disabled
+RUN Remove-WebSite -Name 'Default Web Site'; \
+    $ENABLE_32_BIT='false'; \
+    if($env:TARGET_PLATFORM -eq 'x86') { $ENABLE_32_BIT='true' }; \
+    Write-Host "Creating website with 32 bit enabled: $env:ENABLE_32_BIT"; \
+    c:\Windows\System32\inetsrv\appcmd add apppool /startMode:"AlwaysRunning" /autoStart:"false" /name:AspNetCorePool /managedRuntimeVersion:"" /enable32bitapponwin64:$ENABLE_32_BIT; \
+    New-Website -Name 'SmokeTest' -Port 5000 -PhysicalPath 'c:\app' -ApplicationPool 'AspNetCorePool'; \
+    Set-ItemProperty "IIS:\Sites\SmokeTest" -Name applicationDefaults.preloadEnabled -Value True;
+
+# We override the normal service monitor entrypoint, because we want the container to shut down after the request is sent
+# We would really like to get the pid of the worker processes, but we can't do that easily
+# This is all way more convoluted than it feels like it should be, but it's the only way I could find to get things to work as required
+RUN echo 'Write-Host \"Running servicemonitor to copy variables\"; Start-Process -NoNewWindow -PassThru -FilePath \"c:/ServiceMonitor.exe\" -ArgumentList @(\"w3svc\", \"AspNetCorePool\");' > C:\app\entrypoint.ps1; \
+    echo 'Write-Host \"Starting AspNetCorePool app pool\"; Start-WebAppPool -Name \"AspNetCorePool\" -PassThru;' >> C:\app\entrypoint.ps1; \
+    echo 'Write-Host \"Making 404 request\"; curl http://localhost:5000;' >> C:\app\entrypoint.ps1; \
+    echo 'Write-Host \"Stopping pool\";Stop-WebAppPool \"AspNetCorePool\" -PassThru;' >> C:\app\entrypoint.ps1;  \
+    echo 'Write-Host \"Shutting down\"' >> C:\app\entrypoint.ps1;
+
+# Set the script as the entrypoint
+ENTRYPOINT ["powershell", "-File", "C:\\app\\entrypoint.ps1"]