From c741f9294366fdf079a225799173e81b1d521365 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Wed, 5 Feb 2025 12:30:29 +0100 Subject: [PATCH] [Fleet Automation] Add a flag to force install package (#33600) --- .gitlab/deploy_packages/e2e.yml | 20 +- .gitlab/deploy_packages/windows.yml | 18 +- .gitlab/e2e/e2e.yml | 4 +- .gitlab/e2e_install_packages/installer.yml | 1 + .gitlab/package_build/installer.yml | 21 ++ .../subcommands/installer/command.go | 5 + pkg/fleet/daemon/daemon_test.go | 8 +- pkg/fleet/installer/installer.go | 24 +- pkg/fleet/installer/installer_test.go | 219 +++++++++++------- pkg/fleet/installer/setup/setup.go | 2 +- .../internal/bootstrap/bootstrap_windows.go | 2 +- pkg/fleet/internal/exec/installer_exec.go | 24 +- pkg/fleet/internal/msi/msiexec.go | 23 +- .../installer/windows/datadog_installer.go | 6 +- .../remote_windows_host_asserts.go | 28 +++ .../suites/install-script/install_test.go | 79 +++++++ .../Install-Datadog.ps1 | 2 + 17 files changed, 365 insertions(+), 121 deletions(-) create mode 100644 test/new-e2e/tests/installer/windows/suites/install-script/install_test.go diff --git a/.gitlab/deploy_packages/e2e.yml b/.gitlab/deploy_packages/e2e.yml index fd3fca5403a8c1..c33cfef5fe4d5e 100644 --- a/.gitlab/deploy_packages/e2e.yml +++ b/.gitlab/deploy_packages/e2e.yml @@ -1,6 +1,8 @@ -# Jobs that deploy agent packages on QA environment, to be used by e2e tests +# Jobs that deploy agent packages on QA environment, to be used by e2e tests. +# We use two separate jobs for Windows and Linux so that a failure in deploying the +# Linux / Windows script doesn't impact the other OS (i.e. Windows scripts failing to be signed blocking Linux E2E tests). -qa_installer_script: +qa_installer_script_linux: image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES stage: deploy_packages tags: ["arch:amd64"] @@ -14,3 +16,17 @@ qa_installer_script: script: - $S3_CP_CMD --recursive --exclude "*" --include "install*.sh" "$OMNIBUS_PACKAGE_DIR" "s3://${INSTALLER_TESTING_S3_BUCKET}/${CI_COMMIT_SHA}/scripts/" - $S3_CP_CMD --recursive --exclude "*" --include "install*.sh" "$OMNIBUS_PACKAGE_DIR" "s3://${INSTALLER_TESTING_S3_BUCKET}/pipeline-${CI_PIPELINE_ID}/scripts/" + +qa_installer_script_windows: + image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + stage: deploy_packages + tags: ["arch:amd64"] + rules: + - !reference [.on_installer_or_e2e_changes] + - !reference [.manual] + needs: + - powershell_script_signing + before_script: + - ls $WINDOWS_POWERSHELL_DIR + script: + - $S3_CP_CMD $WINDOWS_POWERSHELL_DIR/Install-Datadog.ps1 s3://${INSTALLER_TESTING_S3_BUCKET}/pipeline-${CI_PIPELINE_ID}/scripts/Install-Datadog.ps1 diff --git a/.gitlab/deploy_packages/windows.yml b/.gitlab/deploy_packages/windows.yml index 204bcb5786df75..c53b70310f7ec2 100644 --- a/.gitlab/deploy_packages/windows.yml +++ b/.gitlab/deploy_packages/windows.yml @@ -51,23 +51,6 @@ deploy_staging_windows_tags-7: full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732 # Datadog Installer -powershell_script_signing: - extends: .windows_docker_default - stage: deploy_packages - needs: [] - variables: - ARCH: "x64" - rules: - !reference [.on_deploy_installer] - artifacts: - expire_in: 2 weeks - paths: - - $WINDOWS_POWERSHELL_DIR - script: - - mkdir $WINDOWS_POWERSHELL_DIR - - docker run --rm -v "$(Get-Location):c:\mnt" -e AWS_NETWORKING=true -e IS_AWS_CONTAINER=true ${WINBUILDIMAGE} powershell -C "dd-wcs sign \mnt\tools\windows\DatadogAgentInstallScript\Install-Datadog.ps1" - - copy .\tools\windows\DatadogAgentInstallScript\Install-Datadog.ps1 $WINDOWS_POWERSHELL_DIR\Install-Datadog.ps1 - deploy_installer_packages_windows-x64: rules: !reference [.on_deploy_installer] @@ -77,6 +60,7 @@ deploy_installer_packages_windows-x64: needs: ["windows-installer-amd64", "powershell_script_signing"] before_script: - ls $OMNIBUS_PACKAGE_DIR + - ls $WINDOWS_POWERSHELL_DIR script: - $S3_CP_CMD --recursive diff --git a/.gitlab/e2e/e2e.yml b/.gitlab/e2e/e2e.yml index 82692b64e5289d..9f62c998aa17c4 100644 --- a/.gitlab/e2e/e2e.yml +++ b/.gitlab/e2e/e2e.yml @@ -460,7 +460,7 @@ new-e2e-installer-script: - deploy_suse_rpm_testing_arm64-a7 - deploy_suse_rpm_testing_x64-a7 - deploy_installer_oci - - qa_installer_script + - qa_installer_script_linux variables: TARGETS: ./tests/installer/script TEAM: fleet @@ -497,6 +497,7 @@ new-e2e-installer-windows: - deploy_windows_testing-a7 - deploy_installer_oci - deploy_agent_oci + - qa_installer_script_windows before_script: # CURRENT_AGENT_VERSION is used to verify the installed agent version # Must run before new_e2e_template changes the aws profile @@ -517,6 +518,7 @@ new-e2e-installer-windows: - EXTRA_PARAMS: --run "TestAgentInstalls$" - EXTRA_PARAMS: --run "TestAgentUpgrades$" # install-script + - EXTRA_PARAMS: --run "TestInstallScript$" - EXTRA_PARAMS: --run "TestInstallScriptWithAgentUser$" # installer-package - EXTRA_PARAMS: --run "TestInstaller$" diff --git a/.gitlab/e2e_install_packages/installer.yml b/.gitlab/e2e_install_packages/installer.yml index d6af7e7339878a..f136c86e9b7a98 100644 --- a/.gitlab/e2e_install_packages/installer.yml +++ b/.gitlab/e2e_install_packages/installer.yml @@ -13,3 +13,4 @@ qa_installer_script_main: - ls $OMNIBUS_PACKAGE_DIR script: - $S3_CP_CMD --recursive --exclude "*" --include "install*.sh" "$OMNIBUS_PACKAGE_DIR" "s3://${INSTALLER_TESTING_S3_BUCKET}/scripts/" + diff --git a/.gitlab/package_build/installer.yml b/.gitlab/package_build/installer.yml index 199cc52b82414d..5210e76f41957b 100644 --- a/.gitlab/package_build/installer.yml +++ b/.gitlab/package_build/installer.yml @@ -115,6 +115,27 @@ installer-install-scripts: paths: - $OMNIBUS_PACKAGE_DIR +# +# Windows install script +# +powershell_script_signing: + extends: .windows_docker_default + stage: package_build + needs: [] + variables: + ARCH: "x64" + rules: + - !reference [.except_mergequeue] + - when: on_success + artifacts: + expire_in: 2 weeks + paths: + - $WINDOWS_POWERSHELL_DIR + script: + - mkdir $WINDOWS_POWERSHELL_DIR + - docker run --rm -v "$(Get-Location):c:\mnt" -e AWS_NETWORKING=true -e IS_AWS_CONTAINER=true ${WINBUILDIMAGE} powershell -C "dd-wcs sign \mnt\tools\windows\DatadogAgentInstallScript\Install-Datadog.ps1" + - copy .\tools\windows\DatadogAgentInstallScript\Install-Datadog.ps1 $WINDOWS_POWERSHELL_DIR\Install-Datadog.ps1 + # # The installer program # diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index 0642cacda83106..a1e7ac10423c5b 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -284,6 +284,7 @@ func setupCommand() *cobra.Command { func installCommand() *cobra.Command { var installArgs []string + var forceInstall bool cmd := &cobra.Command{ Use: "install ", Short: "Install a package", @@ -296,10 +297,14 @@ func installCommand() *cobra.Command { } defer func() { i.stop(err) }() i.span.SetTag("params.url", args[0]) + if forceInstall { + return i.ForceInstall(i.ctx, args[0], installArgs) + } return i.Install(i.ctx, args[0], installArgs) }, } cmd.Flags().StringArrayVarP(&installArgs, "install_args", "A", nil, "Arguments to pass to the package") + cmd.Flags().BoolVar(&forceInstall, "force", false, "Install packages, even if they are already up-to-date.") return cmd } diff --git a/pkg/fleet/daemon/daemon_test.go b/pkg/fleet/daemon/daemon_test.go index 28c6d5c61abec4..88c5710b235ca1 100644 --- a/pkg/fleet/daemon/daemon_test.go +++ b/pkg/fleet/daemon/daemon_test.go @@ -3,9 +3,6 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// for now the installer is not supported on windows -//go:build !windows - package daemon import ( @@ -67,6 +64,11 @@ func (m *testPackageManager) Install(ctx context.Context, url string, installArg return args.Error(0) } +func (m *testPackageManager) ForceInstall(ctx context.Context, url string, installArgs []string) error { + args := m.Called(ctx, url, installArgs) + return args.Error(0) +} + func (m *testPackageManager) Remove(ctx context.Context, pkg string) error { args := m.Called(ctx, pkg) return args.Error(0) diff --git a/pkg/fleet/installer/installer.go b/pkg/fleet/installer/installer.go index 7e6da446d153a4..d8339db6f67c4a 100644 --- a/pkg/fleet/installer/installer.go +++ b/pkg/fleet/installer/installer.go @@ -50,6 +50,7 @@ type Installer interface { ConfigStates() (map[string]repository.State, error) Install(ctx context.Context, url string, args []string) error + ForceInstall(ctx context.Context, url string, args []string) error Remove(ctx context.Context, pkg string) error Purge(ctx context.Context) @@ -151,8 +152,28 @@ func (i *installerImpl) IsInstalled(_ context.Context, pkg string) (bool, error) return hasPackage, nil } +// ForceInstall installs or updates a package, even if it's already installed +func (i *installerImpl) ForceInstall(ctx context.Context, url string, args []string) error { + return i.doInstall(ctx, url, args, func(dbPkg db.Package, pkg *oci.DownloadedPackage) bool { + if dbPkg.Name == pkg.Name && dbPkg.Version == pkg.Version { + log.Warnf("package %s version %s is already installed, updating it anyway", pkg.Name, pkg.Version) + } + return true + }) +} + // Install installs or updates a package. func (i *installerImpl) Install(ctx context.Context, url string, args []string) error { + return i.doInstall(ctx, url, args, func(dbPkg db.Package, pkg *oci.DownloadedPackage) bool { + if dbPkg.Name == pkg.Name && dbPkg.Version == pkg.Version { + log.Warnf("package %s version %s is already installed", pkg.Name, pkg.Version) + return false + } + return true + }) +} + +func (i *installerImpl) doInstall(ctx context.Context, url string, args []string, shouldInstallPredicate func(dbPkg db.Package, pkg *oci.DownloadedPackage) bool) error { i.m.Lock() defer i.m.Unlock() pkg, err := i.downloader.Download(ctx, url) // Downloads pkg metadata only @@ -168,8 +189,7 @@ func (i *installerImpl) Install(ctx context.Context, url string, args []string) if err != nil && !errors.Is(err, db.ErrPackageNotFound) { return fmt.Errorf("could not get package: %w", err) } - if dbPkg.Name == pkg.Name && dbPkg.Version == pkg.Version { - log.Infof("package %s version %s is already installed", pkg.Name, pkg.Version) + if !shouldInstallPredicate(dbPkg, pkg) { return nil } err = i.preparePackage(ctx, pkg.Name, args) // Preinst diff --git a/pkg/fleet/installer/installer_test.go b/pkg/fleet/installer/installer_test.go index 1527993014605e..92e79fb9d55ee6 100644 --- a/pkg/fleet/installer/installer_test.go +++ b/pkg/fleet/installer/installer_test.go @@ -11,6 +11,8 @@ import ( "os" "path" "path/filepath" + "reflect" + "runtime" "testing" "time" @@ -25,6 +27,9 @@ import ( var testCtx = context.TODO() +type installFn = func(context.Context, string, []string) error +type installFnFactory = func(manager *testPackageManager) installFn + type testPackageManager struct { installerImpl } @@ -52,79 +57,87 @@ func (i *testPackageManager) ConfigFS(f fixtures.Fixture) fs.FS { } func TestInstallStable(t *testing.T) { - s := fixtures.NewServer(t) - installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) - defer installer.db.Close() - - err := installer.Install(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) - assert.NoError(t, err) - r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) - state, err := r.GetState() - assert.NoError(t, err) - assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) - assert.False(t, state.HasExperiment()) - fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV1), r.StableFS()) - fixtures.AssertEqualFS(t, s.ConfigFS(fixtures.FixtureSimpleV1), installer.ConfigFS(fixtures.FixtureSimpleV1)) + doTestInstallers(t, func(instFactory installFnFactory, t *testing.T) { + s := fixtures.NewServer(t) + installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) + defer installer.db.Close() + + err := instFactory(installer)(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) + assert.NoError(t, err) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) + state, err := r.GetState() + assert.NoError(t, err) + assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) + assert.False(t, state.HasExperiment()) + fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV1), r.StableFS()) + fixtures.AssertEqualFS(t, s.ConfigFS(fixtures.FixtureSimpleV1), installer.ConfigFS(fixtures.FixtureSimpleV1)) + }) } func TestInstallExperiment(t *testing.T) { - s := fixtures.NewServer(t) - installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) - defer installer.db.Close() - - err := installer.Install(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) - assert.NoError(t, err) - err = installer.InstallExperiment(testCtx, s.PackageURL(fixtures.FixtureSimpleV2)) - assert.NoError(t, err) - r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) - state, err := r.GetState() - assert.NoError(t, err) - assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) - assert.Equal(t, fixtures.FixtureSimpleV2.Version, state.Experiment) - fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV1), r.StableFS()) - fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV2), r.ExperimentFS()) - fixtures.AssertEqualFS(t, s.ConfigFS(fixtures.FixtureSimpleV2), installer.ConfigFS(fixtures.FixtureSimpleV2)) + doTestInstallers(t, func(instFactory installFnFactory, t *testing.T) { + s := fixtures.NewServer(t) + installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) + defer installer.db.Close() + + err := instFactory(installer)(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) + assert.NoError(t, err) + err = installer.InstallExperiment(testCtx, s.PackageURL(fixtures.FixtureSimpleV2)) + assert.NoError(t, err) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) + state, err := r.GetState() + assert.NoError(t, err) + assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) + assert.Equal(t, fixtures.FixtureSimpleV2.Version, state.Experiment) + fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV1), r.StableFS()) + fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV2), r.ExperimentFS()) + fixtures.AssertEqualFS(t, s.ConfigFS(fixtures.FixtureSimpleV2), installer.ConfigFS(fixtures.FixtureSimpleV2)) + }) } func TestInstallPromoteExperiment(t *testing.T) { - s := fixtures.NewServer(t) - installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) - defer installer.db.Close() - - err := installer.Install(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) - assert.NoError(t, err) - err = installer.InstallExperiment(testCtx, s.PackageURL(fixtures.FixtureSimpleV2)) - assert.NoError(t, err) - err = installer.PromoteExperiment(testCtx, fixtures.FixtureSimpleV1.Package) - assert.NoError(t, err) - r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) - state, err := r.GetState() - assert.NoError(t, err) - assert.Equal(t, fixtures.FixtureSimpleV2.Version, state.Stable) - assert.False(t, state.HasExperiment()) - fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV2), r.StableFS()) - fixtures.AssertEqualFS(t, s.ConfigFS(fixtures.FixtureSimpleV2), installer.ConfigFS(fixtures.FixtureSimpleV2)) + doTestInstallers(t, func(instFactory installFnFactory, t *testing.T) { + s := fixtures.NewServer(t) + installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) + defer installer.db.Close() + + err := instFactory(installer)(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) + assert.NoError(t, err) + err = installer.InstallExperiment(testCtx, s.PackageURL(fixtures.FixtureSimpleV2)) + assert.NoError(t, err) + err = installer.PromoteExperiment(testCtx, fixtures.FixtureSimpleV1.Package) + assert.NoError(t, err) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) + state, err := r.GetState() + assert.NoError(t, err) + assert.Equal(t, fixtures.FixtureSimpleV2.Version, state.Stable) + assert.False(t, state.HasExperiment()) + fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV2), r.StableFS()) + fixtures.AssertEqualFS(t, s.ConfigFS(fixtures.FixtureSimpleV2), installer.ConfigFS(fixtures.FixtureSimpleV2)) + }) } func TestUninstallExperiment(t *testing.T) { - s := fixtures.NewServer(t) - installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) - defer installer.db.Close() - - err := installer.Install(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) - assert.NoError(t, err) - err = installer.InstallExperiment(testCtx, s.PackageURL(fixtures.FixtureSimpleV2)) - assert.NoError(t, err) - err = installer.RemoveExperiment(testCtx, fixtures.FixtureSimpleV1.Package) - assert.NoError(t, err) - r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) - state, err := r.GetState() - assert.NoError(t, err) - assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) - assert.False(t, state.HasExperiment()) - fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV1), r.StableFS()) - // we do not rollback configuration examples to their previous versions currently - fixtures.AssertEqualFS(t, s.ConfigFS(fixtures.FixtureSimpleV2), installer.ConfigFS(fixtures.FixtureSimpleV2)) + doTestInstallers(t, func(instFactory installFnFactory, t *testing.T) { + s := fixtures.NewServer(t) + installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) + defer installer.db.Close() + + err := instFactory(installer)(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) + assert.NoError(t, err) + err = installer.InstallExperiment(testCtx, s.PackageURL(fixtures.FixtureSimpleV2)) + assert.NoError(t, err) + err = installer.RemoveExperiment(testCtx, fixtures.FixtureSimpleV1.Package) + assert.NoError(t, err) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) + state, err := r.GetState() + assert.NoError(t, err) + assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) + assert.False(t, state.HasExperiment()) + fixtures.AssertEqualFS(t, s.PackageFS(fixtures.FixtureSimpleV1), r.StableFS()) + // we do not rollback configuration examples to their previous versions currently + fixtures.AssertEqualFS(t, s.ConfigFS(fixtures.FixtureSimpleV2), installer.ConfigFS(fixtures.FixtureSimpleV2)) + }) } func TestInstallSkippedWhenAlreadyInstalled(t *testing.T) { @@ -146,7 +159,7 @@ func TestInstallSkippedWhenAlreadyInstalled(t *testing.T) { assert.Equal(t, lastModTime, newLastModTime) } -func TestReinstallAfterDBClean(t *testing.T) { +func TestForceInstallWhenAlreadyInstalled(t *testing.T) { s := fixtures.NewServer(t) installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) defer installer.db.Close() @@ -157,9 +170,7 @@ func TestReinstallAfterDBClean(t *testing.T) { lastModTime, err := latestModTimeFS(r.StableFS(), ".") assert.NoError(t, err) - installer.db.DeletePackage(fixtures.FixtureSimpleV1.Package) - - err = installer.Install(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) + err = installer.ForceInstall(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) assert.NoError(t, err) r = installer.packages.Get(fixtures.FixtureSimpleV1.Package) newLastModTime, err := latestModTimeFS(r.StableFS(), ".") @@ -167,6 +178,29 @@ func TestReinstallAfterDBClean(t *testing.T) { assert.NotEqual(t, lastModTime, newLastModTime) } +func TestReinstallAfterDBClean(t *testing.T) { + doTestInstallers(t, func(instFactory installFnFactory, t *testing.T) { + s := fixtures.NewServer(t) + installer := newTestPackageManager(t, s, t.TempDir(), t.TempDir()) + defer installer.db.Close() + + err := instFactory(installer)(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) + assert.NoError(t, err) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) + lastModTime, err := latestModTimeFS(r.StableFS(), ".") + assert.NoError(t, err) + + installer.db.DeletePackage(fixtures.FixtureSimpleV1.Package) + + err = instFactory(installer)(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) + assert.NoError(t, err) + r = installer.packages.Get(fixtures.FixtureSimpleV1.Package) + newLastModTime, err := latestModTimeFS(r.StableFS(), ".") + assert.NoError(t, err) + assert.NotEqual(t, lastModTime, newLastModTime) + }) +} + func latestModTimeFS(fsys fs.FS, dirPath string) (time.Time, error) { var latestTime time.Time @@ -208,20 +242,39 @@ func latestModTimeFS(fsys fs.FS, dirPath string) (time.Time, error) { } func TestPurge(t *testing.T) { - s := fixtures.NewServer(t) - rootPath := t.TempDir() - installer := newTestPackageManager(t, s, rootPath, t.TempDir()) - - err := installer.Install(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) - assert.NoError(t, err) - r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) - - state, err := r.GetState() - assert.NoError(t, err) - assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) + doTestInstallers(t, func(instFactory installFnFactory, t *testing.T) { + s := fixtures.NewServer(t) + rootPath := t.TempDir() + installer := newTestPackageManager(t, s, rootPath, t.TempDir()) + + err := instFactory(installer)(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) + assert.NoError(t, err) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) + + state, err := r.GetState() + assert.NoError(t, err) + assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) + + installer.Purge(testCtx) + assert.NoFileExists(t, filepath.Join(rootPath, "packages.db"), "purge should remove the packages database") + assert.NoDirExists(t, rootPath, "purge should remove the packages directory") + assert.Nil(t, installer.db, "purge should close the packages database") + }) +} - installer.Purge(testCtx) - assert.NoFileExists(t, filepath.Join(rootPath, "packages.db"), "purge should remove the packages database") - assert.NoDirExists(t, rootPath, "purge should remove the packages directory") - assert.Nil(t, installer.db, "purge should close the packages database") +func doTestInstallers(t *testing.T, testFunc func(installFnFactory, *testing.T)) { + t.Helper() + installers := []installFnFactory{ + func(manager *testPackageManager) installFn { + return manager.Install + }, + func(manager *testPackageManager) installFn { + return manager.ForceInstall + }, + } + for _, inst := range installers { + t.Run(runtime.FuncForPC(reflect.ValueOf(inst).Pointer()).Name(), func(t *testing.T) { + testFunc(inst, t) + }) + } } diff --git a/pkg/fleet/installer/setup/setup.go b/pkg/fleet/installer/setup/setup.go index 4a356182137bdd..cc305c3942214d 100644 --- a/pkg/fleet/installer/setup/setup.go +++ b/pkg/fleet/installer/setup/setup.go @@ -56,7 +56,7 @@ func Agent7InstallScript(ctx context.Context, env *env.Env) error { return fmt.Errorf("failed to get default packages: %w", err) } for _, url := range defaultPackages { - err = cmd.Install(ctx, url, nil) + err = cmd.ForceInstall(ctx, url, nil) if err != nil { return fmt.Errorf("failed to install package %s: %w", url, err) } diff --git a/pkg/fleet/internal/bootstrap/bootstrap_windows.go b/pkg/fleet/internal/bootstrap/bootstrap_windows.go index 7172d02a1b6a96..aacd15bd40e16d 100644 --- a/pkg/fleet/internal/bootstrap/bootstrap_windows.go +++ b/pkg/fleet/internal/bootstrap/bootstrap_windows.go @@ -43,7 +43,7 @@ func install(ctx context.Context, env *env.Env, url string, experiment bool) err if experiment { return cmd.InstallExperiment(ctx, url) } - return cmd.Install(ctx, url, nil) + return cmd.ForceInstall(ctx, url, nil) } // downloadInstaller downloads the installer package from the registry and returns the path to the executable. diff --git a/pkg/fleet/internal/exec/installer_exec.go b/pkg/fleet/internal/exec/installer_exec.go index 87f8cce841f300..0eeafa2de558bd 100644 --- a/pkg/fleet/internal/exec/installer_exec.go +++ b/pkg/fleet/internal/exec/installer_exec.go @@ -10,14 +10,13 @@ import ( "bytes" "context" "fmt" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" + "github.com/DataDog/datadog-agent/pkg/util/log" "os" "os/exec" "runtime" "strings" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" - "github.com/DataDog/datadog-agent/pkg/util/log" - "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" @@ -69,8 +68,23 @@ func (i *InstallerExec) newInstallerCmd(ctx context.Context, command string, arg } // Install installs a package. -func (i *InstallerExec) Install(ctx context.Context, url string, _ []string) (err error) { - cmd := i.newInstallerCmd(ctx, "install", url) +func (i *InstallerExec) Install(ctx context.Context, url string, args []string) (err error) { + var cmdLineArgs = []string{url} + if len(args) > 0 { + cmdLineArgs = append(cmdLineArgs, "--install_args", strings.Join(args, ",")) + } + cmd := i.newInstallerCmd(ctx, "install", cmdLineArgs...) + defer func() { cmd.span.Finish(err) }() + return cmd.Run() +} + +// ForceInstall installs a package, even if it's already installed. +func (i *InstallerExec) ForceInstall(ctx context.Context, url string, args []string) (err error) { + var cmdLineArgs = []string{url, "--force"} + if len(args) > 0 { + cmdLineArgs = append(cmdLineArgs, "--install_args", strings.Join(args, ",")) + } + cmd := i.newInstallerCmd(ctx, "install", cmdLineArgs...) defer func() { cmd.span.Finish(err) }() return cmd.Run() } diff --git a/pkg/fleet/internal/msi/msiexec.go b/pkg/fleet/internal/msi/msiexec.go index f2ca387fb3aa2e..bfea8f7428002d 100644 --- a/pkg/fleet/internal/msi/msiexec.go +++ b/pkg/fleet/internal/msi/msiexec.go @@ -18,6 +18,8 @@ import ( "path" "path/filepath" "regexp" + "strings" + "syscall" ) type msiexecArgs struct { @@ -47,6 +49,14 @@ func Install() MsiexecOption { } } +// AdministrativeInstall specifies that msiexec will be invoked to extract the product +func AdministrativeInstall() MsiexecOption { + return func(a *msiexecArgs) error { + a.msiAction = "/a" + return nil + } +} + // Uninstall specifies that msiexec will be invoked to uninstall a product func Uninstall() MsiexecOption { return func(a *msiexecArgs) error { @@ -277,12 +287,19 @@ func Cmd(options ...MsiexecOption) (*Msiexec, error) { _ = os.RemoveAll(tempDir) }) } - args := append(a.additionalArgs, a.msiAction, a.target, "/qn", "MSIFASTINSTALL=7", "/log", a.logFile) if a.ddagentUserName != "" { - args = append(args, fmt.Sprintf("DDAGENTUSER_NAME=%s", a.ddagentUserName)) + a.additionalArgs = append(a.additionalArgs, fmt.Sprintf("DDAGENTUSER_NAME=%s", a.ddagentUserName)) + } + if a.msiAction == "/i" { + a.additionalArgs = append(a.additionalArgs, "MSIFASTINSTALL=7") } - cmd.Cmd = exec.Command("msiexec", args...) + // Do NOT pass the args to msiexec in exec.Command as it will apply some quoting algorithm (CommandLineToArgvW) that is + // incompatible with msiexec. It will make arguments like `TARGETDIR` fail because they will be quoted. + // Instead, we use the SysProcAttr.CmdLine option and do the quoting ourselves. + args := append([]string{`"C:\Windows\system32\msiexec.exe"`, a.msiAction, fmt.Sprintf(`"%s"`, a.target), "/qn", "/log", fmt.Sprintf(`"%s"`, a.logFile)}, a.additionalArgs...) + cmd.Cmd = exec.Command("msiexec") + cmd.SysProcAttr = &syscall.SysProcAttr{CmdLine: strings.Join(args, " ")} cmd.logFile = a.logFile return cmd, nil diff --git a/test/new-e2e/tests/installer/windows/datadog_installer.go b/test/new-e2e/tests/installer/windows/datadog_installer.go index 244a826fd6ae4a..0aa4be9c9f00ba 100644 --- a/test/new-e2e/tests/installer/windows/datadog_installer.go +++ b/test/new-e2e/tests/installer/windows/datadog_installer.go @@ -153,10 +153,10 @@ func (d *DatadogInstaller) RunInstallScript(extraEnvVars map[string]string) (str envVars[k] = v } envVars["DD_INSTALLER_URL"] = artifactURL - // TODO: Use install script from pipeline - cmd := `Set-ExecutionPolicy Bypass -Scope Process -Force; + + cmd := fmt.Sprintf(`Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; - iex ((New-Object System.Net.WebClient).DownloadString('https://s3.amazonaws.com/dd-agent-mstesting/Install-Datadog.ps1'));` + iex ((New-Object System.Net.WebClient).DownloadString('https://installtesting.datad0g.com/pipeline-%s/scripts/Install-Datadog.ps1'))`, d.env.Environment.PipelineID()) return d.env.RemoteHost.Execute(cmd, client.WithEnvVariables(envVars)) } diff --git a/test/new-e2e/tests/installer/windows/remote-host-assertions/remote_windows_host_asserts.go b/test/new-e2e/tests/installer/windows/remote-host-assertions/remote_windows_host_asserts.go index ea3fa8fec3f884..5c3c1a1171b1aa 100644 --- a/test/new-e2e/tests/installer/windows/remote-host-assertions/remote_windows_host_asserts.go +++ b/test/new-e2e/tests/installer/windows/remote-host-assertions/remote_windows_host_asserts.go @@ -181,3 +181,31 @@ func (r *RemoteWindowsHostAssertions) HasNoNamedPipe(pipeName string) *RemoteWin return r } + +// HasARunningDatadogInstallerService verifies that the Datadog Installer service is installed and correctly configured. +func (r *RemoteWindowsHostAssertions) HasARunningDatadogInstallerService() *RemoteWindowsHostAssertions { + r.suite.T().Helper() + + r.HasAService("Datadog Installer"). + WithStatus("Running"). + HasNamedPipe(`\\.\pipe\dd_installer`). + WithSecurity( + // Only accessible to Administrators and LocalSystem + common.NewProtectedSecurityInfo( + common.GetIdentityForSID(common.AdministratorsSID), + common.GetIdentityForSID(common.LocalSystemSID), + []common.AccessRule{ + common.NewExplicitAccessRule( + common.GetIdentityForSID(common.LocalSystemSID), + common.FileFullControl, + common.AccessControlTypeAllow, + ), + common.NewExplicitAccessRule( + common.GetIdentityForSID(common.AdministratorsSID), + common.FileFullControl, + common.AccessControlTypeAllow, + ), + }, + )) + return r +} diff --git a/test/new-e2e/tests/installer/windows/suites/install-script/install_test.go b/test/new-e2e/tests/installer/windows/suites/install-script/install_test.go new file mode 100644 index 00000000000000..153f2a0ad94ef7 --- /dev/null +++ b/test/new-e2e/tests/installer/windows/suites/install-script/install_test.go @@ -0,0 +1,79 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package agenttests + +import ( + "fmt" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" + installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" + "testing" +) + +type testInstallScriptSuite struct { + installerwindows.BaseInstallerSuite +} + +// TestAgentInstalls tests the usage of the Datadog installer to install the Datadog Agent package. +func TestInstallScript(t *testing.T) { + e2e.Run(t, &testInstallScriptSuite{}, + e2e.WithProvisioner( + winawshost.ProvisionerNoAgentNoFakeIntake(), + ), + ) +} + +// TestInstallAgentPackage tests installing and uninstalling the Datadog Agent using the Datadog installer. +func (s *testInstallScriptSuite) TestInstallAgentPackage() { + s.Run("Fresh install", func() { + s.InstallLastStable() + s.Run("Install different Agent version", func() { + s.UpgradeToLatestExperiment() + s.Run("Reinstall last stable", func() { + s.InstallLastStable() + }) + }) + }) +} + +func (s *testInstallScriptSuite) InstallLastStable() { + // Arrange + + // Act + output, err := s.Installer().RunInstallScript(map[string]string{ + "DD_INSTALLER_DEFAULT_PKG_VERSION_DATADOG_AGENT": s.StableAgentVersion().PackageVersion(), + "DD_INSTALLER_REGISTRY_URL_AGENT_PACKAGE": "install.datadoghq.com", + }) + + // Assert + if s.NoError(err) { + fmt.Printf("%s\n", output) + } + s.Require().NoErrorf(err, "failed to install the Datadog Agent package: %s", output) + s.Require().Host(s.Env().RemoteHost). + HasARunningDatadogInstallerService(). + HasARunningDatadogAgentService(). + WithVersionMatchPredicate(func(version string) { + s.Require().Contains(version, s.StableAgentVersion().Version()) + }) +} + +func (s *testInstallScriptSuite) UpgradeToLatestExperiment() { + // Act + output, err := s.Installer().InstallExperiment("datadog-agent") + + // Assert + if s.NoError(err) { + fmt.Printf("%s\n", output) + } + s.Require().NoErrorf(err, "failed to install the Datadog Agent package: %s", output) + s.Require().Host(s.Env().RemoteHost). + HasARunningDatadogInstallerService(). + HasARunningDatadogAgentService(). + WithVersionMatchPredicate(func(version string) { + s.Require().Contains(version, s.CurrentAgentVersion().GetNumberAndPre()) + }) +} diff --git a/tools/windows/DatadogAgentInstallScript/Install-Datadog.ps1 b/tools/windows/DatadogAgentInstallScript/Install-Datadog.ps1 index 8724a7800831b2..7c4c1ed62b4df4 100644 --- a/tools/windows/DatadogAgentInstallScript/Install-Datadog.ps1 +++ b/tools/windows/DatadogAgentInstallScript/Install-Datadog.ps1 @@ -273,9 +273,11 @@ try { } catch [ExitCodeException] { Show-Error $_.Exception.Message $_.Exception.LastExitCode + Exit $_.Exception.LastExitCode } catch { Show-Error $_.Exception.Message $GENERAL_ERROR_CODE + Exit $GENERAL_ERROR_CODE } finally { Write-Host "Cleaning up..."