Merge branch 'master' into 1191-support-running-features

e2e/_suites/fleet/stand-alone.go

@@ -51,7 +51,10 @@ func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error {
 	maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2
 	minimumHitsCount := 50
 
-	result, err := searchAgentData(fts.Hostname, fts.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout)
+	agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(fts.Image)
+
+	manifest, _ := fts.deployer.Inspect(agentService)
+	result, err := searchAgentData(manifest.Hostname, fts.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout)
 	if err != nil {
 		return err
 	}
@@ -62,11 +65,8 @@ func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error {
 }
 
 func (fts *FleetTestSuite) theDockerContainerIsStopped(serviceName string) error {
-	services := []deploy.ServiceRequest{
-		deploy.NewServiceRequest(common.FleetProfileName),
-		deploy.NewServiceRequest(serviceName),
-	}
-	err := fts.deployer.Remove(services, common.ProfileEnv)
+	agentService := deploy.NewServiceRequest(serviceName)
+	err := fts.deployer.Stop(agentService)
 	if err != nil {
 		return err
 	}
@@ -79,7 +79,9 @@ func (fts *FleetTestSuite) thereIsNoNewDataInTheIndexAfterAgentShutsDown() error
 	maxTimeout := time.Duration(30) * time.Second
 	minimumHitsCount := 1
 
-	result, err := searchAgentData(fts.Hostname, fts.AgentStoppedDate, minimumHitsCount, maxTimeout)
+	agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName)
+	manifest, _ := fts.deployer.Inspect(agentService)
+	result, err := searchAgentData(manifest.Hostname, fts.AgentStoppedDate, minimumHitsCount, maxTimeout)
 	if err != nil {
 		if strings.Contains(err.Error(), "type:index_not_found_exception") {
 			return err
@@ -140,14 +142,7 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, en
 		return err
 	}
 
-	// get container hostname once
-	hostname, err := deploy.GetContainerHostname(containerName)
-	if err != nil {
-		return err
-	}
-
 	fts.Image = image
-	fts.Hostname = hostname
 
 	err = fts.installTestTools(containerName)
 	if err != nil {
@@ -208,7 +203,7 @@ func (fts *FleetTestSuite) installTestTools(containerName string) error {
 		"containerName": containerName,
 	}).Trace("Installing test tools ")
 
-	_, err := deploy.ExecCommandIntoContainer(context.Background(), deploy.NewServiceRequest(containerName), "root", cmd)
+	_, err := deploy.ExecCommandIntoContainer(context.Background(), containerName, "root", cmd)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"command":       cmd,

e2e/_suites/fleet/world.go

@@ -23,7 +23,11 @@ type IngestManagerTestSuite struct {
 }
 
 func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state string) error {
-	return imts.thereAreInstancesOfTheProcessInTheState("1", process, state)
+	ocurrences := "1"
+	if state == "uninstalled" || state == "stopped" {
+		ocurrences = "0"
+	}
+	return imts.thereAreInstancesOfTheProcessInTheState(ocurrences, process, state)
 }
 
 func (imts *IngestManagerTestSuite) thereAreInstancesOfTheProcessInTheState(ocurrences string, process string, state string) error {
@@ -34,8 +38,9 @@ func (imts *IngestManagerTestSuite) thereAreInstancesOfTheProcessInTheState(ocur
 	if imts.Fleet.StandAlone {
 		containerName = fmt.Sprintf("%s_%s_%d", profile, common.ElasticAgentServiceName, 1)
 	} else {
-		agentInstaller := imts.Fleet.getInstaller()
-		containerName = imts.Fleet.getContainerName(agentInstaller)
+		agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName)
+		manifest, _ := imts.Fleet.deployer.Inspect(agentService)
+		containerName = manifest.Name
 	}
 
 	count, err := strconv.Atoi(ocurrences)
@@ -100,6 +105,22 @@ func waitForProcess(deployer deploy.Deployment, service string, process string,
 		cmds := []string{"pgrep", "-d", ",", process}
 		output, err := deployer.ExecIn(serviceRequest, cmds)
 		if err != nil {
+
+			if !mustBePresent && ocurrences == 0 {
+				log.WithFields(log.Fields{
+					"cmds":          cmds,
+					"desiredState":  desiredState,
+					"elapsedTime":   exp.GetElapsedTime(),
+					"error":         err,
+					"service":       service,
+					"mustBePresent": mustBePresent,
+					"ocurrences":    ocurrences,
+					"process":       process,
+					"retry":         retryCount,
+				}).Warn("Process is not present and number of occurences is 0")
+				return nil
+			}
+
 			log.WithFields(log.Fields{
 				"cmds":          cmds,
 				"desiredState":  desiredState,

internal/deploy/base.go

@@ -12,19 +12,47 @@ import (
 // required for testing
 type Deployment interface {
 	Add(services []ServiceRequest, env map[string]string) error    // adds a service to deployment
+	AddFiles(service ServiceRequest, files []string) error         // adds files to a service
 	Bootstrap(waitCB func() error) error                           // will bootstrap or reuse existing cluster if kubernetes is selected
 	Destroy() error                                                // Teardown deployment
 	ExecIn(service ServiceRequest, cmd []string) (string, error)   // Execute arbitrary commands in service
 	Inspect(service ServiceRequest) (*ServiceManifest, error)      // inspects service
+	Logs(service ServiceRequest) error                             // prints logs of deployed service
 	Remove(services []ServiceRequest, env map[string]string) error // Removes services from deployment
+	Start(service ServiceRequest) error                            // Starts a service or container depending on Deployment
+	Stop(service ServiceRequest) error                             // Stop a service or container depending on deployment
+}
+
+// ServiceOperator represents the operations that can be performed by a service
+type ServiceOperator interface {
+	AddFiles(files []string) error             // adds files to service environment
+	Enroll(token string) error                 // handle any enrollment/registering of service
+	Exec(args []string) (string, error)        // exec arbitrary commands in service environment
+	Inspect() (ServiceOperatorManifest, error) // returns manifest for package
+	Install() error
+	InstallCerts() error
+	Logs() error
+	Postinstall() error
+	Preinstall() error
+	Start() error // will start a service
+	Stop() error  // will stop a service
+	Uninstall() error
+}
+
+// ServiceOperatorManifest is state information for each service operator
+type ServiceOperatorManifest struct {
+	CommitFile string
+	WorkDir    string
 }
 
 // ServiceManifest information about a service in a deployment
 type ServiceManifest struct {
 	ID         string
 	Name       string
 	Connection string // a string representing how to connect to service
+	Alias      string // container network aliases
 	Hostname   string
+	Platform   string // running in linux, macos, windows
 }
 
 // ServiceRequest represents the service to be created using the provider

internal/deploy/docker.go

@@ -6,9 +6,11 @@ package deploy
 
 import (
 	"context"
+	"path/filepath"
 	"strings"
 
 	"github.com/elastic/e2e-testing/internal/common"
+	"github.com/elastic/e2e-testing/internal/shell"
 	"github.com/elastic/e2e-testing/internal/utils"
 	log "github.com/sirupsen/logrus"
 )
@@ -58,6 +60,23 @@ func (c *dockerDeploymentManifest) Bootstrap(waitCB func() error) error {
 	return nil
 }
 
+// AddFiles - add files to service
+func (c *dockerDeploymentManifest) AddFiles(service ServiceRequest, files []string) error {
+	container, _ := c.Inspect(service)
+	for _, file := range files {
+		isTar := true
+		fileExt := filepath.Ext(file)
+		if fileExt == ".rpm" || fileExt == ".deb" {
+			isTar = false
+		}
+		err := CopyFileToContainer(c.Context, container.Name, file, "/", isTar)
+		if err != nil {
+			log.WithField("error", err).Fatal("Unable to copy file to service")
+		}
+	}
+	return nil
+}
+
 // Destroy teardown docker environment
 func (c *dockerDeploymentManifest) Destroy() error {
 	serviceManager := NewServiceManager()
@@ -73,7 +92,12 @@ func (c *dockerDeploymentManifest) Destroy() error {
 
 // ExecIn execute command in service
 func (c *dockerDeploymentManifest) ExecIn(service ServiceRequest, cmd []string) (string, error) {
-	output, err := ExecCommandIntoContainer(c.Context, service, "root", cmd)
+	inspect, _ := c.Inspect(service)
+	args := []string{"exec", "-u", "root", "-i", inspect.Name}
+	for _, cmdArg := range cmd {
+		args = append(args, cmdArg)
+	}
+	output, err := shell.Execute(c.Context, ".", "docker", args...)
 	if err != nil {
 		return "", err
 	}
@@ -86,17 +110,54 @@ func (c *dockerDeploymentManifest) Inspect(service ServiceRequest) (*ServiceMani
 	if err != nil {
 		return &ServiceManifest{}, err
 	}
+
 	return &ServiceManifest{
 		ID:         inspect.ID,
 		Name:       strings.TrimPrefix(inspect.Name, "/"),
 		Connection: service.Name,
-		Hostname:   inspect.NetworkSettings.Networks["fleet_default"].Aliases[0],
+		Alias:      inspect.NetworkSettings.Networks["fleet_default"].Aliases[0],
+		Hostname:   inspect.Config.Hostname,
+		Platform:   inspect.Platform,
 	}, nil
 }
 
+// Logs print logs of service
+func (c *dockerDeploymentManifest) Logs(service ServiceRequest) error {
+	manifest, _ := c.Inspect(service)
+	_, err := shell.Execute(c.Context, ".", "docker", "logs", manifest.Name)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":   err,
+			"service": service.Name,
+		}).Error("Could not retrieve Elastic Agent logs")
+
+		return err
+	}
+	return nil
+}
+
 // Remove remove services from deployment
 func (c *dockerDeploymentManifest) Remove(services []ServiceRequest, env map[string]string) error {
-	serviceManager := NewServiceManager()
+	for _, service := range services[1:] {
+		manifest, _ := c.Inspect(service)
+		_, err := shell.Execute(c.Context, ".", "docker", "rm", "-fv", manifest.Name)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Start a container
+func (c *dockerDeploymentManifest) Start(service ServiceRequest) error {
+	manifest, _ := c.Inspect(service)
+	_, err := shell.Execute(c.Context, ".", "docker", "start", manifest.Name)
+	return err
+}
 
-	return serviceManager.RemoveServicesFromCompose(c.Context, services[0], services[1:], env)
+// Stop a container
+func (c *dockerDeploymentManifest) Stop(service ServiceRequest) error {
+	manifest, _ := c.Inspect(service)
+	_, err := shell.Execute(c.Context, ".", "docker", "stop", manifest.Name)
+	return err
 }

internal/deploy/docker_client.go

@@ -135,18 +135,18 @@ func CopyFileToContainer(ctx context.Context, containerName string, srcPath stri
 }
 
 // ExecCommandIntoContainer executes a command, as a user, into a container
-func ExecCommandIntoContainer(ctx context.Context, container ServiceRequest, user string, cmd []string) (string, error) {
+func ExecCommandIntoContainer(ctx context.Context, container string, user string, cmd []string) (string, error) {
 	return ExecCommandIntoContainerWithEnv(ctx, container, user, cmd, []string{})
 }
 
 // ExecCommandIntoContainerWithEnv executes a command, as a user, with env, into a container
-func ExecCommandIntoContainerWithEnv(ctx context.Context, container ServiceRequest, user string, cmd []string, env []string) (string, error) {
+func ExecCommandIntoContainerWithEnv(ctx context.Context, container string, user string, cmd []string, env []string) (string, error) {
 	dockerClient := getDockerClient()
 
 	detach := false
 	tty := false
 
-	containerName := container.Name
+	containerName := container
 
 	log.WithFields(log.Fields{
 		"container": containerName,
@@ -192,6 +192,8 @@ func ExecCommandIntoContainerWithEnv(ctx context.Context, container ServiceReque
 		Detach: detach,
 		Tty:    tty,
 	})
+	defer resp.Close()
+
 	if err != nil {
 		log.WithFields(log.Fields{
 			"container": containerName,
@@ -203,7 +205,6 @@ func ExecCommandIntoContainerWithEnv(ctx context.Context, container ServiceReque
 		}).Error("Could not execute command in container")
 		return "", err
 	}
-	defer resp.Close()
 
 	// see https://stackoverflow.com/a/57132902
 	var execRes execResult
@@ -268,7 +269,7 @@ func GetContainerHostname(containerName string) (string, error) {
 		"containerName": containerName,
 	}).Trace("Retrieving container name from the Docker client")
 
-	hostname, err := ExecCommandIntoContainer(context.Background(), NewServiceRequest(containerName), "root", []string{"cat", "/etc/hostname"})
+	hostname, err := ExecCommandIntoContainer(context.Background(), containerName, "root", []string{"cat", "/etc/hostname"})
 	if err != nil {
 		log.WithFields(log.Fields{
 			"containerName": containerName,

internal/deploy/docker_client_test.go

@@ -40,7 +40,7 @@ func Test_CopyFile(t *testing.T) {
 		err = CopyFileToContainer(ctx, containerName, src, target, false)
 		assert.Nil(t, err)
 
-		output, err := ExecCommandIntoContainer(ctx, NewServiceRequest(containerName), "root", []string{"cat", "/tmp/dockerCopy.txt"})
+		output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"cat", "/tmp/dockerCopy.txt"})
 		assert.Nil(t, err)
 		assert.True(t, strings.HasSuffix(output, "OK!"), "File contains the 'OK!' string")
 	})
@@ -76,7 +76,7 @@ func Test_CopyFile(t *testing.T) {
 		err = CopyFileToContainer(ctx, containerName, src, target, true)
 		assert.Nil(t, err)
 
-		output, err := ExecCommandIntoContainer(ctx, NewServiceRequest(containerName), "root", []string{"ls", "/project/txtr/kermit.jpg"})
+		output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"ls", "/project/txtr/kermit.jpg"})
 		assert.Nil(t, err)
 		assert.True(t, strings.Contains(output, "/project/txtr/kermit.jpg"), "File '/project/txtr/kermit.jpg' should be present")
 	})