diff --git a/scripts/modules/cli.py b/scripts/modules/cli.py index 555b0f9ea..158e697cd 100644 --- a/scripts/modules/cli.py +++ b/scripts/modules/cli.py @@ -22,7 +22,7 @@ Packetbeat, Metricbeat, Heartbeat, Filebeat ) from .elastic_stack import ( # noqa: F401 - ApmServer, Elasticsearch, EnterpriseSearch, Kibana + ApmServer, ElasticAgent, Elasticsearch, EnterpriseSearch, Kibana ) from .aux_services import ( # noqa: F401 Kafka, Logstash, Postgres, Redis, Zookeeper, WaitService diff --git a/scripts/modules/elastic_stack.py b/scripts/modules/elastic_stack.py index f6cf76210..0d4e1d093 100644 --- a/scripts/modules/elastic_stack.py +++ b/scripts/modules/elastic_stack.py @@ -6,7 +6,7 @@ import json import os -from .helpers import curl_healthcheck, try_to_set_slowlog +from .helpers import curl_healthcheck, try_to_set_slowlog, urlparse from .service import StackService, Service, DEFAULT_APM_SERVER_URL @@ -587,6 +587,69 @@ def render_tee(self): return {self.name(): content} +class ElasticAgent(StackService, Service): + docker_path = "beats" + + def __init__(self, **options): + super(ElasticAgent, self).__init__(**options) + if not self.at_least_version("7.8"): + raise Exception("Elastic Agent is only available in 7.8+") + + # build deps + self.depends_on = {"kibana": {"condition": "service_healthy"}} if options.get("enable_kibana", True) else {} + + # build environment + # Environment variables consumed by the Elastic Agent entrypoint + # https://github.com/elastic/beats/blob/4f4a5536b72f4a25962d56262f31e3b8533b252e/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl + # FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll + # FLEET_ENROLL - if set to 1 enroll will be performed + # FLEET_ENROLL_INSECURE - if set to 1, agent will enroll with fleet using --insecure flag + # FLEET_SETUP - if set to 1 fleet setup will be performed + # FLEET_TOKEN_NAME - token name for a token to be created + # KIBANA_HOST - actual kibana host [http://localhost:5601] + # KIBANA_PASSWORD - password for accessing kibana API [changeme] + # KIBANA_USERNAME - username for accessing kibana API [elastic] + kibana_url = options.get("elastic_agent_kibana_url") + if not kibana_url: + kibana_scheme = "https" if self.options.get("kibana_enable_tls", False) else "http" + # TODO(gr): add default elastic-agent user + kibana_url = kibana_scheme + "://admin:changeme@" + self.DEFAULT_KIBANA_HOST + + kibana_parsed_url = urlparse(kibana_url) + self.environment = { + "FLEET_ENROLL": "1", + "FLEET_SETUP": "1", + "KIBANA_HOST": kibana_url, + } + if kibana_parsed_url.password: + self.environment["KIBANA_PASSWORD"] = kibana_parsed_url.password + if kibana_parsed_url.username: + self.environment["KIBANA_USERNAME"] = kibana_parsed_url.username + if not kibana_url.startswith("https://"): + self.environment["FLEET_ENROLL_INSECURE"] = 1 + + def _content(self): + return dict( + depends_on=self.depends_on, + environment=self.environment, + healthcheck={ + "test": ["CMD", "/bin/true"], + }, + volumes=[ + "/var/run/docker.sock:/var/run/docker.sock", + ] + ) + + @classmethod + def add_arguments(cls, parser): + super(ElasticAgent, cls).add_arguments(parser) + parser.add_argument( + "--elastic-agent-kibana-url", + default="http://admin:changeme@" + cls.DEFAULT_KIBANA_HOST, + help="Elastic Agent's Kibana URL, including username:password" + ) + + class Elasticsearch(StackService, Service): default_environment = [ "bootstrap.memory_lock=true", @@ -859,6 +922,10 @@ def __init__(self, **options): if self.at_least_version("7.7"): self.environment["XPACK_SECURITY_ENCRYPTIONKEY"] = "fhjskloppd678ehkdfdlliverpoolfcr" self.environment["XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY"] = "fhjskloppd678ehkdfdlliverpoolfcr" + if self.at_least_version("7.8"): + self.environment["XPACK_FLEET_AGENTS_ELASTICSEARCH_HOST"] = urls[0] + self.environment["XPACK_FLEET_AGENTS_KIBANA_HOST"] = "{}://kibana:{}".format( + "https" if self.kibana_tls else "http", self.SERVICE_PORT) if options.get("xpack_secure"): self.environment["ELASTICSEARCH_PASSWORD"] = "changeme" self.environment["ELASTICSEARCH_USERNAME"] = "kibana_system_user" diff --git a/scripts/modules/helpers.py b/scripts/modules/helpers.py index 79daf0380..fb8c3227c 100644 --- a/scripts/modules/helpers.py +++ b/scripts/modules/helpers.py @@ -15,9 +15,12 @@ try: from urllib.request import urlopen, urlretrieve, Request + from urllib.parse import urlparse except ImportError: from urllib import urlretrieve from urllib2 import urlopen, Request + import urllib2 + urlparse = urllib2.urlparse.urlparse def _camel_hyphen(string): diff --git a/scripts/tests/localsetup_tests.py b/scripts/tests/localsetup_tests.py index da8c516cc..97f763746 100644 --- a/scripts/tests/localsetup_tests.py +++ b/scripts/tests/localsetup_tests.py @@ -964,6 +964,8 @@ def test_start_master_default(self): XPACK_MONITORING_ENABLED: 'true', XPACK_SECURITY_ENCRYPTIONKEY: 'fhjskloppd678ehkdfdlliverpoolfcr', XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'fhjskloppd678ehkdfdlliverpoolfcr', + XPACK_FLEET_AGENTS_ELASTICSEARCH_HOST: 'http://elasticsearch:9200', + XPACK_FLEET_AGENTS_KIBANA_HOST: 'http://kibana:5601', XPACK_FLEET_AGENTS_TLSCHECKDISABLED: 'true', XPACK_XPACK_MAIN_TELEMETRY_ENABLED: 'false', XPACK_SECURITY_LOGINASSISTANCEMESSAGE: 'Login details: `admin/changeme`. Further details [here](https://github.com/elastic/apm-integration-testing#logging-in).' diff --git a/scripts/tests/service_tests.py b/scripts/tests/service_tests.py index 1ed0b57a4..4c226df93 100644 --- a/scripts/tests/service_tests.py +++ b/scripts/tests/service_tests.py @@ -12,7 +12,7 @@ ) from ..modules.aux_services import Logstash, Kafka, Zookeeper from ..modules.beats import Filebeat, Heartbeat, Metricbeat, Packetbeat -from ..modules.elastic_stack import ApmServer, Elasticsearch, EnterpriseSearch, Kibana +from ..modules.elastic_stack import ApmServer, ElasticAgent, Elasticsearch, EnterpriseSearch, Kibana class ServiceTest(unittest.TestCase): @@ -887,6 +887,50 @@ def test_debug(self): self.assertTrue("*" in apm_server["command"]) +class ElasticAgentServiceTest(ServiceTest): + def test_default(self): + ea = ElasticAgent(version="7.12.345").render()["elastic-agent"] + self.assertEqual( + ea, {"container_name": "localtesting_7.12.345_elastic-agent", + "depends_on": {"kibana": {"condition": "service_healthy"}}, + "environment": {"FLEET_ENROLL": "1", + "FLEET_ENROLL_INSECURE": 1, + "FLEET_SETUP": "1", + "KIBANA_HOST": "http://admin:changeme@kibana:5601", + "KIBANA_PASSWORD": "changeme", + "KIBANA_USERNAME": "admin"}, + "healthcheck": {"test": ["CMD", "/bin/true"]}, + "image": "docker.elastic.co/beats/elastic-agent:7.12.345-SNAPSHOT", + "labels": ["co.elastic.apm.stack-version=7.12.345"], + "logging": {"driver": "json-file", + "options": {"max-file": "5", "max-size": "2m"}}, + "volumes": ["/var/run/docker.sock:/var/run/docker.sock"]} + ) + + def test_default_snapshot(self): + ea = ElasticAgent(version="7.12.345", snapshot=True).render()["elastic-agent"] + self.assertEqual( + "docker.elastic.co/beats/elastic-agent:7.12.345-SNAPSHOT", ea["image"] + ) + + def test_kibana_tls(self): + ea = ElasticAgent(version="7.12.345", kibana_enable_tls=True).render()["elastic-agent"] + self.assertEqual( + "https://admin:changeme@kibana:5601", ea["environment"]["KIBANA_HOST"] + ) + + def test_kibana_url(self): + ea = ElasticAgent(version="7.12.345", elastic_agent_kibana_url="http://foo").render()["elastic-agent"] + self.assertEqual("http://foo", ea["environment"]["KIBANA_HOST"]) + self.assertNotIn("KIBANA_PASSWORD", ea["environment"]) + self.assertNotIn("KIBANA_USERNAME", ea["environment"]) + + ea = ElasticAgent(version="7.12.345", elastic_agent_kibana_url="http://u:p@h:123").render()["elastic-agent"] + self.assertEqual("http://u:p@h:123", ea["environment"]["KIBANA_HOST"]) + self.assertEqual("p", ea["environment"]["KIBANA_PASSWORD"]) + self.assertEqual("u", ea["environment"]["KIBANA_USERNAME"]) + + class ElasticsearchServiceTest(ServiceTest): def test_6_2_release(self): elasticsearch = Elasticsearch(version="6.2.4", release=True).render()["elasticsearch"]