Add Elastic Agent

scripts/modules/cli.py

@@ -22,7 +22,7 @@
     Packetbeat, Metricbeat, Heartbeat, Filebeat
 )
 from .elastic_stack import (  # noqa: F401
-    ApmServer, Elasticsearch, EnterpriseSearch, Kibana
+    ApmServer, ElasticAgent, Elasticsearch, EnterpriseSearch, Kibana
 )
 from .aux_services import (  # noqa: F401
     Kafka, Logstash, Postgres, Redis, Zookeeper, WaitService

scripts/modules/elastic_stack.py

@@ -6,7 +6,7 @@
 import json
 import os
 
-from .helpers import curl_healthcheck, try_to_set_slowlog
+from .helpers import curl_healthcheck, try_to_set_slowlog, urlparse
 from .service import StackService, Service, DEFAULT_APM_SERVER_URL
 
 
@@ -587,6 +587,69 @@ def render_tee(self):
         return {self.name(): content}
 
 
+class ElasticAgent(StackService, Service):
+    docker_path = "beats"
+
+    def __init__(self, **options):
+        super(ElasticAgent, self).__init__(**options)
+        if not self.at_least_version("7.8"):
+            raise Exception("Elastic Agent is only available in 7.8+")
+
+        # build deps
+        self.depends_on = {"kibana": {"condition": "service_healthy"}} if options.get("enable_kibana", True) else {}
+
+        # build environment
+        # Environment variables consumed by the Elastic Agent entrypoint
+        # https://github.com/elastic/beats/blob/4f4a5536b72f4a25962d56262f31e3b8533b252e/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl
+        # FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll
+        # FLEET_ENROLL - if set to 1 enroll will be performed
+        # FLEET_ENROLL_INSECURE - if set to 1, agent will enroll with fleet using --insecure flag
+        # FLEET_SETUP - if  set to 1 fleet setup will be performed
+        # FLEET_TOKEN_NAME - token name for a token to be created
+        # KIBANA_HOST - actual kibana host [http://localhost:5601]
+        # KIBANA_PASSWORD - password for accessing kibana API [changeme]
+        # KIBANA_USERNAME - username for accessing kibana API [elastic]
+        kibana_url = options.get("elastic_agent_kibana_url")
+        if not kibana_url:
+            kibana_scheme = "https" if self.options.get("kibana_enable_tls", False) else "http"
+            # TODO(gr): add default elastic-agent user
+            kibana_url = kibana_scheme + "://admin:changeme@" + self.DEFAULT_KIBANA_HOST
+
+        kibana_parsed_url = urlparse(kibana_url)
+        self.environment = {
+            "FLEET_ENROLL": "1",
+            "FLEET_SETUP": "1",
+            "KIBANA_HOST": kibana_url,
+        }
+        if kibana_parsed_url.password:
+            self.environment["KIBANA_PASSWORD"] = kibana_parsed_url.password
+        if kibana_parsed_url.username:
+            self.environment["KIBANA_USERNAME"] = kibana_parsed_url.username
+        if not kibana_url.startswith("https://"):
+            self.environment["FLEET_ENROLL_INSECURE"] = 1
+
+    def _content(self):
+        return dict(
+            depends_on=self.depends_on,
+            environment=self.environment,
+            healthcheck={
+                "test": ["CMD", "/bin/true"],
+            },
+            volumes=[
+                "/var/run/docker.sock:/var/run/docker.sock",
+            ]
+        )
+
+    @classmethod
+    def add_arguments(cls, parser):
+        super(ElasticAgent, cls).add_arguments(parser)
+        parser.add_argument(
+            "--elastic-agent-kibana-url",
+            default="http://admin:changeme@" + cls.DEFAULT_KIBANA_HOST,
+            help="Elastic Agent's Kibana URL, including username:password"
+        )
+
+
 class Elasticsearch(StackService, Service):
     default_environment = [
         "bootstrap.memory_lock=true",
@@ -859,6 +922,10 @@ def __init__(self, **options):
             if self.at_least_version("7.7"):
                 self.environment["XPACK_SECURITY_ENCRYPTIONKEY"] = "fhjskloppd678ehkdfdlliverpoolfcr"
                 self.environment["XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY"] = "fhjskloppd678ehkdfdlliverpoolfcr"
+            if self.at_least_version("7.8"):
+                self.environment["XPACK_FLEET_AGENTS_ELASTICSEARCH_HOST"] = urls[0]
+                self.environment["XPACK_FLEET_AGENTS_KIBANA_HOST"] = "{}://kibana:{}".format(
+                    "https" if self.kibana_tls else "http", self.SERVICE_PORT)
             if options.get("xpack_secure"):
                 self.environment["ELASTICSEARCH_PASSWORD"] = "changeme"
                 self.environment["ELASTICSEARCH_USERNAME"] = "kibana_system_user"

scripts/modules/helpers.py

@@ -15,9 +15,12 @@
 
 try:
     from urllib.request import urlopen, urlretrieve, Request
+    from urllib.parse import urlparse
 except ImportError:
     from urllib import urlretrieve
     from urllib2 import urlopen, Request
+    import urllib2
+    urlparse = urllib2.urlparse.urlparse
 
 
 def _camel_hyphen(string):

scripts/tests/localsetup_tests.py

@@ -964,6 +964,8 @@ def test_start_master_default(self):
                     XPACK_MONITORING_ENABLED: 'true',
                     XPACK_SECURITY_ENCRYPTIONKEY: 'fhjskloppd678ehkdfdlliverpoolfcr',
                     XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'fhjskloppd678ehkdfdlliverpoolfcr',
+                    XPACK_FLEET_AGENTS_ELASTICSEARCH_HOST: 'http://elasticsearch:9200',
+                    XPACK_FLEET_AGENTS_KIBANA_HOST: 'http://kibana:5601',
                     XPACK_FLEET_AGENTS_TLSCHECKDISABLED: 'true',
                     XPACK_XPACK_MAIN_TELEMETRY_ENABLED: 'false',
                     XPACK_SECURITY_LOGINASSISTANCEMESSAGE: 'Login details: `admin/changeme`. Further details [here](https://github.com/elastic/apm-integration-testing#logging-in).'

scripts/tests/service_tests.py

@@ -12,7 +12,7 @@
 )
 from ..modules.aux_services import Logstash, Kafka, Zookeeper
 from ..modules.beats import Filebeat, Heartbeat, Metricbeat, Packetbeat
-from ..modules.elastic_stack import ApmServer, Elasticsearch, EnterpriseSearch, Kibana
+from ..modules.elastic_stack import ApmServer, ElasticAgent, Elasticsearch, EnterpriseSearch, Kibana
 
 
 class ServiceTest(unittest.TestCase):
@@ -887,6 +887,50 @@ def test_debug(self):
         self.assertTrue("*" in apm_server["command"])
 
 
+class ElasticAgentServiceTest(ServiceTest):
+    def test_default(self):
+        ea = ElasticAgent(version="7.12.345").render()["elastic-agent"]
+        self.assertEqual(
+            ea, {"container_name": "localtesting_7.12.345_elastic-agent",
+                 "depends_on": {"kibana": {"condition": "service_healthy"}},
+                 "environment": {"FLEET_ENROLL": "1",
+                                 "FLEET_ENROLL_INSECURE": 1,
+                                 "FLEET_SETUP": "1",
+                                 "KIBANA_HOST": "http://admin:changeme@kibana:5601",
+                                 "KIBANA_PASSWORD": "changeme",
+                                 "KIBANA_USERNAME": "admin"},
+                 "healthcheck": {"test": ["CMD", "/bin/true"]},
+                 "image": "docker.elastic.co/beats/elastic-agent:7.12.345-SNAPSHOT",
+                 "labels": ["co.elastic.apm.stack-version=7.12.345"],
+                 "logging": {"driver": "json-file",
+                             "options": {"max-file": "5", "max-size": "2m"}},
+                 "volumes": ["/var/run/docker.sock:/var/run/docker.sock"]}
+        )
+
+    def test_default_snapshot(self):
+        ea = ElasticAgent(version="7.12.345", snapshot=True).render()["elastic-agent"]
+        self.assertEqual(
+            "docker.elastic.co/beats/elastic-agent:7.12.345-SNAPSHOT", ea["image"]
+        )
+
+    def test_kibana_tls(self):
+        ea = ElasticAgent(version="7.12.345", kibana_enable_tls=True).render()["elastic-agent"]
+        self.assertEqual(
+            "https://admin:changeme@kibana:5601", ea["environment"]["KIBANA_HOST"]
+        )
+
+    def test_kibana_url(self):
+        ea = ElasticAgent(version="7.12.345", elastic_agent_kibana_url="http://foo").render()["elastic-agent"]
+        self.assertEqual("http://foo", ea["environment"]["KIBANA_HOST"])
+        self.assertNotIn("KIBANA_PASSWORD", ea["environment"])
+        self.assertNotIn("KIBANA_USERNAME", ea["environment"])
+
+        ea = ElasticAgent(version="7.12.345", elastic_agent_kibana_url="http://u:p@h:123").render()["elastic-agent"]
+        self.assertEqual("http://u:p@h:123", ea["environment"]["KIBANA_HOST"])
+        self.assertEqual("p", ea["environment"]["KIBANA_PASSWORD"])
+        self.assertEqual("u", ea["environment"]["KIBANA_USERNAME"])
+
+
 class ElasticsearchServiceTest(ServiceTest):
     def test_6_2_release(self):
         elasticsearch = Elasticsearch(version="6.2.4", release=True).render()["elasticsearch"]