Feature/tilt 2 (#32326)

* new dev tools under dev-tools/kubernetes to remote-debug and hot reload filebeat/metricbeat running on kubernetes

Co-authored-by: Tetiana Kravchenko <[email protected]>

dev-tools/kubernetes/.gitignore

@@ -0,0 +1 @@
+/build

dev-tools/kubernetes/README.md

@@ -0,0 +1,94 @@
+# Readme
+
+This folder container some dev tools that make it easier to develop and deploy filebeat and metricbeat running inside a Kubernetes cluster. This is especially useful when developing the metricbeat module `kubernetes` since it requires metricbeat to run inside a Kubernetes cluster in order to interact with kube-state-metrics and the Kubernetes APIs.
+
+In details, a combination of Dockerfiles, Kubernetes manifests and Tiltfile make it possible to have features like:
+- hot reloading of code running in Kubernetes, without re-applying the Kubernetes manifest
+- remote debugging (with breakpoints) both metricbeat/filebeat running as a pod in a Kind Kubernetes cluster
+
+[Tilt](https://tilt.dev/) is a dev toolkit for microservices.
+
+
+## Setup
+You can install Tilt by using the command
+
+```shell
+curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
+```
+
+
+## How to run
+The Tiltfile that orchestrates everything is located at `dev-tools/kubernetes/Tiltfile`. All the following commands need to be run from `dev-tools/kubernetes`, in the same folder where the Tiltfile is located.
+
+How to run Tilt
+
+```shell
+tilt up
+```
+
+This will open a terminal and optionally a web UI where you can interact with Tilt, see container logs and restart resources.
+
+Once you are done with Tilt, you can simply `CTRL+C` from the open Tilt terminal. The resources that you started in k8s will still be running though.
+
+If you want to remove all the k8s resources that you started with Tilt, you can run
+
+```shell
+tilt down
+```
+
+If you want to use a remote debugger with Visual Studio code, you need to provide a `.vscode/launch.json` similar to the following file. In order for this to work on your laptop, you need to replace `<absolute_path_to_beats_folder>` with the absolute path of the root folder in this project. This file is currently not under git because it depends on the user configuration, it is only useful for VisualStudio Code and in a folder usually ignored by git.
+
+```json
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Connect to server",
+            "type": "go",
+            "request": "attach",
+            "mode": "remote",
+            "debugAdapter": "dlv-dap",
+            "port": 56268,
+            "host": "127.0.0.1",
+            "showLog": true,
+            "trace": "trace",
+            "cwd": "${workspaceFolder}",
+            "substitutePath": [
+                {
+	                "from": "${workspaceFolder}",
+                    "to": "<absolute_path_to_beats_folder>"
+                }
+            ]
+        }
+
+    ]
+}
+```
+
+
+## Run vs debug mode
+Currently the Tiltfile is configured in `run` mode
+
+```python
+beat(mode="run")
+# beat(mode="debug")
+```
+
+This mode runs metricbeat like a single process in a pod.
+
+If you want to switch to `debug` mode you have to modify the Tiltfile like the following code
+
+```python
+# beat(mode="run")
+beat(mode="debug")
+```
+
+You can make this change while Tilt is running in the background. Tilt will
+stop the running container and swap it with a debug container instead.
+
+Both mode support `hot reloading`, meaning that if Tilt is running, when you make a change to
+the source code, Tilt will:
+
+1. Automatically compile the source code
+2. Live sync the new binary to the running container
+3. Restart the container to run the new binary

dev-tools/kubernetes/Tiltfile

@@ -0,0 +1,106 @@
+
+load('ext://restart_process', 'docker_build_with_restart')
+
+def beat(
+        mode="run",  # or debug
+        beat="metricbeat", # or filebeat
+        arch="arm64", # or amd64
+        k8s_env="dev", # or prod
+        **kwargs):
+
+    if mode not in ["run", "debug"]:
+        exit(-1)
+
+    print("Mode: {}".format(mode))
+
+    ## Compile
+    flags = ""
+    if mode == "debug":
+        flags = '-gcflags "-N -l"'
+    build_cmd = "CGO_ENABLED=0 GOOS=linux GOARCH={} go build {} -o build/{} ../../{}".format(arch, flags, beat, beat)
+    print("Build cmd: {}".format(build_cmd))
+
+    local_resource(
+        '{}-compile'.format(beat),
+        build_cmd,
+        deps=['.'],
+        ignore=[
+            "build",
+            "Tiltfile"
+        ])
+
+
+    ## Docker Build
+    docker_image = '{}-{}-image'.format(beat, mode)
+    if k8s_env == "prod":
+        # More info at https://ttl.sh/
+        # docker_registry = 'ttl.sh/{}'.format(docker_image)
+
+        # In order to push to GCR you need
+        #   gcloud auth login
+        #   gcloud auth configure-docker
+        #
+        # More info at https://cloud.google.com/container-registry/docs/advanced-authentication#gcloud-helper
+        docker_registry = "us.gcr.io/elastic-observability".format(docker_image)
+
+        default_registry(docker_registry)
+        print("Docker registry: {}".format(docker_registry))
+
+    print("Docker image: {}".format(docker_image))
+
+    docker_file = '{}/Dockerfile.{}'.format(beat, mode)
+    print("Docker file: {}".format(docker_file))
+
+    if mode == "run":
+        docker_entrypoint = [
+                '/usr/share/{}/{}'.format(beat, beat),
+                "-c",
+                "/etc/{}.yml".format(beat),
+                "-e"
+            ]
+    else:
+        docker_entrypoint = [
+            "dlv",
+            "--headless=true",
+            "--listen=:56268",
+            "--api-version=2",
+            "--log",
+            "exec",
+            "/usr/share/{}/{}".format(beat, beat),
+            "--"
+        ]
+    print("Docker entrypoint: {}".format(docker_entrypoint))
+
+    docker_build_with_restart(
+        docker_image,
+        '.',
+        entrypoint=docker_entrypoint,
+        dockerfile=docker_file,
+        only=["build"],
+        live_update=[
+            sync('./build', '/usr/share/{}'.format(beat)),
+        ],
+    )
+
+    ## Deploy to k8s
+    k8s_manifest = '{}/manifest.{}.{}.yaml'.format(beat, mode, k8s_env)
+    print("K8s manifest: {}".format(k8s_manifest))
+
+    k8s_yaml(k8s_manifest)
+
+    ## Expose resource
+    if mode == "run":
+        k8s_resource(beat)
+    else:
+        k8s_resource(beat, port_forwards=56268)
+
+    allow_k8s_contexts(k8s_context())
+
+beat(beat="metricbeat", # change to `filebeat` to test filebeat
+    mode="debug", # debug mode only works with k8s_env=`dev`
+    arch="arm64", # change if on different laptop architecture
+    k8s_env="dev") # change to `prod` to use with EKS or GKE.
+
+
+
+

dev-tools/kubernetes/filebeat/Dockerfile.debug

@@ -0,0 +1,21 @@
+FROM golang:alpine3.15 as builder
+
+ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
+
+ENV CGO_ENABLED=0
+
+RUN go install github.com/go-delve/delve/cmd/[email protected]
+
+COPY build/filebeat /usr/share/filebeat/filebeat
+
+FROM alpine:3.15
+
+ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin
+
+WORKDIR /usr/share/filebeat
+
+COPY --from=builder /go/bin/dlv /go/bin/dlv
+COPY --from=builder /usr/share/filebeat/filebeat /usr/share/filebeat/filebeat
+
+ENTRYPOINT ["dlv", "--headless=true", "--listen=:56268", "--api-version=2", "--log", "exec", "./filebeat", "--"]
+CMD [ "-e" ]

dev-tools/kubernetes/filebeat/Dockerfile.run

@@ -0,0 +1,11 @@
+FROM alpine:3.15
+
+ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin
+
+WORKDIR /usr/share/filebeat
+
+COPY build/filebeat /usr/share/filebeat/filebeat
+
+ENTRYPOINT ["./filebeat"]
+
+CMD [ "-e" ]