Merge branch 'main' into baptiste.foy/RCM-983/update-go-tuf--3

DataDog · Jul 21, 2023 · 3bdb963 · 3bdb963
2 parents 99ad631 + 9be0c94
commit 3bdb963
Show file tree

Hide file tree

Showing 308 changed files with 11,036 additions and 5,881 deletions.
diff --git a/.circleci/README.md b/.circleci/README.md
@@ -4,28 +4,19 @@ CircleCI is used to run unit tests on Unix env.
 
 ## Upgrading Golang version
 
-/!\ Disclaimer: the datadog-agent-runner-circle image should never be used for anything else than CircleCI tests /!\
+/!\ Disclaimer: the datadog/agent-buildimages-circleci-runner image should never be used for anything else than CircleCI tests /!\
 
-Change the Golang version in this file `images/runner/Dockerfile`.
+This image is now built alongside other images in [agent-buildimages](https://github.com/DataDog/datadog-agent-buildimages) repository. Change of Golang version must occur in this repository.
 
-Then locally build and push the new image using
-`datadog/datadog-agent-runner-circle:go<new golang version>` for the image's
-name. You will need write access to that repo on DockerHub (the Agent's team
-should have it).
+Once you have created a new image by building a new version of agent-buildimages, you can test your modification with the associated invoke task:
 
-Example:
 ```bash
-cd .circleci/images/runner
-docker build --platform=linux/amd64 -t datadog/datadog-agent-runner-circle:go1205 .
-docker login
-docker push datadog/datadog-agent-runner-circle:go1205
+invoke -e pipeline.update-buildimages --image-tag v12345678-c0mm1t5
+```
+This will update the configuration of circleci and gitlab to use the __test version__ of these images.
+Once your test is successful, you can either move the `_test_version` from files or invoke
+```bash
+invoke -e pipeline.update-buildimages --image-tag v12345678-c0mm1t5 --no-test-version
 ```
-
-Once your image is pushed, update this file:
-https://github.com/DataDog/datadog-agent/blob/main/.circleci/config.yml.
-Change `image: datadog/datadog-agent-runner-circle:goXXXX` for the tag you
-just pushed.
-
-Push your change as a new PR to see if CircleCI is still green.
 
 If everything is green, get a review and merge the PR.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -15,7 +15,7 @@ experimental:
 templates:
   job_template: &job_template
     docker:
-      - image: datadog/datadog-agent-runner-circle:go1206
+      - image: datadog/agent-buildimages-circleci-runner:v18021293-6b0809a
         environment:
           USE_SYSTEM_LIBS: "1"
     working_directory: /go/src/github.com/DataDog/datadog-agent
@@ -29,11 +29,11 @@ templates:
           # If incremental dep fails, increase the cache gen number
           # in restore_deps AND save_deps
           # See https://github.com/DataDog/datadog-agent/pull/2384
-          - gen19-godeps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/requirements.txt" }}-{{ checksum ".circleci/images/runner/Dockerfile" }}-{{ .Branch }}-{{ .Revision }}
-          - gen19-godeps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/requirements.txt" }}-{{ checksum ".circleci/images/runner/Dockerfile" }}-{{ .Branch }}-
-          - gen19-godeps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/requirements.txt" }}-{{ checksum ".circleci/images/runner/Dockerfile" }}-main-
+          - gen19-godeps-{{ checksum ".circleci/config.yml" }}-{{ .Branch }}-{{ .Revision }}
+          - gen19-godeps-{{ checksum ".circleci/config.yml" }}-{{ .Branch }}-
+          - gen19-godeps-{{ checksum ".circleci/config.yml" }}-main-
     - save_cache: &save_deps
-        key: gen19-godeps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/requirements.txt" }}-{{ checksum ".circleci/images/runner/Dockerfile" }}-{{ .Branch }}-{{ .Revision }}
+        key: gen19-godeps-{{ checksum ".circleci/config.yml" }}-{{ .Branch }}-{{ .Revision }}
     - restore_cache: &restore_source
         keys:
           # Cache retrieval is faster than full git checkout
@@ -58,12 +58,6 @@ jobs:
     steps:
       - restore_cache: *restore_source
       - restore_cache: *restore_deps
-      - run:
-          name: setup python deps
-          command: |
-            python3 -m pip install wheel
-            python3 -m pip install -r requirements.txt
-            python3 -m pip install -r .circleci/requirements.txt
       - run:
           name: check go version
           command: |
@@ -95,7 +89,6 @@ jobs:
             - /go/pkg/mod
             - /go/bin
             - /go/src/github.com/DataDog/datadog-agent/dev
-            - /usr/local/lib/python3.8/dist-packages
             - /usr/local/bin
 
   unit_tests:

diff --git a/.circleci/images/builder/Dockerfile b/.circleci/images/builder/Dockerfile
diff --git a/.circleci/images/runner/Dockerfile b/.circleci/images/runner/Dockerfile
diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -335,6 +335,9 @@
 /pkg/network/driver/                    @DataDog/windows-kernel-integrations
 /pkg/network/ebpf/c/prebuilt/usm*      @DataDog/universal-service-monitoring
 /pkg/network/ebpf/c/runtime/usm*       @DataDog/universal-service-monitoring
+/pkg/network/ebpf/c/prebuilt/shared-libraries*  @DataDog/universal-service-monitoring
+/pkg/network/ebpf/c/runtime/shared-libraries*   @DataDog/universal-service-monitoring
+/pkg/network/ebpf/c/shared-libraries/           @DataDog/universal-service-monitoring
 /pkg/network/ebpf/c/prebuilt/http*      @DataDog/universal-service-monitoring
 /pkg/network/ebpf/c/runtime/http*       @DataDog/universal-service-monitoring
 /pkg/network/ebpf/c/protocols/          @DataDog/universal-service-monitoring

diff --git a/.gitignore b/.gitignore
@@ -116,6 +116,7 @@ pkg/ebpf/bytecode/build/**/*.d
 pkg/ebpf/bytecode/runtime/conntrack.go
 pkg/ebpf/bytecode/runtime/http.go
 pkg/ebpf/bytecode/runtime/usm.go
+pkg/ebpf/bytecode/runtime/shared-libraries.go
 pkg/ebpf/bytecode/runtime/offsetguess-test.go
 pkg/ebpf/bytecode/runtime/oom-kill.go
 pkg/ebpf/bytecode/runtime/runtime-security.go

diff --git a/.gitlab/binary_build/system_probe.yml b/.gitlab/binary_build/system_probe.yml
@@ -19,6 +19,8 @@
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/offset-guess-debug.o $S3_ARTIFACTS_URI/offset-guess-debug.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/usm.o $S3_ARTIFACTS_URI/usm.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/usm-debug.o $S3_ARTIFACTS_URI/usm-debug.o.$ARCH
+    - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/shared-libraries.o $S3_ARTIFACTS_URI/shared-libraries.o.$ARCH
+    - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/shared-libraries-debug.o $S3_ARTIFACTS_URI/shared-libraries-debug.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/dns.o $S3_ARTIFACTS_URI/dns.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/dns-debug.o $S3_ARTIFACTS_URI/dns-debug.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/conntrack.o $S3_ARTIFACTS_URI/conntrack.o.$ARCH
@@ -28,6 +30,8 @@
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/runtime-security-offset-guesser.o $S3_ARTIFACTS_URI/runtime-security-offset-guesser.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re/usm.o $S3_ARTIFACTS_URI/usm-co-re.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re/usm-debug.o $S3_ARTIFACTS_URI/usm-debug-co-re.o.$ARCH
+    - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re/shared-libraries.o $S3_ARTIFACTS_URI/shared-libraries-co-re.o.$ARCH
+    - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re/shared-libraries-debug.o $S3_ARTIFACTS_URI/shared-libraries-debug-co-re.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re/oom-kill.o $S3_ARTIFACTS_URI/oom-kill-co-re.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re/tcp-queue-length.o $S3_ARTIFACTS_URI/tcp-queue-length-co-re.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re/ebpf.o $S3_ARTIFACTS_URI/ebpf-co-re.o.$ARCH
@@ -38,6 +42,7 @@
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/co-re/tracer-debug.o $S3_ARTIFACTS_URI/tracer-debug-co-re.o.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/runtime/tracer.c $S3_ARTIFACTS_URI/tracer.c.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/runtime/usm.c $S3_ARTIFACTS_URI/usm.c.$ARCH
+    - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/runtime/shared-libraries.c $S3_ARTIFACTS_URI/shared-libraries.c.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/runtime/runtime-security.c $S3_ARTIFACTS_URI/runtime-security.c.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/runtime/conntrack.c $S3_ARTIFACTS_URI/conntrack.c.$ARCH
     - $S3_CP_CMD $CI_PROJECT_DIR/pkg/ebpf/bytecode/build/runtime/oom-kill.c $S3_ARTIFACTS_URI/oom-kill.c.$ARCH

diff --git a/.gitlab/internal_image_deploy.yml b/.gitlab/internal_image_deploy.yml
@@ -27,7 +27,7 @@ docker_trigger_internal:
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
     - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi
     - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi
-    - inv pipeline.trigger-child-pipeline --project-name "DataDog/images" --git-ref "master" --variables "IMAGE_VERSION,IMAGE_NAME,RELEASE_TAG,BUILD_TAG,TMPL_SRC_IMAGE,TMPL_SRC_REPO,RELEASE_STAGING,RELEASE_PROD"
+    - inv pipeline.trigger-child-pipeline --project-name "DataDog/images" --git-ref "master" --variables "IMAGE_VERSION,IMAGE_NAME,RELEASE_TAG,BUILD_TAG,TMPL_SRC_IMAGE,TMPL_SRC_REPO,RELEASE_STAGING,RELEASE_PROD,DYNAMIC_BUILD_RENDER_RULES"
 
 
 docker_trigger_cluster_agent_internal:
@@ -56,4 +56,4 @@ docker_trigger_cluster_agent_internal:
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
     - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi
     - if [ "$BUCKET_BRANCH" = "nightly" ]; then RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi
-    - inv pipeline.trigger-child-pipeline --project-name "DataDog/images" --git-ref "master" --variables "IMAGE_VERSION,IMAGE_NAME,RELEASE_TAG,BUILD_TAG,TMPL_SRC_IMAGE,TMPL_SRC_REPO,RELEASE_STAGING,RELEASE_PROD"
+    - inv pipeline.trigger-child-pipeline --project-name "DataDog/images" --git-ref "master" --variables "IMAGE_VERSION,IMAGE_NAME,RELEASE_TAG,BUILD_TAG,TMPL_SRC_IMAGE,TMPL_SRC_REPO,RELEASE_STAGING,RELEASE_PROD,DYNAMIC_BUILD_RENDER_RULES"
diff --git a/.gitlab/package_build/deb.yml b/.gitlab/package_build/deb.yml
@@ -26,6 +26,8 @@
     - $S3_CP_CMD $S3_ARTIFACTS_URI/offset-guess-debug.o.${PACKAGE_ARCH} /tmp/system-probe/offset-guess-debug.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm.o.${PACKAGE_ARCH} /tmp/system-probe/usm.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm-debug.o.${PACKAGE_ARCH} /tmp/system-probe/usm-debug.o
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries.o.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries.o
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries-debug.o.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries-debug.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/dns.o.${PACKAGE_ARCH} /tmp/system-probe/dns.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/dns-debug.o.${PACKAGE_ARCH} /tmp/system-probe/dns-debug.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/conntrack.o.${PACKAGE_ARCH} /tmp/system-probe/conntrack.o
@@ -43,8 +45,11 @@
     - $S3_CP_CMD $S3_ARTIFACTS_URI/tracer-fentry-debug-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/tracer-fentry-debug-co-re.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/usm-co-re.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm-debug-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/usm-debug-co-re.o
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries-co-re.o
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries-debug-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries-debug-co-re.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/tracer.c.${PACKAGE_ARCH} /tmp/system-probe/tracer.c
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm.c.${PACKAGE_ARCH} /tmp/system-probe/usm.c
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries.c.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries.c
     - $S3_CP_CMD $S3_ARTIFACTS_URI/runtime-security.c.${PACKAGE_ARCH} /tmp/system-probe/runtime-security.c
     - $S3_CP_CMD $S3_ARTIFACTS_URI/conntrack.c.${PACKAGE_ARCH} /tmp/system-probe/conntrack.c
     - $S3_CP_CMD $S3_ARTIFACTS_URI/oom-kill.c.${PACKAGE_ARCH} /tmp/system-probe/oom-kill.c

diff --git a/.gitlab/package_build/rpm.yml b/.gitlab/package_build/rpm.yml
@@ -23,6 +23,8 @@
     - $S3_CP_CMD $S3_ARTIFACTS_URI/offset-guess-debug.o.${PACKAGE_ARCH} /tmp/system-probe/offset-guess-debug.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm.o.${PACKAGE_ARCH} /tmp/system-probe/usm.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm-debug.o.${PACKAGE_ARCH} /tmp/system-probe/usm-debug.o
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries.o.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries.o
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries-debug.o.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries-debug.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/dns.o.${PACKAGE_ARCH} /tmp/system-probe/dns.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/dns-debug.o.${PACKAGE_ARCH} /tmp/system-probe/dns-debug.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/conntrack.o.${PACKAGE_ARCH} /tmp/system-probe/conntrack.o
@@ -40,8 +42,11 @@
     - $S3_CP_CMD $S3_ARTIFACTS_URI/tracer-fentry-debug-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/tracer-fentry-debug-co-re.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/usm-co-re.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm-debug-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/usm-debug-co-re.o
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries-co-re.o
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries-debug-co-re.o.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries-debug-co-re.o
     - $S3_CP_CMD $S3_ARTIFACTS_URI/tracer.c.${PACKAGE_ARCH} /tmp/system-probe/tracer.c
     - $S3_CP_CMD $S3_ARTIFACTS_URI/usm.c.${PACKAGE_ARCH} /tmp/system-probe/usm.c
+    - $S3_CP_CMD $S3_ARTIFACTS_URI/shared-libraries.c.${PACKAGE_ARCH} /tmp/system-probe/shared-libraries.c
     - $S3_CP_CMD $S3_ARTIFACTS_URI/runtime-security.c.${PACKAGE_ARCH} /tmp/system-probe/runtime-security.c
     - $S3_CP_CMD $S3_ARTIFACTS_URI/conntrack.c.${PACKAGE_ARCH} /tmp/system-probe/conntrack.c
     - $S3_CP_CMD $S3_ARTIFACTS_URI/oom-kill.c.${PACKAGE_ARCH} /tmp/system-probe/oom-kill.c