diff --git a/.github/actions/pull_images/action.yml b/.github/actions/pull_images/action.yml index b091a792cd..4e31aaa50e 100644 --- a/.github/actions/pull_images/action.yml +++ b/.github/actions/pull_images/action.yml @@ -59,4 +59,4 @@ runs: - name: Pull shell: bash - run: docker compose pull + run: cat compose.yaml && docker compose pull diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74e30d667a..755d9a1a49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,11 @@ jobs: uses: actions/checkout@v4 - name: Get library artifact run: ./utils/scripts/load-binary.sh ${{ matrix.library }} + - name: Get nginx module + if: matrix.library == 'cpp' + run: ./utils/scripts/load-binary.sh nginx + env: + CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }} - name: Get agent artifact run: ./utils/scripts/load-binary.sh agent diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 301e1221aa..0bd98e5a07 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -193,15 +193,15 @@ jobs: if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, 'AGENT_NOT_SUPPORTING_SPAN_EVENTS') && (inputs.library != 'ruby' || matrix.weblog == 'rack') run: ./run.sh AGENT_NOT_SUPPORTING_SPAN_EVENTS - name: Run APPSEC_MISSING_RULES scenario - # C++ 1.2.0 freeze when the rules file is missing - if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_MISSING_RULES"') && inputs.library != 'cpp' + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_MISSING_RULES"') && matrix.weblog != 'nginx' + # nginx 1.2.0 refuses to start without a valid rules files run: ./run.sh APPSEC_MISSING_RULES - name: Run APPSEC_CUSTOM_RULES scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_CUSTOM_RULES"') run: ./run.sh APPSEC_CUSTOM_RULES - name: Run APPSEC_CORRUPTED_RULES scenario - # C++ 1.2.0 freeze when the rules file is missing - if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_CORRUPTED_RULES"') && inputs.library != 'cpp' + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_CORRUPTED_RULES"') && matrix.weblog != 'nginx' + # nginx 1.2.0 refuses to start without a valid rules files run: ./run.sh APPSEC_CORRUPTED_RULES - name: Run APPSEC_RULES_MONITORING_WITH_ERRORS scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_RULES_MONITORING_WITH_ERRORS"') diff --git a/manifests/cpp.yml b/manifests/cpp.yml index b258828047..6f3dddc300 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -8,7 +8,37 @@ tests/: api_security/: test_api_security_rc.py: irrelevant (ASM is not implemented in C++) test_apisec_sampling.py: irrelevant (ASM is not implemented in C++) - test_schemas.py: irrelevant (ASM is not implemented in C++) + test_schemas.py: + Test_Scanners: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Schema_Request_Cookies: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Schema_Request_FormUrlEncoded_Body: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Schema_Request_Headers: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Schema_Request_Json_Body: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Schema_Request_Path_Parameters: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Schema_Request_Query_Parameters: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Schema_Response_Body: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Schema_Response_Body_env_var: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Schema_Response_Headers: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature iast/: sink/: test_code_injection.py: irrelevant (ASM is not implemented in C++) @@ -62,38 +92,270 @@ tests/: test_sqli.py: irrelevant (ASM is not implemented in C++) test_ssrf.py: irrelevant (ASM is not implemented in C++) waf/: - test_addresses.py: irrelevant (ASM is not implemented in C++) - test_blocking.py: irrelevant (ASM is not implemented in C++) - test_custom_rules.py: irrelevant (ASM is not implemented in C++) - test_exclusions.py: irrelevant (ASM is not implemented in C++) - test_miscs.py: irrelevant (ASM is not implemented in C++) - test_reports.py: irrelevant (ASM is not implemented in C++) - test_rules.py: irrelevant (ASM is not implemented in C++) + test_addresses.py: + Test_BodyJson: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.4.0 + Test_BodyRaw: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_BodyUrlEncoded: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.4.0 + Test_BodyXml: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Cookies: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_FullGrpc: irrelevant + Test_GraphQL: irrelevant + Test_GrpcServerMethod: irrelevant + Test_Headers: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_PathParams: + "*": irrelevant (ASM is not implemented in C++) + nginx: irrelevant (path params waf address unfilled) + Test_ResponseStatus: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_UrlQuery: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_UrlQueryKey: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_UrlRaw: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_gRPC: irrelevant + test_blocking.py: + Test_Blocking: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_Blocking_strip_response_headers: + "*": irrelevant (ASM is not implemented in C++) + nginx: irrelevant (no response headers on 1st waf run, which is where blocking is possible) + Test_CustomBlockingResponse: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + test_custom_rules.py: + Test_CustomRules: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + test_exclusions.py: + Test_Exclusions: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + test_miscs.py: + Test_404: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_CorrectOptionProcessing: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_MultipleAttacks: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_MultipleHighlight: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + test_reports.py: + Test_Monitoring: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature (WAF monitoring tags not implemented) + test_rules.py: + Test_CommandInjection: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_DiscoveryScan: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_HttpProtocol: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_JavaCodeInjection: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_JsInjection: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_LFI: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_NoSqli: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_PhpCodeInjection: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_RFI: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_SQLI: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_SSRF: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_Scanners: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_XSS: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 test_telemetry.py: irrelevant (ASM is not implemented in C++) test_alpha.py: irrelevant (ASM is not implemented in C++) test_asm_standalone.py: irrelevant (ASM is not implemented in C++) - test_automated_login_events.py: irrelevant (ASM is not implemented in C++) + test_automated_login_events.py: + Test_Login_Events: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_Login_Events_Extended: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_V2_Login_Events: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_V2_Login_Events_Anon: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_V2_Login_Events_RC: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_V3_Auto_User_Instrum_Mode_Capability: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_V3_Login_Events: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_V3_Login_Events_Anon: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_V3_Login_Events_Blocking: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_V3_Login_Events_RC: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature test_automated_user_and_session_tracking.py: irrelevant (ASM is not implemented in C++) - test_blocking_addresses.py: irrelevant (ASM is not implemented in C++) - test_client_ip.py: irrelevant (ASM is not implemented in C++) - test_conf.py: irrelevant (ASM is not implemented in C++) - test_customconf.py: irrelevant (ASM is not implemented in C++) + test_blocking_addresses.py: + Test_BlockingGraphqlResolvers: + "*": irrelevant (ASM is not implemented in C++) + nginx: irrelevant + Test_Blocking_client_ip: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_Blocking_request_body: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.4.0 + Test_Blocking_request_body_multipart: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.4.0 + Test_Blocking_request_cookies: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_Blocking_request_headers: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_Blocking_request_method: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_Blocking_request_path_params: + "*": irrelevant (ASM is not implemented in C++) + nginx: irrelevant + Test_Blocking_request_query: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_Blocking_request_uri: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_Blocking_response_headers: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature (blocking on final WAF run not possible) + Test_Blocking_response_status: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature (blocking on final WAF run not possible) + Test_Blocking_user_id: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature (users not supported) + Test_Suspicious_Request_Blocking: + "*": irrelevant (ASM is not implemented in C++) + nginx: irrelevant (path params waf address unfilled) + test_client_ip.py: + Test_StandardTagsClientIp: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature (Standard logs not implemented) + test_conf.py: + Test_ConfigurationVariables: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + test_customconf.py: + Test_ConfRuleSet: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_CorruptedRules: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_MissingRules: + "*": irrelevant (ASM is not implemented in C++) + nginx: irrelevant (fatal error on missing rules) + Test_NoLimitOnWafRules: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 test_event_tracking.py: irrelevant (ASM is not implemented in C++) test_fingerprinting.py: irrelevant (ASM is not implemented in C++) test_identify.py: irrelevant (ASM is not implemented in C++) - test_ip_blocking_full_denylist.py: irrelevant (ASM is not implemented in C++) + test_ip_blocking_full_denylist.py: + Test_AppSecIPBlockingFullDenylist: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.3.0 test_logs.py: irrelevant (ASM is not implemented in C++) test_metastruct.py: irrelevant (ASM is not implemented in C++) - test_rate_limiter.py: irrelevant (ASM is not implemented in C++) + test_rate_limiter.py: + Test_Main: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature (Rate limiting not implemented) test_remote_config_rule_changes.py: irrelevant (ASM is not implemented in C++) test_reports.py: irrelevant (ASM is not implemented in C++) - test_request_blocking.py: irrelevant (ASM is not implemented in C++) - test_runtime_activation.py: irrelevant (ASM is not implemented in C++) + test_request_blocking.py: + Test_AppSecRequestBlocking: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.3.0 + test_runtime_activation.py: + Test_RuntimeActivation: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.3.0 test_shell_execution.py: irrelevant (ASM is not implemented in C++) test_suspicious_attacker_blocking.py: irrelevant (ASM is not implemented in C++) - test_traces.py: irrelevant (ASM is not implemented in C++) - test_user_blocking_full_denylist.py: irrelevant (ASM is not implemented in C++) - test_versions.py: irrelevant (ASM is not implemented in C++) + test_traces.py: + Test_AppSecEventSpanTags: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_AppSecObfuscator: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_CollectDefaultRequestHeader: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_CollectRespondHeaders: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_ExternalWafRequestsIdentification: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + Test_RetainTraces: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 + test_user_blocking_full_denylist.py: + Test_UserBlocking_FullDenylist: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature (User blocking not implemented) + test_versions.py: + Test_Events: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.2.0 debugger/: test_debugger_exception_replay.py: Test_Debugger_Exception_Replay: irrelevant @@ -214,12 +476,22 @@ tests/: test_tracer_flare.py: missing_feature remote_config/: test_remote_configuration.py: - Test_RemoteConfigurationExtraServices: missing_feature - Test_RemoteConfigurationUpdateSequenceASMDD: missing_feature - Test_RemoteConfigurationUpdateSequenceASMDDNoCache: missing_feature - Test_RemoteConfigurationUpdateSequenceFeatures: missing_feature - Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: missing_feature - Test_RemoteConfigurationUpdateSequenceLiveDebugging: missing_feature + Test_RemoteConfigurationExtraServices: + "*": irrelevant (ASM is not implemented in C++) + nginx: missing_feature + Test_RemoteConfigurationUpdateSequenceASMDD: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.3.0 + Test_RemoteConfigurationUpdateSequenceASMDDNoCache: + "*": irrelevant (ASM is not implemented in C++) + nginx: irrelevant (we opt into cache) + Test_RemoteConfigurationUpdateSequenceFeatures: + "*": irrelevant (ASM is not implemented in C++) + nginx: v1.3.0 + Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: + "*": irrelevant (ASM is not implemented in C++) + nginx: irrelevant (we opt into cache) + Test_RemoteConfigurationUpdateSequenceLiveDebugging: irrelevant serverless/: span_pointers/: aws/: @@ -262,8 +534,13 @@ tests/: test_span_events.py: incomplete_test_app (Weblog `/add_event` not implemented) test_standard_tags.py: irrelevant test_telemetry.py: + Test_APMOnboardingInstallID: missing_feature + Test_DependencyEnable: missing_feature Test_Log_Generation: missing_feature Test_MessageBatch: missing_feature + Test_Metric_Generation_Disabled: missing_feature Test_Metric_Generation_Enabled: missing_feature Test_ProductsDisabled: irrelevant + Test_Telemetry: missing_feature Test_TelemetrySCAEnvVar: missing_feature + Test_TelemetryV2: missing_feature diff --git a/tests/appsec/test_blocking_addresses.py b/tests/appsec/test_blocking_addresses.py index ac6fa51431..1ac55a2e65 100644 --- a/tests/appsec/test_blocking_addresses.py +++ b/tests/appsec/test_blocking_addresses.py @@ -3,6 +3,9 @@ # Copyright 2021 Datadog, Inc. import json + +import pytest + from utils import ( bug, context, @@ -54,6 +57,7 @@ def test_blocking(self): def setup_blocking_before(self): self.block_req2 = weblog.get("/tag_value/tainted_value_6512/200", headers={"X-Forwarded-For": "1.1.1.1"}) + @irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run") def test_blocking_before(self): """Test that blocked requests are blocked before being processed""" # second request should block and must not set the tag in span @@ -106,6 +110,7 @@ def setup_blocking_before(self): self.block_req2 = weblog.request("OPTIONS", path="/tag_value/tainted_value_6512/200") @flaky(context.library < "java@1.16.0", reason="APMRP-360") + @irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run") def test_blocking_before(self): """Test that blocked requests are blocked before being processed""" # first request should not block and must set the tag in span accordingly @@ -158,6 +163,7 @@ def setup_blocking_before(self): self.set_req1 = weblog.get("/tag_value/clean_value_3877/200") self.block_req2 = weblog.get("/tag_value/tainted_value_6512.git/200") + @irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run") def test_blocking_before(self): """Test that blocked requests are blocked before being processed""" # first request should not block and must set the tag in span accordingly @@ -202,6 +208,7 @@ def setup_blocking_before(self): self.set_req1 = weblog.get("/tag_value/clean_value_3878/200") self.block_req2 = weblog.get("/tag_value/tainted_value_AiKfOeRcvG45/200") + @irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run") def test_blocking_before(self): """Test that blocked requests are blocked before being processed""" # first request should not block and must set the tag in span accordingly @@ -249,6 +256,7 @@ def setup_blocking_before(self): self.set_req1 = weblog.get("/tag_value/clean_value_3879/200") self.block_req2 = weblog.get("/tag_value/tainted_value_a1b2c3/200?foo=xtrace") + @irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run") def test_blocking_before(self): """Test that blocked requests are blocked before being processed""" # first request should not block and must set the tag in span accordingly @@ -296,6 +304,7 @@ def setup_blocking_before(self): self.set_req1 = weblog.get("/tag_value/clean_value_3880/200") self.block_req2 = weblog.get("/tag_value/tainted_value_xyz/200", headers={"foo": "asldhkuqwgervf"}) + @irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run") def test_blocking_before(self): """Test that blocked requests are blocked before being processed""" # first request should not block and must set the tag in span accordingly @@ -343,6 +352,7 @@ def setup_blocking_before(self): self.set_req1 = weblog.get("/tag_value/clean_value_3881/200") self.block_req2 = weblog.get("/tag_value/tainted_value_cookies/200", cookies={"foo": "jdfoSDGFkivRG_234"}) + @irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run") def test_blocking_before(self): """Test that blocked requests are blocked before being processed""" # first request should not block and must set the tag in span accordingly @@ -394,7 +404,7 @@ def setup_non_blocking_plain_text(self): ) @irrelevant( - context.weblog_variant in ("akka-http", "play", "jersey-grizzly2", "resteasy-netty3"), + context.weblog_variant in ("akka-http", "play", "jersey-grizzly2", "resteasy-netty3", "nginx"), reason="Blocks on text/plain if parsed to a String", ) def test_non_blocking_plain_text(self): @@ -407,6 +417,7 @@ def setup_blocking_before(self): self.set_req1 = weblog.post("/tag_value/clean_value_3882/200", data={"good": "value"}) self.block_req2 = weblog.post("/tag_value/tainted_value_body/200", data={"value5": "bsldhkuqwgervf"}) + @irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run") def test_blocking_before(self): """Test that blocked requests are blocked before being processed""" # first request should not block and must set the tag in span accordingly diff --git a/tests/appsec/test_traces.py b/tests/appsec/test_traces.py index 185340c91b..d128274777 100644 --- a/tests/appsec/test_traces.py +++ b/tests/appsec/test_traces.py @@ -7,7 +7,7 @@ from utils.tools import nested_lookup -RUNTIME_FAMILIES = ["nodejs", "ruby", "jvm", "dotnet", "go", "php", "python"] +RUNTIME_FAMILIES = ["nodejs", "ruby", "jvm", "dotnet", "go", "php", "python", "cpp"] @bug(context.library == "python@1.1.0", reason="APMRP-360") diff --git a/tests/appsec/waf/test_addresses.py b/tests/appsec/waf/test_addresses.py index 537b90f1a4..d4109477a0 100644 --- a/tests/appsec/waf/test_addresses.py +++ b/tests/appsec/waf/test_addresses.py @@ -100,6 +100,7 @@ def setup_specific_key2(self): @irrelevant(library="ruby", reason="Rack transforms underscores into dashes") @irrelevant(library="php", reason="PHP normalizes into dashes; additionally, matching on keys is not supported") + @irrelevant(weblog_variant="nginx", reason="Header rejected by nginx ('client sent invalid header line'") @missing_feature(weblog_variant="spring-boot-3-native", reason="GraalVM. Tracing support only") def test_specific_key2(self): """attacks on specific header X_Filename, and report it""" diff --git a/utils/_context/containers.py b/utils/_context/containers.py index c91c398465..97eccc9445 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -172,6 +172,10 @@ def start(self, network: Network) -> Container: logger.info(f"Start container {self.container_name}") + # the whole thing is reimplemented in python... + if self.healthcheck is not None: + self.kwargs["healthcheck"] = {"test": ["NONE"]} + self._container = _get_client().containers.run( image=self.image.name, name=self.container_name, @@ -994,9 +998,10 @@ def __init__(self, host_log_folder) -> None: class MySqlContainer(SqlDbTestedContainer): def __init__(self, host_log_folder) -> None: super().__init__( - image_name="mysql/mysql-server:latest", + image_name="mysql/mysql-server:8.0.32", name="mysqldb", - command="--default-authentication-plugin=mysql_native_password", + command="--lc-messages-dir=/usr/share/mysql-8.0/english " + "--default-authentication-plugin=mysql_native_password", environment={ "MYSQL_DATABASE": "mysql_dbname", "MYSQL_USER": "mysqldb", diff --git a/utils/build/docker/cpp/nginx.Dockerfile b/utils/build/docker/cpp/nginx.Dockerfile index 6fcf38abf7..dad81fce61 100644 --- a/utils/build/docker/cpp/nginx.Dockerfile +++ b/utils/build/docker/cpp/nginx.Dockerfile @@ -4,16 +4,24 @@ ARG NGINX_VERSION="1.25.4" ENV NGINX_VERSION=${NGINX_VERSION} RUN apt-get update \ - && apt-get install -y wget tar jq curl xz-utils stress-ng binutils + && apt-get install -y \ + wget tar jq curl xz-utils stress-ng binutils gcc libmicrohttpd-dev \ + procps gdb -RUN mkdir /builds +RUN mkdir /builds /binaries -COPY utils/build/docker/cpp/nginx/nginx.conf /etc/nginx/nginx.conf +COPY utils/build/docker/cpp/nginx/nginx.conf /etc/nginx/nginx.conf.no-waf +COPY utils/build/docker/cpp/nginx/nginx-waf.conf /etc/nginx/nginx.conf.waf COPY utils/build/docker/cpp/nginx/hello.html /builds/hello.html COPY utils/build/docker/cpp/nginx/install_ddtrace.sh /builds/ COPY utils/build/docker/cpp/install_ddprof.sh /builds/ COPY utils/build/docker/cpp/nginx/app.sh /builds/ COPY utils/build/docker/cpp/ binaries* /builds/ +COPY binaries/* /binaries/ +COPY utils/build/docker/cpp/nginx/backend.c /tmp/ + +# install backend app +RUN gcc -O0 -g -o /usr/local/bin/backend /tmp/backend.c -lmicrohttpd && rm /tmp/backend.c WORKDIR /builds diff --git a/utils/build/docker/cpp/nginx/app.sh b/utils/build/docker/cpp/nginx/app.sh index 50792b637c..0711ba28d6 100755 --- a/utils/build/docker/cpp/nginx/app.sh +++ b/utils/build/docker/cpp/nginx/app.sh @@ -1,5 +1,7 @@ #!/bin/bash +backend & + if [[ "${DDPROF_ENABLE:-,,}" == "yes" ]]; then ddprof -l notice nginx -g 'daemon off;' else diff --git a/utils/build/docker/cpp/nginx/backend.c b/utils/build/docker/cpp/nginx/backend.c new file mode 100644 index 0000000000..86d2f731be --- /dev/null +++ b/utils/build/docker/cpp/nginx/backend.c @@ -0,0 +1,114 @@ +#include +#include +#include +#include +#include +#include +#include + +#define PORT 7778 + +static struct MHD_Daemon *daemon_; + +static ssize_t read_data(void *cls, uint64_t pos, char *buf, size_t max) { + int fd = (int)(uintptr_t)cls; + ssize_t num_read = read(fd, buf, max); + if (num_read == 0) { + return -1; + } + if (num_read < 0) { + return -2; + } + return num_read; +} +static void close_fd(void *cls) { + close((int)(uintptr_t)cls); +} + +static enum MHD_Result answer_to_connection(void *cls, struct MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls) +{ + const char *status_str = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "status"); + const char *value = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "value"); + + if (strcmp(url, "/read_file") == 0) { + const char *val = MHD_lookup_connection_value (connection, MHD_GET_ARGUMENT_KIND, "file"); + if (!val) { + return MHD_NO; + } + + const int fd = open(val, O_RDONLY); + if (fd < 0) { + return MHD_NO; + } + + struct stat st; + if (fstat(fd, &st) != 0) { + close(fd); + return MHD_NO; + } + const size_t file_size = st.st_size; + + struct MHD_Response *response; + if (file_size == 0) { + response = MHD_create_response_from_callback(-1, 512, read_data, (void*)(uintptr_t)fd, close_fd); + } else { + response = MHD_create_response_from_fd((uint64_t)file_size, fd); + } + + MHD_add_response_header(response, "Content-Type", "application/octet-stream"); + int ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + return ret; + } + + if (strcmp(url, "/content") != 0 || !status_str || !value) + return MHD_NO; // Only respond to the correct URL and if all parameters are present + + int status_code = atoi(status_str); + if (status_code <= 0) + return MHD_NO; // Ensure the status code is a valid positive integer + + const char *page = value; + struct MHD_Response *response = MHD_create_response_from_buffer(strlen(page), (void*)page, MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(response, "Content-Type", "text/html"); + + int ret = MHD_queue_response(connection, status_code, response); + MHD_destroy_response(response); + return ret; +} + +void stop_server(int signum) +{ + if (daemon_) + { + MHD_stop_daemon(daemon_); + daemon_ = NULL; + printf("Server has been stopped.\n"); + } + exit(0); +} + +int main() +{ + struct sigaction action; + memset(&action, 0, sizeof(struct sigaction)); + action.sa_handler = stop_server; + sigaction(SIGINT, &action, NULL); // Handle SIGINT + sigaction(SIGTERM, &action, NULL); // Handle SIGTERM + + daemon_ = MHD_start_daemon(MHD_USE_INTERNAL_POLLING_THREAD, PORT, NULL, NULL, + &answer_to_connection, NULL, MHD_OPTION_END); + if (!daemon_) + { + fprintf(stderr, "Failed to start the daemon.\n"); + return 1; + } + + printf("Server running on port %d\n", PORT); + pause(); // Wait for signals + + return 0; +} diff --git a/utils/build/docker/cpp/nginx/install_ddtrace.sh b/utils/build/docker/cpp/nginx/install_ddtrace.sh index 45406d46af..940f3d90ac 100755 --- a/utils/build/docker/cpp/nginx/install_ddtrace.sh +++ b/utils/build/docker/cpp/nginx/install_ddtrace.sh @@ -1,60 +1,148 @@ #!/bin/bash set -eu +set -o pipefail +set -x + +function fetch_version { + declare -r what=$1 + strings /usr/lib/nginx/modules/ngx_http_datadog_module.so | \ + grep -F "[${what} version" | \ + sed 's/.* version \([^]]\+\).*/\1/' +} + +function epilogue { + local module_version=$1 + if [[ -z $module_version ]]; then + echo "ERROR: Missing module version." + exit 1 + fi + + if [[ $module_version = unknown_mod_version ]]; then + module_version=v$(fetch_version nginx_mod) + if [[ $module_version == v ]]; then + echo "ERROR: could not determine module version" + exit 1 + fi + fi + + echo "DataDog/nginx-datadog version: ${module_version}" + echo "$module_version" | tr -d v > SYSTEM_TESTS_LIBRARY_VERSION + + rm -f /etc/nginx/nginx.conf + if version_first_is_greater "$module_version" "v1.1.0"; then + ln -s nginx.conf.waf /etc/nginx/nginx.conf + fetch_version libddwaf > SYSTEM_TESTS_LIBDDWAF_VERSION + fetch_version waf_rules > SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION + else + ln -s nginx.conf.no-waf /etc/nginx/nginx.conf + echo "0.0.0" > SYSTEM_TESTS_LIBDDWAF_VERSION + echo "0.0.0" > SYSTEM_TESTS_APPSEC_EVENT_RULES_VERSION + fi + + printf '{"status":"ok","library":{"language":"cpp","version":"%s"}}' "$(< SYSTEM_TESTS_LIBRARY_VERSION)" \ + > healthcheck.json +} + +version_first_is_greater() { + local v1=(${1//./ }) + local v2=(${2//./ }) + + # Remove the 'v' prefix from the version numbers + v1[0]=${v1[0]//v/} + v2[0]=${v2[0]//v/} + + # Compare the major, minor, and patch numbers + for i in {0..2}; do + if (( v1[i] > v2[i] )); then + return 0 + elif (( v1[i] < v2[i] )); then + return 1 + fi + done + + # equal (not greater) + return 1 +} + +if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 0 ]]; then + echo "Found module in /binaries" + + if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | wc -l) -gt 1 ]]; then + echo "ERROR: Found several ngx_http_datadog_module-*.so.tgz files in binaries/, abort." + exit 1 + fi + + NGINX_VERSION_OF_MODULE=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz' | grep -Po '(\d+\.\d+\.\d+)') + if [[ $NGINX_VERSION_OF_MODULE != $NGINX_VERSION ]]; then + echo "ERROR: nginx mismatch: module for $NGINX_VERSION_OF_MODULE, but base image of $NGINX_VERSION" + exit 1 + fi + + MAIN_TARBALL=$(find /binaries -name 'ngx_http_datadog_module-*.so.tgz') + tar -xzvf "$MAIN_TARBALL" -C /usr/lib/nginx/modules + if [[ $(find /binaries -name 'ngx_http_datadog_module-*.so.debug.tgz' | wc -l) -eq 1 ]]; then + tar -xzvf /binaries/ngx_http_datadog_module-*.so.debug.tgz -C /usr/lib/nginx/modules + fi + + epilogue unknown_mod_version + exit 0 +fi + +if [[ -f /binaries/ngx_http_datadog_module.so ]]; then + cp -v /binaries/ngx_http_datadog_module.so /usr/lib/nginx/modules + if [[ -f /binaries/ngx_http_datadog_module.so.debug ]]; then + cp -v /binaries/ngx_http_datadog_module.so.debug /usr/lib/nginx/modules + fi + + epilogue unknown_mod_version + exit 0 +fi get_latest_release() { - wget -qO- "https://api.github.com/repos/$1/releases/latest" | jq -r '.tag_name' + wget -qO- "https://api.github.com/repos/DataDog/nginx-datadog/releases/latest" \ + | jq -r '.tag_name' } get_architecture() { - case "$(uname -m)" in - aarch64) - echo "arm64" - ;; - arm64) - echo "arm64" - ;; - x86_64) - echo "amd64" - ;; - amd64) - echo "amd64" - ;; - *) - echo "" - ;; - esac + arch | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/' } + if [ NGINX_VERSION == "" ]; then echo 1>&2 "ERROR: Missing NGINX_VERSION." exit 1 fi -ARCH=$(get_architecture) +readonly ARCH=$(get_architecture) -if [ -z "$ARCH" ]; then - echo 1>&2 "ERROR: Architecture $(uname -m) is not supported." +if [[ $ARCH != "amd64" && $ARCH != "arm64" ]]; then + echo 1>&2 "ERROR: Architecture ${ARCH} is not supported." exit 1 fi -FILENAME=ngx_http_datadog_module-appsec-$ARCH-$NGINX_VERSION.so +readonly NGINX_DATADOG_VERSION="$(get_latest_release)" -if [ -f "$FILENAME" ]; then - echo "Install NGINX plugin from binaries/$FILENAME" - cp $FILENAME /usr/lib/nginx/modules/ngx_http_datadog_module.so - NGINX_DATADOG_VERSION="6.6.6" # TODO : get the version from the file ? +if version_first_is_greater "$NGINX_DATADOG_VERSION" "v1.1.0"; then + TARBALLS=( + "ngx_http_datadog_module-appsec-${ARCH}-${NGINX_VERSION}.so.tgz" + "ngx_http_datadog_module-appsec-${ARCH}-${NGINX_VERSION}.so.debug.tgz" + ) + for FILE in "${TARBALLS[@]}"; do + wget -O - \ + "https://github.com/DataDog/nginx-datadog/releases/download/$NGINX_DATADOG_VERSION/${FILE}" \ + | tar -xzf - -C /usr/lib/nginx/modules + done else - NGINX_DATADOG_VERSION="$(get_latest_release DataDog/nginx-datadog)" - TARBALL="$FILENAME.tgz" - echo "Get NGINX plugin from last github release of nginx-datadog" + # old versions + BASE_IMAGE="nginx:${NGINX_VERSION}" + BASE_IMAGE_WITHOUT_COLONS=$(echo "$BASE_IMAGE" | tr ':' '_') + TARBALL="$BASE_IMAGE_WITHOUT_COLONS-$ARCH-ngx_http_datadog_module.so.tgz" + + # Install NGINX plugin wget "https://github.com/DataDog/nginx-datadog/releases/download/${NGINX_DATADOG_VERSION}/${TARBALL}" tar -xzf "${TARBALL}" -C /usr/lib/nginx/modules rm "$TARBALL" fi -VERSION=$(strings /usr/lib/nginx/modules/ngx_http_datadog_module.so | grep -F "[dd-trace-cpp version" | sed 's/.* version \([^]]\+\).*/\1/') -echo '{"status": "ok", "library": {"language": "cpp", "version": "'$VERSION'"}}' > /builds/healthcheck.json -echo $VERSION > SYSTEM_TESTS_LIBRARY_VERSION -echo "Library version : $VERSION" - +epilogue "${NGINX_DATADOG_VERSION:1}" diff --git a/utils/build/docker/cpp/nginx/nginx-waf.conf b/utils/build/docker/cpp/nginx/nginx-waf.conf new file mode 100644 index 0000000000..e7f987c1ff --- /dev/null +++ b/utils/build/docker/cpp/nginx/nginx-waf.conf @@ -0,0 +1,93 @@ +error_log /var/log/nginx/error.log info; + +load_module modules/ngx_http_datadog_module.so; +thread_pool waf_thread_pool threads=2 max_queue=1000; + +events { + worker_connections 1024; +} + +http { + datadog_trace_locations off; + + # enabled via ENV var: + # datadog_appsec_enabled on; + # datadog_appsec_waf_timeout 2s; + + datadog_waf_thread_pool_name waf_thread_pool; + + server { + listen 7777; + server_name 0.0.0.0; + + root /builds; + + # `return` directives are not traced + # See, + location ~* /sample_rate/([0-9]+) { + try_files /hello.html =404; + } + + location /mysql { + # can't use `return` + try_files /non_existent =404; + } + + location /healthcheck { + root /builds; + try_files /healthcheck.json =404; + } + + location ~ ^/tag_value/([^/]+)/(\d+) { + datadog_tag appsec.events.system_tests_appsec_event.value $1; + proxy_pass http://127.0.0.1:7778/content?status=$2&value=Value+tagged; + } + + location =/status { + proxy_pass http://127.0.0.1:7778/content?status=$arg_code&value=something; + } + + location /headers { + add_header Content-Language "en-US" always; + add_header Content-Type "text/plain" always; + add_header Content-Length "0" always; + try_files /hello.html =404; + } + + location =/ { + try_files /hello.html =404; + } + + location =/params { + try_files /hello.html =404; + } + + location /waf { + proxy_pass http://127.0.0.1:7778/content?status=200&value=Hello,+World!; + } + + location =/read_file { + limit_except GET { + deny all; + } + if ($arg_file = "") { + return 400 "Bad Request: 'file' parameter is required."; + } + + proxy_pass http://127.0.0.1:7778; + } + + location ~ ^/sample_rate_route/\d+$ { + rewrite ^/sample_rate_route/\d+$ /content?status=200&value=Hello+world! break; + proxy_pass http://127.0.0.1:7778; + } + + location /path_that_doesn't_exists { + try_files /non_existent =404; + } + + location / { + try_files $uri =404; + } + } +} diff --git a/utils/build/docker/cpp/nginx/nginx.conf b/utils/build/docker/cpp/nginx/nginx.conf index cae6c69715..bedb6d6d4b 100644 --- a/utils/build/docker/cpp/nginx/nginx.conf +++ b/utils/build/docker/cpp/nginx/nginx.conf @@ -18,11 +18,32 @@ http { try_files /hello.html =404; } + location /mysql { + # can't use `return` + try_files /non_existent =404; + } + location /healthcheck { root /builds; try_files /healthcheck.json =404; } + location =/read_file { + limit_except GET { + deny all; + } + if ($arg_file = "") { + return 400 "Bad Request: 'file' parameter is required."; + } + + proxy_pass http://127.0.0.1:7778; + } + + location ~ ^/sample_rate_route/\d+$ { + rewrite ^/sample_rate_route/\d+$ /content?status=200&value=Hello+world! break; + proxy_pass http://127.0.0.1:7778; + } + location / { root /builds; try_files /hello.html =404; diff --git a/utils/build/docker/php/common/install_ddtrace.sh b/utils/build/docker/php/common/install_ddtrace.sh index cb59347547..626b69d1af 100755 --- a/utils/build/docker/php/common/install_ddtrace.sh +++ b/utils/build/docker/php/common/install_ddtrace.sh @@ -44,6 +44,7 @@ fi #Extract version info php -d error_reporting='' -d extension=ddtrace.so -d extension=ddappsec.so -r 'echo phpversion("ddtrace");' > \ /binaries/SYSTEM_TESTS_LIBRARY_VERSION +#echo 1.2.0+f70ce41e4dfa0342fe5fac418fa1d8bffb02a52d > /binaries/SYSTEM_TESTS_LIBRARY_VERSION find /opt -name ddappsec-helper -exec ln -s '{}' /usr/local/bin/ \; diff --git a/utils/interfaces/_logs.py b/utils/interfaces/_logs.py index cff94e38db..e2df4893ea 100644 --- a/utils/interfaces/_logs.py +++ b/utils/interfaces/_logs.py @@ -159,7 +159,15 @@ def init_patterns(self, library): timestamp = p("timestamp", r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d") klass = p("klass", r"[\w\.$\[\]/]+") self._parsers.append(re.compile(rf"^{timestamp} +{level} \d -+ \[ *{thread}\] +{klass} *: *{message}")) - + elif context.library == "cpp": + # TODO: should be limited to nginx, but context.weblog_variant is unset at this point + # messages are like '2024/04/30 09:31:49 [info] 25#25: AppSec loaded 154 rules from file embedded ruleset' + self._new_log_line_pattern = re.compile(r".") + timestamp = p("timestamp", r"\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}") + level = p("level", r"\w+") + thread = p("thread", r"\d+#\d+") + message = p("message", r".+") + self._parsers.append(re.compile(rf"^{timestamp} \[{level}\] {thread}: {message}")) elif library == "dotnet": self._new_log_line_pattern = re.compile(r"^\s*(info|debug|error)") elif library == "php": @@ -183,7 +191,7 @@ def _clean_line(self, line): return line def _get_standardized_level(self, level): - if self.library == "php": + if context.library == "php" or context.weblog_variant == "nginx": return level.upper() return super()._get_standardized_level(level) diff --git a/utils/scripts/get-image-list.py b/utils/scripts/get-image-list.py index 39a26a24c3..1f80747e05 100644 --- a/utils/scripts/get-image-list.py +++ b/utils/scripts/get-image-list.py @@ -29,6 +29,6 @@ images.sort() - compose_data = {"services": {re.sub(r"[/:\.]", "-", image): {"image": image} for image in images}} + compose_data = {"services": {re.sub(r"[/:\.@]", "-", image): {"image": image} for image in images}} print(yaml.dump(compose_data, default_flow_style=False)) diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index e9acd68826..a139282198 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. # This product includes software developed at Datadog (https://www.datadoghq.com/). @@ -20,6 +20,7 @@ # * Python: Direct from github source # * Ruby: Direct from github source # * WAF: Direct from github source, but not working, as this repo is now private +# * nginx: From circle ci ########################################################################################## set -eu @@ -89,20 +90,23 @@ get_circleci_artifact() { ARTIFACTS=$(curl --silent https://circleci.com/api/v2/project/$SLUG/$JOB_NUMBER/artifacts -H "Circle-Token: $CIRCLECI_TOKEN") QUERY=".items[] | select(.path | test(\"$ARTIFACT_PATTERN\"))" - ARTIFACT_URL=$(echo $ARTIFACTS | jq -r "$QUERY | .url") + readarray -t ARTIFACT_URLS < <(jq -r "$QUERY | .url" <<< "$ARTIFACTS") - if [ -z "$ARTIFACT_URL" ]; then + if [[ ${#ARTIFACT_URLS[@]} -eq 0 ]]; then echo "Oooops, I did not found any artifact that satisfy this pattern: $ARTIFACT_PATTERN. Here is the list:" echo $ARTIFACTS | jq -r ".items[] | .path" exit 1 fi - - ARTIFACT_NAME=$(echo $ARTIFACTS | jq -r "$QUERY | .path" | sed -E 's/libs\///') - echo "Artifact URL: $ARTIFACT_URL" - echo "Artifact name: $ARTIFACT_NAME" - echo "Downloading artifact..." - - curl --silent -L $ARTIFACT_URL --output $ARTIFACT_NAME + echo "${ARTIFACT_URLS[@]}" + + local aurl= aname= + for aurl in "${ARTIFACT_URLS[@]}"; do + aname="${aurl##*/}" + echo "Artifact URL: $aurl" + echo "Artifact Name: $aname" + echo "Downloading artifact..." + curl --silent -L "$aurl" --output "$aname" + done } get_github_action_artifact() { @@ -207,6 +211,10 @@ elif [ "$TARGET" = "cpp" ]; then # Not handled for now for system-tests. this handles artifact for parametric echo "Using https://github.com/DataDog/dd-trace-cpp@main" echo "https://github.com/DataDog/dd-trace-cpp@main" > cpp-load-from-git +elif [ "$TARGET" = "nginx" ]; then + assert_version_is_dev + ARCH=$(arch | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/) + get_circleci_artifact gh/DataDog/nginx-datadog build-and-test "build 1.25.4 on ${ARCH} WAF ON" 'ngx_http_datadog_module\\.so.*' elif [ "$TARGET" = "agent" ]; then assert_version_is_dev echo "datadog/agent-dev:master-py3" > agent-image