diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index c8a0b287f7..3f6bcec808 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -81,6 +81,8 @@ jobs: scenario: APPSEC_LOW_WAF_TIMEOUT - weblog-variant: net-http scenario: APPSEC_STANDALONE + - weblog-variant: net-http + scenario: APPSEC_META_STRUCT_DISABLED - weblog-variant: net-http scenario: APPSEC_CUSTOM_OBFUSCATION # APM scenarios requiring specific environment settings diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index c1ad01edf9..49e2845f9c 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -722,6 +722,9 @@ type agentFeatures struct { // defaultEnv is the trace-agent's default env, used for stats calculation if no env override is present defaultEnv string + + // metaStructAvailable reports whether the trace-agent can receive spans with the `meta_struct` field. + metaStructAvailable bool } // HasFlag reports whether the agent has set the feat feature flag. @@ -748,11 +751,12 @@ func loadAgentFeatures(agentDisabled bool, agentURL *url.URL, httpClient *http.C } defer resp.Body.Close() type infoResponse struct { - Endpoints []string `json:"endpoints"` - ClientDropP0s bool `json:"client_drop_p0s"` - FeatureFlags []string `json:"feature_flags"` - PeerTags []string `json:"peer_tags"` - Config struct { + Endpoints []string `json:"endpoints"` + ClientDropP0s bool `json:"client_drop_p0s"` + FeatureFlags []string `json:"feature_flags"` + PeerTags []string `json:"peer_tags"` + SpanMetaStruct bool `json:"span_meta_structs"` + Config struct { StatsdPort int `json:"statsd_port"` } `json:"config"` } @@ -762,8 +766,10 @@ func loadAgentFeatures(agentDisabled bool, agentURL *url.URL, httpClient *http.C log.Error("Decoding features: %v", err) return } + features.DropP0s = info.ClientDropP0s features.StatsdPort = info.Config.StatsdPort + features.metaStructAvailable = info.SpanMetaStruct for _, endpoint := range info.Endpoints { switch endpoint { case "/v0.6/stats": diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 25e08b7cb3..ce1f5b0f23 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -195,7 +195,7 @@ func Start(opts ...StartOption) { // client is appropriately configured. appsecopts := make([]appsecConfig.StartOption, 0, len(t.config.appsecStartOptions)+1) appsecopts = append(appsecopts, t.config.appsecStartOptions...) - appsecopts = append(appsecopts, appsecConfig.WithRCConfig(cfg)) + appsecopts = append(appsecopts, appsecConfig.WithRCConfig(cfg), appsecConfig.WithMetaStructAvailable(t.config.agent.metaStructAvailable)) appsec.Start(appsecopts...) if t.config.logStartup { diff --git a/internal/appsec/config/config.go b/internal/appsec/config/config.go index 7d334017f2..fe34fa4332 100644 --- a/internal/appsec/config/config.go +++ b/internal/appsec/config/config.go @@ -58,6 +58,8 @@ type StartConfig struct { // IsEnabled is a function that determines whether AppSec is enabled or not. When unset, the // default [IsEnabled] function is used. EnablementMode func() (EnablementMode, Origin, error) + // MetaStructAvailable is true if meta struct is supported by the trace agent. + MetaStructAvailable bool } type EnablementMode int8 @@ -123,6 +125,12 @@ func WithRCConfig(cfg remoteconfig.ClientConfig) StartOption { } } +func WithMetaStructAvailable(available bool) StartOption { + return func(c *StartConfig) { + c.MetaStructAvailable = available + } +} + // Config is the AppSec configuration. type Config struct { // rules loaded via the env var DD_APPSEC_RULES. When not set, the builtin rules will be used @@ -141,6 +149,8 @@ type Config struct { RASP bool // SupportedAddresses are the addresses that the AppSec listener will bind to. SupportedAddresses AddressSet + // MetaStructAvailable is true if meta struct is supported by the trace agent. + MetaStructAvailable bool } // AddressSet is a set of WAF addresses. @@ -201,12 +211,13 @@ func (c *StartConfig) NewConfig() (*Config, error) { } return &Config{ - RulesManager: r, - WAFTimeout: internal.WAFTimeoutFromEnv(), - TraceRateLimit: int64(internal.RateLimitFromEnv()), - Obfuscator: internal.NewObfuscatorConfig(), - APISec: internal.NewAPISecConfig(), - RASP: internal.RASPEnabled(), - RC: c.RC, + RulesManager: r, + WAFTimeout: internal.WAFTimeoutFromEnv(), + TraceRateLimit: int64(internal.RateLimitFromEnv()), + Obfuscator: internal.NewObfuscatorConfig(), + APISec: internal.NewAPISecConfig(), + RASP: internal.RASPEnabled(), + RC: c.RC, + MetaStructAvailable: c.MetaStructAvailable, }, nil } diff --git a/internal/appsec/listener/waf/waf.go b/internal/appsec/listener/waf/waf.go index 7607f3245b..30d5ce83d8 100644 --- a/internal/appsec/listener/waf/waf.go +++ b/internal/appsec/listener/waf/waf.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/appsec-internal-go/limiter" wafv3 "github.com/DataDog/go-libddwaf/v3" + "gopkg.in/DataDog/dd-trace-go.v1/internal" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener" "gopkg.in/DataDog/dd-trace-go.v1/appsec/events" @@ -30,6 +31,10 @@ type Feature struct { handle *wafv3.Handle supportedAddrs config.AddressSet reportRulesTags sync.Once + + // Determine if we can use [internal.MetaStructValue] to delegate the WAF events serialization to the trace writer + // or if we have to use the [SerializableTag] method to serialize the events + metaStructAvailable bool } func NewWAFFeature(cfg *config.Config, rootOp dyngo.Operation) (listener.Feature, error) { @@ -55,10 +60,11 @@ func NewWAFFeature(cfg *config.Config, rootOp dyngo.Operation) (listener.Feature tokenTicker.Start() feature := &Feature{ - handle: newHandle, - timeout: cfg.WAFTimeout, - limiter: tokenTicker, - supportedAddrs: cfg.SupportedAddresses, + handle: newHandle, + timeout: cfg.WAFTimeout, + limiter: tokenTicker, + supportedAddrs: cfg.SupportedAddresses, + metaStructAvailable: cfg.MetaStructAvailable, } dyngo.On(rootOp, feature.onStart) @@ -116,7 +122,12 @@ func (waf *Feature) onFinish(op *waf.ContextOperation, _ waf.ContextRes) { AddWAFMonitoringTags(op, waf.handle.Diagnostics().Version, ctx.Stats().Metrics()) if wafEvents := op.Events(); len(wafEvents) > 0 { - op.SetSerializableTag("_dd.appsec.json", map[string][]any{"triggers": op.Events()}) + tagValue := map[string][]any{"triggers": wafEvents} + if waf.metaStructAvailable { + op.SetTag("appsec", internal.MetaStructValue{Value: tagValue}) + } else { + op.SetSerializableTag("_dd.appsec.json", tagValue) + } } op.SetSerializableTags(op.Derivatives()) if stacks := op.StackTraces(); len(stacks) > 0 { diff --git a/internal/appsec/waf_test.go b/internal/appsec/waf_test.go index 347d4f802c..7e3387cd4c 100644 --- a/internal/appsec/waf_test.go +++ b/internal/appsec/waf_test.go @@ -31,6 +31,7 @@ import ( httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + globalinternal "gopkg.in/DataDog/dd-trace-go.v1/internal" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/config" @@ -863,6 +864,81 @@ func TestSuspiciousAttackerBlocking(t *testing.T) { } } +func TestWafEventsInMetaStruct(t *testing.T) { + t.Setenv("DD_APPSEC_RULES", "testdata/user_rules.json") + appsec.Start(config.WithMetaStructAvailable(true)) + defer appsec.Stop() + + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + // Start and trace an HTTP server + mux := httptrace.NewServeMux() + mux.HandleFunc("/hello", func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("Hello World!\n")) + }) + mux.HandleFunc("/response-header", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("match-response-header", "match-response-header") + w.WriteHeader(204) + }) + + srv := httptest.NewServer(mux) + defer srv.Close() + + for _, tc := range []struct { + name string + url string + rule string + }{ + { + name: "custom-001", + url: "/hello", + rule: "custom-001", + }, + { + name: "custom-action", + url: "/hello?match=match-request-query", + rule: "query-002", + }, + { + name: "response-headers", + url: "/response-header", + rule: "headers-003", + }, + } { + t.Run(tc.name, func(t *testing.T) { + mt := mocktracer.Start() + defer mt.Stop() + + req, err := http.NewRequest("GET", srv.URL+tc.url, nil) + require.NoError(t, err) + + res, err := srv.Client().Do(req) + require.NoError(t, err) + defer res.Body.Close() + + spans := mt.FinishedSpans() + require.Len(t, spans, 1) + + tag := spans[0].Tag("appsec") + require.IsType(t, globalinternal.MetaStructValue{}, tag) + + events := tag.(globalinternal.MetaStructValue).Value + require.IsType(t, map[string][]any{}, events) + + triggers := events.(map[string][]any)["triggers"] + ids := make([]string, 0, len(triggers)) + for _, trigger := range triggers { + ids = append(ids, trigger.(map[string]any)["rule"].(map[string]any)["id"].(string)) + } + + require.Contains(t, ids, tc.rule) + }) + } + +} + // BenchmarkSampleWAFContext benchmarks the creation of a WAF context and running the WAF on a request/response pair // This is a basic sample of what could happen in a real-world scenario. func BenchmarkSampleWAFContext(b *testing.B) {