appsec: meta_struct Security Events (#3098)

Signed-off-by: Eliott Bouhana <[email protected]>

.github/workflows/system-tests.yml

@@ -81,6 +81,8 @@ jobs:
             scenario: APPSEC_LOW_WAF_TIMEOUT
           - weblog-variant: net-http
             scenario: APPSEC_STANDALONE
+          - weblog-variant: net-http
+            scenario: APPSEC_META_STRUCT_DISABLED
           - weblog-variant: net-http
             scenario: APPSEC_CUSTOM_OBFUSCATION
           # APM scenarios requiring specific environment settings

ddtrace/tracer/option.go

@@ -722,6 +722,9 @@ type agentFeatures struct {
 
 	// defaultEnv is the trace-agent's default env, used for stats calculation if no env override is present
 	defaultEnv string
+
+	// metaStructAvailable reports whether the trace-agent can receive spans with the `meta_struct` field.
+	metaStructAvailable bool
 }
 
 // HasFlag reports whether the agent has set the feat feature flag.
@@ -748,11 +751,12 @@ func loadAgentFeatures(agentDisabled bool, agentURL *url.URL, httpClient *http.C
 	}
 	defer resp.Body.Close()
 	type infoResponse struct {
-		Endpoints     []string `json:"endpoints"`
-		ClientDropP0s bool     `json:"client_drop_p0s"`
-		FeatureFlags  []string `json:"feature_flags"`
-		PeerTags      []string `json:"peer_tags"`
-		Config        struct {
+		Endpoints      []string `json:"endpoints"`
+		ClientDropP0s  bool     `json:"client_drop_p0s"`
+		FeatureFlags   []string `json:"feature_flags"`
+		PeerTags       []string `json:"peer_tags"`
+		SpanMetaStruct bool     `json:"span_meta_structs"`
+		Config         struct {
 			StatsdPort int `json:"statsd_port"`
 		} `json:"config"`
 	}
@@ -762,8 +766,10 @@ func loadAgentFeatures(agentDisabled bool, agentURL *url.URL, httpClient *http.C
 		log.Error("Decoding features: %v", err)
 		return
 	}
+
 	features.DropP0s = info.ClientDropP0s
 	features.StatsdPort = info.Config.StatsdPort
+	features.metaStructAvailable = info.SpanMetaStruct
 	for _, endpoint := range info.Endpoints {
 		switch endpoint {
 		case "/v0.6/stats":

ddtrace/tracer/tracer.go

@@ -195,7 +195,7 @@ func Start(opts ...StartOption) {
 	// client is appropriately configured.
 	appsecopts := make([]appsecConfig.StartOption, 0, len(t.config.appsecStartOptions)+1)
 	appsecopts = append(appsecopts, t.config.appsecStartOptions...)
-	appsecopts = append(appsecopts, appsecConfig.WithRCConfig(cfg))
+	appsecopts = append(appsecopts, appsecConfig.WithRCConfig(cfg), appsecConfig.WithMetaStructAvailable(t.config.agent.metaStructAvailable))
 	appsec.Start(appsecopts...)
 
 	if t.config.logStartup {

internal/appsec/config/config.go

@@ -58,6 +58,8 @@ type StartConfig struct {
 	// IsEnabled is a function that determines whether AppSec is enabled or not. When unset, the
 	// default [IsEnabled] function is used.
 	EnablementMode func() (EnablementMode, Origin, error)
+	// MetaStructAvailable is true if meta struct is supported by the trace agent.
+	MetaStructAvailable bool
 }
 
 type EnablementMode int8
@@ -123,6 +125,12 @@ func WithRCConfig(cfg remoteconfig.ClientConfig) StartOption {
 	}
 }
 
+func WithMetaStructAvailable(available bool) StartOption {
+	return func(c *StartConfig) {
+		c.MetaStructAvailable = available
+	}
+}
+
 // Config is the AppSec configuration.
 type Config struct {
 	// rules loaded via the env var DD_APPSEC_RULES. When not set, the builtin rules will be used
@@ -141,6 +149,8 @@ type Config struct {
 	RASP bool
 	// SupportedAddresses are the addresses that the AppSec listener will bind to.
 	SupportedAddresses AddressSet
+	// MetaStructAvailable is true if meta struct is supported by the trace agent.
+	MetaStructAvailable bool
 }
 
 // AddressSet is a set of WAF addresses.
@@ -201,12 +211,13 @@ func (c *StartConfig) NewConfig() (*Config, error) {
 	}
 
 	return &Config{
-		RulesManager:   r,
-		WAFTimeout:     internal.WAFTimeoutFromEnv(),
-		TraceRateLimit: int64(internal.RateLimitFromEnv()),
-		Obfuscator:     internal.NewObfuscatorConfig(),
-		APISec:         internal.NewAPISecConfig(),
-		RASP:           internal.RASPEnabled(),
-		RC:             c.RC,
+		RulesManager:        r,
+		WAFTimeout:          internal.WAFTimeoutFromEnv(),
+		TraceRateLimit:      int64(internal.RateLimitFromEnv()),
+		Obfuscator:          internal.NewObfuscatorConfig(),
+		APISec:              internal.NewAPISecConfig(),
+		RASP:                internal.RASPEnabled(),
+		RC:                  c.RC,
+		MetaStructAvailable: c.MetaStructAvailable,
 	}, nil
 }

internal/appsec/listener/waf/waf.go

@@ -13,6 +13,7 @@ import (
 	"github.com/DataDog/appsec-internal-go/limiter"
 	wafv3 "github.com/DataDog/go-libddwaf/v3"
 
+	"gopkg.in/DataDog/dd-trace-go.v1/internal"
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/listener"
 
 	"gopkg.in/DataDog/dd-trace-go.v1/appsec/events"
@@ -30,6 +31,10 @@ type Feature struct {
 	handle          *wafv3.Handle
 	supportedAddrs  config.AddressSet
 	reportRulesTags sync.Once
+
+	// Determine if we can use [internal.MetaStructValue] to delegate the WAF events serialization to the trace writer
+	// or if we have to use the [SerializableTag] method to serialize the events
+	metaStructAvailable bool
 }
 
 func NewWAFFeature(cfg *config.Config, rootOp dyngo.Operation) (listener.Feature, error) {
@@ -55,10 +60,11 @@ func NewWAFFeature(cfg *config.Config, rootOp dyngo.Operation) (listener.Feature
 	tokenTicker.Start()
 
 	feature := &Feature{
-		handle:         newHandle,
-		timeout:        cfg.WAFTimeout,
-		limiter:        tokenTicker,
-		supportedAddrs: cfg.SupportedAddresses,
+		handle:              newHandle,
+		timeout:             cfg.WAFTimeout,
+		limiter:             tokenTicker,
+		supportedAddrs:      cfg.SupportedAddresses,
+		metaStructAvailable: cfg.MetaStructAvailable,
 	}
 
 	dyngo.On(rootOp, feature.onStart)
@@ -116,7 +122,12 @@ func (waf *Feature) onFinish(op *waf.ContextOperation, _ waf.ContextRes) {
 
 	AddWAFMonitoringTags(op, waf.handle.Diagnostics().Version, ctx.Stats().Metrics())
 	if wafEvents := op.Events(); len(wafEvents) > 0 {
-		op.SetSerializableTag("_dd.appsec.json", map[string][]any{"triggers": op.Events()})
+		tagValue := map[string][]any{"triggers": wafEvents}
+		if waf.metaStructAvailable {
+			op.SetTag("appsec", internal.MetaStructValue{Value: tagValue})
+		} else {
+			op.SetSerializableTag("_dd.appsec.json", tagValue)
+		}
 	}
 	op.SetSerializableTags(op.Derivatives())
 	if stacks := op.StackTraces(); len(stacks) > 0 {

internal/appsec/waf_test.go

@@ -31,6 +31,7 @@ import (
 	httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+	globalinternal "gopkg.in/DataDog/dd-trace-go.v1/internal"
 
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec"
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/config"
@@ -863,6 +864,81 @@ func TestSuspiciousAttackerBlocking(t *testing.T) {
 	}
 }
 
+func TestWafEventsInMetaStruct(t *testing.T) {
+	t.Setenv("DD_APPSEC_RULES", "testdata/user_rules.json")
+	appsec.Start(config.WithMetaStructAvailable(true))
+	defer appsec.Stop()
+
+	if !appsec.Enabled() {
+		t.Skip("appsec disabled")
+	}
+
+	// Start and trace an HTTP server
+	mux := httptrace.NewServeMux()
+	mux.HandleFunc("/hello", func(w http.ResponseWriter, _ *http.Request) {
+		w.Write([]byte("Hello World!\n"))
+	})
+	mux.HandleFunc("/response-header", func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("match-response-header", "match-response-header")
+		w.WriteHeader(204)
+	})
+
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	for _, tc := range []struct {
+		name string
+		url  string
+		rule string
+	}{
+		{
+			name: "custom-001",
+			url:  "/hello",
+			rule: "custom-001",
+		},
+		{
+			name: "custom-action",
+			url:  "/hello?match=match-request-query",
+			rule: "query-002",
+		},
+		{
+			name: "response-headers",
+			url:  "/response-header",
+			rule: "headers-003",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			mt := mocktracer.Start()
+			defer mt.Stop()
+
+			req, err := http.NewRequest("GET", srv.URL+tc.url, nil)
+			require.NoError(t, err)
+
+			res, err := srv.Client().Do(req)
+			require.NoError(t, err)
+			defer res.Body.Close()
+
+			spans := mt.FinishedSpans()
+			require.Len(t, spans, 1)
+
+			tag := spans[0].Tag("appsec")
+			require.IsType(t, globalinternal.MetaStructValue{}, tag)
+
+			events := tag.(globalinternal.MetaStructValue).Value
+			require.IsType(t, map[string][]any{}, events)
+
+			triggers := events.(map[string][]any)["triggers"]
+			ids := make([]string, 0, len(triggers))
+			for _, trigger := range triggers {
+				ids = append(ids, trigger.(map[string]any)["rule"].(map[string]any)["id"].(string))
+			}
+
+			require.Contains(t, ids, tc.rule)
+		})
+	}
+
+}
+
 // BenchmarkSampleWAFContext benchmarks the creation of a WAF context and running the WAF on a request/response pair
 // This is a basic sample of what could happen in a real-world scenario.
 func BenchmarkSampleWAFContext(b *testing.B) {