Apply configuration of SnippetsFilters to NGINX (#2604)

Problem: As a user of NGF, I want my SnippetsFilters configuration
applied to NGF's data plane, so that I can leverage NGINX features
not yet available in NGF.

Solution: Apply configuration of valid SnippetsFilters referenced in
HTTPRoutes and GRPCRoutes to the appropriate contexts in the
NGINX config. If the SnippetsFilter referenced is invalid
(wrong group or kind), the routing rule is not configured.
If the SnippetsFilter cannot be resolved, the routing rule is configured,
but the route will return a 500.

deploy/snippets-filters-nginx-plus/deploy.yaml

@@ -257,8 +257,8 @@ spec:
           name: nginx-conf
         - mountPath: /etc/nginx/stream-conf.d
           name: nginx-stream-conf
-        - mountPath: /etc/nginx/module-includes
-          name: module-includes
+        - mountPath: /etc/nginx/main-includes
+          name: nginx-main-includes
         - mountPath: /etc/nginx/secrets
           name: nginx-secrets
         - mountPath: /var/run/nginx
@@ -289,8 +289,8 @@ spec:
           name: nginx-conf
         - mountPath: /etc/nginx/stream-conf.d
           name: nginx-stream-conf
-        - mountPath: /etc/nginx/module-includes
-          name: module-includes
+        - mountPath: /etc/nginx/main-includes
+          name: nginx-main-includes
         - mountPath: /etc/nginx/secrets
           name: nginx-secrets
         - mountPath: /var/run/nginx
@@ -311,7 +311,7 @@ spec:
       - emptyDir: {}
         name: nginx-stream-conf
       - emptyDir: {}
-        name: module-includes
+        name: nginx-main-includes
       - emptyDir: {}
         name: nginx-secrets
       - emptyDir: {}

deploy/snippets-filters/deploy.yaml

@@ -248,8 +248,8 @@ spec:
           name: nginx-conf
         - mountPath: /etc/nginx/stream-conf.d
           name: nginx-stream-conf
-        - mountPath: /etc/nginx/module-includes
-          name: module-includes
+        - mountPath: /etc/nginx/main-includes
+          name: nginx-main-includes
         - mountPath: /etc/nginx/secrets
           name: nginx-secrets
         - mountPath: /var/run/nginx
@@ -280,8 +280,8 @@ spec:
           name: nginx-conf
         - mountPath: /etc/nginx/stream-conf.d
           name: nginx-stream-conf
-        - mountPath: /etc/nginx/module-includes
-          name: module-includes
+        - mountPath: /etc/nginx/main-includes
+          name: nginx-main-includes
         - mountPath: /etc/nginx/secrets
           name: nginx-secrets
         - mountPath: /var/run/nginx
@@ -302,7 +302,7 @@ spec:
       - emptyDir: {}
         name: nginx-stream-conf
       - emptyDir: {}
-        name: module-includes
+        name: nginx-main-includes
       - emptyDir: {}
         name: nginx-secrets
       - emptyDir: {}

examples/snippets-filter/example.yaml

@@ -0,0 +1,86 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coffee
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: coffee
+  template:
+    metadata:
+      labels:
+        app: coffee
+    spec:
+      containers:
+        - name: coffee
+          image: nginxdemos/nginx-hello:plain-text
+          ports:
+            - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: coffee
+spec:
+  ports:
+    - port: 80
+      targetPort: 8080
+      protocol: TCP
+      name: http
+  selector:
+    app: coffee
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: gateway
+spec:
+  gatewayClassName: nginx
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      hostname: "*.example.com"
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: coffee
+spec:
+  parentRefs:
+    - name: gateway
+      sectionName: http
+  hostnames:
+    - "cafe.example.com"
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /coffee
+      filters:
+        - type: ExtensionRef
+          extensionRef:
+            group: gateway.nginx.org
+            kind: SnippetsFilter
+            name: test-all-contexts
+      backendRefs:
+        - name: coffee
+          port: 80
+---
+apiVersion: gateway.nginx.org/v1alpha1
+kind: SnippetsFilter
+metadata:
+  name: test-all-contexts
+spec:
+  snippets:
+    - context: main
+      value: worker_shutdown_timeout 120s;
+    - context: http
+      value: aio on;
+    - context: http.server
+      value: auth_delay 10s;
+    - context: http.server.location
+      value: |
+        allow 10.0.0.0/8;
+        deny all;

internal/framework/kinds/kinds.go

@@ -31,6 +31,8 @@ const (
 	ObservabilityPolicy = "ObservabilityPolicy"
 	// NginxProxy is the NginxProxy kind.
 	NginxProxy = "NginxProxy"
+	// SnippetsFilter is the SnippetsFilter kind.
+	SnippetsFilter = "SnippetsFilter"
 )
 
 // MustExtractGVK is a function that extracts the GroupVersionKind (GVK) of a client.object.

internal/mode/static/nginx/config/base_http_config.go

@@ -4,16 +4,31 @@ import (
 	gotemplate "text/template"
 
 	"github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers"
+	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/shared"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane"
 )
 
 var baseHTTPTemplate = gotemplate.Must(gotemplate.New("baseHttp").Parse(baseHTTPTemplateText))
 
+type httpConfig struct {
+	Includes []shared.Include
+	HTTP2    bool
+}
+
 func executeBaseHTTPConfig(conf dataplane.Configuration) []executeResult {
-	result := executeResult{
-		dest: httpConfigFile,
-		data: helpers.MustExecuteTemplate(baseHTTPTemplate, conf.BaseHTTPConfig),
+	includes := createIncludesFromSnippets(conf.BaseHTTPConfig.Snippets)
+
+	hc := httpConfig{
+		HTTP2:    conf.BaseHTTPConfig.HTTP2,
+		Includes: includes,
 	}
 
-	return []executeResult{result}
+	results := make([]executeResult, 0, len(includes)+1)
+	results = append(results, executeResult{
+		dest: httpConfigFile,
+		data: helpers.MustExecuteTemplate(baseHTTPTemplate, hc),
+	})
+	results = append(results, createIncludeExecuteResults(includes)...)
+
+	return results
 }

internal/mode/static/nginx/config/base_http_config_template.go

@@ -23,4 +23,8 @@ map $http_upgrade $connection_upgrade {
 map $request_uri $request_uri_path {
   "~^(?P<path>[^?]*)(\?.*)?$"  $path;
 }
+
+{{ range $i := .Includes -}}
+include {{ $i.Name }};
+{{ end -}}
 `

internal/mode/static/nginx/config/base_http_config_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"sort"
 	"strings"
 	"testing"
 
@@ -9,7 +10,7 @@ import (
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane"
 )
 
-func TestExecuteBaseHttp(t *testing.T) {
+func TestExecuteBaseHttp_HTTP2(t *testing.T) {
 	t.Parallel()
 	confOn := dataplane.Configuration{
 		BaseHTTPConfig: dataplane.BaseHTTPConfig{
@@ -56,3 +57,53 @@ func TestExecuteBaseHttp(t *testing.T) {
 		})
 	}
 }
+
+func TestExecuteBaseHttp_Snippets(t *testing.T) {
+	t.Parallel()
+
+	conf := dataplane.Configuration{
+		BaseHTTPConfig: dataplane.BaseHTTPConfig{
+			Snippets: []dataplane.Snippet{
+				{
+					Name:     "snippet1",
+					Contents: "contents1",
+				},
+				{
+					Name:     "snippet2",
+					Contents: "contents2",
+				},
+			},
+		},
+	}
+
+	g := NewWithT(t)
+
+	res := executeBaseHTTPConfig(conf)
+	g.Expect(res).To(HaveLen(3))
+
+	sort.Slice(
+		res, func(i, j int) bool {
+			return res[i].dest < res[j].dest
+		},
+	)
+
+	/*
+		Order of files:
+		/etc/nginx/conf.d/http.conf
+		/etc/nginx/includes/snippet1.conf
+		/etc/nginx/includes/snippet2.conf
+	*/
+
+	httpRes := string(res[0].data)
+	g.Expect(httpRes).To(ContainSubstring("map $http_host $gw_api_compliant_host {"))
+	g.Expect(httpRes).To(ContainSubstring("map $http_upgrade $connection_upgrade {"))
+	g.Expect(httpRes).To(ContainSubstring("map $request_uri $request_uri_path {"))
+	g.Expect(httpRes).To(ContainSubstring("include /etc/nginx/includes/snippet1.conf;"))
+	g.Expect(httpRes).To(ContainSubstring("include /etc/nginx/includes/snippet2.conf;"))
+
+	snippet1IncludeRes := string(res[1].data)
+	g.Expect(snippet1IncludeRes).To(ContainSubstring("contents1"))
+
+	snippet2IncludeRes := string(res[2].data)
+	g.Expect(snippet2IncludeRes).To(ContainSubstring("contents2"))
+}

internal/mode/static/nginx/config/generator.go

@@ -24,6 +24,7 @@ const (
 	streamFolder = configFolder + "/stream-conf.d"
 
 	// mainIncludesFolder is the folder where NGINX main context configuration files are stored.
+	// For example, these files include load_module directives and snippets that target the main context.
 	mainIncludesFolder = configFolder + "/main-includes"
 
 	// secretsFolder is the folder where secrets (like TLS certs/keys) are stored.
@@ -61,9 +62,7 @@ type Generator interface {
 
 // GeneratorImpl is an implementation of Generator.
 //
-// It generates files to be written to the following locations, which must exist and available for writing:
-// - httpFolder, for HTTP configuration files.
-// - secretsFolder, for secrets.
+// It generates files to be written to the ConfigFolders locations, which must exist and available for writing.
 //
 // It also expects that the main NGINX configuration file nginx.conf is located in configFolder and nginx.conf
 // includes (https://nginx.org/en/docs/ngx_core_module.html#include) the files from httpFolder.
@@ -100,7 +99,7 @@ func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File {
 		observability.NewGenerator(conf.Telemetry),
 	)
 
-	files = append(files, g.runExecuteFuncs(conf, policyGenerator)...)
+	files = append(files, g.executeConfigTemplates(conf, policyGenerator)...)
 
 	for id, bundle := range conf.CertBundles {
 		files = append(files, generateCertBundle(id, bundle))
@@ -109,36 +108,7 @@ func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File {
 	return files
 }
 
-func generatePEM(id dataplane.SSLKeyPairID, cert []byte, key []byte) file.File {
-	c := make([]byte, 0, len(cert)+len(key)+1)
-	c = append(c, cert...)
-	c = append(c, '\n')
-	c = append(c, key...)
-
-	return file.File{
-		Content: c,
-		Path:    generatePEMFileName(id),
-		Type:    file.TypeSecret,
-	}
-}
-
-func generatePEMFileName(id dataplane.SSLKeyPairID) string {
-	return filepath.Join(secretsFolder, string(id)+".pem")
-}
-
-func generateCertBundle(id dataplane.CertBundleID, cert []byte) file.File {
-	return file.File{
-		Content: cert,
-		Path:    generateCertBundleFileName(id),
-		Type:    file.TypeRegular,
-	}
-}
-
-func generateCertBundleFileName(id dataplane.CertBundleID) string {
-	return filepath.Join(secretsFolder, string(id)+".crt")
-}
-
-func (g GeneratorImpl) runExecuteFuncs(
+func (g GeneratorImpl) executeConfigTemplates(
 	conf dataplane.Configuration,
 	generator policies.Generator,
 ) []file.File {
@@ -165,6 +135,7 @@ func (g GeneratorImpl) runExecuteFuncs(
 
 func (g GeneratorImpl) getExecuteFuncs(generator policies.Generator) []executeFunc {
 	return []executeFunc{
+		executeMainConfig,
 		executeBaseHTTPConfig,
 		g.newExecuteServersFunc(generator),
 		g.executeUpstreams,
@@ -178,3 +149,32 @@ func (g GeneratorImpl) getExecuteFuncs(generator policies.Generator) []executeFu
 		executeMainIncludesConfig,
 	}
 }
+
+func generatePEM(id dataplane.SSLKeyPairID, cert []byte, key []byte) file.File {
+	c := make([]byte, 0, len(cert)+len(key)+1)
+	c = append(c, cert...)
+	c = append(c, '\n')
+	c = append(c, key...)
+
+	return file.File{
+		Content: c,
+		Path:    generatePEMFileName(id),
+		Type:    file.TypeSecret,
+	}
+}
+
+func generatePEMFileName(id dataplane.SSLKeyPairID) string {
+	return filepath.Join(secretsFolder, string(id)+".pem")
+}
+
+func generateCertBundle(id dataplane.CertBundleID, cert []byte) file.File {
+	return file.File{
+		Content: cert,
+		Path:    generateCertBundleFileName(id),
+		Type:    file.TypeRegular,
+	}
+}
+
+func generateCertBundleFileName(id dataplane.CertBundleID) string {
+	return filepath.Join(secretsFolder, string(id)+".crt")
+}