Merge branch 'main' of ssh://github.com/hashicorp/consul into NET-4240

.changelog/18184.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+api: Fix client deserialization errors by marking new Enterprise-only prepared query fields as omit empty
+```

.changelog/18223.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+cli: `consul members` command uses `-filter` expression to filter members based on bexpr.
+```

.changelog/18291.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+api-gateway: fix race condition in proxy config generation when Consul is notified of the bound-api-gateway config entry before it is notified of the api-gateway config entry.
+```

.changelog/18300.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+connect: update supported envoy versions to 1.24.10, 1.25.9, 1.26.4, 1.27.0
+```

.github/workflows/build.yml

@@ -282,7 +282,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: ["386", "amd64", "arm", "arm64"]
+        include:
+          - { arch: "386" }
+          - { arch: "arm" }
+          - { arch: "amd64" }
+          - { arch: "arm64" }
     env:
       repo: ${{github.event.repository.name}}
       version: ${{needs.set-product-version.outputs.product-version}}
@@ -371,7 +375,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: ["386", "amd64", "arm", "arm64"]
+        include:
+          - { arch: "386" }
+          - { arch: "arm" }
+          - { arch: "amd64" }
+          - { arch: "arm64" }
       fail-fast: true
     env:
       version: ${{ needs.set-product-version.outputs.product-version }}
@@ -432,7 +440,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        arch: ["i386", "amd64", "armhf", "arm64"]
+        include:
+          - { arch: "i386" }
+          - { arch: "armhf" }
+          - { arch: "amd64" }
+          - { arch: "arm64" }
       # fail-fast: true
     env:
       version: ${{ needs.set-product-version.outputs.product-version }}
@@ -469,8 +481,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+        include:
+          - { arch: "i386" }
+          - { arch: "x86_64" }
         # TODO(eculver): re-enable when there is a smaller verification container available
-        arch: ["i386", "x86_64"] #, "armv7hl", "aarch64"]
+          # - { arch: "armv7hl" }
+          # - { arch: "aarch64" }
     env:
       version: ${{ needs.set-product-version.outputs.product-version }}
 

.github/workflows/go-tests.yml

@@ -2,13 +2,14 @@ name: go-tests
 
 on:
   pull_request:
-    paths-ignore: 
-      - '.changelog/**'
-      - '.github/ISSUE_TEMPLATE/**'
-      - 'contributing/**'
+    branches-ignore:
+      - stable-website
       - 'docs/**'
       - 'ui/**'
-      - 'website/**'
+      - 'mktg-**' # Digital Team Terraform-generated branches' prefix
+      - 'backport/docs/**'
+      - 'backport/ui/**'
+      - 'backport/mktg-**'
   push:
     branches:
       # Push events on the main branch
@@ -183,18 +184,18 @@ jobs:
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
 
-  dev-build-s390x:
-    if: ${{ endsWith(github.repository, '-enterprise') }}
-    needs:
-    - setup
-    uses: ./.github/workflows/reusable-dev-build.yml
-    with:
-      uploaded-binary-name: 'consul-bin-s390x'
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
-      go-arch: "s390x"
-      repository-name: ${{ github.repository }}
-    secrets:
-      elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+  # dev-build-s390x:
+  #   if: ${{ endsWith(github.repository, '-enterprise') }}
+  #   needs:
+  #   - setup
+  #   uses: ./.github/workflows/reusable-dev-build.yml
+  #   with:
+  #     uploaded-binary-name: 'consul-bin-s390x'
+  #     runs-on: ${{ needs.setup.outputs.compute-xl }}
+  #     go-arch: "s390x"
+  #     repository-name: ${{ github.repository }}
+  #   secrets:
+  #     elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
 
   # dev-build-arm64:
   #   # only run on enterprise because GHA does not have arm64 runners in OSS
@@ -308,26 +309,26 @@ jobs:
       consul-license: ${{secrets.CONSUL_LICENSE}}
       datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
-  go-test-s390x:
-    if: ${{ endsWith(github.repository, '-enterprise') }}
-    needs:
-    - setup
-    - dev-build-s390x
-    uses: ./.github/workflows/reusable-unit.yml
-    with:
-      uploaded-binary-name: 'consul-bin-s390x'
-      directory: .
-      go-test-flags: 'export GO_TEST_FLAGS="-short"'
-      runs-on: ${{ needs.setup.outputs.compute-xl }}
-      repository-name: ${{ github.repository }}
-      go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
-    permissions:
-      id-token: write # NOTE: this permission is explicitly required for Vault auth.
-      contents: read
-    secrets:
-      elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
-      consul-license: ${{secrets.CONSUL_LICENSE}}
-      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
+  # go-test-s390x:
+  #   if: ${{ endsWith(github.repository, '-enterprise') }}
+  #   needs:
+  #   - setup
+  #   - dev-build-s390x
+  #   uses: ./.github/workflows/reusable-unit.yml
+  #   with:
+  #     uploaded-binary-name: 'consul-bin-s390x'
+  #     directory: .
+  #     go-test-flags: 'export GO_TEST_FLAGS="-short"'
+  #     runs-on: ${{ needs.setup.outputs.compute-xl }}
+  #     repository-name: ${{ github.repository }}
+  #     go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+  #   permissions:
+  #     id-token: write # NOTE: this permission is explicitly required for Vault auth.
+  #     contents: read
+  #   secrets:
+  #     elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+  #     consul-license: ${{secrets.CONSUL_LICENSE}}
+  #     datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-envoyextensions:
     needs:
@@ -482,7 +483,7 @@ jobs:
     - go-test-sdk-1-19
     - go-test-sdk-1-20
     - go-test-32bit
-    - go-test-s390x
+    # - go-test-s390x
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
     if: ${{ always() }}
     steps:

.github/workflows/test-integrations-windows.yml

@@ -57,7 +57,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: [ "1.23.10", "1.24.8", "1.25.7", "1.26.2" ]
+        envoy-version: [ "1.24.10", "1.25.9", "1.26.4", "1.27.0" ]
         xds-target: [ "server", "client" ]
     env:
       ENVOY_VERSION: ${{ matrix.envoy-version }}

.github/workflows/test-integrations.yml

@@ -5,13 +5,14 @@ name: test-integrations
 
 on:
   pull_request:
-    paths-ignore:
-      - '.changelog/**'
-      - '.github/ISSUE_TEMPLATE/**'
-      - 'contributing/**'
+    branches-ignore:
+      - stable-website
       - 'docs/**'
       - 'ui/**'
-      - 'website/**'
+      - 'mktg-**' # Digital Team Terraform-generated branch prefix
+      - 'backport/docs/**'
+      - 'backport/ui/**'
+      - 'backport/mktg-**'
 
 env:
   TEST_RESULTS_DIR: /tmp/test-results
@@ -239,7 +240,7 @@ jobs:
           # this is further going to multiplied in envoy-integration tests by the 
           # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
           # multiplied by 8 based on these values:
-          # envoy-version: ["1.23.10", "1.24.8", "1.25.7", "1.26.2"]
+          # envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"]
           # xds-target: ["server", "client"]
           TOTAL_RUNNERS: 4 
           JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
@@ -273,7 +274,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: ["1.23.10", "1.24.8", "1.25.7", "1.26.2"]
+        envoy-version: ["1.24.10", "1.25.9", "1.26.4", "1.27.0"]
         xds-target: ["server", "client"]
         test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
     env:

agent/agent_endpoint.go

@@ -619,6 +619,21 @@ func (s *HTTPHandlers) AgentMembers(resp http.ResponseWriter, req *http.Request)
 		}
 	}
 
+	// filter the members by parsed filter expression
+	var filterExpression string
+	s.parseFilter(req, &filterExpression)
+	if filterExpression != "" {
+		filter, err := bexpr.CreateFilter(filterExpression, nil, members)
+		if err != nil {
+			return nil, err
+		}
+		raw, err := filter.Execute(members)
+		if err != nil {
+			return nil, err
+		}
+		members = raw.([]serf.Member)
+	}
+
 	total := len(members)
 	if err := s.agent.filterMembers(token, &members); err != nil {
 		return nil, err

agent/proxycfg/api_gateway.go

@@ -54,12 +54,6 @@ func (h *handlerAPIGateway) initialize(ctx context.Context) (ConfigSnapshot, err
 		return snap, err
 	}
 
-	// Watch the bound-api-gateway's config entry
-	err = h.subscribeToConfigEntry(ctx, structs.BoundAPIGateway, h.service, h.proxyID.EnterpriseMeta, boundGatewayConfigWatchID)
-	if err != nil {
-		return snap, err
-	}
-
 	snap.APIGateway.Listeners = make(map[string]structs.APIGatewayListener)
 	snap.APIGateway.BoundListeners = make(map[string]structs.BoundAPIGatewayListener)
 	snap.APIGateway.HTTPRoutes = watch.NewMap[structs.ResourceReference, *structs.HTTPRouteConfigEntry]()
@@ -143,10 +137,12 @@ func (h *handlerAPIGateway) handleRootCAUpdate(u UpdateEvent, snap *ConfigSnapsh
 	return nil
 }
 
-// handleGatewayConfigUpdate responds to changes in the watched config entry for a gateway.
-// In particular, we want to make sure that we're subscribing to any attached resources such
-// as routes and certificates. These additional subscriptions will enable us to update the
-// config snapshot appropriately for any route or certificate changes.
+// handleGatewayConfigUpdate responds to changes in the watched config entries for a gateway.
+// Once the base api-gateway config entry has been seen, we store the list of listeners and
+// then subscribe to the corresponding bound-api-gateway config entry. We use the bound-api-gateway
+// config entry to subscribe to any attached resources, including routes and certificates.
+// These additional subscriptions will enable us to update the config snapshot appropriately
+// for any route or certificate changes.
 func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u UpdateEvent, snap *ConfigSnapshot, correlationID string) error {
 	resp, ok := u.Result.(*structs.ConfigEntryResponse)
 	if !ok {
@@ -244,6 +240,12 @@ func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u Upd
 		}
 
 		snap.APIGateway.GatewayConfigLoaded = true
+
+		// Watch the corresponding bound-api-gateway config entry
+		err := h.subscribeToConfigEntry(ctx, structs.BoundAPIGateway, h.service, h.proxyID.EnterpriseMeta, boundGatewayConfigWatchID)
+		if err != nil {
+			return err
+		}
 		break
 	default:
 		return fmt.Errorf("invalid type for config entry: %T", resp.Entry)

api/agent.go

@@ -274,6 +274,8 @@ type MembersOpts struct {
 	// Segment is the LAN segment to show members for. Setting this to the
 	// AllSegments value above will show members in all segments.
 	Segment string
+
+	Filter string
 }
 
 // AgentServiceRegistration is used to register a new service
@@ -790,6 +792,10 @@ func (a *Agent) MembersOpts(opts MembersOpts) ([]*AgentMember, error) {
 		r.params.Set("wan", "1")
 	}
 
+	if opts.Filter != "" {
+		r.params.Set("filter", opts.Filter)
+	}
+
 	_, resp, err := a.c.doRequest(r)
 	if err != nil {
 		return nil, err

api/agent_test.go

@@ -155,6 +155,31 @@ func TestAPI_AgentMembersOpts(t *testing.T) {
 	if len(members) != 2 {
 		t.Fatalf("bad: %v", members)
 	}
+
+	members, err = agent.MembersOpts(MembersOpts{
+		WAN:    true,
+		Filter: `Tags["dc"] == dc2`,
+	})
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	require.Equal(t, 1, len(members))
+
+	members, err = agent.MembersOpts(MembersOpts{
+		WAN:    true,
+		Filter: `Tags["dc"] == "not-Exist"`,
+	})
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	require.Equal(t, 0, len(members))
+
+	_, err = agent.MembersOpts(MembersOpts{
+		WAN:    true,
+		Filter: `Tags["dc"] == invalid-bexpr-value`,
+	})
+	require.ErrorContains(t, err, "Failed to create boolean expression evaluator")
 }
 
 func TestAPI_AgentMembers(t *testing.T) {

api/api.go

@@ -1005,8 +1005,10 @@ func (r *request) toHTTP() (*http.Request, error) {
 	// this is required since go started validating req.host in 1.20.6 and 1.19.11.
 	// prior to that they would strip out the slashes for you.  They removed that
 	// behavior and added more strict validation as part of a CVE.
-	// https://github.com/golang/go/issues/60374
-	// the hope is that
+	// This issue is being tracked by the Go team:
+	// https://github.com/golang/go/issues/61431
+	// If there is a resolution in this issue, we will remove this code.
+	// In the time being, this is the accepted workaround.
 	if strings.HasPrefix(r.url.Host, "/") {
 		r.url.Host = "localhost"
 	}

command/members/members.go

@@ -33,6 +33,7 @@ type cmd struct {
 	wan          bool
 	statusFilter string
 	segment      string
+	filter       string
 }
 
 func New(ui cli.Ui) *cmd {
@@ -54,6 +55,7 @@ func (c *cmd) init() {
 	c.flags.StringVar(&c.segment, "segment", consulapi.AllSegments,
 		"(Enterprise-only) If provided, output is filtered to only nodes in"+
 			"the given segment.")
+	c.flags.StringVar(&c.filter, "filter", "", "Filter to use with the request")
 
 	c.http = &flags.HTTPFlags{}
 	flags.Merge(c.flags, c.http.ClientFlags())
@@ -83,6 +85,7 @@ func (c *cmd) Run(args []string) int {
 	opts := consulapi.MembersOpts{
 		Segment: c.segment,
 		WAN:     c.wan,
+		Filter:  c.filter,
 	}
 	members, err := client.Agent().MembersOpts(opts)
 	if err != nil {

envoyextensions/xdscommon/envoy_versioning_test.go

@@ -151,10 +151,10 @@ func TestDetermineSupportedProxyFeaturesFromString(t *testing.T) {
 	}
 	*/
 	for _, v := range []string{
-		"1.23.0", "1.23.1", "1.23.2", "1.23.3", "1.23.4", "1.23.5", "1.23.6", "1.23.7", "1.23.8", "1.23.9", "1.23.10",
-		"1.24.0", "1.24.1", "1.24.2", "1.24.3", "1.24.4", "1.24.5", "1.24.6", "1.24.7", "1.24.8",
-		"1.25.0", "1.25.1", "1.25.2", "1.25.3", "1.25.4", "1.25.5", "1.25.6", "1.25.7",
-		"1.26.0", "1.26.1", "1.26.2",
+		"1.24.0", "1.24.1", "1.24.2", "1.24.3", "1.24.4", "1.24.5", "1.24.6", "1.24.7", "1.24.8", "1.24.9", "1.24.10",
+		"1.25.0", "1.25.1", "1.25.2", "1.25.3", "1.25.4", "1.25.5", "1.25.6", "1.25.7", "1.25.8", "1.25.9",
+		"1.26.0", "1.26.1", "1.26.2", "1.26.3", "1.26.4",
+		"1.27.0",
 	} {
 		cases[v] = testcase{expect: SupportedProxyFeatures{}}
 	}