Merge branch 'master' into autopilot

GoogleCloudPlatform · Mar 7, 2023 · 9a25aed · 9a25aed
2 parents 14981ab + ca31192
commit 9a25aed
Show file tree

Hide file tree

Showing 47 changed files with 1,296 additions and 380 deletions.
diff --git a/.github/actions/fabric-tests/action.yml b/.github/actions/fabric-tests/action.yml
@@ -0,0 +1,55 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: fabric-tests
+description: Set up Fabric testing environment
+inputs:
+  PYTHON_VERSION:
+    required: true
+  TERRAFORM_VERSION:
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Config auth
+      shell: bash
+      run: |
+        echo '{"type": "service_account", "project_id": "test-only"}' \
+          | tee -a $GOOGLE_APPLICATION_CREDENTIALS
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.PYTHON_VERSION }}
+        cache: 'pip'
+        cache-dependency-path: 'tests/requirements.txt'
+    - name: Set up Terraform
+      uses: hashicorp/setup-terraform@v2
+      with:
+        terraform_version: ${{ inputs.TERRAFORM_VERSION }}
+        terraform_wrapper: false
+    - name: Configure provider cache
+      shell: bash
+      run: |
+        echo 'plugin_cache_dir = "/home/runner/.terraform.d/plugin-cache"' \
+          | tee -a /home/runner/.terraformrc
+        echo 'disable_checkpoint = true' \
+          | tee -a /home/runner/.terraformrc
+        mkdir -p /home/runner/.terraform.d/plugin-cache
+    # avoid conflicts with user-installed providers on local machines
+    - name: Pin provider versions
+      shell: bash
+      run: |
+        for f in $(find . -name versions.tf); do
+          sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
+        done
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -31,72 +31,51 @@ env:
   TF_VERSION: 1.3.9
 
 jobs:
-  examples:
+  examples-blueprints:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
-      - name: Config auth
+      - name: Call composite action fabric-tests
+        uses: ./.github/actions/fabric-tests
+        with:
+          PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
+          TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
+
+      - name: Run tests on documentation examples
+        id: pytest
         run: |
-          echo '{"type": "service_account", "project_id": "test-only"}' \
-            | tee -a $GOOGLE_APPLICATION_CREDENTIALS
+          pip install -r tests/requirements.txt
+          pytest -vv -k blueprints/ tests/examples
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          cache: 'pip'
-          cache-dependency-path: 'tests/requirements.txt'
+  examples-modules:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
 
-      - name: Set up Terraform
-        uses: hashicorp/setup-terraform@v2
+      - name: Call composite action fabric-tests
+        uses: ./.github/actions/fabric-tests
         with:
-          terraform_version: ${{ env.TF_VERSION }}
-          terraform_wrapper: false
-
-      # avoid conflicts with user-installed providers on local machines
-      - name: Pin provider versions
-        run: |
-          for f in $(find . -name versions.tf); do
-            sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
-          done
+          PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
+          TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
 
       - name: Run tests on documentation examples
         id: pytest
         run: |
           mkdir -p ${{ env.TF_PLUGIN_CACHE_DIR }}
           pip install -r tests/requirements.txt
-          pytest -vv tests/examples
+          pytest -vv -k modules/ tests/examples
 
   blueprints:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
-      - name: Config auth
-        run: |
-          echo '{"type": "service_account", "project_id": "test-only"}' \
-            | tee -a $GOOGLE_APPLICATION_CREDENTIALS
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          cache: 'pip'
-          cache-dependency-path: 'tests/requirements.txt'
-
-      - name: Set up Terraform
-        uses: hashicorp/setup-terraform@v2
+      - name: Call composite action fabric-tests
+        uses: ./.github/actions/fabric-tests
         with:
-          terraform_version: ${{ env.TF_VERSION }}
-          terraform_wrapper: false
-
-      # avoid conflicts with user-installed providers on local machines
-      - name: Pin provider versions
-        run: |
-          for f in $(find . -name versions.tf); do
-            sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
-          done
+          PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
+          TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
 
       - name: Run tests environments
         id: pytest
@@ -110,30 +89,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Config auth
-        run: |
-          echo '{"type": "service_account", "project_id": "test-only"}' \
-            | tee -a $GOOGLE_APPLICATION_CREDENTIALS
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - name: Call composite action fabric-tests
+        uses: ./.github/actions/fabric-tests
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          cache: 'pip'
-          cache-dependency-path: 'tests/requirements.txt'
-
-      - name: Set up Terraform
-        uses: hashicorp/setup-terraform@v2
-        with:
-          terraform_version: ${{ env.TF_VERSION }}
-          terraform_wrapper: false
-
-      # avoid conflicts with user-installed providers on local machines
-      - name: Pin provider versions
-        run: |
-          for f in $(find . -name versions.tf); do
-            sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
-          done
+          PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
+          TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
 
       - name: Run tests modules
         id: pytest
@@ -147,30 +107,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Config auth
-        run: |
-          echo '{"type": "service_account", "project_id": "test-only"}' \
-            | tee -a $GOOGLE_APPLICATION_CREDENTIALS
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - name: Call composite action fabric-tests
+        uses: ./.github/actions/fabric-tests
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          cache: 'pip'
-          cache-dependency-path: 'tests/requirements.txt'
-
-      - name: Set up Terraform
-        uses: hashicorp/setup-terraform@v2
-        with:
-          terraform_version: ${{ env.TF_VERSION }}
-          terraform_wrapper: false
-
-      # avoid conflicts with user-installed providers on local machines
-      - name: Pin provider versions
-        run: |
-          for f in $(find . -name versions.tf); do
-            sed -i 's/>=\(.*# tftest\)/=\1/g' $f;
-          done
+          PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
+          TERRAFORM_VERSION: ${{ env.TERRAFORM_VERSION }}
 
       - name: Run tests on FAST stages
         id: pytest

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -45,6 +45,7 @@ All notable changes to this project will be documented in this file.
 
 ### FAST
 
+- [[#1211](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1211)] **incompatible change:** Add support for proxy and psc subnets to net-vpc module factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-05 16:08:43+00:00 -->
 - [[#1209](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1209)] Billing exclusion support for FAST mt resman ([ludoo](https://github.com/ludoo)) <!-- 2023-03-03 16:23:37+00:00 -->
 - [[#1207](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1207)] Allow preventing creation of billing IAM roles in FAST, add instructions on delayed billing association ([ludoo](https://github.com/ludoo)) <!-- 2023-03-03 08:24:42+00:00 -->
 - [[#1184](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1184)] **incompatible change:** Allow multiple peer gateways in VPN HA module ([ludoo](https://github.com/ludoo)) <!-- 2023-02-27 10:19:00+00:00 -->
@@ -63,6 +64,7 @@ All notable changes to this project will be documented in this file.
 
 ### MODULES
 
+- [[#1211](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1211)] **incompatible change:** Add support for proxy and psc subnets to net-vpc module factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-05 16:08:43+00:00 -->
 - [[#1206](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1206)] Dataproc module. Fix output. ([lcaggio](https://github.com/lcaggio)) <!-- 2023-03-02 12:59:19+00:00 -->
 - [[#1205](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1205)] Fix issue with GKE cluster notifications topic & static output for pubsub module ([rosmo](https://github.com/rosmo)) <!-- 2023-03-02 10:43:40+00:00 -->
 - [[#1204](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1204)] Fix url_redirect issue on net-glb module ([erabusi](https://github.com/erabusi)) <!-- 2023-03-02 06:51:40+00:00 -->
@@ -100,6 +102,7 @@ All notable changes to this project will be documented in this file.
 
 ### TOOLS
 
+- [[#1211](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1211)] **incompatible change:** Add support for proxy and psc subnets to net-vpc module factory ([ludoo](https://github.com/ludoo)) <!-- 2023-03-05 16:08:43+00:00 -->
 - [[#1209](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1209)] Billing exclusion support for FAST mt resman ([ludoo](https://github.com/ludoo)) <!-- 2023-03-03 16:23:37+00:00 -->
 - [[#1208](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1208)] Fix outdated go deps, dependabot alerts ([averbuks](https://github.com/averbuks)) <!-- 2023-03-03 06:15:09+00:00 -->
 - [[#1182](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1182)] Bump actions versions ([juliocc](https://github.com/juliocc)) <!-- 2023-02-25 16:27:20+00:00 -->

diff --git a/blueprints/README.md b/blueprints/README.md
@@ -6,7 +6,7 @@ Currently available blueprints:
 
 - **apigee** - [Apigee Hybrid on GKE](./apigee/hybrid-gke/), [Apigee X analytics in BigQuery](./apigee/bigquery-analytics), [Apigee network patterns](./apigee/network-patterns/)
 - **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Networking Dashboard](./cloud-operations/network-dashboard), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation)
-- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder)
+- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/bq-ml)
 - **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory)
 - **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [GKE Autopilot](./gke/autopilot)
 - **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [Network filtering with Squid](./networking/filtering-proxy), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), [Network filtering with Squid with isolated VPCs using Private Service Connect](./networking/filtering-proxy-psc), On-prem DNS and Google Private Access,  [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke)

diff --git a/blueprints/data-solutions/README.md b/blueprints/data-solutions/README.md
@@ -69,3 +69,9 @@ This [blueprint](./vertex-mlops/) implements the infrastructure required to have
 This [blueprint](./shielded-folder/) implements an opinionated folder configuration according to GCP best practices. Configurations implemented on the folder would be beneficial to host workloads inheriting constraints from the folder they belong to.
 
 <br clear="left">
+
+### BigQuery ML and Vertex AI Pipeline
+
+<a href="./bq-ml/" title="BigQuery ML and Vertex AI Pipeline"><img src="./bq-ml/images/diagram.png" align="left" width="280px"></a>
+This [blueprint](./bq-ml/) provides the necessary infrastructure to create a complete development environment for building and deploying machine learning models using BigQuery ML and Vertex AI. With this blueprint, you can deploy your models to a Vertex AI endpoint or use them within BigQuery ML.
+<br clear="left">