Merge branch 'kubeflow:master' into patch-14

.github/workflows/bentoml_kind_test.yaml

@@ -0,0 +1,29 @@
+name: Build & Apply BentoML Yatai Stack manifests in KinD
+on:
+  pull_request:
+    paths:
+      - contrib/bentoml/**
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Install KinD
+      run: ./tests/gh-actions/install_kind.sh
+
+    - name: Create KinD Cluster
+      run: kind create cluster --config tests/gh-actions/kind-cluster-1-24.yaml
+
+    - name: Install kustomize
+      run: ./tests/gh-actions/install_kustomize.sh
+
+    - name: Install cert-manager
+      run: ./tests/gh-actions/install_cert_manager.sh
+
+    - name: Build & Apply manifests
+      run: |
+        cd contrib/bentoml/
+        make test

.github/workflows/centraldb_kind_test.yaml

@@ -3,6 +3,8 @@ on:
   pull_request:
     paths:
       - apps/centraldashboard/upstream/**
+      - tests/gh-actions/kind-cluster.yaml
+      - tests/gh-actions/install_istio.sh
 
 jobs:
   build:
@@ -15,7 +17,7 @@ jobs:
       run: ./tests/gh-actions/install_kind.sh
 
     - name: Create KinD Cluster
-      run: kind create cluster --config tests/gh-actions/kind-cluster-1-25.yaml
+      run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
 
     - name: Install kustomize
       run: ./tests/gh-actions/install_kustomize.sh

.github/workflows/dex_kind_test.yaml

@@ -3,6 +3,8 @@ on:
   pull_request:
     paths:
       - common/dex/base/**
+      - tests/gh-actions/kind-cluster.yaml
+      - tests/gh-actions/install_istio.sh
 
 jobs:
   build:
@@ -15,7 +17,7 @@ jobs:
       run: ./tests/gh-actions/install_kind.sh
 
     - name: Create KinD Cluster
-      run: kind create cluster --config tests/gh-actions/kind-cluster-1-25.yaml
+      run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
 
     - name: Install kustomize
       run: ./tests/gh-actions/install_kustomize.sh

.github/workflows/jwa_kind_test.yaml

@@ -3,6 +3,8 @@ on:
   pull_request:
     paths:
       - apps/jupyter/jupyter-web-app/upstream/**
+      - tests/gh-actions/kind-cluster.yaml
+      - tests/gh-actions/install_istio.sh
 
 jobs:
   build:
@@ -15,7 +17,7 @@ jobs:
       run: ./tests/gh-actions/install_kind.sh
 
     - name: Create KinD Cluster
-      run: kind create cluster --config tests/gh-actions/kind-cluster-1-25.yaml
+      run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
 
     - name: Install kustomize
       run: ./tests/gh-actions/install_kustomize.sh

.github/workflows/katib_kind_test.yaml

@@ -3,6 +3,8 @@ on:
   pull_request:
     paths:
       - apps/katib/upstream/**
+      - tests/gh-actions/kind-cluster.yaml
+      - tests/gh-actions/install_istio.sh
 
 jobs:
   build:
@@ -15,7 +17,7 @@ jobs:
       run: ./tests/gh-actions/install_kind.sh
 
     - name: Create KinD Cluster
-      run: kind create cluster --config tests/gh-actions/kind-cluster-1-25.yaml
+      run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
 
     - name: Install kustomize
       run: ./tests/gh-actions/install_kustomize.sh
@@ -26,17 +28,31 @@ jobs:
     - name: Install cert-manager
       run: ./tests/gh-actions/install_cert_manager.sh
 
+    # https://kind.sigs.k8s.io/docs/user/known-issues/#apparmor
+    - name: AppArmor
+      run: |
+        set -x
+        sudo apt-get install apparmor-profiles
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+
     - name: Build & Apply manifests
       run: |
         cd apps/katib/upstream
         kubectl create ns kubeflow
         kustomize build installs/katib-with-kubeflow | kubectl apply -f -
-        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 300s
+        kubectl wait --for=condition=Ready pods --all -n kubeflow --timeout 300s
 
     - name: Create katib experiment
       run: |
         kubectl create namespace kubeflow-user
         kubectl label namespace kubeflow-user katib.kubeflow.org/metrics-collector-injection=enabled
         kubectl apply -f tests/gh-actions/kf-objects/katib_test.yaml
+
+        echo "Waiting for Experiment to become Running..."
+        kubectl wait --for=condition=Running experiments.kubeflow.org -n kubeflow-user --all --timeout 300s
+
+        echo "Waiting for all Trials to become Succeeded..."
         kubectl wait --for=condition=Succeeded trials.kubeflow.org -n kubeflow-user --all --timeout 300s
+
+        echo "Waiting for the Experiment to become Succeeded..."
         kubectl wait --for=condition=Succeeded experiments.kubeflow.org -n kubeflow-user --all --timeout 300s

.github/workflows/kserve_kind_cni_test.yaml

@@ -0,0 +1,61 @@
+name: Build & Apply KServe manifests in KinD, using istio CNI
+on:
+  pull_request:
+    paths:
+      - common/istio-cni-1-17/**
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install KinD
+        run: ./tests/gh-actions/install_kind.sh
+
+      - name: Create KinD Cluster
+        run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
+
+      - name: Install kustomize
+        run: ./tests/gh-actions/install_kustomize.sh
+
+      - name: Create kubeflow namespace
+        run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
+
+      - name: Install Istio CNI
+        run: ./tests/gh-actions/install_istio-cni.sh
+
+      - name: Install cert-manager
+        run: ./tests/gh-actions/install_cert_manager.sh
+
+      - name: Install knative CNI
+        run: ./tests/gh-actions/install_knative-cni.sh
+
+      - name: Build & Apply manifests
+        run: ./tests/gh-actions/install_kserve.sh
+
+      - name: Create test namespace
+        run: kubectl create ns kserve-test
+
+      - name: Setup python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install test dependencies
+        run: pip install -r ./contrib/kserve/tests/requirements.txt
+
+      - name: Port forward
+        run: |
+          INGRESS_GATEWAY_SERVICE=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
+          nohup kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 &
+
+      - name: Run kserve tests
+        run: |
+          export KSERVE_INGRESS_HOST_PORT=localhost:8080
+          cd ./contrib/kserve/tests && pytest .
+
+      - name: Run kserve models webapp test
+        run: |
+          kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app

.github/workflows/kserve_kind_test.yaml

@@ -3,34 +3,60 @@ on:
   pull_request:
     paths:
       - contrib/kserve/**
+      - common/knative/**
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout
-      uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v3
 
-    - name: Install KinD
-      run: ./tests/gh-actions/install_kind.sh
+      - name: Install KinD
+        run: ./tests/gh-actions/install_kind.sh
 
-    - name: Create KinD Cluster
-      run: kind create cluster --config tests/gh-actions/kind-cluster-1-25.yaml
+      - name: Create KinD Cluster
+        run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
 
-    - name: Install kustomize
-      run: ./tests/gh-actions/install_kustomize.sh
+      - name: Install kustomize
+        run: ./tests/gh-actions/install_kustomize.sh
 
-    - name: Install Istio
-      run: ./tests/gh-actions/install_istio.sh
+      - name: Create kubeflow namespace
+        run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
 
-    - name: Install cert-manager
-      run: ./tests/gh-actions/install_cert_manager.sh
+      - name: Install Istio
+        run: ./tests/gh-actions/install_istio.sh
 
-    - name: Build & Apply manifests
-      run: ./tests/gh-actions/install_kserve.sh
+      - name: Install cert-manager
+        run: ./tests/gh-actions/install_cert_manager.sh
 
-    - name: Create kserve experiment
-      run: |
-        ./tests/gh-actions/install_knative.sh
-        kubectl apply -f tests/gh-actions/kf-objects/kserve_test.yaml
-        kubectl wait --for=condition=ready inferenceservices.serving.kserve.io --all-namespaces --all --timeout 300s
+      - name: Install knative
+        run: ./tests/gh-actions/install_knative.sh
+
+      - name: Build & Apply manifests
+        run: ./tests/gh-actions/install_kserve.sh
+
+      - name: Create test namespace
+        run: kubectl create ns kserve-test
+
+      - name: Setup python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install test dependencies
+        run: pip install -r ./contrib/kserve/tests/requirements.txt
+
+      - name: Port forward
+        run: |
+          INGRESS_GATEWAY_SERVICE=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
+          nohup kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8080:80 &
+
+      - name: Run kserve tests
+        run: |
+          export KSERVE_INGRESS_HOST_PORT=localhost:8080
+          cd ./contrib/kserve/tests && pytest .
+
+      - name: Run kserve models webapp test
+        run: |
+          kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app

.github/workflows/manifests_unittests.yaml

@@ -13,12 +13,8 @@ jobs:
     - name: Check out repo
       uses: actions/checkout@v2
 
-    - name: Install Kustomize
-      working-directory: ./tests/e2e
-      run: |
-        curl -Lo ./kustomize https://github.com/kubernetes-sigs/kustomize/releases/download/v3.2.0/kustomize_3.2.0_linux_amd64
-        chmod +x ./kustomize
-        sudo mv kustomize /usr/local/bin
+    - name: Install kustomize
+      run: ./tests/gh-actions/install_kustomize.sh
 
     - name: Unit Test
       run: |

.github/workflows/metacontroller_kind_test.yaml

@@ -0,0 +1,32 @@
+name: Build & Apply contrib/metacontroller in KinD
+on:
+  pull_request:
+    paths:
+      - contrib/metacontroller/**
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Install KinD
+      run: ./tests/gh-actions/install_kind.sh
+
+    - name: Create KinD Cluster
+      run: kind create cluster --config ./tests/gh-actions/kind-cluster.yaml
+
+    - name: Install kustomize
+      run: ./tests/gh-actions/install_kustomize.sh
+
+    - name: Install Istio
+      run: ./tests/gh-actions/install_istio.sh
+
+    - name: Install cert-manager
+      run: ./tests/gh-actions/install_cert_manager.sh
+
+    - name: Build & Apply manifests
+      run: |
+        cd contrib/metacontroller/
+        make test

.github/workflows/nb_controller_kind_test.yaml

@@ -3,6 +3,8 @@ on:
   pull_request:
     paths:
       - apps/jupyter/notebook-controller/upstream/**
+      - tests/gh-actions/kind-cluster.yaml
+      - tests/gh-actions/install_istio.sh
 
 jobs:
   build:
@@ -15,7 +17,7 @@ jobs:
       run: ./tests/gh-actions/install_kind.sh
 
     - name: Create KinD Cluster
-      run: kind create cluster --config tests/gh-actions/kind-cluster-1-25.yaml
+      run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
 
     - name: Install kustomize
       run: ./tests/gh-actions/install_kustomize.sh

.github/workflows/pipeline_kind_test.yaml

@@ -3,6 +3,9 @@ on:
   pull_request:
     paths:
       - apps/pipeline/upstream/**
+      - tests/gh-actions/kind-cluster.yaml
+      - tests/gh-actions/install_istio.sh
+      - common/cert-manager/**
 
 jobs:
   build:
@@ -15,7 +18,7 @@ jobs:
       run: ./tests/gh-actions/install_kind.sh
 
     - name: Create KinD Cluster
-      run: kind create cluster --config tests/gh-actions/kind-cluster-1-25.yaml
+      run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
 
     - name: Install kustomize
       run: ./tests/gh-actions/install_kustomize.sh
@@ -31,7 +34,7 @@ jobs:
 
     - name: Deploy test pipeline
       run: |
-        pip3 install kfp==1.8.4
+        pip3 install kfp==1.8.22
         kustomize build apps/profiles/upstream/overlays/kubeflow | kubectl apply -f -
         kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 180s
         kustomize build common/user-namespace/base | kubectl apply -f -

.github/workflows/poddefaults_kind_test.yaml

@@ -3,6 +3,9 @@ on:
   pull_request:
     paths:
       - apps/admission-webhook/upstream/**
+      - tests/gh-actions/kind-cluster.yaml
+      - tests/gh-actions/install_istio.sh
+      - common/cert-manager/**
 
 jobs:
   build:
@@ -15,7 +18,7 @@ jobs:
       run: ./tests/gh-actions/install_kind.sh
 
     - name: Create KinD Cluster
-      run: kind create cluster --config tests/gh-actions/kind-cluster-1-25.yaml
+      run: kind create cluster --config tests/gh-actions/kind-cluster.yaml
 
     - name: Install kustomize
       run: ./tests/gh-actions/install_kustomize.sh