Storage account crd

.github/ISSUE_TEMPLATE/config.yml

@@ -8,4 +8,4 @@ contact_links:
       about: Share ideas for new features
     - name: Bug Report
       url: https://github.com/fybrik/fybrik/issues/new
-      about: Report a reproducable bug
+      about: Report a reproducible bug

.github/workflows/build.yml

@@ -14,10 +14,9 @@ on:
 env:
   GO_VERSION: 1.17
   # the newest supported K8s version and its respective kind image
-  NEW_K8S_VERSION: v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047
+  NEW_K8S_VERSION: v1.24.0@sha256:406fd86d48eaf4c04c7280cd1d2ca1d61e7d0d61ddef0125cb097bc7b82ed6a1
   # the oldest supported K8s version and its respective kind image
-  OLD_K8S_VERSION: v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729
-
+  OLD_K8S_VERSION: v1.22.9@sha256:6e57a6b0c493c7d7183a1151acff0bfa44bf37eb668826bf00da5637c55b6d5e 
   LATEST_BACKWARD_SUPPORTED_VERSION: 0.7.0
 
 jobs:
@@ -33,15 +32,16 @@ jobs:
       with:
         fetch-depth: 0
     - name: Cache dependencies
+      id: cache
       uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
           ${{ github.workspace }}/hack/tools/bin
-        key: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go
-    - name: install json-schema-generator
-      run: ./hack/tools/install_json-schema-generator.sh
+          ${{ github.workspace }}/hack/tools/lib
+        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
+    - name: Install tools
+      run: make install-tools
     - name: Verify
       run: |
         make generate manifests generate-docs verify
@@ -55,18 +55,21 @@ jobs:
       with:
         go-version: ${{ env.GO_VERSION }}
     - uses: actions/checkout@v3
+    - name: Set Swap Space
+      uses: pierotofy/set-swap-space@master
+      with:
+        swap-size-gb: 10
     - name: Cache dependencies
+      id: cache
       uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
           ${{ github.workspace }}/hack/tools/bin
-        key: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go
+          ${{ github.workspace }}/hack/tools/lib
+        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
     - name: Install tools
       run: make install-tools
-    - name: install json-schema-generator
-      run: ./hack/tools/install_json-schema-generator.sh
     - name: Test
       run: make test
 
@@ -86,13 +89,14 @@ jobs:
         go-version: ${{ env.GO_VERSION }}
     - uses: actions/checkout@v3
     - name: Cache dependencies
+      id: cache
       uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
           ${{ github.workspace }}/hack/tools/bin
-        key: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go
+          ${{ github.workspace }}/hack/tools/lib
+        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
     - name: Install tools
       run: make install-tools
     - name: Namescope integration tests
@@ -101,7 +105,7 @@ jobs:
         K8S_VERSION: ${{ env.NEW_K8S_VERSION }}
       run: make run-namescope-integration-tests
     - name: Integration tests old
-      if: ${{ matrix.k8s_version == 'old' }}
+      if: ${{ matrix.k8s_version == 'old' && ! github.event.pull_request.draft }}
       env:
         K8S_VERSION: ${{ env.OLD_K8S_VERSION }}
       run: make run-integration-tests
@@ -138,17 +142,18 @@ jobs:
         go-version: ${{ env.GO_VERSION }}
     - uses: actions/checkout@v3
     - name: Cache dependencies
+      id: cache
       uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
           ${{ github.workspace }}/hack/tools/bin
-        key: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go
+          ${{ github.workspace }}/hack/tools/lib
+        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
     - name: Install tools
       run: make install-tools
     - name: Read flow notebook tests old
-      if: ${{ matrix.k8s_version == 'old' }}
+      if: ${{ matrix.k8s_version == 'old' && ! github.event.pull_request.draft }}
       env:
         K8S_VERSION: ${{ env.OLD_K8S_VERSION }}
       run: make run-notebook-readflow-tests
@@ -176,8 +181,8 @@ jobs:
 #        path: |
 #          ~/go/pkg/mod
 #          ${{ github.workspace }}/hack/tools/bin
-#        key: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go-${{ hashFiles('**/go.sum') }}
-#        restore-keys: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go
+#          ${{ github.workspace }}/hack/tools/lib
+#        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
 #    - name: Install tools
 #      run: make install-tools
 #    - name: Read flow notebook tests bc version
@@ -200,13 +205,14 @@ jobs:
         go-version: ${{ env.GO_VERSION }}
     - uses: actions/checkout@v3
     - name: Cache dependencies
+      id: cache
       uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
           ${{ github.workspace }}/hack/tools/bin
-        key: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go
+          ${{ github.workspace }}/hack/tools/lib
+        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
     - name: Install tools
       run: make install-tools
     - name: Write flow notebook tests
@@ -226,13 +232,14 @@ jobs:
         go-version: ${{ env.GO_VERSION }}
     - uses: actions/checkout@v3
     - name: Cache dependencies
+      id: cache
       uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
           ${{ github.workspace }}/hack/tools/bin
-        key: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go
+          ${{ github.workspace }}/hack/tools/lib
+        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
     - name: Install tools
       run: make install-tools
     - name: Read flow notebook tests with tls
@@ -252,19 +259,47 @@ jobs:
         go-version: ${{ env.GO_VERSION }}
     - uses: actions/checkout@v3
     - name: Cache dependencies
+      id: cache
       uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
           ${{ github.workspace }}/hack/tools/bin
-        key: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go
+          ${{ github.workspace }}/hack/tools/lib
+        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
     - name: Install tools
       run: make install-tools
     - name: Read flow notebook tests with tls using system CA certs
       run: make run-notebook-readflow-tls-system-cacerts-tests
 
+  notebook-sample-read-flow-katalog:
+    name: Read flow notebook sample with katalog catalog
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_HOSTNAME: "localhost:5000"
+      DOCKER_NAMESPACE: fybrik-system
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - uses: actions/checkout@v3
+    - name: Cache dependencies
+      id: cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/go/pkg/mod
+          ${{ github.workspace }}/hack/tools/bin
+          ${{ github.workspace }}/hack/tools/lib
+        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
+    - name: Install tools
+      run: make install-tools
+    - name: Read flow notebook tests with katalog
+      run: make run-notebook-readflow-tests-katalog
+
   tekton-pipelines:
+    if: ${{ ! github.event.pull_request.draft }}
     name: Tekton Tests
     runs-on: ubuntu-latest
     env:
@@ -276,20 +311,23 @@ jobs:
       with:
         go-version: ${{ env.GO_VERSION }}
     - uses: actions/checkout@v3
+    - name: Set Swap Space
+      uses: pierotofy/set-swap-space@master
+      with:
+        swap-size-gb: 10
     - name: install nfs rpms
       run: sudo apt-get update && sudo apt-get install -y nfs-kernel-server nfs-common && sudo modprobe nfsd && sudo modprobe nfs
     - name: Cache dependencies
+      id: cache
       uses: actions/cache@v3
       with:
         path: |
           ~/go/pkg/mod
           ${{ github.workspace }}/hack/tools/bin
-        key: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: ${{ runner.os }}-t-${{ hashFiles('hack/make-rules/tools.mk') }}-go
+          ${{ github.workspace }}/hack/tools/lib
+        key: ${{ runner.os }}-c-${{ hashFiles('hack/make-rules/tools.mk') }}-${{ hashFiles('hack/tools/requirements.sh') }}-go-${{ hashFiles('**/go.sum') }}
     - name: Install tools
       run: make install-tools
-    - name: install json-schema-generator
-      run: ./hack/tools/install_json-schema-generator.sh
     - name: Integration tests
       env:
         image_source_repo_password: "fake"

.github/workflows/license-check.yml

@@ -2,7 +2,7 @@ name: license-check
 on:
   push:
     branches:
-      - main
+      - master
   pull_request:
 jobs:
   addlicense-check:

.github/workflows/python-publish.yaml

@@ -8,6 +8,9 @@ on:
       vault-version:
         required: true
         default: 'do-not-publish'
+      tls-version:
+        required: true
+        default: 'do-not-publish'
       pypi-repository-url:
         required: true
         default: 'https://upload.pypi.org/legacy/'
@@ -20,7 +23,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v4.2.0
+      uses: actions/setup-python@v4.3.0
       with:
         python-version: '3.x'
     - name: Install dependencies
@@ -43,6 +46,14 @@ jobs:
         else
           VERSION=${{ github.event.inputs.vault-version }} python -m build ./python/vault --outdir ./dist
         fi
+    - name: Build tls package
+      run: |
+        if [ ${{ github.event.inputs.tls-version }} == "do-not-publish" ]
+        then
+          echo "do not publish tls library"
+        else
+          VERSION=${{ github.event.inputs.tls-version }} python -m build ./python/tls --outdir ./dist
+        fi
     - name: Publish package
       uses: pypa/gh-action-pypi-publish@release/v1
       with:

.github/workflows/website-deploy.yml

@@ -22,7 +22,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Install Python
-        uses: actions/setup-python@v4.2.0
+        uses: actions/setup-python@v4.3.0
         with:
           python-version: 3.8
       - name: Install Material for MkDocs