From b4d1b725eb50c6757c65a8417411cf2a465729ce Mon Sep 17 00:00:00 2001 From: nassarmu10 Date: Wed, 2 Feb 2022 21:50:25 +0200 Subject: [PATCH 1/2] Add a workflow to test the module. Signed-off-by: nassarmu10 --- .github/workflows/test-module.yaml | 44 +++++ hack/make-rules/tools.mk | 24 +++ hack/tools/install_kubebuilder.sh | 21 +++ hack/tools/test-script/Asset-0.5.0.yaml | 29 +++ hack/tools/test-script/Asset-0.6.0.yaml | 30 +++ .../PS_20174392719_1491204439457_log.csv | 102 +++++++++++ hack/tools/test-script/expected.txt | 14 ++ .../test-script/fybrikStorage-0.5.0.yaml | 10 + .../test-script/fybrikStorage-0.6.0.yaml | 10 + .../test-script/fybrikapplication-0.5.0.yaml | 19 ++ .../test-script/fybrikapplication-0.6.0.yaml | 18 ++ .../test-script/sample-policy-0.5.0.rego | 9 + .../test-script/sample-policy-0.6.0.rego | 9 + hack/tools/test-script/test.py | 23 +++ hack/tools/test_module.sh | 173 ++++++++++++++++++ 15 files changed, 535 insertions(+) create mode 100644 .github/workflows/test-module.yaml create mode 100755 hack/tools/install_kubebuilder.sh create mode 100644 hack/tools/test-script/Asset-0.5.0.yaml create mode 100644 hack/tools/test-script/Asset-0.6.0.yaml create mode 100644 hack/tools/test-script/PS_20174392719_1491204439457_log.csv create mode 100644 hack/tools/test-script/expected.txt create mode 100644 hack/tools/test-script/fybrikStorage-0.5.0.yaml create mode 100644 hack/tools/test-script/fybrikStorage-0.6.0.yaml create mode 100644 hack/tools/test-script/fybrikapplication-0.5.0.yaml create mode 100644 hack/tools/test-script/fybrikapplication-0.6.0.yaml create mode 100644 hack/tools/test-script/sample-policy-0.5.0.rego create mode 100644 hack/tools/test-script/sample-policy-0.6.0.rego create mode 100644 hack/tools/test-script/test.py create mode 100755 hack/tools/test_module.sh diff --git a/.github/workflows/test-module.yaml b/.github/workflows/test-module.yaml new file mode 100644 index 0000000..1d9a74f --- /dev/null +++ b/.github/workflows/test-module.yaml @@ -0,0 +1,44 @@ +name: Test +on: + workflow_dispatch: + inputs: + kind-version: + required: true + default: 'kind19' + fybrik-version: + required: true + default: '0.6.0' + module-version: + required: true + default: '0.6.0' + module-type: + required: true + default: 'stream' + cert-manager-version: + required: true + default: '1.6.2' + +env: + GO_VERSION: 1.17 + AWS_PAGER: "" + AWS_EC2_METADATA_DISABLED: true + +jobs: + test-module: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install tools + run: make install-tools + - name: Install AWS cli + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2 # default + verbose: false # default + - name: run-test + run: pushd hack/tools && ./test_module.sh ${{ github.event.inputs.kind-version }} ${{ github.event.inputs.fybrik-version }} ${{ github.event.inputs.module-version }} ${{ github.event.inputs.module-type }} ${{ github.event.inputs.cert-manager-version }} && popd + diff --git a/hack/make-rules/tools.mk b/hack/make-rules/tools.mk index 109e931..828d78f 100644 --- a/hack/make-rules/tools.mk +++ b/hack/make-rules/tools.mk @@ -1,5 +1,29 @@ +INSTALL_TOOLS += $(TOOLBIN)/yq $(TOOLBIN)/yq: cd $(TOOLS_DIR); ./install_yq.sh + $(call post-install-check) +INSTALL_TOOLS += $(TOOLBIN)/helm $(TOOLBIN)/helm: cd $(TOOLS_DIR); ./install_helm.sh + $(call post-install-check) + +INSTALL_TOOLS += $(TOOLBIN)/kind +$(TOOLBIN)/kind: + GOBIN=$(ABSTOOLBIN) go install sigs.k8s.io/kind@v0.11.1 + $(call post-install-check) + +INSTALL_TOOLS += $(TOOLBIN)/kubebuilder +$(TOOLBIN)/kubebuilder $(TOOLBIN)/etcd $(TOOLBIN)/kube-apiserver $(TOOLBIN)/kubectl: + cd $(TOOLS_DIR); ./install_kubebuilder.sh + $(call post-install-check) + +.PHONY: install-tools +install-tools: $(INSTALL_TOOLS) + +.PHONY: uninstall-tools +uninstall-tools: + rm -rf $(INSTALL_TOOLS) + + + diff --git a/hack/tools/install_kubebuilder.sh b/hack/tools/install_kubebuilder.sh new file mode 100755 index 0000000..57dfba0 --- /dev/null +++ b/hack/tools/install_kubebuilder.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Copyright 2020 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +source ./common.sh + +version=2.3.1 + +header_text "Checking for bin/etcd" +[[ -f bin/etcd ]] && exit 0 + +header_text "Installing bin/etcd" +mkdir -p ./bin + +K8S_VERSION=1.19.2 +curl -sSLo envtest-bins.tar.gz "https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-${K8S_VERSION}-$(go env GOOS)-$(go env GOARCH).tar.gz" + +tar -zvxf envtest-bins.tar.gz +mv kubebuilder/bin/* bin +rm envtest-bins.tar.gz +rm -r kubebuilder \ No newline at end of file diff --git a/hack/tools/test-script/Asset-0.5.0.yaml b/hack/tools/test-script/Asset-0.5.0.yaml new file mode 100644 index 0000000..81a3522 --- /dev/null +++ b/hack/tools/test-script/Asset-0.5.0.yaml @@ -0,0 +1,29 @@ +apiVersion: katalog.fybrik.io/v1alpha1 +kind: Asset +metadata: + name: paysim-csv +spec: + secretRef: + name: paysim-csv + assetDetails: + dataFormat: csv + connection: + type: s3 + s3: + endpoint: "http://localstack.fybrik-notebook-sample.svc.cluster.local:4566" + bucket: "demo" + objectKey: "PS_20174392719_1491204439457_log.csv" + assetMetadata: + geography: theshire + tags: + - finance + componentsMetadata: + nameOrig: + tags: + - PII + oldbalanceOrg: + tags: + - PII + newbalanceOrig: + tags: + - PII \ No newline at end of file diff --git a/hack/tools/test-script/Asset-0.6.0.yaml b/hack/tools/test-script/Asset-0.6.0.yaml new file mode 100644 index 0000000..f81183b --- /dev/null +++ b/hack/tools/test-script/Asset-0.6.0.yaml @@ -0,0 +1,30 @@ +apiVersion: katalog.fybrik.io/v1alpha1 +kind: Asset +metadata: + name: paysim-csv +spec: + secretRef: + name: paysim-csv + details: + dataFormat: csv + connection: + name: s3 + s3: + endpoint: "http://localstack.fybrik-notebook-sample.svc.cluster.local:4566" + bucket: "demo" + object_key: "PS_20174392719_1491204439457_log.csv" + metadata: + name: Synthetic Financial Datasets For Fraud Detection + geography: theshire + tags: + finance: true + columns: + - name: nameOrig + tags: + PII: true + - name: oldbalanceOrg + tags: + PII: true + - name: newbalanceOrig + tags: + PII: true \ No newline at end of file diff --git a/hack/tools/test-script/PS_20174392719_1491204439457_log.csv b/hack/tools/test-script/PS_20174392719_1491204439457_log.csv new file mode 100644 index 0000000..6c5a246 --- /dev/null +++ b/hack/tools/test-script/PS_20174392719_1491204439457_log.csv @@ -0,0 +1,102 @@ +step,type,amount,nameOrig,oldbalanceOrg,newbalanceOrig,nameDest,oldbalanceDest,newbalanceDest,isFraud,isFlaggedFraud +1,PAYMENT,9839.64,C1231006815,170136,160296.36,M1979787155,0,0,0,0 +1,PAYMENT,1864.28,C1666544295,21249,19384.72,M2044282225,0,0,0,0 +1,TRANSFER,181,C1305486145,181,0,C553264065,0,0,1,0 +1,CASH_OUT,181,C840083671,181,0,C38997010,21182,0,1,0 +1,PAYMENT,11668.14,C2048537720,41554,29885.86,M1230701703,0,0,0,0 +1,PAYMENT,7817.71,C90045638,53860,46042.29,M573487274,0,0,0,0 +1,PAYMENT,7107.77,C154988899,183195,176087.23,M408069119,0,0,0,0 +1,PAYMENT,7861.64,C1912850431,176087.23,168225.59,M633326333,0,0,0,0 +1,PAYMENT,4024.36,C1265012928,2671,0,M1176932104,0,0,0,0 +1,DEBIT,5337.77,C712410124,41720,36382.23,C195600860,41898,40348.79,0,0 +1,DEBIT,9644.94,C1900366749,4465,0,C997608398,10845,157982.12,0,0 +1,PAYMENT,3099.97,C249177573,20771,17671.03,M2096539129,0,0,0,0 +1,PAYMENT,2560.74,C1648232591,5070,2509.26,M972865270,0,0,0,0 +1,PAYMENT,11633.76,C1716932897,10127,0,M801569151,0,0,0,0 +1,PAYMENT,4098.78,C1026483832,503264,499165.22,M1635378213,0,0,0,0 +1,CASH_OUT,229133.94,C905080434,15325,0,C476402209,5083,51513.44,0,0 +1,PAYMENT,1563.82,C761750706,450,0,M1731217984,0,0,0,0 +1,PAYMENT,1157.86,C1237762639,21156,19998.14,M1877062907,0,0,0,0 +1,PAYMENT,671.64,C2033524545,15123,14451.36,M473053293,0,0,0,0 +1,TRANSFER,215310.3,C1670993182,705,0,C1100439041,22425,0,0,0 +1,PAYMENT,1373.43,C20804602,13854,12480.57,M1344519051,0,0,0,0 +1,DEBIT,9302.79,C1566511282,11299,1996.21,C1973538135,29832,16896.7,0,0 +1,DEBIT,1065.41,C1959239586,1817,751.59,C515132998,10330,0,0,0 +1,PAYMENT,3876.41,C504336483,67852,63975.59,M1404932042,0,0,0,0 +1,TRANSFER,311685.89,C1984094095,10835,0,C932583850,6267,2719172.89,0,0 +1,PAYMENT,6061.13,C1043358826,443,0,M1558079303,0,0,0,0 +1,PAYMENT,9478.39,C1671590089,116494,107015.61,M58488213,0,0,0,0 +1,PAYMENT,8009.09,C1053967012,10968,2958.91,M295304806,0,0,0,0 +1,PAYMENT,8901.99,C1632497828,2958.91,0,M33419717,0,0,0,0 +1,PAYMENT,9920.52,C764826684,0,0,M1940055334,0,0,0,0 +1,PAYMENT,3448.92,C2103763750,0,0,M335107734,0,0,0,0 +1,PAYMENT,4206.84,C215078753,0,0,M1757317128,0,0,0,0 +1,PAYMENT,5885.56,C840514538,0,0,M1804441305,0,0,0,0 +1,PAYMENT,5307.88,C1768242710,0,0,M1971783162,0,0,0,0 +1,PAYMENT,5031.22,C247113419,0,0,M151442075,0,0,0,0 +1,PAYMENT,24213.67,C1238616099,0,0,M70695990,0,0,0,0 +1,PAYMENT,8603.42,C1608633989,253,0,M1615617512,0,0,0,0 +1,PAYMENT,2791.42,C923341586,300481,297689.58,M107994825,0,0,0,0 +1,PAYMENT,7413.54,C1470868839,297689.58,290276.03,M1426725223,0,0,0,0 +1,PAYMENT,3295.19,C711197015,233633,230337.81,M1384454980,0,0,0,0 +1,PAYMENT,1684.81,C1481594086,297,0,M1569435561,0,0,0,0 +1,DEBIT,5758.59,C1466917878,32604,26845.41,C1297685781,209699,16997.22,0,0 +1,CASH_OUT,110414.71,C768216420,26845.41,0,C1509514333,288800,2415.16,0,0 +1,PAYMENT,7823.46,C260084831,998,0,M267814113,0,0,0,0 +1,PAYMENT,5086.48,C598357562,0,0,M1593224710,0,0,0,0 +1,PAYMENT,5281.48,C1440738283,152019,146737.52,M1849015357,0,0,0,0 +1,PAYMENT,13875.98,C484199463,15818,1942.02,M2008106788,0,0,0,0 +1,CASH_OUT,56953.9,C1570470538,1942.02,0,C824009085,70253,64106.18,0,0 +1,CASH_OUT,5346.89,C512549200,0,0,C248609774,652637,6453430.91,0,0 +1,PAYMENT,2204.04,C1615801298,586,0,M490391704,0,0,0,0 +1,PAYMENT,2641.47,C460570271,23053,20411.53,M1653361344,0,0,0,0 +1,CASH_OUT,23261.3,C2072313080,20411.53,0,C2001112025,25742,0,0,0 +1,PAYMENT,2330.64,C816944408,203543,201212.36,M909132503,0,0,0,0 +1,PAYMENT,1614.64,C912966811,41276,39661.36,M1792384402,0,0,0,0 +1,PAYMENT,9164.71,C1458621573,47235.77,38071.06,M1658980982,0,0,0,0 +1,PAYMENT,2970.97,C46941357,38071.06,35100.09,M1152606315,0,0,0,0 +1,PAYMENT,38.66,C343345308,16174,16135.34,M1714688478,0,0,0,0 +1,PAYMENT,2252.44,C104716441,1627,0,M1506951181,0,0,0,0 +1,TRANSFER,62610.8,C1976401987,79114,16503.2,C1937962514,517,8383.29,0,0 +1,DEBIT,5529.13,C867288517,8547,3017.87,C242131142,10206,0,0,0 +1,CASH_OUT,82940.31,C1528834618,3017.87,0,C476800120,132372,49864.36,0,0 +1,DEBIT,4510.22,C280615803,10256,5745.78,C1254526270,10697,0,0,0 +1,DEBIT,8727.74,C166694583,882770,874042.26,C1129670968,12636,0,0,0 +1,PAYMENT,2599.46,C885910946,874042.26,871442.79,M1860591867,0,0,0,0 +1,DEBIT,4874.49,C811207775,153,0,C1971489295,253104,0,0,0 +1,PAYMENT,6440.78,C1161148117,2192,0,M516875052,0,0,0,0 +1,PAYMENT,4910.14,C1131592118,41551,36640.86,M589987187,0,0,0,0 +1,PAYMENT,6444.64,C1262609629,12019,5574.36,M587180314,0,0,0,0 +1,DEBIT,5149.66,C1955990522,4782,0,C1330106945,52752,24044.18,0,0 +1,PAYMENT,7292.16,C69673470,216827,209534.84,M1082411691,0,0,0,0 +1,CASH_OUT,47458.86,C527211736,209534.84,162075.98,C2096057945,52120,0,0,0 +1,CASH_OUT,136872.92,C1533123860,162075.98,25203.05,C766572210,217806,0,0,0 +1,CASH_OUT,94253.33,C1718906711,25203.05,0,C977993101,99773,965870.05,0,0 +1,PAYMENT,2998.04,C71802912,12030,9031.96,M2134271532,0,0,0,0 +1,PAYMENT,3454.08,C686349795,9031.96,5577.88,M1831010686,0,0,0,0 +1,PAYMENT,4316.2,C1423768154,10999,6682.8,M404222443,0,0,0,0 +1,PAYMENT,2131.84,C1987977423,224,0,M61073295,0,0,0,0 +1,PAYMENT,12986.61,C807322507,23350,10363.39,M396485834,0,0,0,0 +1,TRANSFER,42712.39,C283039401,10363.39,0,C1330106945,57901.66,24044.18,0,0 +1,TRANSFER,77957.68,C207471778,0,0,C1761291320,94900,22233.65,0,0 +1,TRANSFER,17231.46,C1243171897,0,0,C783286238,24672,0,0,0 +1,TRANSFER,78766.03,C1376151044,0,0,C1749186397,103772,277515.05,0,0 +1,TRANSFER,224606.64,C873175411,0,0,C766572210,354678.92,0,0,0 +1,TRANSFER,125872.53,C1443967876,0,0,C392292416,348512,3420103.09,0,0 +1,TRANSFER,379856.23,C1449772539,0,0,C1590550415,900180,19169204.93,0,0 +1,TRANSFER,1505626.01,C926859124,0,0,C665576141,29031,5515763.34,0,0 +1,TRANSFER,554026.99,C1603696865,0,0,C766572210,579285.56,0,0,0 +1,TRANSFER,147543.1,C12905860,0,0,C1359044626,223220,16518.36,0,0 +1,TRANSFER,761507.39,C412788346,0,0,C1590550415,1280036.23,19169204.93,0,0 +1,TRANSFER,1429051.47,C1520267010,0,0,C1590550415,2041543.62,19169204.93,0,0 +1,TRANSFER,358831.92,C908084672,0,0,C392292416,474384.53,3420103.09,0,0 +1,TRANSFER,367768.4,C288306765,0,0,C1359044626,370763.1,16518.36,0,0 +1,TRANSFER,209711.11,C1556867940,0,0,C1509514333,399214.71,2415.16,0,0 +1,TRANSFER,583848.46,C1839168128,0,0,C1286084959,667778,2107778.11,0,0 +1,TRANSFER,1724887.05,C1495608502,0,0,C1590550415,3470595.1,19169204.93,0,0 +1,TRANSFER,710544.77,C835773569,0,0,C1359044626,738531.5,16518.36,0,0 +1,TRANSFER,581294.26,C843299092,0,0,C1590550415,5195482.15,19169204.93,0,0 +1,TRANSFER,11996.58,C605982374,0,0,C1225616405,40255,0,0,0 +1,PAYMENT,2875.1,C1412322831,15443,12567.9,M1651262695,0,0,0,0 +1,PAYMENT,8586.98,C1305004711,3763,0,M494077446,0,0,0,0 + diff --git a/hack/tools/test-script/expected.txt b/hack/tools/test-script/expected.txt new file mode 100644 index 0000000..b3e7a65 --- /dev/null +++ b/hack/tools/test-script/expected.txt @@ -0,0 +1,14 @@ + amount oldbalanceOrg +0 9839.64 XXXXX +1 1864.28 XXXXX +2 181.00 XXXXX +3 181.00 XXXXX +4 11668.14 XXXXX +.. ... ... +95 710544.77 XXXXX +96 581294.26 XXXXX +97 11996.58 XXXXX +98 2875.10 XXXXX +99 8586.98 XXXXX + +[100 rows x 2 columns] diff --git a/hack/tools/test-script/fybrikStorage-0.5.0.yaml b/hack/tools/test-script/fybrikStorage-0.5.0.yaml new file mode 100644 index 0000000..487a63e --- /dev/null +++ b/hack/tools/test-script/fybrikStorage-0.5.0.yaml @@ -0,0 +1,10 @@ +apiVersion: app.fybrik.io/v1alpha1 +kind: FybrikStorageAccount +metadata: + name: storage-account + namespace: fybrik-system +spec: + endpoint: "http://localstack.fybrik-notebook-sample.svc.cluster.local:4566" + regions: + - theshire + secretRef: bucket-creds \ No newline at end of file diff --git a/hack/tools/test-script/fybrikStorage-0.6.0.yaml b/hack/tools/test-script/fybrikStorage-0.6.0.yaml new file mode 100644 index 0000000..966ce04 --- /dev/null +++ b/hack/tools/test-script/fybrikStorage-0.6.0.yaml @@ -0,0 +1,10 @@ +apiVersion: app.fybrik.io/v1alpha1 +kind: FybrikStorageAccount +metadata: + name: storage-account + namespace: fybrik-system +spec: + id: theshire + endpoints: + theshire: "http://localstack.fybrik-notebook-sample.svc.cluster.local:4566" + secretRef: bucket-creds \ No newline at end of file diff --git a/hack/tools/test-script/fybrikapplication-0.5.0.yaml b/hack/tools/test-script/fybrikapplication-0.5.0.yaml new file mode 100644 index 0000000..cb4cbdd --- /dev/null +++ b/hack/tools/test-script/fybrikapplication-0.5.0.yaml @@ -0,0 +1,19 @@ +apiVersion: app.fybrik.io/v1alpha1 +kind: FybrikApplication +metadata: + name: my-notebook + labels: + app: my-notebook +spec: + selector: + workloadSelector: + matchLabels: + app: my-notebook + appInfo: + intent: fraud-detection + data: + - dataSetID: "fybrik-notebook-sample/paysim-csv" + requirements: + interface: + protocol: fybrik-arrow-flight + dataformat: arrow diff --git a/hack/tools/test-script/fybrikapplication-0.6.0.yaml b/hack/tools/test-script/fybrikapplication-0.6.0.yaml new file mode 100644 index 0000000..62d1ccb --- /dev/null +++ b/hack/tools/test-script/fybrikapplication-0.6.0.yaml @@ -0,0 +1,18 @@ +apiVersion: app.fybrik.io/v1alpha1 +kind: FybrikApplication +metadata: + name: my-notebook + labels: + app: my-notebook +spec: + selector: + workloadSelector: + matchLabels: + app: my-notebook + appInfo: + intent: Fraud Detection + data: + - dataSetID: "fybrik-notebook-sample/paysim-csv" + requirements: + interface: + protocol: fybrik-arrow-flight diff --git a/hack/tools/test-script/sample-policy-0.5.0.rego b/hack/tools/test-script/sample-policy-0.5.0.rego new file mode 100644 index 0000000..3cc08b3 --- /dev/null +++ b/hack/tools/test-script/sample-policy-0.5.0.rego @@ -0,0 +1,9 @@ +package dataapi.authz + +rule[{"action": {"name":"RedactAction", "columns": column_names}, "policy": description}] { + description := "Redact columns tagged as PII in datasets tagged with finance = true" + input.action.actionType == "read" + input.resource.tags.finance + column_names := [input.resource.columns[i].name | input.resource.columns[i].tags.PII] + count(column_names) > 0 +} diff --git a/hack/tools/test-script/sample-policy-0.6.0.rego b/hack/tools/test-script/sample-policy-0.6.0.rego new file mode 100644 index 0000000..2b1f3d3 --- /dev/null +++ b/hack/tools/test-script/sample-policy-0.6.0.rego @@ -0,0 +1,9 @@ +package dataapi.authz + +rule[{"action": {"name":"RedactAction", "columns": column_names}, "policy": description}] { + description := "Redact columns tagged as PII in datasets tagged with finance = true" + input.action.actionType == "read" + input.resource.metadata.tags.finance + column_names := [input.resource.metadata.columns[i].name | input.resource.metadata.columns[i].tags.PII] + count(column_names) > 0 +} diff --git a/hack/tools/test-script/test.py b/hack/tools/test-script/test.py new file mode 100644 index 0000000..78356ff --- /dev/null +++ b/hack/tools/test-script/test.py @@ -0,0 +1,23 @@ + + + +import json +import pyarrow.flight as fl +import pandas as pd + +# Create a Flight client +client = fl.connect('grpc://my-notebook-fybrik-notebook-sample-arrow-flight-aef23.fybrik-blueprints:80') + +# Prepare the request +request = { + "asset": "fybrik-notebook-sample/paysim-csv", + # To request specific columns add to the request a "columns" key with a list of column names + "columns": ["amount", "oldbalanceOrg"] +} + +# Send request and fetch result as a pandas DataFrame +info = client.get_flight_info(fl.FlightDescriptor.for_command(json.dumps(request))) +reader: fl.FlightStreamReader = client.do_get(info.endpoints[0].ticket) +df: pd.DataFrame = reader.read_pandas() +print(df) + diff --git a/hack/tools/test_module.sh b/hack/tools/test_module.sh new file mode 100755 index 0000000..1147570 --- /dev/null +++ b/hack/tools/test_module.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash + +set -x +set -e + + +export WORKING_DIR=test-script +export ACCESS_KEY=1234 +export SECRET_KEY=1234 + +kubernetesVersion=$1 +fybrikVersion=$2 +moduleVersion=$3 +module=$4 +certManagerVersion=$5 + +if [ $kubernetesVersion == "kind19" ] +then + bin/kind delete cluster + bin/kind create cluster --image=kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729 +elif [ $kubernetesVersion == "kind20" ] +then + bin/kind delete cluster + bin/kind create cluster --image=kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9 +elif [ $kubernetesVersion == "kind21" ] +then + bin/kind delete cluster + bin/kind create cluster --image=kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 +elif [ $kubernetesVersion == "kind22" ] +then + bin/kind delete cluster + bin/kind create cluster --image=kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047 +else + echo "Unsupported kind version" + exit 1 +fi + + +#quick start + +bin/helm repo add jetstack https://charts.jetstack.io +bin/helm repo add hashicorp https://helm.releases.hashicorp.com +bin/helm repo add fybrik-charts https://fybrik.github.io/charts +bin/helm repo update + + +bin/helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --version v1.2.0 \ + --create-namespace \ + --set installCRDs=true \ + --wait --timeout 400s + + + +bin/helm install vault fybrik-charts/vault --create-namespace -n fybrik-system \ + --set "vault.injector.enabled=false" \ + --set "vault.server.dev.enabled=true" \ + --values https://raw.githubusercontent.com/fybrik/fybrik/v0.5.3/charts/vault/env/dev/vault-single-cluster-values.yaml + bin/kubectl wait --for=condition=ready --all pod -n fybrik-system --timeout=400s + +bin/helm install fybrik-crd fybrik-charts/fybrik-crd -n fybrik-system --version v$fybrikVersion --wait +bin/helm install fybrik fybrik-charts/fybrik -n fybrik-system --version v$fybrikVersion --wait + + +#cd /data/fybrik-release-0.5.0/fybrik +#helm install fybrik-crd charts/fybrik-crd -n fybrik-system --wait +#helm install fybrik charts/fybrik --set global.tag=0.5.3 --set global.imagePullPolicy=Always -n fybrik-system --wait + + + +# apply modules + + +bin/kubectl apply -f https://github.com/fybrik/arrow-flight-module/releases/download/v$moduleVersion/module.yaml -n fybrik-system + + + + +# #datashim +# bin/kubectl apply -f https://raw.githubusercontent.com/datashim-io/datashim/master/release-tools/manifests/dlf.yaml +# bin/kubectl wait --for=condition=ready pods -l app.kubernetes.io/name=dlf -n dlf --timeout=600s +# # cd $PATH_TO_LOCAL_FYBRIK/third_party/datashim/ +# # make deploy + + + +# source ${EXPORT_FILE} + +sleep 10 + +# Notebook sample + +bin/kubectl create namespace fybrik-notebook-sample +bin/kubectl config set-context --current --namespace=fybrik-notebook-sample + +#localstack +bin/helm repo add localstack-charts https://localstack.github.io/helm-charts +bin/helm install localstack localstack-charts/localstack --set startServices="s3" --set service.type=ClusterIP +bin/kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=600s + +bin/kubectl port-forward svc/localstack 4566:4566 & + +sleep 10 + +export ENDPOINT="http://127.0.0.1:4566" +export BUCKET="demo" +export OBJECT_KEY="PS_20174392719_1491204439457_log.csv" +export FILEPATH="$WORKING_DIR/PS_20174392719_1491204439457_log.csv" +aws configure set aws_access_key_id ${ACCESS_KEY} && aws configure set aws_secret_access_key ${SECRET_KEY} && aws --endpoint-url=${ENDPOINT} s3api create-bucket --bucket ${BUCKET} && aws --endpoint-url=${ENDPOINT} s3api put-object --bucket ${BUCKET} --key ${OBJECT_KEY} --body ${FILEPATH} + + +cat << EOF | bin/kubectl apply -f - +apiVersion: v1 +kind: Secret +metadata: + name: paysim-csv +type: Opaque +stringData: + access_key: "${ACCESS_KEY}" + secret_key: "${SECRET_KEY}" +EOF + + +bin/kubectl apply -f $WORKING_DIR/Asset-$moduleVersion.yaml -n fybrik-notebook-sample +sleep 10 +bin/kubectl describe Asset paysim-csv -n fybrik-notebook-sample + + + +bin/kubectl -n fybrik-system create configmap sample-policy --from-file=$WORKING_DIR/sample-policy-$moduleVersion.rego +bin/kubectl -n fybrik-system label configmap sample-policy openpolicyagent.org/policy=rego + +c=0 +while [[ $(bin/kubectl get cm sample-policy -n fybrik-system -o 'jsonpath={.metadata.annotations.openpolicyagent\.org/policy-status}') != '{"status":"ok"}' ]] +do + echo "waiting" + ((c++)) && ((c==25)) && break + sleep 5 +done + + +bin/kubectl apply -f $WORKING_DIR/fybrikapplication-$moduleVersion.yaml + +c=0 +while [[ $(bin/kubectl get fybrikapplication my-notebook -o 'jsonpath={.status.ready}') != "true" ]] +do + echo "waiting" + ((c++)) && ((c==30)) && break + sleep 6 +done + +sleep 10 + + +bin/kubectl get pods -n fybrik-blueprints +POD_NAME=$(bin/kubectl get pods -n fybrik-blueprints -o=name | sed "s/^.\{4\}//") + + +bin/kubectl cp $WORKING_DIR/test.py ${POD_NAME}:/tmp -n fybrik-blueprints +bin/kubectl exec -i ${POD_NAME} -n fybrik-blueprints -- python /tmp/test.py > res.out + + +bin/kubectl logs ${POD_NAME} -n fybrik-blueprints + +DIFF=$(diff -b $WORKING_DIR/expected.txt res.out) +if [ "${DIFF}" == "" ] +then + echo "test succeeded" +else + echo "test failed" + exit 1 +fi From 4629512fbf3ff2fedb5722e40f9820c4c7c663d2 Mon Sep 17 00:00:00 2001 From: nassarmu10 Date: Thu, 3 Feb 2022 08:30:19 +0200 Subject: [PATCH 2/2] Remove unwanted code. Signed-off-by: nassarmu10 --- .github/workflows/test-module.yaml | 3 --- hack/make-rules/tools.mk | 4 ---- hack/tools/test-script/fybrikStorage-0.5.0.yaml | 10 ---------- hack/tools/test-script/fybrikStorage-0.6.0.yaml | 10 ---------- hack/tools/test_module.sh | 15 +++++++++------ 5 files changed, 9 insertions(+), 33 deletions(-) delete mode 100644 hack/tools/test-script/fybrikStorage-0.5.0.yaml delete mode 100644 hack/tools/test-script/fybrikStorage-0.6.0.yaml diff --git a/.github/workflows/test-module.yaml b/.github/workflows/test-module.yaml index 1d9a74f..e636d88 100644 --- a/.github/workflows/test-module.yaml +++ b/.github/workflows/test-module.yaml @@ -11,9 +11,6 @@ on: module-version: required: true default: '0.6.0' - module-type: - required: true - default: 'stream' cert-manager-version: required: true default: '1.6.2' diff --git a/hack/make-rules/tools.mk b/hack/make-rules/tools.mk index 828d78f..97d7e8b 100644 --- a/hack/make-rules/tools.mk +++ b/hack/make-rules/tools.mk @@ -1,22 +1,18 @@ INSTALL_TOOLS += $(TOOLBIN)/yq $(TOOLBIN)/yq: cd $(TOOLS_DIR); ./install_yq.sh - $(call post-install-check) INSTALL_TOOLS += $(TOOLBIN)/helm $(TOOLBIN)/helm: cd $(TOOLS_DIR); ./install_helm.sh - $(call post-install-check) INSTALL_TOOLS += $(TOOLBIN)/kind $(TOOLBIN)/kind: GOBIN=$(ABSTOOLBIN) go install sigs.k8s.io/kind@v0.11.1 - $(call post-install-check) INSTALL_TOOLS += $(TOOLBIN)/kubebuilder $(TOOLBIN)/kubebuilder $(TOOLBIN)/etcd $(TOOLBIN)/kube-apiserver $(TOOLBIN)/kubectl: cd $(TOOLS_DIR); ./install_kubebuilder.sh - $(call post-install-check) .PHONY: install-tools install-tools: $(INSTALL_TOOLS) diff --git a/hack/tools/test-script/fybrikStorage-0.5.0.yaml b/hack/tools/test-script/fybrikStorage-0.5.0.yaml deleted file mode 100644 index 487a63e..0000000 --- a/hack/tools/test-script/fybrikStorage-0.5.0.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: app.fybrik.io/v1alpha1 -kind: FybrikStorageAccount -metadata: - name: storage-account - namespace: fybrik-system -spec: - endpoint: "http://localstack.fybrik-notebook-sample.svc.cluster.local:4566" - regions: - - theshire - secretRef: bucket-creds \ No newline at end of file diff --git a/hack/tools/test-script/fybrikStorage-0.6.0.yaml b/hack/tools/test-script/fybrikStorage-0.6.0.yaml deleted file mode 100644 index 966ce04..0000000 --- a/hack/tools/test-script/fybrikStorage-0.6.0.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: app.fybrik.io/v1alpha1 -kind: FybrikStorageAccount -metadata: - name: storage-account - namespace: fybrik-system -spec: - id: theshire - endpoints: - theshire: "http://localstack.fybrik-notebook-sample.svc.cluster.local:4566" - secretRef: bucket-creds \ No newline at end of file diff --git a/hack/tools/test_module.sh b/hack/tools/test_module.sh index 1147570..c0f3693 100755 --- a/hack/tools/test_module.sh +++ b/hack/tools/test_module.sh @@ -11,8 +11,11 @@ export SECRET_KEY=1234 kubernetesVersion=$1 fybrikVersion=$2 moduleVersion=$3 -module=$4 -certManagerVersion=$5 +certManagerVersion=$4 + +# Trim the last two charts of the module version +# to construct the module resource path +moduleResourceVersion=${moduleVersion%??}".0" if [ $kubernetesVersion == "kind19" ] then @@ -46,7 +49,7 @@ bin/helm repo update bin/helm install cert-manager jetstack/cert-manager \ --namespace cert-manager \ - --version v1.2.0 \ + --version v$certManagerVersion \ --create-namespace \ --set installCRDs=true \ --wait --timeout 400s @@ -122,13 +125,13 @@ stringData: EOF -bin/kubectl apply -f $WORKING_DIR/Asset-$moduleVersion.yaml -n fybrik-notebook-sample +bin/kubectl apply -f $WORKING_DIR/Asset-$moduleResourceVersion.yaml -n fybrik-notebook-sample sleep 10 bin/kubectl describe Asset paysim-csv -n fybrik-notebook-sample -bin/kubectl -n fybrik-system create configmap sample-policy --from-file=$WORKING_DIR/sample-policy-$moduleVersion.rego +bin/kubectl -n fybrik-system create configmap sample-policy --from-file=$WORKING_DIR/sample-policy-$moduleResourceVersion.rego bin/kubectl -n fybrik-system label configmap sample-policy openpolicyagent.org/policy=rego c=0 @@ -140,7 +143,7 @@ do done -bin/kubectl apply -f $WORKING_DIR/fybrikapplication-$moduleVersion.yaml +bin/kubectl apply -f $WORKING_DIR/fybrikapplication-$moduleResourceVersion.yaml c=0 while [[ $(bin/kubectl get fybrikapplication my-notebook -o 'jsonpath={.status.ready}') != "true" ]]