Add a workflow to test the module.

.github/workflows/test-module.yaml

@@ -0,0 +1,41 @@
+name: Test
+on: 
+  workflow_dispatch:
+    inputs:
+      kind-version:
+        required: true
+        default: 'kind19'
+      fybrik-version:
+        required: true
+        default: '0.6.0'
+      module-version:
+        required: true
+        default: '0.6.0'
+      cert-manager-version:
+        required: true
+        default: '1.6.2'
+
+env:
+  GO_VERSION: 1.17
+  AWS_PAGER: ""
+  AWS_EC2_METADATA_DISABLED: true
+
+jobs:
+  test-module:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - name: Install tools
+      run: make install-tools
+    - name: Install AWS cli
+      uses: unfor19/install-aws-cli-action@v1
+      with:
+        version: 2     # default
+        verbose: false # default
+    - name: run-test
+      run: pushd hack/tools && ./test_module.sh ${{ github.event.inputs.kind-version }} ${{ github.event.inputs.fybrik-version }} ${{ github.event.inputs.module-version }} ${{ github.event.inputs.module-type }} ${{ github.event.inputs.cert-manager-version }} && popd
+

hack/make-rules/tools.mk

@@ -1,5 +1,25 @@
+INSTALL_TOOLS += $(TOOLBIN)/yq
 $(TOOLBIN)/yq:
 	cd $(TOOLS_DIR); ./install_yq.sh
 
+INSTALL_TOOLS += $(TOOLBIN)/helm
 $(TOOLBIN)/helm:
 	cd $(TOOLS_DIR); ./install_helm.sh
+
+INSTALL_TOOLS += $(TOOLBIN)/kind
+$(TOOLBIN)/kind:
+	GOBIN=$(ABSTOOLBIN) go install sigs.k8s.io/[email protected]
+
+INSTALL_TOOLS += $(TOOLBIN)/kubebuilder
+$(TOOLBIN)/kubebuilder $(TOOLBIN)/etcd $(TOOLBIN)/kube-apiserver $(TOOLBIN)/kubectl:
+	cd $(TOOLS_DIR); ./install_kubebuilder.sh
+
+.PHONY: install-tools
+install-tools: $(INSTALL_TOOLS)
+
+.PHONY: uninstall-tools
+uninstall-tools:
+	rm -rf $(INSTALL_TOOLS)
+
+
+

hack/tools/install_kubebuilder.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright 2020 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+source ./common.sh
+
+version=2.3.1
+
+header_text "Checking for bin/etcd"
+[[ -f bin/etcd ]] && exit 0
+
+header_text "Installing bin/etcd"
+mkdir -p ./bin
+
+K8S_VERSION=1.19.2
+curl -sSLo envtest-bins.tar.gz "https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-${K8S_VERSION}-$(go env GOOS)-$(go env GOARCH).tar.gz"
+
+tar -zvxf envtest-bins.tar.gz
+mv kubebuilder/bin/* bin
+rm envtest-bins.tar.gz
+rm -r kubebuilder

hack/tools/test-script/Asset-0.5.0.yaml

@@ -0,0 +1,29 @@
+apiVersion: katalog.fybrik.io/v1alpha1
+kind: Asset
+metadata:
+  name: paysim-csv
+spec:
+  secretRef: 
+    name: paysim-csv
+  assetDetails:
+    dataFormat: csv
+    connection:
+      type: s3
+      s3:
+        endpoint: "http://localstack.fybrik-notebook-sample.svc.cluster.local:4566"
+        bucket: "demo"
+        objectKey: "PS_20174392719_1491204439457_log.csv"
+  assetMetadata:
+    geography: theshire
+    tags:
+    - finance
+    componentsMetadata:
+      nameOrig: 
+        tags:
+        - PII
+      oldbalanceOrg:
+        tags:
+        - PII
+      newbalanceOrig:
+        tags:
+        - PII

hack/tools/test-script/Asset-0.6.0.yaml

@@ -0,0 +1,30 @@
+apiVersion: katalog.fybrik.io/v1alpha1
+kind: Asset
+metadata:
+  name: paysim-csv
+spec:
+  secretRef: 
+    name: paysim-csv
+  details:
+    dataFormat: csv
+    connection:
+      name: s3
+      s3:
+        endpoint: "http://localstack.fybrik-notebook-sample.svc.cluster.local:4566"
+        bucket: "demo"
+        object_key: "PS_20174392719_1491204439457_log.csv"
+  metadata:
+    name: Synthetic Financial Datasets For Fraud Detection
+    geography: theshire 
+    tags:
+      finance: true
+    columns:
+      - name: nameOrig
+        tags:
+          PII: true
+      - name: oldbalanceOrg
+        tags:
+          PII: true
+      - name: newbalanceOrig
+        tags:
+          PII: true

hack/tools/test-script/PS_20174392719_1491204439457_log.csv

@@ -0,0 +1,102 @@
+step,type,amount,nameOrig,oldbalanceOrg,newbalanceOrig,nameDest,oldbalanceDest,newbalanceDest,isFraud,isFlaggedFraud
+1,PAYMENT,9839.64,C1231006815,170136,160296.36,M1979787155,0,0,0,0
+1,PAYMENT,1864.28,C1666544295,21249,19384.72,M2044282225,0,0,0,0
+1,TRANSFER,181,C1305486145,181,0,C553264065,0,0,1,0
+1,CASH_OUT,181,C840083671,181,0,C38997010,21182,0,1,0
+1,PAYMENT,11668.14,C2048537720,41554,29885.86,M1230701703,0,0,0,0
+1,PAYMENT,7817.71,C90045638,53860,46042.29,M573487274,0,0,0,0
+1,PAYMENT,7107.77,C154988899,183195,176087.23,M408069119,0,0,0,0
+1,PAYMENT,7861.64,C1912850431,176087.23,168225.59,M633326333,0,0,0,0
+1,PAYMENT,4024.36,C1265012928,2671,0,M1176932104,0,0,0,0
+1,DEBIT,5337.77,C712410124,41720,36382.23,C195600860,41898,40348.79,0,0
+1,DEBIT,9644.94,C1900366749,4465,0,C997608398,10845,157982.12,0,0
+1,PAYMENT,3099.97,C249177573,20771,17671.03,M2096539129,0,0,0,0
+1,PAYMENT,2560.74,C1648232591,5070,2509.26,M972865270,0,0,0,0
+1,PAYMENT,11633.76,C1716932897,10127,0,M801569151,0,0,0,0
+1,PAYMENT,4098.78,C1026483832,503264,499165.22,M1635378213,0,0,0,0
+1,CASH_OUT,229133.94,C905080434,15325,0,C476402209,5083,51513.44,0,0
+1,PAYMENT,1563.82,C761750706,450,0,M1731217984,0,0,0,0
+1,PAYMENT,1157.86,C1237762639,21156,19998.14,M1877062907,0,0,0,0
+1,PAYMENT,671.64,C2033524545,15123,14451.36,M473053293,0,0,0,0
+1,TRANSFER,215310.3,C1670993182,705,0,C1100439041,22425,0,0,0
+1,PAYMENT,1373.43,C20804602,13854,12480.57,M1344519051,0,0,0,0
+1,DEBIT,9302.79,C1566511282,11299,1996.21,C1973538135,29832,16896.7,0,0
+1,DEBIT,1065.41,C1959239586,1817,751.59,C515132998,10330,0,0,0
+1,PAYMENT,3876.41,C504336483,67852,63975.59,M1404932042,0,0,0,0
+1,TRANSFER,311685.89,C1984094095,10835,0,C932583850,6267,2719172.89,0,0
+1,PAYMENT,6061.13,C1043358826,443,0,M1558079303,0,0,0,0
+1,PAYMENT,9478.39,C1671590089,116494,107015.61,M58488213,0,0,0,0
+1,PAYMENT,8009.09,C1053967012,10968,2958.91,M295304806,0,0,0,0
+1,PAYMENT,8901.99,C1632497828,2958.91,0,M33419717,0,0,0,0
+1,PAYMENT,9920.52,C764826684,0,0,M1940055334,0,0,0,0
+1,PAYMENT,3448.92,C2103763750,0,0,M335107734,0,0,0,0
+1,PAYMENT,4206.84,C215078753,0,0,M1757317128,0,0,0,0
+1,PAYMENT,5885.56,C840514538,0,0,M1804441305,0,0,0,0
+1,PAYMENT,5307.88,C1768242710,0,0,M1971783162,0,0,0,0
+1,PAYMENT,5031.22,C247113419,0,0,M151442075,0,0,0,0
+1,PAYMENT,24213.67,C1238616099,0,0,M70695990,0,0,0,0
+1,PAYMENT,8603.42,C1608633989,253,0,M1615617512,0,0,0,0
+1,PAYMENT,2791.42,C923341586,300481,297689.58,M107994825,0,0,0,0
+1,PAYMENT,7413.54,C1470868839,297689.58,290276.03,M1426725223,0,0,0,0
+1,PAYMENT,3295.19,C711197015,233633,230337.81,M1384454980,0,0,0,0
+1,PAYMENT,1684.81,C1481594086,297,0,M1569435561,0,0,0,0
+1,DEBIT,5758.59,C1466917878,32604,26845.41,C1297685781,209699,16997.22,0,0
+1,CASH_OUT,110414.71,C768216420,26845.41,0,C1509514333,288800,2415.16,0,0
+1,PAYMENT,7823.46,C260084831,998,0,M267814113,0,0,0,0
+1,PAYMENT,5086.48,C598357562,0,0,M1593224710,0,0,0,0
+1,PAYMENT,5281.48,C1440738283,152019,146737.52,M1849015357,0,0,0,0
+1,PAYMENT,13875.98,C484199463,15818,1942.02,M2008106788,0,0,0,0
+1,CASH_OUT,56953.9,C1570470538,1942.02,0,C824009085,70253,64106.18,0,0
+1,CASH_OUT,5346.89,C512549200,0,0,C248609774,652637,6453430.91,0,0
+1,PAYMENT,2204.04,C1615801298,586,0,M490391704,0,0,0,0
+1,PAYMENT,2641.47,C460570271,23053,20411.53,M1653361344,0,0,0,0
+1,CASH_OUT,23261.3,C2072313080,20411.53,0,C2001112025,25742,0,0,0
+1,PAYMENT,2330.64,C816944408,203543,201212.36,M909132503,0,0,0,0
+1,PAYMENT,1614.64,C912966811,41276,39661.36,M1792384402,0,0,0,0
+1,PAYMENT,9164.71,C1458621573,47235.77,38071.06,M1658980982,0,0,0,0
+1,PAYMENT,2970.97,C46941357,38071.06,35100.09,M1152606315,0,0,0,0
+1,PAYMENT,38.66,C343345308,16174,16135.34,M1714688478,0,0,0,0
+1,PAYMENT,2252.44,C104716441,1627,0,M1506951181,0,0,0,0
+1,TRANSFER,62610.8,C1976401987,79114,16503.2,C1937962514,517,8383.29,0,0
+1,DEBIT,5529.13,C867288517,8547,3017.87,C242131142,10206,0,0,0
+1,CASH_OUT,82940.31,C1528834618,3017.87,0,C476800120,132372,49864.36,0,0
+1,DEBIT,4510.22,C280615803,10256,5745.78,C1254526270,10697,0,0,0
+1,DEBIT,8727.74,C166694583,882770,874042.26,C1129670968,12636,0,0,0
+1,PAYMENT,2599.46,C885910946,874042.26,871442.79,M1860591867,0,0,0,0
+1,DEBIT,4874.49,C811207775,153,0,C1971489295,253104,0,0,0
+1,PAYMENT,6440.78,C1161148117,2192,0,M516875052,0,0,0,0
+1,PAYMENT,4910.14,C1131592118,41551,36640.86,M589987187,0,0,0,0
+1,PAYMENT,6444.64,C1262609629,12019,5574.36,M587180314,0,0,0,0
+1,DEBIT,5149.66,C1955990522,4782,0,C1330106945,52752,24044.18,0,0
+1,PAYMENT,7292.16,C69673470,216827,209534.84,M1082411691,0,0,0,0
+1,CASH_OUT,47458.86,C527211736,209534.84,162075.98,C2096057945,52120,0,0,0
+1,CASH_OUT,136872.92,C1533123860,162075.98,25203.05,C766572210,217806,0,0,0
+1,CASH_OUT,94253.33,C1718906711,25203.05,0,C977993101,99773,965870.05,0,0
+1,PAYMENT,2998.04,C71802912,12030,9031.96,M2134271532,0,0,0,0
+1,PAYMENT,3454.08,C686349795,9031.96,5577.88,M1831010686,0,0,0,0
+1,PAYMENT,4316.2,C1423768154,10999,6682.8,M404222443,0,0,0,0
+1,PAYMENT,2131.84,C1987977423,224,0,M61073295,0,0,0,0
+1,PAYMENT,12986.61,C807322507,23350,10363.39,M396485834,0,0,0,0
+1,TRANSFER,42712.39,C283039401,10363.39,0,C1330106945,57901.66,24044.18,0,0
+1,TRANSFER,77957.68,C207471778,0,0,C1761291320,94900,22233.65,0,0
+1,TRANSFER,17231.46,C1243171897,0,0,C783286238,24672,0,0,0
+1,TRANSFER,78766.03,C1376151044,0,0,C1749186397,103772,277515.05,0,0
+1,TRANSFER,224606.64,C873175411,0,0,C766572210,354678.92,0,0,0
+1,TRANSFER,125872.53,C1443967876,0,0,C392292416,348512,3420103.09,0,0
+1,TRANSFER,379856.23,C1449772539,0,0,C1590550415,900180,19169204.93,0,0
+1,TRANSFER,1505626.01,C926859124,0,0,C665576141,29031,5515763.34,0,0
+1,TRANSFER,554026.99,C1603696865,0,0,C766572210,579285.56,0,0,0
+1,TRANSFER,147543.1,C12905860,0,0,C1359044626,223220,16518.36,0,0
+1,TRANSFER,761507.39,C412788346,0,0,C1590550415,1280036.23,19169204.93,0,0
+1,TRANSFER,1429051.47,C1520267010,0,0,C1590550415,2041543.62,19169204.93,0,0
+1,TRANSFER,358831.92,C908084672,0,0,C392292416,474384.53,3420103.09,0,0
+1,TRANSFER,367768.4,C288306765,0,0,C1359044626,370763.1,16518.36,0,0
+1,TRANSFER,209711.11,C1556867940,0,0,C1509514333,399214.71,2415.16,0,0
+1,TRANSFER,583848.46,C1839168128,0,0,C1286084959,667778,2107778.11,0,0
+1,TRANSFER,1724887.05,C1495608502,0,0,C1590550415,3470595.1,19169204.93,0,0
+1,TRANSFER,710544.77,C835773569,0,0,C1359044626,738531.5,16518.36,0,0
+1,TRANSFER,581294.26,C843299092,0,0,C1590550415,5195482.15,19169204.93,0,0
+1,TRANSFER,11996.58,C605982374,0,0,C1225616405,40255,0,0,0
+1,PAYMENT,2875.1,C1412322831,15443,12567.9,M1651262695,0,0,0,0
+1,PAYMENT,8586.98,C1305004711,3763,0,M494077446,0,0,0,0
+

hack/tools/test-script/expected.txt

@@ -0,0 +1,14 @@
+       amount oldbalanceOrg
+0     9839.64         XXXXX
+1     1864.28         XXXXX
+2      181.00         XXXXX
+3      181.00         XXXXX
+4    11668.14         XXXXX
+..        ...           ...
+95  710544.77         XXXXX
+96  581294.26         XXXXX
+97   11996.58         XXXXX
+98    2875.10         XXXXX
+99    8586.98         XXXXX
+
+[100 rows x 2 columns]

hack/tools/test-script/fybrikapplication-0.5.0.yaml

@@ -0,0 +1,19 @@
+apiVersion: app.fybrik.io/v1alpha1
+kind: FybrikApplication
+metadata:
+  name: my-notebook
+  labels:
+    app: my-notebook
+spec:
+  selector:
+    workloadSelector:
+      matchLabels:
+        app: my-notebook
+  appInfo:
+    intent: fraud-detection
+  data:
+    - dataSetID: "fybrik-notebook-sample/paysim-csv"
+      requirements:
+        interface: 
+          protocol: fybrik-arrow-flight
+          dataformat: arrow

hack/tools/test-script/fybrikapplication-0.6.0.yaml

@@ -0,0 +1,18 @@
+apiVersion: app.fybrik.io/v1alpha1
+kind: FybrikApplication
+metadata:
+  name: my-notebook
+  labels:
+    app: my-notebook
+spec:
+  selector:
+    workloadSelector:
+      matchLabels:
+        app: my-notebook
+  appInfo:
+    intent: Fraud Detection
+  data:
+    - dataSetID: "fybrik-notebook-sample/paysim-csv"
+      requirements:
+        interface: 
+          protocol: fybrik-arrow-flight

hack/tools/test-script/sample-policy-0.5.0.rego

@@ -0,0 +1,9 @@
+package dataapi.authz
+
+rule[{"action": {"name":"RedactAction", "columns": column_names}, "policy": description}] {
+  description := "Redact columns tagged as PII in datasets tagged with finance = true"
+  input.action.actionType == "read"
+  input.resource.tags.finance
+  column_names := [input.resource.columns[i].name | input.resource.columns[i].tags.PII]
+  count(column_names) > 0
+}

hack/tools/test-script/sample-policy-0.6.0.rego

@@ -0,0 +1,9 @@
+package dataapi.authz
+
+rule[{"action": {"name":"RedactAction", "columns": column_names}, "policy": description}] {
+  description := "Redact columns tagged as PII in datasets tagged with finance = true"
+  input.action.actionType == "read"
+  input.resource.metadata.tags.finance
+  column_names := [input.resource.metadata.columns[i].name | input.resource.metadata.columns[i].tags.PII]
+  count(column_names) > 0
+}

hack/tools/test-script/test.py

@@ -0,0 +1,23 @@
+
+
+
+import json
+import pyarrow.flight as fl
+import pandas as pd
+
+# Create a Flight client
+client = fl.connect('grpc://my-notebook-fybrik-notebook-sample-arrow-flight-aef23.fybrik-blueprints:80')
+
+# Prepare the request
+request = {
+    "asset": "fybrik-notebook-sample/paysim-csv",
+    # To request specific columns add to the request a "columns" key with a list of column names
+    "columns": ["amount", "oldbalanceOrg"]
+}
+
+# Send request and fetch result as a pandas DataFrame
+info = client.get_flight_info(fl.FlightDescriptor.for_command(json.dumps(request)))
+reader: fl.FlightStreamReader = client.do_get(info.endpoints[0].ticket)
+df: pd.DataFrame = reader.read_pandas()
+print(df)
+