Added some more template structure files

Signed-off-by: Raviv Schaffer <[email protected]>

Pipfile

@@ -5,6 +5,7 @@ name = "pypi"
 
 [packages]
 requests = "*"
+boto3 = "*"
 
 [dev-packages]
 

delete-module/main.py

@@ -1,5 +1,8 @@
 import requests
+import boto3
 
 response = requests.get('https://httpbin.org/ip')
 
-print('Your IP is {0}'.format(response.json()['origin']))
+print('Your IP is {0}'.format(response.json()['origin']))
+print("Changed")
+s3_resource = boto3.resource("s3", endpoint_url="http://127.0.0.1:4566", aws_access_key_id="1234", aws_secret_access_key="1234")

hack/localstack.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -x
+set -e
+
+
+export WORKING_DIR=test-script
+export ACCESS_KEY=1234
+export SECRET_KEY=1234
+
+
+# Notebook sample
+
+kubectl create namespace fybrik-notebook-sample
+kubectl config set-context --current --namespace=fybrik-notebook-sample
+
+
+#localstack
+helm repo add localstack-charts https://localstack.github.io/helm-charts
+helm install localstack localstack-charts/localstack --set startServices="s3" --set service.type=ClusterIP
+kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=600s
+
+kubectl port-forward svc/localstack 4566:4566 &
+
+
+export ENDPOINT="http://127.0.0.1:4566"
+export BUCKET="demo"
+export OBJECT_KEY="PS_20174392719_1491204439457_log.csv"
+export FILEPATH="$WORKING_DIR/PS_20174392719_1491204439457_log.csv"
+aws configure set aws_access_key_id ${ACCESS_KEY} && aws configure set aws_secret_access_key ${SECRET_KEY} && aws --endpoint-url=${ENDPOINT} s3api create-bucket --bucket ${BUCKET} && aws --endpoint-url=${ENDPOINT} s3api put-object --bucket ${BUCKET} --key ${OBJECT_KEY} --body ${FILEPATH}
+alias awsls="aws --endpoint-url=$ENDPOINT"

hack/make-rules/tools.mk

@@ -11,7 +11,7 @@ $(TOOLBIN)/kind:
 	GOBIN=$(ABSTOOLBIN) go install sigs.k8s.io/[email protected]
 
 INSTALL_TOOLS += $(TOOLBIN)/kubebuilder
-$(TOOLBIN)/kubebuilder $(TOOLBIN)/etcd $(TOOLBIN)/kube-apiserver $(TOOLBIN)/kubectl:
+$(TOOLBIN)/kubebuilder $(TOOLBIN)/etcd $(TOOLBIN)/kubectl:
 	cd $(TOOLS_DIR); ./install_kubebuilder.sh
 
 .PHONY: install-tools

hack/test-script/PS_20174392719_1491204439457_log.csv

@@ -0,0 +1,102 @@
+step,type,amount,nameOrig,oldbalanceOrg,newbalanceOrig,nameDest,oldbalanceDest,newbalanceDest,isFraud,isFlaggedFraud
+1,PAYMENT,9839.64,C1231006815,170136,160296.36,M1979787155,0,0,0,0
+1,PAYMENT,1864.28,C1666544295,21249,19384.72,M2044282225,0,0,0,0
+1,TRANSFER,181,C1305486145,181,0,C553264065,0,0,1,0
+1,CASH_OUT,181,C840083671,181,0,C38997010,21182,0,1,0
+1,PAYMENT,11668.14,C2048537720,41554,29885.86,M1230701703,0,0,0,0
+1,PAYMENT,7817.71,C90045638,53860,46042.29,M573487274,0,0,0,0
+1,PAYMENT,7107.77,C154988899,183195,176087.23,M408069119,0,0,0,0
+1,PAYMENT,7861.64,C1912850431,176087.23,168225.59,M633326333,0,0,0,0
+1,PAYMENT,4024.36,C1265012928,2671,0,M1176932104,0,0,0,0
+1,DEBIT,5337.77,C712410124,41720,36382.23,C195600860,41898,40348.79,0,0
+1,DEBIT,9644.94,C1900366749,4465,0,C997608398,10845,157982.12,0,0
+1,PAYMENT,3099.97,C249177573,20771,17671.03,M2096539129,0,0,0,0
+1,PAYMENT,2560.74,C1648232591,5070,2509.26,M972865270,0,0,0,0
+1,PAYMENT,11633.76,C1716932897,10127,0,M801569151,0,0,0,0
+1,PAYMENT,4098.78,C1026483832,503264,499165.22,M1635378213,0,0,0,0
+1,CASH_OUT,229133.94,C905080434,15325,0,C476402209,5083,51513.44,0,0
+1,PAYMENT,1563.82,C761750706,450,0,M1731217984,0,0,0,0
+1,PAYMENT,1157.86,C1237762639,21156,19998.14,M1877062907,0,0,0,0
+1,PAYMENT,671.64,C2033524545,15123,14451.36,M473053293,0,0,0,0
+1,TRANSFER,215310.3,C1670993182,705,0,C1100439041,22425,0,0,0
+1,PAYMENT,1373.43,C20804602,13854,12480.57,M1344519051,0,0,0,0
+1,DEBIT,9302.79,C1566511282,11299,1996.21,C1973538135,29832,16896.7,0,0
+1,DEBIT,1065.41,C1959239586,1817,751.59,C515132998,10330,0,0,0
+1,PAYMENT,3876.41,C504336483,67852,63975.59,M1404932042,0,0,0,0
+1,TRANSFER,311685.89,C1984094095,10835,0,C932583850,6267,2719172.89,0,0
+1,PAYMENT,6061.13,C1043358826,443,0,M1558079303,0,0,0,0
+1,PAYMENT,9478.39,C1671590089,116494,107015.61,M58488213,0,0,0,0
+1,PAYMENT,8009.09,C1053967012,10968,2958.91,M295304806,0,0,0,0
+1,PAYMENT,8901.99,C1632497828,2958.91,0,M33419717,0,0,0,0
+1,PAYMENT,9920.52,C764826684,0,0,M1940055334,0,0,0,0
+1,PAYMENT,3448.92,C2103763750,0,0,M335107734,0,0,0,0
+1,PAYMENT,4206.84,C215078753,0,0,M1757317128,0,0,0,0
+1,PAYMENT,5885.56,C840514538,0,0,M1804441305,0,0,0,0
+1,PAYMENT,5307.88,C1768242710,0,0,M1971783162,0,0,0,0
+1,PAYMENT,5031.22,C247113419,0,0,M151442075,0,0,0,0
+1,PAYMENT,24213.67,C1238616099,0,0,M70695990,0,0,0,0
+1,PAYMENT,8603.42,C1608633989,253,0,M1615617512,0,0,0,0
+1,PAYMENT,2791.42,C923341586,300481,297689.58,M107994825,0,0,0,0
+1,PAYMENT,7413.54,C1470868839,297689.58,290276.03,M1426725223,0,0,0,0
+1,PAYMENT,3295.19,C711197015,233633,230337.81,M1384454980,0,0,0,0
+1,PAYMENT,1684.81,C1481594086,297,0,M1569435561,0,0,0,0
+1,DEBIT,5758.59,C1466917878,32604,26845.41,C1297685781,209699,16997.22,0,0
+1,CASH_OUT,110414.71,C768216420,26845.41,0,C1509514333,288800,2415.16,0,0
+1,PAYMENT,7823.46,C260084831,998,0,M267814113,0,0,0,0
+1,PAYMENT,5086.48,C598357562,0,0,M1593224710,0,0,0,0
+1,PAYMENT,5281.48,C1440738283,152019,146737.52,M1849015357,0,0,0,0
+1,PAYMENT,13875.98,C484199463,15818,1942.02,M2008106788,0,0,0,0
+1,CASH_OUT,56953.9,C1570470538,1942.02,0,C824009085,70253,64106.18,0,0
+1,CASH_OUT,5346.89,C512549200,0,0,C248609774,652637,6453430.91,0,0
+1,PAYMENT,2204.04,C1615801298,586,0,M490391704,0,0,0,0
+1,PAYMENT,2641.47,C460570271,23053,20411.53,M1653361344,0,0,0,0
+1,CASH_OUT,23261.3,C2072313080,20411.53,0,C2001112025,25742,0,0,0
+1,PAYMENT,2330.64,C816944408,203543,201212.36,M909132503,0,0,0,0
+1,PAYMENT,1614.64,C912966811,41276,39661.36,M1792384402,0,0,0,0
+1,PAYMENT,9164.71,C1458621573,47235.77,38071.06,M1658980982,0,0,0,0
+1,PAYMENT,2970.97,C46941357,38071.06,35100.09,M1152606315,0,0,0,0
+1,PAYMENT,38.66,C343345308,16174,16135.34,M1714688478,0,0,0,0
+1,PAYMENT,2252.44,C104716441,1627,0,M1506951181,0,0,0,0
+1,TRANSFER,62610.8,C1976401987,79114,16503.2,C1937962514,517,8383.29,0,0
+1,DEBIT,5529.13,C867288517,8547,3017.87,C242131142,10206,0,0,0
+1,CASH_OUT,82940.31,C1528834618,3017.87,0,C476800120,132372,49864.36,0,0
+1,DEBIT,4510.22,C280615803,10256,5745.78,C1254526270,10697,0,0,0
+1,DEBIT,8727.74,C166694583,882770,874042.26,C1129670968,12636,0,0,0
+1,PAYMENT,2599.46,C885910946,874042.26,871442.79,M1860591867,0,0,0,0
+1,DEBIT,4874.49,C811207775,153,0,C1971489295,253104,0,0,0
+1,PAYMENT,6440.78,C1161148117,2192,0,M516875052,0,0,0,0
+1,PAYMENT,4910.14,C1131592118,41551,36640.86,M589987187,0,0,0,0
+1,PAYMENT,6444.64,C1262609629,12019,5574.36,M587180314,0,0,0,0
+1,DEBIT,5149.66,C1955990522,4782,0,C1330106945,52752,24044.18,0,0
+1,PAYMENT,7292.16,C69673470,216827,209534.84,M1082411691,0,0,0,0
+1,CASH_OUT,47458.86,C527211736,209534.84,162075.98,C2096057945,52120,0,0,0
+1,CASH_OUT,136872.92,C1533123860,162075.98,25203.05,C766572210,217806,0,0,0
+1,CASH_OUT,94253.33,C1718906711,25203.05,0,C977993101,99773,965870.05,0,0
+1,PAYMENT,2998.04,C71802912,12030,9031.96,M2134271532,0,0,0,0
+1,PAYMENT,3454.08,C686349795,9031.96,5577.88,M1831010686,0,0,0,0
+1,PAYMENT,4316.2,C1423768154,10999,6682.8,M404222443,0,0,0,0
+1,PAYMENT,2131.84,C1987977423,224,0,M61073295,0,0,0,0
+1,PAYMENT,12986.61,C807322507,23350,10363.39,M396485834,0,0,0,0
+1,TRANSFER,42712.39,C283039401,10363.39,0,C1330106945,57901.66,24044.18,0,0
+1,TRANSFER,77957.68,C207471778,0,0,C1761291320,94900,22233.65,0,0
+1,TRANSFER,17231.46,C1243171897,0,0,C783286238,24672,0,0,0
+1,TRANSFER,78766.03,C1376151044,0,0,C1749186397,103772,277515.05,0,0
+1,TRANSFER,224606.64,C873175411,0,0,C766572210,354678.92,0,0,0
+1,TRANSFER,125872.53,C1443967876,0,0,C392292416,348512,3420103.09,0,0
+1,TRANSFER,379856.23,C1449772539,0,0,C1590550415,900180,19169204.93,0,0
+1,TRANSFER,1505626.01,C926859124,0,0,C665576141,29031,5515763.34,0,0
+1,TRANSFER,554026.99,C1603696865,0,0,C766572210,579285.56,0,0,0
+1,TRANSFER,147543.1,C12905860,0,0,C1359044626,223220,16518.36,0,0
+1,TRANSFER,761507.39,C412788346,0,0,C1590550415,1280036.23,19169204.93,0,0
+1,TRANSFER,1429051.47,C1520267010,0,0,C1590550415,2041543.62,19169204.93,0,0
+1,TRANSFER,358831.92,C908084672,0,0,C392292416,474384.53,3420103.09,0,0
+1,TRANSFER,367768.4,C288306765,0,0,C1359044626,370763.1,16518.36,0,0
+1,TRANSFER,209711.11,C1556867940,0,0,C1509514333,399214.71,2415.16,0,0
+1,TRANSFER,583848.46,C1839168128,0,0,C1286084959,667778,2107778.11,0,0
+1,TRANSFER,1724887.05,C1495608502,0,0,C1590550415,3470595.1,19169204.93,0,0
+1,TRANSFER,710544.77,C835773569,0,0,C1359044626,738531.5,16518.36,0,0
+1,TRANSFER,581294.26,C843299092,0,0,C1590550415,5195482.15,19169204.93,0,0
+1,TRANSFER,11996.58,C605982374,0,0,C1225616405,40255,0,0,0
+1,PAYMENT,2875.1,C1412322831,15443,12567.9,M1651262695,0,0,0,0
+1,PAYMENT,8586.98,C1305004711,3763,0,M494077446,0,0,0,0
+

helm/delete-module/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
-description: A Helm chart for installing AFM in Kubernetes
-name: arrow-flight-module-chart
+description: A Helm chart for deleting data in Fybrik
+name: delete-module
 version: "0.0.0"

helm/delete-module/templates/deployment.yaml → helm/delete-module/templates/job.yaml

@@ -1,5 +1,5 @@
-apiVersion: apps/v1
-kind: Deployment
+apiVersion: batch/v1
+kind: Job
 metadata:
   name: {{ include "arrow-flight-module.fullname" . }}
   labels:
@@ -8,11 +8,6 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
-  replicas: {{ .Values.replicaCount }}
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: {{ include "arrow-flight-module.name" . }}
-      app.kubernetes.io/instance: {{ .Release.Name }}
   template:
     metadata:
       labels:
@@ -24,6 +19,7 @@ spec:
       annotations:
         sidecar.istio.io/inject: "true"
     spec:
+      restartPolicy: OnFailure
       serviceAccountName: {{ include "arrow-flight-module.fullname" . }}
       containers:
         - name: {{ .Chart.Name }}

helm/delete-module/templates/service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}
+  labels:
+    app.kubernetes.io/name: {{ include "arrow-flight-module.name" . }}
+    helm.sh/chart: {{ include "arrow-flight-module.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: 8080
+      protocol: TCP
+      name: grpc
+  selector:
+    app.kubernetes.io/name: {{ include "arrow-flight-module.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}

helm/delete-module/templates/serviceaccount.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "arrow-flight-module.fullname" . }}
+{{- if .Values.image.pullSecret }}
+imagePullSecrets: 
+  - name: {{ .Values.image.pullSecret }}
+{{- end }}

helm/delete-module/values.yaml

@@ -5,7 +5,7 @@
 replicaCount: 1
 
 image:
-  repository: ghcr.io/raviv-s/arrow-flight-module
+  repository: ghcr.io/raviv-s/delete-module
   tag: 0.0.0
   pullPolicy: Always
   pullSecret: null