Added the main module logic

Signed-off-by: Raviv Schaffer <[email protected]>

.gitignore

@@ -0,0 +1,8 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# Test scrips
+run.log
+update.sh
+install-sample.sh

Pipfile

@@ -3,9 +3,17 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 name = "pypi"
 
+[[source]]
+name = "testpypi"
+url = "https://test.pypi.org/simple"
+verify_ssl = true
+
 [packages]
 requests = "*"
 boto3 = "*"
+fybrik_python_vault = {version="0.1.0", index="testpypi"}
+pyyaml = "*"
+fybrik-python-logging = "*"
 
 [dev-packages]
 

build/Dockerfile

@@ -3,15 +3,15 @@ FROM python:3.8-slim
 ENV USER_UID=1001 \
     USER_NAME=delete-module
 
-
 COPY  build/bin /usr/local/bin
 RUN  /usr/local/bin/user_setup
 
 COPY requirements.txt /tmp/requirements.txt
 RUN python3 -m pip install --no-cache-dir -r /tmp/requirements.txt
 
 WORKDIR /app
-COPY ./delete-module/main.py main.py
+COPY main.py main.py
+COPY deletem deletem
 
 ENTRYPOINT ["python3"]
 CMD ["main.py"]

deletem/config.py

@@ -0,0 +1,43 @@
+#
+# Copyright 2020 IBM Corp.
+# SPDX-License-Identifier: Apache-2.0
+#
+import yaml
+
+
+class Config:
+    def __init__(self, config_path):
+        # TODO: change to schemed yaml using schemed-yaml-config
+        with open(config_path, 'r') as stream:
+            self.values = yaml.safe_load(stream)
+
+    def for_asset(self, asset_name: str, capability="") -> dict:
+        for asset_info in self.values.get('data', []):
+            if asset_info['name'] == asset_name and (capability == "" or asset_info['capability'] == capability):
+                return asset_info
+        raise ValueError(
+            "Requested config for undefined asset: {}".format(asset_name))
+
+    @property
+    def app_uuid(self) -> str:
+        return self.values.get('app-uuid', '')
+
+    @property
+    def workers(self) -> list:
+        return self.values.get('workers', [])
+
+    @property
+    def auth(self) -> dict:
+        return self.values.get('auth', {})
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def connection_type(self, asset_name: str, capability="") -> str:
+       asset_info = self.for_asset(asset_name, capability)
+       if 'connection' in asset_info:
+           return asset_info['connection'].get('type')
+       return None

deletem/vault.py

@@ -0,0 +1,35 @@
+from fybrik_python_logging import logger, Error, DataSetID, ForUser
+from fybrik_python_vault import get_jwt_from_file, get_raw_secret_from_vault
+
+
+def get_credentials_from_vault(vault_credentials, datasetID):
+    jwt_file_path = vault_credentials.get('jwt_file_path', '/var/run/secrets/kubernetes.io/serviceaccount/token')
+    jwt = get_jwt_from_file(jwt_file_path)
+    vault_address = vault_credentials.get('address', 'https://localhost:8200')
+    secret_path = vault_credentials.get('secretPath', '/v1/secret/data/cred')
+    vault_auth = vault_credentials.get('authPath', '/v1/auth/kubernetes/login')
+    role = vault_credentials.get('role', 'demo')
+    logger.trace('getting vault credentials',
+        extra={'jwt_file_path': str(jwt_file_path),
+               'vault_address': str(vault_address),
+               'secret_path': str(secret_path),
+               'vault_auth': str(vault_auth),
+               'role': str(role),
+               DataSetID: datasetID,
+               ForUser: True})
+    credentials = get_raw_secret_from_vault(jwt, secret_path, vault_address, vault_auth, role, datasetID)
+    if not credentials:
+        raise ValueError("Vault credentials are missing")
+    if 'access_key' in credentials and 'secret_key' in credentials:
+        if credentials['access_key'] and credentials['secret_key']:
+            return credentials['access_key'], credentials['secret_key']
+        else:
+            if not credentials['access_key']:
+                logger.error("'access_key' must be non-empty",
+                             extra={DataSetID: datasetID, ForUser: True})
+            if not credentials['secret_key']:
+                logger.error("'secret_key' must be non-empty",
+                             extra={DataSetID: datasetID, ForUser: True})
+    logger.error("Expected both 'access_key' and 'secret_key' fields in vault secret",
+                 extra={DataSetID: datasetID, ForUser: True})
+    raise ValueError("Vault credentials are missing")

hack/localstack.sh

@@ -3,29 +3,36 @@
 set -x
 set -e
 
-
-export WORKING_DIR=test-script
-export ACCESS_KEY=1234
-export SECRET_KEY=1234
-
-
 # Notebook sample
-
 kubectl create namespace fybrik-notebook-sample
 kubectl config set-context --current --namespace=fybrik-notebook-sample
 
-
-#localstack
+# localstack installation
 helm repo add localstack-charts https://localstack.github.io/helm-charts
 helm install localstack localstack-charts/localstack --set startServices="s3" --set service.type=ClusterIP
 kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=600s
 
 kubectl port-forward svc/localstack 4566:4566 &
 
-
+# Configure aws, add objects
+export WORKING_DIR=/home/ravivs/delete-module/hack/test-script
+export ACCESS_KEY=1234
+export SECRET_KEY=1234
 export ENDPOINT="http://127.0.0.1:4566"
 export BUCKET="demo"
 export OBJECT_KEY="PS_20174392719_1491204439457_log.csv"
 export FILEPATH="$WORKING_DIR/PS_20174392719_1491204439457_log.csv"
 aws configure set aws_access_key_id ${ACCESS_KEY} && aws configure set aws_secret_access_key ${SECRET_KEY} && aws --endpoint-url=${ENDPOINT} s3api create-bucket --bucket ${BUCKET} && aws --endpoint-url=${ENDPOINT} s3api put-object --bucket ${BUCKET} --key ${OBJECT_KEY} --body ${FILEPATH}
-alias awsls="aws --endpoint-url=$ENDPOINT"
+
+alias awsls="aws --endpoint-url=$ENDPOINT"
+
+cat << EOF | kubectl apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: paysim-csv
+type: Opaque
+stringData:
+  access_key: "${ACCESS_KEY}"
+  secret_key: "${SECRET_KEY}"
+EOF

main.py

@@ -0,0 +1,52 @@
+from deletem.config import Config
+import deletem.vault as vault
+from fybrik_python_logging import init_logger, logger
+import boto3
+
+def s3_connection(endpoint, aws_access_key, aws_secret_key):
+    try:
+        logger.info("Connecting S3 client")
+        s3_resource = boto3.resource("s3", endpoint_url=endpoint, aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key)
+        s3_client = boto3.client("s3", endpoint_url=endpoint, aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key)
+    except:
+        logger.info("Could not connect to S3 client")
+        raise
+    else:
+        return s3_resource, s3_client
+
+def delete_object(s3_client, bucket_name, object_key):
+    try:
+        logger.info(f"Deleting object: '{object_key}' from bucket: '{bucket_name}'")
+        response = s3_client.delete_object(Bucket=bucket_name, Key=object_key)
+    except:
+        logger.error("Could not delete object")
+        raise
+    else:
+        return response
+
+
+if __name__ == "__main__":
+    init_logger("TRACE", "7d320bd3-df69-4c66-ba58-f6de26fa1744", 'delete-module')
+    logger.info('Delete module initialized')
+
+    conf = Config("/etc/conf/conf.yaml")
+    conf_data = conf.values["data"][0]
+    dataID = conf_data["name"]
+    endpoint = conf_data["connection"]["s3"]["endpoint_url"]
+    vault_cred = conf_data["connection"]["s3"]["vault_credentials"]
+
+    access_key, secret_key = vault.get_credentials_from_vault(vault_cred, dataID)
+
+    s3_resource, s3_client = s3_connection(endpoint, access_key, secret_key)
+
+    bucket_name, object_key = conf_data["path"].split("/")
+    logger.info("Deleting object")
+    delete_object(s3_client, bucket_name, object_key)
+    logger.info(f"Object '{object_key}' deleted.")
+
+
+
+
+
+
+