From d78075eebcc583d45bd569e6a1ec6a0328687b67 Mon Sep 17 00:00:00 2001 From: Sivakumar Srinivasulu Date: Tue, 21 Nov 2023 01:29:51 -0600 Subject: [PATCH 1/7] created fms file retrieve python script --- buildspec-dev.yml | 18 ++++++++ tools/scripts/retrieve-fms-file.py | 74 ++++++++++++++++++++++++++++++ upload-part-file.sh | 47 +++++++++++-------- 3 files changed, 120 insertions(+), 19 deletions(-) create mode 100644 buildspec-dev.yml create mode 100755 tools/scripts/retrieve-fms-file.py mode change 100644 => 100755 upload-part-file.sh diff --git a/buildspec-dev.yml b/buildspec-dev.yml new file mode 100644 index 000000000..db26e8bf8 --- /dev/null +++ b/buildspec-dev.yml @@ -0,0 +1,18 @@ +version: 0.2 +env: + parameter-store: + AWS_ACCOUNT_ID: "/qppar-sf/account_id" + AUTH_URL: "/qppar-sf/${ENVIRONMENT}/conversion_tool/QPPA_TOKEN_URL" + FMS_URL: "/qppar-sf/${ENVIRONMENT}/conversion_tool/FMS_URL" + FMS_TOKEN: "/qppar-sf/${ENVIRONMENT}/conversion_tool/FMS_TOKEN" + FMS_PATH: "/qppar-sf/${ENVIRONMENT}/conversion_tool/FMS_PATH" +phases: + install: + runtime-versions: + python: 3.8 + test_build: + commands: + - chmod +x ./qppsfct-copy-certs.sh + - ./qppsfct-copy-certs.sh $ENVIRONMENT $AWS_DEFAULT_REGION + - chmod +x ./upload-part-file.sh + - ./upload-part-file.sh $AUTH_URL $FMS_URL $FMS_TOKEN $FMS_PATH \ No newline at end of file diff --git a/tools/scripts/retrieve-fms-file.py b/tools/scripts/retrieve-fms-file.py new file mode 100755 index 000000000..d1d7045d5 --- /dev/null +++ b/tools/scripts/retrieve-fms-file.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import argparse +import sys + +import requests + + +def get_user_inputs(): + parser = argparse.ArgumentParser() + parser.add_argument('-au', '--auth-url', required=True, type=str, + help='QPP Auth token retrieval url. Example: https://imp.qpp.cms.gov/api/auth/oauth/token') + parser.add_argument('-fu', '--fms-url', required=True, type=str, + help='FMS Base url. Example: https://impl.ar.qpp.internal/dataservices') + parser.add_argument('-t', '--fms-token', required=True, type=str, + help='QPP Auth client assertion token to retrieve the FMS S2S token') + parser.add_argument('-p', '--fms-path', required=True, type=str, + help='FMS path with file name and extension. Example: /folder/file.xlsx') + args = parser.parse_args() + + return args + + +def download_from_fms(auth_url, fms_url, fms_token, fms_path): + try: + d = {'client_assertion': fms_token, + 'client_assertion_type': 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer', + 'grant_type': 'client_credentials', + 'scope': 'analyticsAndReporting' + } + # print('starting s2s token retrieval request from qpp auth') + get_s2s_token = requests.post( + url=auth_url, + data=d, + headers={ + 'Content-Type': 'application/x-www-form-urlencoded', + 'Accept': 'application/vnd.qpp.cms.gov.v2+json' + } + ) + s2s_token = get_s2s_token.json()["data"]["token"] + # print('starting download from fms for file - ' + fms_path) + get_download_url = requests.post( + url=fms_url + '/get-file', + json={"path": fms_path}, + verify=False, + headers={ + 'Accept': 'application/vnd.qpp.cms.gov.v2+json', + 'Authorization': 'Bearer ' + s2s_token + } + ) + download_url = get_download_url.json()['presigned_url'] + + # upload_status = s3_client.put_object( + # Bucket=S3_BUCKET, + # Key=s3_path.split(PII_BUCKET_PATH)[1], + # Body=download_result.content, + # ServerSideEncryption='aws:kms' + # ) + print(download_url) + return download_url + + except Exception as err: + print(f"Unexpected Error. {err = }, {type(err) = }") + sys.exit(1) + + +def main(): + args = get_user_inputs() + s3_url = download_from_fms(args.auth_url, args.fms_url, args.fms_token, args.fms_path) + return s3_url + + +if __name__ == '__main__': + url = main() diff --git a/upload-part-file.sh b/upload-part-file.sh old mode 100644 new mode 100755 index 6da2341dc..3e094cf08 --- a/upload-part-file.sh +++ b/upload-part-file.sh @@ -1,25 +1,34 @@ #!/bin/bash -PART_FILE_BUCKET=$1 -PART_FILE=$2 -FORMATTED_FILE_NAME=$3 -AWS_REGION=$4 +AUTH_URL=$1 +FMS_URL=$2 +FMS_TOKEN=$3 +FMS_PATH=$4 -export AWS_REGION=${AWS_REGION} +chmod +x ./tools/scripts/retrieve-fms-file.py +s3_url=`python ./tools/scripts/retrieve-fms-file.py -au ${AUTH_TOKEN} -fu ${FMS_URL} -t ${FMS_TOKEN} -p ${FMS_PATH}` +echo $s3_url -pip install openpyxl -pip install simplejson +# PART_FILE_BUCKET=$1 +# PART_FILE=$2 +# FORMATTED_FILE_NAME=$3 +# AWS_REGION=$4 -aws s3 cp s3://${PART_FILE_BUCKET}/${PART_FILE} . -chmod +x ./tools/scripts/format-participation-file.py -python ./tools/scripts/format-participation-file.py ${PART_FILE} ${FORMATTED_FILE_NAME} -aws s3 mv ${FORMATTED_FILE_NAME} s3://${PART_FILE_BUCKET}/${FORMATTED_FILE_NAME} +# export AWS_REGION=${AWS_REGION} -if test -f "$FORMATTED_FILE_NAME" -then - echo 'Removing Formatted Participation file localy...' - rm ${FORMATTED_FILE_NAME} - echo 'Formatted Participation file has been removed locally.' -else - echo 'Formatted Participation file has been removed locally.' -fi \ No newline at end of file +# pip install openpyxl +# pip install simplejson + +# aws s3 cp s3://${PART_FILE_BUCKET}/${PART_FILE} . +# chmod +x ./tools/scripts/format-participation-file.py +# python ./tools/scripts/format-participation-file.py ${PART_FILE} ${FORMATTED_FILE_NAME} +# aws s3 mv ${FORMATTED_FILE_NAME} s3://${PART_FILE_BUCKET}/${FORMATTED_FILE_NAME} + +# if test -f "$FORMATTED_FILE_NAME" +# then +# echo 'Removing Formatted Participation file localy...' +# rm ${FORMATTED_FILE_NAME} +# echo 'Formatted Participation file has been removed locally.' +# else +# echo 'Formatted Participation file has been removed locally.' +# fi \ No newline at end of file From 645ae8a43e84cfa111cde80d376d8e38a19ce356 Mon Sep 17 00:00:00 2001 From: Sivakumar Srinivasulu Date: Tue, 21 Nov 2023 01:33:56 -0600 Subject: [PATCH 2/7] updated build phase for buildspec-dev --- buildspec-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec-dev.yml b/buildspec-dev.yml index db26e8bf8..d7d42a794 100644 --- a/buildspec-dev.yml +++ b/buildspec-dev.yml @@ -10,7 +10,7 @@ phases: install: runtime-versions: python: 3.8 - test_build: + pre_build: commands: - chmod +x ./qppsfct-copy-certs.sh - ./qppsfct-copy-certs.sh $ENVIRONMENT $AWS_DEFAULT_REGION From 296b4659ac2c046121b298f1ba85646d15444493 Mon Sep 17 00:00:00 2001 From: Sivakumar Srinivasulu Date: Tue, 21 Nov 2023 01:38:22 -0600 Subject: [PATCH 3/7] updated the variable name to reflect the arguments passed --- upload-part-file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upload-part-file.sh b/upload-part-file.sh index 3e094cf08..f95468bbb 100755 --- a/upload-part-file.sh +++ b/upload-part-file.sh @@ -6,7 +6,7 @@ FMS_TOKEN=$3 FMS_PATH=$4 chmod +x ./tools/scripts/retrieve-fms-file.py -s3_url=`python ./tools/scripts/retrieve-fms-file.py -au ${AUTH_TOKEN} -fu ${FMS_URL} -t ${FMS_TOKEN} -p ${FMS_PATH}` +s3_url=`python ./tools/scripts/retrieve-fms-file.py -au ${AUTH_URL} -fu ${FMS_URL} -t ${FMS_TOKEN} -p ${FMS_PATH}` echo $s3_url # PART_FILE_BUCKET=$1 From 98d6d3489e005bf2544c4a3fa91fdcf0f4a954a3 Mon Sep 17 00:00:00 2001 From: Sivakumar Srinivasulu Date: Tue, 21 Nov 2023 02:04:53 -0600 Subject: [PATCH 4/7] updated dev buildspec commands to verify the values are valid --- buildspec-dev.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/buildspec-dev.yml b/buildspec-dev.yml index d7d42a794..42212342a 100644 --- a/buildspec-dev.yml +++ b/buildspec-dev.yml @@ -12,7 +12,10 @@ phases: python: 3.8 pre_build: commands: + - echo ${AUTH_URL} + - echo ${FMS_URL} + - echo ${FMS_PATH} - chmod +x ./qppsfct-copy-certs.sh - ./qppsfct-copy-certs.sh $ENVIRONMENT $AWS_DEFAULT_REGION - chmod +x ./upload-part-file.sh - - ./upload-part-file.sh $AUTH_URL $FMS_URL $FMS_TOKEN $FMS_PATH \ No newline at end of file + - ./upload-part-file.sh ${AUTH_URL} ${FMS_URL} ${FMS_TOKEN} ${FMS_PATH} \ No newline at end of file From 68ad39e268f42e632943c59256b074d36f5a5047 Mon Sep 17 00:00:00 2001 From: Sivakumar Srinivasulu Date: Tue, 21 Nov 2023 17:14:10 -0600 Subject: [PATCH 5/7] updated the fms retrieval to run manually --- .gitignore | 1 + DEVELOPER.md | 14 ++++ buildspec-dev.yml | 21 ------ tools/scripts/retrieve-fms-file.py | 115 ++++++++++++++++++----------- upload-part-file.sh | 49 ++++++------ 5 files changed, 107 insertions(+), 93 deletions(-) delete mode 100644 buildspec-dev.yml diff --git a/.gitignore b/.gitignore index 527752346..fc1e90a8e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ bin/ .factorypath *.swp qrdaToQppAssociations.txt +local.env */.checkstyle diff --git a/DEVELOPER.md b/DEVELOPER.md index 65e2c9059..7bb318f0c 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -92,3 +92,17 @@ Finished a cool new feature or fix for the conversion tool? Please feel free sub See the [contribution guidelines](/.github/CONTRIBUTING.md) on how to submit a Pull Request. [readme]: /README.md + +## FMS Retrieval + +In order to retrieve files from FMS, `local.env` should contain values for below parameters. These contain sensitive information and cannot be committed to the repository. + +``` +auth_url='url' +fms_url='url' +fms_token='token' +fms_path='folder/file.ext' +s3_bucket='aws-pii-bucket' +``` + +With access to the conversion tool environment, update your local `~/.aws/credentials` file with new short-term AWS credentials before initiating the request ([learn more](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/setup-credentials.html)). \ No newline at end of file diff --git a/buildspec-dev.yml b/buildspec-dev.yml deleted file mode 100644 index 42212342a..000000000 --- a/buildspec-dev.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 0.2 -env: - parameter-store: - AWS_ACCOUNT_ID: "/qppar-sf/account_id" - AUTH_URL: "/qppar-sf/${ENVIRONMENT}/conversion_tool/QPPA_TOKEN_URL" - FMS_URL: "/qppar-sf/${ENVIRONMENT}/conversion_tool/FMS_URL" - FMS_TOKEN: "/qppar-sf/${ENVIRONMENT}/conversion_tool/FMS_TOKEN" - FMS_PATH: "/qppar-sf/${ENVIRONMENT}/conversion_tool/FMS_PATH" -phases: - install: - runtime-versions: - python: 3.8 - pre_build: - commands: - - echo ${AUTH_URL} - - echo ${FMS_URL} - - echo ${FMS_PATH} - - chmod +x ./qppsfct-copy-certs.sh - - ./qppsfct-copy-certs.sh $ENVIRONMENT $AWS_DEFAULT_REGION - - chmod +x ./upload-part-file.sh - - ./upload-part-file.sh ${AUTH_URL} ${FMS_URL} ${FMS_TOKEN} ${FMS_PATH} \ No newline at end of file diff --git a/tools/scripts/retrieve-fms-file.py b/tools/scripts/retrieve-fms-file.py index d1d7045d5..9f406e6de 100755 --- a/tools/scripts/retrieve-fms-file.py +++ b/tools/scripts/retrieve-fms-file.py @@ -1,9 +1,16 @@ #!/usr/bin/env python3 -import argparse import sys - +import boto3 +import argparse import requests +import simplejson as json +from io import BytesIO +from dotenv import dotenv_values +from openpyxl import load_workbook + +config = dotenv_values("local.env") +s3_client = boto3.client('s3') def get_user_inputs(): @@ -22,53 +29,73 @@ def get_user_inputs(): def download_from_fms(auth_url, fms_url, fms_token, fms_path): - try: - d = {'client_assertion': fms_token, - 'client_assertion_type': 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer', - 'grant_type': 'client_credentials', - 'scope': 'analyticsAndReporting' - } - # print('starting s2s token retrieval request from qpp auth') - get_s2s_token = requests.post( - url=auth_url, - data=d, - headers={ - 'Content-Type': 'application/x-www-form-urlencoded', - 'Accept': 'application/vnd.qpp.cms.gov.v2+json' - } - ) - s2s_token = get_s2s_token.json()["data"]["token"] - # print('starting download from fms for file - ' + fms_path) - get_download_url = requests.post( - url=fms_url + '/get-file', - json={"path": fms_path}, - verify=False, - headers={ - 'Accept': 'application/vnd.qpp.cms.gov.v2+json', - 'Authorization': 'Bearer ' + s2s_token - } - ) - download_url = get_download_url.json()['presigned_url'] + d = {'client_assertion': fms_token, + 'client_assertion_type': 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer', + 'grant_type': 'client_credentials', + 'scope': 'analyticsAndReporting' + } + # print('starting s2s token retrieval request from qpp auth') + get_s2s_token = requests.post( + url=auth_url, + data=d, + headers={ + 'Content-Type': 'application/x-www-form-urlencoded', + 'Accept': 'application/vnd.qpp.cms.gov.v2+json' + } + ) + s2s_token = get_s2s_token.json()["data"]["token"] + # print('starting download from fms for file - ' + fms_path) + get_download_url = requests.post( + url=fms_url + '/get-file', + json={"path": fms_path}, + verify=False, + headers={ + 'Accept': 'application/vnd.qpp.cms.gov.v2+json', + 'Authorization': 'Bearer ' + s2s_token + } + ) + download_url = get_download_url.json()['presigned_url'] + download_result = requests.get(url=download_url) + return download_result - # upload_status = s3_client.put_object( - # Bucket=S3_BUCKET, - # Key=s3_path.split(PII_BUCKET_PATH)[1], - # Body=download_result.content, - # ServerSideEncryption='aws:kms' - # ) - print(download_url) - return download_url - except Exception as err: - print(f"Unexpected Error. {err = }, {type(err) = }") - sys.exit(1) +def process_file(download_result): + print('processing file') + file_object = BytesIO(download_result.content) + wb = load_workbook(file_object) + sh = wb['2023_Practices'] + data_list = [] + for row in sh.iter_rows(sh.min_row + 1, sh.max_row): + data_list.append(row[0].value) + j = json.dumps(data_list) + with open('./converter/src/main/resources/pcf_apm_entity_ids.json', 'w') as f: + f.write(j) + + +def upload_to_s3(download_result): + filename = config.get('fms_path').split('/')[-1] + # print('starting to upload file to s3 bucket - ' + filename) + upload_status = s3_client.put_object( + Bucket=config.get('s3_bucket'), + Key=filename, + Body=download_result.content, + ServerSideEncryption='aws:kms' + ) + print(upload_status) def main(): - args = get_user_inputs() - s3_url = download_from_fms(args.auth_url, args.fms_url, args.fms_token, args.fms_path) - return s3_url + try: + # args = get_user_inputs() + # s3_url = download_from_fms(args.auth_url, args.fms_url, args.fms_token, args.fms_path) + download_result = download_from_fms(config.get('auth_url'), config.get('fms_url'), config.get('fms_token'), + config.get('fms_path')) + process_file(download_result) + upload_to_s3(download_result) + except Exception as err: + print(f"Unexpected Error. {err = }, {type(err) = }") + sys.exit(1) if __name__ == '__main__': - url = main() + main() diff --git a/upload-part-file.sh b/upload-part-file.sh index f95468bbb..16d7d4f74 100755 --- a/upload-part-file.sh +++ b/upload-part-file.sh @@ -1,34 +1,27 @@ #!/bin/bash -AUTH_URL=$1 -FMS_URL=$2 -FMS_TOKEN=$3 -FMS_PATH=$4 +PART_FILE_BUCKET=$1 +PART_FILE=$2 +FORMATTED_FILE_NAME=$3 +AWS_REGION=$4 -chmod +x ./tools/scripts/retrieve-fms-file.py -s3_url=`python ./tools/scripts/retrieve-fms-file.py -au ${AUTH_URL} -fu ${FMS_URL} -t ${FMS_TOKEN} -p ${FMS_PATH}` -echo $s3_url +export AWS_REGION=${AWS_REGION} -# PART_FILE_BUCKET=$1 -# PART_FILE=$2 -# FORMATTED_FILE_NAME=$3 -# AWS_REGION=$4 +pip install openpyxl +pip install simplejson -# export AWS_REGION=${AWS_REGION} +#chmod +x ./tools/scripts/retrieve-fms-file.py +#python ./tools/scripts/retrieve-fms-file.py -au ${AUTH_URL} -fu ${FMS_URL} -t ${FMS_TOKEN} -p ${FMS_PATH} +aws s3 cp s3://${PART_FILE_BUCKET}/${PART_FILE} . +chmod +x ./tools/scripts/format-participation-file.py +python ./tools/scripts/format-participation-file.py ${PART_FILE} ${FORMATTED_FILE_NAME} +aws s3 mv ${FORMATTED_FILE_NAME} s3://${PART_FILE_BUCKET}/${FORMATTED_FILE_NAME} -# pip install openpyxl -# pip install simplejson - -# aws s3 cp s3://${PART_FILE_BUCKET}/${PART_FILE} . -# chmod +x ./tools/scripts/format-participation-file.py -# python ./tools/scripts/format-participation-file.py ${PART_FILE} ${FORMATTED_FILE_NAME} -# aws s3 mv ${FORMATTED_FILE_NAME} s3://${PART_FILE_BUCKET}/${FORMATTED_FILE_NAME} - -# if test -f "$FORMATTED_FILE_NAME" -# then -# echo 'Removing Formatted Participation file localy...' -# rm ${FORMATTED_FILE_NAME} -# echo 'Formatted Participation file has been removed locally.' -# else -# echo 'Formatted Participation file has been removed locally.' -# fi \ No newline at end of file +if test -f "$FORMATTED_FILE_NAME" +then + echo 'Removing Formatted Participation file localy...' + rm ${FORMATTED_FILE_NAME} + echo 'Formatted Participation file has been removed locally.' +else + echo 'Formatted Participation file has been removed locally.' +fi \ No newline at end of file From bc2c3faa0c53bf062ba625a003e78f91ccba3e4f Mon Sep 17 00:00:00 2001 From: Sivakumar Srinivasulu Date: Tue, 12 Dec 2023 13:58:30 -0600 Subject: [PATCH 6/7] added codeowners --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 93f1b1bce..daa61f5d3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,7 +4,7 @@ # You can also use email addresses if you prefer. # Default reviewers -* @saquino0827 @omnivion +* @saquino0827 @sivaksb @jpec07 # CoreVPC Nava reviewers *.tf @gabesmed @ohlol From f7c535a7e087eefe3122d4041896641fc46521c6 Mon Sep 17 00:00:00 2001 From: Sivakumar Srinivasulu Date: Tue, 12 Dec 2023 20:20:21 -0600 Subject: [PATCH 7/7] updated codeowners --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index daa61f5d3..affe2b67d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,5 +7,5 @@ * @saquino0827 @sivaksb @jpec07 # CoreVPC Nava reviewers -*.tf @gabesmed @ohlol -*.tpl @gabesmed @ohlol +# *.tf @gabesmed @ohlol +# *.tpl @gabesmed @ohlol