From bcf7091135ce9a3eecd18fc137b194f235ea4a79 Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 15 Jan 2025 18:51:58 -0500 Subject: [PATCH 1/7] [COST-5368] prep for AWS Glue Data Catalogue migration --- deploy/clowdapp.yaml | 223 ++++++++++++++++-- deploy/kustomize/base/base.yaml | 66 ++++-- deploy/kustomize/patches/koku-reads.yaml | 6 + deploy/kustomize/patches/koku-writes.yaml | 6 + deploy/kustomize/patches/listener.yaml | 6 + deploy/kustomize/patches/masu.yaml | 6 + deploy/kustomize/patches/nginx.yaml | 6 + deploy/kustomize/patches/scheduler.yaml | 6 + deploy/kustomize/patches/sources-client.yaml | 6 + .../kustomize/patches/sources-listener.yaml | 6 + deploy/kustomize/patches/worker-celery.yaml | 6 + .../patches/worker-cost-model-penalty.yaml | 6 + .../patches/worker-cost-model-xl.yaml | 6 + .../kustomize/patches/worker-cost-model.yaml | 6 + .../patches/worker-download-penalty.yaml | 6 + .../kustomize/patches/worker-download-xl.yaml | 6 + deploy/kustomize/patches/worker-download.yaml | 6 + deploy/kustomize/patches/worker-hcs.yaml | 6 + .../kustomize/patches/worker-ocp-penalty.yaml | 6 + deploy/kustomize/patches/worker-ocp-xl.yaml | 6 + deploy/kustomize/patches/worker-ocp.yaml | 6 + .../patches/worker-priority-penalty.yaml | 6 + .../kustomize/patches/worker-priority-xl.yaml | 6 + deploy/kustomize/patches/worker-priority.yaml | 6 + .../patches/worker-refresh-penalty.yaml | 6 + .../kustomize/patches/worker-refresh-xl.yaml | 6 + deploy/kustomize/patches/worker-refresh.yaml | 6 + .../patches/worker-subs-extraction.yaml | 6 + .../patches/worker-subs-transmission.yaml | 6 + .../patches/worker-summary-penalty.yaml | 6 + .../kustomize/patches/worker-summary-xl.yaml | 6 + deploy/kustomize/patches/worker-summary.yaml | 6 + .../report/test/util/model_bakery_loader.py | 3 +- koku/koku/dev_middleware.py | 7 +- koku/koku/middleware.py | 5 +- koku/koku/settings.py | 23 +- koku/masu/database/aws_report_db_accessor.py | 5 +- .../masu/database/azure_report_db_accessor.py | 5 +- koku/masu/database/gcp_report_db_accessor.py | 7 +- koku/masu/database/ocp_report_db_accessor.py | 15 +- koku/masu/database/report_db_accessor_base.py | 15 +- .../reporting_ocpaws_compute_summary_p.sql | 4 +- ...rting_ocpaws_cost_summary_by_account_p.sql | 4 +- ...orting_ocpaws_cost_summary_by_region_p.sql | 4 +- ...rting_ocpaws_cost_summary_by_service_p.sql | 4 +- .../reporting_ocpaws_cost_summary_p.sql | 4 +- .../reporting_ocpaws_database_summary_p.sql | 4 +- .../reporting_ocpaws_network_summary_p.sql | 4 +- .../reporting_ocpaws_storage_summary_p.sql | 4 +- .../reporting_ocpazure_compute_summary_p.sql | 4 +- ...ing_ocpazure_cost_summary_by_account_p.sql | 4 +- ...ng_ocpazure_cost_summary_by_location_p.sql | 4 +- ...ing_ocpazure_cost_summary_by_service_p.sql | 4 +- .../reporting_ocpazure_cost_summary_p.sql | 4 +- .../reporting_ocpazure_database_summary_p.sql | 4 +- .../reporting_ocpazure_network_summary_p.sql | 4 +- .../reporting_ocpazure_storage_summary_p.sql | 4 +- .../reporting_ocpgcp_compute_summary_p.sql | 4 +- ...rting_ocpgcp_cost_summary_by_account_p.sql | 4 +- ...g_ocpgcp_cost_summary_by_gcp_project_p.sql | 4 +- ...orting_ocpgcp_cost_summary_by_region_p.sql | 4 +- ...rting_ocpgcp_cost_summary_by_service_p.sql | 4 +- .../reporting_ocpgcp_cost_summary_p.sql | 4 +- .../reporting_ocpgcp_database_summary_p.sql | 4 +- .../reporting_ocpgcp_network_summary_p.sql | 4 +- .../reporting_ocpgcp_storage_summary_p.sql | 4 +- .../reporting_ocpall_compute_summary_pt.sql | 6 +- ...ting_ocpall_cost_summary_by_account_pt.sql | 6 +- ...rting_ocpall_cost_summary_by_region_pt.sql | 6 +- ...ting_ocpall_cost_summary_by_service_pt.sql | 6 +- .../all/reporting_ocpall_cost_summary_pt.sql | 6 +- .../reporting_ocpall_database_summary_pt.sql | 6 +- .../reporting_ocpall_network_summary_pt.sql | 6 +- .../reporting_ocpall_storage_summary_pt.sql | 6 +- ...g_ocpallcostlineitem_daily_summary_aws.sql | 6 +- ...ocpallcostlineitem_daily_summary_azure.sql | 6 +- ...g_ocpallcostlineitem_daily_summary_gcp.sql | 6 +- ...costlineitem_project_daily_summary_aws.sql | 6 +- ...stlineitem_project_daily_summary_azure.sql | 6 +- ...costlineitem_project_daily_summary_gcp.sql | 6 +- .../reporting_ocpaws_compute_summary_p.sql | 4 +- ...rting_ocpaws_cost_summary_by_account_p.sql | 4 +- ...orting_ocpaws_cost_summary_by_region_p.sql | 4 +- ...rting_ocpaws_cost_summary_by_service_p.sql | 4 +- .../reporting_ocpaws_cost_summary_p.sql | 4 +- .../reporting_ocpaws_database_summary_p.sql | 4 +- .../reporting_ocpaws_network_summary_p.sql | 4 +- .../reporting_ocpaws_storage_summary_p.sql | 4 +- .../reporting_ocpazure_compute_summary_p.sql | 4 +- ...ing_ocpazure_cost_summary_by_account_p.sql | 4 +- ...ng_ocpazure_cost_summary_by_location_p.sql | 4 +- ...ing_ocpazure_cost_summary_by_service_p.sql | 4 +- .../reporting_ocpazure_cost_summary_p.sql | 4 +- .../reporting_ocpazure_database_summary_p.sql | 4 +- .../reporting_ocpazure_network_summary_p.sql | 4 +- .../reporting_ocpazure_storage_summary_p.sql | 4 +- .../reporting_ocpgcp_compute_summary_p.sql | 4 +- ...rting_ocpgcp_cost_summary_by_account_p.sql | 4 +- ...g_ocpgcp_cost_summary_by_gcp_project_p.sql | 4 +- ...orting_ocpgcp_cost_summary_by_region_p.sql | 4 +- ...rting_ocpgcp_cost_summary_by_service_p.sql | 4 +- .../reporting_ocpgcp_cost_summary_p.sql | 4 +- .../reporting_ocpgcp_database_summary_p.sql | 4 +- .../reporting_ocpgcp_network_summary_p.sql | 4 +- .../reporting_ocpgcp_storage_summary_p.sql | 4 +- ...rting_ocpgcpcostlineitem_daily_summary.sql | 4 +- ...costlineitem_daily_summary_resource_id.sql | 6 +- koku/masu/processor/_tasks/data_validation.py | 2 +- .../aws/aws_report_parquet_processor.py | 2 +- .../oci/oci_report_parquet_processor.py | 2 +- .../ocp/ocp_cloud_parquet_summary_updater.py | 6 +- .../report_parquet_processor_base.py | 24 +- .../test_report_parquet_processor_base.py | 5 +- koku/sources/api/serializers.py | 7 +- koku/sources/kafka_message_processor.py | 4 +- koku/subs/subs_data_extractor.py | 6 +- koku/subs/subs_data_messenger.py | 6 +- 117 files changed, 674 insertions(+), 238 deletions(-) diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 514031bcdc..91b615d86f 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -22,6 +22,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DJANGO_SECRET_KEY valueFrom: secretKeyRef: @@ -209,6 +215,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DJANGO_SECRET_KEY valueFrom: secretKeyRef: @@ -387,6 +399,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: APP_POD_NAME valueFrom: fieldRef: @@ -444,6 +462,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: KAFKA_CONNECT value: ${KAFKA_CONNECT} - name: AWS_SHARED_CREDENTIALS_FILE @@ -590,6 +614,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: MASU value: "true" - name: DJANGO_SECRET_KEY @@ -788,6 +818,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -948,6 +984,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: SOURCES value: "true" - name: SOURCES_PROBE_HEADER @@ -1115,6 +1157,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: SOURCES value: "true" - name: SOURCES_PROBE_HEADER @@ -1279,6 +1327,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -1461,6 +1515,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -1649,6 +1709,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -1837,6 +1903,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2025,6 +2097,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2217,6 +2295,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2409,6 +2493,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2601,6 +2691,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2791,6 +2887,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2981,6 +3083,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3171,6 +3279,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3367,6 +3481,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3563,6 +3683,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3759,6 +3885,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3947,6 +4079,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4135,6 +4273,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4323,6 +4467,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4515,6 +4665,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4707,6 +4863,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4899,6 +5061,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -5085,6 +5253,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -5259,6 +5433,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -5434,6 +5614,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DEVELOPMENT value: ${DEVELOPMENT} - name: PROMETHEUS_MULTIPROC_DIR @@ -5467,6 +5653,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DEVELOPMENT value: ${DEVELOPMENT} - name: MGMT_IMAGE @@ -5521,25 +5713,6 @@ objects: kind: Secret metadata: name: koku-secret -- apiVersion: v1 - data: - aws-credentials: ${AWS_CREDENTIALS_EPH} - kind: Secret - metadata: - name: koku-aws -- apiVersion: v1 - data: - gcp-credentials: ${GCP_CREDENTIALS_EPH} - kind: Secret - metadata: - name: koku-gcp -- apiVersion: v1 - data: - oci-config: ${OCI_CONFIG_EPH} - oci-credentials: ${OCI_CREDENTIALS_EPH} - kind: Secret - metadata: - name: koku-oci - apiVersion: v1 data: nginx.conf: |- @@ -5637,6 +5810,12 @@ parameters: name: IMAGE required: true value: quay.io/redhat-services-prod/cost-mgmt-dev-tenant/koku +- description: DEVELOPMENT ONLY - used to prevent clashing Glue databases during ephemeral + testing + name: SCHEMA_SUFFIX + value: "" +- name: TRINO_SCHEMA_PREFIX + value: "" - name: S3_BUCKET_NAME value: hccm-eph-s3 - name: S3_ROS_BUCKET_NAME @@ -5948,6 +6127,12 @@ parameters: name: AWS_CREDENTIALS_EPH required: true value: W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkPUFXU19BQ0NFU1NfS0VZX0lECmF3c19zZWNyZXRfYWNjZXNzX2tleT1BV1NfU0VDUkVUX0FDQ0VTU19LRVkK +- name: AWS_ACCESS_KEY_ID_EPH + required: false + value: AWS_ACCESS_KEY_ID +- name: AWS_SECRET_ACCESS_KEY_EPH + required: false + value: AWS_SECRET_ACCESS_KEY - displayName: GCP Credentials (Ephemeral) name: GCP_CREDENTIALS_EPH required: true diff --git a/deploy/kustomize/base/base.yaml b/deploy/kustomize/base/base.yaml index 7722916698..488bc3ca87 100644 --- a/deploy/kustomize/base/base.yaml +++ b/deploy/kustomize/base/base.yaml @@ -73,6 +73,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DEVELOPMENT value: ${DEVELOPMENT} - name: PROMETHEUS_MULTIPROC_DIR @@ -109,6 +115,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DEVELOPMENT value: ${DEVELOPMENT} - name: MGMT_IMAGE @@ -153,25 +165,28 @@ objects: name: koku-secret data: django-secret-key: "${SECRET_KEY}" -- apiVersion: v1 - kind: Secret # For ephemeral/local environment only - metadata: - name: koku-aws - data: - aws-credentials: "${AWS_CREDENTIALS_EPH}" -- apiVersion: v1 - kind: Secret # For ephemeral/local environment only - metadata: - name: koku-gcp - data: - gcp-credentials: "${GCP_CREDENTIALS_EPH}" -- apiVersion: v1 - kind: Secret # For ephemeral/local environment only - metadata: - name: koku-oci - data: - oci-credentials: "${OCI_CREDENTIALS_EPH}" - oci-config: "${OCI_CONFIG_EPH}" +# - apiVersion: v1 +# kind: Secret +# metadata: +# name: koku-aws +# data: +# aws-credentials: ${AWS_CREDENTIALS_EPH} +# stringData: +# aws-access-key-id: "${AWS_ACCESS_KEY_ID_EPH}" +# aws-secret-access-key: "${AWS_SECRET_ACCESS_KEY_EPH}" +# - apiVersion: v1 +# kind: Secret # For ephemeral/local environment only +# metadata: +# name: koku-gcp +# data: +# gcp-credentials: "${GCP_CREDENTIALS_EPH}" +# - apiVersion: v1 +# kind: Secret # For ephemeral/local environment only +# metadata: +# name: koku-oci +# data: +# oci-credentials: "${OCI_CREDENTIALS_EPH}" +# oci-config: "${OCI_CONFIG_EPH}" - apiVersion: v1 kind: ConfigMap @@ -275,6 +290,13 @@ parameters: name: IMAGE required: true value: quay.io/redhat-services-prod/cost-mgmt-dev-tenant/koku +- name: SCHEMA_SUFFIX + description: DEVELOPMENT ONLY - used to prevent clashing Glue databases during ephemeral testing + value: "" +- name: TRINO_SCHEMA_PREFIX + value: "" +- name: TRINO_S3A_OR_S3 + value: "s3a" - name: S3_BUCKET_NAME value: hccm-eph-s3 - name: S3_ROS_BUCKET_NAME @@ -601,6 +623,12 @@ parameters: name: AWS_CREDENTIALS_EPH required: true value: W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkPUFXU19BQ0NFU1NfS0VZX0lECmF3c19zZWNyZXRfYWNjZXNzX2tleT1BV1NfU0VDUkVUX0FDQ0VTU19LRVkK +- name: AWS_ACCESS_KEY_ID_EPH + required: false + value: AWS_ACCESS_KEY_ID +- name: AWS_SECRET_ACCESS_KEY_EPH + required: false + value: AWS_SECRET_ACCESS_KEY - name: GCP_CREDENTIALS_EPH displayName: GCP Credentials (Ephemeral) required: true diff --git a/deploy/kustomize/patches/koku-reads.yaml b/deploy/kustomize/patches/koku-reads.yaml index 646b53bde2..bc052e5883 100644 --- a/deploy/kustomize/patches/koku-reads.yaml +++ b/deploy/kustomize/patches/koku-reads.yaml @@ -13,6 +13,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DJANGO_SECRET_KEY valueFrom: secretKeyRef: diff --git a/deploy/kustomize/patches/koku-writes.yaml b/deploy/kustomize/patches/koku-writes.yaml index ce730a4cfb..06146c181d 100644 --- a/deploy/kustomize/patches/koku-writes.yaml +++ b/deploy/kustomize/patches/koku-writes.yaml @@ -20,6 +20,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DJANGO_SECRET_KEY valueFrom: secretKeyRef: diff --git a/deploy/kustomize/patches/listener.yaml b/deploy/kustomize/patches/listener.yaml index 86462896d7..020e8c13c6 100644 --- a/deploy/kustomize/patches/listener.yaml +++ b/deploy/kustomize/patches/listener.yaml @@ -17,6 +17,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: KAFKA_CONNECT value: ${KAFKA_CONNECT} - name: AWS_SHARED_CREDENTIALS_FILE diff --git a/deploy/kustomize/patches/masu.yaml b/deploy/kustomize/patches/masu.yaml index 6c087557e5..71c262dacd 100644 --- a/deploy/kustomize/patches/masu.yaml +++ b/deploy/kustomize/patches/masu.yaml @@ -19,6 +19,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: MASU value: "true" - name: DJANGO_SECRET_KEY diff --git a/deploy/kustomize/patches/nginx.yaml b/deploy/kustomize/patches/nginx.yaml index ede4369b5c..35fb8759c7 100644 --- a/deploy/kustomize/patches/nginx.yaml +++ b/deploy/kustomize/patches/nginx.yaml @@ -18,6 +18,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: APP_POD_NAME valueFrom: fieldRef: diff --git a/deploy/kustomize/patches/scheduler.yaml b/deploy/kustomize/patches/scheduler.yaml index 2f0bbc6576..08808a93f3 100644 --- a/deploy/kustomize/patches/scheduler.yaml +++ b/deploy/kustomize/patches/scheduler.yaml @@ -29,6 +29,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/sources-client.yaml b/deploy/kustomize/patches/sources-client.yaml index 04d9dd6d5b..56aaf66d01 100644 --- a/deploy/kustomize/patches/sources-client.yaml +++ b/deploy/kustomize/patches/sources-client.yaml @@ -19,6 +19,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: SOURCES value: "true" - name: SOURCES_PROBE_HEADER diff --git a/deploy/kustomize/patches/sources-listener.yaml b/deploy/kustomize/patches/sources-listener.yaml index 28627a9226..d1449ba07c 100644 --- a/deploy/kustomize/patches/sources-listener.yaml +++ b/deploy/kustomize/patches/sources-listener.yaml @@ -23,6 +23,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: SOURCES value: "true" - name: SOURCES_PROBE_HEADER diff --git a/deploy/kustomize/patches/worker-celery.yaml b/deploy/kustomize/patches/worker-celery.yaml index 648ba377f8..1c6a7b8fa8 100644 --- a/deploy/kustomize/patches/worker-celery.yaml +++ b/deploy/kustomize/patches/worker-celery.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-cost-model-penalty.yaml b/deploy/kustomize/patches/worker-cost-model-penalty.yaml index d89cf88732..097229eb11 100644 --- a/deploy/kustomize/patches/worker-cost-model-penalty.yaml +++ b/deploy/kustomize/patches/worker-cost-model-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-cost-model-xl.yaml b/deploy/kustomize/patches/worker-cost-model-xl.yaml index 9981c39008..a4a188dbcc 100644 --- a/deploy/kustomize/patches/worker-cost-model-xl.yaml +++ b/deploy/kustomize/patches/worker-cost-model-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-cost-model.yaml b/deploy/kustomize/patches/worker-cost-model.yaml index 0c5364e2a8..093e3f3186 100644 --- a/deploy/kustomize/patches/worker-cost-model.yaml +++ b/deploy/kustomize/patches/worker-cost-model.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-download-penalty.yaml b/deploy/kustomize/patches/worker-download-penalty.yaml index e8f9b9c728..1f24068381 100644 --- a/deploy/kustomize/patches/worker-download-penalty.yaml +++ b/deploy/kustomize/patches/worker-download-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-download-xl.yaml b/deploy/kustomize/patches/worker-download-xl.yaml index 19c98fc607..4dcda30ce5 100644 --- a/deploy/kustomize/patches/worker-download-xl.yaml +++ b/deploy/kustomize/patches/worker-download-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-download.yaml b/deploy/kustomize/patches/worker-download.yaml index 2492985ec0..11a1ccbfd8 100644 --- a/deploy/kustomize/patches/worker-download.yaml +++ b/deploy/kustomize/patches/worker-download.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-hcs.yaml b/deploy/kustomize/patches/worker-hcs.yaml index e67a1d0905..45d7039b99 100644 --- a/deploy/kustomize/patches/worker-hcs.yaml +++ b/deploy/kustomize/patches/worker-hcs.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-ocp-penalty.yaml b/deploy/kustomize/patches/worker-ocp-penalty.yaml index 30dc4c67cd..f22ddab309 100644 --- a/deploy/kustomize/patches/worker-ocp-penalty.yaml +++ b/deploy/kustomize/patches/worker-ocp-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-ocp-xl.yaml b/deploy/kustomize/patches/worker-ocp-xl.yaml index fd748efb1f..b15ab6df68 100644 --- a/deploy/kustomize/patches/worker-ocp-xl.yaml +++ b/deploy/kustomize/patches/worker-ocp-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-ocp.yaml b/deploy/kustomize/patches/worker-ocp.yaml index 8cd798ddc0..fd90beca6f 100644 --- a/deploy/kustomize/patches/worker-ocp.yaml +++ b/deploy/kustomize/patches/worker-ocp.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-priority-penalty.yaml b/deploy/kustomize/patches/worker-priority-penalty.yaml index c4fc9314cd..42b4c889da 100644 --- a/deploy/kustomize/patches/worker-priority-penalty.yaml +++ b/deploy/kustomize/patches/worker-priority-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-priority-xl.yaml b/deploy/kustomize/patches/worker-priority-xl.yaml index ed4c9f319a..8786b15518 100644 --- a/deploy/kustomize/patches/worker-priority-xl.yaml +++ b/deploy/kustomize/patches/worker-priority-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-priority.yaml b/deploy/kustomize/patches/worker-priority.yaml index 05b64f3bc5..9a5eafee10 100644 --- a/deploy/kustomize/patches/worker-priority.yaml +++ b/deploy/kustomize/patches/worker-priority.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-refresh-penalty.yaml b/deploy/kustomize/patches/worker-refresh-penalty.yaml index 7185748424..09b121737c 100644 --- a/deploy/kustomize/patches/worker-refresh-penalty.yaml +++ b/deploy/kustomize/patches/worker-refresh-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-refresh-xl.yaml b/deploy/kustomize/patches/worker-refresh-xl.yaml index 30526c8825..a5ba56897b 100644 --- a/deploy/kustomize/patches/worker-refresh-xl.yaml +++ b/deploy/kustomize/patches/worker-refresh-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-refresh.yaml b/deploy/kustomize/patches/worker-refresh.yaml index c1b5979770..cf32f20568 100644 --- a/deploy/kustomize/patches/worker-refresh.yaml +++ b/deploy/kustomize/patches/worker-refresh.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-subs-extraction.yaml b/deploy/kustomize/patches/worker-subs-extraction.yaml index b3b213bc02..c8e6e4a352 100644 --- a/deploy/kustomize/patches/worker-subs-extraction.yaml +++ b/deploy/kustomize/patches/worker-subs-extraction.yaml @@ -19,6 +19,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-subs-transmission.yaml b/deploy/kustomize/patches/worker-subs-transmission.yaml index 23e89d85f9..ea1b32deb7 100644 --- a/deploy/kustomize/patches/worker-subs-transmission.yaml +++ b/deploy/kustomize/patches/worker-subs-transmission.yaml @@ -19,6 +19,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-summary-penalty.yaml b/deploy/kustomize/patches/worker-summary-penalty.yaml index f2c0c02c18..6069f46799 100644 --- a/deploy/kustomize/patches/worker-summary-penalty.yaml +++ b/deploy/kustomize/patches/worker-summary-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-summary-xl.yaml b/deploy/kustomize/patches/worker-summary-xl.yaml index 080247e158..57e079b708 100644 --- a/deploy/kustomize/patches/worker-summary-xl.yaml +++ b/deploy/kustomize/patches/worker-summary-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-summary.yaml b/deploy/kustomize/patches/worker-summary.yaml index a00a0487c9..f8126f6ab2 100644 --- a/deploy/kustomize/patches/worker-summary.yaml +++ b/deploy/kustomize/patches/worker-summary.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/koku/api/report/test/util/model_bakery_loader.py b/koku/api/report/test/util/model_bakery_loader.py index 1643cd7c79..fa8d01cf66 100644 --- a/koku/api/report/test/util/model_bakery_loader.py +++ b/koku/api/report/test/util/model_bakery_loader.py @@ -35,7 +35,6 @@ from reporting.models import AWSAccountAlias from reporting.models import AWSOrganizationalUnit - BILL_MODELS = { Provider.PROVIDER_AWS: "AWSCostEntryBill", Provider.PROVIDER_AWS_LOCAL: "AWSCostEntryBill", @@ -533,7 +532,7 @@ def load_openshift_on_cloud_data(self, provider_type, cluster_id, bills, report_ # update ui tables sql_params = { - "schema_name": self.schema, + "schema": self.schema, "start_date": self.first_start_date, "end_date": self.last_end_date, "source_uuid": provider.uuid, diff --git a/koku/koku/dev_middleware.py b/koku/koku/dev_middleware.py index a980072257..8e034371d2 100644 --- a/koku/koku/dev_middleware.py +++ b/koku/koku/dev_middleware.py @@ -60,6 +60,9 @@ def process_request(self, request): identity_header = settings.DEVELOPMENT_IDENTITY user_dict = identity_header.get("identity", {}).get("user") + org_id = identity_header.get("org_id") or "1234567" + if not org_id.endswith(settings.SCHEMA_SUFFIX): + org_id = f"{org_id}{settings.SCHEMA_SUFFIX}" user = Mock( spec=User, access=user_dict.get("access", {}), @@ -68,8 +71,8 @@ def process_request(self, request): admin=user_dict.get("is_org_admin", False), customer=Mock( account_id=identity_header.get("account_number", "10001"), - org_id=identity_header.get("org_id", "1234567"), - schema_name=f'org{identity_header.get("org_id", "1234567")}', + org_id=org_id, + schema_name=f"org{org_id}", ), req_id="DEVELOPMENT", ) diff --git a/koku/koku/middleware.py b/koku/koku/middleware.py index cc5c9ffeaf..4d14b1fae0 100644 --- a/koku/koku/middleware.py +++ b/koku/koku/middleware.py @@ -42,7 +42,6 @@ from koku.rbac import RbacConnectionError from koku.rbac import RbacService - MAX_CACHE_SIZE = 10000 USER_CACHE = TTLCache(maxsize=MAX_CACHE_SIZE, ttl=settings.MIDDLEWARE_TIME_TO_LIVE) @@ -331,7 +330,9 @@ def process_request(self, request): # noqa: C901 # Check for customer creation & user creation query_string = "" if request.META["QUERY_STRING"]: - query_string = "?{}".format(request.META["QUERY_STRING"]) + query_string = f"?{request.META['QUERY_STRING']}" + if not org_id.endswith(settings.SCHEMA_SUFFIX): + org_id = f"{org_id}{settings.SCHEMA_SUFFIX}" stmt = { "method": request.method, "path": request.path + query_string, diff --git a/koku/koku/settings.py b/koku/koku/settings.py index c2b1d61098..be070b5427 100644 --- a/koku/koku/settings.py +++ b/koku/koku/settings.py @@ -15,6 +15,7 @@ """ import logging import os +import re import sys from json import JSONDecodeError from zoneinfo import ZoneInfo @@ -137,7 +138,14 @@ MIDDLEWARE_TIME_TO_LIVE = ENVIRONMENT.int("MIDDLEWARE_TIME_TO_LIVE", default=900) # in seconds (default = 15 minutes) DEVELOPMENT = ENVIRONMENT.bool("DEVELOPMENT", default=False) +SCHEMA_SUFFIX = re.sub("[^a-zA-Z0-9_]", "_", ENVIRONMENT.get_value("SCHEMA_SUFFIX", default="")) +print(f"ORG ID SUFFIX: '{SCHEMA_SUFFIX}'") if DEVELOPMENT: + if SCHEMA_SUFFIX == "": + SCHEMA_SUFFIX = f"_{ENVIRONMENT.get_value('USER', default='')}" + if not SCHEMA_SUFFIX.startswith("_"): + SCHEMA_SUFFIX = f"_{SCHEMA_SUFFIX}" + print(f"ORG ID SUFFIX: '{SCHEMA_SUFFIX}'") DEFAULT_IDENTITY = { "identity": { "account_number": "10001", @@ -487,19 +495,29 @@ REQUESTED_ROS_BUCKET = ENVIRONMENT.get_value("REQUESTED_ROS_BUCKET", default="ros-report") REQUESTED_SUBS_BUCKET = ENVIRONMENT.get_value("REQUESTED_SUBS_BUCKET", default="subs-report") S3_TIMEOUT = ENVIRONMENT.int("S3_CONNECTION_TIMEOUT", default=60) -S3_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() +S3_DEFAULT_ENDPOINT = ENVIRONMENT.get_value("S3_ENDPOINT", default="https://s3.amazonaws.com") S3_REGION = ENVIRONMENT.get_value("S3_REGION", default="us-east-1") S3_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET) S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET) S3_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_BUCKET) +# HCS +S3_HCS_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET) +S3_HCS_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET) +S3_HCS_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_BUCKET) +S3_HCS_REGION = CONFIGURATOR.get_object_store_region(REQUESTED_BUCKET) +S3_HCS_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() +# ROS S3_ROS_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_ROS_BUCKET) S3_ROS_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_ROS_BUCKET) S3_ROS_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_ROS_BUCKET) S3_ROS_REGION = CONFIGURATOR.get_object_store_region(REQUESTED_ROS_BUCKET) +S3_ROS_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() +# SUBS S3_SUBS_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_SUBS_BUCKET) S3_SUBS_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_SUBS_BUCKET) S3_SUBS_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_SUBS_BUCKET) S3_SUBS_REGION = CONFIGURATOR.get_object_store_region(REQUESTED_SUBS_BUCKET) +S3_SUBS_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() SKIP_MINIO_DATA_DELETION = ENVIRONMENT.bool("SKIP_MINIO_DATA_DELETION", default=False) PARQUET_PROCESSING_BATCH_SIZE = ENVIRONMENT.int("PARQUET_PROCESSING_BATCH_SIZE", default=200000) @@ -520,6 +538,9 @@ TRINO_HOST = ENVIRONMENT.get_value("TRINO_HOST", default=None) TRINO_PORT = ENVIRONMENT.get_value("TRINO_PORT", default=None) TRINO_DATE_STEP = ENVIRONMENT.int("TRINO_DATE_STEP", default=5) +TRINO_SCHEMA_PREFIX = re.sub("[^a-zA-Z0-9_]", "_", ENVIRONMENT.get_value("TRINO_SCHEMA_PREFIX", default="")) +TRINO_S3A_OR_S3 = ENVIRONMENT.get_value("TRINO_S3A_OR_S3", default="s3a") +TRINO_SCHEMA_PREFIX_KEY = "trino_schema_prefix" # IBM Settings IBM_SERVICE_URL = ENVIRONMENT.get_value("IBM_SERVICE_URL", default="https://enterprise.cloud.ibm.com") diff --git a/koku/masu/database/aws_report_db_accessor.py b/koku/masu/database/aws_report_db_accessor.py index 730705d3fd..838b74a075 100644 --- a/koku/masu/database/aws_report_db_accessor.py +++ b/koku/masu/database/aws_report_db_accessor.py @@ -9,6 +9,7 @@ import uuid from dateutil.parser import parse +from django.conf import settings from django.db import connection from django.db.models import F from django.db.models import Q @@ -165,7 +166,7 @@ def populate_ocp_on_aws_ui_summary_tables_trino( sql = pkgutil.get_data("masu.database", f"trino_sql/aws/openshift/{table_name}.sql") sql = sql.decode("utf-8") sql_params = { - "schema_name": self.schema, + "schema": self.schema, "start_date": start_date, "end_date": end_date, "year": year, @@ -199,7 +200,7 @@ def delete_ocp_on_aws_hive_partition_by_day( else: column_name = "aws_source" sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {column_name} = '{aws_source}' AND ocp_source = '{ocp_source}' AND year = '{year}' diff --git a/koku/masu/database/azure_report_db_accessor.py b/koku/masu/database/azure_report_db_accessor.py index ad14e936a7..79c4227804 100644 --- a/koku/masu/database/azure_report_db_accessor.py +++ b/koku/masu/database/azure_report_db_accessor.py @@ -9,6 +9,7 @@ import uuid from dateutil.parser import parse +from django.conf import settings from django.db import connection from django.db.models import F from django.db.models import Q @@ -210,7 +211,7 @@ def populate_ocp_on_azure_ui_summary_tables_trino( sql = pkgutil.get_data("masu.database", f"trino_sql/azure/openshift/{table_name}.sql") sql = sql.decode("utf-8") sql_params = { - "schema_name": self.schema, + "schema": self.schema, "start_date": start_date, "end_date": end_date, "year": year, @@ -244,7 +245,7 @@ def delete_ocp_on_azure_hive_partition_by_day( else: column_name = "azure_source" sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {column_name} = '{az_source}' AND ocp_source = '{ocp_source}' AND year = '{year}' diff --git a/koku/masu/database/gcp_report_db_accessor.py b/koku/masu/database/gcp_report_db_accessor.py index 1d3da7cae4..66fe65dcc7 100644 --- a/koku/masu/database/gcp_report_db_accessor.py +++ b/koku/masu/database/gcp_report_db_accessor.py @@ -12,6 +12,7 @@ from dateutil.parser import parse from dateutil.relativedelta import relativedelta +from django.conf import settings from django.db import connection from django.db.models import F from django.db.models import Q @@ -270,7 +271,7 @@ def get_gcp_topology_trino(self, source_uuid, start_date, end_date, invoice_mont service_id, service_description, location_region - FROM hive.{self.schema}.gcp_line_items as gcp + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.gcp_line_items as gcp WHERE gcp.source = '{source_uuid}' AND gcp.year = '{invoice_month_date.strftime("%Y")}' AND gcp.month = '{invoice_month_date.strftime("%m")}' @@ -396,7 +397,7 @@ def populate_ocp_on_gcp_ui_summary_tables_trino( sql = pkgutil.get_data("masu.database", f"trino_sql/gcp/openshift/{table_name}.sql") sql = sql.decode("utf-8") sql_params = { - "schema_name": self.schema, + "schema": self.schema, "start_date": start_date, "end_date": end_date, "year": year, @@ -431,7 +432,7 @@ def delete_ocp_on_gcp_hive_partition_by_day( else: column_name = "gcp_source" sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {column_name} = '{gcp_source}' AND ocp_source = '{ocp_source}' AND year = '{year}' diff --git a/koku/masu/database/ocp_report_db_accessor.py b/koku/masu/database/ocp_report_db_accessor.py index 3f851ea989..aa596b1e5f 100644 --- a/koku/masu/database/ocp_report_db_accessor.py +++ b/koku/masu/database/ocp_report_db_accessor.py @@ -11,6 +11,7 @@ import uuid from dateutil.parser import parse +from django.conf import settings from django.db.models import DecimalField from django.db.models import F from django.db.models import Value @@ -226,7 +227,7 @@ def delete_ocp_hive_partition_by_day(self, days, source, year, month): ) for day in days: sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE source = '{source}' AND year = '{year}' AND (month = replace(ltrim(replace('{month}', '0', ' ')),' ', '0') OR month = '{month}') @@ -248,7 +249,7 @@ def delete_hive_partitions_by_source(self, table, partition_column, provider_uui } LOG.info(log_json(msg="deleting Hive partitions by source", context=ctx)) sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {partition_column} = '{provider_uuid}' """ self._execute_trino_raw_sql_query( @@ -920,8 +921,8 @@ def get_nodes_trino(self, source_uuid, start_date, end_date): WHEN any_match(array_agg(DISTINCT nl.node_labels), element -> element like '%"node_role_kubernetes_io": "infra"%') THEN 'infra' ELSE 'worker' END) as node_role - FROM hive.{self.schema}.openshift_pod_usage_line_items_daily as ocp - LEFT JOIN hive.{self.schema}.openshift_node_labels_line_items_daily as nl + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_pod_usage_line_items_daily as ocp + LEFT JOIN hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_node_labels_line_items_daily as nl ON ocp.node = nl.node WHERE ocp.source = '{source_uuid}' AND ocp.year = '{start_date.strftime("%Y")}' @@ -947,7 +948,7 @@ def get_pvcs_trino(self, source_uuid, start_date, end_date): SELECT distinct persistentvolume, persistentvolumeclaim, csi_volume_handle - FROM hive.{self.schema}.openshift_storage_usage_line_items_daily as ocp + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_storage_usage_line_items_daily as ocp WHERE ocp.source = '{source_uuid}' AND ocp.year = '{start_date.strftime("%Y")}' AND ocp.month = '{start_date.strftime("%m")}' @@ -961,7 +962,7 @@ def get_projects_trino(self, source_uuid, start_date, end_date): """Get the nodes from an OpenShift cluster.""" sql = f""" SELECT distinct namespace - FROM hive.{self.schema}.openshift_pod_usage_line_items_daily as ocp + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_pod_usage_line_items_daily as ocp WHERE ocp.source = '{source_uuid}' AND ocp.year = '{start_date.strftime("%Y")}' AND ocp.month = '{start_date.strftime("%m")}' @@ -1119,7 +1120,7 @@ def get_max_min_timestamp_from_parquet(self, source_uuid, start_date, end_date): sql = f""" SELECT min(interval_start) as min_timestamp, max(interval_start) as max_timestamp - FROM hive.{self.schema}.openshift_pod_usage_line_items_daily as ocp + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_pod_usage_line_items_daily as ocp WHERE ocp.source = '{source_uuid}' AND ocp.year = '{start_date.strftime("%Y")}' AND ocp.month = '{start_date.strftime("%m")}' diff --git a/koku/masu/database/report_db_accessor_base.py b/koku/masu/database/report_db_accessor_base.py index 0e64def15b..ac390b1d06 100644 --- a/koku/masu/database/report_db_accessor_base.py +++ b/koku/masu/database/report_db_accessor_base.py @@ -109,6 +109,8 @@ def _execute_raw_sql_query(self, table, sql, bind_params=None, operation="UPDATE def _execute_trino_raw_sql_query(self, sql, *, sql_params=None, context=None, log_ref=None): """Execute a single trino query returning only the fetchall results""" + if sql_params is None: + sql_params = {} results, _ = self._execute_trino_raw_sql_query_with_description( sql, sql_params=sql_params, context=context, log_ref=log_ref ) @@ -119,14 +121,12 @@ def _execute_trino_raw_sql_query_with_description( self, sql, *, - sql_params=None, + sql_params: dict, context=None, log_ref="Trino query", conn_params=None, ): """Execute a single trino query and return cur.fetchall and cur.description""" - if sql_params is None: - sql_params = {} if context is None: context = {} if conn_params is None: @@ -137,6 +137,8 @@ def _execute_trino_raw_sql_query_with_description( ctx = self.extract_context_from_sql_params(context) else: ctx = {} + if sql_params and settings.TRINO_SCHEMA_PREFIX_KEY not in sql_params: + sql_params[settings.TRINO_SCHEMA_PREFIX_KEY] = settings.TRINO_SCHEMA_PREFIX sql, bind_params = self.trino_prepare_query(sql, sql_params) t1 = time.time() trino_conn = trino_db.connect(schema=self.schema, **conn_params) @@ -169,12 +171,13 @@ def _execute_trino_raw_sql_query_with_description( def _execute_trino_multipart_sql_query(self, sql, *, bind_params=None): """Execute multiple related SQL queries in Trino.""" trino_conn = trino_db.connect(schema=self.schema) + if isinstance(bind_params, dict) and settings.TRINO_SCHEMA_PREFIX_KEY not in bind_params: + bind_params[settings.TRINO_SCHEMA_PREFIX_KEY] = settings.TRINO_SCHEMA_PREFIX return trino_db.executescript(trino_conn, sql, params=bind_params, preprocessor=self.trino_prepare_query) def delete_line_item_daily_summary_entries_for_date_range_raw( self, source_uuid, start_date, end_date, filters=None, null_filters=None, table=None ): - if table is None: table = self.line_item_daily_summary_table if not isinstance(table, str): @@ -243,7 +246,7 @@ def schema_exists_trino(self): cache_key = build_trino_schema_exists_key(self.schema) if result := get_value_from_cache(cache_key): return result - check_sql = f"SHOW SCHEMAS LIKE '{self.schema}'" + check_sql = f"SHOW SCHEMAS LIKE '{settings.TRINO_SCHEMA_PREFIX}{self.schema}'" exists = bool(self._execute_trino_raw_sql_query(check_sql, log_ref="schema_exists_trino")) set_value_in_cache(cache_key, exists) return exists @@ -264,7 +267,7 @@ def delete_hive_partition_by_month(self, table, source, year, month, source_colu for i in range(retries): try: sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {source_column} = '{source}' AND year = '{year}' AND (month = replace(ltrim(replace('{month}', '0', ' ')),' ', '0') OR month = '{month}') diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql index b6b1e5ef8d..e1fec9af05 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_compute_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_compute_summary_p ( id, usage_start, usage_end, @@ -38,7 +38,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_compute_summary_p ( sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql index 2b93eef4f0..c8f974c4c8 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p ( id, usage_start, usage_end, @@ -30,7 +30,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql index dd57bb1bf7..59b59066c8 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p ( id, usage_start, usage_end, @@ -34,7 +34,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql index 18a9c07911..7fbe344f2b 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p ( id, usage_start, usage_end, @@ -34,7 +34,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql index 6bbb23b5e2..6c0a7bbcea 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_cost_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_p ( max(currency_code), {{source_uuid}}::uuid, max(cost_category_id) - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_database_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_database_summary_p.sql index b7ed3a240c..50ad4cee71 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_database_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_database_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_database_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_database_summary_p ( id, usage_start, usage_end, @@ -36,7 +36,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_database_summary_p ( sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE product_code IN ('AmazonRDS','AmazonDynamoDB','AmazonElastiCache','AmazonNeptune','AmazonRedshift','AmazonDocumentDB') AND usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_network_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_network_summary_p.sql index 03618d88cb..4c3b495e8b 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_network_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_network_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_network_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_network_summary_p ( id, usage_start, usage_end, @@ -36,7 +36,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_network_summary_p ( sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE product_code IN ('AmazonVPC','AmazonCloudFront','AmazonRoute53','AmazonAPIGateway') AND usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql index 9814e3e2e4..6c7133f9b6 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_storage_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_storage_summary_p ( id, usage_start, usage_end, @@ -36,7 +36,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_storage_summary_p ( sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE product_family LIKE '%%Storage%%' AND unit = 'GB-Mo' AND usage_start >= {{start_date}}::date diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql index c0b3c9e1b2..6ab5002e06 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_compute_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_compute_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_compute_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql index 67c43e15bd..120c90db8c 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_account_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_account_p ( id, usage_start, usage_end, @@ -20,7 +20,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_account sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql index 1d483127fe..977e243b7e 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_location_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_location_p ( id, usage_start, usage_end, @@ -22,7 +22,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_locatio sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql index 534b15351c..7c86e20f54 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_service_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_service_p ( id, usage_start, usage_end, @@ -22,7 +22,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_service sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql index aa48e80457..b32e0d03f9 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_cost_summary_p ( id, usage_start, usage_end, @@ -20,7 +20,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_p ( max(currency) as currency, {{source_uuid}}::uuid as source_uuid, max(cost_category_id) as cost_category_id - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_database_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_database_summary_p.sql index 275f68116c..782421d033 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_database_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_database_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_database_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_database_summary_p ( id, usage_start, usage_end, @@ -26,7 +26,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_database_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE service_name IN ('Cosmos DB','Cache for Redis') OR service_name ILIKE '%%database%%' AND usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_network_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_network_summary_p.sql index 159bf3a433..25d509d62d 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_network_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_network_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_network_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_network_summary_p ( id, usage_start, usage_end, @@ -26,7 +26,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_network_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE service_name IN ('Virtual Network','VPN','DNS','Traffic Manager','ExpressRoute','Load Balancer','Application Gateway') AND usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql index 8cb55a800c..796536289a 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_storage_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_storage_summary_p ( id, usage_start, usage_end, @@ -26,7 +26,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_storage_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE service_name LIKE '%%Storage%%' AND unit_of_measure = 'GB-Mo' AND usage_start >= {{start_date}}::date diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql index 79c8828470..5b8f6483f0 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( id, account_id, cluster_id, @@ -31,7 +31,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql index 15eeadc463..efe2fe4fa0 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql @@ -1,6 +1,6 @@ -- Clear out old entries first -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p ( id, cluster_id, cluster_alias, @@ -26,7 +26,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql index facde9c808..41f4f0d510 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_project_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_project_p ( id, cluster_id, cluster_alias, @@ -27,7 +27,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_proje source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql index 545368a1b3..7295acb5c3 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p ( id, cluster_id, cluster_alias, @@ -27,7 +27,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql index 8a33d04d3f..ab141ce832 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql @@ -1,6 +1,6 @@ -- Clear out old entries first -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p ( id, cluster_id, cluster_alias, @@ -30,7 +30,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql index b8a0694441..578aec8e4f 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( id, cluster_id, cluster_alias, @@ -25,7 +25,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( sum(credit_amount) as credit_amount, invoice_month, max(cost_category_id) - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql index d174f19327..1c66fd17a3 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_database_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_database_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_database_summary_p ( source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p -- Get data for this month or last month WHERE (service_alias LIKE '%%SQL%%' OR service_alias LIKE '%%Spanner%%' diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql index 2fb32836e0..a0a0743263 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_network_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_network_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_network_summary_p ( source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p -- Get data for this month or last month WHERE (service_alias LIKE '%%Network%%' OR service_alias LIKE '%%VPC%%' diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql index 343cb479fa..61ab65ae24 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p -- Get data for this month or last month WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_compute_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_compute_summary_pt.sql index b74ba3c3e5..ae9a6bd503 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_compute_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_compute_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_compute_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_compute_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_compute_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -15,7 +15,7 @@ DELETE -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} -- TESTING!! INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_compute_summary_pt + INTO {{schema | sqlsafe}}.reporting_ocpall_compute_summary_pt ( source_type, usage_start, @@ -52,7 +52,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}} AND usage_end <= {{end_date}} AND source_type = {{source_type}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_account_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_account_pt.sql index a98287f10a..7ef19ad777 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_account_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_account_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_cost_summary_by_account_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_account_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_account_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_cost_summary_by_account_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_account_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_account_pt ( source_type, usage_start, usage_end, @@ -39,7 +39,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_end <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_region_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_region_pt.sql index a5f9629e1e..54c2489059 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_region_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_region_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_cost_summary_by_region_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_region_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_region_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_cost_summary_by_region_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_region_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_region_pt ( source_type, usage_start, usage_end, @@ -43,7 +43,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_service_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_service_pt.sql index 6a39ffbd4e..0fd98f12c6 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_service_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_service_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_cost_summary_by_service_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_service_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_service_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -14,7 +14,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_cost_summary_by_service_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_service_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_service_pt ( source_type, usage_start, usage_end, @@ -44,7 +44,7 @@ SELECT {{source_type}}, max(currency_code) as currency_code, max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_pt.sql index 16ed7f5355..afbdcb1b42 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_cost_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_cost_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_cost_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_pt + INTO {{schema | sqlsafe}}.reporting_ocpall_cost_summary_pt ( source_type, usage_start, @@ -36,7 +36,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_database_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_database_summary_pt.sql index ebb4fffc6d..799d70763b 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_database_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_database_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_database_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_database_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_database_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_database_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_database_summary_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_database_summary_pt ( source_type, usage_start, usage_end, @@ -45,7 +45,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_network_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_network_summary_pt.sql index 63f79c4843..8b1f1ad689 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_network_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_network_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_network_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_network_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_network_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_network_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_network_summary_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_network_summary_pt ( source_type, usage_start, usage_end, @@ -45,7 +45,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_storage_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_storage_summary_pt.sql index 664a7ae46c..e09511ed34 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_storage_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_storage_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_storage_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_storage_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_storage_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_storage_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_storage_summary_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_storage_summary_pt ( source_type, usage_start, usage_end, @@ -47,7 +47,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_aws.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_aws.sql index f53d2f3fe6..9ed29281fb 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_aws.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_aws.sql @@ -1,7 +1,7 @@ -- OCP ON ALL DAILY SUMMARY PROCESSING (AWS DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -61,7 +61,7 @@ SELECT 'AWS'::text AS source_type, max(cost_category_id) as cost_category_id, cast(1 as decimal) as shared_projects, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p AS aws + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p AS aws WHERE aws.usage_start >= {{start_date}}::date AND aws.usage_start <= {{end_date}}::date AND aws.cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_azure.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_azure.sql index 85096de2ff..3a7a5c0f58 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_azure.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_azure.sql @@ -1,7 +1,7 @@ -- OCP ON ALL DAILY SUMMARY PROCESSING (AZURE DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -60,7 +60,7 @@ SELECT 'Azure'::text AS source_type, max(cost_category_id) as cost_category_id, cast(1 as decimal) as shared_projects, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p AS azure + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p AS azure WHERE azure.usage_start >= {{start_date}}::date AND azure.usage_start <= {{end_date}}::date AND azure.cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_gcp.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_gcp.sql index aff598f70e..b649c9f226 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_gcp.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_gcp.sql @@ -1,7 +1,7 @@ -- OCP ON ALL DAILY SUMMARY PROCESSING (GCP DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -60,7 +60,7 @@ SELECT 'GCP'::text AS source_type, max(cost_category_id) as cost_category_id, cast(1 as decimal) as shared_projects, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p AS gcp + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p AS gcp WHERE gcp.usage_start >= {{start_date}}::date AND gcp.usage_start <= {{end_date}}::date AND gcp.cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_aws.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_aws.sql index 4a425dbc54..074db95b0c 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_aws.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_aws.sql @@ -1,7 +1,7 @@ -- OCP ON ALL PROJECT DAILY SUMMARY PROCESSING (AWS DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -63,7 +63,7 @@ SELECT 'AWS' as source_type, max(currency_code) as currency_code, max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_azure.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_azure.sql index b5b828d6a1..dbd6b3b4b3 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_azure.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_azure.sql @@ -1,7 +1,7 @@ -- OCP ON ALL PROJECT DAILY SUMMARY PROCESSING (AZURE DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -62,7 +62,7 @@ SELECT 'Azure' as source_type, max(currency) as currency_code, max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_gcp.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_gcp.sql index 88d1b1defd..3d52a21c46 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_gcp.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_gcp.sql @@ -1,7 +1,7 @@ -- OCP ON ALL PROJECT DAILY SUMMARY PROCESSING (GCP DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -65,7 +65,7 @@ SELECT 'GCP' as source_type, max(currency) as currency_code, max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql index cfd5c43177..3b20e001ac 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_compute_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_compute_summary_p ( id, usage_start, usage_end, @@ -42,7 +42,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_compute_summary_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql index 4e573a04eb..2cdc055dcd 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p ( id, usage_start, usage_end, @@ -34,7 +34,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql index 843138622e..352feacb63 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p ( id, usage_start, usage_end, @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql index ec91d8592f..ef74f74521 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p ( id, usage_start, usage_end, @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql index 04342f54b4..1938331aff 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_p ( id, usage_start, usage_end, @@ -32,7 +32,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_p ( max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid, max(cost_category_id) - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql index 96dc30fb51..03abcbd246 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_database_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_database_summary_p ( id, usage_start, usage_end, @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_database_summary sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql index 8e62f0244a..04e1f702d1 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_network_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_network_summary_p ( id, usage_start, usage_end, @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_network_summary_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql index 3606a46904..a376e83d33 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_storage_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_storage_summary_p ( id, usage_start, usage_end, @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_storage_summary_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql index 53f6f48e3c..bd323cc2c2 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_compute_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_compute_summary_p ( id, usage_start, usage_end, @@ -30,7 +30,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_compute_summar sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql index 6ba4285d9d..7d6ff2fb50 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_account_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_account_p ( id, usage_start, usage_end, @@ -22,7 +22,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_b sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql index d91043b1a5..4df03bdbec 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_location_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_location_p ( id, usage_start, usage_end, @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_b sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql index 94ee3cb686..9943268054 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_service_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_service_p ( id, usage_start, usage_end, @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_b sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql index be2a7d438c..2233cc81df 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_p ( id, usage_start, usage_end, @@ -20,7 +20,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_p max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid, max(cost_category_id) as cost_category_id - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql index 1e3d02a9aa..3ebe4d4dfe 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_database_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_database_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_database_summa sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql index 3e07ce82a6..150118b550 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_network_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_network_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_network_summar sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql index 579b939b0c..29d6c87236 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_storage_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_storage_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_storage_summar sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql index f7a727fce9..bb99732422 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( id, account_id, cluster_id, @@ -31,7 +31,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_compute_summary_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql index ee64ca9491..9fdbe05d37 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p ( id, cluster_id, cluster_alias, @@ -25,7 +25,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql index 6e4162e37b..2fafd275ca 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_project_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_project_p ( id, cluster_id, cluster_alias, @@ -27,7 +27,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql index ff9ccf4613..ef85f0a62c 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p ( id, cluster_id, cluster_alias, @@ -27,7 +27,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql index 8702c72ef6..c135be4690 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p ( id, cluster_id, cluster_alias, @@ -29,7 +29,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql index d5d53e5052..780ccac4c2 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( id, cluster_id, cluster_alias, @@ -25,7 +25,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( sum(credit_amount) as credit_amount, invoice_month, max(cost_category_id) - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql index ac0c9700f8..fafbd6ead9 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_database_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_database_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_database_summary cast({{gcp_source_uuid}} as uuid) as gcp_source, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql index 491a5126c2..2cd2b457b2 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_network_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_network_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_network_summary_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql index 3c13580034..7231fd38e5 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_storage_summary_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql index 33f0b44690..f82528848b 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql @@ -211,7 +211,7 @@ SELECT gcp.uuid as gcp_uuid, max(gcp.year) as year, max(gcp.month) as month FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp -JOIN hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON date(gcp.usage_start_time) = ocp.usage_start AND (strpos(gcp.labels, 'kubernetes-io-cluster-{{cluster_id | sqlsafe}}') != 0 -- THIS IS THE SPECIFIC TO OCP ON GCP TAG MATCH OR strpos(gcp.labels, 'kubernetes-io-cluster-{{cluster_alias | sqlsafe}}') != 0) @@ -329,7 +329,7 @@ SELECT gcp.uuid as gcp_uuid, max(gcp.year) as year, max(gcp.month) as month FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp -JOIN hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON date(gcp.usage_start_time) = ocp.usage_start AND ( json_query(gcp.labels, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql index 23b06cd841..15f0fec8d6 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql @@ -430,7 +430,7 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( @@ -553,7 +553,7 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( @@ -806,7 +806,7 @@ SELECT gcp.uuid as gcp_uuid, cast(year(max(gcp.usage_start)) as varchar) as year, cast(month(max(gcp.usage_start)) as varchar) as month, cast(day(max(gcp.usage_start)) as varchar) as day -FROM hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( diff --git a/koku/masu/processor/_tasks/data_validation.py b/koku/masu/processor/_tasks/data_validation.py index 33ec6ac36d..5dac103619 100644 --- a/koku/masu/processor/_tasks/data_validation.py +++ b/koku/masu/processor/_tasks/data_validation.py @@ -162,7 +162,7 @@ def execute_relevant_query(self, provider_type, cluster_id=None, trino=False): source = "source" if not self.ocp_on_cloud_type else "cluster_id" sql = f""" SELECT sum({query_filters.get("metric")}) as metric, {query_filters.get("date")} as date - FROM hive.{self.schema}.{table} + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {source} = '{provider_filter}' AND {query_filters.get("date")} >= date('{start}') AND {query_filters.get("date")} <= date('{end}') diff --git a/koku/masu/processor/aws/aws_report_parquet_processor.py b/koku/masu/processor/aws/aws_report_parquet_processor.py index e1cdce25d7..97d9a66296 100644 --- a/koku/masu/processor/aws/aws_report_parquet_processor.py +++ b/koku/masu/processor/aws/aws_report_parquet_processor.py @@ -83,7 +83,7 @@ def create_bill(self, bill_date): AND month = '{bill_date.strftime("%m")}' """ - rows = self._execute_sql(sql, self._schema_name) + rows = self._execute_trino_sql(sql, self._schema_name) payer_account_id = None if rows: payer_account_id = rows[0][0] diff --git a/koku/masu/processor/oci/oci_report_parquet_processor.py b/koku/masu/processor/oci/oci_report_parquet_processor.py index af47589e86..0bce01dc04 100644 --- a/koku/masu/processor/oci/oci_report_parquet_processor.py +++ b/koku/masu/processor/oci/oci_report_parquet_processor.py @@ -67,7 +67,7 @@ def create_bill(self, bill_date): AND year = '{bill_date.strftime("%Y")}' AND month = '{bill_date.strftime("%m")}' """ - rows = self._execute_sql(sql, self._schema_name) + rows = self._execute_trino_sql(sql, self._schema_name) payer_tenant_id = None if rows: payer_tenant_id = rows[0][0] diff --git a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py index 7eb52f224c..904af56a0f 100644 --- a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py +++ b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py @@ -273,7 +273,7 @@ def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, # OpenShift on AWS sql_params = { - "schema_name": self._schema, + "schema": self._schema, "start_date": start_date, "end_date": end_date, "source_uuid": aws_provider_uuid, @@ -403,7 +403,7 @@ def update_azure_summary_tables(self, openshift_provider_uuid, azure_provider_uu # OpenShift on Azure sql_params = { - "schema_name": self._schema, + "schema": self._schema, "start_date": start_date, "end_date": end_date, "source_uuid": azure_provider_uuid, @@ -527,7 +527,7 @@ def update_gcp_summary_tables(self, openshift_provider_uuid, gcp_provider_uuid, # OpenShift on GCP sql_params = { - "schema_name": self._schema, + "schema": self._schema, "start_date": start_date, "end_date": end_date, "source_uuid": gcp_provider_uuid, diff --git a/koku/masu/processor/report_parquet_processor_base.py b/koku/masu/processor/report_parquet_processor_base.py index 73892e87af..cb05080127 100644 --- a/koku/masu/processor/report_parquet_processor_base.py +++ b/koku/masu/processor/report_parquet_processor_base.py @@ -53,9 +53,11 @@ def postgres_summary_table(self): """Return error if unimplemented in subclass.""" raise PostgresSummaryTableError("This must be a property on the sub class.") - def _execute_sql(self, sql, schema_name): # pragma: no cover + def _execute_trino_sql(self, sql, schema_name: str): # pragma: no cover """Execute Trino SQL.""" rows = [] + if not schema_name.startswith(settings.TRINO_SCHEMA_PREFIX): + schema_name = f"{settings.TRINO_SCHEMA_PREFIX}{schema_name}" try: with trino.dbapi.connect( host=settings.TRINO_HOST, port=settings.TRINO_PORT, user="admin", catalog="hive", schema=schema_name @@ -91,8 +93,8 @@ def schema_exists(self): cache_key = build_trino_schema_exists_key(self._schema_name) if result := get_value_from_cache(cache_key): return result - schema_check_sql = f"SHOW SCHEMAS LIKE '{self._schema_name}'" - exists = bool(self._execute_sql(schema_check_sql, "default")) + schema_check_sql = f"SHOW SCHEMAS LIKE '{settings.TRINO_SCHEMA_PREFIX}{self._schema_name}'" + exists = bool(self._execute_trino_sql(schema_check_sql, "default")) set_value_in_cache(cache_key, exists) return exists @@ -103,15 +105,15 @@ def table_exists(self): if result := get_value_from_cache(cache_key): return result table_check_sql = f"SHOW TABLES LIKE '{self._table_name}'" - exists = bool(self._execute_sql(table_check_sql, self._schema_name)) + exists = bool(self._execute_trino_sql(table_check_sql, self._schema_name)) set_value_in_cache(cache_key, exists) return exists def create_schema(self): """Create Trino schema.""" LOG.info(log_json(msg="create trino/hive schema sql", schema=self._schema_name)) - schema_create_sql = f"CREATE SCHEMA IF NOT EXISTS {self._schema_name}" - self._execute_sql(schema_create_sql, "default") + schema_create_sql = f"CREATE SCHEMA IF NOT EXISTS {settings.TRINO_SCHEMA_PREFIX}{self._schema_name}" + self._execute_trino_sql(schema_create_sql, "default") return self._schema_name def _generate_column_list(self): @@ -124,7 +126,7 @@ def _generate_create_table_sql(self, partition_map=None): parquet_columns = self._generate_column_list() s3_path = f"{settings.S3_BUCKET_NAME}/{self._s3_path}" - sql = f"CREATE TABLE IF NOT EXISTS {self._schema_name}.{self._table_name} (" + sql = f"CREATE TABLE IF NOT EXISTS {settings.TRINO_SCHEMA_PREFIX}{self._schema_name}.{self._table_name} (" for idx, col in enumerate(parquet_columns): norm_col = strip_characters_from_column_name(col) if norm_col in self._column_types["numeric_columns"]: @@ -145,7 +147,7 @@ def _generate_create_table_sql(self, partition_map=None): sql += ",".join([f"{item[0]} {item[1]} " for item in list(partition_map.items())]) partition_column_str = ", ".join([f"'{key}'" for key in partition_map.keys()]) sql += ( - f") WITH(external_location = 's3a://{s3_path}', format = 'PARQUET'," + f") WITH(external_location = '{settings.TRINO_S3A_OR_S3}://{s3_path}', format = 'PARQUET'," f" partitioned_by=ARRAY[{partition_column_str}])" ) else: @@ -153,7 +155,7 @@ def _generate_create_table_sql(self, partition_map=None): sql += ",source varchar, year varchar, month varchar" sql += ( - f") WITH(external_location = 's3a://{s3_path}', format = 'PARQUET'," + f") WITH(external_location = '{settings.TRINO_S3A_OR_S3}://{s3_path}', format = 'PARQUET'," " partitioned_by=ARRAY['source', 'year', 'month'])" ) return sql @@ -162,7 +164,7 @@ def create_table(self, partition_map=None): """Create Trino SQL table.""" sql = self._generate_create_table_sql(partition_map=partition_map) LOG.info(log_json(msg="attempting to create parquet table", table=self._table_name, schema=self._schema_name)) - self._execute_sql(sql, self._schema_name) + self._execute_trino_sql(sql, self._schema_name) LOG.info(log_json(msg="trino parquet table created", table=self._table_name, schema=self._schema_name)) def get_or_create_postgres_partition(self, bill_date, **kwargs): @@ -228,4 +230,4 @@ def sync_hive_partitions(self): ) sql = f"CALL system.sync_partition_metadata('{self._schema_name}', '{self._table_name}', 'FULL')" LOG.info(sql) - self._execute_sql(sql, self._schema_name) + self._execute_trino_sql(sql, self._schema_name) diff --git a/koku/masu/test/processor/test_report_parquet_processor_base.py b/koku/masu/test/processor/test_report_parquet_processor_base.py index 2fcb592d1d..920ff9873f 100644 --- a/koku/masu/test/processor/test_report_parquet_processor_base.py +++ b/koku/masu/test/processor/test_report_parquet_processor_base.py @@ -9,6 +9,7 @@ from unittest.mock import patch import pandas as pd +from django.conf import settings from django.test.utils import override_settings from api.common import log_json @@ -97,7 +98,7 @@ def test_generate_create_table_sql(self, mock_execute): expected_start = f"CREATE TABLE IF NOT EXISTS {self.schema}.{self.table_name}" expected_end = ( - f"WITH(external_location = 's3a://test-bucket/{self.temp_dir}', " + f"WITH(external_location = '{settings.TRINO_S3A_OR_S3}://test-bucket/{self.temp_dir}', " "format = 'PARQUET', partitioned_by=ARRAY['source', 'year', 'month'])" ) self.assertTrue(generated_sql.startswith(expected_start)) @@ -123,7 +124,7 @@ def test_generate_create_table_sql_with_provider_map(self, mock_execute): expected_start = f"CREATE TABLE IF NOT EXISTS {self.schema}.{self.table_name}" expected_end = ( - f"WITH(external_location = 's3a://test-bucket/{self.temp_dir}', " + f"WITH(external_location = '{settings.TRINO_S3A_OR_S3}://test-bucket/{self.temp_dir}', " "format = 'PARQUET', partitioned_by=ARRAY['source', 'year', 'month', 'day'])" ) self.assertTrue(generated_sql.startswith(expected_start)) diff --git a/koku/sources/api/serializers.py b/koku/sources/api/serializers.py index 097a27bbcb..0cc9356ecd 100644 --- a/koku/sources/api/serializers.py +++ b/koku/sources/api/serializers.py @@ -6,6 +6,7 @@ import logging from uuid import uuid4 +from django.conf import settings from django.db import transaction from rest_framework import serializers @@ -18,7 +19,6 @@ from sources.api import get_auth_header from sources.api import get_param_from_header - LOG = logging.getLogger(__name__) ALLOWED_BILLING_SOURCE_PROVIDERS = ( @@ -104,7 +104,10 @@ def _validate_account_id(self, account_id): return get_param_from_header(self.context.get("request"), "account_number") def _validate_org_id(self, account_id): - return get_param_from_header(self.context.get("request"), "org_id") + org_id = get_param_from_header(self.context.get("request"), "org_id") + if not org_id.endswith(settings.SCHEMA_SUFFIX): + org_id = f"{org_id}{settings.SCHEMA_SUFFIX}" + return org_id def validate(self, data): data["source_id"] = self._validate_source_id(data.get("id")) diff --git a/koku/sources/kafka_message_processor.py b/koku/sources/kafka_message_processor.py index 173b9557b7..92afcb30dd 100644 --- a/koku/sources/kafka_message_processor.py +++ b/koku/sources/kafka_message_processor.py @@ -5,6 +5,7 @@ import json import logging +from django.conf import settings from rest_framework.exceptions import ValidationError from api.provider.models import Provider @@ -17,7 +18,6 @@ from sources.sources_http_client import SourcesHTTPClient from sources.sources_http_client import SourcesHTTPClientError - LOG = logging.getLogger(__name__) KAFKA_APPLICATION_CREATE = "Application.create" @@ -101,6 +101,8 @@ def __init__(self, msg, event_type, cost_mgmt_id): msg = f"[KafkaMessageProcessor] missing `{KAFKA_HDR_RH_IDENTITY}` or org_id: {msg.headers()}" LOG.warning(msg) raise SourcesMessageError(msg) + if isinstance(self.org_id, str) and not self.org_id.endswith(settings.SCHEMA_SUFFIX): + self.org_id = f"{self.org_id}{settings.SCHEMA_SUFFIX}" self.source_id = None self.application_type_id = None diff --git a/koku/subs/subs_data_extractor.py b/koku/subs/subs_data_extractor.py index 2c93b48970..7fa447bde0 100644 --- a/koku/subs/subs_data_extractor.py +++ b/koku/subs/subs_data_extractor.py @@ -22,7 +22,6 @@ from reporting.models import SubsIDMap from reporting.models import SubsLastProcessed - LOG = logging.getLogger(__name__) @@ -40,7 +39,10 @@ def __init__(self, tracing_id, context): self.creation_processing_time = self.creation_processing_time - timedelta(days=1) self.tracing_id = tracing_id self.s3_resource = get_s3_resource( - settings.S3_SUBS_ACCESS_KEY, settings.S3_SUBS_SECRET, settings.S3_SUBS_REGION + access_key=settings.S3_SUBS_ACCESS_KEY, + secret_key=settings.S3_SUBS_SECRET, + region=settings.S3_SUBS_REGION, + endpoint_url=settings.S3_SUBS_ENDPOINT, ) self.context = context diff --git a/koku/subs/subs_data_messenger.py b/koku/subs/subs_data_messenger.py index 32c78ca042..58f7d1cec6 100644 --- a/koku/subs/subs_data_messenger.py +++ b/koku/subs/subs_data_messenger.py @@ -25,7 +25,6 @@ from masu.prometheus_stats import RHEL_ELS_VCPU_HOURS from masu.util.aws.common import get_s3_resource - LOG = logging.getLogger(__name__) HPC_ROLE = "Red Hat Enterprise Linux Compute Node" @@ -51,7 +50,10 @@ def __init__(self, context, schema_name, tracing_id): self.tracing_id = tracing_id self.schema_name = schema_name self.s3_resource = get_s3_resource( - settings.S3_SUBS_ACCESS_KEY, settings.S3_SUBS_SECRET, settings.S3_SUBS_REGION + access_key=settings.S3_SUBS_ACCESS_KEY, + secret_key=settings.S3_SUBS_SECRET, + region=settings.S3_SUBS_REGION, + endpoint_url=settings.S3_SUBS_ENDPOINT, ) subs_cust = Customer.objects.get(schema_name=schema_name) self.account_id = subs_cust.account_id From d21adc7b706362f02c9b1afc614b18d5ecd22f31 Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 15 Jan 2025 19:31:17 -0500 Subject: [PATCH 2/7] move hcs copy to bucket --- koku/hcs/csv_file_handler.py | 41 ++++++++++++++++---- koku/hcs/test/test_csv_file_handler.py | 15 ++++---- koku/koku/settings.py | 2 +- koku/masu/external/ros_report_shipper.py | 3 +- koku/masu/test/util/aws/test_common.py | 49 +++++++++--------------- koku/masu/util/aws/common.py | 15 -------- tox.ini | 3 ++ 7 files changed, 64 insertions(+), 64 deletions(-) diff --git a/koku/hcs/csv_file_handler.py b/koku/hcs/csv_file_handler.py index 200ad18db7..8ae53d9909 100644 --- a/koku/hcs/csv_file_handler.py +++ b/koku/hcs/csv_file_handler.py @@ -2,10 +2,13 @@ import os import pandas as pd +from botocore.exceptions import ClientError +from botocore.exceptions import EndpointConnectionError from dateutil import relativedelta +from django.conf import settings from api.common import log_json -from masu.util.aws.common import copy_local_hcs_report_file_to_s3_bucket +from masu.util.aws.common import get_s3_resource LOG = logging.getLogger(__name__) @@ -18,6 +21,12 @@ def __init__(self, schema_name, provider, provider_uuid): self._schema_name = str(schema_name).strip("acct") self._provider = provider self._provider_uuid = provider_uuid + self._s3_resource = get_s3_resource( + access_key=settings.S3_HCS_ACCESS_KEY, + secret_key=settings.S3_HCS_SECRET, + region=settings.S3_HCS_REGION, + endpoint_url=settings.S3_HCS_ENDPOINT, + ) def write_csv_to_s3(self, date, data, cols, finalize=False, tracing_id=None): """ @@ -30,7 +39,6 @@ def write_csv_to_s3(self, date, data, cols, finalize=False, tracing_id=None): :return none """ - my_df = pd.DataFrame(data) filename = f"hcs_{date}.csv" month = date.strftime("%m") year = date.strftime("%Y") @@ -44,13 +52,32 @@ def write_csv_to_s3(self, date, data, cols, finalize=False, tracing_id=None): if finalize: # reports are finalized on the 15th of the month following the report date finalize_date = (date.replace(day=15) + relativedelta.relativedelta(months=1)).strftime("%Y-%m-%d") - s3_csv_path = ( + s3_path = ( f"hcs/csv/{self._schema_name}/{self._provider}/source={self._provider_uuid}/year={year}/month={month}" ) - LOG.info(log_json(tracing_id, msg="preparing to write file to object storage")) + LOG.info(log_json(tracing_id, msg="preparing to write file to object storage", context=context)) + self.copy_data_to_s3_bucket(tracing_id, data, cols, filename, s3_path, finalize, finalize_date, context) + LOG.info(log_json(tracing_id, msg="wrote file to object storage", context=context)) + + def copy_data_to_s3_bucket( + self, tracing_id, data, cols, filename, s3_path, finalize, finalize_date, context + ): # pragma: no cover + """Copy hcs data to S3 bucket.""" + my_df = pd.DataFrame(data) my_df.to_csv(filename, header=cols, index=False) - copy_local_hcs_report_file_to_s3_bucket( - tracing_id, s3_csv_path, filename, filename, finalize, finalize_date, context - ) + metadata = {"finalized": str(finalize)} + if finalize and finalize_date: + metadata["finalized-date"] = finalize_date + extra_args = {"Metadata": metadata} + with open(filename, "rb") as fin: + try: + upload_key = f"{s3_path}/{filename}" + s3_obj = {"bucket_name": settings.S3_HCS_BUCKET_NAME, "key": upload_key} + upload = self._s3_resource.Object(**s3_obj) + upload.upload_fileobj(fin, ExtraArgs=extra_args) + except (EndpointConnectionError, ClientError) as err: + msg = f"unable to copy data to {upload_key}, bucket {settings.S3_HCS_BUCKET_NAME}. Reason: {str(err)}" + LOG.warning(log_json(tracing_id, msg=msg, context=context)) + return os.remove(filename) diff --git a/koku/hcs/test/test_csv_file_handler.py b/koku/hcs/test/test_csv_file_handler.py index 672aa287b2..e38094819c 100644 --- a/koku/hcs/test/test_csv_file_handler.py +++ b/koku/hcs/test/test_csv_file_handler.py @@ -13,6 +13,7 @@ from hcs.test import HCSTestCase +@patch("hcs.csv_file_handler.get_s3_resource") class TestHCSCSVFileHandler(HCSTestCase): """Test cases for HCS CSV File Handler""" @@ -24,14 +25,13 @@ def setUpClass(cls): cls.provider = Provider.PROVIDER_AWS cls.provider_uuid = "cabfdddb-4ed5-421e-a041-311b75daf235" - def test_init(self): + def test_init(self, *args): """Test the initializer.""" fh = CSVFileHandler(self.schema, self.provider, self.provider_uuid) self.assertEqual(fh._schema_name, "org1234567") self.assertEqual(fh._provider, "AWS") self.assertEqual(fh._provider_uuid, "cabfdddb-4ed5-421e-a041-311b75daf235") - @patch("masu.util.aws.common.get_s3_resource") def test_write_df_to_csv(self, *args): data = {"x": "123", "y": "456", "z": "456"} with self.assertLogs("hcs.csv_file_handler", "INFO") as _logs: @@ -40,10 +40,8 @@ def test_write_df_to_csv(self, *args): self.assertIn("preparing to write file to object storage", _logs.output[0]) - @patch("subs.subs_data_messenger.os.remove") - @patch("pandas.DataFrame") - @patch("hcs.csv_file_handler.copy_local_hcs_report_file_to_s3_bucket") - def test_write_csv_to_s3_finalize(self, mock_copy, mock_df, mock_remove): + @patch("hcs.csv_file_handler.CSVFileHandler.copy_data_to_s3_bucket") + def test_write_csv_to_s3_finalize(self, mock_copy, *args): """Test the right finalized date is determine when finalizing a report""" data = {"x": "123", "y": "456", "z": "456"} date = "2023-08-01" @@ -67,9 +65,10 @@ def test_write_csv_to_s3_finalize(self, mock_copy, mock_df, mock_remove): fh.write_csv_to_s3(datetime_date, data.items(), ["x", "y", "z"], finalize, mock_trace) mock_copy.assert_called_once_with( mock_trace, - expected_s3_path, - expected_filename, + data.items(), + ["x", "y", "z"], expected_filename, + expected_s3_path, finalize, expected_finalize_date, mock_context, diff --git a/koku/koku/settings.py b/koku/koku/settings.py index be070b5427..19cc54f93e 100644 --- a/koku/koku/settings.py +++ b/koku/koku/settings.py @@ -495,7 +495,7 @@ REQUESTED_ROS_BUCKET = ENVIRONMENT.get_value("REQUESTED_ROS_BUCKET", default="ros-report") REQUESTED_SUBS_BUCKET = ENVIRONMENT.get_value("REQUESTED_SUBS_BUCKET", default="subs-report") S3_TIMEOUT = ENVIRONMENT.int("S3_CONNECTION_TIMEOUT", default=60) -S3_DEFAULT_ENDPOINT = ENVIRONMENT.get_value("S3_ENDPOINT", default="https://s3.amazonaws.com") +S3_ENDPOINT = ENVIRONMENT.get_value("S3_ENDPOINT", default="https://s3.amazonaws.com") S3_REGION = ENVIRONMENT.get_value("S3_REGION", default="us-east-1") S3_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET) S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET) diff --git a/koku/masu/external/ros_report_shipper.py b/koku/masu/external/ros_report_shipper.py index c787b1401a..a26731701f 100644 --- a/koku/masu/external/ros_report_shipper.py +++ b/koku/masu/external/ros_report_shipper.py @@ -22,7 +22,6 @@ from masu.prometheus_stats import KAFKA_CONNECTION_ERRORS_COUNTER from masu.util.ocp import common as utils - LOG = logging.getLogger(__name__) @@ -34,7 +33,7 @@ def get_ros_s3_client(): # pragma: no cover aws_secret_access_key=settings.S3_ROS_SECRET, region_name=settings.S3_ROS_REGION, ) - return s3_session.client("s3", endpoint_url=settings.S3_ENDPOINT, config=config) + return s3_session.client("s3", endpoint_url=settings.S3_ROS_ENDPOINT, config=config) def generate_s3_object_url(client, upload_key): # pragma: no cover diff --git a/koku/masu/test/util/aws/test_common.py b/koku/masu/test/util/aws/test_common.py index fadc43aa92..a37887d78f 100644 --- a/koku/masu/test/util/aws/test_common.py +++ b/koku/masu/test/util/aws/test_common.py @@ -9,7 +9,6 @@ from unittest import TestCase from unittest.mock import call from unittest.mock import Mock -from unittest.mock import mock_open from unittest.mock import patch from unittest.mock import PropertyMock from uuid import uuid4 @@ -262,8 +261,9 @@ def test_update_account_aliases_no_aliases(self): mock_account_id = self.account_id - with patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, patch( - "masu.util.aws.common.get_account_names_by_organization" + with ( + patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, + patch("masu.util.aws.common.get_account_names_by_organization"), ): mock_get.return_value = (mock_account_id, mock_account_id) utils.update_account_aliases(self.aws_provider) @@ -279,8 +279,9 @@ def test_update_account_aliases_with_aliases(self): mock_account_id = self.account_id mock_alias = "mock_alias" - with patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, patch( - "masu.util.aws.common.get_account_names_by_organization" + with ( + patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, + patch("masu.util.aws.common.get_account_names_by_organization"), ): mock_get.return_value = (mock_account_id, mock_alias) utils.update_account_aliases(self.aws_provider) @@ -298,9 +299,10 @@ def test_update_account_aliases_with_aliases_and_orgs(self): mock_alias = "mock_alias" mock_alias2 = "mock_alias2" - with patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, patch( - "masu.util.aws.common.get_account_names_by_organization" - ) as mock_get_orgs: + with ( + patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, + patch("masu.util.aws.common.get_account_names_by_organization") as mock_get_orgs, + ): mock_get.return_value = (mock_account_id, mock_alias) mock_get_orgs.return_value = [{"id": mock_account_id2, "name": mock_alias2}] utils.update_account_aliases(self.aws_provider) @@ -536,9 +538,10 @@ def test_remove_s3_objects_not_matching_metadata(self): ) self.assertListEqual(removed, []) - with patch("masu.util.aws.common.get_s3_objects_not_matching_metadata") as mock_get_objects, patch( - "masu.util.aws.common.get_s3_resource" - ) as mock_s3: + with ( + patch("masu.util.aws.common.get_s3_objects_not_matching_metadata") as mock_get_objects, + patch("masu.util.aws.common.get_s3_resource") as mock_s3, + ): mock_s3.return_value.Object.return_value.delete.side_effect = ClientError({}, "Error") mock_get_objects.return_value = [] removed = utils.delete_s3_objects_not_matching_metadata( @@ -644,9 +647,10 @@ def test_remove_s3_objects_matching_metadata(self): ) self.assertListEqual(removed, []) - with patch("masu.util.aws.common.get_s3_objects_matching_metadata") as mock_get_objects, patch( - "masu.util.aws.common.get_s3_resource" - ) as mock_s3: + with ( + patch("masu.util.aws.common.get_s3_objects_matching_metadata") as mock_get_objects, + patch("masu.util.aws.common.get_s3_resource") as mock_s3, + ): mock_s3.return_value.Object.return_value.delete.side_effect = ClientError({}, "Error") mock_get_objects.return_value = [] removed = utils.delete_s3_objects_matching_metadata( @@ -665,23 +669,6 @@ def test_copy_data_to_s3_bucket(self): with self.assertRaises(utils.UploadError): utils.copy_data_to_s3_bucket("request_id", "path", "filename", "data", "manifest_id") - @patch("masu.util.aws.common.copy_data_to_s3_bucket") - def test_copy_local_hcs_report_file_to_s3_bucket_with_finalize(self, mock_copy): - """Test that the proper metadata is used when a finalized date is passed in with the finalize option""" - fake_request_id = "fake_id" - fake_s3_path = "fake_path" - fake_filename = "fake_filename" - expected_metadata = {"finalized": "True", "finalized-date": "2023-08-15"} - expected_context = {} - mock_op = mock_open(read_data="x,y,z") - with patch("builtins.open", mock_op): - utils.copy_local_hcs_report_file_to_s3_bucket( - fake_request_id, fake_s3_path, fake_filename, fake_filename, True, "2023-08-15", expected_context - ) - mock_copy.assert_called_once_with( - fake_request_id, fake_s3_path, fake_filename, mock_op(), expected_metadata, expected_context - ) - def test_match_openshift_resources_and_labels(self): """Test OCP on AWS data matching.""" cluster_topology = [ diff --git a/koku/masu/util/aws/common.py b/koku/masu/util/aws/common.py index 32521b37c6..2030f3d67f 100644 --- a/koku/masu/util/aws/common.py +++ b/koku/masu/util/aws/common.py @@ -561,21 +561,6 @@ def copy_local_report_file_to_s3_bucket( copy_data_to_s3_bucket(request_id, s3_path, local_filename, fin, metadata, context) -def copy_local_hcs_report_file_to_s3_bucket( - request_id, s3_path, full_file_path, local_filename, finalize=False, finalize_date=None, context={} -): - """ - Copies local report file to s3 bucket - """ - if s3_path: - LOG.info(f"copy_local_HCS_report_file_to_s3_bucket: {s3_path} {full_file_path}") - with open(full_file_path, "rb") as fin: - metadata = {"finalized": str(finalize)} - if finalize and finalize_date: - metadata["finalized-date"] = finalize_date - copy_data_to_s3_bucket(request_id, s3_path, local_filename, fin, metadata, context) - - def _get_s3_objects(s3_path): s3_resource = get_s3_resource(settings.S3_ACCESS_KEY, settings.S3_SECRET, settings.S3_REGION) return s3_resource.Bucket(settings.S3_BUCKET_NAME).objects.filter(Prefix=s3_path) diff --git a/tox.ini b/tox.ini index 8202011209..1af91cac09 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,8 @@ envlist = py311 skipsdist = True + + [flake8] ; E203 = Whitespace before ':', conflicts with black ; W503 = Line break before binary operator @@ -22,6 +24,7 @@ import-order-style = pycharm application-import-names = koku, api, providers, reporting, reporting_common, cost_models, masu [testenv] +allowlist_externals=/bin/sh setenv = DATABASE_SERVICE_NAME={env:DATABASE_SERVICE_NAME:POSTGRES_SQL} DATABASE_ENGINE={env:DATABASE_ENGINE:postgresql} From f74851c2bcae996786ced9465b36f739b94dfeb0 Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 15 Jan 2025 19:34:23 -0500 Subject: [PATCH 3/7] add trino_schema_prefix to sql --- .../sql/reporting_aws_hcs_daily_summary.sql | 2 +- .../sql/reporting_azure_hcs_daily_summary.sql | 2 +- .../sql/reporting_gcp_hcs_daily_summary.sql | 2 +- koku/koku/test_trino_db_utils.py | 16 ++-- .../openshift/managed_aws_openshift_daily.sql | 12 +-- .../reporting_ocpaws_compute_summary_p.sql | 2 +- ...rting_ocpaws_cost_summary_by_account_p.sql | 2 +- ...orting_ocpaws_cost_summary_by_region_p.sql | 2 +- ...rting_ocpaws_cost_summary_by_service_p.sql | 2 +- .../reporting_ocpaws_cost_summary_p.sql | 2 +- .../reporting_ocpaws_database_summary_p.sql | 2 +- .../reporting_ocpaws_network_summary_p.sql | 2 +- .../reporting_ocpaws_storage_summary_p.sql | 2 +- ...porting_ocpinfrastructure_provider_map.sql | 4 +- .../managed_azure_openshift_daily.sql | 12 +-- .../reporting_ocpazure_compute_summary_p.sql | 2 +- ...ing_ocpazure_cost_summary_by_account_p.sql | 2 +- ...ng_ocpazure_cost_summary_by_location_p.sql | 2 +- ...ing_ocpazure_cost_summary_by_service_p.sql | 2 +- .../reporting_ocpazure_cost_summary_p.sql | 2 +- .../reporting_ocpazure_database_summary_p.sql | 2 +- .../reporting_ocpazure_network_summary_p.sql | 2 +- .../reporting_ocpazure_storage_summary_p.sql | 2 +- ...porting_ocpinfrastructure_provider_map.sql | 4 +- .../openshift/managed_gcp_openshift_daily.sql | 12 +-- .../reporting_ocpgcp_compute_summary_p.sql | 2 +- ...rting_ocpgcp_cost_summary_by_account_p.sql | 2 +- ...g_ocpgcp_cost_summary_by_gcp_project_p.sql | 2 +- ...orting_ocpgcp_cost_summary_by_region_p.sql | 2 +- ...rting_ocpgcp_cost_summary_by_service_p.sql | 2 +- .../reporting_ocpgcp_cost_summary_p.sql | 2 +- .../reporting_ocpgcp_database_summary_p.sql | 2 +- .../reporting_ocpgcp_matched_tags.sql | 4 +- .../reporting_ocpgcp_network_summary_p.sql | 2 +- .../reporting_ocpgcp_storage_summary_p.sql | 2 +- ...rting_ocpgcpcostlineitem_daily_summary.sql | 26 +++---- ...costlineitem_daily_summary_resource_id.sql | 38 +++++----- ...porting_ocpinfrastructure_provider_map.sql | 6 +- .../trino_sql/ocp_special_matched_tags.sql | 4 +- ...ing_awscostentrylineitem_daily_summary.sql | 2 +- ...entrylineitem_summary_by_ec2_compute_p.sql | 6 +- ...g_azurecostentrylineitem_daily_summary.sql | 2 +- ...ing_gcpcostentrylineitem_daily_summary.sql | 2 +- ...ing_ocicostentrylineitem_daily_summary.sql | 4 +- .../reporting_ocpaws_matched_tags.sql | 4 +- ...rting_ocpawscostlineitem_daily_summary.sql | 76 +++++++++---------- .../reporting_ocpazure_matched_tags.sql | 4 +- ...ing_ocpazurecostlineitem_daily_summary.sql | 68 ++++++++--------- ...porting_ocpusagelineitem_daily_summary.sql | 24 +++--- .../gcp/managed_ocp_on_gcp_verification.sql | 4 +- .../verify/gcp/managed_resources.sql | 6 +- .../managed_ocp_on_aws_verification.sql | 4 +- .../managed_ocp_on_azure_verification.sql | 4 +- .../aws/determine_ids_for_provider.sql | 2 +- ...termine_resource_ids_for_usage_account.sql | 2 +- koku/subs/trino_sql/aws/subs_row_count.sql | 2 +- koku/subs/trino_sql/aws/subs_summary.sql | 2 +- .../azure/determine_ids_for_provider.sql | 2 +- ...termine_resource_ids_for_usage_account.sql | 2 +- koku/subs/trino_sql/azure/subs_row_count.sql | 2 +- koku/subs/trino_sql/azure/subs_summary.sql | 2 +- 61 files changed, 211 insertions(+), 209 deletions(-) diff --git a/koku/hcs/database/sql/reporting_aws_hcs_daily_summary.sql b/koku/hcs/database/sql/reporting_aws_hcs_daily_summary.sql index 173d7dca04..24199ab832 100644 --- a/koku/hcs/database/sql/reporting_aws_hcs_daily_summary.sql +++ b/koku/hcs/database/sql/reporting_aws_hcs_daily_summary.sql @@ -52,7 +52,7 @@ SELECT {{ebs_acct_num}} as ebs_account_id, {{org_id}} as org_id FROM - hive.{{schema | sqlsafe}}.{{table | sqlsafe}} + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}} WHERE source = {{provider_uuid}} AND year = {{year}} diff --git a/koku/hcs/database/sql/reporting_azure_hcs_daily_summary.sql b/koku/hcs/database/sql/reporting_azure_hcs_daily_summary.sql index c51313a2a0..f5908b51cf 100644 --- a/koku/hcs/database/sql/reporting_azure_hcs_daily_summary.sql +++ b/koku/hcs/database/sql/reporting_azure_hcs_daily_summary.sql @@ -3,7 +3,7 @@ SELECT {{ebs_acct_num}} as ebs_account_id, {{org_id}} as org_id FROM - hive.{{schema | sqlsafe}}.{{table | sqlsafe}} + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}} WHERE source = {{provider_uuid}} AND year = {{year}} diff --git a/koku/hcs/database/sql/reporting_gcp_hcs_daily_summary.sql b/koku/hcs/database/sql/reporting_gcp_hcs_daily_summary.sql index d80cdafc96..c6429876f4 100644 --- a/koku/hcs/database/sql/reporting_gcp_hcs_daily_summary.sql +++ b/koku/hcs/database/sql/reporting_gcp_hcs_daily_summary.sql @@ -1,5 +1,5 @@ SELECT *, {{ebs_acct_num}} as ebs_account_id, {{org_id}} as org_id -FROM hive.{{schema | sqlsafe}}.{{table | sqlsafe}} +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}} WHERE ( sku_description LIKE '%RedHat%' OR sku_description LIKE '%Red Hat%' OR service_description LIKE '%Red Hat%') diff --git a/koku/koku/test_trino_db_utils.py b/koku/koku/test_trino_db_utils.py index e7732be08f..f154e43701 100644 --- a/koku/koku/test_trino_db_utils.py +++ b/koku/koku/test_trino_db_utils.py @@ -30,7 +30,7 @@ def test_executescript_jinja_conditionals(self): """ sqlscript = """ {% if populate %} -SELECT * FROM hive.{{schema | sqlsafe}}.fake_table +SELECT * FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.fake_table {% endif %} ; """ @@ -61,23 +61,25 @@ def test_executescript(self): Test execution of a buffer containing multiple statements """ sqlscript = """ -drop table if exists hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}; -create table hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} +drop table if exists hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}; +create table hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} ( id varchar, i_data integer, t_data varchar ); -insert into hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data) +insert into hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data) values (cast(uuid() as varchar), 10, 'default'); -insert into hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data) +insert into hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data) values (cast(uuid() as varchar), {{int_data}}, {{txt_data}}); -select t_data from hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} where i_data = {{int_data}}; +select t_data + from hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} +where i_data = {{int_data}}; -drop table if exists hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}; +drop table if exists hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}; """ conn = FakeTrinoConn() params = { diff --git a/koku/masu/database/trino_sql/aws/openshift/managed_aws_openshift_daily.sql b/koku/masu/database/trino_sql/aws/openshift/managed_aws_openshift_daily.sql index 65d70bb6ae..38a1879690 100644 --- a/koku/masu/database/trino_sql/aws/openshift/managed_aws_openshift_daily.sql +++ b/koku/masu/database/trino_sql/aws/openshift/managed_aws_openshift_daily.sql @@ -1,5 +1,5 @@ -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_aws_openshift_daily +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_aws_openshift_daily ( lineitem_resourceid varchar, lineitem_usagestartdate timestamp(3), @@ -45,7 +45,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_aws_openshift_daily ; -- Direct resource matching -INSERT INTO hive.{{schema | sqlsafe}}.managed_aws_openshift_daily ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_aws_openshift_daily ( lineitem_resourceid, lineitem_usagestartdate, bill_payeraccountid, @@ -85,7 +85,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.managed_aws_openshift_daily ( ) WITH cte_aws_resource_names AS ( SELECT DISTINCT lineitem_resourceid - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily WHERE source = {{aws_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -94,7 +94,7 @@ WITH cte_aws_resource_names AS ( ), cte_array_agg_nodes AS ( SELECT DISTINCT resource_id - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND resource_id != '' AND year = {{year}} @@ -104,7 +104,7 @@ cte_array_agg_nodes AS ( ), cte_array_agg_volumes AS ( SELECT DISTINCT persistentvolume, csi_volume_handle - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND persistentvolume != '' AND year = {{year}} @@ -174,7 +174,7 @@ SELECT aws.lineitem_resourceid, aws.year, aws.month, cast(day(aws.lineitem_usagestartdate) as varchar) as day -FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS aws LEFT JOIN cte_matchable_resource_names AS resource_names ON resource_names.lineitem_resourceid = aws.lineitem_resourceid LEFT JOIN cte_agg_tags AS tag_matches diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql index 3b20e001ac..c663fc26c7 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql @@ -42,7 +42,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_compute_summary_p ( sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql index 2cdc055dcd..ff850851e5 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql @@ -34,7 +34,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_accou sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql index 352feacb63..cbd37f5bf2 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_regio sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql index ef74f74521..b8fd333686 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_servi sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql index 1938331aff..0889a449b8 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql @@ -32,7 +32,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_p ( max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid, max(cost_category_id) - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql index 03abcbd246..d1b65b1d33 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_database_summary_p ( sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql index 04e1f702d1..9136747baa 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_network_summary_p ( sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql index a376e83d33..ce6de6528a 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_storage_summary_p ( sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/reporting_ocpinfrastructure_provider_map.sql b/koku/masu/database/trino_sql/aws/reporting_ocpinfrastructure_provider_map.sql index 11ce0b6397..e6b01d8d99 100644 --- a/koku/masu/database/trino_sql/aws/reporting_ocpinfrastructure_provider_map.sql +++ b/koku/masu/database/trino_sql/aws/reporting_ocpinfrastructure_provider_map.sql @@ -2,7 +2,7 @@ WITH cte_aws_resource_ids AS ( SELECT DISTINCT lineitem_resourceid, aws.source - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS aws WHERE aws.lineitem_usagestartdate >= {{start_date}} AND aws.lineitem_usagestartdate < date_add('day', 1, {{end_date}}) AND aws.lineitem_resourceid IS NOT NULL @@ -16,7 +16,7 @@ WITH cte_aws_resource_ids AS ( cte_ocp_resource_ids AS ( SELECT DISTINCT resource_id, ocp.source - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp WHERE ocp.interval_start >= {{start_date}} AND ocp.interval_start < date_add('day', 1, {{end_date}}) AND ocp.resource_id IS NOT NULL diff --git a/koku/masu/database/trino_sql/azure/openshift/managed_azure_openshift_daily.sql b/koku/masu/database/trino_sql/azure/openshift/managed_azure_openshift_daily.sql index c342e1989c..2da50a3195 100644 --- a/koku/masu/database/trino_sql/azure/openshift/managed_azure_openshift_daily.sql +++ b/koku/masu/database/trino_sql/azure/openshift/managed_azure_openshift_daily.sql @@ -1,5 +1,5 @@ -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_azure_openshift_daily +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_azure_openshift_daily ( accountname varchar, additionalinfo varchar, @@ -45,7 +45,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_azure_openshift_dai ; -- Direct resource matching -INSERT INTO hive.{{schema | sqlsafe}}.managed_azure_openshift_daily ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_azure_openshift_daily ( accountname, additionalinfo, billingcurrency, @@ -89,7 +89,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.managed_azure_openshift_daily ( ) WITH cte_azure_resource_names AS ( SELECT DISTINCT resourceid - FROM hive.{{schema | sqlsafe}}.azure_line_items + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source = {{azure_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -98,7 +98,7 @@ WITH cte_azure_resource_names AS ( ), cte_array_agg_nodes AS ( SELECT DISTINCT node - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -107,7 +107,7 @@ cte_array_agg_nodes AS ( ), cte_array_agg_volumes AS ( SELECT DISTINCT persistentvolume, csi_volume_handle - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -180,7 +180,7 @@ SELECT azure.accountname, azure.year, azure.month, cast(day(azure.date) as varchar) as day -FROM hive.{{schema | sqlsafe}}.azure_line_items AS azure +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items AS azure LEFT JOIN cte_matchable_resource_names AS resource_names ON azure.resourceid = resource_names.resourceid LEFT JOIN cte_agg_tags AS tag_matches diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql index bd323cc2c2..7832cc1efe 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql @@ -30,7 +30,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_compute_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql index 7d6ff2fb50..5601f4c6a7 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql @@ -22,7 +22,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_acc sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql index 4df03bdbec..8bab6c5086 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_loc sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql index 9943268054..1e2e25cf9a 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_ser sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql index 2233cc81df..81c8193302 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql @@ -20,7 +20,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_p ( max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid, max(cost_category_id) as cost_category_id - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql index 3ebe4d4dfe..1b124c80d9 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_database_summary_p sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql index 150118b550..4930019aeb 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_network_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql index 29d6c87236..8fb3559f9e 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_storage_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/reporting_ocpinfrastructure_provider_map.sql b/koku/masu/database/trino_sql/azure/reporting_ocpinfrastructure_provider_map.sql index 2b7eb0ef10..c4b44414cc 100644 --- a/koku/masu/database/trino_sql/azure/reporting_ocpinfrastructure_provider_map.sql +++ b/koku/masu/database/trino_sql/azure/reporting_ocpinfrastructure_provider_map.sql @@ -2,7 +2,7 @@ WITH cte_azure_instances AS ( SELECT DISTINCT split_part(azure.resourceid, '/', 9) as instance, azure.source - FROM hive.{{schema | sqlsafe}}.azure_line_items AS azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items AS azure WHERE azure.date >= {{start_date}} AND azure.date < date_add('day', 1, {{end_date}}) {% if azure_provider_uuid %} @@ -14,7 +14,7 @@ WITH cte_azure_instances AS ( cte_ocp_nodes AS ( SELECT DISTINCT ocp.node, ocp.source - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp WHERE ocp.interval_start >= {{start_date}} AND ocp.interval_start < date_add('day', 1, {{end_date}}) AND ocp.node IS NOT NULL diff --git a/koku/masu/database/trino_sql/gcp/openshift/managed_gcp_openshift_daily.sql b/koku/masu/database/trino_sql/gcp/openshift/managed_gcp_openshift_daily.sql index 8718576a0e..b1c1f040a3 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/managed_gcp_openshift_daily.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/managed_gcp_openshift_daily.sql @@ -1,5 +1,5 @@ -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_gcp_openshift_daily ( invoice_month varchar, billing_account_id varchar, @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily ; -- Direct resource matching -INSERT INTO hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_gcp_openshift_daily ( invoice_month, billing_account_id, project_id, @@ -63,7 +63,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily ( ) WITH cte_gcp_resource_names AS ( SELECT DISTINCT resource_name - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily WHERE source = {{gcp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -72,7 +72,7 @@ WITH cte_gcp_resource_names AS ( ), cte_array_agg_nodes AS ( SELECT DISTINCT node - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -81,7 +81,7 @@ cte_array_agg_nodes AS ( ), cte_array_agg_volumes AS ( SELECT DISTINCT persistentvolume, csi_volume_handle - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -141,7 +141,7 @@ SELECT gcp.invoice_month, gcp.year, gcp.month, cast(day(gcp.usage_start_time) as varchar) as day -FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily AS gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily AS gcp LEFT JOIN cte_matchable_resource_names AS resource_names ON gcp.resource_name = resource_names.resource_name LEFT JOIN cte_agg_tags AS tag_matches diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql index bb99732422..bbaad6a8bd 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql @@ -31,7 +31,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql index 9fdbe05d37..2647a6f7cc 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql @@ -25,7 +25,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_accou cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql index 2fafd275ca..448d0e2585 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql @@ -27,7 +27,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_p cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql index ef85f0a62c..81e692ff5b 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql @@ -27,7 +27,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_regio cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql index c135be4690..70fac01c78 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql @@ -29,7 +29,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_servi cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql index 780ccac4c2..da2ea2d37b 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql @@ -25,7 +25,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( sum(credit_amount) as credit_amount, invoice_month, max(cost_category_id) - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql index fafbd6ead9..a9ec9c1004 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_database_summary_p ( cast({{gcp_source_uuid}} as uuid) as gcp_source, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_matched_tags.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_matched_tags.sql index f38a50e803..360a2c6b8f 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_matched_tags.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_matched_tags.sql @@ -1,7 +1,7 @@ WITH cte_unnested_gcp_tags AS ( SELECT DISTINCT key, value - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily AS gcp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily AS gcp CROSS JOIN UNNEST(cast(json_parse(labels) as map(varchar, varchar))) AS tags(key, value) WHERE source = {{gcp_source_uuid}} AND year = {{year}} @@ -14,7 +14,7 @@ cte_unnested_ocp_tags AS ( pod_value, volume_key, volume_value - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp CROSS JOIN UNNEST( cast(json_parse(pod_labels) as map(varchar, varchar)), cast(json_parse(volume_labels) as map(varchar, varchar)) diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql index 2cd2b457b2..52d7734c7f 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_network_summary_p ( cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql index 7231fd38e5..3e9136354a 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql index f82528848b..1881cc9506 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql @@ -1,5 +1,5 @@ -- First we'll store the data in a "temp" table to do our grouping against -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid varchar, cluster_id varchar, @@ -57,7 +57,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineite ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid varchar, cluster_id varchar, @@ -114,7 +114,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineite ; -- OCP ON GCP kubernetes-io-cluster-{cluster_id} label is applied on the VM and is exclusively a pod cost -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid, cluster_id, cluster_alias, @@ -210,8 +210,8 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp -JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as gcp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON date(gcp.usage_start_time) = ocp.usage_start AND (strpos(gcp.labels, 'kubernetes-io-cluster-{{cluster_id | sqlsafe}}') != 0 -- THIS IS THE SPECIFIC TO OCP ON GCP TAG MATCH OR strpos(gcp.labels, 'kubernetes-io-cluster-{{cluster_alias | sqlsafe}}') != 0) @@ -232,7 +232,7 @@ GROUP BY gcp.uuid, ocp.namespace, gcp.invoice_month, ocp.data_source, ocp.pod_la ; -- direct tag matching, these costs are split evenly between pod and storage since we don't have the info to quantify them separately -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid, cluster_id, cluster_alias, @@ -328,8 +328,8 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp -JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as gcp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON date(gcp.usage_start_time) = ocp.usage_start AND ( json_query(gcp.labels, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace @@ -343,7 +343,7 @@ JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp AND ocp.namespace != 'Platform unallocated' AND ocp.namespace != 'Network unattributed' AND ocp.namespace != 'Storage unattributed' -LEFT JOIN hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds +LEFT JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds ON gcp.uuid = pds.gcp_uuid WHERE gcp.source = {{gcp_source_uuid}} AND gcp.year = {{year}} @@ -361,7 +361,7 @@ GROUP BY gcp.uuid, ocp.namespace, ocp.data_source, gcp.invoice_month ; -- Group by to calculate proper cost per project -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid, cluster_id, cluster_alias, @@ -413,7 +413,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily WITH cte_rankings AS ( SELECT pds.gcp_uuid, count(*) as gcp_uuid_count - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds GROUP BY gcp_uuid ) SELECT pds.gcp_uuid, @@ -498,7 +498,7 @@ SELECT pds.gcp_uuid, cast(year(usage_start) as varchar) as year, cast(month(usage_start) as varchar) as month, cast(day(usage_start) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp as pds +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp as pds JOIN cte_rankings as r ON pds.gcp_uuid = r.gcp_uuid WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month = {{month}} @@ -580,7 +580,7 @@ SELECT uuid(), cast(gcp_source as UUID), credit_amount, invoice_month -FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql index 15f0fec8d6..342bd9d024 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql @@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS {{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_ ) WITH(format = 'PARQUET', partitioned_by=ARRAY['ocp_source', 'year', 'month']) ; -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid varchar, cluster_id varchar, @@ -114,7 +114,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineite ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid varchar, cluster_id varchar, @@ -171,7 +171,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineite ) WITH(format = 'PARQUET', partitioned_by=ARRAY['gcp_source', 'ocp_source', 'year', 'month', 'day']) ; -INSERT INTO hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp ( uuid, usage_start, account_id, @@ -225,7 +225,7 @@ SELECT cast(uuid() as varchar), {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as gcp WHERE gcp.source = {{gcp_source_uuid}} AND gcp.year = {{year}} AND gcp.month = {{month}} @@ -243,7 +243,7 @@ GROUP BY gcp.usage_start_time, 10 -- data transfer direction ; -INSERT INTO hive.{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp ( uuid, usage_start, account_id, @@ -309,7 +309,7 @@ SELECT cast(uuid() as varchar), {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as gcp CROSS JOIN cte_enabled_tag_keys as etk WHERE gcp.source = {{gcp_source_uuid}} AND gcp.year = {{year}} @@ -328,7 +328,7 @@ GROUP BY gcp.usage_start_time, ; -- Direct resource_id matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid, cluster_id, cluster_alias, @@ -430,8 +430,8 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( (strpos(gcp.resource_name, ocp.node) != 0 AND ocp.data_source='Pod') @@ -451,7 +451,7 @@ GROUP BY gcp.uuid, ocp.namespace, ocp.data_source, ocp.pod_labels, ocp.volume_la ; -- direct tag matching, these costs are split evenly between pod and storage since we don't have the info to quantify them separately -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid, cluster_id, cluster_alias, @@ -553,8 +553,8 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( json_query(gcp.labels, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace @@ -580,7 +580,7 @@ GROUP BY gcp.uuid, ocp.namespace, ocp.data_source, gcp.invoice_month ; -- Group by to calculate proper cost per project -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid, cluster_id, cluster_alias, @@ -635,7 +635,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily WITH cte_rankings AS ( SELECT pds.gcp_uuid, count(*) as gcp_uuid_count - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds GROUP BY gcp_uuid ) SELECT pds.gcp_uuid, @@ -717,7 +717,7 @@ SELECT pds.gcp_uuid, cast(year(usage_start) as varchar) as year, cast(month(usage_start) as varchar) as month, cast(day(usage_start) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp as pds +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp as pds JOIN cte_rankings as r ON pds.gcp_uuid = r.gcp_uuid WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month = {{month}} @@ -728,7 +728,7 @@ WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month -- and costs are split out by pod metrics, this puts all network costs per node -- into a "Network unattributed" project with no cost split and one record per -- data direction -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid, cluster_id, cluster_alias, @@ -806,8 +806,8 @@ SELECT gcp.uuid as gcp_uuid, cast(year(max(gcp.usage_start)) as varchar) as year, cast(month(max(gcp.usage_start)) as varchar) as month, cast(day(max(gcp.usage_start)) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( (strpos(gcp.resource_name, ocp.node) != 0 AND ocp.data_source='Pod') @@ -923,7 +923,7 @@ SELECT uuid(), cast(gcp_source as UUID), credit_amount, invoice_month -FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/gcp/reporting_ocpinfrastructure_provider_map.sql b/koku/masu/database/trino_sql/gcp/reporting_ocpinfrastructure_provider_map.sql index 741293fc17..30be38b3c3 100644 --- a/koku/masu/database/trino_sql/gcp/reporting_ocpinfrastructure_provider_map.sql +++ b/koku/masu/database/trino_sql/gcp/reporting_ocpinfrastructure_provider_map.sql @@ -8,7 +8,7 @@ cte_distinct_gcp_labels AS ( SELECT DISTINCT labels, source - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily WHERE source = {{gcp_provider_uuid}} AND year = {{year}} AND month = {{month}} @@ -34,7 +34,7 @@ WITH cte_gcp_resource_name AS ( SELECT DISTINCT gcp.resource_name, gcp.source - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily AS gcp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily AS gcp WHERE gcp.usage_start_time >= {{start_date}} AND gcp.usage_start_time < date_add('day', 1, {{end_date}}) {% if gcp_provider_uuid %} @@ -46,7 +46,7 @@ cte_ocp_nodes AS ( SELECT DISTINCT ocp.node, ocp.source - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp WHERE ocp.interval_start >= {{start_date}} AND ocp.interval_start < date_add('day', 1, {{end_date}}) AND ocp.node IS NOT NULL diff --git a/koku/masu/database/trino_sql/ocp_special_matched_tags.sql b/koku/masu/database/trino_sql/ocp_special_matched_tags.sql index 4b39c4ad41..1d62ae7dba 100644 --- a/koku/masu/database/trino_sql/ocp_special_matched_tags.sql +++ b/koku/masu/database/trino_sql/ocp_special_matched_tags.sql @@ -1,6 +1,6 @@ WITH cte_array_agg_nodes AS ( SELECT DISTINCT node - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -34,7 +34,7 @@ cte_tag_matches AS ( UNION SELECT distinct format('"openshift_project": "%s"', namespace) - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND month = {{month}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql index f8f99d3e9e..e52d8060f3 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql @@ -122,7 +122,7 @@ FROM ( max(pricing_publicondemandrate) as public_on_demand_rate, array_agg(DISTINCT lineitem_resourceid) as resource_ids, count(DISTINCT lineitem_resourceid) as resource_count - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily WHERE source = '{{source_uuid | sqlsafe}}' AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql index 0b014be4a1..ab5d111d6d 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql @@ -48,7 +48,7 @@ cte_latest_values as ( costcategory as cost_category, nullif(product_memory, '') as memory, cast(nullif(product_vcpu, '') AS INTEGER) as vcpu - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily as alid + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily as alid WHERE source = '{{source_uuid | sqlsafe}}' AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' @@ -57,7 +57,7 @@ cte_latest_values as ( AND lineitem_resourceid != '' AND lineitem_usagestartdate = ( SELECT max(date(lv.lineitem_usagestartdate)) AS usage_start - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS lv + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS lv WHERE lineitem_resourceid = alid.lineitem_resourceid AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' @@ -164,7 +164,7 @@ FROM ( ) as calculated_amortized_cost, sum(pricing_publicondemandcost) as public_on_demand_cost, max(pricing_publicondemandrate) as public_on_demand_rate - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily as lid + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily as lid WHERE source = '{{source_uuid | sqlsafe}}' AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_azurecostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_azurecostentrylineitem_daily_summary.sql index 0315d23ca2..4c16e9e9f3 100644 --- a/koku/masu/database/trino_sql/reporting_azurecostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_azurecostentrylineitem_daily_summary.sql @@ -46,7 +46,7 @@ WITH cte_line_items AS ( THEN split_part(unitofmeasure, ' ', 2) ELSE unitofmeasure END as unit_of_measure - FROM hive.{{schema | sqlsafe}}.azure_line_items + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source = '{{source_uuid | sqlsafe}}' AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_gcpcostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_gcpcostentrylineitem_daily_summary.sql index 8b274d1201..f205880b6d 100644 --- a/koku/masu/database/trino_sql/reporting_gcpcostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_gcpcostentrylineitem_daily_summary.sql @@ -57,7 +57,7 @@ SELECT uuid() as uuid, UUID '{{source_uuid | sqlsafe}}' as source_uuid, invoice_month, sum(((cast(COALESCE(json_extract_scalar(json_parse(credits), '$["amount"]'), '0')AS decimal(24,9)))*1000000)/1000000) as credit_amount -FROM hive.{{schema | sqlsafe}}.{{table | sqlsafe}} +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}} CROSS JOIN cte_pg_enabled_keys as pek WHERE source = '{{source_uuid | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_ocicostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocicostentrylineitem_daily_summary.sql index 80324b255e..78c288697f 100644 --- a/koku/masu/database/trino_sql/reporting_ocicostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocicostentrylineitem_daily_summary.sql @@ -87,8 +87,8 @@ FROM ( max(c.cost_currencycode) as currency, sum(c.cost_mycost) as cost, c.tags as tags - FROM hive.{{schema | sqlsafe}}.oci_cost_line_items as c - JOIN hive.{{schema | sqlsafe}}.oci_usage_line_items as u + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.oci_cost_line_items as c + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.oci_usage_line_items as u ON c.lineItem_intervalUsageStart = u.lineItem_intervalUsageStart AND c.product_resourceId = u.product_resourceId WHERE c.source = '{{source_uuid | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_ocpaws_matched_tags.sql b/koku/masu/database/trino_sql/reporting_ocpaws_matched_tags.sql index 0e8809707a..c28b2f4b8c 100644 --- a/koku/masu/database/trino_sql/reporting_ocpaws_matched_tags.sql +++ b/koku/masu/database/trino_sql/reporting_ocpaws_matched_tags.sql @@ -13,7 +13,7 @@ WITH cte_enabled_tag_keys AS ( cte_unnested_aws_tags AS ( SELECT DISTINCT key, value - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS aws CROSS JOIN UNNEST(cast(json_parse(resourcetags) as map(varchar, varchar))) AS tags(key, value) JOIN cte_enabled_tag_keys AS etk ON any_match(etk.key_array, x->strpos(aws.resourcetags, x) != 0) @@ -28,7 +28,7 @@ cte_unnested_ocp_tags AS ( pod_value, volume_key, volume_value - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp CROSS JOIN UNNEST( cast(json_parse(pod_labels) as map(varchar, varchar)), cast(json_parse(volume_labels) as map(varchar, varchar)) diff --git a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql index 677fa1dc0a..a48d863ab9 100644 --- a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql @@ -54,7 +54,7 @@ CREATE TABLE IF NOT EXISTS {{schema | sqlsafe}}.aws_openshift_daily_tag_matched_ ) WITH(format = 'PARQUET', partitioned_by=ARRAY['ocp_source', 'year', 'month']) ; -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid varchar, cluster_id varchar, @@ -113,7 +113,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpawscostlineite ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( aws_uuid varchar, cluster_id varchar, @@ -162,7 +162,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpawscostlineite ) WITH(format = 'PARQUET', partitioned_by=ARRAY['aws_source', 'ocp_source', 'year', 'month', 'day']) ; -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp ( resource_id varchar, capacity integer, @@ -172,7 +172,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.aws_openshift_disk_capaciti month varchar ) WITH(format = 'PARQUET', partitioned_by=ARRAY['ocp_source', 'year', 'month']); -INSERT INTO hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp ( uuid, usage_start, resource_id, @@ -255,7 +255,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.aws_openshift_daily as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily as aws WHERE aws.source = {{aws_source_uuid}} AND aws.year = {{year}} AND aws.month = {{month}} @@ -276,7 +276,7 @@ GROUP BY aws.lineitem_usagestartdate, lineitem_operation ; -INSERT INTO hive.{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp ( uuid, usage_start, resource_id, @@ -362,7 +362,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.aws_openshift_daily as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily as aws CROSS JOIN cte_enabled_tag_keys as etk WHERE aws.source = {{aws_source_uuid}} AND aws.year = {{year}} @@ -387,7 +387,7 @@ GROUP BY aws.lineitem_usagestartdate, -- Developer notes -- We can't use the aws_openshift_daily table to calcualte the capacity -- because it has already aggregated cost per each hour. -INSERT INTO hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp ( resource_id, capacity, usage_start, @@ -405,8 +405,8 @@ cte_ocp_filtered_resources as ( DATE(aws.usage_start) as usage_start, aws.year as year, aws.month as month - FROM hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws - JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.csi_volume_handle) != 0 AND ocp.csi_volume_handle is not null @@ -427,7 +427,7 @@ calculated_capacity AS ( {{ocp_source_uuid}} as ocp_source, {{year}} as year, {{month}} as month - FROM hive.{{schema | sqlsafe}}.aws_line_items as aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items as aws INNER JOIN cte_ocp_filtered_resources as ocpaws ON aws.lineitem_resourceid = ocpaws.resource_id AND DATE(aws.lineitem_usagestartdate) = ocpaws.usage_start @@ -448,7 +448,7 @@ WHERE capacity > 0 -- Maintain tag matching logic for disk resources -- until unattributed storage is released -- FIXME: Remove this section when the unleash flag is removed -INSERT INTO hive.{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp ( uuid, usage_start, resource_id, @@ -484,7 +484,7 @@ WITH cte_enabled_tag_keys AS ( ), cte_csi_volume_handles as ( SELECT distinct csi_volume_handle as csi_volume_handle - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily as ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily as ocp WHERE ocp.source = {{ocp_source_uuid}} AND ocp.year = {{year}} AND ocp.month = {{month}} @@ -541,7 +541,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.aws_openshift_daily as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily as aws CROSS JOIN cte_enabled_tag_keys as etk CROSS JOIN cte_csi_volume_handles as csi WHERE aws.source = {{aws_source_uuid}} @@ -573,7 +573,7 @@ GROUP BY aws.lineitem_usagestartdate, -- Algorhtim: -- (PV Capacity) / Disk Capacity * Cost of Disk -- PV without PVCs are unattributed storage -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid, cluster_id, cluster_alias, @@ -675,13 +675,13 @@ SELECT cast(uuid() as varchar) as aws_uuid, -- need a new uuid or it will dedup {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.csi_volume_handle) != 0 AND ocp.csi_volume_handle is not null AND ocp.csi_volume_handle != '' -JOIN hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk ON aws_disk.usage_start = aws.usage_start AND aws_disk.resource_id = aws.resource_id WHERE ocp.source = {{ocp_source_uuid}} @@ -706,7 +706,7 @@ GROUP BY aws.uuid, ocp.namespace, ocp.pod_labels, ocp.volume_labels {% if unattributed_storage %} -- Unattributed Storage Cost: -- ((Disk Capacity - Sum(PV capacity) / Disk Capacity) * Cost of Disk -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid, cluster_id, cluster_alias, @@ -749,8 +749,8 @@ WITH cte_total_pv_capacity as ( ocp.persistentvolume, max(ocp.persistentvolumeclaim_capacity_gigabyte) as capacity, aws.resource_id as aws_resource_id - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws ON (aws.usage_start = ocp.usage_start) AND strpos(aws.resource_id, ocp.csi_volume_handle) > 0 AND ocp.csi_volume_handle is not null @@ -797,13 +797,13 @@ SELECT cast(uuid() as varchar) as aws_uuid, -- need a new uuid or it will dedup {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.csi_volume_handle) != 0 AND ocp.csi_volume_handle is not null AND ocp.csi_volume_handle != '' -JOIN hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk ON aws_disk.usage_start = aws.usage_start AND aws_disk.resource_id = aws.resource_id LEFT JOIN cte_total_pv_capacity as pv_cap @@ -828,7 +828,7 @@ GROUP BY aws.uuid, aws.resource_id ; -- Direct resource_id matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid, cluster_id, cluster_alias, @@ -932,11 +932,11 @@ SELECT aws.uuid as aws_uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.resource_id) != 0 - LEFT JOIN hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk + LEFT JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk ON aws_disk.usage_start = aws.usage_start AND aws_disk.resource_id = aws.resource_id AND aws_disk.year = aws.year @@ -957,7 +957,7 @@ SELECT aws.uuid as aws_uuid, ; -- Tag matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid, cluster_id, cluster_alias, @@ -1061,8 +1061,8 @@ SELECT aws.uuid as aws_uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp as aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp as aws ON aws.usage_start = ocp.usage_start AND ( json_query(aws.tags, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace @@ -1087,7 +1087,7 @@ SELECT aws.uuid as aws_uuid, ; -- Group by to calculate proper cost per project -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( aws_uuid, cluster_id, cluster_alias, @@ -1134,7 +1134,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily WITH cte_rankings AS ( SELECT pds.aws_uuid, count(*) as aws_uuid_count - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp AS pds + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp AS pds GROUP BY aws_uuid ) SELECT pds.aws_uuid, @@ -1220,7 +1220,7 @@ SELECT pds.aws_uuid, cast(year(usage_start) as varchar) as year, cast(month(usage_start) as varchar) as month, cast(day(usage_start) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp AS pds +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp AS pds JOIN cte_rankings as r ON pds.aws_uuid = r.aws_uuid LEFT JOIN postgres.{{schema | sqlsafe}}.reporting_awsaccountalias AS aa @@ -1229,7 +1229,7 @@ WHERE pds.ocp_source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{mon ; -- Put Node Network Costs into the Network unattributed namespace -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( aws_uuid, cluster_id, cluster_alias, @@ -1314,8 +1314,8 @@ SELECT max(cast(year(aws.usage_start) AS varchar)) AS year, max(cast(month(aws.usage_start) AS varchar)) AS month, max(cast(day(aws.usage_start) AS varchar)) AS day -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp -JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp AS aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp AS aws ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.resource_id) != 0 LEFT JOIN postgres.{{schema | sqlsafe}}.reporting_awsaccountalias AS aa @@ -1430,7 +1430,7 @@ SELECT uuid(), json_parse(aws_cost_category), cost_category_id, cast(aws_source as UUID) -FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/reporting_ocpazure_matched_tags.sql b/koku/masu/database/trino_sql/reporting_ocpazure_matched_tags.sql index cd55dbc35e..fea5dd92ff 100644 --- a/koku/masu/database/trino_sql/reporting_ocpazure_matched_tags.sql +++ b/koku/masu/database/trino_sql/reporting_ocpazure_matched_tags.sql @@ -3,7 +3,7 @@ WITH cte_unnested_azure_tags AS ( SELECT DISTINCT key, value - FROM hive.{{schema | sqlsafe}}.azure_line_items AS azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items AS azure CROSS JOIN UNNEST(cast(json_parse(tags) as map(varchar, varchar))) AS tags(key, value) WHERE source = {{azure_source_uuid}} AND year = {{year}} @@ -16,7 +16,7 @@ cte_unnested_ocp_tags AS ( pod_value, volume_key, volume_value - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp CROSS JOIN UNNEST( cast(json_parse(pod_labels) as map(varchar, varchar)), cast(json_parse(volume_labels) as map(varchar, varchar)) diff --git a/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql index 0334e1d042..6ffcb9a886 100644 --- a/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql @@ -44,7 +44,7 @@ CREATE TABLE IF NOT EXISTS {{schema | sqlsafe}}.azure_openshift_daily_tag_matche ) WITH(format = 'PARQUET', partitioned_by=ARRAY['ocp_source', 'year', 'month']) ; -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid varchar, cluster_id varchar, @@ -95,7 +95,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpazurecostlinei ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( azure_uuid varchar, cluster_id varchar, @@ -136,7 +136,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpazurecostlinei ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp ( resource_id varchar, capacity integer, @@ -148,7 +148,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.azure_openshift_disk_capaci ; -INSERT INTO hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp ( uuid, usage_start, resource_id, @@ -198,7 +198,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month -FROM hive.{{schema | sqlsafe}}.azure_openshift_daily as azure +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily as azure WHERE azure.source = {{azure_source_uuid}} AND azure.year = {{year}} AND azure.month = {{month}} @@ -218,7 +218,7 @@ GROUP BY azure.date, ; -INSERT INTO hive.{{schema | sqlsafe}}.azure_openshift_daily_tag_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_tag_matched_temp ( uuid, usage_start, resource_id, @@ -279,7 +279,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month -FROM hive.{{schema | sqlsafe}}.azure_openshift_daily as azure +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily as azure CROSS JOIN cte_enabled_tag_keys as etk WHERE azure.source = {{azure_source_uuid}} AND azure.year = {{year}} @@ -299,7 +299,7 @@ GROUP BY azure.date, ; {% if unattributed_storage %} -INSERT INTO hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp ( resource_id, capacity, usage_start, @@ -310,8 +310,8 @@ INSERT INTO hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp ( WITH cte_ocp_filtered_resources as ( SELECT distinct azure.resource_id as azure_partial_resource_id - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') @@ -356,7 +356,7 @@ GROUP BY ocp_filtered.azure_partial_resource_id, date(date) -- (PV’s Capacity) / Disk Capacity * Cost of Disk -- PV without PVCs are Unattributed Storage -- 2 volumes can share the same disk id -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid, cluster_id, cluster_alias, @@ -434,15 +434,15 @@ SELECT cast(uuid() as varchar) as azure_uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') OR (lower(ocp.csi_volume_handle) = lower(azure.resource_id)) ) - JOIN hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk ON az_disk.usage_start = azure.usage_start AND az_disk.resource_id = azure.resource_id WHERE ocp.source = {{ocp_source_uuid}} @@ -467,7 +467,7 @@ SELECT cast(uuid() as varchar) as azure_uuid, {% if unattributed_storage %} -- Unallocated Cost: ((Disk Capacity - Sum(PV capacity) / Disk Capacity) * Cost of Disk -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid, cluster_id, cluster_alias, @@ -502,8 +502,8 @@ WITH cte_total_pv_capacity as ( ocp.persistentvolume, max(ocp.persistentvolumeclaim_capacity_gigabyte) as capacity, azure.resource_id as azure_resource_id - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') @@ -544,15 +544,15 @@ SELECT cast(uuid() as varchar) as azure_uuid, -- need a new uuid or it will dedu {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') OR (lower(ocp.csi_volume_handle) = lower(azure.resource_id)) ) - JOIN hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk ON az_disk.usage_start = azure.usage_start AND az_disk.resource_id = azure.resource_id LEFT JOIN cte_total_pv_capacity as pv_cap @@ -574,7 +574,7 @@ SELECT cast(uuid() as varchar) as azure_uuid, -- need a new uuid or it will dedu ; -- Directly Pod resource_id matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid, cluster_id, cluster_alias, @@ -662,14 +662,14 @@ SELECT azure.uuid as azure_uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (replace(lower(azure.resource_id), '_osdisk', '') = lower(ocp.node) AND ocp.data_source = 'Pod') OR (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') ) - LEFT JOIN hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp as disk_cap + LEFT JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp as disk_cap ON azure.resource_id = disk_cap.resource_id AND disk_cap.year = azure.year AND disk_cap.month = azure.month @@ -690,7 +690,7 @@ SELECT azure.uuid as azure_uuid, ; -- Tag matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid, cluster_id, cluster_alias, @@ -752,8 +752,8 @@ SELECT azure.uuid as azure_uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_tag_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_tag_matched_temp as azure ON azure.usage_start = ocp.usage_start AND ( json_query(azure.tags, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace @@ -780,7 +780,7 @@ SELECT azure.uuid as azure_uuid, -- Group by to calculate proper cost per project -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( azure_uuid, cluster_id, cluster_alias, @@ -818,7 +818,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_dai WITH cte_rankings AS ( SELECT pds.azure_uuid, count(*) as azure_uuid_count - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp AS pds + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp AS pds GROUP BY azure_uuid ) SELECT pds.azure_uuid, @@ -877,7 +877,7 @@ SELECT pds.azure_uuid, cast(year(usage_start) as varchar) as year, cast(month(usage_start) as varchar) as month, cast(day(usage_start) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp AS pds +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp AS pds JOIN cte_rankings as r ON pds.azure_uuid = r.azure_uuid WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month = {{month}} @@ -888,7 +888,7 @@ WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month -- and costs are split out by pod metrics, this puts all network costs per node -- into a "Network unattributed" project with no cost split and one record per -- data direction -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( azure_uuid, cluster_id, cluster_alias, @@ -952,8 +952,8 @@ SELECT azure.uuid as azure_uuid, max(azure.year) as year, max(azure.month) as month, cast(day(max(azure.usage_start)) as varchar) as day - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON azure.usage_start = ocp.usage_start AND (azure.resource_id = ocp.node AND ocp.data_source = 'Pod') WHERE ocp.source = {{ocp_source_uuid}} @@ -1049,7 +1049,7 @@ SELECT uuid(), json_parse(tags), cost_category_id, cast(azure_source as UUID) -FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/reporting_ocpusagelineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpusagelineitem_daily_summary.sql index 65b857d067..1d52397b0d 100644 --- a/koku/masu/database/trino_sql/reporting_ocpusagelineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpusagelineitem_daily_summary.sql @@ -3,7 +3,7 @@ * This SQL will utilize Trino for the raw line-item data aggregating * and store the results into the koku database summary tables. */ -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( uuid varchar, report_period_id int, cluster_id varchar, @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_ ) WITH(format = 'PARQUET', partitioned_by=ARRAY['source', 'year', 'month', 'day']) ; -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( uuid, report_period_id, cluster_id, @@ -107,7 +107,7 @@ cte_ocp_node_label_line_item_daily AS ( (k,v) -> contains(pek.keys, k) ) as json ) as node_labels - FROM hive.{{schema | sqlsafe}}.openshift_node_labels_line_items_daily AS nli + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_node_labels_line_items_daily AS nli CROSS JOIN cte_pg_enabled_keys AS pek WHERE nli.source = {{source}} AND nli.year = {{year}} @@ -128,7 +128,7 @@ cte_ocp_namespace_label_line_item_daily AS ( (k,v) -> contains(pek.keys, k) ) as json ) as namespace_labels - FROM hive.{{schema | sqlsafe}}.openshift_namespace_labels_line_items_daily AS nli + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_namespace_labels_line_items_daily AS nli CROSS JOIN cte_pg_enabled_keys AS pek WHERE nli.source = {{source}} AND nli.year = {{year}} @@ -150,7 +150,7 @@ cte_ocp_node_capacity AS ( li.node, max(li.node_capacity_cpu_core_seconds) as node_capacity_cpu_core_seconds, max(li.node_capacity_memory_byte_seconds) as node_capacity_memory_byte_seconds - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items AS li + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items AS li WHERE li.source = {{source}} AND li.year = {{year}} AND li.month = {{month}} @@ -180,8 +180,8 @@ cte_volume_nodes AS ( sli.namespace, uli.node, uli.resource_id - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily as sli - JOIN hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily as uli + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily as sli + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily as uli ON uli.source = sli.source AND uli.namespace = sli.namespace AND uli.pod = sli.pod @@ -286,7 +286,7 @@ FROM ( max(nc.node_capacity_memory_byte_seconds) / 3600.0 * power(2, -30) as node_capacity_memory_gigabyte_hours, max(cc.cluster_capacity_cpu_core_seconds) / 3600.0 as cluster_capacity_cpu_core_hours, max(cc.cluster_capacity_memory_byte_seconds) / 3600.0 * power(2, -30) as cluster_capacity_memory_gigabyte_hours - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily as li + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily as li CROSS JOIN cte_pg_enabled_keys AS pek LEFT JOIN cte_ocp_node_label_line_item_daily as nli ON nli.node = li.node @@ -408,7 +408,7 @@ FROM ( -- Divide volume usage and requests by the number of nodes that volume is mounted on sum(sli.volume_request_storage_byte_seconds) / max(nc.node_count) as volume_request_storage_byte_seconds, sum(sli.persistentvolumeclaim_usage_byte_seconds) / max(nc.node_count) as persistentvolumeclaim_usage_byte_seconds - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily sli + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily sli CROSS JOIN cte_pg_enabled_keys AS pek LEFT JOIN cte_volume_nodes as vn ON vn.usage_start = date(sli.interval_start) @@ -457,7 +457,7 @@ What was selected from unallocated capacity. AND lids.namespace != 'Platform unallocated' AND lids.namespace != 'Worker unallocated' */ -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( uuid, report_period_id, cluster_id, @@ -528,7 +528,7 @@ cte_unallocated_capacity AS ( cast(year(lids.usage_start) as varchar) as year, cast(month(lids.usage_start) as varchar) as month, cast(day(lids.usage_start) as varchar) as day - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as lids + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as lids LEFT JOIN cte_node_role AS nodes ON lids.node = nodes.node AND lids.resource_id = nodes.resource_id @@ -658,7 +658,7 @@ SELECT uuid(), cast(source_uuid as UUID), json_parse(infrastructure_usage_cost), cost_category_id -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS lids +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS lids WHERE lids.source = {{source}} AND lids.year = {{year}} AND lpad(lids.month, 2, '0') = {{month}} -- Zero pad the month when fewer than 2 characters diff --git a/koku/masu/database/trino_sql/verify/gcp/managed_ocp_on_gcp_verification.sql b/koku/masu/database/trino_sql/verify/gcp/managed_ocp_on_gcp_verification.sql index b842d1d94c..0dff2ab8cc 100644 --- a/koku/masu/database/trino_sql/verify/gcp/managed_ocp_on_gcp_verification.sql +++ b/koku/masu/database/trino_sql/verify/gcp/managed_ocp_on_gcp_verification.sql @@ -9,7 +9,7 @@ SELECT FROM ( SELECT sum(cost) AS managed_total_cost - FROM hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily as managed_ocpcloud + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_gcp_openshift_daily as managed_ocpcloud WHERE managed_ocpcloud.source = {{cloud_source_uuid}} AND managed_ocpcloud.year = {{year}} AND managed_ocpcloud.month = {{month}} @@ -17,7 +17,7 @@ FROM ) t1, ( SELECT sum(cost) AS parquet_total_cost - FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as parquet_table + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as parquet_table LEFT JOIN cte_agg_tags AS tag_matches ON any_match(tag_matches.matched_tags, x->strpos(parquet_table.labels, x) != 0) AND parquet_table.ocp_matched = False diff --git a/koku/masu/database/trino_sql/verify/gcp/managed_resources.sql b/koku/masu/database/trino_sql/verify/gcp/managed_resources.sql index 46f3520b4e..ccaa00f6ab 100644 --- a/koku/masu/database/trino_sql/verify/gcp/managed_resources.sql +++ b/koku/masu/database/trino_sql/verify/gcp/managed_resources.sql @@ -9,7 +9,7 @@ cte_resource_breakdown AS ( SUM(cost) AS cost FROM ( SELECT 'parquet' AS source_type, resource_name, usage_start_time, cost - FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily parquet_table + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily parquet_table WHERE source = {{cloud_source_uuid}} AND year = {{year}} AND month = {{month}} AND (ocp_matched = TRUE OR EXISTS ( @@ -19,7 +19,7 @@ cte_resource_breakdown AS ( )) UNION ALL SELECT 'managed' AS source_type, resource_name, usage_start_time, cost - FROM hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_gcp_openshift_daily WHERE source = {{cloud_source_uuid}} AND year = {{year}} AND month = {{month}} AND (resource_id_matched = TRUE OR matched_tag != '') @@ -51,7 +51,7 @@ cte_initial_cost_check AS ( WHEN SUM(gcp.cost) < MAX(d.parquet_cost) AND SUM(gcp.cost) = MAX(d.managed_cost) THEN TRUE ELSE FALSE END AS parquet_issue - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily gcp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily gcp JOIN cte_discrepancies d ON gcp.resource_name = d.resource_name AND gcp.usage_start_time = d.usage_start_time diff --git a/koku/masu/database/trino_sql/verify/managed_ocp_on_aws_verification.sql b/koku/masu/database/trino_sql/verify/managed_ocp_on_aws_verification.sql index 4806b0bc87..6fc97f1ec0 100644 --- a/koku/masu/database/trino_sql/verify/managed_ocp_on_aws_verification.sql +++ b/koku/masu/database/trino_sql/verify/managed_ocp_on_aws_verification.sql @@ -11,7 +11,7 @@ SELECT FROM ( SELECT sum(lineitem_unblendedcost) AS managed_total_cost - FROM hive.{{schema | sqlsafe}}.managed_aws_openshift_daily as managed_ocpcloud + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_aws_openshift_daily as managed_ocpcloud WHERE managed_ocpcloud.source = {{cloud_source_uuid}} AND managed_ocpcloud.year = {{year}} AND managed_ocpcloud.month = {{month}} @@ -21,7 +21,7 @@ FROM ) t1, ( SELECT sum(lineitem_unblendedcost) as parquet_total_cost - FROM hive.{{schema | sqlsafe}}.aws_openshift_daily as parquet_table + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily as parquet_table LEFT JOIN cte_agg_tags AS tag_matches ON any_match(tag_matches.matched_tags, x->strpos(parquet_table.resourcetags, x) != 0) AND parquet_table.resource_id_matched = False diff --git a/koku/masu/database/trino_sql/verify/managed_ocp_on_azure_verification.sql b/koku/masu/database/trino_sql/verify/managed_ocp_on_azure_verification.sql index 7f0d68abc6..5f0e0c1c99 100644 --- a/koku/masu/database/trino_sql/verify/managed_ocp_on_azure_verification.sql +++ b/koku/masu/database/trino_sql/verify/managed_ocp_on_azure_verification.sql @@ -11,7 +11,7 @@ SELECT FROM ( SELECT sum(costinbillingcurrency) AS managed_total_cost - FROM hive.{{schema | sqlsafe}}.managed_azure_openshift_daily as managed_ocpcloud + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_azure_openshift_daily as managed_ocpcloud WHERE managed_ocpcloud.source = {{cloud_source_uuid}} AND managed_ocpcloud.year = {{year}} AND managed_ocpcloud.month = {{month}} @@ -19,7 +19,7 @@ FROM ) t1, ( SELECT sum(costinbillingcurrency) AS parquet_total_cost - FROM hive.{{schema | sqlsafe}}.azure_openshift_daily as parquet_table + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily as parquet_table LEFT JOIN cte_agg_tags AS tag_matches ON any_match(tag_matches.matched_tags, x->strpos(parquet_table.tags, x) != 0) AND parquet_table.resource_id_matched = False diff --git a/koku/subs/trino_sql/aws/determine_ids_for_provider.sql b/koku/subs/trino_sql/aws/determine_ids_for_provider.sql index a6bcfad89b..029e9b816c 100644 --- a/koku/subs/trino_sql/aws/determine_ids_for_provider.sql +++ b/koku/subs/trino_sql/aws/determine_ids_for_provider.sql @@ -1,6 +1,6 @@ SELECT DISTINCT lineitem_usageaccountid -FROM hive.{{schema | sqlsafe}}.aws_line_items +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items WHERE source={{source_uuid}} AND year={{year}} AND month={{month}} diff --git a/koku/subs/trino_sql/aws/determine_resource_ids_for_usage_account.sql b/koku/subs/trino_sql/aws/determine_resource_ids_for_usage_account.sql index a697c56577..9fc7fed824 100644 --- a/koku/subs/trino_sql/aws/determine_resource_ids_for_usage_account.sql +++ b/koku/subs/trino_sql/aws/determine_resource_ids_for_usage_account.sql @@ -1,7 +1,7 @@ SELECT lineitem_resourceid, max(lineitem_usagestartdate) -FROM hive.{{schema | sqlsafe}}.aws_line_items +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items WHERE source={{source_uuid}} AND year={{year}} AND month={{month}} diff --git a/koku/subs/trino_sql/aws/subs_row_count.sql b/koku/subs/trino_sql/aws/subs_row_count.sql index 7bbb67ba80..a60c25e712 100644 --- a/koku/subs/trino_sql/aws/subs_row_count.sql +++ b/koku/subs/trino_sql/aws/subs_row_count.sql @@ -1,6 +1,6 @@ SELECT count(*) FROM - hive.{{schema | sqlsafe}}.aws_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items WHERE source = {{ source_uuid }} AND year = {{ year }} diff --git a/koku/subs/trino_sql/aws/subs_summary.sql b/koku/subs/trino_sql/aws/subs_summary.sql index 480fad481d..a5d2520e1d 100644 --- a/koku/subs/trino_sql/aws/subs_summary.sql +++ b/koku/subs/trino_sql/aws/subs_summary.sql @@ -58,7 +58,7 @@ FROM ) as json ) as tags from - hive.{{schema | sqlsafe}}.aws_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items WHERE source = {{ source_uuid }} AND year = {{ year }} diff --git a/koku/subs/trino_sql/azure/determine_ids_for_provider.sql b/koku/subs/trino_sql/azure/determine_ids_for_provider.sql index b50e9a7bfe..a19a931cbc 100644 --- a/koku/subs/trino_sql/azure/determine_ids_for_provider.sql +++ b/koku/subs/trino_sql/azure/determine_ids_for_provider.sql @@ -1,6 +1,6 @@ SELECT DISTINCT COALESCE(NULLIF(subscriptionid, ''), subscriptionguid) -FROM hive.{{schema | sqlsafe}}.azure_line_items +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source={{source_uuid}} AND year={{year}} AND month={{month}} diff --git a/koku/subs/trino_sql/azure/determine_resource_ids_for_usage_account.sql b/koku/subs/trino_sql/azure/determine_resource_ids_for_usage_account.sql index 58f194b5b3..d89de5f46d 100644 --- a/koku/subs/trino_sql/azure/determine_resource_ids_for_usage_account.sql +++ b/koku/subs/trino_sql/azure/determine_resource_ids_for_usage_account.sql @@ -14,7 +14,7 @@ FROM ( COALESCE(json_extract_scalar(lower(additionalinfo), '$.vmname'), regexp_extract(resourceid, '([^/]+$)')) as vmname, max(date) max_date FROM - hive.{{schema | sqlsafe}}.azure_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source={{source_uuid}} AND year={{year}} diff --git a/koku/subs/trino_sql/azure/subs_row_count.sql b/koku/subs/trino_sql/azure/subs_row_count.sql index 7e30c310ca..85540bd69e 100644 --- a/koku/subs/trino_sql/azure/subs_row_count.sql +++ b/koku/subs/trino_sql/azure/subs_row_count.sql @@ -1,7 +1,7 @@ SELECT COUNT(*) FROM - hive.{{schema | sqlsafe}}.azure_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source = {{ source_uuid }} AND year = {{ year }} diff --git a/koku/subs/trino_sql/azure/subs_summary.sql b/koku/subs/trino_sql/azure/subs_summary.sql index 24d5e4ec0d..7268f62aa8 100644 --- a/koku/subs/trino_sql/azure/subs_summary.sql +++ b/koku/subs/trino_sql/azure/subs_summary.sql @@ -39,7 +39,7 @@ SELECT -- if the VMName isn't present in additionalinfo, the end of the resourceid should be the VMName COALESCE(json_extract_scalar(lower(additionalinfo), '$.vmname'), regexp_extract(resourceid, '([^/]+$)')) as subs_vmname FROM - hive.{{schema | sqlsafe}}.azure_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source = {{ source_uuid }} AND year = {{ year }} From 0dcabe2c29bf24a92f8ebfc99fa154bc052be563 Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 15 Jan 2025 19:41:49 -0500 Subject: [PATCH 4/7] comment out SCHEMA_SUFFIX for now --- koku/koku/settings.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/koku/koku/settings.py b/koku/koku/settings.py index 19cc54f93e..3be93221fa 100644 --- a/koku/koku/settings.py +++ b/koku/koku/settings.py @@ -141,11 +141,11 @@ SCHEMA_SUFFIX = re.sub("[^a-zA-Z0-9_]", "_", ENVIRONMENT.get_value("SCHEMA_SUFFIX", default="")) print(f"ORG ID SUFFIX: '{SCHEMA_SUFFIX}'") if DEVELOPMENT: - if SCHEMA_SUFFIX == "": - SCHEMA_SUFFIX = f"_{ENVIRONMENT.get_value('USER', default='')}" - if not SCHEMA_SUFFIX.startswith("_"): - SCHEMA_SUFFIX = f"_{SCHEMA_SUFFIX}" - print(f"ORG ID SUFFIX: '{SCHEMA_SUFFIX}'") + # if SCHEMA_SUFFIX == "": + # SCHEMA_SUFFIX = f"_{ENVIRONMENT.get_value('USER', default='')}" + # if not SCHEMA_SUFFIX.startswith("_"): + # SCHEMA_SUFFIX = f"_{SCHEMA_SUFFIX}" + # print(f"ORG ID SUFFIX: '{SCHEMA_SUFFIX}'") DEFAULT_IDENTITY = { "identity": { "account_number": "10001", @@ -495,7 +495,7 @@ REQUESTED_ROS_BUCKET = ENVIRONMENT.get_value("REQUESTED_ROS_BUCKET", default="ros-report") REQUESTED_SUBS_BUCKET = ENVIRONMENT.get_value("REQUESTED_SUBS_BUCKET", default="subs-report") S3_TIMEOUT = ENVIRONMENT.int("S3_CONNECTION_TIMEOUT", default=60) -S3_ENDPOINT = ENVIRONMENT.get_value("S3_ENDPOINT", default="https://s3.amazonaws.com") +S3_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() S3_REGION = ENVIRONMENT.get_value("S3_REGION", default="us-east-1") S3_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET) S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET) From 083bfe06015531d61e0106d08bcec68d2124bc36 Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 15 Jan 2025 19:52:21 -0500 Subject: [PATCH 5/7] fix clowdapp --- deploy/clowdapp.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 91b615d86f..e1617a2395 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -5816,6 +5816,8 @@ parameters: value: "" - name: TRINO_SCHEMA_PREFIX value: "" +- name: TRINO_S3A_OR_S3 + value: s3a - name: S3_BUCKET_NAME value: hccm-eph-s3 - name: S3_ROS_BUCKET_NAME From 8f5771f26b60ec22aa8188b47366aca6a51f1537 Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 15 Jan 2025 20:01:54 -0500 Subject: [PATCH 6/7] fix deploy --- ci/functions.sh | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/ci/functions.sh b/ci/functions.sh index d8abf6cb37..cefb710cb1 100755 --- a/ci/functions.sh +++ b/ci/functions.sh @@ -94,14 +94,9 @@ function run_smoke_tests_stage() { source ${CICD_ROOT}/_common_deploy_logic.sh export NAMESPACE=$(bonfire namespace reserve --duration ${RESERVATION_TIMEOUT}) - oc get secret/koku-aws -o json -n ephemeral-base | jq -r '.data' > aws-creds.json - oc get secret/koku-gcp -o json -n ephemeral-base | jq -r '.data' > gcp-creds.json - oc get secret/koku-oci -o json -n ephemeral-base | jq -r '.data' > oci-creds.json - - AWS_CREDENTIALS_EPH=$(jq -r '."aws-credentials"' < aws-creds.json) - GCP_CREDENTIALS_EPH=$(jq -r '."gcp-credentials"' < gcp-creds.json) - OCI_CREDENTIALS_EPH=$(jq -r '."oci-credentials"' < oci-creds.json) - OCI_CONFIG_EPH=$(jq -r '."oci-config"' < oci-creds.json) + oc get secret koku-aws -o yaml -n ephemeral-base | grep -v '^\s*namespace:\s' | oc apply --namespace=${NAMESPACE} -f - + oc get secret koku-gcp -o yaml -n ephemeral-base | grep -v '^\s*namespace:\s' | oc apply --namespace=${NAMESPACE} -f - + oc get secret koku-oci -o yaml -n ephemeral-base | grep -v '^\s*namespace:\s' | oc apply --namespace=${NAMESPACE} -f - bonfire deploy \ ${APP_NAME} \ @@ -113,10 +108,6 @@ function run_smoke_tests_stage() { ${COMPONENTS_RESOURCES_ARG} \ --optional-deps-method hybrid \ --set-parameter rbac/MIN_REPLICAS=1 \ - --set-parameter koku/AWS_CREDENTIALS_EPH=${AWS_CREDENTIALS_EPH} \ - --set-parameter koku/GCP_CREDENTIALS_EPH=${GCP_CREDENTIALS_EPH} \ - --set-parameter koku/OCI_CREDENTIALS_EPH=${OCI_CREDENTIALS_EPH} \ - --set-parameter koku/OCI_CONFIG_EPH=${OCI_CONFIG_EPH} \ --set-parameter koku/DBM_IMAGE=${IMAGE} \ --set-parameter koku/DBM_IMAGE_TAG=${IMAGE_TAG} \ --set-parameter koku/DBM_INVOCATION=${DBM_INVOCATION} \ From b9c41546f3c2b4142f83b2fc0f19d1c78e21965f Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 15 Jan 2025 21:02:45 -0500 Subject: [PATCH 7/7] some fixes --- .../report_parquet_processor_base.py | 2 +- .../aws/test_aws_report_parquet_processor.py | 4 ++-- .../gcp/test_gcp_report_parquet_processor.py | 4 ++-- .../oci/test_oci_report_parquet_processor.py | 4 ++-- .../test_report_parquet_processor_base.py | 22 +++++++++---------- koku/masu/util/aws/common.py | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/koku/masu/processor/report_parquet_processor_base.py b/koku/masu/processor/report_parquet_processor_base.py index cb05080127..ec77863a0b 100644 --- a/koku/masu/processor/report_parquet_processor_base.py +++ b/koku/masu/processor/report_parquet_processor_base.py @@ -65,7 +65,7 @@ def _execute_trino_sql(self, sql, schema_name: str): # pragma: no cover cur = conn.cursor() cur.execute(sql) rows = cur.fetchall() - LOG.debug(f"_execute_sql rows: {str(rows)}. Type: {type(rows)}") + LOG.debug(f"_execute_trino_sql rows: {str(rows)}. Type: {type(rows)}") except TrinoUserError as err: LOG.warning(err) except TrinoExternalError as err: diff --git a/koku/masu/test/processor/aws/test_aws_report_parquet_processor.py b/koku/masu/test/processor/aws/test_aws_report_parquet_processor.py index 99feecb9c0..ccccb4cb00 100644 --- a/koku/masu/test/processor/aws/test_aws_report_parquet_processor.py +++ b/koku/masu/test/processor/aws/test_aws_report_parquet_processor.py @@ -54,7 +54,7 @@ def test_postgres_summary_table(self): """Test that the correct table is returned.""" self.assertEqual(self.processor.postgres_summary_table, AWSCostEntryLineItemDailySummary) - @patch("masu.processor.aws.aws_report_parquet_processor.AWSReportParquetProcessor._execute_sql") + @patch("masu.processor.aws.aws_report_parquet_processor.AWSReportParquetProcessor._execute_trino_sql") def test_create_bill(self, mock_execute_sql): """Test that a bill is created in the Postgres database.""" bill_date = DateHelper().this_month_start @@ -73,7 +73,7 @@ def test_create_bill(self, mock_execute_sql): ) self.assertIsNotNone(bill.first()) - @patch("masu.processor.aws.aws_report_parquet_processor.AWSReportParquetProcessor._execute_sql") + @patch("masu.processor.aws.aws_report_parquet_processor.AWSReportParquetProcessor._execute_trino_sql") def test_create_bill_with_string_arg(self, mock_execute_sql): """Test that a bill is created in the Postgres database.""" bill_date = DateHelper().this_month_start diff --git a/koku/masu/test/processor/gcp/test_gcp_report_parquet_processor.py b/koku/masu/test/processor/gcp/test_gcp_report_parquet_processor.py index 784f538747..197fd5d093 100644 --- a/koku/masu/test/processor/gcp/test_gcp_report_parquet_processor.py +++ b/koku/masu/test/processor/gcp/test_gcp_report_parquet_processor.py @@ -54,7 +54,7 @@ def test_postgres_summary_table(self): """Test that the correct table is returned.""" self.assertEqual(self.processor.postgres_summary_table, GCPCostEntryLineItemDailySummary) - @patch("masu.processor.gcp.gcp_report_parquet_processor.GCPReportParquetProcessor._execute_sql") + @patch("masu.processor.gcp.gcp_report_parquet_processor.GCPReportParquetProcessor._execute_trino_sql") def test_create_bill(self, mock_execute_sql): """Test that a bill is created in the Postgres database.""" bill_date = DateHelper().this_month_start @@ -72,7 +72,7 @@ def test_create_bill(self, mock_execute_sql): ) self.assertIsNotNone(bill.first()) - @patch("masu.processor.gcp.gcp_report_parquet_processor.GCPReportParquetProcessor._execute_sql") + @patch("masu.processor.gcp.gcp_report_parquet_processor.GCPReportParquetProcessor._execute_trino_sql") def test_create_bill_with_string_arg(self, mock_execute_sql): """Test that a bill is created in the Postgres database.""" bill_date = DateHelper().this_month_start diff --git a/koku/masu/test/processor/oci/test_oci_report_parquet_processor.py b/koku/masu/test/processor/oci/test_oci_report_parquet_processor.py index 6b30080c00..beb1cfc147 100644 --- a/koku/masu/test/processor/oci/test_oci_report_parquet_processor.py +++ b/koku/masu/test/processor/oci/test_oci_report_parquet_processor.py @@ -47,7 +47,7 @@ def test_postgres_summary_table(self): """Test that the correct table is returned.""" self.assertEqual(self.processor.postgres_summary_table, OCICostEntryLineItemDailySummary) - @patch("masu.processor.oci.oci_report_parquet_processor.OCIReportParquetProcessor._execute_sql") + @patch("masu.processor.oci.oci_report_parquet_processor.OCIReportParquetProcessor._execute_trino_sql") def test_create_bill(self, mock_execute_sql): """Test that a bill is created in the Postgres database.""" bill_date = DateHelper().this_month_start @@ -66,7 +66,7 @@ def test_create_bill(self, mock_execute_sql): ) self.assertIsNotNone(bill.first()) - @patch("masu.processor.oci.oci_report_parquet_processor.OCIReportParquetProcessor._execute_sql") + @patch("masu.processor.oci.oci_report_parquet_processor.OCIReportParquetProcessor._execute_trino_sql") def test_create_bill_with_string_arg(self, mock_execute_sql): """Test that a bill is created in the Postgres database.""" bill_date = DateHelper().this_month_start diff --git a/koku/masu/test/processor/test_report_parquet_processor_base.py b/koku/masu/test/processor/test_report_parquet_processor_base.py index 920ff9873f..97edcf44f2 100644 --- a/koku/masu/test/processor/test_report_parquet_processor_base.py +++ b/koku/masu/test/processor/test_report_parquet_processor_base.py @@ -91,7 +91,7 @@ def test_postgres_summary_table(self): self.processor.postgres_summary_table @override_settings(S3_BUCKET_NAME="test-bucket") - @patch("masu.processor.aws.aws_report_parquet_processor.ReportParquetProcessorBase._execute_sql") + @patch("masu.processor.aws.aws_report_parquet_processor.ReportParquetProcessorBase._execute_trino_sql") def test_generate_create_table_sql(self, mock_execute): """Test the generate parquet table sql.""" generated_sql = self.processor._generate_create_table_sql() @@ -111,7 +111,7 @@ def test_generate_create_table_sql(self, mock_execute): self.assertTrue(generated_sql.endswith(expected_end)) @override_settings(S3_BUCKET_NAME="test-bucket") - @patch("masu.processor.aws.aws_report_parquet_processor.ReportParquetProcessorBase._execute_sql") + @patch("masu.processor.aws.aws_report_parquet_processor.ReportParquetProcessorBase._execute_trino_sql") def test_generate_create_table_sql_with_provider_map(self, mock_execute): """Test the generate parquet table sql.""" partition_map = { @@ -136,7 +136,7 @@ def test_generate_create_table_sql_with_provider_map(self, mock_execute): self.assertIn(f"{other_col} varchar", generated_sql) self.assertTrue(generated_sql.endswith(expected_end)) - @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_sql") + @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_trino_sql") def test_create_table(self, mock_execute): """Test the Trino/Hive create table method.""" expected_logs = [] @@ -150,7 +150,7 @@ def test_create_table(self, mock_execute): for expected_log in expected_logs: self.assertIn(expected_log, logger.output) - @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_sql") + @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_trino_sql") def test_sync_hive_partitions(self, mock_execute): """Test that hive partitions are synced.""" expected_log = self.log_output_info + str( @@ -160,7 +160,7 @@ def test_sync_hive_partitions(self, mock_execute): self.processor.sync_hive_partitions() self.assertIn(expected_log, logger.output) - @patch.object(ReportParquetProcessorBase, "_execute_sql") + @patch.object(ReportParquetProcessorBase, "_execute_trino_sql") def test_schema_exists_cache_value_in_cache(self, trino_mock): with patch( "masu.processor.report_parquet_processor_base.get_value_from_cache", @@ -169,7 +169,7 @@ def test_schema_exists_cache_value_in_cache(self, trino_mock): self.assertTrue(self.processor.schema_exists()) trino_mock.assert_not_called() - @patch.object(ReportParquetProcessorBase, "_execute_sql") + @patch.object(ReportParquetProcessorBase, "_execute_trino_sql") def test_schema_exists_cache_value_not_in_cache(self, trino_mock): trino_mock.return_value = True key = build_trino_schema_exists_key(self.account) @@ -177,7 +177,7 @@ def test_schema_exists_cache_value_not_in_cache(self, trino_mock): self.assertTrue(self.processor.schema_exists()) mock_cache_set.assert_called_with(key, True) - @patch.object(ReportParquetProcessorBase, "_execute_sql") + @patch.object(ReportParquetProcessorBase, "_execute_trino_sql") def test_schema_exists_cache_value_not_in_cache_not_exists(self, trino_mock): trino_mock.return_value = False key = build_trino_schema_exists_key(self.account) @@ -185,7 +185,7 @@ def test_schema_exists_cache_value_not_in_cache_not_exists(self, trino_mock): self.assertFalse(self.processor.schema_exists()) mock_cache_set.assert_called_with(key, False) - @patch.object(ReportParquetProcessorBase, "_execute_sql") + @patch.object(ReportParquetProcessorBase, "_execute_trino_sql") def test_table_exists_cache_value_in_cache(self, trino_mock): with patch( "masu.processor.report_parquet_processor_base.get_value_from_cache", @@ -194,7 +194,7 @@ def test_table_exists_cache_value_in_cache(self, trino_mock): self.assertTrue(self.processor.table_exists()) trino_mock.assert_not_called() - @patch.object(ReportParquetProcessorBase, "_execute_sql") + @patch.object(ReportParquetProcessorBase, "_execute_trino_sql") def test_table_exists_cache_value_not_in_cache(self, trino_mock): trino_mock.return_value = True key = build_trino_table_exists_key(self.account, self.table_name) @@ -202,7 +202,7 @@ def test_table_exists_cache_value_not_in_cache(self, trino_mock): self.assertTrue(self.processor.table_exists()) mock_cache_set.assert_called_with(key, True) - @patch.object(ReportParquetProcessorBase, "_execute_sql") + @patch.object(ReportParquetProcessorBase, "_execute_trino_sql") def test_table_exists_cache_value_not_in_cache_not_exists(self, trino_mock): trino_mock.return_value = False key = build_trino_table_exists_key(self.account, self.table_name) @@ -210,7 +210,7 @@ def test_table_exists_cache_value_not_in_cache_not_exists(self, trino_mock): self.assertFalse(self.processor.table_exists()) mock_cache_set.assert_called_with(key, False) - @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_sql") + @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_trino_sql") def test_create_schema(self, mock_execute): """Test that hive partitions are synced.""" expected_log = self.log_output_info + str( diff --git a/koku/masu/util/aws/common.py b/koku/masu/util/aws/common.py index 2030f3d67f..27ece66873 100644 --- a/koku/masu/util/aws/common.py +++ b/koku/masu/util/aws/common.py @@ -511,7 +511,7 @@ def get_bills_from_provider( return bills -def get_s3_resource(access_key, secret_key, region): # pragma: no cover +def get_s3_resource(access_key, secret_key, region, endpoint_url=settings.S3_ENDPOINT): # pragma: no cover """ Obtain the s3 session client """ @@ -521,7 +521,7 @@ def get_s3_resource(access_key, secret_key, region): # pragma: no cover aws_secret_access_key=secret_key, region_name=region, ) - return aws_session.resource("s3", endpoint_url=settings.S3_ENDPOINT, config=config) + return aws_session.resource("s3", endpoint_url=endpoint_url, config=config) def copy_data_to_s3_bucket(request_id, path, filename, data, metadata=None, context=None):