From c32d7fd36650649ad5d26859707d5c1b9e7624bd Mon Sep 17 00:00:00 2001 From: Mohamad Agil Date: Wed, 29 Jan 2025 14:56:13 +0000 Subject: [PATCH 1/2] Convert glue catalog database to athena database --- .../hmpps-audit-dev/resources/athena.tf | 116 +++++------------- 1 file changed, 30 insertions(+), 86 deletions(-) diff --git a/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena.tf b/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena.tf index aa911eff2f0..0b80788a8f1 100644 --- a/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena.tf +++ b/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena.tf @@ -1,6 +1,6 @@ -resource "aws_glue_catalog_database" "audit_database" { - name = "audit_${var.environment-name}" - location_uri = "s3://${module.s3.bucket_name}/" +resource "aws_athena_database" "audit_database" { + name = "audit_${var.environment-name}" + bucket = module.s3.bucket_name } resource "aws_athena_workgroup" "queries" { @@ -16,87 +16,31 @@ resource "aws_athena_workgroup" "queries" { } } -resource "aws_glue_catalog_table" "audit_event_table" { - database_name = aws_glue_catalog_database.audit_database.name - name = "audit_event" - - table_type = "EXTERNAL_TABLE" - - storage_descriptor { - columns { - name = "id" - type = "string" - } - columns { - name = "what" - type = "string" - } - columns { - name = "when" - type = "string" - } - columns { - name = "operationId" - type = "string" - } - columns { - name = "subjectId" - type = "string" - } - columns { - name = "subjectType" - type = "string" - } - columns { - name = "correlationId" - type = "string" - } - columns { - name = "who" - type = "string" - } - columns { - name = "service" - type = "string" - } - columns { - name = "details" - type = "string" - } - - location = "s3://${module.s3.bucket_name}/" - input_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat" - output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" - compressed = true - ser_de_info { - serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" - } - - stored_as_sub_directories = false - } - - partition_keys { - name = "year" - type = "string" - } - - partition_keys { - name = "month" - type = "string" - } - - partition_keys { - name = "day" - type = "string" - } - - partition_keys { - name = "user" - type = "string" - } - - parameters = { - EXTERNAL = "TRUE" - "parquet.compression" = "SNAPPY" - } +resource "aws_athena_named_query" "audit_event_table" { + name = "audit_event_table" + database = aws_athena_database.audit_database.name + + query = < Date: Wed, 29 Jan 2025 15:30:17 +0000 Subject: [PATCH 2/2] fix --- .../hmpps-audit-dev/resources/athena-iam.tf | 8 ++++---- .../hmpps-audit-dev/resources/irsa.tf | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena-iam.tf b/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena-iam.tf index c174a6e67b5..d1d2dbe5c18 100644 --- a/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena-iam.tf +++ b/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena-iam.tf @@ -40,10 +40,10 @@ data "aws_iam_policy_document" "athena" { aws_athena_workgroup.queries.arn, "${aws_athena_workgroup.queries.arn}/*", "arn:aws:glue:eu-west-2:*:catalog", - "arn:aws:glue:eu-west-2:*:database/${aws_glue_catalog_database.audit_database.id}", - "arn:aws:glue:eu-west-2:*:database/${aws_glue_catalog_database.audit_database.id}/*", - "arn:aws:glue:eu-west-2:*:table/${aws_glue_catalog_database.audit_database.id}", - "arn:aws:glue:eu-west-2:*:table/${aws_glue_catalog_database.audit_database.id}/*", + "arn:aws:glue:eu-west-2:*:database/${aws_athena_database.audit_database.id}", + "arn:aws:glue:eu-west-2:*:database/${aws_athena_database.audit_database.id}/*", + "arn:aws:glue:eu-west-2:*:table/${aws_athena_database.audit_database.id}", + "arn:aws:glue:eu-west-2:*:table/${aws_athena_database.audit_database.id}/*", module.s3.bucket_arn, "${module.s3.bucket_arn}/*", diff --git a/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/irsa.tf b/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/irsa.tf index 16c0831d436..54eeab96840 100644 --- a/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/irsa.tf +++ b/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/irsa.tf @@ -83,10 +83,10 @@ data "aws_iam_policy_document" "document" { aws_athena_workgroup.queries.arn, "${aws_athena_workgroup.queries.arn}/*", "arn:aws:glue:eu-west-2:*:catalog", - "arn:aws:glue:eu-west-2:*:database/${aws_glue_catalog_database.audit_database.id}", - "arn:aws:glue:eu-west-2:*:database/${aws_glue_catalog_database.audit_database.id}/*", - "arn:aws:glue:eu-west-2:*:table/${aws_glue_catalog_database.audit_database.id}", - "arn:aws:glue:eu-west-2:*:table/${aws_glue_catalog_database.audit_database.id}/*", + "arn:aws:glue:eu-west-2:*:database/${aws_athena_database.audit_database.id}", + "arn:aws:glue:eu-west-2:*:database/${aws_athena_database.audit_database.id}/*", + "arn:aws:glue:eu-west-2:*:table/${aws_athena_database.audit_database.id}", + "arn:aws:glue:eu-west-2:*:table/${aws_athena_database.audit_database.id}/*", module.s3.bucket_arn, "${module.s3.bucket_arn}/*",