Merge pull request #29207 from ministryofjustice/fix-audit-dev-permis…

…sions Convert glue catalog database to athena database for audit-dev
ministryofjustice · Jan 29, 2025 · 9915d7d · 9915d7d
2 parents f8d287f + 69e81bd
commit 9915d7d
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 94 deletions.
diff --git a/...spaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena-iam.tf b/...spaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena-iam.tf
@@ -40,10 +40,10 @@ data "aws_iam_policy_document" "athena" {
       aws_athena_workgroup.queries.arn,
       "${aws_athena_workgroup.queries.arn}/*",
       "arn:aws:glue:eu-west-2:*:catalog",
-      "arn:aws:glue:eu-west-2:*:database/${aws_glue_catalog_database.audit_database.id}",
-      "arn:aws:glue:eu-west-2:*:database/${aws_glue_catalog_database.audit_database.id}/*",
-      "arn:aws:glue:eu-west-2:*:table/${aws_glue_catalog_database.audit_database.id}",
-      "arn:aws:glue:eu-west-2:*:table/${aws_glue_catalog_database.audit_database.id}/*",
+      "arn:aws:glue:eu-west-2:*:database/${aws_athena_database.audit_database.id}",
+      "arn:aws:glue:eu-west-2:*:database/${aws_athena_database.audit_database.id}/*",
+      "arn:aws:glue:eu-west-2:*:table/${aws_athena_database.audit_database.id}",
+      "arn:aws:glue:eu-west-2:*:table/${aws_athena_database.audit_database.id}/*",
       module.s3.bucket_arn,
       "${module.s3.bucket_arn}/*",
 

diff --git a/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena.tf b/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/athena.tf
@@ -1,6 +1,6 @@
-resource "aws_glue_catalog_database" "audit_database" {
-  name = "audit_${var.environment-name}"
-  location_uri = "s3://${module.s3.bucket_name}/"
+resource "aws_athena_database" "audit_database" {
+  name   = "audit_${var.environment-name}"
+  bucket = module.s3.bucket_name
 }
 
 resource "aws_athena_workgroup" "queries" {
@@ -16,87 +16,31 @@ resource "aws_athena_workgroup" "queries" {
   }
 }
 
-resource "aws_glue_catalog_table" "audit_event_table" {
-  database_name = aws_glue_catalog_database.audit_database.name
-  name          = "audit_event"
-
-  table_type = "EXTERNAL_TABLE"
-
-  storage_descriptor {
-    columns {
-      name = "id"
-      type = "string"
-    }
-    columns {
-      name = "what"
-      type = "string"
-    }
-    columns {
-      name = "when"
-      type = "string"
-    }
-    columns {
-      name = "operationId"
-      type = "string"
-    }
-    columns {
-      name = "subjectId"
-      type = "string"
-    }
-    columns {
-      name = "subjectType"
-      type = "string"
-    }
-    columns {
-      name = "correlationId"
-      type = "string"
-    }
-    columns {
-      name = "who"
-      type = "string"
-    }
-    columns {
-      name = "service"
-      type = "string"
-    }
-    columns {
-      name = "details"
-      type = "string"
-    }
-
-    location      = "s3://${module.s3.bucket_name}/"
-    input_format  = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"
-    output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"
-    compressed    = true
-    ser_de_info {
-      serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
-    }
-
-    stored_as_sub_directories = false
-  }
-
-  partition_keys {
-    name = "year"
-    type = "string"
-  }
-
-  partition_keys {
-    name = "month"
-    type = "string"
-  }
-
-  partition_keys {
-    name = "day"
-    type = "string"
-  }
-
-  partition_keys {
-    name = "user"
-    type = "string"
-  }
-
-  parameters = {
-    EXTERNAL              = "TRUE"
-    "parquet.compression" = "SNAPPY"
-  }
+resource "aws_athena_named_query" "audit_event_table" {
+  name        = "audit_event_table"
+  database    = aws_athena_database.audit_database.name
+
+  query = <<EOT
+    CREATE EXTERNAL TABLE IF NOT EXISTS audit_event (
+      id STRING,
+      what STRING,
+      `when` STRING,
+      operationId STRING,
+      subjectId STRING,
+      subjectType STRING,
+      correlationId STRING,
+      who STRING,
+      service STRING,
+      details STRING
+    )
+    PARTITIONED BY (
+      year STRING,
+      month STRING,
+      day STRING,
+      user STRING
+    )
+    STORED AS PARQUET
+    LOCATION 's3://${module.s3.bucket_name}/'
+    TBLPROPERTIES ("parquet.compression"="SNAPPY");
+  EOT
 }
diff --git a/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/irsa.tf b/namespaces/live.cloud-platform.service.justice.gov.uk/hmpps-audit-dev/resources/irsa.tf
@@ -83,10 +83,10 @@ data "aws_iam_policy_document" "document" {
       aws_athena_workgroup.queries.arn,
       "${aws_athena_workgroup.queries.arn}/*",
       "arn:aws:glue:eu-west-2:*:catalog",
-      "arn:aws:glue:eu-west-2:*:database/${aws_glue_catalog_database.audit_database.id}",
-      "arn:aws:glue:eu-west-2:*:database/${aws_glue_catalog_database.audit_database.id}/*",
-      "arn:aws:glue:eu-west-2:*:table/${aws_glue_catalog_database.audit_database.id}",
-      "arn:aws:glue:eu-west-2:*:table/${aws_glue_catalog_database.audit_database.id}/*",
+      "arn:aws:glue:eu-west-2:*:database/${aws_athena_database.audit_database.id}",
+      "arn:aws:glue:eu-west-2:*:database/${aws_athena_database.audit_database.id}/*",
+      "arn:aws:glue:eu-west-2:*:table/${aws_athena_database.audit_database.id}",
+      "arn:aws:glue:eu-west-2:*:table/${aws_athena_database.audit_database.id}/*",
       module.s3.bucket_arn,
       "${module.s3.bucket_arn}/*",