New Relic Table Check (#4192)

* Add check_tables command * Add check-tables workflow * Fix naming * fix command * Check Tables post deploy * Change file name * Prep v0.1.6 version change * Prep schedule for checking tables * Test v0.1.6 * Add check tables to dashboard * Add post deploy table check * Setup check table workflow * setup check table scheduler * Fix version reference * Add widget for Submissions within a backup window Testing dual query on the widget * Change backup log capture * Remove unnecessary columns * add row_count to check_tables operation * Version bump to v0.1.7 * Adds RowCount() prior to backup on the schedule * Version bump to v0.1.8 * Add widgets for Row Count
GSA-TTS · Oct 16, 2024 · 6ba74a0 · 6ba74a0
1 parent 5674d3e
commit 6ba74a0
Show file tree

Hide file tree

Showing 8 changed files with 216 additions and 5 deletions.
diff --git a/.github/workflows/deploy-application.yml b/.github/workflows/deploy-application.yml
@@ -126,3 +126,11 @@ jobs:
           cf_space: ${{ env.space }}
           command: cf restart logshipper
 
+  check-tables:
+    needs: [push-with-creds]
+    uses: ./.github/workflows/fac-check-tables.yml
+    secrets: inherit
+    with:
+      environment: ${{ inputs.environment }}
+      util_version: "v0.1.8"
+      backup_operation: "check_tables"
diff --git a/.github/workflows/fac-backup-scheduler.yml b/.github/workflows/fac-backup-scheduler.yml
@@ -19,6 +19,6 @@ jobs:
     secrets: inherit
     with:
       environment: ${{ matrix.environment.name }}
-      util_version: "v0.1.5"
+      util_version: "v0.1.8"
       backup_operation: "scheduled_backup"
 
diff --git a/.github/workflows/fac-backup-util-scheduled.yml b/.github/workflows/fac-backup-util-scheduled.yml
@@ -1,8 +1,8 @@
 ---
 name: Backup the database with fac-backup-utility
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.5 scheduled_backup
-# ./fac-backup-util.sh v0.1.5 daily_backup
+# ./fac-backup-util.sh v0.1.8 scheduled_backup
+# ./fac-backup-util.sh v0.1.8 daily_backup
 on:
   workflow_call:
     inputs:

diff --git a/.github/workflows/fac-backup-util.yml b/.github/workflows/fac-backup-util.yml
@@ -1,8 +1,8 @@
 ---
 name: Backup the database with fac-backup-utility
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.5 initial_backup
-# ./fac-backup-util.sh v0.1.5 deploy_backup
+# ./fac-backup-util.sh v0.1.8 initial_backup
+# ./fac-backup-util.sh v0.1.8 deploy_backup
 on:
   workflow_dispatch:
     inputs:

diff --git a/.github/workflows/fac-check-tables-scheduler.yml b/.github/workflows/fac-check-tables-scheduler.yml
@@ -0,0 +1,24 @@
+---
+name: Check Tables Daily
+on:
+  schedule:
+    # Invoke every 12 hours
+    - cron: '0 */12 * * *'
+  workflow_dispatch: null
+
+jobs:
+  check-tables:
+    strategy:
+      fail-fast: false
+      matrix:
+        environment:
+          - name: dev
+          - name: staging
+          - name: production
+          - name: preview
+    uses: ./.github/workflows/fac-check-tables.yml
+    secrets: inherit
+    with:
+      environment: ${{ matrix.environment.name }}
+      util_version: "v0.1.8"
+      backup_operation: "check_tables"
diff --git a/.github/workflows/fac-check-tables.yml b/.github/workflows/fac-check-tables.yml
@@ -0,0 +1,54 @@
+---
+name: Check existing tables in an environment
+### Common Commands:
+# ./fac-backup-util.sh v0.1.8 check_tables
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        required: true
+        type: choice
+        options:
+          - 'dev'
+          - 'preview'
+          - 'staging'
+          - 'production'
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: choice
+        options:
+          - 'check_tables'
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: string
+jobs:
+  fac-check-tables:
+    name: Check tables in FAC Database
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+    steps:
+      - name: Check tables in ${{ env.space }}
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 2G -m 3G --name check_tables --command "./fac-backup-util.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}"
diff --git a/backend/fac-backup-util.sh b/backend/fac-backup-util.sh
@@ -30,6 +30,12 @@ RDSToS3Dump() {
 RDSToRDS() {
     ./gov.gsa.fac.cgov-util db_to_db --src_db "$1" --dest_db "$2" --operation "$3"
 }
+CheckTables() {
+    ./gov.gsa.fac.cgov-util check_db --db "$1"
+}
+RowCount() {
+    ./gov.gsa.fac.cgov-util row_count --db "$1"
+}
 
 if [ "$run_option" == "initial_backup" ]; then
     GetUtil
@@ -53,6 +59,8 @@ elif [ "$run_option" == "scheduled_backup" ]; then
     GetUtil
     InstallAWS
     gonogo "install_aws"
+    RowCount "$db_name"
+    gonogo "row_count"
     RDSToS3Dump "$db_name" "$backup_s3_name" "scheduled/$scheduled_date"
     gonogo "db_to_s3"
     AWSS3Sync "$s3_name" "$backup_s3_name"
@@ -71,4 +79,12 @@ elif [ "$run_option" == "media_sync" ]; then
     gonogo "install_aws"
     AWSS3Sync "$s3_name" "$backup_s3_name"
     gonogo "s3_sync"
+elif [ "$run_option" == "check_tables" ]; then
+    GetUtil
+    InstallAWS
+    gonogo "install_aws"
+    CheckTables "$db_name"
+    gonogo "check_tables"
+    RowCount "$db_name"
+    gonogo "row_count"
 fi
diff --git a/terraform/shared/modules/newrelic/logreview.tf b/terraform/shared/modules/newrelic/logreview.tf
@@ -222,5 +222,114 @@ resource "newrelic_one_dashboard" "log_review_dashboard" {
 
       legend_enabled = true
     }
+
+    widget_log_table {
+      title = "${var.cf_space_name} Check Tables Logs"
+
+      row    = 8
+      column = 1
+      width  = 8
+      height = 3
+
+      nrql_query {
+        query = "SELECT `message` FROM Log WHERE `tags.space_name` = '${var.cf_space_name}' AND allColumnSearch('CHECKTABLESPASS', insensitive: true) SINCE 7 days ago"
+      }
+
+      legend_enabled = true
+    }
+
+    widget_billboard {
+      title = "${var.cf_space_name} Table Check Count - Pass"
+
+      row    = 8
+      column = 9
+      width  = 3
+      height = 3
+
+      nrql_query {
+        query = "SELECT count(*) FROM Log WHERE `tags.space_name` = '${var.cf_space_name}' AND allColumnSearch('CHECKTABLESPASS', insensitive: true) SINCE 7 days ago"
+      }
+
+      legend_enabled = true
+    }
+
+    widget_log_table {
+      title = "${var.cf_space_name} Missing Tables Logs"
+
+      row    = 9
+      column = 1
+      width  = 8
+      height = 3
+
+      nrql_query {
+        query = "SELECT `message` FROM Log WHERE `tags.space_name` = '${var.cf_space_name}' AND allColumnSearch('DBMISSINGTABLES', insensitive: true) SINCE 7 days ago"
+      }
+
+      legend_enabled = true
+    }
+
+    widget_billboard {
+      title = "${var.cf_space_name} Table Check Count - Fail"
+
+      row    = 9
+      column = 9
+      width  = 3
+      height = 3
+
+      nrql_query {
+        query = "SELECT count(*) FROM Log WHERE `tags.space_name` = '${var.cf_space_name}' AND allColumnSearch('DBMISSINGTABLES', insensitive: true) SINCE 7 days ago"
+      }
+
+      legend_enabled = true
+    }
+
+    widget_table {
+      title = "${var.cf_space_name} Backups and Submissions"
+
+      row    = 10
+      column = 1
+      width  = 8
+      height = 3
+
+      nrql_query {
+        query = "SELECT `message` FROM Log WHERE allColumnSearch('POST', insensitive: true) AND allColumnSearch('/submission/', insensitive: true) AND `newrelic.source` = 'logs.APM' AND entity.name ='${var.cf_space_name}' SINCE 2 hours ago"
+      }
+      nrql_query {
+        query = "SELECT `message`,`timestamp` FROM Log WHERE allColumnSearch('STARTUP_CHECK', insensitive: true) AND `message` LIKE '%db_to_s3%' AND `message` LIKE '%PASS%' AND tags.space_name ='${var.cf_space_name}' SINCE 2 hours ago"
+      }
+
+      legend_enabled = true
+    }
+    widget_table {
+      title = "${var.cf_space_name} Backup and Row Count"
+
+      row    = 11
+      column = 1
+      width  = 8
+      height = 3
+
+      nrql_query {
+        query = "SELECT `message` FROM Log WHERE `tags.space_name` = '${var.cf_space_name}' AND allColumnSearch('\"TABLEROWCOUNT\"', insensitive: true) SINCE 2 hours ago"
+      }
+      nrql_query {
+        query = "SELECT `message`,`timestamp` FROM Log WHERE allColumnSearch('STARTUP_CHECK', insensitive: true) AND `message` LIKE '%db_to_s3%' AND `message` LIKE '%PASS%' AND tags.space_name ='${var.cf_space_name}' SINCE 2 hours ago"
+      }
+
+      legend_enabled = true
+    }
+    widget_table {
+      title = "${var.cf_space_name} Row Count"
+
+      row    = 12
+      column = 1
+      width  = 8
+      height = 3
+
+      nrql_query {
+        query = "SELECT `message` FROM Log WHERE `tags.space_name` = '${var.cf_space_name}' AND allColumnSearch('\"TABLEROWCOUNT\"', insensitive: true) SINCE 7 days ago"
+      }
+
+      legend_enabled = true
+    }
   }
 }