Concd 199 feature branch wait dataload (#1997)

* add wait condition to data load host

* Remove psql statements that require the fargate-feautrebranch to have already run migrations against the database

* Switching to sleep and adding back psql commands assuming migrations has already by fargate app task launch

* CONCD-199 add github workflow badging

* CONCD-199 update ami

* remove problematic install

* add configuration to ignore cfn warning W2001 for a UserData completion signal in the featurebranch nested clouformation stacks

* incrememnt EngineVersion. AWS would no longer launch 12.8

* CONCD-199 Add DataLoadHost completion signal using cfn signal helper scripts, cfn output and input parameter to featurebranch cloudformation nested stack. This is to allow UserData in DataLoadHost to complete before launching the nested FargateCluster stack

.cfnlintrc.yaml

@@ -0,0 +1,5 @@
+# The W2001 check is used to ignore the featurebranch.yaml DataLoadStackName parameter in the nested
+# stack fargate-featurebranch.yaml used to signal when the DataLoadHost UserData commands are complete.
+# Check if Parameters are Used
+ignore_checks:
+    - W2001

.gitignore

@@ -13,6 +13,7 @@ deploystatus.log
 artifacts/
 /.*
 !.gitignore
+!.cfnlintrc.yaml
 !.github
 !.dockerignore
 .DS_Store

README.md

@@ -1,4 +1,6 @@
-[![Build Status](https://travis-ci.org/LibraryOfCongress/concordia.svg?branch=main)](https://travis-ci.org/LibraryOfCongress/concordia)
+[![Lint](https://github.com/LibraryOfCongress/concordia/actions/workflows/black.yml/badge.svg)](https://github.com/LibraryOfCongress/concordia/actions/workflows/black.yml)
+[![Test](https://github.com/LibraryOfCongress/concordia/actions/workflows/test.yml/badge.svg)](https://github.com/LibraryOfCongress/concordia/actions/workflows/test.yml)
+[![Build](https://github.com/LibraryOfCongress/concordia/actions/workflows/build.yml/badge.svg)](https://github.com/LibraryOfCongress/concordia/actions/workflows/build.yml)
 [![Coverage Status](https://coveralls.io/repos/github/LibraryOfCongress/concordia/badge.svg?branch=main)](https://coveralls.io/github/LibraryOfCongress/concordia?branch=main)
 
 # Welcome to Concordia

cloudformation/featurebranch.yaml

@@ -70,3 +70,4 @@ Resources:
                 MemcachedPort: !GetAtt ElastiCache.Outputs.MemcachedPort
                 DatabaseEndpoint: !GetAtt RDS.Outputs.DatabaseHostName
                 Priority: !Ref Priority
+                DataLoadStackName: !GetAtt DataLoadHost.Outputs.StackName

cloudformation/infrastructure/data-load.yaml

@@ -24,7 +24,7 @@ Parameters:
 Mappings:
     AWSRegionToAMI:
         us-east-1:
-            AMI: ami-0cabc39acf991f4f1
+            AMI: ami-0806bc468ce3a22ec
 
     EnvironmentMapping:
         IamInstanceProfileName:
@@ -50,6 +50,9 @@ Mappings:
 Resources:
     DataLoadHost:
         Type: AWS::EC2::Instance
+        CreationPolicy:
+            ResourceSignal:
+                Timeout: PT30M
         Properties:
             ImageId:
                 Fn::FindInMap:
@@ -79,18 +82,25 @@ Resources:
             UserData:
                 Fn::Base64: !Sub |
                     #!/bin/bash -xe
+                    trap '/opt/aws/bin/cfn-signal --exit-code 1 --resource DataLoadHost --region ${AWS::Region} --stack ${AWS::StackName}' ERR
                     echo "Running userdata for ${EnvironmentName}"
                     echo "export ENV_NAME=${EnvironmentName}" >> /home/ec2-user/.bash_profile
                     source /home/ec2-user/.bash_profile
-                    sudo yum -y update
-                    sudo amazon-linux-extras install -y postgresql12
+                    yum -y update
+                    amazon-linux-extras install -y postgresql12
                     aws s3 cp s3://crowd-deployment/database-dumps/concordia.latest.dmp concordia.dmp
                     echo "${PostgresqlHost}:5432:*:concordia:${PostgresqlPassword}" >> /root/.pgpass
                     chmod 0600 /root/.pgpass
-                    psql -U concordia -h "$POSTGRESQL_HOST" -d postgres -c "select pg_terminate_backend(pid) from pg_stat_activity where datname='concordia';"
-                    psql -U concordia -h "$POSTGRESQL_HOST" -d postgres -c "drop database concordia;"
+                    psql -U concordia -h ${PostgresqlHost} -d postgres -c "select pg_terminate_backend(pid) from pg_stat_activity where datname='concordia';"
+                    psql -U concordia -h ${PostgresqlHost} -d postgres -c "drop database concordia;"
                     pg_restore --create -Fc -U concordia -h ${PostgresqlHost} --dbname=postgres --no-password --no-owner --no-acl concordia.dmp
+                    # Signal the status from cfn-init
+                    /opt/aws/bin/cfn-signal --exit-code 0 --resource DataLoadHost --region ${AWS::Region} --stack ${AWS::StackName}
                     shutdown -h now
             Tags:
                 - Key: Name
                   Value: !Sub ${EnvironmentName}-DataLoadHost
+Outputs:
+    StackName:
+        Description: 'Stackname for the DataLoadHost'
+        Value: !Ref AWS::StackName

cloudformation/infrastructure/fargate-featurebranch.yaml

@@ -71,6 +71,10 @@ Parameters:
         Description: Priority of the subdomain listener rule, must be unique in the set of listener rules
         Default: 100
 
+    DataLoadStackName:
+        Type: String
+        Description: Signal that the DataLoadHost UserData has completed
+
 Resources:
     ConcordiaAppLogsGroup:
         Type: AWS::Logs::LogGroup

cloudformation/infrastructure/rds.yaml

@@ -49,7 +49,7 @@ Resources:
             PreferredMaintenanceWindow: sun:07:14-sun:07:44
             DBName: concordia
             Engine: postgres
-            EngineVersion: '12.8'
+            EngineVersion: '12.11'
             LicenseModel: postgresql-license
             DBSubnetGroupName:
                 Ref: PostgresSubnetGroup