Skip to content

Commit

Permalink
split into multiple environments; implements #16 except pull from s3
Browse files Browse the repository at this point in the history
  • Loading branch information
rivernews committed Aug 23, 2021
1 parent fadcee5 commit c27ace3
Show file tree
Hide file tree
Showing 36 changed files with 359 additions and 187 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
**credential**
**/builds/**

scraper_lambda/main
scraper_lambda/**/main
venv

# Binaries for programs and plugins
Expand Down
2 changes: 1 addition & 1 deletion cloud_environments/dev/module.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
module main {
source = "../../cloud_src"
source = "../../cloud_module"
environment_name = var.environment_name
project_alias = var.project_alias
slack_signing_secret = var.slack_signing_secret
Expand Down
37 changes: 2 additions & 35 deletions cloud_environments/dev/terraform.sh
Original file line number Diff line number Diff line change
@@ -1,37 +1,4 @@
set -e

ENV=dev

set -o allexport

. ./local.backend.credentials.tfvars
. ./local.credentials
AWS_ACCESS_KEY_ID=${access_key}
AWS_SECRET_ACCESS_KEY=${secret_key}
AWS_DEFAULT_REGION=${region}
TF_VAR_project_alias=media-literacy
TF_VAR_environment_name=${ENV}
TF_VAR_slack_signing_secret=${slack_signing_secret}
TF_VAR_slack_post_webhook_url=${slack_post_webhook_url}
set +o allexport


DEPLOY_DIR=$(git rev-parse --show-toplevel)/cloud_environments/${ENV}
SCRAPER_SRC_DIR=$(git rev-parse --show-toplevel)/scraper_lambda

cd $SCRAPER_SRC_DIR

if go build -o main; then
cd $DEPLOY_DIR

# if deploy the first time, uncomment below
# to avoid "Invalid for_each argument" error
# https://github.com/terraform-aws-modules/terraform-aws-step-functions/issues/20
# terraform "$@" -target=module.main.module.scraper_lambda

terraform "$@"
else
echo "Go build failed"
cd $DEPLOY_DIR
exit 1
fi
REPO_DIR=$(git rev-parse --show-toplevel)
ENV=dev sh "${REPO_DIR}/cloud_environments/terraform.sh" "$@"
2 changes: 1 addition & 1 deletion cloud_environments/production/module.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
module main {
source = "../../cloud_src"
source = "../../cloud_module"
environment_name = var.environment_name
project_alias = var.project_alias
slack_signing_secret = var.slack_signing_secret
Expand Down
34 changes: 3 additions & 31 deletions cloud_environments/production/terraform.sh
Original file line number Diff line number Diff line change
@@ -1,32 +1,4 @@
set -o allexport
set -e

. ./local.backend.credentials.tfvars
. ./local.credentials
AWS_ACCESS_KEY_ID=${access_key}
AWS_SECRET_ACCESS_KEY=${secret_key}
AWS_DEFAULT_REGION=${region}
TF_VAR_project_alias=media-literacy
TF_VAR_environment_name=
TF_VAR_slack_signing_secret=${slack_signing_secret}
TF_VAR_slack_post_webhook_url=${slack_post_webhook_url}
set +o allexport

DEPLOY_DIR=$(git rev-parse --show-toplevel)/cloud_environments/production
SCRAPER_SRC_DIR=$(git rev-parse --show-toplevel)/scraper_lambda

cd $SCRAPER_SRC_DIR

if go build -o main; then
cd $DEPLOY_DIR

# if deploy the first time, uncomment below
# to avoid "Invalid for_each argument" error
# https://github.com/terraform-aws-modules/terraform-aws-step-functions/issues/20
# terraform "$@" -target=module.main.module.scraper_lambda

terraform "$@"
else
echo "Go build failed"
cd $DEPLOY_DIR
exit 1
fi
REPO_DIR=$(git rev-parse --show-toplevel)
ENV= sh "${REPO_DIR}/cloud_environments/terraform.sh" "$@"
41 changes: 41 additions & 0 deletions cloud_environments/terraform.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
set -e

DEPLOY_DIR=$(git rev-parse --show-toplevel)/cloud_environments/${ENV:-production}
SCRAPER_SRC_DIR=$(git rev-parse --show-toplevel)/scraper_lambda

set -o allexport
. ${DEPLOY_DIR}/local.backend.credentials.tfvars
. ${DEPLOY_DIR}/local.credentials
AWS_ACCESS_KEY_ID=${access_key}
AWS_SECRET_ACCESS_KEY=${secret_key}
AWS_DEFAULT_REGION=${region}
TF_VAR_project_alias=media-literacy
TF_VAR_environment_name=${ENV:-}
TF_VAR_slack_signing_secret=${slack_signing_secret}
TF_VAR_slack_post_webhook_url=${slack_post_webhook_url}
set +o allexport


if (
cd $SCRAPER_SRC_DIR/landing && go build -o main && \
cd $SCRAPER_SRC_DIR/stories && go build -o main
); then
cd $DEPLOY_DIR

echo "Go built success"
echo "Launching terraform..."

# if deploy the first time, uncomment below
# to avoid "Invalid for_each argument" error
# https://github.com/terraform-aws-modules/terraform-aws-step-functions/issues/20
# terraform "$@" \
# -target=module.main.module.scraper_lambda \
# -target=module.main.module.batch_stories_fetch_parse_lambda

terraform "$@"
else
cd $DEPLOY_DIR

echo "Go build failed"
exit 1
fi
File renamed without changes.
128 changes: 128 additions & 0 deletions cloud_module/api.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
# Based on
# https://github.com/terraform-aws-modules/terraform-aws-apigateway-v2#http-api-gateway
module "api" {
source = "terraform-aws-modules/apigateway-v2/aws"

name = "${local.project_name}-api-gateway"
description = "HTTP API Gateway of project ${local.project_name}"
protocol_type = "HTTP"

cors_configuration = {
allow_headers = ["content-type"]
allow_methods = ["OPTIONS", "POST", "GET"]
allow_origins = ["*"]
}

# Custom domain
domain_name = local.api_domain_name
# Note that the certificate has to be in same region if using HTTP API
domain_name_certificate_arn = aws_acm_certificate_validation.api.certificate_arn

# Access logs
default_stage_access_log_destination_arn = aws_cloudwatch_log_group.api.arn
default_stage_access_log_format = "$context.identity.sourceIp - - [$context.requestTime] \"$context.httpMethod $context.routeKey $context.protocol\" $context.status $context.responseLength $context.requestId $context.integrationErrorMessage"

# Routes and integrations
integrations = {
"POST /slack/command" = {
lambda_arn = module.slack_command_lambda.lambda_function_arn
payload_format_version = "2.0"
timeout_milliseconds = 29000
}
}

default_route_settings = {
detailed_metrics_enabled = true
throttling_burst_limit = 5
throttling_rate_limit = 10
logging_level = "INFO"
}

tags = {
Project = local.project_name
}
}

resource "aws_cloudwatch_log_group" "api" {
name = "/aws/api/${local.project_name}"
retention_in_days = 7
}

module "slack_command_lambda" {
source = "terraform-aws-modules/lambda/aws"

create_function = true
function_name = "${local.project_name}-slack-command-lambda"
description = "Lambda function for slack command for environment ${local.project_name}"
handler = "slack_command_controller.lambda_handler"
runtime = "python3.8"
source_path = "${path.module}/../lambda/src/slack_command_controller.py"

layers = [
module.lambda_layer.lambda_layer_arn
]

# Maximum lambda execution time - 15m
timeout = 20
cloudwatch_logs_retention_in_days = 7

# Enable publish to create versions for lambda;
# otherwise will use $LATEST instead and will cause trouble creating permission for allowing API Gateway invocation:
# `We currently do not support adding policies for $LATEST.`
publish = true
allowed_triggers = {
APIGatewayAny = {
service = "apigateway"
source_arn = "${module.api.apigatewayv2_api_execution_arn}/*/POST/slack/command"
}
}

attach_policy_statements = true
policy_statements = {
pipeline_sqs = {
effect = "Allow",
actions = ["sqs:SendMessage", "sqs:GetQueueUrl"],
resources = [module.pipeline_queue.this_sqs_queue_arn]
}
s3_archive_bucket = {
effect = "Allow",
actions = [
"s3:ListBucket",
],
resources = ["${data.aws_s3_bucket.archive.arn}"]
}
}

# allow lambda to invoke step function
attach_policy_json = true
policy_json = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"states:StartExecution"
],
"Resource": ["${module.batch_stories_sfn.state_machine_arn}"]
}
]
}
EOF

environment_variables = {
SLACK_SIGNING_SECRET = var.slack_signing_secret
SLACK_POST_WEBHOOK_URL = var.slack_post_webhook_url

PIPELINE_QUEUE_NAME = module.pipeline_queue.this_sqs_queue_name
BATCH_STORIES_SFN_ARN = module.batch_stories_sfn.state_machine_arn

LOGLEVEL = "DEBUG"

S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
}

tags = {
Project = local.project_name
}
}
File renamed without changes.
File renamed without changes.
File renamed without changes.
56 changes: 2 additions & 54 deletions cloud_src/lambda.tf → cloud_module/lambda.tf
Original file line number Diff line number Diff line change
@@ -1,58 +1,6 @@
# Based on
# https://github.com/terraform-aws-modules/terraform-aws-lambda/blob/master/examples/build-package/main.tf

module "slack_command_lambda" {
source = "terraform-aws-modules/lambda/aws"

create_function = true
function_name = "${local.project_name}-slack-command-lambda"
description = "Lambda function for slack command for environment ${local.project_name}"
handler = "slack_command_controller.lambda_handler"
runtime = "python3.8"
source_path = "${path.module}/../lambda/src/slack_command_controller.py"

layers = [
module.lambda_layer.lambda_layer_arn
]

# Maximum lambda execution time - 15m
timeout = 20
cloudwatch_logs_retention_in_days = 7

# Enable publish to create versions for lambda;
# otherwise will use $LATEST instead and will cause trouble creating permission for allowing API Gateway invocation:
# `We currently do not support adding policies for $LATEST.`
publish = true
allowed_triggers = {
APIGatewayAny = {
service = "apigateway"
source_arn = "${module.api.apigatewayv2_api_execution_arn}/*/POST/slack/command"
}
}

attach_policy_statements = true
policy_statements = {
pipeline_sqs = {
effect = "Allow",
actions = ["sqs:SendMessage", "sqs:GetQueueUrl"],
resources = [module.pipeline_queue.this_sqs_queue_arn]
}
}

environment_variables = {
SLACK_SIGNING_SECRET = var.slack_signing_secret
SLACK_POST_WEBHOOK_URL = var.slack_post_webhook_url
PIPELINE_QUEUE_NAME = module.pipeline_queue.this_sqs_queue_name
LOGLEVEL = "DEBUG"

S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
}

tags = {
Project = local.project_name
}
}

module "lambda_layer" {
source = "terraform-aws-modules/lambda/aws"

Expand Down Expand Up @@ -83,7 +31,7 @@ module "step_function" {
name = "${local.project_name}-step-function"

# TODO: change to yaml
definition = templatefile("${path.module}/state_machine_definition.json", {
definition = templatefile("${path.module}/sfn_def/state_machine_definition.json", {
SCRAPER_LAMBDA_ARN = module.scraper_lambda.lambda_function_arn
})

Expand Down Expand Up @@ -119,7 +67,7 @@ module "scraper_lambda" {
# Based on tf https://github.com/terraform-aws-modules/terraform-aws-lambda/blob/master/examples/build-package/main.tf#L111
# Based on golang https://github.com/snsinfu/terraform-lambda-example/blob/master/Makefile#L23
source_path = [{
path = "${path.module}/../scraper_lambda/"
path = "${path.module}/../scraper_lambda/landing"
commands = ["go build -o main", ":zip"]
patterns = ["main"]
}]
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
13 changes: 13 additions & 0 deletions cloud_module/sfn_def/batch_stories_def.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"Comment": "Batch processing stories for a landing page",
"StartAt": "BatchStoriesFetchParse",
"States": {
"BatchStoriesFetchParse":{
"Comment": "Fetch and parse all stories",
"Type":"Task",
"Resource":"${BATCH_STORIES_FETCH_PARSE_LAMBDA_ARN}",
"Parameters":{},
"End":true
}
}
}
File renamed without changes.
File renamed without changes.
Loading

0 comments on commit c27ace3

Please sign in to comment.