From 5c60040f92222fd0a51427ef8de98b0324a3c745 Mon Sep 17 00:00:00 2001 From: Shaung Date: Thu, 15 Sep 2022 00:53:08 -0700 Subject: [PATCH 1/3] Try S3 notification --- cloud_module/landing_s3_trigger.tf | 20 ++++++++++++++++++++ lambda_golang/cmd/stories/main.go | 28 +++++++++++++++++++--------- 2 files changed, 39 insertions(+), 9 deletions(-) create mode 100644 cloud_module/landing_s3_trigger.tf diff --git a/cloud_module/landing_s3_trigger.tf b/cloud_module/landing_s3_trigger.tf new file mode 100644 index 0000000..2b382cf --- /dev/null +++ b/cloud_module/landing_s3_trigger.tf @@ -0,0 +1,20 @@ +resource "aws_lambda_permission" "allow_bucket" { + statement_id = "AllowExecutionFromS3Bucket" + action = "lambda:InvokeFunction" + function_name = module.batch_stories_fetch_parse_lambda.lambda_function_arn + principal = "s3.amazonaws.com" + source_arn = data.aws_s3_bucket.archive.arn +} + +resource "aws_s3_bucket_notification" "bucket_notification" { + bucket = data.aws_s3_bucket.archive.id + + lambda_function { + lambda_function_arn = module.batch_stories_fetch_parse_lambda.lambda_function_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "media-literacy-archives/" + filter_suffix = ".html" + } + + depends_on = [aws_lambda_permission.allow_bucket] +} diff --git a/lambda_golang/cmd/stories/main.go b/lambda_golang/cmd/stories/main.go index 2aff590..c0b8af8 100644 --- a/lambda_golang/cmd/stories/main.go +++ b/lambda_golang/cmd/stories/main.go @@ -5,19 +5,20 @@ package main import ( "context" "fmt" - "strings" - "math" + //"strings" + //"math" "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-lambda-go/events" // https://github.com/aws/aws-lambda-go/blob/main/events/README_S3.md "github.com/rivernews/GoTools" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/sqs" + //"github.com/aws/aws-sdk-go-v2/aws" + //"github.com/aws/aws-sdk-go-v2/config" + //"github.com/aws/aws-sdk-go-v2/service/sqs" // local packages - "github.com/rivernews/media-literacy/pkg/cloud" - "github.com/rivernews/media-literacy/pkg/newssite" + //"github.com/rivernews/media-literacy/pkg/cloud" + //"github.com/rivernews/media-literacy/pkg/newssite" ) @@ -34,9 +35,16 @@ type LambdaResponse struct { Message string `json:"message:"` } -func HandleRequest(ctx context.Context, stepFunctionInput StepFunctionInput) (LambdaResponse, error) { - GoTools.Logger("INFO", fmt.Sprintf("Batch stories lambda started! Landing page S3 path: `%s`; going to test delayed messages...", stepFunctionInput.LandingS3Key)) +func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, error) { + // GoTools.Logger("INFO", fmt.Sprintf("Batch stories lambda started! Landing page S3 path: `%s`; going to test delayed messages...", stepFunctionInput.LandingS3Key)) + GoTools.SendSlackMessage(fmt.Sprintf("Hello! ``` %v ```", s3Event)) + return LambdaResponse{ + OK: true, + Message: "Done", + }, nil + + /* landingPageHtmlText := cloud.Pull(stepFunctionInput.LandingS3Key) stories := newssite.GetStoriesFromEconomy(landingPageHtmlText) @@ -100,4 +108,6 @@ func HandleRequest(ctx context.Context, stepFunctionInput StepFunctionInput) (La OK: true, Message: fmt.Sprintf("Sent %d messages OK", len(linkChunks)), }, nil + + */ } From d5d8186b3c18d59392067e65227c0d5d0bfcf2ed Mon Sep 17 00:00:00 2001 From: Shaung Date: Thu, 15 Sep 2022 10:47:30 -0700 Subject: [PATCH 2/3] Fix prefix to include newssite alias --- cloud_module/global_ssm.tf | 5 +++++ cloud_module/landing_s3_trigger.tf | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cloud_module/global_ssm.tf b/cloud_module/global_ssm.tf index 9145f08..ce2affb 100644 --- a/cloud_module/global_ssm.tf +++ b/cloud_module/global_ssm.tf @@ -1,3 +1,8 @@ data aws_ssm_parameter newssite_economy { name = "/app/media-literacy/newssites/ECONOMY" } + +locals { + newssite_economy_tokens = split(",", data.aws_ssm_parameter.newssite_economy.value) + newssite_economy_alias = local.newssite_economy_tokens[2] +} diff --git a/cloud_module/landing_s3_trigger.tf b/cloud_module/landing_s3_trigger.tf index 2b382cf..037f8b9 100644 --- a/cloud_module/landing_s3_trigger.tf +++ b/cloud_module/landing_s3_trigger.tf @@ -12,7 +12,7 @@ resource "aws_s3_bucket_notification" "bucket_notification" { lambda_function { lambda_function_arn = module.batch_stories_fetch_parse_lambda.lambda_function_arn events = ["s3:ObjectCreated:*"] - filter_prefix = "media-literacy-archives/" + filter_prefix = "${local.newssite_economy_alias}/" filter_suffix = ".html" } From fa266bf9935a3eba738e9604922468d531505ad9 Mon Sep 17 00:00:00 2001 From: Shaung Date: Thu, 15 Sep 2022 23:18:49 -0700 Subject: [PATCH 3/3] Fix aws lambda PathError issue --- cloud_environments/terraform.sh | 2 +- cloud_module/lambda.tf | 9 ++++++++- cloud_module/stories_sfn.tf | 4 ++-- cloud_module/stories_sqs.tf | 2 +- lambda_golang/cmd/stories/main.go | 1 + 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/cloud_environments/terraform.sh b/cloud_environments/terraform.sh index 2efaada..9a7cea1 100755 --- a/cloud_environments/terraform.sh +++ b/cloud_environments/terraform.sh @@ -16,7 +16,7 @@ TF_VAR_slack_signing_secret=${slack_signing_secret} TF_VAR_slack_post_webhook_url=${slack_post_webhook_url} set +o allexport - +# below is just for testing golang build! The actual build command is in terraform lambda module `command` property if ( cd $GOLANG_SRC_DIR && \ go build ./cmd/landing && \ diff --git a/cloud_module/lambda.tf b/cloud_module/lambda.tf index b54260b..634ac8a 100755 --- a/cloud_module/lambda.tf +++ b/cloud_module/lambda.tf @@ -68,7 +68,7 @@ module "scraper_lambda" { # Based on golang https://github.com/snsinfu/terraform-lambda-example/blob/master/Makefile#L23 source_path = [{ path = "${path.module}/../lambda_golang/" - commands = ["go build ./cmd/landing", ":zip"] + commands = ["${local.go_build_flags} go build ./cmd/landing", ":zip"] patterns = ["landing"] }] @@ -104,3 +104,10 @@ module "scraper_lambda" { Project = local.project_name } } + +locals { + # amd64 is the x86 instruction set + # arm is not (like M1), not supported by AWS lambda go runtime yet + # https://stackoverflow.com/questions/26951940/how-do-i-make-go-get-to-build-against-x86-64-instead-of-i386 + go_build_flags = "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 " +} diff --git a/cloud_module/stories_sfn.tf b/cloud_module/stories_sfn.tf index c700472..15b7aea 100644 --- a/cloud_module/stories_sfn.tf +++ b/cloud_module/stories_sfn.tf @@ -33,11 +33,11 @@ module batch_stories_fetch_parse_lambda { function_name = "${local.project_name}-batch-stories-fetch-parse" description = "Batch fetch and parse all stories of a landing page" handler = "stories" - runtime = "go1.x" + runtime = "go1.x" source_path = [{ path = "${path.module}/../lambda_golang/" - commands = ["go build ./cmd/stories", ":zip"] + commands = ["${local.go_build_flags} go build ./cmd/stories", ":zip"] patterns = ["stories"] }] diff --git a/cloud_module/stories_sqs.tf b/cloud_module/stories_sqs.tf index e99ceb3..f353ad8 100644 --- a/cloud_module/stories_sqs.tf +++ b/cloud_module/stories_sqs.tf @@ -35,7 +35,7 @@ module "stories_queue_consumer_lambda" { runtime = "go1.x" source_path = [{ path = "${path.module}/../lambda_golang/" - commands = ["go build ./cmd/story", ":zip"] + commands = ["${local.go_build_flags} go build ./cmd/story", ":zip"] patterns = ["story"] }] publish = true diff --git a/lambda_golang/cmd/stories/main.go b/lambda_golang/cmd/stories/main.go index c0b8af8..b002eb1 100644 --- a/lambda_golang/cmd/stories/main.go +++ b/lambda_golang/cmd/stories/main.go @@ -37,6 +37,7 @@ type LambdaResponse struct { func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, error) { // GoTools.Logger("INFO", fmt.Sprintf("Batch stories lambda started! Landing page S3 path: `%s`; going to test delayed messages...", stepFunctionInput.LandingS3Key)) + GoTools.SendSlackMessage("Ha ha!") GoTools.SendSlackMessage(fmt.Sprintf("Hello! ``` %v ```", s3Event)) return LambdaResponse{