From a62c33d602ac2b02728ae9a8e80b09baf76bf3ef Mon Sep 17 00:00:00 2001 From: Shaung Date: Sat, 17 Sep 2022 00:19:00 -0700 Subject: [PATCH] Metadata.json trigger computing env --- cloud_module/landing_s3_trigger.tf | 22 +++++++- lambda_golang/cmd/landing_metadata/main.go | 7 +-- lambda_golang/cmd/story/main.go | 59 ++++++++++++---------- lambda_golang/go.mod | 18 ++++++- lambda_golang/go.sum | 4 -- lambda_golang/pkg/newssite/utilities.go | 22 +++++--- 6 files changed, 86 insertions(+), 46 deletions(-) diff --git a/cloud_module/landing_s3_trigger.tf b/cloud_module/landing_s3_trigger.tf index ac2303b..22c5bb9 100644 --- a/cloud_module/landing_s3_trigger.tf +++ b/cloud_module/landing_s3_trigger.tf @@ -1,4 +1,4 @@ -resource "aws_lambda_permission" "allow_bucket" { +resource "aws_lambda_permission" "allow_bucket_trigger_by_landing" { statement_id = "AllowExecutionFromS3Bucket" action = "lambda:InvokeFunction" function_name = module.landing_parse_metadata_lambda.lambda_function_arn @@ -6,6 +6,14 @@ resource "aws_lambda_permission" "allow_bucket" { source_arn = data.aws_s3_bucket.archive.arn } +resource "aws_lambda_permission" "allow_bucket_trigger_by_landing_metadata" { + statement_id = "AllowExecutionFromS3Bucket" + action = "lambda:InvokeFunction" + function_name = module.stories_queue_consumer_lambda.lambda_function_arn + principal = "s3.amazonaws.com" + source_arn = data.aws_s3_bucket.archive.arn +} + resource "aws_s3_bucket_notification" "bucket_notification" { bucket = data.aws_s3_bucket.archive.id @@ -16,5 +24,15 @@ resource "aws_s3_bucket_notification" "bucket_notification" { filter_suffix = ".html" } - depends_on = [aws_lambda_permission.allow_bucket] + lambda_function { + lambda_function_arn = module.stories_queue_consumer_lambda.lambda_function_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "${local.newssite_economy_alias}/" + filter_suffix = "/metadata.json" + } + + depends_on = [ + aws_lambda_permission.allow_bucket_trigger_by_landing, + aws_lambda_permission.allow_bucket_trigger_by_landing_metadata + ] } diff --git a/lambda_golang/cmd/landing_metadata/main.go b/lambda_golang/cmd/landing_metadata/main.go index 1ce3d36..24562bc 100644 --- a/lambda_golang/cmd/landing_metadata/main.go +++ b/lambda_golang/cmd/landing_metadata/main.go @@ -4,7 +4,6 @@ package main import ( "context" - "encoding/json" "fmt" "strings" @@ -55,11 +54,7 @@ func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, result := newssite.GetStoriesFromEconomy(landingPageHtmlText) - metadataJSONBytes, metadataJSONStringError := json.Marshal(LandingPageMetadata{Stories: result.Topics, UntitledStories: result.UntitledTopics}) - if metadataJSONStringError != nil { - GoTools.Logger("ERROR", metadataJSONStringError.Error()) - } - metadataJSONString := string(metadataJSONBytes) + metadataJSONString := newssite.AsJson(LandingPageMetadata{Stories: result.Topics, UntitledStories: result.UntitledTopics}) cloud.Archive(cloud.ArchiveArgs{ BodyText: metadataJSONString, diff --git a/lambda_golang/cmd/story/main.go b/lambda_golang/cmd/story/main.go index 682047d..b7220fd 100644 --- a/lambda_golang/cmd/story/main.go +++ b/lambda_golang/cmd/story/main.go @@ -1,17 +1,17 @@ package main import ( + "fmt" + "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-lambda-go/lambda" "context" - "fmt" "math/rand" - "strings" "time" "github.com/rivernews/GoTools" - "github.com/rivernews/media-literacy/pkg/cloud" + "github.com/rivernews/media-literacy/pkg/newssite" ) func main() { @@ -26,35 +26,42 @@ type LambdaResponse struct { // SQS event // refer to https://github.com/aws/aws-lambda-go/blob/v1.26.0/events/README_SQS.md -func HandleRequest(ctx context.Context, event events.SQSEvent) (LambdaResponse, error) { - for _, message := range event.Records { - storyChunk := message.Body - GoTools.Logger("INFO", fmt.Sprintf("Story consumer! story chunk: %s", storyChunk)) +func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse, error) { + GoTools.Logger("INFO", "Fetch stories lambda launched ... triggered by metadata.json creation.") + + for _, record := range S3Event.Records { + + GoTools.Logger("INFO", fmt.Sprintf("S3 event ``` %s ```\n ", newssite.AsJson(record))) + /* + storyChunk := message.Body + GoTools.Logger("INFO", fmt.Sprintf("Story consumer! story chunk: %s", storyChunk)) + + // TODO: fetch and archive for the chunk of storyURLs + storyURLs := strings.Split(storyChunk, " ") - // TODO: fetch and archive for the chunk of storyURLs - storyURLs := strings.Split(storyChunk, " ") + for _, storyURL := range storyURLs { + // TODO: randomized interval + time.Sleep(time.Duration(rand.Intn(5)+5) * time.Second) - for _, storyURL := range storyURLs { - // TODO: randomized interval - time.Sleep(time.Duration(rand.Intn(5)+5) * time.Second) + storyS3Path := fmt.Sprintf("story/%s", storyURL) - storyS3Path := fmt.Sprintf("story/%s", storyURL) + if !cloud.IsDuplicated(storyS3Path) { + GoTools.Logger("INFO", fmt.Sprintf("Archiving story %s", storyURL)) + } else { + GoTools.Logger("DEBUG", fmt.Sprintf("Skip archiving story %s", storyURL)) + } - if !cloud.IsDuplicated(storyS3Path) { - GoTools.Logger("INFO", fmt.Sprintf("Archiving story %s", storyURL)) - } else { - GoTools.Logger("DEBUG", fmt.Sprintf("Skip archiving story %s", storyURL)) + // _, resMessage, err := GoTools.Fetch(GoTools.FetchOption{ + // Method: "GET", + // URL: "https://ipv4bot.whatismyipaddress.com", + // }) + // if err != nil { + // GoTools.Logger("ERROR", err.Error()) + // } + // GoTools.Logger("INFO", fmt.Sprintf("%s-%s ip: %s", storyChunkId, storyURL, resMessage)) } - // _, resMessage, err := GoTools.Fetch(GoTools.FetchOption{ - // Method: "GET", - // URL: "https://ipv4bot.whatismyipaddress.com", - // }) - // if err != nil { - // GoTools.Logger("ERROR", err.Error()) - // } - // GoTools.Logger("INFO", fmt.Sprintf("%s-%s ip: %s", storyChunkId, storyURL, resMessage)) - } + */ } return LambdaResponse{ diff --git a/lambda_golang/go.mod b/lambda_golang/go.mod index 38fab05..09f5563 100644 --- a/lambda_golang/go.mod +++ b/lambda_golang/go.mod @@ -1,6 +1,6 @@ module github.com/rivernews/media-literacy -go 1.16 +go 1.19 require ( github.com/PuerkitoBio/goquery v1.7.1 @@ -9,7 +9,21 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.7.0 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0 - github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0 github.com/rivernews/GoTools v0.1.8 golang.org/x/net v0.0.0-20210825183410-e898025ed96a ) + +require ( + github.com/andybalholm/cascadia v1.2.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect + github.com/aws/smithy-go v1.8.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + golang.org/x/text v0.3.6 // indirect +) diff --git a/lambda_golang/go.sum b/lambda_golang/go.sum index ca4515a..90fef04 100644 --- a/lambda_golang/go.sum +++ b/lambda_golang/go.sum @@ -25,8 +25,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0 h1:B/1pIeV/oFnrOwh github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0/go.mod h1:LKb3cKNQIMh+itGnEpKGcnL/6OIjPZqrtYah1w5f+3o= github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0 h1:nR9j0xMxpXk6orC/C03fbHNrbb1NaXp8LdVV7V1oVLE= github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0/go.mod h1:Qit9H3zjAmF7CLHOkrepE9b2ndX/2l3scstsM5g2jSk= -github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0 h1:BI05Jbkaqp5IDxiobr3B59mX07lfpLJDv5NwAEx3wSs= -github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0/go.mod h1:BXA1CVaEd9TBOQ8G2ke7lMWdVggAeh35+h2HDO50z7s= github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 h1:sHXMIKYS6YiLPzmKSvDpPmOpJDHxmAUgbiF49YNVztg= github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA= github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 h1:1at4e5P+lvHNl2nUktdM2/v+rpICg/QSEr9TO/uW9vU= @@ -47,8 +45,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivernews/GoTools v0.1.7 h1:7NGCw25eRg+gQo9A2h45rXWN+i64RPOoMm9UlvWgxKM= -github.com/rivernews/GoTools v0.1.7/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY= github.com/rivernews/GoTools v0.1.8 h1:niNLFlg0k4W6zIGrlXrdkL/Cy0yoyMTBIMNmQxPLcnc= github.com/rivernews/GoTools v0.1.8/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/lambda_golang/pkg/newssite/utilities.go b/lambda_golang/pkg/newssite/utilities.go index a73d151..11b332f 100644 --- a/lambda_golang/pkg/newssite/utilities.go +++ b/lambda_golang/pkg/newssite/utilities.go @@ -1,14 +1,16 @@ package newssite import ( + "encoding/json" "strings" + "github.com/rivernews/GoTools" ) type NewsSite struct { - Key string - Name string - Alias string + Key string + Name string + Alias string LandingURL string } @@ -18,9 +20,17 @@ func GetNewsSite(envVar string) NewsSite { tokens := strings.Split(ssmValue, ",") return NewsSite{ - Key: tokens[0], - Name: tokens[1], - Alias: tokens[2], + Key: tokens[0], + Name: tokens[1], + Alias: tokens[2], LandingURL: tokens[3], } } + +func AsJson(v any) string { + jsonBytes, err := json.Marshal(v) + if err != nil { + GoTools.Logger("ERROR", err.Error()) + } + return string(jsonBytes) +}