Metadata.json trigger computing env

rivernews · Sep 17, 2022 · a62c33d · a62c33d
1 parent 9eca283
commit a62c33d
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 46 deletions.
diff --git a/cloud_module/landing_s3_trigger.tf b/cloud_module/landing_s3_trigger.tf
@@ -1,11 +1,19 @@
-resource "aws_lambda_permission" "allow_bucket" {
+resource "aws_lambda_permission" "allow_bucket_trigger_by_landing" {
   statement_id  = "AllowExecutionFromS3Bucket"
   action        = "lambda:InvokeFunction"
   function_name = module.landing_parse_metadata_lambda.lambda_function_arn
   principal     = "s3.amazonaws.com"
   source_arn    = data.aws_s3_bucket.archive.arn
 }
 
+resource "aws_lambda_permission" "allow_bucket_trigger_by_landing_metadata" {
+  statement_id  = "AllowExecutionFromS3Bucket"
+  action        = "lambda:InvokeFunction"
+  function_name = module.stories_queue_consumer_lambda.lambda_function_arn
+  principal     = "s3.amazonaws.com"
+  source_arn    = data.aws_s3_bucket.archive.arn
+}
+
 resource "aws_s3_bucket_notification" "bucket_notification" {
   bucket = data.aws_s3_bucket.archive.id
 
@@ -16,5 +24,15 @@ resource "aws_s3_bucket_notification" "bucket_notification" {
     filter_suffix       = ".html"
   }
 
-  depends_on = [aws_lambda_permission.allow_bucket]
+  lambda_function {
+    lambda_function_arn = module.stories_queue_consumer_lambda.lambda_function_arn
+    events              = ["s3:ObjectCreated:*"]
+    filter_prefix       = "${local.newssite_economy_alias}/"
+    filter_suffix       = "/metadata.json"
+  }
+
+  depends_on = [
+    aws_lambda_permission.allow_bucket_trigger_by_landing,
+    aws_lambda_permission.allow_bucket_trigger_by_landing_metadata
+  ]
 }
diff --git a/lambda_golang/cmd/landing_metadata/main.go b/lambda_golang/cmd/landing_metadata/main.go
@@ -4,7 +4,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -55,11 +54,7 @@ func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse,
 
 		result := newssite.GetStoriesFromEconomy(landingPageHtmlText)
 
-		metadataJSONBytes, metadataJSONStringError := json.Marshal(LandingPageMetadata{Stories: result.Topics, UntitledStories: result.UntitledTopics})
-		if metadataJSONStringError != nil {
-			GoTools.Logger("ERROR", metadataJSONStringError.Error())
-		}
-		metadataJSONString := string(metadataJSONBytes)
+		metadataJSONString := newssite.AsJson(LandingPageMetadata{Stories: result.Topics, UntitledStories: result.UntitledTopics})
 
 		cloud.Archive(cloud.ArchiveArgs{
 			BodyText:          metadataJSONString,

diff --git a/lambda_golang/cmd/story/main.go b/lambda_golang/cmd/story/main.go
@@ -1,17 +1,17 @@
 package main
 
 import (
+	"fmt"
+
 	"github.com/aws/aws-lambda-go/events"
 	"github.com/aws/aws-lambda-go/lambda"
 
 	"context"
-	"fmt"
 	"math/rand"
-	"strings"
 	"time"
 
 	"github.com/rivernews/GoTools"
-	"github.com/rivernews/media-literacy/pkg/cloud"
+	"github.com/rivernews/media-literacy/pkg/newssite"
 )
 
 func main() {
@@ -26,35 +26,42 @@ type LambdaResponse struct {
 
 // SQS event
 // refer to https://github.com/aws/aws-lambda-go/blob/v1.26.0/events/README_SQS.md
-func HandleRequest(ctx context.Context, event events.SQSEvent) (LambdaResponse, error) {
-	for _, message := range event.Records {
-		storyChunk := message.Body
-		GoTools.Logger("INFO", fmt.Sprintf("Story consumer! story chunk: %s", storyChunk))
+func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse, error) {
+	GoTools.Logger("INFO", "Fetch stories lambda launched ... triggered by metadata.json creation.")
+
+	for _, record := range S3Event.Records {
+
+		GoTools.Logger("INFO", fmt.Sprintf("S3 event ``` %s ```\n ", newssite.AsJson(record)))
+		/*
+			storyChunk := message.Body
+			GoTools.Logger("INFO", fmt.Sprintf("Story consumer! story chunk: %s", storyChunk))
+
+			// TODO: fetch and archive for the chunk of storyURLs
+			storyURLs := strings.Split(storyChunk, " ")
 
-		// TODO: fetch and archive for the chunk of storyURLs
-		storyURLs := strings.Split(storyChunk, " ")
+			for _, storyURL := range storyURLs {
+				// TODO: randomized interval
+				time.Sleep(time.Duration(rand.Intn(5)+5) * time.Second)
 
-		for _, storyURL := range storyURLs {
-			// TODO: randomized interval
-			time.Sleep(time.Duration(rand.Intn(5)+5) * time.Second)
+				storyS3Path := fmt.Sprintf("story/%s", storyURL)
 
-			storyS3Path := fmt.Sprintf("story/%s", storyURL)
+				if !cloud.IsDuplicated(storyS3Path) {
+					GoTools.Logger("INFO", fmt.Sprintf("Archiving story %s", storyURL))
+				} else {
+					GoTools.Logger("DEBUG", fmt.Sprintf("Skip archiving story %s", storyURL))
+				}
 
-			if !cloud.IsDuplicated(storyS3Path) {
-				GoTools.Logger("INFO", fmt.Sprintf("Archiving story %s", storyURL))
-			} else {
-				GoTools.Logger("DEBUG", fmt.Sprintf("Skip archiving story %s", storyURL))
+				// _, resMessage, err := GoTools.Fetch(GoTools.FetchOption{
+				// 	Method: "GET",
+				// 	URL: "https://ipv4bot.whatismyipaddress.com",
+				// })
+				// if err != nil {
+				// 	GoTools.Logger("ERROR", err.Error())
+				// }
+				// GoTools.Logger("INFO", fmt.Sprintf("%s-%s ip: %s", storyChunkId, storyURL, resMessage))
 			}
 
-			// _, resMessage, err := GoTools.Fetch(GoTools.FetchOption{
-			// 	Method: "GET",
-			// 	URL: "https://ipv4bot.whatismyipaddress.com",
-			// })
-			// if err != nil {
-			// 	GoTools.Logger("ERROR", err.Error())
-			// }
-			// GoTools.Logger("INFO", fmt.Sprintf("%s-%s ip: %s", storyChunkId, storyURL, resMessage))
-		}
+		*/
 	}
 
 	return LambdaResponse{

diff --git a/lambda_golang/go.mod b/lambda_golang/go.mod
@@ -1,6 +1,6 @@
 module github.com/rivernews/media-literacy
 
-go 1.16
+go 1.19
 
 require (
 	github.com/PuerkitoBio/goquery v1.7.1
@@ -9,7 +9,21 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.7.0
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0
 	github.com/rivernews/GoTools v0.1.8
 	golang.org/x/net v0.0.0-20210825183410-e898025ed96a
 )
+
+require (
+	github.com/andybalholm/cascadia v1.2.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect
+	github.com/aws/smithy-go v1.8.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	golang.org/x/text v0.3.6 // indirect
+)
diff --git a/lambda_golang/go.sum b/lambda_golang/go.sum
@@ -25,8 +25,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0 h1:B/1pIeV/oFnrOwh
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0/go.mod h1:LKb3cKNQIMh+itGnEpKGcnL/6OIjPZqrtYah1w5f+3o=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0 h1:nR9j0xMxpXk6orC/C03fbHNrbb1NaXp8LdVV7V1oVLE=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0/go.mod h1:Qit9H3zjAmF7CLHOkrepE9b2ndX/2l3scstsM5g2jSk=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0 h1:BI05Jbkaqp5IDxiobr3B59mX07lfpLJDv5NwAEx3wSs=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0/go.mod h1:BXA1CVaEd9TBOQ8G2ke7lMWdVggAeh35+h2HDO50z7s=
 github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 h1:sHXMIKYS6YiLPzmKSvDpPmOpJDHxmAUgbiF49YNVztg=
 github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA=
 github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 h1:1at4e5P+lvHNl2nUktdM2/v+rpICg/QSEr9TO/uW9vU=
@@ -47,8 +45,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rivernews/GoTools v0.1.7 h1:7NGCw25eRg+gQo9A2h45rXWN+i64RPOoMm9UlvWgxKM=
-github.com/rivernews/GoTools v0.1.7/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY=
 github.com/rivernews/GoTools v0.1.8 h1:niNLFlg0k4W6zIGrlXrdkL/Cy0yoyMTBIMNmQxPLcnc=
 github.com/rivernews/GoTools v0.1.8/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

diff --git a/lambda_golang/pkg/newssite/utilities.go b/lambda_golang/pkg/newssite/utilities.go
@@ -1,14 +1,16 @@
 package newssite
 
 import (
+	"encoding/json"
 	"strings"
+
 	"github.com/rivernews/GoTools"
 )
 
 type NewsSite struct {
-	Key string
-	Name string
-	Alias string
+	Key        string
+	Name       string
+	Alias      string
 	LandingURL string
 }
 
@@ -18,9 +20,17 @@ func GetNewsSite(envVar string) NewsSite {
 	tokens := strings.Split(ssmValue, ",")
 
 	return NewsSite{
-		Key: tokens[0],
-		Name: tokens[1],
-		Alias: tokens[2],
+		Key:        tokens[0],
+		Name:       tokens[1],
+		Alias:      tokens[2],
 		LandingURL: tokens[3],
 	}
 }
+
+func AsJson(v any) string {
+	jsonBytes, err := json.Marshal(v)
+	if err != nil {
+		GoTools.Logger("ERROR", err.Error())
+	}
+	return string(jsonBytes)
+}