Improved metadata access s3 event

rivernews · Sep 17, 2022 · 9eca283 · 9eca283
1 parent 512f729
commit 9eca283
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 26 deletions.
diff --git a/lambda_golang/cmd/landing_metadata/main.go b/lambda_golang/cmd/landing_metadata/main.go
@@ -44,32 +44,32 @@ type LandingPageMetadata struct {
 func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, error) {
 	GoTools.Logger("INFO", "Landing page metadata.json generator launched")
 
-	landingPageS3Key := s3Event.Records[0].S3.Object.URLDecodedKey
-	GoTools.Logger("INFO", fmt.Sprintf("Captured landing page at %s", landingPageS3Key))
+	for _, record := range s3Event.Records {
+		landingPageS3Key := record.S3.Object.URLDecodedKey
+		GoTools.Logger("INFO", fmt.Sprintf("Captured landing page at %s", landingPageS3Key))
 
-	landingPageHtmlText := cloud.Pull(landingPageS3Key)
-	landingPageS3KeyTokens := strings.Split(landingPageS3Key, "/")
-	metadataS3DirKeyTokens := landingPageS3KeyTokens[:len(landingPageS3KeyTokens)-1]
-	metadataS3Key := fmt.Sprintf("%s/metadata.json", strings.Join(metadataS3DirKeyTokens, "/"))
+		landingPageHtmlText := cloud.Pull(landingPageS3Key)
+		landingPageS3KeyTokens := strings.Split(landingPageS3Key, "/")
+		metadataS3DirKeyTokens := landingPageS3KeyTokens[:len(landingPageS3KeyTokens)-1]
+		metadataS3Key := fmt.Sprintf("%s/metadata.json", strings.Join(metadataS3DirKeyTokens, "/"))
 
-	result := newssite.GetStoriesFromEconomy(landingPageHtmlText)
+		result := newssite.GetStoriesFromEconomy(landingPageHtmlText)
 
-	metadataJSONBytes, metadataJSONStringError := json.Marshal(LandingPageMetadata{Stories: result.Topics, UntitledStories: result.UntitledTopics})
-	if metadataJSONStringError != nil {
-		GoTools.Logger("ERROR", metadataJSONStringError.Error())
-	}
-	metadataJSONString := string(metadataJSONBytes)
-
-	cloud.Archive(cloud.ArchiveArgs{
-		BodyText:          metadataJSONString,
-		Key:               metadataS3Key,
-		FileTypeExtension: "json",
-	})
+		metadataJSONBytes, metadataJSONStringError := json.Marshal(LandingPageMetadata{Stories: result.Topics, UntitledStories: result.UntitledTopics})
+		if metadataJSONStringError != nil {
+			GoTools.Logger("ERROR", metadataJSONStringError.Error())
+		}
+		metadataJSONString := string(metadataJSONBytes)
 
-	bucket := GoTools.GetEnvVarHelper("S3_ARCHIVE_BUCKET")
-	GoTools.Logger("INFO", fmt.Sprintf("Saved landing page metadata to s3://%s/%s", bucket, metadataS3Key))
+		cloud.Archive(cloud.ArchiveArgs{
+			BodyText:          metadataJSONString,
+			Key:               metadataS3Key,
+			FileTypeExtension: "json",
+		})
 
-	// TODO: filter stories?
+		bucket := GoTools.GetEnvVarHelper("S3_ARCHIVE_BUCKET")
+		GoTools.Logger("INFO", fmt.Sprintf("Saved landing page metadata to s3://%s/%s", bucket, metadataS3Key))
+	}
 
 	return LambdaResponse{
 		OK:      true,

diff --git a/lambda_golang/cmd/story/main.go b/lambda_golang/cmd/story/main.go
@@ -6,22 +6,21 @@ import (
 
 	"context"
 	"fmt"
+	"math/rand"
 	"strings"
 	"time"
-	"math/rand"
 
 	"github.com/rivernews/GoTools"
 	"github.com/rivernews/media-literacy/pkg/cloud"
 )
 
-
 func main() {
 	rand.Seed(time.Now().UnixNano())
 	lambda.Start(HandleRequest)
 }
 
 type LambdaResponse struct {
-	OK bool `json:"OK:"`
+	OK      bool   `json:"OK:"`
 	Message string `json:"message:"`
 }
 
@@ -37,7 +36,7 @@ func HandleRequest(ctx context.Context, event events.SQSEvent) (LambdaResponse,
 
 		for _, storyURL := range storyURLs {
 			// TODO: randomized interval
-			time.Sleep(time.Duration(rand.Intn(5) + 5) * time.Second)
+			time.Sleep(time.Duration(rand.Intn(5)+5) * time.Second)
 
 			storyS3Path := fmt.Sprintf("story/%s", storyURL)
 
@@ -59,7 +58,7 @@ func HandleRequest(ctx context.Context, event events.SQSEvent) (LambdaResponse,
 	}
 
 	return LambdaResponse{
-		OK: true,
+		OK:      true,
 		Message: "Story consumer fetch parse ok",
 	}, nil
 }