parameters can flow through sfn, sqs and consumer ok

rivernews · Aug 29, 2021 · e345c7b · e345c7b
1 parent 2c06a67
commit e345c7b
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 14 deletions.
diff --git a/cloud_module/sfn_def/batch_stories_def.json b/cloud_module/sfn_def/batch_stories_def.json
@@ -5,8 +5,13 @@
       "BatchStoriesFetchParse":{
         "Comment": "Fetch and parse all stories",
         "Type":"Task",
-        "Resource":"${BATCH_STORIES_FETCH_PARSE_LAMBDA_ARN}",
-        "Parameters":{},
+        "Resource": "arn:aws:states:::lambda:invoke",
+        "Parameters":{
+          "FunctionName": "${BATCH_STORIES_FETCH_PARSE_LAMBDA_ARN}",
+          "Payload": {
+            "landingS3Key.$": "$.landingS3Key"
+          }
+        },
         "End":true
      }
     }

diff --git a/lambda/src/slack_command_controller.py b/lambda/src/slack_command_controller.py
@@ -52,7 +52,7 @@ def lambda_handler(request: APIGatewayRequest, context):
                 'landingS3Key': landing_html_key
             })
         )
-        SlackService.send(f'landing page key: `{landing_html_key}`')
+        loop.run_until_complete(SlackService.send(f'received landing page key: `{landing_html_key}`'))
 
     loop.run_until_complete(SlackService.send(f'You sent a slack command. Processed response: {res}'))
 

diff --git a/scraper_lambda/stories/main.go b/scraper_lambda/stories/main.go
@@ -19,7 +19,7 @@ func main() {
 	lambda.Start(HandleRequest)
 }
 
-type LambdaEvent struct {
+type StepFunctionInput struct {
 	LandingS3Key string `json:"landingS3Key"`
 }
 
@@ -28,8 +28,8 @@ type LambdaResponse struct {
 	Message string `json:"message:"`
 }
 
-func HandleRequest(ctx context.Context, event LambdaEvent) (LambdaResponse, error) {
-	GoTools.SendSlackMessage(fmt.Sprintf("Batch stories lambda started! Landing page S3 path: %s", event.LandingS3Key))
+func HandleRequest(ctx context.Context, stepFunctionInput StepFunctionInput) (LambdaResponse, error) {
+	GoTools.Logger("INFO", fmt.Sprintf("Batch stories lambda started! Landing page S3 path: `%s`", stepFunctionInput.LandingS3Key))
 
 	// TODO: get all story links
 	link := "http://story.com"
@@ -51,9 +51,11 @@ func HandleRequest(ctx context.Context, event LambdaEvent) (LambdaResponse, erro
 
 	res, err := sqsClient.SendMessage(context.TODO(), &sqs.SendMessageInput{
 		QueueUrl: queueURL,
-		MessageBody: aws.String(fmt.Sprintf("{\"storyURL\": \"%s\"}", link)),
+		MessageBody: aws.String(link),
 		// TODO: better group id naming for multiplexing
 		MessageGroupId: aws.String(fmt.Sprintf("%s-00", queueName)),
+
+		// TODO: add delay
 	})
 
 	if err != nil {

diff --git a/scraper_lambda/story/main.go b/scraper_lambda/story/main.go
@@ -2,6 +2,8 @@ package main
 
 import (
 	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/aws/aws-lambda-go/events"
+
 	"context"
 	"github.com/rivernews/GoTools"
 	"fmt"
@@ -12,19 +14,19 @@ func main() {
 	lambda.Start(HandleRequest)
 }
 
-type LambdaEvent struct {
-	StoryURL string `json:"storyURL"`
-}
-
 type LambdaResponse struct {
 	OK bool `json:"OK:"`
 	Message string `json:"message:"`
 }
 
-func HandleRequest(ctx context.Context, event LambdaEvent) (LambdaResponse, error) {
-	// TODO
+// SQS event
+// refer to https://github.com/aws/aws-lambda-go/blob/v1.26.0/events/README_SQS.md
+func HandleRequest(ctx context.Context, event events.SQSEvent) (LambdaResponse, error) {
+	for _, message := range event.Records {
+		GoTools.Logger("INFO", fmt.Sprintf("Story consumer! story URL: %s", message.Body))
 
-	GoTools.SendSlackMessage(fmt.Sprintf("Story consumer! story URL: %s", event.StoryURL))
+		// TODO: fetch and archive
+	}
 
 	return LambdaResponse{
 		OK: true,