Skip to content

Commit

Permalink
set up sqs flow through ok (but URL cannot show up in lambda event)
Browse files Browse the repository at this point in the history
  • Loading branch information
rivernews committed Aug 29, 2021
1 parent 08c31dd commit 2c06a67
Show file tree
Hide file tree
Showing 11 changed files with 222 additions and 8 deletions.
5 changes: 4 additions & 1 deletion cloud_environments/terraform.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ set -e

DEPLOY_DIR=$(git rev-parse --show-toplevel)/cloud_environments/${ENV:-production}
SCRAPER_SRC_DIR=$(git rev-parse --show-toplevel)/scraper_lambda
PYTHON_SRC_DIR=$(git rev-parse --show-toplevel)/lambda

set -o allexport
. ${DEPLOY_DIR}/local.backend.credentials.tfvars
Expand All @@ -18,7 +19,9 @@ set +o allexport

if (
cd $SCRAPER_SRC_DIR/landing && go build -o main && \
cd $SCRAPER_SRC_DIR/stories && go build -o main
cd $SCRAPER_SRC_DIR/stories && go build -o main && \
cd $SCRAPER_SRC_DIR/story && go build -o main && \
cd $PYTHON_SRC_DIR && python -m compileall layer src
); then
cd $DEPLOY_DIR

Expand Down
1 change: 1 addition & 0 deletions cloud_module/api.tf
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ EOF
BATCH_STORIES_SFN_ARN = module.batch_stories_sfn.state_machine_arn

LOGLEVEL = "DEBUG"
ENV = local.environment

S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ module batch_stories_fetch_parse_lambda {

attach_policy_statements = true
policy_statements = {
pipeline_sqs = {
effect = "Allow",
actions = ["sqs:SendMessage", "sqs:GetQueueUrl"],
resources = [module.stories_queue.this_sqs_queue_arn]
}
s3_archive_bucket = {
effect = "Allow",
actions = [
Expand All @@ -58,6 +63,8 @@ module batch_stories_fetch_parse_lambda {
}

environment_variables = {
STORIES_QUEUE_NAME = module.stories_queue.this_sqs_queue_name

SLACK_WEBHOOK_URL = var.slack_post_webhook_url
LOG_LEVEL = "DEBUG"
DEBUG = "true"
Expand Down
71 changes: 71 additions & 0 deletions cloud_module/stories_sqs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
module "stories_queue" {
source = "terraform-aws-modules/sqs/aws"
version = ">= 2.0, < 3.0"

name = "${local.project_name}-stories-queue.fifo"
delay_seconds = 1

fifo_queue = true
content_based_deduplication = true
visibility_timeout_seconds = 60

# enable long polling
receive_wait_time_seconds = 10

tags = {
Project = local.project_name
}
}

module "stories_queue_consumer_lambda" {
source = "terraform-aws-modules/lambda/aws"

create_function = true
function_name = "${local.project_name}-stories-queue-consumer-lambda"
description = "Consumes ${local.project_name} stories queue"
handler = "main"
runtime = "go1.x"
source_path = [{
path = "${path.module}/../scraper_lambda/story"
commands = ["go build -o main", ":zip"]
patterns = ["main"]
}]
publish = true

timeout = 30
cloudwatch_logs_retention_in_days = 7

reserved_concurrent_executions = -1

# event source mapping for long polling
event_source_mapping = {
sqs = {
event_source_arn = module.stories_queue.this_sqs_queue_arn
batch_size = 1
}
}
allowed_triggers = {
sqs = {
principal = "sqs.amazonaws.com"
source_arn = module.stories_queue.this_sqs_queue_arn
}
}
attach_policy_statements = true
policy_statements = {
pull_sqs = {
effect = "Allow",
actions = ["sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes"],
resources = [module.stories_queue.this_sqs_queue_arn]
}
}

environment_variables = {
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
LOGLEVEL = "DEBUG"
ENV = local.environment
}

tags = {
Project = local.project_name
}
}
3 changes: 2 additions & 1 deletion lambda/src/slack_command_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,10 @@ def lambda_handler(request: APIGatewayRequest, context):
stateMachineArn=os.environ['BATCH_STORIES_SFN_ARN'],
name=f'media-literacy-sf-batch-stories-{datetime.now().strftime("%Y-%H-%M")}-{secrets.token_hex(nbytes=5)}',
input=json.dumps({
'landingURL': landing_html_key
'landingS3Key': landing_html_key
})
)
SlackService.send(f'landing page key: `{landing_html_key}`')

loop.run_until_complete(SlackService.send(f'You sent a slack command. Processed response: {res}'))

Expand Down
3 changes: 3 additions & 0 deletions scraper_lambda/stories/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,8 @@ go 1.16

require (
github.com/aws/aws-lambda-go v1.26.0
github.com/aws/aws-sdk-go-v2 v1.9.0
github.com/aws/aws-sdk-go-v2/config v1.7.0
github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0
github.com/rivernews/GoTools v0.1.5
)
32 changes: 32 additions & 0 deletions scraper_lambda/stories/go.sum
Original file line number Diff line number Diff line change
@@ -1,19 +1,51 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA=
github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
github.com/aws/aws-sdk-go-v2 v1.9.0 h1:+S+dSqQCN3MSU5vJRu1HqHrq00cJn6heIMU7X9hcsoo=
github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
github.com/aws/aws-sdk-go-v2/config v1.7.0 h1:J2cZ7qe+3IpqBEXnHUrFrOjoB9BlsXg7j53vxcl5IVg=
github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY=
github.com/aws/aws-sdk-go-v2/credentials v1.4.0 h1:kmvesfjY861FzlCU9mvAfe01D9aeXcG2ZuC+k9F2YLM=
github.com/aws/aws-sdk-go-v2/credentials v1.4.0/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 h1:OxTAgH8Y4BXHD6PGCJ8DHx2kaZPCQfSTqmDsdRZFezE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0/go.mod h1:CpNzHK9VEFUCknu50kkB8z58AH2B5DvPP7ea1LHve/Y=
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 h1:d95cddM3yTm4qffj3P6EnP+TzX1SSkWaQypXSgT/hpA=
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2/go.mod h1:BQV0agm+JEhqR+2RT5e1XTFIDcAAV0eW6z2trp+iduw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 h1:VNJ5NLBteVXEwE2F1zEXVmyIH58mZ6kIQGJoC7C+vkg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk=
github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0 h1:BI05Jbkaqp5IDxiobr3B59mX07lfpLJDv5NwAEx3wSs=
github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0/go.mod h1:BXA1CVaEd9TBOQ8G2ke7lMWdVggAeh35+h2HDO50z7s=
github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 h1:sHXMIKYS6YiLPzmKSvDpPmOpJDHxmAUgbiF49YNVztg=
github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA=
github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 h1:1at4e5P+lvHNl2nUktdM2/v+rpICg/QSEr9TO/uW9vU=
github.com/aws/aws-sdk-go-v2/service/sts v1.7.0/go.mod h1:0qcSMCyASQPN2sk/1KQLQ2Fh6yq8wm0HSDAimPhzCoM=
github.com/aws/smithy-go v1.8.0 h1:AEwwwXQZtUwP5Mz506FeXXrKBe0jA8gVM+1gEcSRooc=
github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivernews/GoTools v0.1.5 h1:bKC+z4yVzLCyrqd2wM9LZ3vmxd+xAF11nnScacVOaaw=
github.com/rivernews/GoTools v0.1.5/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
44 changes: 38 additions & 6 deletions scraper_lambda/stories/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,15 @@
package main

import (
"github.com/aws/aws-lambda-go/lambda"
"context"
"github.com/rivernews/GoTools"
"fmt"

"github.com/aws/aws-lambda-go/lambda"
"github.com/rivernews/GoTools"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/sqs"
)


Expand All @@ -15,7 +20,7 @@ func main() {
}

type LambdaEvent struct {
LandingURL string `json:"landingURL"`
LandingS3Key string `json:"landingS3Key"`
}

type LambdaResponse struct {
Expand All @@ -24,12 +29,39 @@ type LambdaResponse struct {
}

func HandleRequest(ctx context.Context, event LambdaEvent) (LambdaResponse, error) {
// TODO
GoTools.SendSlackMessage(fmt.Sprintf("Batch stories lambda started! Landing page S3 path: %s", event.LandingS3Key))

// TODO: get all story links
link := "http://story.com"

// send SQS
// refer to
// https://aws.github.io/aws-sdk-go-v2/docs/code-examples/sqs/sendmessage/
queueName := GoTools.GetEnvVarHelper("STORIES_QUEUE_NAME")
awsConfig, configErr := config.LoadDefaultConfig(context.TODO())
if configErr != nil {
GoTools.Logger("ERROR", "AWS shared configuration failed", configErr.Error())
}
sqsClient := sqs.NewFromConfig(awsConfig)
getQueueResponse, getQueueError := sqsClient.GetQueueUrl(context.TODO(), &sqs.GetQueueUrlInput{ QueueName: aws.String(queueName) })
if getQueueError != nil {
GoTools.Logger("ERROR", fmt.Sprintf("Error getting queue URL: %s", getQueueError.Error()))
}
queueURL := getQueueResponse.QueueUrl

res, err := sqsClient.SendMessage(context.TODO(), &sqs.SendMessageInput{
QueueUrl: queueURL,
MessageBody: aws.String(fmt.Sprintf("{\"storyURL\": \"%s\"}", link)),
// TODO: better group id naming for multiplexing
MessageGroupId: aws.String(fmt.Sprintf("%s-00", queueName)),
})

GoTools.SendSlackMessage(fmt.Sprintf("Batch stories lambda started! Landing page S3 path: %s", event.LandingURL))
if err != nil {
GoTools.Logger("ERROR", fmt.Sprintf("Error sending message: %s", err.Error()))
}

return LambdaResponse{
OK: true,
Message: "Batch story fetch parse ok",
Message: fmt.Sprintf("Sent message OK: %s", *res.MessageId),
}, nil
}
8 changes: 8 additions & 0 deletions scraper_lambda/story/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
module github.com/rivernews/media-literacy/story

go 1.16

require (
github.com/aws/aws-lambda-go v1.26.0
github.com/rivernews/GoTools v0.1.5
)
23 changes: 23 additions & 0 deletions scraper_lambda/story/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA=
github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivernews/GoTools v0.1.5 h1:bKC+z4yVzLCyrqd2wM9LZ3vmxd+xAF11nnScacVOaaw=
github.com/rivernews/GoTools v0.1.5/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
33 changes: 33 additions & 0 deletions scraper_lambda/story/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package main

import (
"github.com/aws/aws-lambda-go/lambda"
"context"
"github.com/rivernews/GoTools"
"fmt"
)


func main() {
lambda.Start(HandleRequest)
}

type LambdaEvent struct {
StoryURL string `json:"storyURL"`
}

type LambdaResponse struct {
OK bool `json:"OK:"`
Message string `json:"message:"`
}

func HandleRequest(ctx context.Context, event LambdaEvent) (LambdaResponse, error) {
// TODO

GoTools.SendSlackMessage(fmt.Sprintf("Story consumer! story URL: %s", event.StoryURL))

return LambdaResponse{
OK: true,
Message: "Story consumer fetch parse ok",
}, nil
}

0 comments on commit 2c06a67

Please sign in to comment.