From bda216e44f822a88b3e73fe7e0ab6094d5e05e53 Mon Sep 17 00:00:00 2001 From: Shaung Date: Sat, 17 Sep 2022 01:06:59 -0700 Subject: [PATCH 1/4] read parse metadata.json --- lambda_golang/cmd/landing/main.go | 4 ++-- lambda_golang/cmd/landing_metadata/main.go | 8 +------- lambda_golang/cmd/story/main.go | 8 ++++++++ lambda_golang/pkg/newssite/economy.go | 14 +++++++------- lambda_golang/pkg/newssite/utilities.go | 6 ++++++ 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/lambda_golang/cmd/landing/main.go b/lambda_golang/cmd/landing/main.go index 18c5240..9855af7 100644 --- a/lambda_golang/cmd/landing/main.go +++ b/lambda_golang/cmd/landing/main.go @@ -61,7 +61,7 @@ func HandleRequest(ctx context.Context, name LambdaEvent) (LambdaResponse, error // print out all story titles var slackMessage strings.Builder - for i, topic := range result.Topics { + for i, topic := range result.Stories { slackMessage.WriteString(topic.Name) slackMessage.WriteString(" ") slackMessage.WriteString(topic.Description) @@ -76,7 +76,7 @@ func HandleRequest(ctx context.Context, name LambdaEvent) (LambdaResponse, error } GoTools.SendSlackMessage(slackMessage.String()) - successMessage := fmt.Sprintf("Scraper finished - %d links found", len(result.Topics)) + successMessage := fmt.Sprintf("Scraper finished - %d links found", len(result.Stories)) GoTools.Logger("INFO", successMessage) // S3 archive diff --git a/lambda_golang/cmd/landing_metadata/main.go b/lambda_golang/cmd/landing_metadata/main.go index 24562bc..443d1a5 100644 --- a/lambda_golang/cmd/landing_metadata/main.go +++ b/lambda_golang/cmd/landing_metadata/main.go @@ -35,11 +35,6 @@ type LambdaResponse struct { Message string `json:"message:"` } -type LandingPageMetadata struct { - Stories []newssite.Topic `json:"stories"` - UntitledStories []newssite.Topic `json:"untitledstories"` -} - func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, error) { GoTools.Logger("INFO", "Landing page metadata.json generator launched") @@ -53,8 +48,7 @@ func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, metadataS3Key := fmt.Sprintf("%s/metadata.json", strings.Join(metadataS3DirKeyTokens, "/")) result := newssite.GetStoriesFromEconomy(landingPageHtmlText) - - metadataJSONString := newssite.AsJson(LandingPageMetadata{Stories: result.Topics, UntitledStories: result.UntitledTopics}) + metadataJSONString := newssite.AsJson(result) cloud.Archive(cloud.ArchiveArgs{ BodyText: metadataJSONString, diff --git a/lambda_golang/cmd/story/main.go b/lambda_golang/cmd/story/main.go index b7220fd..3780cb1 100644 --- a/lambda_golang/cmd/story/main.go +++ b/lambda_golang/cmd/story/main.go @@ -11,6 +11,7 @@ import ( "time" "github.com/rivernews/GoTools" + "github.com/rivernews/media-literacy/pkg/cloud" "github.com/rivernews/media-literacy/pkg/newssite" ) @@ -32,6 +33,13 @@ func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse, for _, record := range S3Event.Records { GoTools.Logger("INFO", fmt.Sprintf("S3 event ``` %s ```\n ", newssite.AsJson(record))) + + metadataJSONString := cloud.Pull(record.S3.Object.URLDecodedKey) + var metadata newssite.LandingPageMetadata + newssite.FromJson([]byte(metadataJSONString), &metadata) + + GoTools.Logger("INFO", fmt.Sprintf("Test first story: %d:%d", len(metadata.Stories), len(metadata.UntitledStories))) + /* storyChunk := message.Body GoTools.Logger("INFO", fmt.Sprintf("Story consumer! story chunk: %s", storyChunk)) diff --git a/lambda_golang/pkg/newssite/economy.go b/lambda_golang/pkg/newssite/economy.go index 16a81a4..449c944 100644 --- a/lambda_golang/pkg/newssite/economy.go +++ b/lambda_golang/pkg/newssite/economy.go @@ -13,12 +13,12 @@ type Topic struct { URL string `json:"url"` } -type GetStoriesFromEconomyResult struct { - Topics []Topic - UntitledTopics []Topic +type LandingPageMetadata struct { + Stories []Topic `json:"stories"` + UntitledStories []Topic `json:"untitledstories"` } -func GetStoriesFromEconomy(body string) GetStoriesFromEconomyResult { +func GetStoriesFromEconomy(body string) LandingPageMetadata { // Load the HTML document doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) if err != nil { @@ -50,8 +50,8 @@ func GetStoriesFromEconomy(body string) GetStoriesFromEconomyResult { }) GoTools.Logger("INFO", "Skipped due to empty title URLs:\n", emptyTitleURLs.String()) - return GetStoriesFromEconomyResult{ - Topics: topics, - UntitledTopics: untitledTopics, + return LandingPageMetadata{ + Stories: topics, + UntitledStories: untitledTopics, } } diff --git a/lambda_golang/pkg/newssite/utilities.go b/lambda_golang/pkg/newssite/utilities.go index 11b332f..7b80682 100644 --- a/lambda_golang/pkg/newssite/utilities.go +++ b/lambda_golang/pkg/newssite/utilities.go @@ -34,3 +34,9 @@ func AsJson(v any) string { } return string(jsonBytes) } + +func FromJson(b []byte, structInstance any) { + if err := json.Unmarshal(b, structInstance); err != nil { + GoTools.Logger("ERROR", err.Error()) + } +} From e29fdc8cf0b104c973cc1a0153d4323f2437ce0b Mon Sep 17 00:00:00 2001 From: Shaung Date: Sat, 17 Sep 2022 01:56:14 -0700 Subject: [PATCH 2/4] fetch a story POC https://github.com/rivernews/media-literacy/issues/24 --- cloud_module/landing_s3_trigger.tf | 2 +- lambda_golang/cmd/landing/main.go | 25 +---------------------- lambda_golang/cmd/story/main.go | 12 +++++++++++ lambda_golang/pkg/newssite/economy.go | 2 +- lambda_golang/pkg/newssite/utilities.go | 27 +++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 26 deletions(-) diff --git a/cloud_module/landing_s3_trigger.tf b/cloud_module/landing_s3_trigger.tf index 22c5bb9..ef0b96d 100644 --- a/cloud_module/landing_s3_trigger.tf +++ b/cloud_module/landing_s3_trigger.tf @@ -21,7 +21,7 @@ resource "aws_s3_bucket_notification" "bucket_notification" { lambda_function_arn = module.landing_parse_metadata_lambda.lambda_function_arn events = ["s3:ObjectCreated:*"] filter_prefix = "${local.newssite_economy_alias}/" - filter_suffix = ".html" + filter_suffix = "landing.html" } lambda_function { diff --git a/lambda_golang/cmd/landing/main.go b/lambda_golang/cmd/landing/main.go index 9855af7..ecdb4a4 100644 --- a/lambda_golang/cmd/landing/main.go +++ b/lambda_golang/cmd/landing/main.go @@ -3,13 +3,9 @@ package main import ( "context" "fmt" - "io" - "net/http" "strings" "time" - "golang.org/x/net/html/charset" - "github.com/aws/aws-lambda-go/lambda" "github.com/rivernews/GoTools" @@ -33,27 +29,8 @@ type LambdaResponse struct { func HandleRequest(ctx context.Context, name LambdaEvent) (LambdaResponse, error) { newsSite := newssite.GetNewsSite("NEWSSITE_ECONOMY") - resp, err := http.Get(newsSite.LandingURL) - if err != nil { - // handle error - GoTools.Logger("ERROR", err.Error()) - } - defer resp.Body.Close() - - contentType := resp.Header.Get("Content-Type") // Optional, better guessing - GoTools.Logger("INFO", "ContentType is ", contentType) - utf8reader, err := charset.NewReader(resp.Body, contentType) - if err != nil { - GoTools.Logger("ERROR", err.Error()) - } - - body, err := io.ReadAll(utf8reader) - if err != nil { - // handle error - GoTools.Logger("ERROR", err.Error()) - } - bodyText := string(body) + bodyText := newssite.Fetch(newsSite.LandingURL) GoTools.Logger("INFO", "In golang runtime now!\n\n```\n "+bodyText[:500]+"\n ...```\n End of message") // scraper diff --git a/lambda_golang/cmd/story/main.go b/lambda_golang/cmd/story/main.go index 3780cb1..f1a35a6 100644 --- a/lambda_golang/cmd/story/main.go +++ b/lambda_golang/cmd/story/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strings" "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-lambda-go/lambda" @@ -34,12 +35,23 @@ func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse, GoTools.Logger("INFO", fmt.Sprintf("S3 event ``` %s ```\n ", newssite.AsJson(record))) + metadataS3KeyTokens := strings.Split(record.S3.Object.URLDecodedKey, "/") + newsSiteAlias := metadataS3KeyTokens[0] + landingPageTimeStamp := metadataS3KeyTokens[len(metadataS3KeyTokens)-2] + metadataJSONString := cloud.Pull(record.S3.Object.URLDecodedKey) var metadata newssite.LandingPageMetadata newssite.FromJson([]byte(metadataJSONString), &metadata) GoTools.Logger("INFO", fmt.Sprintf("Test first story: %d:%d", len(metadata.Stories), len(metadata.UntitledStories))) + story := metadata.Stories[0] + storyHtmlBodyText := newssite.Fetch(story.URL) + cloud.Archive(cloud.ArchiveArgs{ + BodyText: storyHtmlBodyText, + Key: fmt.Sprintf("%s/stories/%s-%s/story.html", newsSiteAlias, landingPageTimeStamp, story.Name), + }) + /* storyChunk := message.Body GoTools.Logger("INFO", fmt.Sprintf("Story consumer! story chunk: %s", storyChunk)) diff --git a/lambda_golang/pkg/newssite/economy.go b/lambda_golang/pkg/newssite/economy.go index 449c944..6f14ff8 100644 --- a/lambda_golang/pkg/newssite/economy.go +++ b/lambda_golang/pkg/newssite/economy.go @@ -35,7 +35,7 @@ func GetStoriesFromEconomy(body string) LandingPageMetadata { var emptyTitleURLs strings.Builder doc.Find("a[href$=html]").Each(func(i int, anchor *goquery.Selection) { topic := Topic{ - Name: strings.TrimSpace(anchor.Text()), + Name: strings.ReplaceAll(strings.TrimSpace(anchor.Text()), "/", "-"), Description: "", URL: strings.TrimSpace(anchor.AttrOr("href", "-")), } diff --git a/lambda_golang/pkg/newssite/utilities.go b/lambda_golang/pkg/newssite/utilities.go index 7b80682..1fbfbf9 100644 --- a/lambda_golang/pkg/newssite/utilities.go +++ b/lambda_golang/pkg/newssite/utilities.go @@ -2,8 +2,12 @@ package newssite import ( "encoding/json" + "io" + "net/http" "strings" + "golang.org/x/net/html/charset" + "github.com/rivernews/GoTools" ) @@ -40,3 +44,26 @@ func FromJson(b []byte, structInstance any) { GoTools.Logger("ERROR", err.Error()) } } + +func Fetch(url string) string { + resp, err := http.Get(url) + if err != nil { + // handle error + GoTools.Logger("ERROR", err.Error()) + } + defer resp.Body.Close() + + contentType := resp.Header.Get("Content-Type") // Optional, better guessing + GoTools.Logger("DEBUG", "ContentType is ", contentType) + utf8reader, err := charset.NewReader(resp.Body, contentType) + if err != nil { + GoTools.Logger("ERROR", err.Error()) + } + + body, err := io.ReadAll(utf8reader) + if err != nil { + // handle error + GoTools.Logger("ERROR", err.Error()) + } + return string(body) +} From d2b025e86d7353846850b395d662b388a4b4952c Mon Sep 17 00:00:00 2001 From: Shaung Date: Sat, 17 Sep 2022 04:28:54 -0700 Subject: [PATCH 3/4] Sfn map parallism POC https://github.com/rivernews/media-literacy/issues/24 --- .gitignore | 3 +- cloud_environments/terraform.sh | 1 + cloud_module/api.tf | 18 --- cloud_module/sfn_def/batch_stories_def.json | 40 ++++--- cloud_module/stories_sfn.tf | 60 +++++++++- cloud_module/stories_sqs.tf | 28 ++++- lambda/src/slack_command_controller.py | 3 +- lambda_golang/cmd/stories/main.go | 117 ++++++++++++++++++++ lambda_golang/cmd/story/main.go | 66 +---------- lambda_golang/go.mod | 10 +- lambda_golang/go.sum | 11 ++ lambda_golang/pkg/cloud/aws_common.go | 32 ++++++ lambda_golang/pkg/cloud/s3_common.go | 20 +--- lambda_golang/pkg/cloud/sfn_common.go | 34 ++++++ lambda_golang/pkg/newssite/economy.go | 12 ++ 15 files changed, 334 insertions(+), 121 deletions(-) create mode 100644 lambda_golang/cmd/stories/main.go create mode 100644 lambda_golang/pkg/cloud/aws_common.go create mode 100644 lambda_golang/pkg/cloud/sfn_common.go diff --git a/.gitignore b/.gitignore index 1e9fc3a..d25bdfe 100755 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,9 @@ **/builds/** lambda_golang/landing -lambda_golang/story +lambda_golang/stories lambda_golang/landing_metadata +lambda_golang/story venv # Binaries for programs and plugins diff --git a/cloud_environments/terraform.sh b/cloud_environments/terraform.sh index c882829..0414f72 100755 --- a/cloud_environments/terraform.sh +++ b/cloud_environments/terraform.sh @@ -21,6 +21,7 @@ if ( cd $GOLANG_SRC_DIR && \ go build ./cmd/landing && \ go build ./cmd/landing_metadata && \ + go build ./cmd/stories && \ go build ./cmd/story && \ cd $PYTHON_SRC_DIR && python -m compileall layer src ); then diff --git a/cloud_module/api.tf b/cloud_module/api.tf index db1daa0..6e7603f 100755 --- a/cloud_module/api.tf +++ b/cloud_module/api.tf @@ -93,29 +93,11 @@ module "slack_command_lambda" { } } - # allow lambda to invoke step function - attach_policy_json = true - policy_json = < Date: Sat, 17 Sep 2022 18:42:10 -0700 Subject: [PATCH 4/4] randomize requests --- cloud_module/sfn_def/batch_stories_def.json | 10 ++++++++- lambda_golang/cmd/landing_metadata/main.go | 2 +- lambda_golang/cmd/stories/main.go | 16 ++++---------- lambda_golang/cmd/story/main.go | 23 ++++++++++++++++++++- lambda_golang/go.mod | 2 +- lambda_golang/go.sum | 9 +++----- lambda_golang/pkg/newssite/economy.go | 2 +- lambda_golang/pkg/newssite/utilities.go | 15 -------------- 8 files changed, 41 insertions(+), 38 deletions(-) diff --git a/cloud_module/sfn_def/batch_stories_def.json b/cloud_module/sfn_def/batch_stories_def.json index b9bd661..76194b8 100644 --- a/cloud_module/sfn_def/batch_stories_def.json +++ b/cloud_module/sfn_def/batch_stories_def.json @@ -6,7 +6,7 @@ "Type": "Map", "InputPath": "$", "ItemsPath": "$.stories", - "MaxConcurrency": 1, + "MaxConcurrency": 28, "ResultPath": "$.stories", "Comment": "`Parameters` replaces `InputPath`, see https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-map-state.html ", "Parameters": { @@ -20,6 +20,14 @@ "Fetch": { "Type": "Task", "Resource": "${FETCH_STORY_LAMBDA_ARN}", + "Retry" : [ + { + "ErrorEquals": [ "States.Timeout", "Lambda.Unknown" ], + "IntervalSeconds": 3, + "MaxAttempts": 99, + "BackoffRate": 1.5 + } + ], "End": true } } diff --git a/lambda_golang/cmd/landing_metadata/main.go b/lambda_golang/cmd/landing_metadata/main.go index 443d1a5..0d4ff2d 100644 --- a/lambda_golang/cmd/landing_metadata/main.go +++ b/lambda_golang/cmd/landing_metadata/main.go @@ -48,7 +48,7 @@ func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, metadataS3Key := fmt.Sprintf("%s/metadata.json", strings.Join(metadataS3DirKeyTokens, "/")) result := newssite.GetStoriesFromEconomy(landingPageHtmlText) - metadataJSONString := newssite.AsJson(result) + metadataJSONString := GoTools.AsJson(result) cloud.Archive(cloud.ArchiveArgs{ BodyText: metadataJSONString, diff --git a/lambda_golang/cmd/stories/main.go b/lambda_golang/cmd/stories/main.go index b64ec8e..5181d66 100644 --- a/lambda_golang/cmd/stories/main.go +++ b/lambda_golang/cmd/stories/main.go @@ -37,7 +37,7 @@ func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse, for _, record := range S3Event.Records { - GoTools.Logger("INFO", fmt.Sprintf("S3 event ``` %s ```\n ", newssite.AsJson(record))) + GoTools.Logger("INFO", fmt.Sprintf("S3 event ``` %s ```\n ", GoTools.AsJson(record))) metadataS3KeyTokens := strings.Split(record.S3.Object.URLDecodedKey, "/") newsSiteAlias := metadataS3KeyTokens[0] @@ -45,20 +45,12 @@ func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse, metadataJSONString := cloud.Pull(record.S3.Object.URLDecodedKey) var metadata newssite.LandingPageMetadata - newssite.FromJson([]byte(metadataJSONString), &metadata) + GoTools.FromJson([]byte(metadataJSONString), &metadata) GoTools.Logger("INFO", fmt.Sprintf("Test first story: %d:%d", len(metadata.Stories), len(metadata.UntitledStories))) - // TODO: move below into sfn map lambda - story := metadata.Stories[0] - storyHtmlBodyText := newssite.Fetch(story.URL) - cloud.Archive(cloud.ArchiveArgs{ - BodyText: storyHtmlBodyText, - Key: fmt.Sprintf("%s/stories/%s-%s/story.html", newsSiteAlias, landingPageTimeStamp, story.Name), - }) - // fire step function, input = {stories: [{}, {}, {}...], newsSiteAlias:"", landingPageTimeStamp:""} - sfnInput := newssite.AsJson(&newssite.StepFunctionInput{ + sfnInput := GoTools.AsJson(&newssite.StepFunctionInput{ Stories: metadata.Stories, NewsSiteAlias: newsSiteAlias, LandingPageTimeStamp: landingPageTimeStamp, @@ -76,7 +68,7 @@ func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse, }, ) - GoTools.Logger("INFO", fmt.Sprintf("Sfn output ``` %s ```\n", newssite.AsJson(sfnOutput))) + GoTools.Logger("INFO", fmt.Sprintf("Sfn output ``` %s ```\n", GoTools.AsJson(sfnOutput))) /* storyChunk := message.Body diff --git a/lambda_golang/cmd/story/main.go b/lambda_golang/cmd/story/main.go index bd97498..275c9c3 100644 --- a/lambda_golang/cmd/story/main.go +++ b/lambda_golang/cmd/story/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "fmt" "math/rand" "time" @@ -10,6 +11,7 @@ import ( "context" "github.com/rivernews/GoTools" + "github.com/rivernews/media-literacy/pkg/cloud" "github.com/rivernews/media-literacy/pkg/newssite" ) @@ -26,7 +28,26 @@ type LambdaResponse struct { func HandleRequest(ctx context.Context, stepFunctionMapIterationInput newssite.StepFunctionMapIterationInput) (LambdaResponse, error) { GoTools.Logger("INFO", "Fetch single story launched") - GoTools.Logger("INFO", fmt.Sprintf("%s", stepFunctionMapIterationInput.Story.Name)) + baseWait := 4 + waitRange := 100 + totalWait := rand.Intn(waitRange) + baseWait + time.Sleep(time.Duration(totalWait) * time.Second) + + responseBody, _, _ := GoTools.Fetch(GoTools.FetchOption{ + URL: "https://checkip.amazonaws.com", + QueryParams: map[string]string{ + "format": "json", + }, + Method: "GET", + }) + + GoTools.Logger("INFO", fmt.Sprintf("IP=`%s` waited %d - %s", bytes.TrimSpace(responseBody), totalWait, stepFunctionMapIterationInput.Story.Name)) + + storyHtmlBodyText := newssite.Fetch(stepFunctionMapIterationInput.Story.URL) + cloud.Archive(cloud.ArchiveArgs{ + BodyText: storyHtmlBodyText, + Key: fmt.Sprintf("%s/stories/%s-%s/story.html", stepFunctionMapIterationInput.NewsSiteAlias, stepFunctionMapIterationInput.LandingPageTimeStamp, stepFunctionMapIterationInput.Story.Name), + }) return LambdaResponse{ OK: true, diff --git a/lambda_golang/go.mod b/lambda_golang/go.mod index 34966d6..1ca720b 100644 --- a/lambda_golang/go.mod +++ b/lambda_golang/go.mod @@ -9,7 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.7.0 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0 - github.com/rivernews/GoTools v0.1.8 + github.com/rivernews/GoTools v0.2.2 golang.org/x/net v0.0.0-20210825183410-e898025ed96a ) diff --git a/lambda_golang/go.sum b/lambda_golang/go.sum index e2eb28b..aab9b6f 100644 --- a/lambda_golang/go.sum +++ b/lambda_golang/go.sum @@ -5,7 +5,6 @@ github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5 github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA= github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU= -github.com/aws/aws-sdk-go-v2 v1.9.0 h1:+S+dSqQCN3MSU5vJRu1HqHrq00cJn6heIMU7X9hcsoo= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.16.15 h1:2sInOWGE4HV54R90Pj8QgqBBw3Qf1I0husqbqjPZzys= github.com/aws/aws-sdk-go-v2 v1.16.15/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= @@ -37,7 +36,6 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 h1:sHXMIKYS6YiLPzmKSvDpPmOpJDHxm github.com/aws/aws-sdk-go-v2/service/sso v1.4.0/go.mod h1:+1fpWnL96DL23aXPpMGbsmKe8jLTEfbjuQoA4WS1VaA= github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 h1:1at4e5P+lvHNl2nUktdM2/v+rpICg/QSEr9TO/uW9vU= github.com/aws/aws-sdk-go-v2/service/sts v1.7.0/go.mod h1:0qcSMCyASQPN2sk/1KQLQ2Fh6yq8wm0HSDAimPhzCoM= -github.com/aws/smithy-go v1.8.0 h1:AEwwwXQZtUwP5Mz506FeXXrKBe0jA8gVM+1gEcSRooc= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= @@ -47,8 +45,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -56,8 +54,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivernews/GoTools v0.1.8 h1:niNLFlg0k4W6zIGrlXrdkL/Cy0yoyMTBIMNmQxPLcnc= -github.com/rivernews/GoTools v0.1.8/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY= +github.com/rivernews/GoTools v0.2.2 h1:++Rn9ISg47ykzNaDTwYEM55PZDp9fd0VaNCvSGbMOX0= +github.com/rivernews/GoTools v0.2.2/go.mod h1:rwhBYt6u+VTJwdeAwFkB7ZBMc/RZGvZlf9yJLubIQUI= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -74,7 +72,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/lambda_golang/pkg/newssite/economy.go b/lambda_golang/pkg/newssite/economy.go index 13fb04f..86f08b5 100644 --- a/lambda_golang/pkg/newssite/economy.go +++ b/lambda_golang/pkg/newssite/economy.go @@ -53,7 +53,7 @@ func GetStoriesFromEconomy(body string) LandingPageMetadata { } if topic.Name != "" { topics = append(topics, topic) - GoTools.Logger("DEBUG", "Found an anchor ", topic.Name, topic.URL) + GoTools.Logger("VERBOSE", "Found an anchor ", topic.Name, topic.URL) } else { untitledTopics = append(untitledTopics, topic) emptyTitleURLs.WriteString(topic.URL) diff --git a/lambda_golang/pkg/newssite/utilities.go b/lambda_golang/pkg/newssite/utilities.go index 1fbfbf9..27c2c9b 100644 --- a/lambda_golang/pkg/newssite/utilities.go +++ b/lambda_golang/pkg/newssite/utilities.go @@ -1,7 +1,6 @@ package newssite import ( - "encoding/json" "io" "net/http" "strings" @@ -31,20 +30,6 @@ func GetNewsSite(envVar string) NewsSite { } } -func AsJson(v any) string { - jsonBytes, err := json.Marshal(v) - if err != nil { - GoTools.Logger("ERROR", err.Error()) - } - return string(jsonBytes) -} - -func FromJson(b []byte, structInstance any) { - if err := json.Unmarshal(b, structInstance); err != nil { - GoTools.Logger("ERROR", err.Error()) - } -} - func Fetch(url string) string { resp, err := http.Get(url) if err != nil {