Refactor file structure to share common golang code within module

.gitignore

@@ -1,7 +1,9 @@
 **credential**
 **/builds/**
 
-scraper_lambda/**/main
+lambda_golang/landing
+lambda_golang/story
+lambda_golang/stories
 venv
 
 # Binaries for programs and plugins

cloud_environments/terraform.sh

@@ -1,7 +1,7 @@
 set -e
 
 DEPLOY_DIR=$(git rev-parse --show-toplevel)/cloud_environments/${ENV:-production}
-SCRAPER_SRC_DIR=$(git rev-parse --show-toplevel)/scraper_lambda
+GOLANG_SRC_DIR=$(git rev-parse --show-toplevel)/lambda_golang
 PYTHON_SRC_DIR=$(git rev-parse --show-toplevel)/lambda
 
 set -o allexport
@@ -18,9 +18,10 @@ set +o allexport
 
 
 if (
-    cd $SCRAPER_SRC_DIR/landing && go build -o main && \
-    cd $SCRAPER_SRC_DIR/stories && go build -o main && \
-    cd $SCRAPER_SRC_DIR/story && go build -o main && \
+    cd $GOLANG_SRC_DIR && \
+    go build ./cmd/landing && \
+    go build ./cmd/stories && \
+    go build ./cmd/story && \
     cd $PYTHON_SRC_DIR && python -m compileall layer src
 ); then
     cd $DEPLOY_DIR

cloud_module/lambda.tf

@@ -61,15 +61,15 @@ module "scraper_lambda" {
   create_function = true
   function_name = "${local.project_name}-scraper-lambda"
   description   = "Lambda function for scraping"
-  handler       = "main"
+  handler       = "landing"
   runtime     = "go1.x"
 
   # Based on tf https://github.com/terraform-aws-modules/terraform-aws-lambda/blob/master/examples/build-package/main.tf#L111
   # Based on golang https://github.com/snsinfu/terraform-lambda-example/blob/master/Makefile#L23
   source_path = [{
-    path = "${path.module}/../scraper_lambda/landing"
-    commands = ["go build -o main", ":zip"]
-    patterns = ["main"]
+    path = "${path.module}/../lambda_golang/"
+    commands = ["go build ./cmd/landing", ":zip"]
+    patterns = ["landing"]
   }]
 
   timeout = 900

cloud_module/stories_sfn.tf

@@ -32,13 +32,13 @@ module batch_stories_fetch_parse_lambda {
   create_function = true
   function_name = "${local.project_name}-batch-stories-fetch-parse"
   description   = "Batch fetch and parse all stories of a landing page"
-  handler       = "main"
+  handler       = "stories"
   runtime     = "go1.x"
 
   source_path = [{
-    path = "${path.module}/../scraper_lambda/stories"
-    commands = ["go build -o main", ":zip"]
-    patterns = ["main"]
+    path = "${path.module}/../lambda_golang/"
+    commands = ["go build ./cmd/stories", ":zip"]
+    patterns = ["stories"]
   }]
 
   timeout = 900

cloud_module/stories_sqs.tf

@@ -31,12 +31,12 @@ module "stories_queue_consumer_lambda" {
   create_function = true
   function_name = "${local.project_name}-stories-queue-consumer-lambda"
   description   = "Consumes ${local.project_name} stories queue"
-  handler       = "main"
+  handler       = "story"
   runtime     = "go1.x"
   source_path = [{
-    path = "${path.module}/../scraper_lambda/story"
-    commands = ["go build -o main", ":zip"]
-    patterns = ["main"]
+    path = "${path.module}/../lambda_golang/"
+    commands = ["go build ./cmd/story", ":zip"]
+    patterns = ["story"]
   }]
   publish = true
 

scraper_lambda/landing/main.go → lambda_golang/cmd/landing/main.go

@@ -12,6 +12,9 @@ import (
 	"github.com/aws/aws-lambda-go/lambda"
 
 	"github.com/rivernews/GoTools"
+
+	"github.com/rivernews/media-literacy/pkg/cloud"
+	"github.com/rivernews/media-literacy/pkg/newssite"
 )
 
 func main() {
@@ -28,7 +31,7 @@ type LambdaResponse struct {
 }
 
 func HandleRequest(ctx context.Context, name LambdaEvent) (LambdaResponse, error) {
-	newsSite := GetNewsSite("NEWSSITE_ECONOMY")
+	newsSite :=  newssite.GetNewsSite("NEWSSITE_ECONOMY")
 	resp, err := http.Get(newsSite.LandingURL)
 	if err != nil {
 		// handle error
@@ -79,7 +82,7 @@ func HandleRequest(ctx context.Context, name LambdaEvent) (LambdaResponse, error
 	GoTools.Logger("INFO", successMessage)
 
 	// S3 archive
-	archive(strings.NewReader(bodyText), fmt.Sprintf("%s/daily-headlines/%s/landing.html", newsSite.Alias, time.Now().Format(time.RFC3339)))
+	cloud.Archive(strings.NewReader(bodyText), fmt.Sprintf("%s/daily-headlines/%s/landing.html", newsSite.Alias, time.Now().Format(time.RFC3339)))
 
 	return LambdaResponse{
 		OK: true,

lambda_golang/go.mod

@@ -0,0 +1,15 @@
+module github.com/rivernews/media-literacy
+
+go 1.16
+
+require (
+	github.com/PuerkitoBio/goquery v1.7.1
+	github.com/aws/aws-lambda-go v1.26.0
+	github.com/aws/aws-sdk-go-v2 v1.9.0
+	github.com/aws/aws-sdk-go-v2/config v1.7.0
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0
+	github.com/rivernews/GoTools v0.1.6
+	golang.org/x/net v0.0.0-20210825183410-e898025ed96a
+)

scraper_lambda/stories/go.sum → lambda_golang/go.sum

@@ -1,4 +1,8 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
+github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
+github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
+github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
 github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA=
 github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
 github.com/aws/aws-sdk-go-v2 v1.9.0 h1:+S+dSqQCN3MSU5vJRu1HqHrq00cJn6heIMU7X9hcsoo=
@@ -9,10 +13,18 @@ github.com/aws/aws-sdk-go-v2/credentials v1.4.0 h1:kmvesfjY861FzlCU9mvAfe01D9aeX
 github.com/aws/aws-sdk-go-v2/credentials v1.4.0/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 h1:OxTAgH8Y4BXHD6PGCJ8DHx2kaZPCQfSTqmDsdRZFezE=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0/go.mod h1:CpNzHK9VEFUCknu50kkB8z58AH2B5DvPP7ea1LHve/Y=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0 h1:zzH1xd1/PX7bFO4/BQwVQP+UcBfYieI1sMH9DA68xZY=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0/go.mod h1:fPU0eFGnS47RyKHHs8BNcCKOm5oOA5xm0BlrZWsQT/A=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 h1:d95cddM3yTm4qffj3P6EnP+TzX1SSkWaQypXSgT/hpA=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2/go.mod h1:BQV0agm+JEhqR+2RT5e1XTFIDcAAV0eW6z2trp+iduw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 h1:gceOysEWNNwLd6cki65IMBZ4WAM0MwgBQq2n7kejoT8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0/go.mod h1:v8ygadNyATSm6elwJ/4gzJwcFhri9RqS8skgHKiwXPU=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 h1:VNJ5NLBteVXEwE2F1zEXVmyIH58mZ6kIQGJoC7C+vkg=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0 h1:B/1pIeV/oFnrOwhoMA6ASX+qT4FzMqn1MYsPiIXgMqQ=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0/go.mod h1:LKb3cKNQIMh+itGnEpKGcnL/6OIjPZqrtYah1w5f+3o=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0 h1:nR9j0xMxpXk6orC/C03fbHNrbb1NaXp8LdVV7V1oVLE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0/go.mod h1:Qit9H3zjAmF7CLHOkrepE9b2ndX/2l3scstsM5g2jSk=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0 h1:BI05Jbkaqp5IDxiobr3B59mX07lfpLJDv5NwAEx3wSs=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0/go.mod h1:BXA1CVaEd9TBOQ8G2ke7lMWdVggAeh35+h2HDO50z7s=
 github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 h1:sHXMIKYS6YiLPzmKSvDpPmOpJDHxmAUgbiF49YNVztg=
@@ -24,28 +36,33 @@ github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivernews/GoTools v0.1.6 h1:jO33haprmuzKi/cRlcX9/cjWMhAf3xFlyasXVot/mrM=
 github.com/rivernews/GoTools v0.1.6/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210825183410-e898025ed96a h1:bRuuGXV8wwSdGTB+CtJf+FjgO1APK1CoO39T4BN/XBw=
+golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

scraper_lambda/landing/s3_archiver.go → lambda_golang/pkg/cloud/s3_archiver.go

@@ -1,4 +1,4 @@
-package main
+package cloud
 
 import (
 	"context"
@@ -14,7 +14,7 @@ import (
 	"github.com/rivernews/GoTools"
 )
 
-func archive(body io.Reader, key string) (bool, error) {
+func Archive(body io.Reader, key string) (bool, error) {
 	bucket := GoTools.GetEnvVarHelper("S3_ARCHIVE_BUCKET")
 	GoTools.Logger("INFO", "Bucket to archive: s3://", bucket, "Key:", key)
 

scraper_lambda/landing/utilities.go → lambda_golang/pkg/newssite/utilities.go

@@ -1,4 +1,4 @@
-package main
+package newssite
 
 import (
 	"strings"