Implement S3 Pull; pulling landing page in stories fetch lambda

item in #15
rivernews · Aug 31, 2021 · aff9d43 · aff9d43
1 parent 05a2a5f
commit aff9d43
Show file tree

Hide file tree

Showing 6 changed files with 97 additions and 16 deletions.
diff --git a/cloud_module/stories_sfn.tf b/cloud_module/stories_sfn.tf
@@ -57,6 +57,7 @@ module batch_stories_fetch_parse_lambda {
       effect    = "Allow",
       actions   = [
         "s3:PutObject",
+        "s3:GetObject"
       ],
       resources = ["${data.aws_s3_bucket.archive.arn}/*"]
     }

diff --git a/lambda_golang/cmd/stories/main.go b/lambda_golang/cmd/stories/main.go
@@ -12,6 +12,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
+
+	// local packages
+	"github.com/rivernews/media-literacy/pkg/cloud"
 )
 
 
@@ -31,6 +34,10 @@ type LambdaResponse struct {
 func HandleRequest(ctx context.Context, stepFunctionInput StepFunctionInput) (LambdaResponse, error) {
 	GoTools.Logger("INFO", fmt.Sprintf("Batch stories lambda started! Landing page S3 path: `%s`; going to test delayed messages...", stepFunctionInput.LandingS3Key))
 
+	landingPageHtmlText := cloud.Pull(stepFunctionInput.LandingS3Key)
+
+	GoTools.Logger("INFO", fmt.Sprintf("Pulled landing page content:\n ``` %s ``` \n ", landingPageHtmlText[:500]))
+
 	// TODO: get all story links
 	links := []string{
 		"chunk-00", "chunk-01", "chunk-02", "chunk-03",

diff --git a/lambda_golang/go.mod b/lambda_golang/go.mod
@@ -10,6 +10,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.8.0
-	github.com/rivernews/GoTools v0.1.6
+	github.com/rivernews/GoTools v0.1.7
 	golang.org/x/net v0.0.0-20210825183410-e898025ed96a
 )
diff --git a/lambda_golang/go.sum b/lambda_golang/go.sum
@@ -36,18 +36,23 @@ github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rivernews/GoTools v0.1.6 h1:jO33haprmuzKi/cRlcX9/cjWMhAf3xFlyasXVot/mrM=
-github.com/rivernews/GoTools v0.1.6/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY=
+github.com/rivernews/GoTools v0.1.7 h1:7NGCw25eRg+gQo9A2h45rXWN+i64RPOoMm9UlvWgxKM=
+github.com/rivernews/GoTools v0.1.7/go.mod h1:MUyn34yir1uYn9XUDFnoMwXHANg/DlYyaNMyVK44NQY=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -60,9 +65,12 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/lambda_golang/pkg/cloud/s3_archiver.go b/lambda_golang/pkg/cloud/s3_archiver.go
@@ -7,30 +7,19 @@ import (
   	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 
 	"github.com/rivernews/GoTools"
+
 )
 
 func Archive(body io.Reader, key string) (bool, error) {
 	bucket := GoTools.GetEnvVarHelper("S3_ARCHIVE_BUCKET")
 	GoTools.Logger("INFO", "Bucket to archive: s3://", bucket, "Key:", key)
 
-	// Based on
-	// https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/
-	awsConfig, configErr := config.LoadDefaultConfig(
-		context.TODO(),
-		config.WithRegion("us-west-2"),
-	)
-	if configErr != nil {
-		GoTools.Logger("ERROR", "AWS shared configuration failed", configErr.Error())
-	}
-
   	timeout := time.Second * 30
-
-  	client := s3.NewFromConfig(awsConfig)
+  	client := SharedS3Client()
 
 	// Based on
 	// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#Uploader
@@ -65,3 +54,43 @@ func Archive(body io.Reader, key string) (bool, error) {
 
 	return true, nil
 }
+
+func Pull(key string) string {
+	bucket := GoTools.GetEnvVarHelper("S3_ARCHIVE_BUCKET")
+	client := SharedS3Client()
+
+	// based on
+	// https://stackoverflow.com/a/65710928/9814131
+	headObject, headError := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key: aws.String(key),
+	})
+	if headError != nil {
+		GoTools.Logger("ERROR", headError.Error())
+	}
+
+	// main idea
+	// https://stackoverflow.com/a/41645765/9814131
+	// code based on
+	// https://github.com/aws/aws-sdk-go-v2/pull/1171/files#diff-c43ccf2f39bfbd136d7f7ddf2a1c88ac983d910b687bca29b4a8e6ea9759551b
+	// pre-allocate in memory buffer, where headObject type is *s3.HeadObjectOutput
+	// and
+	// AWS SDK v2 Doc
+	// https://aws.github.io/aws-sdk-go-v2/docs/sdk-utilities/s3/#download-manager
+
+	downloader := manager.NewDownloader(client)
+	buf := make([]byte, int(headObject.ContentLength))
+	// wrap with aws.WriteAtBuffer
+	w := manager.NewWriteAtBuffer(buf)
+	// download file into the memory
+	numBytesDownloaded, err := downloader.Download(context.TODO(), w, &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		GoTools.Logger("ERROR", err.Error())
+	}
+	GoTools.Logger("INFO", fmt.Sprintf("Downloaded %d for `s3://%s/%s`", numBytesDownloaded, bucket, key))
+
+	return string(w.Bytes())
+}
diff --git a/lambda_golang/pkg/cloud/s3_common.go b/lambda_golang/pkg/cloud/s3_common.go
@@ -0,0 +1,36 @@
+package cloud
+
+import (
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+
+	"github.com/rivernews/GoTools"
+
+	"context"
+	"sync"
+)
+
+var (
+	s3Client *s3.Client
+	s3ClientOnce sync.Once
+)
+
+func SharedS3Client() *s3.Client {
+	// followed
+	// https://stackoverflow.com/a/53504651/9814131
+	s3ClientOnce.Do(func() {
+		// Based on
+		// https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/
+		awsConfig, configErr := config.LoadDefaultConfig(
+			context.TODO(),
+			config.WithRegion("us-west-2"),
+		)
+		if configErr != nil {
+			GoTools.Logger("ERROR", "AWS shared configuration failed", configErr.Error())
+		}
+
+		s3Client = s3.NewFromConfig(awsConfig)
+	})
+
+	return s3Client
+}