From a2856b10509ce54438b1f2ae2021f48448c39a47 Mon Sep 17 00:00:00 2001 From: algolucky <105239720+algolucky@users.noreply.github.com> Date: Fri, 6 Jan 2023 15:58:21 -0600 Subject: [PATCH] util/s3: enable more credential providers (#4929) --- util/s3/s3Helper.go | 108 +++++++++++---------------------------- util/s3/s3Helper_test.go | 34 +++--------- 2 files changed, 35 insertions(+), 107 deletions(-) diff --git a/util/s3/s3Helper.go b/util/s3/s3Helper.go index e7e3537206..5f7acf6edb 100644 --- a/util/s3/s3Helper.go +++ b/util/s3/s3Helper.go @@ -22,6 +22,7 @@ import ( "io" "os" "path/filepath" + "reflect" "regexp" "runtime" "strconv" @@ -32,9 +33,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - - "github.com/algorand/go-algorand/util" - "github.com/algorand/go-algorand/util/codecs" ) const ( @@ -45,9 +43,6 @@ const ( s3DefaultReleaseBucket = "algorand-releases" s3DefaultUploadBucket = "algorand-uploads" s3DefaultRegion = "us-east-1" - - downloadAction = "download" - uploadAction = "upload" ) // Helper encapsulates the s3 session state for interactive with our default S3 bucket with appropriate credentials @@ -84,20 +79,12 @@ func getS3Region() (region string) { // MakeS3SessionForUploadWithBucket upload to bucket func MakeS3SessionForUploadWithBucket(awsBucket string) (helper Helper, err error) { - creds, err := getCredentials(uploadAction, awsBucket) - if err != nil { - return - } - return makeS3Session(creds, awsBucket) + return makeS3Session(awsBucket) } // MakeS3SessionForDownloadWithBucket download from bucket func MakeS3SessionForDownloadWithBucket(awsBucket string) (helper Helper, err error) { - creds, err := getCredentials(downloadAction, awsBucket) - if err != nil { - return - } - return makeS3Session(creds, awsBucket) + return makeS3Session(awsBucket) } // UploadFileStream sends file as stream to s3 @@ -114,84 +101,47 @@ func (helper *Helper) UploadFileStream(targetFile string, reader io.Reader) erro return nil } -type s3Keys struct { - ID string - Secret string -} - -func getCredentials(action string, awsBucket string) (creds *credentials.Credentials, err error) { - awsID, awsKey := getAWSCredentials() - credentailsRequired := checkCredentialsRequired(action, awsBucket) - if !credentailsRequired && (awsID == "" || awsKey == "") { - return credentials.AnonymousCredentials, nil - } - err = validateS3Credentials(awsID, awsKey) - if err != nil { +func validateS3Bucket(awsBucket string) (err error) { + if awsBucket == "" { + err = fmt.Errorf("bucket name is empty") return } - creds = credentials.NewStaticCredentials(awsID, awsKey, "") return - } -func loadS3KeysFromFile(keyFile string) (keys s3Keys, err error) { - err = codecs.LoadObjectFromFile(keyFile, &keys) - return -} - -func getAWSCredentials() (awsID string, awsKey string) { - awsID, _ = os.LookupEnv("AWS_ACCESS_KEY_ID") - awsKey, _ = os.LookupEnv("AWS_SECRET_ACCESS_KEY") - - // If not in environment, try to load from s3.json file in bin dir - if awsID == "" || awsKey == "" { - baseDir, err := util.ExeDir() - if err == nil { - keyFile := filepath.Join(baseDir, "s3.json") - keys, err := loadS3KeysFromFile(keyFile) - if err == nil { - awsID = keys.ID - awsKey = keys.Secret - } - } - } - return -} - -func validateS3Credentials(awsID string, awsKey string) (err error) { - if awsID == "" || awsKey == "" { - err = fmt.Errorf("AWS credentials must be specified in environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY") +func makeS3Session(bucket string) (helper Helper, err error) { + err = validateS3Bucket(bucket) + if err != nil { return } - return -} -func validateS3Bucket(awsBucket string) (err error) { - if awsBucket == "" { - err = fmt.Errorf("bucket name is empty") - return + awsConfig := &aws.Config{ + CredentialsChainVerboseErrors: aws.Bool(true), + Region: aws.String(getS3Region()), } - return -} -func checkCredentialsRequired(action string, bucketName string) (required bool) { - required = true - if action == downloadAction && bucketName == s3DefaultReleaseBucket { - required = false + // s3DefaultReleaseBucket should be public, use AnonymousCredentials + if bucket == s3DefaultReleaseBucket { + awsConfig.Credentials = credentials.AnonymousCredentials } - return -} -func makeS3Session(credentials *credentials.Credentials, bucket string) (helper Helper, err error) { - err = validateS3Bucket(bucket) + sess, err := session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Config: *awsConfig, + }) if err != nil { return } - sess, err := session.NewSession(&aws.Config{Region: aws.String(getS3Region()), - Credentials: credentials}) - if err != nil { - return + + // use AnonymousCredentials if none are found + if creds, err := sess.Config.Credentials.Get(); err != nil && !reflect.DeepEqual(creds, credentials.AnonymousCredentials) { + sess.Config.Credentials = credentials.AnonymousCredentials } + + if reflect.DeepEqual(sess.Config.Credentials, credentials.AnonymousCredentials) { + fmt.Println("Using anonymous credentials") + } + helper = Helper{ session: sess, bucket: bucket, @@ -294,7 +244,7 @@ func (helper *Helper) GetPackageFilesVersion(channel string, pkgFiles string, sp func GetVersionFromName(name string) (version uint64, err error) { re := regexp.MustCompile(`_(\d*)\.(\d*)\.(\d*)`) submatchAll := re.FindAllStringSubmatch(name, -1) - if submatchAll == nil || len(submatchAll) == 0 || len(submatchAll[0]) != 4 { + if len(submatchAll) == 0 || len(submatchAll[0]) != 4 { err = errors.New("unable to parse version from filename " + name) return } diff --git a/util/s3/s3Helper_test.go b/util/s3/s3Helper_test.go index 5f46483309..c0502a3b01 100644 --- a/util/s3/s3Helper_test.go +++ b/util/s3/s3Helper_test.go @@ -114,8 +114,6 @@ func TestMakeS3SessionForUploadWithBucket(t *testing.T) { const emptyBucket = "" type args struct { awsBucket string - awsID string - awsSecret string } tests := []struct { name string @@ -123,21 +121,12 @@ func TestMakeS3SessionForUploadWithBucket(t *testing.T) { wantHelper Helper wantErr bool }{ - {name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false}, - {name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true}, - {name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - {name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - {name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - // public upload bucket requires AWS credentials for uploads - {name: "test6", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false}, - {name: "test7", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true}, - {name: "test8", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true}, - {name: "test9", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true}, + {name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false}, + {name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true}, + {name: "test6", args: args{awsBucket: publicUploadBucket}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID) - os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret) gotHelper, err := MakeS3SessionForUploadWithBucket(tt.args.awsBucket) if (err != nil) != tt.wantErr { t.Errorf("MakeS3SessionForUploadWithBucket() error = %v, wantErr %v", err, tt.wantErr) @@ -158,8 +147,6 @@ func TestMakeS3SessionForDownloadWithBucket(t *testing.T) { const emptyBucket = "" type args struct { awsBucket string - awsID string - awsSecret string } tests := []struct { name string @@ -167,21 +154,12 @@ func TestMakeS3SessionForDownloadWithBucket(t *testing.T) { wantHelper Helper wantErr bool }{ - {name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false}, - {name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true}, - {name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - {name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - {name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - // public release bucket does not require AWS credentials for downloads - {name: "test6", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, - {name: "test7", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, - {name: "test8", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, - {name: "test9", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, + {name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false}, + {name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true}, + {name: "test6", args: args{awsBucket: publicReleaseBucket}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID) - os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret) gotHelper, err := MakeS3SessionForDownloadWithBucket(tt.args.awsBucket) if (err != nil) != tt.wantErr { t.Errorf("MakeS3SessionForDownloadWithBucket() error = %v, wantErr %v", err, tt.wantErr)