Skip to content

Commit

Permalink
util/s3: enable more credential providers (#4929)
Browse files Browse the repository at this point in the history
  • Loading branch information
@algolucky
algolucky authored Jan 6, 2023
1 parent 453a346 commit a2856b1
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 107 deletions.
108 changes: 29 additions & 79 deletions util/s3/s3Helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"io"
"os"
"path/filepath"
"reflect"
"regexp"
"runtime"
"strconv"
Expand All @@ -32,9 +33,6 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"

"github.com/algorand/go-algorand/util"
"github.com/algorand/go-algorand/util/codecs"
)

const (
Expand All @@ -45,9 +43,6 @@ const (
s3DefaultReleaseBucket = "algorand-releases"
s3DefaultUploadBucket = "algorand-uploads"
s3DefaultRegion = "us-east-1"

downloadAction = "download"
uploadAction = "upload"
)

// Helper encapsulates the s3 session state for interactive with our default S3 bucket with appropriate credentials
Expand Down Expand Up @@ -84,20 +79,12 @@ func getS3Region() (region string) {

// MakeS3SessionForUploadWithBucket upload to bucket
func MakeS3SessionForUploadWithBucket(awsBucket string) (helper Helper, err error) {
creds, err := getCredentials(uploadAction, awsBucket)
if err != nil {
return
}
return makeS3Session(creds, awsBucket)
return makeS3Session(awsBucket)
}

// MakeS3SessionForDownloadWithBucket download from bucket
func MakeS3SessionForDownloadWithBucket(awsBucket string) (helper Helper, err error) {
creds, err := getCredentials(downloadAction, awsBucket)
if err != nil {
return
}
return makeS3Session(creds, awsBucket)
return makeS3Session(awsBucket)
}

// UploadFileStream sends file as stream to s3
Expand All @@ -114,84 +101,47 @@ func (helper *Helper) UploadFileStream(targetFile string, reader io.Reader) erro
return nil
}

type s3Keys struct {
ID string
Secret string
}

func getCredentials(action string, awsBucket string) (creds *credentials.Credentials, err error) {
awsID, awsKey := getAWSCredentials()
credentailsRequired := checkCredentialsRequired(action, awsBucket)
if !credentailsRequired && (awsID == "" || awsKey == "") {
return credentials.AnonymousCredentials, nil
}
err = validateS3Credentials(awsID, awsKey)
if err != nil {
func validateS3Bucket(awsBucket string) (err error) {
if awsBucket == "" {
err = fmt.Errorf("bucket name is empty")
return
}
creds = credentials.NewStaticCredentials(awsID, awsKey, "")
return

}

func loadS3KeysFromFile(keyFile string) (keys s3Keys, err error) {
err = codecs.LoadObjectFromFile(keyFile, &keys)
return
}

func getAWSCredentials() (awsID string, awsKey string) {
awsID, _ = os.LookupEnv("AWS_ACCESS_KEY_ID")
awsKey, _ = os.LookupEnv("AWS_SECRET_ACCESS_KEY")

// If not in environment, try to load from s3.json file in bin dir
if awsID == "" || awsKey == "" {
baseDir, err := util.ExeDir()
if err == nil {
keyFile := filepath.Join(baseDir, "s3.json")
keys, err := loadS3KeysFromFile(keyFile)
if err == nil {
awsID = keys.ID
awsKey = keys.Secret
}
}
}
return
}

func validateS3Credentials(awsID string, awsKey string) (err error) {
if awsID == "" || awsKey == "" {
err = fmt.Errorf("AWS credentials must be specified in environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY")
func makeS3Session(bucket string) (helper Helper, err error) {
err = validateS3Bucket(bucket)
if err != nil {
return
}
return
}

func validateS3Bucket(awsBucket string) (err error) {
if awsBucket == "" {
err = fmt.Errorf("bucket name is empty")
return
awsConfig := &aws.Config{
CredentialsChainVerboseErrors: aws.Bool(true),
Region: aws.String(getS3Region()),
}
return
}

func checkCredentialsRequired(action string, bucketName string) (required bool) {
required = true
if action == downloadAction && bucketName == s3DefaultReleaseBucket {
required = false
// s3DefaultReleaseBucket should be public, use AnonymousCredentials
if bucket == s3DefaultReleaseBucket {
awsConfig.Credentials = credentials.AnonymousCredentials
}
return
}

func makeS3Session(credentials *credentials.Credentials, bucket string) (helper Helper, err error) {
err = validateS3Bucket(bucket)
sess, err := session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
Config: *awsConfig,
})
if err != nil {
return
}
sess, err := session.NewSession(&aws.Config{Region: aws.String(getS3Region()),
Credentials: credentials})
if err != nil {
return

// use AnonymousCredentials if none are found
if creds, err := sess.Config.Credentials.Get(); err != nil && !reflect.DeepEqual(creds, credentials.AnonymousCredentials) {
sess.Config.Credentials = credentials.AnonymousCredentials
}

if reflect.DeepEqual(sess.Config.Credentials, credentials.AnonymousCredentials) {
fmt.Println("Using anonymous credentials")
}

helper = Helper{
session: sess,
bucket: bucket,
Expand Down Expand Up @@ -294,7 +244,7 @@ func (helper *Helper) GetPackageFilesVersion(channel string, pkgFiles string, sp
func GetVersionFromName(name string) (version uint64, err error) {
re := regexp.MustCompile(`_(\d*)\.(\d*)\.(\d*)`)
submatchAll := re.FindAllStringSubmatch(name, -1)
if submatchAll == nil || len(submatchAll) == 0 || len(submatchAll[0]) != 4 {
if len(submatchAll) == 0 || len(submatchAll[0]) != 4 {
err = errors.New("unable to parse version from filename " + name)
return
}
Expand Down
34 changes: 6 additions & 28 deletions util/s3/s3Helper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,30 +114,19 @@ func TestMakeS3SessionForUploadWithBucket(t *testing.T) {
const emptyBucket = ""
type args struct {
awsBucket string
awsID string
awsSecret string
}
tests := []struct {
name string
args args
wantHelper Helper
wantErr bool
}{
{name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
{name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
{name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
{name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
{name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
// public upload bucket requires AWS credentials for uploads
{name: "test6", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false},
{name: "test7", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true},
{name: "test8", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true},
{name: "test9", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true},
{name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
{name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
{name: "test6", args: args{awsBucket: publicUploadBucket}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID)
os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret)
gotHelper, err := MakeS3SessionForUploadWithBucket(tt.args.awsBucket)
if (err != nil) != tt.wantErr {
t.Errorf("MakeS3SessionForUploadWithBucket() error = %v, wantErr %v", err, tt.wantErr)
Expand All @@ -158,30 +147,19 @@ func TestMakeS3SessionForDownloadWithBucket(t *testing.T) {
const emptyBucket = ""
type args struct {
awsBucket string
awsID string
awsSecret string
}
tests := []struct {
name string
args args
wantHelper Helper
wantErr bool
}{
{name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
{name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
{name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
{name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
{name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
// public release bucket does not require AWS credentials for downloads
{name: "test6", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
{name: "test7", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
{name: "test8", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
{name: "test9", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
{name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
{name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
{name: "test6", args: args{awsBucket: publicReleaseBucket}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID)
os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret)
gotHelper, err := MakeS3SessionForDownloadWithBucket(tt.args.awsBucket)
if (err != nil) != tt.wantErr {
t.Errorf("MakeS3SessionForDownloadWithBucket() error = %v, wantErr %v", err, tt.wantErr)
Expand Down

0 comments on commit a2856b1

Please sign in to comment.