Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

util/s3: enable more credential providers #4929

Merged
merged 3 commits into from
Jan 6, 2023
Merged

util/s3: enable more credential providers #4929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d8620da
util/s3: enable more credential options
algolucky Dec 21, 2022
5b9cb3b
util/s3: use AnonymousCredentials for algorand-releases
algolucky Dec 21, 2022
3c49eed
s3/util: use AnonymousCredentials if none are found
algolucky Dec 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 29 additions & 79 deletions util/s3/s3Helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"io"
"os"
"path/filepath"
"reflect"
"regexp"
"runtime"
"strconv"
Expand All @@ -32,9 +33,6 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"

"github.com/algorand/go-algorand/util"
"github.com/algorand/go-algorand/util/codecs"
)

const (
Expand All @@ -45,9 +43,6 @@ const (
s3DefaultReleaseBucket = "algorand-releases"
s3DefaultUploadBucket = "algorand-uploads"
s3DefaultRegion = "us-east-1"

downloadAction = "download"
uploadAction = "upload"
)

// Helper encapsulates the s3 session state for interactive with our default S3 bucket with appropriate credentials
Expand Down Expand Up @@ -84,20 +79,12 @@ func getS3Region() (region string) {

// MakeS3SessionForUploadWithBucket upload to bucket
func MakeS3SessionForUploadWithBucket(awsBucket string) (helper Helper, err error) {
creds, err := getCredentials(uploadAction, awsBucket)
if err != nil {
return
}
return makeS3Session(creds, awsBucket)
return makeS3Session(awsBucket)
}

// MakeS3SessionForDownloadWithBucket download from bucket
func MakeS3SessionForDownloadWithBucket(awsBucket string) (helper Helper, err error) {
creds, err := getCredentials(downloadAction, awsBucket)
if err != nil {
return
}
return makeS3Session(creds, awsBucket)
return makeS3Session(awsBucket)
}

// UploadFileStream sends file as stream to s3
Expand All @@ -114,84 +101,47 @@ func (helper *Helper) UploadFileStream(targetFile string, reader io.Reader) erro
return nil
}

type s3Keys struct {
ID string
Secret string
}

func getCredentials(action string, awsBucket string) (creds *credentials.Credentials, err error) {
awsID, awsKey := getAWSCredentials()
credentailsRequired := checkCredentialsRequired(action, awsBucket)
if !credentailsRequired && (awsID == "" || awsKey == "") {
return credentials.AnonymousCredentials, nil
}
err = validateS3Credentials(awsID, awsKey)
if err != nil {
func validateS3Bucket(awsBucket string) (err error) {
if awsBucket == "" {
err = fmt.Errorf("bucket name is empty")
return
}
creds = credentials.NewStaticCredentials(awsID, awsKey, "")
return

}

func loadS3KeysFromFile(keyFile string) (keys s3Keys, err error) {
err = codecs.LoadObjectFromFile(keyFile, &keys)
return
}

func getAWSCredentials() (awsID string, awsKey string) {
awsID, _ = os.LookupEnv("AWS_ACCESS_KEY_ID")
awsKey, _ = os.LookupEnv("AWS_SECRET_ACCESS_KEY")

// If not in environment, try to load from s3.json file in bin dir
if awsID == "" || awsKey == "" {
baseDir, err := util.ExeDir()
if err == nil {
keyFile := filepath.Join(baseDir, "s3.json")
keys, err := loadS3KeysFromFile(keyFile)
if err == nil {
awsID = keys.ID
awsKey = keys.Secret
}
}
}
return
}

func validateS3Credentials(awsID string, awsKey string) (err error) {
if awsID == "" || awsKey == "" {
err = fmt.Errorf("AWS credentials must be specified in environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY")
func makeS3Session(bucket string) (helper Helper, err error) {
err = validateS3Bucket(bucket)
if err != nil {
return
}
return
}

func validateS3Bucket(awsBucket string) (err error) {
if awsBucket == "" {
err = fmt.Errorf("bucket name is empty")
return
awsConfig := &aws.Config{
CredentialsChainVerboseErrors: aws.Bool(true),
Region: aws.String(getS3Region()),
}
return
}

func checkCredentialsRequired(action string, bucketName string) (required bool) {
required = true
if action == downloadAction && bucketName == s3DefaultReleaseBucket {
required = false
// s3DefaultReleaseBucket should be public, use AnonymousCredentials
if bucket == s3DefaultReleaseBucket {
awsConfig.Credentials = credentials.AnonymousCredentials
}
return
}

func makeS3Session(credentials *credentials.Credentials, bucket string) (helper Helper, err error) {
err = validateS3Bucket(bucket)
sess, err := session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
Config: *awsConfig,
})
if err != nil {
return
}
sess, err := session.NewSession(&aws.Config{Region: aws.String(getS3Region()),
Credentials: credentials})
if err != nil {
return

// use AnonymousCredentials if none are found
if creds, err := sess.Config.Credentials.Get(); err != nil && !reflect.DeepEqual(creds, credentials.AnonymousCredentials) {
sess.Config.Credentials = credentials.AnonymousCredentials
}

if reflect.DeepEqual(sess.Config.Credentials, credentials.AnonymousCredentials) {
fmt.Println("Using anonymous credentials")
}

helper = Helper{
session: sess,
bucket: bucket,
Expand Down Expand Up @@ -294,7 +244,7 @@ func (helper *Helper) GetPackageFilesVersion(channel string, pkgFiles string, sp
func GetVersionFromName(name string) (version uint64, err error) {
re := regexp.MustCompile(`_(\d*)\.(\d*)\.(\d*)`)
submatchAll := re.FindAllStringSubmatch(name, -1)
if submatchAll == nil || len(submatchAll) == 0 || len(submatchAll[0]) != 4 {
if len(submatchAll) == 0 || len(submatchAll[0]) != 4 {
err = errors.New("unable to parse version from filename " + name)
return
}
Expand Down
34 changes: 6 additions & 28 deletions util/s3/s3Helper_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,30 +114,19 @@ func TestMakeS3SessionForUploadWithBucket(t *testing.T) {
const emptyBucket = ""
type args struct {
awsBucket string
awsID string
awsSecret string
}
tests := []struct {
name string
args args
wantHelper Helper
wantErr bool
}{
{name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
{name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
{name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
{name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
{name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
// public upload bucket requires AWS credentials for uploads
{name: "test6", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false},
{name: "test7", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true},
{name: "test8", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true},
{name: "test9", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true},
{name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
{name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
{name: "test6", args: args{awsBucket: publicUploadBucket}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID)
os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret)
gotHelper, err := MakeS3SessionForUploadWithBucket(tt.args.awsBucket)
if (err != nil) != tt.wantErr {
t.Errorf("MakeS3SessionForUploadWithBucket() error = %v, wantErr %v", err, tt.wantErr)
Expand All @@ -158,30 +147,19 @@ func TestMakeS3SessionForDownloadWithBucket(t *testing.T) {
const emptyBucket = ""
type args struct {
awsBucket string
awsID string
awsSecret string
}
tests := []struct {
name string
args args
wantHelper Helper
wantErr bool
}{
{name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
{name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
{name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
{name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
{name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true},
// public release bucket does not require AWS credentials for downloads
{name: "test6", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
{name: "test7", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
{name: "test8", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
{name: "test9", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
{name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false},
{name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true},
{name: "test6", args: args{awsBucket: publicReleaseBucket}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID)
os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret)
gotHelper, err := MakeS3SessionForDownloadWithBucket(tt.args.awsBucket)
if (err != nil) != tt.wantErr {
t.Errorf("MakeS3SessionForDownloadWithBucket() error = %v, wantErr %v", err, tt.wantErr)
Expand Down