Use official AWS packages and allow custom endpoint

go.mod

@@ -1,23 +1,24 @@
 module github.com/jessfraz/s3server
 
+go 1.14
+
 require (
 	cloud.google.com/go v0.25.0
+	github.com/aws/aws-sdk-go v1.29.30
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/go-units v0.3.3
 	github.com/genuinetools/pkg v0.0.0-20180717144208-764bcdebd5f7
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
 	github.com/google/go-cmp v0.2.0 // indirect
 	github.com/google/martian v2.0.0-beta.2+incompatible // indirect
 	github.com/googleapis/gax-go v2.0.0+incompatible // indirect
-	github.com/mitchellh/goamz v0.0.0-20150317174335-caaaea8b30ee
+	github.com/mitchellh/goamz v0.0.0-20150317174335-caaaea8b30ee // indirect
 	github.com/motain/gocheck v0.0.0-20131023154940-9beb271d26e6 // indirect
 	github.com/onsi/gomega v1.4.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.0.5
-	github.com/stretchr/testify v1.2.2 // indirect
 	github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec // indirect
 	go.opencensus.io v0.14.0 // indirect
-	golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8 // indirect
 	golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd // indirect
 	google.golang.org/api v0.0.0-20180717000714-0025a57598c0
 	google.golang.org/appengine v1.1.0 // indirect

go.sum

@@ -1,5 +1,8 @@
 cloud.google.com/go v0.25.0 h1:6vD6xZTc8Jo6To8gHxFDRVsMvWFDgY3rugNszcDalN8=
 cloud.google.com/go v0.25.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/aws/aws-sdk-go v1.29.30 h1:HEEb7p5H850+hKVLRif2fWpaoJe5ZkZ5WUJwJ+CKWV4=
+github.com/aws/aws-sdk-go v1.29.30/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
@@ -8,6 +11,7 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/genuinetools/pkg v0.0.0-20180717144208-764bcdebd5f7 h1:EqH4uatdCpl20NeUhYwdS2n2Cjlg5hH+6I6vdSTzvUs=
 github.com/genuinetools/pkg v0.0.0-20180717144208-764bcdebd5f7/go.mod h1:XTcrCYlXPxnxL2UpnwuRn7tcaTn9HAhxFoFJucootk8=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -20,6 +24,8 @@ github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/mitchellh/goamz v0.0.0-20150317174335-caaaea8b30ee h1:Wp4ixY2/QEZOrQrGMF1h1x4yxqsef+aQPse0XMXzZhs=
 github.com/mitchellh/goamz v0.0.0-20150317174335-caaaea8b30ee/go.mod h1:svb8iUupD5i7RyGXoCUrk3EQSaXjWxKuqiZ0j41Jmm8=
 github.com/motain/gocheck v0.0.0-20131023154940-9beb271d26e6 h1:gKdQPVb3yDSbcw4sgNyrt2LP0/4uTdrvTm3e4IcATCE=
@@ -28,26 +34,35 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks=
 github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw=
 go.opencensus.io v0.14.0 h1:1eTLxqxSIAylcKoxnNkdhvvBNZDA8JwkKNXxgyma0IA=
 go.opencensus.io v0.14.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
 golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8 h1:h7zdf0RiEvWbYBKIx4b+q41xoUVnMmvsGZnIVE5syG8=
 golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd h1:QQhib242ErYDSMitlBm8V7wYCm/1a25hV8qMadIKLPA=
 golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 google.golang.org/api v0.0.0-20180717000714-0025a57598c0 h1:H82u+VsvfY63/uydrxjj2DpwjFvroE43qWaIzpa2Xik=
@@ -70,3 +85,4 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -27,6 +27,7 @@ var (
 	bucket   string
 	interval time.Duration
 
+	s3Endpoint  string
 	s3AccessKey string
 	s3SecretKey string
 	s3Region    string
@@ -56,6 +57,7 @@ func main() {
 	p.FlagSet.StringVar(&bucket, "bucket", "", "bucket path from which to serve files")
 	p.FlagSet.DurationVar(&interval, "interval", 5*time.Minute, "interval to generate new index.html's at")
 
+	p.FlagSet.StringVar(&s3Endpoint, "s3endpoint", "amazonaws.com", "s3 endpoint base")
 	p.FlagSet.StringVar(&s3AccessKey, "s3key", "", "s3 access key")
 	p.FlagSet.StringVar(&s3SecretKey, "s3secret", "", "s3 access secret")
 	p.FlagSet.StringVar(&s3Region, "s3region", "us-west-2", "aws region for the bucket")
@@ -98,7 +100,7 @@ func main() {
 		}()
 
 		// create a new provider
-		p, err := newProvider(provider, bucket, s3Region, s3AccessKey, s3SecretKey)
+		p, err := newProvider(provider, bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey)
 		if err != nil {
 			logrus.Fatalf("Creating new provider failed: %v", err)
 		}

provider.go

@@ -6,8 +6,10 @@ import (
 	"strings"
 
 	"cloud.google.com/go/storage"
-	"github.com/mitchellh/goamz/aws"
-	"github.com/mitchellh/goamz/s3"
+	"github.com/aws/aws-sdk-go/aws"
+	awsCredentials "github.com/aws/aws-sdk-go/aws/credentials"
+	awsSession "github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
 )
 
 type cloud interface {
@@ -16,36 +18,27 @@ type cloud interface {
 	BaseURL() string
 }
 
-func newProvider(provider, bucket, s3Region, s3AccessKey, s3SecretKey string) (cloud, error) {
+func newProvider(provider, bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey string) (cloud, error) {
+	bucket, prefix := cleanBucketName(bucket)
+
 	if provider == "s3" {
 		// auth with aws
-		auth, err := aws.GetAuth(s3AccessKey, s3SecretKey)
-		if err != nil {
-			return nil, err
-		}
+		conf := newAwsConfig(s3Endpoint, s3Region, s3AccessKey, s3SecretKey)
 
 		// create the client
-		region, err := getRegion(s3Region)
-		if err != nil {
-			return nil, err
-		}
-
-		p := s3Provider{bucket: bucket}
-		p.client = s3.New(auth, region)
-		bucket, p.prefix = cleanBucketName(p.bucket)
-		p.b = p.client.Bucket(bucket)
-		p.baseURL = p.bucket + ".s3.amazonaws.com"
+		p := s3Provider{bucket: bucket, prefix: prefix}
+		p.client = s3.New(awsSession.New(conf))
+		p.baseURL = fmt.Sprintf("%s.%s", p.bucket, *conf.Endpoint)
 		return &p, nil
 	}
 
-	p := gcsProvider{bucket: bucket}
+	p := gcsProvider{bucket: bucket, prefix: prefix}
 	p.ctx = context.Background()
 	client, err := storage.NewClient(p.ctx)
 	if err != nil {
 		return nil, err
 	}
 	p.client = client
-	p.bucket, p.prefix = cleanBucketName(p.bucket)
 	p.b = client.Bucket(p.bucket)
 	p.baseURL = p.bucket
 	if !strings.Contains(p.bucket, "j3ss.co") {
@@ -61,29 +54,25 @@ func cleanBucketName(bucket string) (string, string) {
 	bucket = strings.TrimPrefix(bucket, "gcs://")
 	parts := strings.SplitN(bucket, "/", 2)
 	if len(parts) == 1 {
-		return bucket, "/"
+		return bucket, ""
 	}
 
 	return parts[0], parts[1]
 }
 
-// getRegion returns the aws region that is matches a given string.
-func getRegion(name string) (aws.Region, error) {
-	var regions = map[string]aws.Region{
-		aws.APNortheast.Name:  aws.APNortheast,
-		aws.APSoutheast.Name:  aws.APSoutheast,
-		aws.APSoutheast2.Name: aws.APSoutheast2,
-		aws.EUCentral.Name:    aws.EUCentral,
-		aws.EUWest.Name:       aws.EUWest,
-		aws.USEast.Name:       aws.USEast,
-		aws.USWest.Name:       aws.USWest,
-		aws.USWest2.Name:      aws.USWest2,
-		aws.USGovWest.Name:    aws.USGovWest,
-		aws.SAEast.Name:       aws.SAEast,
-	}
-	region, ok := regions[name]
-	if !ok {
-		return aws.Region{}, fmt.Errorf("no region matches %s", name)
-	}
-	return region, nil
+func newAwsConfig(endpoint, region, accessKey, secretKey string) *aws.Config {
+	conf := aws.NewConfig()
+	conf.WithEndpoint(fmt.Sprintf("s3.%s.%s", region, endpoint))
+	conf.WithRegion(region)
+	conf.WithCredentials(awsCredentials.NewChainCredentials([]awsCredentials.Provider{
+		&awsCredentials.StaticProvider{
+			Value: awsCredentials.Value{
+				AccessKeyID:     accessKey,
+				SecretAccessKey: secretKey,
+			},
+		},
+		&awsCredentials.EnvProvider{},
+		&awsCredentials.SharedCredentialsProvider{},
+	}))
+	return conf
 }

s3.go

@@ -2,42 +2,39 @@ package main
 
 import (
 	"cloud.google.com/go/storage"
-	"github.com/mitchellh/goamz/s3"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/sirupsen/logrus"
 )
 
 type s3Provider struct {
 	bucket  string
 	prefix  string
 	baseURL string
 	client  *s3.S3
-	b       *s3.Bucket
 }
 
 // List returns the files in an s3 bucket.
 func (c *s3Provider) List(prefix, delimiter, marker string, max int, q *storage.Query) (files []object, err error) {
-	resp, err := c.b.List(prefix, delimiter, marker, max)
-	if err != nil {
-		return nil, err
-	}
-
-	// append to files
-	for _, f := range resp.Contents {
-		files = append(files, object{
-			Name:    f.Key,
-			Size:    f.Size,
-			BaseURL: c.BaseURL(),
-		})
-	}
-
-	// recursion for the recursion god
-	if resp.IsTruncated && resp.NextMarker != "" {
-		f, err := c.List(resp.Prefix, resp.Delimiter, resp.NextMarker, resp.MaxKeys, q)
-		if err != nil {
-			return nil, err
+	err = c.client.ListObjectsV2Pages(&s3.ListObjectsV2Input{
+		Bucket: aws.String(c.bucket),
+		Delimiter: aws.String(delimiter),
+		Prefix: aws.String(prefix),
+		MaxKeys: aws.Int64(int64(max)),
+	}, func(p *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, o := range p.Contents {
+			files = append(files, object{
+				Name:    aws.StringValue(o.Key),
+				Size:    aws.Int64Value(o.Size),
+				BaseURL: c.baseURL,
+			})
 		}
 
-		// append to files
-		files = append(files, f...)
+		return true // continue paging
+	})
+
+ 	if err != nil {
+		logrus.Fatalf("Failed to list objects for bucket, %s, %v", c.bucket, err)
 	}
 
 	return files, nil