Skip to content

Commit

Permalink
internal/resource: support S3 access point URLs
Browse files Browse the repository at this point in the history 
Support S3 access point URLs in ARN format as a source.
This allows valid, opaque S3 URLs such as
`s3:arn:aws:s3:us-west-2:123456789012:accesspoint/test/object`
Being able to use this format will allow S3 URLs on different
partitions and lays the foundation to potentially support
multi-region access points in the future.

Fixes #1091
Signed-off-by: Zeleena Kearney <[email protected]>
  • Loading branch information
@zeleena
zeleena committed Apr 4, 2022
1 parent ec526d2 commit 961d70e
Show file tree
Hide file tree
Showing 7 changed files with 221 additions and 20 deletions.
1 change: 1 addition & 0 deletions config/shared/errors/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ var (
ErrEngineConfiguration = errors.New("engine incorrectly configured")

// AWS S3 specific errors
ErrInvalidS3ARN = errors.New("invalid S3 ARN format")
ErrInvalidS3ObjectVersionId = errors.New("invalid S3 object VersionId")

// Obsolete errors, left here for ABI compatibility
Expand Down
19 changes: 19 additions & 0 deletions config/v3_4_experimental/types/url.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package types
import (
"net/url"

"github.com/aws/aws-sdk-go/aws/arn"
"github.com/vincent-petithory/dataurl"

"github.com/coreos/ignition/v2/config/shared/errors"
Expand All @@ -39,6 +40,24 @@ func validateURL(s string) error {
}
}
return nil
case "arn":
fullURL := u.Scheme + ":" + u.Opaque
if !arn.IsARN(fullURL) {
return errors.ErrInvalidS3ARN
}
s3arn, err := arn.Parse(fullURL)
if err != nil {
return err
}
if s3arn.Service != "s3" {
return errors.ErrInvalidS3ARN
}
if v, ok := u.Query()["versionId"]; ok {
if len(v) == 0 || v[0] == "" {
return errors.ErrInvalidS3ObjectVersionId
}
}
return nil
case "data":
if _, err := dataurl.DecodeString(s); err != nil {
return err
Expand Down
32 changes: 32 additions & 0 deletions config/v3_4_experimental/types/url_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,38 @@ func TestURLValidate(t *testing.T) {
util.StrToPtr("s3://bucket/key?versionId=aVersionHash"),
nil,
},
{
util.StrToPtr("Arn:"),
errors.ErrInvalidS3ARN,
},
{
util.StrToPtr("arn:aws:iam:us-west-2:123456789012:resource"),
errors.ErrInvalidS3ARN,
},
{
util.StrToPtr("arn:aws:s3:us-west-2:123456789012:bucket-name/object-key"),
nil,
},
{
util.StrToPtr("arn:aws:s3:us-west-2:123456789012:bucket-name/object-key?versionId=aVersionHash"),
nil,
},
{
util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/test/object"),
nil,
},
{
util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/test/object?versionId=aVersionHash"),
nil,
},
{
util.StrToPtr("arn:aws:s3:::bucket-name/object-key"),
nil,
},
{
util.StrToPtr("arn:aws:s3:::bucket-name/object-key?versionId=aVersionHash"),
nil,
},
{
util.StrToPtr("gs://bucket/object"),
nil,
Expand Down
12 changes: 6 additions & 6 deletions docs/configuration-v3_4_experimental.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ The Ignition configuration is a JSON document conforming to the following specif
* **version** (string): the semantic version number of the spec. The spec version must be compatible with the latest version (`3.4.0-experimental`). Compatibility requires the major versions to match and the spec version be less than or equal to the latest version. `-experimental` versions compare less than the final version with the same number, and previous experimental versions are not accepted.
* **_config_** (objects): options related to the configuration.
* **_merge_** (list of objects): a list of the configs to be merged to the current config.
* **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **_compression_** (string): the type of compression used on the config (null or gzip). Compression cannot be used with S3.
* **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
* **name** (string): the header name.
* **_value_** (string): the header contents.
* **_verification_** (object): options related to the verification of the config.
* **_hash_** (string): the hash of the config, in the form `<type>-<value>` where type is either `sha512` or `sha256`.
* **_replace_** (object): the config that will replace the current.
* **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **_compression_** (string): the type of compression used on the config (null or gzip). Compression cannot be used with S3.
* **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
* **name** (string): the header name.
Expand All @@ -35,7 +35,7 @@ The Ignition configuration is a JSON document conforming to the following specif
* **_security_** (object): options relating to network security.
* **_tls_** (object): options relating to TLS when fetching resources over `https`.
* **_certificateAuthorities_** (list of objects): the list of additional certificate authorities (in addition to the system authorities) to be used for TLS verification when fetching over `https`. All certificate authorities must have a unique `source`.
* **source** (string): the URL of the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **source** (string): the URL of the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **_compression_** (string): the type of compression used on the certificate (null or gzip). Compression cannot be used with S3.
* **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
* **name** (string): the header name.
Expand Down Expand Up @@ -80,15 +80,15 @@ The Ignition configuration is a JSON document conforming to the following specif
* **_overwrite_** (boolean): whether to delete preexisting nodes at the path. `contents.source` must be specified if `overwrite` is true. Defaults to false.
* **_contents_** (object): options related to the contents of the file.
* **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3.
* **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created.
* **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created.
* **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
* **name** (string): the header name.
* **_value_** (string): the header contents.
* **_verification_** (object): options related to the verification of the file contents.
* **_hash_** (string): the hash of the contents, in the form `<type>-<value>` where type is either `sha512` or `sha256`.
* **_append_** (list of objects): list of contents to be appended to the file. Follows the same stucture as `contents`
* **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3.
* **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
* **name** (string): the header name.
* **_value_** (string): the header contents.
Expand Down Expand Up @@ -127,7 +127,7 @@ The Ignition configuration is a JSON document conforming to the following specif
* **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks.
* **_keyFile_** (string): options related to the contents of the key file.
* **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3.
* **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
* **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
* **name** (string): the header name.
* **_value_** (string): the header contents.
Expand Down
2 changes: 1 addition & 1 deletion docs/supported-platforms.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Ignition is currently only supported for the following platforms:
* [Exoscale] (`exoscale`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately.
* [Google Cloud] (`gcp`) - Ignition will read its configuration from the instance metadata entry named "user-data". Cloud SSH keys are handled separately.
* [IBM Cloud] (`ibmcloud`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately.
* Bare Metal (`metal`) - Use the `ignition.config.url` kernel parameter to provide a URL to the configuration. The URL can use the `http://`, `https://`, `tftp://`, `s3://`, or `gs://` schemes to specify a remote config.
* Bare Metal (`metal`) - Use the `ignition.config.url` kernel parameter to provide a URL to the configuration. The URL can use the `http://`, `https://`, `tftp://`, `s3://`, `arn:`, or `gs://` schemes to specify a remote config.
* [Nutanix] (`nutanix`) - Ignition will read its configuration from the instance userdata via config drive. Cloud SSH keys are handled separately.
* [OpenStack] (`openstack`) - Ignition will read its configuration from the instance userdata via either metadata service or config drive. Cloud SSH keys are handled separately.
* [Equinix Metal] (`packet`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately.
Expand Down
87 changes: 74 additions & 13 deletions internal/resource/url.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"google.golang.org/api/option"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
Expand Down Expand Up @@ -135,7 +136,7 @@ func (f *Fetcher) FetchToBuffer(u url.URL, opts FetchOptions) ([]byte, error) {
err = f.fetchFromTFTP(u, dest, opts)
case "data":
err = f.fetchFromDataURL(u, dest, opts)
case "s3":
case "s3", "arn":
buf := &s3buf{
WriteAtBuffer: aws.NewWriteAtBuffer([]byte{}),
}
Expand Down Expand Up @@ -196,7 +197,7 @@ func (f *Fetcher) Fetch(u url.URL, dest *os.File, opts FetchOptions) error {
return f.fetchFromTFTP(u, dest, opts)
case "data":
return f.fetchFromDataURL(u, dest, opts)
case "s3":
case "s3", "arn":
return f.fetchFromS3(u, dest, opts)
case "gs":
return f.fetchFromGCS(u, dest, opts)
Expand Down Expand Up @@ -407,17 +408,39 @@ func (f *Fetcher) fetchFromS3(u url.URL, dest s3target, opts FetchOptions) error
}
sess := f.AWSSession.Copy()

// Determine the partition and region this bucket is in
regionHint := "us-east-1"
if f.S3RegionHint != "" {
regionHint = f.S3RegionHint
// Determine the bucket and key based on the URL scheme
var bucket, key, region string
var err error
switch u.Scheme {
case "s3":
bucket = u.Host
key = u.Path
case "arn":
fullURL := u.Scheme + ":" + u.Opaque
// Parse the bucket and key from the ARN Resource.
// Also set the region for accesspoints.
// S3 bucket ARNs don't include the region field.
bucket, key, region, err = f.parseARN(fullURL)
if err != nil {
return err
}
default:
return ErrSchemeUnsupported
}
region, err := s3manager.GetBucketRegion(ctx, sess, u.Host, regionHint)
if err != nil {
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
return fmt.Errorf("couldn't determine the region for bucket %q: %v", u.Host, err)

// Determine the partition and region this bucket is in
if region == "" {
regionHint := "us-east-1"
if f.S3RegionHint != "" {
regionHint = f.S3RegionHint
}
region, err = s3manager.GetBucketRegion(ctx, sess, bucket, regionHint)
if err != nil {
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
return fmt.Errorf("couldn't determine the region for bucket %q: %v", u.Host, err)
}
return err
}
return err
}

sess.Config.Region = aws.String(region)
Expand All @@ -428,8 +451,8 @@ func (f *Fetcher) fetchFromS3(u url.URL, dest s3target, opts FetchOptions) error
}

input := &s3.GetObjectInput{
Bucket: &u.Host,
Key: &u.Path,
Bucket: &bucket,
Key: &key,
VersionId: versionId,
}
err = f.fetchFromS3WithCreds(ctx, dest, input, sess)
Expand Down Expand Up @@ -522,3 +545,41 @@ func (f *Fetcher) decompressCopyHashAndVerify(dest io.Writer, src io.Reader, opt
}
return nil
}

// parseARN is a custom wrapper around arn.Parse(); it takes arnURL, a full ARN URL,
// and returns a bucket, a key, a potentially empty region, or an error if the ARN
// is invalid or not for an S3 object.
// If the given arnURL is an accesspoint ARN, the region is set.
// The region is empty for S3 bucket ARNs because they don't include the region field.
func (f *Fetcher) parseARN(arnURL string) (string, string, string, error) {
if !arn.IsARN(arnURL) {
return "", "", "", configErrors.ErrInvalidS3ARN
}
s3arn, err := arn.Parse(arnURL)
if err != nil {
return "", "", "", err
}
if s3arn.Service != "s3" {
return "", "", "", configErrors.ErrInvalidS3ARN
}

// Determine if the ARN is for an access point or a bucket.
var bucket, key string
urlSplit := strings.Split(arnURL, "/")
if strings.HasPrefix(s3arn.Resource, "accesspoint/") {
// When using GetObjectInput with an access point,
// you provide the access point ARN in place of the bucket name.
// For more information about access point ARNs, see Using access points
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html
bucket = strings.Join(urlSplit[:2], "/")
key = strings.Join(urlSplit[2:], "/")
return bucket, key, s3arn.Region, nil
}
// Parse out the bucket name in order to find the region with s3manager.GetBucketRegion.
// If specified, the key is part of the Relative ID which has the format "bucket-name/object-key" according to
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html
bucketUrlSplit := strings.Split(urlSplit[0], ":")
bucket = bucketUrlSplit[len(bucketUrlSplit)-1]
key = strings.Join(urlSplit[1:], "/")
return bucket, key, "", nil
}
88 changes: 88 additions & 0 deletions internal/resource/url_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,20 @@ func TestFetchOffline(t *testing.T) {
},
out: out{err: ErrNeedNet},
},
// arn url specifying bucket
{
in: in{
url: "arn:aws:s3:::kola-fixtures/resources/anonymous",
},
out: out{err: ErrNeedNet},
},
// arn url specifying s3 access point
{
in: in{
url: "arn:aws:s3:us-west-2:123456789012:accesspoint/test/object",
},
out: out{err: ErrNeedNet},
},
// gs url
{
in: in{
Expand Down Expand Up @@ -227,3 +241,77 @@ func TestFetchOffline(t *testing.T) {
}
}
}

func TestParseARN(t *testing.T) {
type in struct {
url string
}
type out struct {
bucket string
key string
region string
err error
}
tests := []struct {
in in
out out
}{
{
in: in{
"arn:aws:iam:us-west-2:123456789012:resource",
},
out: out{
err: errors.ErrInvalidS3ARN,
},
},
{
in: in{
"arn:aws:s3:::kola-fixtures/resources/anonymous",
},
out: out{
bucket: "kola-fixtures", key: "resources/anonymous",
},
},
{
in: in{
"arn:aws:s3:us-west-2:123456789012:accesspoint/test/object",
},
out: out{
bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint/test", key: "object", region: "us-west-2",
},
},
{
in: in{
"arn:aws:s3:us-west-2:123456789012:accesspoint/test/path/object",
},
out: out{
bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint/test", key: "path/object", region: "us-west-2",
},
},
}

logger := log.New(true)
f := Fetcher{
Logger: &logger,
}

for i, test := range tests {
bucket, key, region, err := f.parseARN(test.in.url)
if !reflect.DeepEqual(test.out.err, err) {
t.Errorf("#%d: fetching URL: expected error %+v, got %+v", i, test.out.err, err)
continue
}
if test.out.err == nil && !reflect.DeepEqual(test.out.bucket, bucket) {
t.Errorf("#%d: expected output %+v, got %+v", i, test.out.bucket, bucket)
continue
}
if test.out.err == nil && !reflect.DeepEqual(test.out.key, key) {
t.Errorf("#%d: expected output %+v, got %+v", i, test.out.key, key)
continue
}
if test.out.err == nil && !reflect.DeepEqual(test.out.region, region) {
t.Errorf("#%d: expected output %+v, got %+v", i, test.out.region, region)
continue
}
}
}

0 comments on commit 961d70e

Please sign in to comment.