From bc7795100a1e7c0d535c5d519ec9f925f71fd4e5 Mon Sep 17 00:00:00 2001 From: Matt Boersma Date: Fri, 2 Dec 2016 13:32:43 -0700 Subject: [PATCH 1/2] fix(create_bucket): try default s3 region on create error --- rootfs/bin/create_bucket | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/rootfs/bin/create_bucket b/rootfs/bin/create_bucket index 139f77a..b600840 100755 --- a/rootfs/bin/create_bucket +++ b/rootfs/bin/create_bucket @@ -6,6 +6,7 @@ import boto.s3 import json import swiftclient from boto import config as botoconfig +from boto.exception import S3CreateError from boto.s3.connection import S3Connection, OrdinaryCallingFormat from oauth2client.service_account import ServiceAccountCredentials from gcloud.storage.client import Client @@ -23,9 +24,20 @@ region = os.getenv('AWS_REGION') if os.getenv('DATABASE_STORAGE') == "s3": conn = boto.s3.connect_to_region(region) - if not bucket_exists(conn, bucket_name): - conn.create_bucket(bucket_name, location=region) + try: + conn.create_bucket(bucket_name, location=region) + except S3CreateError as err: + # try again in the default S3 region before giving up + default_region = 'us-east-1' + if region != default_region: + print('Failed to create bucket in {}, trying default region {}...'.format( + region, default_region)) + conn = boto.s3.connect_to_region(default_region) + if not bucket_exists(conn, bucket_name): + conn.create_bucket(bucket_name, location=default_region) + # TODO: if we got here, somehow we need to propagate "us-east-1" + # into WALE_S3_ENDPOINT... elif os.getenv('DATABASE_STORAGE') == "gcs": scopes = ['https://www.googleapis.com/auth/devstorage.full_control'] From 8bf4c52fbc4bd2c54166f7d072578be11c4d90c7 Mon Sep 17 00:00:00 2001 From: Matthew Fisher Date: Fri, 2 Dec 2016 13:16:43 -0800 Subject: [PATCH 2/2] fix(create_bucket): propagate us-east-1 to wal-e --- rootfs/bin/create_bucket | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rootfs/bin/create_bucket b/rootfs/bin/create_bucket index b600840..8df8fca 100755 --- a/rootfs/bin/create_bucket +++ b/rootfs/bin/create_bucket @@ -27,17 +27,17 @@ if os.getenv('DATABASE_STORAGE') == "s3": if not bucket_exists(conn, bucket_name): try: conn.create_bucket(bucket_name, location=region) + # NOTE(bacongobbler): for versions prior to v2.9.0, the bucket is created in the default region. + # if we got here, we need to propagate "us-east-1" into WALE_S3_ENDPOINT because the bucket + # exists in a different region and we cannot find it. + # TODO(bacongobbler): deprecate this once we drop support for v2.8.0 and lower except S3CreateError as err: - # try again in the default S3 region before giving up - default_region = 'us-east-1' - if region != default_region: - print('Failed to create bucket in {}, trying default region {}...'.format( - region, default_region)) - conn = boto.s3.connect_to_region(default_region) - if not bucket_exists(conn, bucket_name): - conn.create_bucket(bucket_name, location=default_region) - # TODO: if we got here, somehow we need to propagate "us-east-1" - # into WALE_S3_ENDPOINT... + if region != 'us-east-1': + print('Failed to create bucket in {}. We are now assuming that the bucket was created in us-east-1.'.format(region)) + with open(os.path.join(os.environ['WALE_ENVDIR'], "WALE_S3_ENDPOINT"), "w+") as file: + file.write('https+path://s3.amazonaws.com:443') + else: + raise elif os.getenv('DATABASE_STORAGE') == "gcs": scopes = ['https://www.googleapis.com/auth/devstorage.full_control']