diff --git a/README.md b/README.md index 5a1d7f1f1bd4..4e39ada35d16 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L | Cognito Identity Provider | @mock_cognitoidp | basic endpoints done | |-------------------------------------------------------------------------------------| | Config | @mock_config | basic endpoints done | +| | | core endpoints done | |-------------------------------------------------------------------------------------| | Data Pipeline | @mock_datapipeline | basic endpoints done | |-------------------------------------------------------------------------------------| @@ -296,6 +297,96 @@ def test_describe_instances_allowed(): See [the related test suite](https://github.com/spulec/moto/blob/master/tests/test_core/test_auth.py) for more examples. +## Very Important -- Recommended Usage +There are some important caveats to be aware of when using moto: + +*Failure to follow these guidelines could result in your tests mutating your __REAL__ infrastructure!* + +### How do I avoid tests from mutating my real infrastructure? +You need to ensure that the mocks are actually in place. Changes made to recent versions of `botocore` +have altered some of the mock behavior. In short, you need to ensure that you _always_ do the following: + +1. Ensure that your tests have dummy environment variables set up: + + export AWS_ACCESS_KEY_ID='testing' + export AWS_SECRET_ACCESS_KEY='testing' + export AWS_SECURITY_TOKEN='testing' + export AWS_SESSION_TOKEN='testing' + +1. __VERY IMPORTANT__: ensure that you have your mocks set up __BEFORE__ your `boto3` client is established. + This can typically happen if you import a module that has a `boto3` client instantiated outside of a function. + See the pesky imports section below on how to work around this. + +### Example on usage? +If you are a user of [pytest](https://pytest.org/en/latest/), you can leverage [pytest fixtures](https://pytest.org/en/latest/fixture.html#fixture) +to help set up your mocks and other AWS resources that you would need. + +Here is an example: +```python +@pytest.fixture(scope='function') +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ['AWS_ACCESS_KEY_ID'] = 'testing' + os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' + os.environ['AWS_SECURITY_TOKEN'] = 'testing' + os.environ['AWS_SESSION_TOKEN'] = 'testing' + +@pytest.fixture(scope='function') +def s3(aws_credentials): + with mock_s3(): + yield boto3.client('s3', region_name='us-east-1') + + +@pytest.fixture(scope='function') +def sts(aws_credentials): + with mock_sts(): + yield boto3.client('sts', region_name='us-east-1') + + +@pytest.fixture(scope='function') +def cloudwatch(aws_credentials): + with mock_cloudwatch(): + yield boto3.client('cloudwatch', region_name='us-east-1') + +... etc. +``` + +In the code sample above, all of the AWS/mocked fixtures take in a parameter of `aws_credentials`, +which sets the proper fake environment variables. The fake environment variables are used so that `botocore` doesn't try to locate real +credentials on your system. + +Next, once you need to do anything with the mocked AWS environment, do something like: +```python +def test_create_bucket(s3): + # s3 is a fixture defined above that yields a boto3 s3 client. + # Feel free to instantiate another boto3 S3 client -- Keep note of the region though. + s3.create_bucket(Bucket="somebucket") + + result = s3.list_buckets() + assert len(result['Buckets']) == 1 + assert result['Buckets'][0]['Name'] == 'somebucket' +``` + +### What about those pesky imports? +Recall earlier, it was mentioned that mocks should be established __BEFORE__ the clients are set up. One way +to avoid import issues is to make use of local Python imports -- i.e. import the module inside of the unit +test you want to run vs. importing at the top of the file. + +Example: +```python +def test_something(s3): + from some.package.that.does.something.with.s3 import some_func # <-- Local import for unit test + # ^^ Importing here ensures that the mock has been established. + + sume_func() # The mock has been established from the "s3" pytest fixture, so this function that uses + # a package-level S3 client will properly use the mock and not reach out to AWS. +``` + +### Other caveats +For Tox, Travis CI, and other build systems, you might need to also perform a `touch ~/.aws/credentials` +command before running the tests. As long as that file is present (empty preferably) and the environment +variables above are set, you should be good to go. + ## Stand-alone Server Mode Moto also has a stand-alone server mode. This allows you to utilize diff --git a/moto/config/exceptions.py b/moto/config/exceptions.py index b2b01d6a0ff8..25749200f246 100644 --- a/moto/config/exceptions.py +++ b/moto/config/exceptions.py @@ -52,6 +52,18 @@ def __init__(self, bad_list, good_list): super(InvalidResourceTypeException, self).__init__("ValidationException", message) +class NoSuchConfigurationAggregatorException(JsonRESTError): + code = 400 + + def __init__(self, number=1): + if number == 1: + message = 'The configuration aggregator does not exist. Check the configuration aggregator name and try again.' + else: + message = 'At least one of the configuration aggregators does not exist. Check the configuration aggregator' \ + ' names and try again.' + super(NoSuchConfigurationAggregatorException, self).__init__("NoSuchConfigurationAggregatorException", message) + + class NoSuchConfigurationRecorderException(JsonRESTError): code = 400 @@ -78,6 +90,14 @@ def __init__(self): super(NoSuchBucketException, self).__init__("NoSuchBucketException", message) +class InvalidNextTokenException(JsonRESTError): + code = 400 + + def __init__(self): + message = 'The nextToken provided is invalid' + super(InvalidNextTokenException, self).__init__("InvalidNextTokenException", message) + + class InvalidS3KeyPrefixException(JsonRESTError): code = 400 @@ -147,3 +167,66 @@ def __init__(self, name): message = 'Failed to delete last specified delivery channel with name \'{name}\', because there, ' \ 'because there is a running configuration recorder.'.format(name=name) super(LastDeliveryChannelDeleteFailedException, self).__init__("LastDeliveryChannelDeleteFailedException", message) + + +class TooManyAccountSources(JsonRESTError): + code = 400 + + def __init__(self, length): + locations = ['com.amazonaws.xyz'] * length + + message = 'Value \'[{locations}]\' at \'accountAggregationSources\' failed to satisfy constraint: ' \ + 'Member must have length less than or equal to 1'.format(locations=', '.join(locations)) + super(TooManyAccountSources, self).__init__("ValidationException", message) + + +class DuplicateTags(JsonRESTError): + code = 400 + + def __init__(self): + super(DuplicateTags, self).__init__( + 'InvalidInput', 'Duplicate tag keys found. Please note that Tag keys are case insensitive.') + + +class TagKeyTooBig(JsonRESTError): + code = 400 + + def __init__(self, tag, param='tags.X.member.key'): + super(TagKeyTooBig, self).__init__( + 'ValidationException', "1 validation error detected: Value '{}' at '{}' failed to satisfy " + "constraint: Member must have length less than or equal to 128".format(tag, param)) + + +class TagValueTooBig(JsonRESTError): + code = 400 + + def __init__(self, tag): + super(TagValueTooBig, self).__init__( + 'ValidationException', "1 validation error detected: Value '{}' at 'tags.X.member.value' failed to satisfy " + "constraint: Member must have length less than or equal to 256".format(tag)) + + +class InvalidParameterValueException(JsonRESTError): + code = 400 + + def __init__(self, message): + super(InvalidParameterValueException, self).__init__('InvalidParameterValueException', message) + + +class InvalidTagCharacters(JsonRESTError): + code = 400 + + def __init__(self, tag, param='tags.X.member.key'): + message = "1 validation error detected: Value '{}' at '{}' failed to satisfy ".format(tag, param) + message += 'constraint: Member must satisfy regular expression pattern: [\\\\p{L}\\\\p{Z}\\\\p{N}_.:/=+\\\\-@]+' + + super(InvalidTagCharacters, self).__init__('ValidationException', message) + + +class TooManyTags(JsonRESTError): + code = 400 + + def __init__(self, tags, param='tags'): + super(TooManyTags, self).__init__( + 'ValidationException', "1 validation error detected: Value '{}' at '{}' failed to satisfy " + "constraint: Member must have length less than or equal to 50.".format(tags, param)) diff --git a/moto/config/models.py b/moto/config/models.py index cd6e07afa14f..6541fc981ee8 100644 --- a/moto/config/models.py +++ b/moto/config/models.py @@ -1,6 +1,9 @@ import json +import re import time import pkg_resources +import random +import string from datetime import datetime @@ -12,37 +15,125 @@ NoSuchConfigurationRecorderException, NoAvailableConfigurationRecorderException, \ InvalidDeliveryChannelNameException, NoSuchBucketException, InvalidS3KeyPrefixException, \ InvalidSNSTopicARNException, MaxNumberOfDeliveryChannelsExceededException, NoAvailableDeliveryChannelException, \ - NoSuchDeliveryChannelException, LastDeliveryChannelDeleteFailedException + NoSuchDeliveryChannelException, LastDeliveryChannelDeleteFailedException, TagKeyTooBig, \ + TooManyTags, TagValueTooBig, TooManyAccountSources, InvalidParameterValueException, InvalidNextTokenException, \ + NoSuchConfigurationAggregatorException, InvalidTagCharacters, DuplicateTags from moto.core import BaseBackend, BaseModel DEFAULT_ACCOUNT_ID = 123456789012 +POP_STRINGS = [ + 'capitalizeStart', + 'CapitalizeStart', + 'capitalizeArn', + 'CapitalizeArn', + 'capitalizeARN', + 'CapitalizeARN' +] +DEFAULT_PAGE_SIZE = 100 def datetime2int(date): return int(time.mktime(date.timetuple())) -def snake_to_camels(original): +def snake_to_camels(original, cap_start, cap_arn): parts = original.split('_') camel_cased = parts[0].lower() + ''.join(p.title() for p in parts[1:]) - camel_cased = camel_cased.replace('Arn', 'ARN') # Config uses 'ARN' instead of 'Arn' + + if cap_arn: + camel_cased = camel_cased.replace('Arn', 'ARN') # Some config services use 'ARN' instead of 'Arn' + + if cap_start: + camel_cased = camel_cased[0].upper() + camel_cased[1::] return camel_cased +def random_string(): + """Returns a random set of 8 lowercase letters for the Config Aggregator ARN""" + chars = [] + for x in range(0, 8): + chars.append(random.choice(string.ascii_lowercase)) + + return "".join(chars) + + +def validate_tag_key(tag_key, exception_param='tags.X.member.key'): + """Validates the tag key. + + :param tag_key: The tag key to check against. + :param exception_param: The exception parameter to send over to help format the message. This is to reflect + the difference between the tag and untag APIs. + :return: + """ + # Validate that the key length is correct: + if len(tag_key) > 128: + raise TagKeyTooBig(tag_key, param=exception_param) + + # Validate that the tag key fits the proper Regex: + # [\w\s_.:/=+\-@]+ SHOULD be the same as the Java regex on the AWS documentation: [\p{L}\p{Z}\p{N}_.:/=+\-@]+ + match = re.findall(r'[\w\s_.:/=+\-@]+', tag_key) + # Kudos if you can come up with a better way of doing a global search :) + if not len(match) or len(match[0]) < len(tag_key): + raise InvalidTagCharacters(tag_key, param=exception_param) + + +def check_tag_duplicate(all_tags, tag_key): + """Validates that a tag key is not a duplicate + + :param all_tags: Dict to check if there is a duplicate tag. + :param tag_key: The tag key to check against. + :return: + """ + if all_tags.get(tag_key): + raise DuplicateTags() + + +def validate_tags(tags): + proper_tags = {} + + if len(tags) > 50: + raise TooManyTags(tags) + + for tag in tags: + # Validate the Key: + validate_tag_key(tag['Key']) + check_tag_duplicate(proper_tags, tag['Key']) + + # Validate the Value: + if len(tag['Value']) > 256: + raise TagValueTooBig(tag['Value']) + + proper_tags[tag['Key']] = tag['Value'] + + return proper_tags + + class ConfigEmptyDictable(BaseModel): """Base class to make serialization easy. This assumes that the sub-class will NOT return 'None's in the JSON.""" + def __init__(self, capitalize_start=False, capitalize_arn=True): + """Assists with the serialization of the config object + :param capitalize_start: For some Config services, the first letter is lowercase -- for others it's capital + :param capitalize_arn: For some Config services, the API expects 'ARN' and for others, it expects 'Arn' + """ + self.capitalize_start = capitalize_start + self.capitalize_arn = capitalize_arn + def to_dict(self): data = {} for item, value in self.__dict__.items(): if value is not None: if isinstance(value, ConfigEmptyDictable): - data[snake_to_camels(item)] = value.to_dict() + data[snake_to_camels(item, self.capitalize_start, self.capitalize_arn)] = value.to_dict() else: - data[snake_to_camels(item)] = value + data[snake_to_camels(item, self.capitalize_start, self.capitalize_arn)] = value + + # Cleanse the extra properties: + for prop in POP_STRINGS: + data.pop(prop, None) return data @@ -50,8 +141,9 @@ def to_dict(self): class ConfigRecorderStatus(ConfigEmptyDictable): def __init__(self, name): - self.name = name + super(ConfigRecorderStatus, self).__init__() + self.name = name self.recording = False self.last_start_time = None self.last_stop_time = None @@ -75,12 +167,16 @@ def stop(self): class ConfigDeliverySnapshotProperties(ConfigEmptyDictable): def __init__(self, delivery_frequency): + super(ConfigDeliverySnapshotProperties, self).__init__() + self.delivery_frequency = delivery_frequency class ConfigDeliveryChannel(ConfigEmptyDictable): def __init__(self, name, s3_bucket_name, prefix=None, sns_arn=None, snapshot_properties=None): + super(ConfigDeliveryChannel, self).__init__() + self.name = name self.s3_bucket_name = s3_bucket_name self.s3_key_prefix = prefix @@ -91,6 +187,8 @@ def __init__(self, name, s3_bucket_name, prefix=None, sns_arn=None, snapshot_pro class RecordingGroup(ConfigEmptyDictable): def __init__(self, all_supported=True, include_global_resource_types=False, resource_types=None): + super(RecordingGroup, self).__init__() + self.all_supported = all_supported self.include_global_resource_types = include_global_resource_types self.resource_types = resource_types @@ -99,6 +197,8 @@ def __init__(self, all_supported=True, include_global_resource_types=False, reso class ConfigRecorder(ConfigEmptyDictable): def __init__(self, role_arn, recording_group, name='default', status=None): + super(ConfigRecorder, self).__init__() + self.name = name self.role_arn = role_arn self.recording_group = recording_group @@ -109,18 +209,118 @@ def __init__(self, role_arn, recording_group, name='default', status=None): self.status = status +class AccountAggregatorSource(ConfigEmptyDictable): + + def __init__(self, account_ids, aws_regions=None, all_aws_regions=None): + super(AccountAggregatorSource, self).__init__(capitalize_start=True) + + # Can't have both the regions and all_regions flag present -- also can't have them both missing: + if aws_regions and all_aws_regions: + raise InvalidParameterValueException('Your configuration aggregator contains a list of regions and also specifies ' + 'the use of all regions. You must choose one of these options.') + + if not (aws_regions or all_aws_regions): + raise InvalidParameterValueException('Your request does not specify any regions. Select AWS Config-supported ' + 'regions and try again.') + + self.account_ids = account_ids + self.aws_regions = aws_regions + + if not all_aws_regions: + all_aws_regions = False + + self.all_aws_regions = all_aws_regions + + +class OrganizationAggregationSource(ConfigEmptyDictable): + + def __init__(self, role_arn, aws_regions=None, all_aws_regions=None): + super(OrganizationAggregationSource, self).__init__(capitalize_start=True, capitalize_arn=False) + + # Can't have both the regions and all_regions flag present -- also can't have them both missing: + if aws_regions and all_aws_regions: + raise InvalidParameterValueException('Your configuration aggregator contains a list of regions and also specifies ' + 'the use of all regions. You must choose one of these options.') + + if not (aws_regions or all_aws_regions): + raise InvalidParameterValueException('Your request does not specify any regions. Select AWS Config-supported ' + 'regions and try again.') + + self.role_arn = role_arn + self.aws_regions = aws_regions + + if not all_aws_regions: + all_aws_regions = False + + self.all_aws_regions = all_aws_regions + + +class ConfigAggregator(ConfigEmptyDictable): + + def __init__(self, name, region, account_sources=None, org_source=None, tags=None): + super(ConfigAggregator, self).__init__(capitalize_start=True, capitalize_arn=False) + + self.configuration_aggregator_name = name + self.configuration_aggregator_arn = 'arn:aws:config:{region}:{id}:config-aggregator/config-aggregator-{random}'.format( + region=region, + id=DEFAULT_ACCOUNT_ID, + random=random_string() + ) + self.account_aggregation_sources = account_sources + self.organization_aggregation_source = org_source + self.creation_time = datetime2int(datetime.utcnow()) + self.last_updated_time = datetime2int(datetime.utcnow()) + + # Tags are listed in the list_tags_for_resource API call ... not implementing yet -- please feel free to! + self.tags = tags or {} + + # Override the to_dict so that we can format the tags properly... + def to_dict(self): + result = super(ConfigAggregator, self).to_dict() + + # Override the account aggregation sources if present: + if self.account_aggregation_sources: + result['AccountAggregationSources'] = [a.to_dict() for a in self.account_aggregation_sources] + + # Tags are listed in the list_tags_for_resource API call ... not implementing yet -- please feel free to! + # if self.tags: + # result['Tags'] = [{'Key': key, 'Value': value} for key, value in self.tags.items()] + + return result + + +class ConfigAggregationAuthorization(ConfigEmptyDictable): + + def __init__(self, current_region, authorized_account_id, authorized_aws_region, tags=None): + super(ConfigAggregationAuthorization, self).__init__(capitalize_start=True, capitalize_arn=False) + + self.aggregation_authorization_arn = 'arn:aws:config:{region}:{id}:aggregation-authorization/' \ + '{auth_account}/{auth_region}'.format(region=current_region, + id=DEFAULT_ACCOUNT_ID, + auth_account=authorized_account_id, + auth_region=authorized_aws_region) + self.authorized_account_id = authorized_account_id + self.authorized_aws_region = authorized_aws_region + self.creation_time = datetime2int(datetime.utcnow()) + + # Tags are listed in the list_tags_for_resource API call ... not implementing yet -- please feel free to! + self.tags = tags or {} + + class ConfigBackend(BaseBackend): def __init__(self): self.recorders = {} self.delivery_channels = {} + self.config_aggregators = {} + self.aggregation_authorizations = {} @staticmethod def _validate_resource_types(resource_list): # Load the service file: resource_package = 'botocore' resource_path = '/'.join(('data', 'config', '2014-11-12', 'service-2.json')) - conifg_schema = json.loads(pkg_resources.resource_string(resource_package, resource_path)) + config_schema = json.loads(pkg_resources.resource_string(resource_package, resource_path)) # Verify that each entry exists in the supported list: bad_list = [] @@ -128,11 +328,11 @@ def _validate_resource_types(resource_list): # For PY2: r_str = str(resource) - if r_str not in conifg_schema['shapes']['ResourceType']['enum']: + if r_str not in config_schema['shapes']['ResourceType']['enum']: bad_list.append(r_str) if bad_list: - raise InvalidResourceTypeException(bad_list, conifg_schema['shapes']['ResourceType']['enum']) + raise InvalidResourceTypeException(bad_list, config_schema['shapes']['ResourceType']['enum']) @staticmethod def _validate_delivery_snapshot_properties(properties): @@ -147,6 +347,158 @@ def _validate_delivery_snapshot_properties(properties): raise InvalidDeliveryFrequency(properties.get('deliveryFrequency', None), conifg_schema['shapes']['MaximumExecutionFrequency']['enum']) + def put_configuration_aggregator(self, config_aggregator, region): + # Validate the name: + if len(config_aggregator['ConfigurationAggregatorName']) > 256: + raise NameTooLongException(config_aggregator['ConfigurationAggregatorName'], 'configurationAggregatorName') + + account_sources = None + org_source = None + + # Tag validation: + tags = validate_tags(config_aggregator.get('Tags', [])) + + # Exception if both AccountAggregationSources and OrganizationAggregationSource are supplied: + if config_aggregator.get('AccountAggregationSources') and config_aggregator.get('OrganizationAggregationSource'): + raise InvalidParameterValueException('The configuration aggregator cannot be created because your request contains both the' + ' AccountAggregationSource and the OrganizationAggregationSource. Include only ' + 'one aggregation source and try again.') + + # If neither are supplied: + if not config_aggregator.get('AccountAggregationSources') and not config_aggregator.get('OrganizationAggregationSource'): + raise InvalidParameterValueException('The configuration aggregator cannot be created because your request is missing either ' + 'the AccountAggregationSource or the OrganizationAggregationSource. Include the ' + 'appropriate aggregation source and try again.') + + if config_aggregator.get('AccountAggregationSources'): + # Currently, only 1 account aggregation source can be set: + if len(config_aggregator['AccountAggregationSources']) > 1: + raise TooManyAccountSources(len(config_aggregator['AccountAggregationSources'])) + + account_sources = [] + for a in config_aggregator['AccountAggregationSources']: + account_sources.append(AccountAggregatorSource(a['AccountIds'], aws_regions=a.get('AwsRegions'), + all_aws_regions=a.get('AllAwsRegions'))) + + else: + org_source = OrganizationAggregationSource(config_aggregator['OrganizationAggregationSource']['RoleArn'], + aws_regions=config_aggregator['OrganizationAggregationSource'].get('AwsRegions'), + all_aws_regions=config_aggregator['OrganizationAggregationSource'].get( + 'AllAwsRegions')) + + # Grab the existing one if it exists and update it: + if not self.config_aggregators.get(config_aggregator['ConfigurationAggregatorName']): + aggregator = ConfigAggregator(config_aggregator['ConfigurationAggregatorName'], region, account_sources=account_sources, + org_source=org_source, tags=tags) + self.config_aggregators[config_aggregator['ConfigurationAggregatorName']] = aggregator + + else: + aggregator = self.config_aggregators[config_aggregator['ConfigurationAggregatorName']] + aggregator.tags = tags + aggregator.account_aggregation_sources = account_sources + aggregator.organization_aggregation_source = org_source + aggregator.last_updated_time = datetime2int(datetime.utcnow()) + + return aggregator.to_dict() + + def describe_configuration_aggregators(self, names, token, limit): + limit = DEFAULT_PAGE_SIZE if not limit or limit < 0 else limit + agg_list = [] + result = {'ConfigurationAggregators': []} + + if names: + for name in names: + if not self.config_aggregators.get(name): + raise NoSuchConfigurationAggregatorException(number=len(names)) + + agg_list.append(name) + + else: + agg_list = list(self.config_aggregators.keys()) + + # Empty? + if not agg_list: + return result + + # Sort by name: + sorted_aggregators = sorted(agg_list) + + # Get the start: + if not token: + start = 0 + else: + # Tokens for this moto feature are just the next names of the items in the list: + if not self.config_aggregators.get(token): + raise InvalidNextTokenException() + + start = sorted_aggregators.index(token) + + # Get the list of items to collect: + agg_list = sorted_aggregators[start:(start + limit)] + result['ConfigurationAggregators'] = [self.config_aggregators[agg].to_dict() for agg in agg_list] + + if len(sorted_aggregators) > (start + limit): + result['NextToken'] = sorted_aggregators[start + limit] + + return result + + def delete_configuration_aggregator(self, config_aggregator): + if not self.config_aggregators.get(config_aggregator): + raise NoSuchConfigurationAggregatorException() + + del self.config_aggregators[config_aggregator] + + def put_aggregation_authorization(self, current_region, authorized_account, authorized_region, tags): + # Tag validation: + tags = validate_tags(tags or []) + + # Does this already exist? + key = '{}/{}'.format(authorized_account, authorized_region) + agg_auth = self.aggregation_authorizations.get(key) + if not agg_auth: + agg_auth = ConfigAggregationAuthorization(current_region, authorized_account, authorized_region, tags=tags) + self.aggregation_authorizations['{}/{}'.format(authorized_account, authorized_region)] = agg_auth + else: + # Only update the tags: + agg_auth.tags = tags + + return agg_auth.to_dict() + + def describe_aggregation_authorizations(self, token, limit): + limit = DEFAULT_PAGE_SIZE if not limit or limit < 0 else limit + result = {'AggregationAuthorizations': []} + + if not self.aggregation_authorizations: + return result + + # Sort by name: + sorted_authorizations = sorted(self.aggregation_authorizations.keys()) + + # Get the start: + if not token: + start = 0 + else: + # Tokens for this moto feature are just the next names of the items in the list: + if not self.aggregation_authorizations.get(token): + raise InvalidNextTokenException() + + start = sorted_authorizations.index(token) + + # Get the list of items to collect: + auth_list = sorted_authorizations[start:(start + limit)] + result['AggregationAuthorizations'] = [self.aggregation_authorizations[auth].to_dict() for auth in auth_list] + + if len(sorted_authorizations) > (start + limit): + result['NextToken'] = sorted_authorizations[start + limit] + + return result + + def delete_aggregation_authorization(self, authorized_account, authorized_region): + # This will always return a 200 -- regardless if there is or isn't an existing + # aggregation authorization. + key = '{}/{}'.format(authorized_account, authorized_region) + self.aggregation_authorizations.pop(key, None) + def put_configuration_recorder(self, config_recorder): # Validate the name: if not config_recorder.get('name'): diff --git a/moto/config/responses.py b/moto/config/responses.py index 286b2349f171..03612d403c89 100644 --- a/moto/config/responses.py +++ b/moto/config/responses.py @@ -13,6 +13,39 @@ def put_configuration_recorder(self): self.config_backend.put_configuration_recorder(self._get_param('ConfigurationRecorder')) return "" + def put_configuration_aggregator(self): + aggregator = self.config_backend.put_configuration_aggregator(json.loads(self.body), self.region) + schema = {'ConfigurationAggregator': aggregator} + return json.dumps(schema) + + def describe_configuration_aggregators(self): + aggregators = self.config_backend.describe_configuration_aggregators(self._get_param('ConfigurationAggregatorNames'), + self._get_param('NextToken'), + self._get_param('Limit')) + return json.dumps(aggregators) + + def delete_configuration_aggregator(self): + self.config_backend.delete_configuration_aggregator(self._get_param('ConfigurationAggregatorName')) + return "" + + def put_aggregation_authorization(self): + agg_auth = self.config_backend.put_aggregation_authorization(self.region, + self._get_param('AuthorizedAccountId'), + self._get_param('AuthorizedAwsRegion'), + self._get_param('Tags')) + schema = {'AggregationAuthorization': agg_auth} + return json.dumps(schema) + + def describe_aggregation_authorizations(self): + authorizations = self.config_backend.describe_aggregation_authorizations(self._get_param('NextToken'), self._get_param('Limit')) + + return json.dumps(authorizations) + + def delete_aggregation_authorization(self): + self.config_backend.delete_aggregation_authorization(self._get_param('AuthorizedAccountId'), self._get_param('AuthorizedAwsRegion')) + + return "" + def describe_configuration_recorders(self): recorders = self.config_backend.describe_configuration_recorders(self._get_param('ConfigurationRecorderNames')) schema = {'ConfigurationRecorders': recorders} diff --git a/moto/iam/models.py b/moto/iam/models.py index bb19b8cade9d..21bb87e0226d 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -694,7 +694,6 @@ def list_role_policies(self, role_name): def _validate_tag_key(self, tag_key, exception_param='tags.X.member.key'): """Validates the tag key. - :param all_tags: Dict to check if there is a duplicate tag. :param tag_key: The tag key to check against. :param exception_param: The exception parameter to send over to help format the message. This is to reflect the difference between the tag and untag APIs. diff --git a/setup.py b/setup.py index 17a4f6691fe9..ff4d9720affa 100755 --- a/setup.py +++ b/setup.py @@ -30,8 +30,8 @@ def get_version(): install_requires = [ "Jinja2>=2.10.1", "boto>=2.36.0", - "boto3>=1.9.86", - "botocore>=1.12.86", + "boto3>=1.9.201", + "botocore>=1.12.201", "cryptography>=2.3.0", "requests>=2.5", "xmltodict", diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 96c62455c827..95e88cab1926 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -123,6 +123,526 @@ def test_put_configuration_recorder(): assert "maximum number of configuration recorders: 1 is reached." in ce.exception.response['Error']['Message'] +@mock_config +def test_put_configuration_aggregator(): + client = boto3.client('config', region_name='us-west-2') + + # With too many aggregation sources: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ] + }, + { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ] + } + ] + ) + assert 'Member must have length less than or equal to 1' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # With an invalid region config (no regions defined): + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AllAwsRegions': False + } + ] + ) + assert 'Your request does not specify any regions' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + OrganizationAggregationSource={ + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole' + } + ) + assert 'Your request does not specify any regions' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + # With both region flags defined: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ], + 'AllAwsRegions': True + } + ] + ) + assert 'You must choose one of these options' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + OrganizationAggregationSource={ + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole', + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ], + 'AllAwsRegions': True + } + ) + assert 'You must choose one of these options' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + # Name too long: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='a' * 257, + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ] + ) + assert 'configurationAggregatorName' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Too many tags (>50): + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': '{}'.format(x), 'Value': '{}'.format(x)} for x in range(0, 51)] + ) + assert 'Member must have length less than or equal to 50' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Tag key is too big (>128 chars): + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': 'a' * 129, 'Value': 'a'}] + ) + assert 'Member must have length less than or equal to 128' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Tag value is too big (>256 chars): + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': 'tag', 'Value': 'a' * 257}] + ) + assert 'Member must have length less than or equal to 256' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Duplicate Tags: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': 'a', 'Value': 'a'}, {'Key': 'a', 'Value': 'a'}] + ) + assert 'Duplicate tag keys found.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidInput' + + # Invalid characters in the tag key: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': '!', 'Value': 'a'}] + ) + assert 'Member must satisfy regular expression pattern:' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # If it contains both the AccountAggregationSources and the OrganizationAggregationSource + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': False + } + ], + OrganizationAggregationSource={ + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole', + 'AllAwsRegions': False + } + ) + assert 'AccountAggregationSource and the OrganizationAggregationSource' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + # If it contains neither: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + ) + assert 'AccountAggregationSource or the OrganizationAggregationSource' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + # Just make one: + account_aggregation_source = { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ], + 'AllAwsRegions': False + } + + result = client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[account_aggregation_source], + ) + assert result['ConfigurationAggregator']['ConfigurationAggregatorName'] == 'testing' + assert result['ConfigurationAggregator']['AccountAggregationSources'] == [account_aggregation_source] + assert 'arn:aws:config:us-west-2:123456789012:config-aggregator/config-aggregator-' in \ + result['ConfigurationAggregator']['ConfigurationAggregatorArn'] + assert result['ConfigurationAggregator']['CreationTime'] == result['ConfigurationAggregator']['LastUpdatedTime'] + + # Update the existing one: + original_arn = result['ConfigurationAggregator']['ConfigurationAggregatorArn'] + account_aggregation_source.pop('AwsRegions') + account_aggregation_source['AllAwsRegions'] = True + result = client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[account_aggregation_source] + ) + + assert result['ConfigurationAggregator']['ConfigurationAggregatorName'] == 'testing' + assert result['ConfigurationAggregator']['AccountAggregationSources'] == [account_aggregation_source] + assert result['ConfigurationAggregator']['ConfigurationAggregatorArn'] == original_arn + + # Make an org one: + result = client.put_configuration_aggregator( + ConfigurationAggregatorName='testingOrg', + OrganizationAggregationSource={ + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole', + 'AwsRegions': ['us-east-1', 'us-west-2'] + } + ) + + assert result['ConfigurationAggregator']['ConfigurationAggregatorName'] == 'testingOrg' + assert result['ConfigurationAggregator']['OrganizationAggregationSource'] == { + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole', + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ], + 'AllAwsRegions': False + } + + +@mock_config +def test_describe_configuration_aggregators(): + client = boto3.client('config', region_name='us-west-2') + + # Without any config aggregators: + assert not client.describe_configuration_aggregators()['ConfigurationAggregators'] + + # Make 10 config aggregators: + for x in range(0, 10): + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing{}'.format(x), + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ] + ) + + # Describe with an incorrect name: + with assert_raises(ClientError) as ce: + client.describe_configuration_aggregators(ConfigurationAggregatorNames=['DoesNotExist']) + assert 'The configuration aggregator does not exist.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationAggregatorException' + + # Error describe with more than 1 item in the list: + with assert_raises(ClientError) as ce: + client.describe_configuration_aggregators(ConfigurationAggregatorNames=['testing0', 'DoesNotExist']) + assert 'At least one of the configuration aggregators does not exist.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationAggregatorException' + + # Get the normal list: + result = client.describe_configuration_aggregators() + assert not result.get('NextToken') + assert len(result['ConfigurationAggregators']) == 10 + + # Test filtered list: + agg_names = ['testing0', 'testing1', 'testing2'] + result = client.describe_configuration_aggregators(ConfigurationAggregatorNames=agg_names) + assert not result.get('NextToken') + assert len(result['ConfigurationAggregators']) == 3 + assert [agg['ConfigurationAggregatorName'] for agg in result['ConfigurationAggregators']] == agg_names + + # Test Pagination: + result = client.describe_configuration_aggregators(Limit=4) + assert len(result['ConfigurationAggregators']) == 4 + assert result['NextToken'] == 'testing4' + assert [agg['ConfigurationAggregatorName'] for agg in result['ConfigurationAggregators']] == \ + ['testing{}'.format(x) for x in range(0, 4)] + result = client.describe_configuration_aggregators(Limit=4, NextToken='testing4') + assert len(result['ConfigurationAggregators']) == 4 + assert result['NextToken'] == 'testing8' + assert [agg['ConfigurationAggregatorName'] for agg in result['ConfigurationAggregators']] == \ + ['testing{}'.format(x) for x in range(4, 8)] + result = client.describe_configuration_aggregators(Limit=4, NextToken='testing8') + assert len(result['ConfigurationAggregators']) == 2 + assert not result.get('NextToken') + assert [agg['ConfigurationAggregatorName'] for agg in result['ConfigurationAggregators']] == \ + ['testing{}'.format(x) for x in range(8, 10)] + + # Test Pagination with Filtering: + result = client.describe_configuration_aggregators(ConfigurationAggregatorNames=['testing2', 'testing4'], Limit=1) + assert len(result['ConfigurationAggregators']) == 1 + assert result['NextToken'] == 'testing4' + assert result['ConfigurationAggregators'][0]['ConfigurationAggregatorName'] == 'testing2' + result = client.describe_configuration_aggregators(ConfigurationAggregatorNames=['testing2', 'testing4'], Limit=1, NextToken='testing4') + assert not result.get('NextToken') + assert result['ConfigurationAggregators'][0]['ConfigurationAggregatorName'] == 'testing4' + + # Test with an invalid filter: + with assert_raises(ClientError) as ce: + client.describe_configuration_aggregators(NextToken='WRONG') + assert 'The nextToken provided is invalid' == ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidNextTokenException' + + +@mock_config +def test_put_aggregation_authorization(): + client = boto3.client('config', region_name='us-west-2') + + # Too many tags (>50): + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': '{}'.format(x), 'Value': '{}'.format(x)} for x in range(0, 51)] + ) + assert 'Member must have length less than or equal to 50' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Tag key is too big (>128 chars): + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': 'a' * 129, 'Value': 'a'}] + ) + assert 'Member must have length less than or equal to 128' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Tag value is too big (>256 chars): + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': 'tag', 'Value': 'a' * 257}] + ) + assert 'Member must have length less than or equal to 256' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Duplicate Tags: + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': 'a', 'Value': 'a'}, {'Key': 'a', 'Value': 'a'}] + ) + assert 'Duplicate tag keys found.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidInput' + + # Invalid characters in the tag key: + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': '!', 'Value': 'a'}] + ) + assert 'Member must satisfy regular expression pattern:' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Put a normal one there: + result = client.put_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-east-1', + Tags=[{'Key': 'tag', 'Value': 'a'}]) + + assert result['AggregationAuthorization']['AggregationAuthorizationArn'] == 'arn:aws:config:us-west-2:123456789012:' \ + 'aggregation-authorization/012345678910/us-east-1' + assert result['AggregationAuthorization']['AuthorizedAccountId'] == '012345678910' + assert result['AggregationAuthorization']['AuthorizedAwsRegion'] == 'us-east-1' + assert isinstance(result['AggregationAuthorization']['CreationTime'], datetime) + + creation_date = result['AggregationAuthorization']['CreationTime'] + + # And again: + result = client.put_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-east-1') + assert result['AggregationAuthorization']['AggregationAuthorizationArn'] == 'arn:aws:config:us-west-2:123456789012:' \ + 'aggregation-authorization/012345678910/us-east-1' + assert result['AggregationAuthorization']['AuthorizedAccountId'] == '012345678910' + assert result['AggregationAuthorization']['AuthorizedAwsRegion'] == 'us-east-1' + assert result['AggregationAuthorization']['CreationTime'] == creation_date + + +@mock_config +def test_describe_aggregation_authorizations(): + client = boto3.client('config', region_name='us-west-2') + + # With no aggregation authorizations: + assert not client.describe_aggregation_authorizations()['AggregationAuthorizations'] + + # Make 10 account authorizations: + for i in range(0, 10): + client.put_aggregation_authorization(AuthorizedAccountId='{}'.format(str(i) * 12), AuthorizedAwsRegion='us-west-2') + + result = client.describe_aggregation_authorizations() + assert len(result['AggregationAuthorizations']) == 10 + assert not result.get('NextToken') + for i in range(0, 10): + assert result['AggregationAuthorizations'][i]['AuthorizedAccountId'] == str(i) * 12 + + # Test Pagination: + result = client.describe_aggregation_authorizations(Limit=4) + assert len(result['AggregationAuthorizations']) == 4 + assert result['NextToken'] == ('4' * 12) + '/us-west-2' + assert [auth['AuthorizedAccountId'] for auth in result['AggregationAuthorizations']] == ['{}'.format(str(x) * 12) for x in range(0, 4)] + + result = client.describe_aggregation_authorizations(Limit=4, NextToken=('4' * 12) + '/us-west-2') + assert len(result['AggregationAuthorizations']) == 4 + assert result['NextToken'] == ('8' * 12) + '/us-west-2' + assert [auth['AuthorizedAccountId'] for auth in result['AggregationAuthorizations']] == ['{}'.format(str(x) * 12) for x in range(4, 8)] + + result = client.describe_aggregation_authorizations(Limit=4, NextToken=('8' * 12) + '/us-west-2') + assert len(result['AggregationAuthorizations']) == 2 + assert not result.get('NextToken') + assert [auth['AuthorizedAccountId'] for auth in result['AggregationAuthorizations']] == ['{}'.format(str(x) * 12) for x in range(8, 10)] + + # Test with an invalid filter: + with assert_raises(ClientError) as ce: + client.describe_aggregation_authorizations(NextToken='WRONG') + assert 'The nextToken provided is invalid' == ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidNextTokenException' + + +@mock_config +def test_delete_aggregation_authorization(): + client = boto3.client('config', region_name='us-west-2') + + client.put_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-west-2') + + # Delete it: + client.delete_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-west-2') + + # Verify that none are there: + assert not client.describe_aggregation_authorizations()['AggregationAuthorizations'] + + # Try it again -- nothing should happen: + client.delete_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-west-2') + + +@mock_config +def test_delete_configuration_aggregator(): + client = boto3.client('config', region_name='us-west-2') + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ] + ) + + client.delete_configuration_aggregator(ConfigurationAggregatorName='testing') + + # And again to confirm that it's deleted: + with assert_raises(ClientError) as ce: + client.delete_configuration_aggregator(ConfigurationAggregatorName='testing') + assert 'The configuration aggregator does not exist.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationAggregatorException' + + @mock_config def test_describe_configurations(): client = boto3.client('config', region_name='us-west-2')