From 2388a89ff270364bc0ebe9968a432d2a26daeda3 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 10 Jul 2020 09:58:35 +0200 Subject: [PATCH 01/11] Integration tests for AnsibleAWSModule --- .../targets/ansible_aws_module/aliases | 2 + .../targets/ansible_aws_module/inventory | 6 +++ .../targets/ansible_aws_module/main.yml | 4 ++ .../targets/ansible_aws_module/meta/main.yml | 3 ++ .../library/boto3_example.py | 41 +++++++++++++++++++ .../roles/ansible_aws_module/meta/main.yml | 5 +++ .../roles/ansible_aws_module/tasks/main.yml | 8 ++++ .../targets/ansible_aws_module/runme.sh | 10 +++++ .../targets/ansible_aws_module/setup.yml | 9 ++++ .../templates/boto_config.j2 | 7 ++++ 10 files changed, 95 insertions(+) create mode 100644 tests/integration/targets/ansible_aws_module/aliases create mode 100644 tests/integration/targets/ansible_aws_module/inventory create mode 100644 tests/integration/targets/ansible_aws_module/main.yml create mode 100644 tests/integration/targets/ansible_aws_module/meta/main.yml create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/meta/main.yml create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml create mode 100755 tests/integration/targets/ansible_aws_module/runme.sh create mode 100644 tests/integration/targets/ansible_aws_module/setup.yml create mode 100644 tests/integration/targets/ansible_aws_module/templates/boto_config.j2 diff --git a/tests/integration/targets/ansible_aws_module/aliases b/tests/integration/targets/ansible_aws_module/aliases new file mode 100644 index 00000000000..6e3860bee23 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/aliases @@ -0,0 +1,2 @@ +cloud/aws +shippable/aws/group2 diff --git a/tests/integration/targets/ansible_aws_module/inventory b/tests/integration/targets/ansible_aws_module/inventory new file mode 100644 index 00000000000..5093e8582bc --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/inventory @@ -0,0 +1,6 @@ +[tests] +localhost + +[all:vars] +ansible_connection=local +ansible_python_interpreter="{{ ansible_playbook_python }}" diff --git a/tests/integration/targets/ansible_aws_module/main.yml b/tests/integration/targets/ansible_aws_module/main.yml new file mode 100644 index 00000000000..e3b381aa8e6 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/main.yml @@ -0,0 +1,4 @@ +- hosts: all + gather_facts: no + roles: + - 'ansible_aws_module' diff --git a/tests/integration/targets/ansible_aws_module/meta/main.yml b/tests/integration/targets/ansible_aws_module/meta/main.yml new file mode 100644 index 00000000000..1f64f1169a9 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - prepare_tests + - setup_ec2 diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py new file mode 100644 index 00000000000..2b6d05e9d69 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py @@ -0,0 +1,41 @@ +#!/usr/bin/python +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# A bare-minimum Ansible Module based on AnsibleAWSModule used for testing some +# of the core behaviour around variables + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +try: + from botocore.exceptions import BotoCoreError, ClientError +except ImportError: + pass # Handled by AnsibleAWSModule + +from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict + +def main(): + module = AnsibleAWSModule( + argument_spec={}, + supports_check_mode=True, + ) + + client = module.client('sts') + + try: + caller_info = client.get_caller_identity() + caller_info.pop('ResponseMetadata', None) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg='Failed to retrieve caller identity') + + # Return something, just because we can. + module.exit_json( + changed=False, + **camel_dict_to_snake_dict(caller_info)) + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/meta/main.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/meta/main.yml new file mode 100644 index 00000000000..77589cc2b48 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/meta/main.yml @@ -0,0 +1,5 @@ +dependencies: + - prepare_tests + - setup_ec2 +collections: + - amazon.aws diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml new file mode 100644 index 00000000000..d48da759558 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: 'run test module' + boto3_example: + #region: '{{ aws_region }}' + #aws_access_key: "{{ aws_access_key }}" + #aws_secret_key: "{{ aws_secret_key }}" + #security_token: "{{ security_token | default(omit) }}" + profile: 'test_profile' diff --git a/tests/integration/targets/ansible_aws_module/runme.sh b/tests/integration/targets/ansible_aws_module/runme.sh new file mode 100755 index 00000000000..6bfdf31f262 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/runme.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -eux + +export ANSIBLE_ROLES_PATH=../ +export AWS_CONFIG_FILE="$( pwd )/boto3_config" +#export AWS_SDK_LOAD_CONFIG=1 + +ansible-playbook setup.yml -i localhost "$@" +ansible-playbook main.yml -i inventory "$@" diff --git a/tests/integration/targets/ansible_aws_module/setup.yml b/tests/integration/targets/ansible_aws_module/setup.yml new file mode 100644 index 00000000000..e9bf1108be6 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/setup.yml @@ -0,0 +1,9 @@ +--- +- hosts: localhost + connection: local + gather_facts: no + tasks: + - name: 'Write out boto config file' + template: + dest: './boto3_config' + src: 'boto_config.j2' diff --git a/tests/integration/targets/ansible_aws_module/templates/boto_config.j2 b/tests/integration/targets/ansible_aws_module/templates/boto_config.j2 new file mode 100644 index 00000000000..363947df054 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/templates/boto_config.j2 @@ -0,0 +1,7 @@ +[profile test_profile] +region = {{ aws_region }} +aws_access_key_id = {{ aws_access_key }} +aws_secret_access_key = {{ aws_secret_key }} +{% if security_token is defined %} +aws_security_token = {{ security_token }} +{% endif %} From 632ed7cd9e6175fd61dcbde94a4164cab8e6fe68 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 10 Jul 2020 11:06:43 +0200 Subject: [PATCH 02/11] Linting fixups --- .../roles/ansible_aws_module/library/boto3_example.py | 3 ++- tests/integration/targets/ansible_aws_module/runme.sh | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py index 2b6d05e9d69..bd6f3cd14a1 100644 --- a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py @@ -3,7 +3,7 @@ # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) # A bare-minimum Ansible Module based on AnsibleAWSModule used for testing some -# of the core behaviour around variables +# of the core behaviour around AWS/Boto3 connection details from __future__ import (absolute_import, division, print_function) __metaclass__ = type @@ -17,6 +17,7 @@ from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict + def main(): module = AnsibleAWSModule( argument_spec={}, diff --git a/tests/integration/targets/ansible_aws_module/runme.sh b/tests/integration/targets/ansible_aws_module/runme.sh index 6bfdf31f262..7bc12a87e41 100755 --- a/tests/integration/targets/ansible_aws_module/runme.sh +++ b/tests/integration/targets/ansible_aws_module/runme.sh @@ -2,9 +2,10 @@ set -eux -export ANSIBLE_ROLES_PATH=../ -export AWS_CONFIG_FILE="$( pwd )/boto3_config" -#export AWS_SDK_LOAD_CONFIG=1 +ANSIBLE_ROLES_PATH="../" +AWS_CONFIG_FILE="$( pwd )/boto3_config" +export ANSIBLE_ROLES_PATH +export AWS_CONFIG_FILE ansible-playbook setup.yml -i localhost "$@" ansible-playbook main.yml -i inventory "$@" From fa58ac707bb2e76a7a2b5156210a0a7c6bdaf258 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sat, 18 Jul 2020 13:17:59 +0200 Subject: [PATCH 03/11] Add support for setting the CA bundle used when verifying the connection. This is primarily used when connecting to AWS-like endpoints. --- plugins/doc_fragments/aws.py | 8 +++++++- plugins/module_utils/ec2.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/plugins/doc_fragments/aws.py b/plugins/doc_fragments/aws.py index 08c1c9154f2..960bcd4a090 100644 --- a/plugins/doc_fragments/aws.py +++ b/plugins/doc_fragments/aws.py @@ -40,6 +40,11 @@ class ModuleDocFragment(object): - AWS STS security token. If not set then the value of the AWS_SECURITY_TOKEN or EC2_SECURITY_TOKEN environment variable is used. type: str aliases: [ access_token ] + aws_ca_bundle: + description: + - The location of a CA Bundle to use when validating SSL certificates. + - Only used for boto3 based modules. + type: path validate_certs: description: - When set to "no", SSL certificates will not be validated for boto versions >= 2.6.0. @@ -65,7 +70,8 @@ class ModuleDocFragment(object): C(AWS_ACCESS_KEY_ID) or C(AWS_ACCESS_KEY) or C(EC2_ACCESS_KEY), C(AWS_SECRET_ACCESS_KEY) or C(AWS_SECRET_KEY) or C(EC2_SECRET_KEY), C(AWS_SECURITY_TOKEN) or C(EC2_SECURITY_TOKEN), - C(AWS_REGION) or C(EC2_REGION) + C(AWS_REGION) or C(EC2_REGION), + C(AWS_CA_BUNDLE) - Ansible uses the boto configuration file (typically ~/.boto) if no credentials are provided. See https://boto.readthedocs.io/en/latest/boto_config_tut.html - C(AWS_REGION) or C(EC2_REGION) can be typically be used to specify the diff --git a/plugins/module_utils/ec2.py b/plugins/module_utils/ec2.py index 7c0abaaadbf..714b566c240 100644 --- a/plugins/module_utils/ec2.py +++ b/plugins/module_utils/ec2.py @@ -193,6 +193,7 @@ def aws_common_argument_spec(): aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True), aws_access_key=dict(aliases=['ec2_access_key', 'access_key']), validate_certs=dict(default=True, type='bool'), + aws_ca_bundle=dict(type='path'), security_token=dict(aliases=['access_token'], no_log=True), profile=dict(), aws_config=dict(type='dict'), @@ -254,6 +255,7 @@ def get_aws_connection_info(module, boto3=False): region = get_aws_region(module, boto3) profile_name = module.params.get('profile') validate_certs = module.params.get('validate_certs') + ca_bundle = module.params.get('aws_ca_bundle') config = module.params.get('aws_config') if not ec2_url: @@ -307,11 +309,18 @@ def get_aws_connection_info(module, boto3=False): # in case secret_token came in as empty string security_token = None + if not ca_bundle: + if os.environ.get('AWS_CA_BUNDLE'): + ca_bundle = os.environ.get('AWS_CA_BUNDLE') + if HAS_BOTO3 and boto3: boto_params = dict(aws_access_key_id=access_key, aws_secret_access_key=secret_key, aws_session_token=security_token) - boto_params['verify'] = validate_certs + if validate_certs and ca_bundle: + boto_params['verify'] = ca_bundle + else: + boto_params['verify'] = validate_certs if profile_name: boto_params = dict(aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None) From 7eeb768417957ffe0c9b88c3d1680db48224c7b8 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sat, 18 Jul 2020 13:19:38 +0200 Subject: [PATCH 04/11] Add support for setting the AWS profile using an environment variable --- plugins/doc_fragments/aws.py | 1 + plugins/module_utils/ec2.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/plugins/doc_fragments/aws.py b/plugins/doc_fragments/aws.py index 960bcd4a090..242938a02d1 100644 --- a/plugins/doc_fragments/aws.py +++ b/plugins/doc_fragments/aws.py @@ -71,6 +71,7 @@ class ModuleDocFragment(object): C(AWS_SECRET_ACCESS_KEY) or C(AWS_SECRET_KEY) or C(EC2_SECRET_KEY), C(AWS_SECURITY_TOKEN) or C(EC2_SECURITY_TOKEN), C(AWS_REGION) or C(EC2_REGION), + C(AWS_PROFILE) or C(AWS_DEFAULT_PROFILE), C(AWS_CA_BUNDLE) - Ansible uses the boto configuration file (typically ~/.boto) if no credentials are provided. See https://boto.readthedocs.io/en/latest/boto_config_tut.html diff --git a/plugins/module_utils/ec2.py b/plugins/module_utils/ec2.py index 714b566c240..8661947a715 100644 --- a/plugins/module_utils/ec2.py +++ b/plugins/module_utils/ec2.py @@ -313,6 +313,12 @@ def get_aws_connection_info(module, boto3=False): if os.environ.get('AWS_CA_BUNDLE'): ca_bundle = os.environ.get('AWS_CA_BUNDLE') + if not profile_name: + if os.environ.get('AWS_PROFILE'): + profile_name = os.environ.get('AWS_PROFILE') + if os.environ.get('AWS_DEFAULT_PROFILE'): + profile_name = os.environ.get('AWS_DEFAULT_PROFILE') + if HAS_BOTO3 and boto3: boto_params = dict(aws_access_key_id=access_key, aws_secret_access_key=secret_key, From 2ff8a3605f7c187a80ba27fa9cfd4142e1915c3d Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sat, 18 Jul 2020 13:21:04 +0200 Subject: [PATCH 05/11] Add some extra aliases for the AWS parameters to try and make them a little more consistent --- plugins/doc_fragments/aws.py | 4 +++- plugins/module_utils/ec2.py | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/doc_fragments/aws.py b/plugins/doc_fragments/aws.py index 242938a02d1..bd40d124a53 100644 --- a/plugins/doc_fragments/aws.py +++ b/plugins/doc_fragments/aws.py @@ -25,6 +25,7 @@ class ModuleDocFragment(object): Ignored for modules where region is required. Must be specified for all other modules if region is not used. If not set then the value of the EC2_URL environment variable, if any, is used. type: str + aliases: [ aws_endpoint_url, endpoint_url ] aws_secret_key: description: - AWS secret key. If not set then the value of the AWS_SECRET_ACCESS_KEY, AWS_SECRET_KEY, or EC2_SECRET_KEY environment variable is used. @@ -39,7 +40,7 @@ class ModuleDocFragment(object): description: - AWS STS security token. If not set then the value of the AWS_SECURITY_TOKEN or EC2_SECURITY_TOKEN environment variable is used. type: str - aliases: [ access_token ] + aliases: [ aws_security_token, access_token ] aws_ca_bundle: description: - The location of a CA Bundle to use when validating SSL certificates. @@ -54,6 +55,7 @@ class ModuleDocFragment(object): description: - Uses a boto profile. Only works with boto >= 2.24.0. type: str + aliases: [ aws_profile ] aws_config: description: - A dictionary to modify the botocore configuration. diff --git a/plugins/module_utils/ec2.py b/plugins/module_utils/ec2.py index 8661947a715..9ec629b0a67 100644 --- a/plugins/module_utils/ec2.py +++ b/plugins/module_utils/ec2.py @@ -189,13 +189,13 @@ def boto_exception(err): def aws_common_argument_spec(): return dict( debug_botocore_endpoint_logs=dict(fallback=(env_fallback, ['ANSIBLE_DEBUG_BOTOCORE_LOGS']), default=False, type='bool'), - ec2_url=dict(), - aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True), + ec2_url=dict(aliases=['aws_endpoint_url', 'endpoint_url']), aws_access_key=dict(aliases=['ec2_access_key', 'access_key']), + aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True), + security_token=dict(aliases=['access_token', 'aws_security_token'], no_log=True), validate_certs=dict(default=True, type='bool'), aws_ca_bundle=dict(type='path'), - security_token=dict(aliases=['access_token'], no_log=True), - profile=dict(), + profile=dict(aliases=['aws_profile']), aws_config=dict(type='dict'), ) From ab6db99e00ac070ab17c98ae8d052194be38fb32 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sat, 18 Jul 2020 13:23:15 +0200 Subject: [PATCH 06/11] Rework example module to use EC2 rather than STS. STS doesn't use regions... --- .../targets/ansible_aws_module/main.yml | 3 + .../library/boto3_example.py | 12 +- .../roles/ansible_aws_module/tasks/main.yml | 9 +- .../ansible_aws_module/tasks/success.yml | 118 ++++++++++++++++++ .../targets/ansible_aws_module/runme.sh | 2 +- .../targets/ansible_aws_module/setup.yml | 31 +++++ .../templates/boto_config.j2 | 8 +- .../templates/session_credentials.yml.j2 | 3 + 8 files changed, 168 insertions(+), 18 deletions(-) create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/success.yml create mode 100644 tests/integration/targets/ansible_aws_module/templates/session_credentials.yml.j2 diff --git a/tests/integration/targets/ansible_aws_module/main.yml b/tests/integration/targets/ansible_aws_module/main.yml index e3b381aa8e6..2fe3a5e8a1a 100644 --- a/tests/integration/targets/ansible_aws_module/main.yml +++ b/tests/integration/targets/ansible_aws_module/main.yml @@ -1,4 +1,7 @@ - hosts: all gather_facts: no + collections: + - community.aws + - amazon.aws roles: - 'ansible_aws_module' diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py index bd6f3cd14a1..86e6ceeae44 100644 --- a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py @@ -15,6 +15,7 @@ pass # Handled by AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict @@ -24,18 +25,19 @@ def main(): supports_check_mode=True, ) - client = module.client('sts') + client = module.client('ec2') + + filters = ansible_dict_to_boto3_filter_list({'name': 'amzn2-ami-hvm-2.0.202006*-x86_64-gp2'}) try: - caller_info = client.get_caller_identity() - caller_info.pop('ResponseMetadata', None) + images = client.describe_images(ImageIds=[], Filters=filters, Owners=['amazon'], ExecutableUsers=[]) except (BotoCoreError, ClientError) as e: - module.fail_json_aws(e, msg='Failed to retrieve caller identity') + module.fail_json_aws(e, msg='Failed to retrieve list of amis') # Return something, just because we can. module.exit_json( changed=False, - **camel_dict_to_snake_dict(caller_info)) + **camel_dict_to_snake_dict(images)) if __name__ == '__main__': diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml index d48da759558..c95be81b4be 100644 --- a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml @@ -1,8 +1,3 @@ --- -- name: 'run test module' - boto3_example: - #region: '{{ aws_region }}' - #aws_access_key: "{{ aws_access_key }}" - #aws_secret_key: "{{ aws_secret_key }}" - #security_token: "{{ security_token | default(omit) }}" - profile: 'test_profile' +- name: 'Test that the varients we expect to succeed, do' + include_tasks: 'success.yml' diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/success.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/success.yml new file mode 100644 index 00000000000..e71f26bbc1c --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/success.yml @@ -0,0 +1,118 @@ +--- +################################################################################## +# Tests using standard credential parameters + +- name: 'Test basic operation using simple credentials (simple-parameters)' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + register: credential_result + +- name: 'Test basic operation using simple credentials (aws-parameters)' + boto3_example: + aws_region: '{{ aws_region }}' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + register: credential_result + +- name: 'Test basic operation using simple credentials (ec2-parameters)' + boto3_example: + ec2_region: '{{ aws_region }}' + ec2_access_key: '{{ aws_access_key }}' + ec2_secret_key: '{{ aws_secret_key }}' + access_token: '{{ security_token }}' + register: credential_result + +################################################################################## +# Tests using standard credentials from environment variables + +- name: 'Test basic operation using simple credentials (aws-environment)' + boto3_example: + environment: + AWS_REGION: '{{ aws_region }}' + AWS_ACCESS_KEY_ID: '{{ aws_access_key }}' + AWS_SECRET_ACCESS_KEY: '{{ aws_secret_key }}' + AWS_SECURITY_TOKEN: '{{ security_token }}' + register: credential_result + +- name: 'Test basic operation using simple credentials (aws2-environment)' + boto3_example: + environment: + AWS_DEFAULT_REGION: '{{ aws_region }}' + AWS_ACCESS_KEY: '{{ aws_access_key }}' + AWS_SECRET_KEY: '{{ aws_secret_key }}' + AWS_SESSION_TOKEN: '{{ security_token }}' + register: credential_result + +- name: 'Test basic operation using simple credentials (ec2-environment)' + boto3_example: + environment: + EC2_REGION: '{{ aws_region }}' + EC2_ACCESS_KEY: '{{ aws_access_key }}' + EC2_SECRET_KEY: '{{ aws_secret_key }}' + EC2_SECURITY_TOKEN: '{{ security_token }}' + register: credential_result + +################################################################################## +# Tests using profiles instead of directly consuming credentials + +- name: 'Test basic operation using profile (simple-parameters)' + boto3_example: + profile: 'test_profile' + register: profile_result + +- name: 'Test basic operation using profile (aws-parameters)' + boto3_example: + profile: 'test_profile' + register: profile_result + +- name: 'Test basic operation using profile (aws-environment)' + boto3_example: + environment: + AWS_PROFILE: 'test_profile' + register: profile_result + +- name: 'Test basic operation using profile (aws2-environment)' + boto3_example: + environment: + AWS_DEFAULT_PROFILE: 'test_profile' + register: profile_result + +################################################################################## +# Tests using profiles instead of directly consuming credentials + +- name: 'Test basic operation using standard endpoint (aws-parameters)' + boto3_example: + region: '{{ aws_region }}' + aws_endpoint_url: 'https://ec2.{{ aws_region }}.amazonaws.com' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + register: standard_endpoint_result + +- name: 'Check that we connected to the standard endpoint' + assert: + that: + - standard_endpoint_result is successful + - '"ec2:DescribeImages" in standard_endpoint_result.resource_actions' + +# The FIPS endpoints aren't available in every region, this will trigger errors +# outside of: [ us-east-1, us-east-2, us-west-1, us-west-2 ] + +- name: 'Test basic operation using FIPS endpoint (aws-parameters)' + boto3_example: + region: '{{ aws_region }}' + aws_endpoint_url: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + register: fips_endpoint_result + +- name: 'Check that we connected to the FIPS endpoint' + assert: + that: + - fips_endpoint_result is successful + - '"ec2-fips:DescribeImages" in fips_endpoint_result.resource_actions' diff --git a/tests/integration/targets/ansible_aws_module/runme.sh b/tests/integration/targets/ansible_aws_module/runme.sh index 7bc12a87e41..3a933f5f61d 100755 --- a/tests/integration/targets/ansible_aws_module/runme.sh +++ b/tests/integration/targets/ansible_aws_module/runme.sh @@ -8,4 +8,4 @@ export ANSIBLE_ROLES_PATH export AWS_CONFIG_FILE ansible-playbook setup.yml -i localhost "$@" -ansible-playbook main.yml -i inventory "$@" +ansible-playbook main.yml -i inventory "$@" -e "@session_credentials.yml" diff --git a/tests/integration/targets/ansible_aws_module/setup.yml b/tests/integration/targets/ansible_aws_module/setup.yml index e9bf1108be6..9b219eb20f9 100644 --- a/tests/integration/targets/ansible_aws_module/setup.yml +++ b/tests/integration/targets/ansible_aws_module/setup.yml @@ -3,6 +3,37 @@ connection: local gather_facts: no tasks: + # =========================================================== + # While CI uses a dedicated session, the easiest way to run + # tests outside of CI is with a simple access/secret key pair. + # + # For consistency, use sts_session_token to grab session + # credentials if we're not already using a session + # Note: this can't be done within a session, hence the slightly + # strange dance + - name: 'Get a session token if we are using a basic key' + when: + - security_token is not defined + block: + - name: 'Get a session token' + sts_session_token: + region: '{{ aws_region }}' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + register: session_token + no_log: true + - name: 'Override initial tokens' + set_fact: + session_access_key: '{{ session_token.sts_creds.access_key }}' + session_secret_key: '{{ session_token.sts_creds.secret_key }}' + session_security_token: '{{ session_token.sts_creds.session_token }}' + no_log: true + + - name: 'Write out credentials' + template: + dest: './session_credentials.yml' + src: 'session_credentials.yml.j2' + - name: 'Write out boto config file' template: dest: './boto3_config' diff --git a/tests/integration/targets/ansible_aws_module/templates/boto_config.j2 b/tests/integration/targets/ansible_aws_module/templates/boto_config.j2 index 363947df054..f8668f057fc 100644 --- a/tests/integration/targets/ansible_aws_module/templates/boto_config.j2 +++ b/tests/integration/targets/ansible_aws_module/templates/boto_config.j2 @@ -1,7 +1,5 @@ [profile test_profile] region = {{ aws_region }} -aws_access_key_id = {{ aws_access_key }} -aws_secret_access_key = {{ aws_secret_key }} -{% if security_token is defined %} -aws_security_token = {{ security_token }} -{% endif %} +aws_access_key_id = {{ session_access_key | default(aws_access_key) }} +aws_secret_access_key = {{ session_secret_key | default(aws_secret_key) }} +aws_security_token = {{ session_security_token | default(security_token) }} diff --git a/tests/integration/targets/ansible_aws_module/templates/session_credentials.yml.j2 b/tests/integration/targets/ansible_aws_module/templates/session_credentials.yml.j2 new file mode 100644 index 00000000000..bb030439392 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/templates/session_credentials.yml.j2 @@ -0,0 +1,3 @@ +aws_access_key: {{ session_access_key | default(aws_access_key) }} +aws_secret_key: {{ session_secret_key | default(aws_secret_key) }} +security_token: {{ session_security_token | default(security_token) }} From 4c99bd56cfe2b563be9da29bb52b5a8f385223c3 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sat, 18 Jul 2020 19:57:51 +0200 Subject: [PATCH 07/11] rework test struture and finish fleshing it out --- .../ansible_aws_module/files/amazonroot.pem | 20 ++ .../ansible_aws_module/files/isrg-x1.pem | 31 +++ .../library/boto3_example.py | 2 +- .../ansible_aws_module/tasks/ca_bundle.yml | 158 ++++++++++++ .../ansible_aws_module/tasks/credentials.yml | 229 ++++++++++++++++++ .../ansible_aws_module/tasks/endpoints.yml | 122 ++++++++++ .../roles/ansible_aws_module/tasks/main.yml | 13 +- .../ansible_aws_module/tasks/profiles.yml | 57 +++++ .../ansible_aws_module/tasks/success.yml | 118 --------- 9 files changed, 629 insertions(+), 121 deletions(-) create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/amazonroot.pem create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/isrg-x1.pem create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/ca_bundle.yml create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/credentials.yml create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/endpoints.yml create mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/profiles.yml delete mode 100644 tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/success.yml diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/amazonroot.pem b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/amazonroot.pem new file mode 100644 index 00000000000..a6f3e92af58 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/amazonroot.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj +ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM +9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw +IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 +VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L +93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm +jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA +A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI +U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs +N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv +o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU +5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy +rqXRfboQnoZsG4q5WTP468SQvvG5 +-----END CERTIFICATE----- diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/isrg-x1.pem b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/isrg-x1.pem new file mode 100644 index 00000000000..b85c8037f6b --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/isrg-x1.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 +WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu +ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY +MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc +h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ +0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U +A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW +T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH +B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC +B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv +KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn +OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn +jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw +qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI +rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq +hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL +ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ +3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK +NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 +ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur +TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC +jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc +oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq +4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA +mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d +emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= +-----END CERTIFICATE----- diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py index 86e6ceeae44..80e7651fc2c 100644 --- a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py @@ -32,7 +32,7 @@ def main(): try: images = client.describe_images(ImageIds=[], Filters=filters, Owners=['amazon'], ExecutableUsers=[]) except (BotoCoreError, ClientError) as e: - module.fail_json_aws(e, msg='Failed to retrieve list of amis') + module.fail_json_aws(e, msg='Fail JSON AWS') # Return something, just because we can. module.exit_json( diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/ca_bundle.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/ca_bundle.yml new file mode 100644 index 00000000000..9ecf813fee5 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/ca_bundle.yml @@ -0,0 +1,158 @@ +--- +- name: 'Create temporary location for CA files' + tempfile: + state: directory + suffix: 'test-CAs' + register: ca_tmp + +- name: 'Ensure we have Amazons root CA available to us' + copy: + src: 'amazonroot.pem' + dest: '{{ ca_tmp.path }}/amazonroot.pem' + +- name: 'Ensure we have a another CA (ISRG-X1) bundle available to us' + copy: + src: 'isrg-x1.pem' + dest: '{{ ca_tmp.path }}/isrg-x1.pem' + +################################################################################## +# Test disabling cert validation (make sure we don't error) + +- name: 'Test basic operation using default CA bundle (no validation) - parameter' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + validate_certs: False + register: default_bundle_result + +- assert: + that: + - default_bundle_result is successful + +################################################################################## +# Tests using Amazon's CA (the one the endpoint certs should be signed with) + +- name: 'Test basic operation using Amazons root CA - parameter' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + aws_ca_bundle: '{{ ca_tmp.path }}/amazonroot.pem' + register: amazon_ca_result + +- assert: + that: + - amazon_ca_result is successful + +- name: 'Test basic operation using Amazons root CA - environment' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + environment: + AWS_CA_BUNDLE: '{{ ca_tmp.path }}/amazonroot.pem' + register: amazon_ca_result + +- assert: + that: + - amazon_ca_result is successful + +- name: 'Test basic operation using Amazons root CA (no validation) - parameter' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + aws_ca_bundle: '{{ ca_tmp.path }}/amazonroot.pem' + validate_certs: False + register: amazon_ca_result + +- assert: + that: + - amazon_ca_result is successful + +- name: 'Test basic operation using Amazons root CA (no validation) - environment' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + validate_certs: False + environment: + AWS_CA_BUNDLE: '{{ ca_tmp.path }}/amazonroot.pem' + register: amazon_ca_result + +- assert: + that: + - amazon_ca_result is successful + +################################################################################## +# Tests using ISRG's CA (one that the endpoint certs *aren't* signed with) + +- name: 'Test basic operation using a different CA - parameter' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + aws_ca_bundle: '{{ ca_tmp.path }}/isrg-x1.pem' + register: isrg_ca_result + ignore_errors: yes + +- assert: + that: + - isrg_ca_result is failed + # Caught when we try to do something, and passed to fail_json_aws + - '"CERTIFICATE_VERIFY_FAILED" in isrg_ca_result.msg' + - '"Fail JSON AWS" in isrg_ca_result.msg' + +- name: 'Test basic operation using a different CA - environment' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + environment: + AWS_CA_BUNDLE: '{{ ca_tmp.path }}/isrg-x1.pem' + register: isrg_ca_result + ignore_errors: yes + +- assert: + that: + - isrg_ca_result is failed + # Caught when we try to do something, and passed to fail_json_aws + - '"CERTIFICATE_VERIFY_FAILED" in isrg_ca_result.msg' + - '"Fail JSON AWS" in isrg_ca_result.msg' + +- name: 'Test basic operation using a different CA (no validation) - parameter' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + aws_ca_bundle: '{{ ca_tmp.path }}/isrg-x1.pem' + validate_certs: False + register: isrg_ca_result + +- assert: + that: + - isrg_ca_result is successful + +- name: 'Test basic operation using a different CA (no validation) - environment' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + validate_certs: False + environment: + AWS_CA_BUNDLE: '{{ ca_tmp.path }}/isrg-x1.pem' + register: isrg_ca_result + +- assert: + that: + - isrg_ca_result is successful diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/credentials.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/credentials.yml new file mode 100644 index 00000000000..a1e99a78e6d --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/credentials.yml @@ -0,0 +1,229 @@ +--- +################################################################################## +# Tests using standard credential parameters + +- name: 'Test basic operation using simple credentials (simple-parameters)' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + register: credential_result + +- assert: + that: + - credential_result is successful + +- name: 'Test basic operation using simple credentials (aws-parameters)' + boto3_example: + aws_region: '{{ aws_region }}' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + register: credential_result + +- assert: + that: + - credential_result is successful + +- name: 'Test basic operation using simple credentials (ec2-parameters)' + boto3_example: + ec2_region: '{{ aws_region }}' + ec2_access_key: '{{ aws_access_key }}' + ec2_secret_key: '{{ aws_secret_key }}' + access_token: '{{ security_token }}' + register: credential_result + +- assert: + that: + - credential_result is successful + +################################################################################## +# Tests using standard credentials from environment variables + +- name: 'Test basic operation using simple credentials (aws-environment)' + boto3_example: + environment: + AWS_REGION: '{{ aws_region }}' + AWS_ACCESS_KEY_ID: '{{ aws_access_key }}' + AWS_SECRET_ACCESS_KEY: '{{ aws_secret_key }}' + AWS_SECURITY_TOKEN: '{{ security_token }}' + register: credential_result + +- assert: + that: + - credential_result is successful + +- name: 'Test basic operation using simple credentials (aws2-environment)' + boto3_example: + environment: + AWS_DEFAULT_REGION: '{{ aws_region }}' + AWS_ACCESS_KEY: '{{ aws_access_key }}' + AWS_SECRET_KEY: '{{ aws_secret_key }}' + AWS_SESSION_TOKEN: '{{ security_token }}' + register: credential_result + +- assert: + that: + - credential_result is successful + +- name: 'Test basic operation using simple credentials (ec2-environment)' + boto3_example: + environment: + EC2_REGION: '{{ aws_region }}' + EC2_ACCESS_KEY: '{{ aws_access_key }}' + EC2_SECRET_KEY: '{{ aws_secret_key }}' + EC2_SECURITY_TOKEN: '{{ security_token }}' + register: credential_result + +- assert: + that: + - credential_result is successful + +################################################################################## +# Tests for missing parameters + +- name: 'Test with missing region' + boto3_example: + region: '{{ omit }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + register: missing_region + ignore_errors: True + +- assert: + that: + - missing_region is failed + - '"requires a region" in missing_region.msg' + +- name: 'Test with missing access key' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ omit }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + register: missing_access + ignore_errors: True + +- assert: + that: + - missing_access is failed + - '"Partial credentials found" in missing_access.msg' + - '"aws_access_key_id" in missing_access.msg' + +- name: 'Test with missing secret key' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ omit }}' + security_token: '{{ security_token }}' + register: missing_secret + ignore_errors: True + +- assert: + that: + - missing_secret is failed + - '"Partial credentials found" in missing_secret.msg' + - '"aws_secret_access_key" in missing_secret.msg' + +- name: 'Test with missing security token' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ omit }}' + register: missing_token + ignore_errors: True + +- assert: + that: + - missing_token is failed + # Caught when we try to do something, and passed to fail_json_aws + - '"AuthFailure" in missing_token.msg' + - '"Fail JSON AWS" in missing_token.msg' + - '"error" in missing_token' + - '"code" in missing_token.error' + - missing_token.error.code == 'AuthFailure' + - '"message" in missing_token.error' + + +################################################################################## +# Tests for bad parameters + +- name: 'Test with bad region' + boto3_example: + region: 'junk-example' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + register: bad_region + ignore_errors: True + +- assert: + that: + - bad_region is failed + - '"msg" in bad_region' + - '"Could not connect to the endpoint URL" in bad_region.msg' + - '"Fail JSON AWS" in bad_region.msg' + - '"ec2.junk-example" in bad_region.msg' + +- name: 'Test with bad access key' + boto3_example: + region: '{{ aws_region }}' + access_key: 'junk-example' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + register: bad_access + ignore_errors: True + +- assert: + that: + - bad_access is failed + # Caught when we try to do something, and passed to fail_json_aws + - '"AuthFailure" in bad_access.msg' + - '"Fail JSON AWS" in bad_access.msg' + - '"error" in bad_access' + - '"code" in bad_access.error' + - bad_access.error.code == 'AuthFailure' + - '"message" in bad_access.error' + +- name: 'Test with bad secret key' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: 'junk-example' + security_token: '{{ security_token }}' + register: bad_secret + ignore_errors: True + +- assert: + that: + - bad_secret is failed + # Caught when we try to do something, and passed to fail_json_aws + - '"AuthFailure" in bad_secret.msg' + - '"Fail JSON AWS" in bad_secret.msg' + - '"error" in bad_secret' + - '"code" in bad_secret.error' + - bad_secret.error.code == 'AuthFailure' + - '"message" in bad_secret.error' + +- name: 'Test with bad security token' + boto3_example: + region: '{{ aws_region }}' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: 'junk-example' + register: bad_token + ignore_errors: True + +- assert: + that: + - bad_token is failed + # Caught when we try to do something, and passed to fail_json_aws + - '"AuthFailure" in bad_token.msg' + - '"Fail JSON AWS" in bad_token.msg' + - '"error" in bad_token' + - '"code" in bad_token.error' + - bad_token.error.code == 'AuthFailure' + - '"message" in bad_token.error' diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/endpoints.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/endpoints.yml new file mode 100644 index 00000000000..03476147858 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/endpoints.yml @@ -0,0 +1,122 @@ +--- +################################################################################## +# Tests using Endpoints + +- name: 'Test basic operation using standard endpoint (aws-parameters)' + boto3_example: + region: '{{ aws_region }}' + aws_endpoint_url: 'https://ec2.{{ aws_region }}.amazonaws.com' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + register: standard_endpoint_result + +- name: 'Check that we connected to the standard endpoint' + assert: + that: + - standard_endpoint_result is successful + - '"ec2:DescribeImages" in standard_endpoint_result.resource_actions' + +# The FIPS endpoints aren't available in every region, this will trigger errors +# outside of: [ us-east-1, us-east-2, us-west-1, us-west-2 ] + +- name: 'Test basic operation using FIPS endpoint (aws-parameters)' + boto3_example: + region: '{{ aws_region }}' + aws_endpoint_url: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + register: fips_endpoint_result + +- name: 'Check that we connected to the FIPS endpoint' + assert: + that: + - fips_endpoint_result is successful + - '"ec2-fips:DescribeImages" in fips_endpoint_result.resource_actions' + +- name: 'Test basic operation using FIPS endpoint (aws-parameters)' + boto3_example: + region: '{{ aws_region }}' + endpoint_url: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + register: fips_endpoint_result + +- name: 'Check that we connected to the FIPS endpoint' + assert: + that: + - fips_endpoint_result is successful + - '"ec2-fips:DescribeImages" in fips_endpoint_result.resource_actions' + +- name: 'Test basic operation using FIPS endpoint (aws-parameters)' + boto3_example: + region: '{{ aws_region }}' + ec2_url: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + register: fips_endpoint_result + +- name: 'Check that we connected to the FIPS endpoint' + assert: + that: + - fips_endpoint_result is successful + - '"ec2-fips:DescribeImages" in fips_endpoint_result.resource_actions' + +################################################################################## +# Tests using environment variables + +- name: 'Test basic operation using FIPS endpoint (aws-environment)' + boto3_example: + region: '{{ aws_region }}' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + environment: + AWS_URL: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + register: fips_endpoint_result + +- name: 'Check that we connected to the FIPS endpoint' + assert: + that: + - fips_endpoint_result is successful + - '"ec2-fips:DescribeImages" in fips_endpoint_result.resource_actions' + +- name: 'Test basic operation using FIPS endpoint (ec2-environment)' + boto3_example: + region: '{{ aws_region }}' + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + aws_security_token: '{{ security_token }}' + environment: + EC2_URL: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + register: fips_endpoint_result + +- name: 'Check that we connected to the FIPS endpoint' + assert: + that: + - fips_endpoint_result is successful + - '"ec2-fips:DescribeImages" in fips_endpoint_result.resource_actions' + +################################################################################## +# Tests using a bad endpoint URL + +- name: 'Test with bad endpoint URL' + boto3_example: + region: '{{ aws_region }}' + endpoint_url: 'https://junk.{{ aws_region }}.amazonaws.com' + access_key: '{{ aws_access_key }}' + secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + register: bad_endpoint + ignore_errors: True + +- assert: + that: + - bad_endpoint is failed + - '"msg" in bad_endpoint' + - '"Could not connect to the endpoint URL" in bad_endpoint.msg' + - '"Fail JSON AWS" in bad_endpoint.msg' + - '"junk.{{ aws_region }}.amazonaws.com" in bad_endpoint.msg' diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml index c95be81b4be..dc61fad6814 100644 --- a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/main.yml @@ -1,3 +1,12 @@ --- -- name: 'Test that the varients we expect to succeed, do' - include_tasks: 'success.yml' +- name: 'Tests around standard credentials' + include_tasks: 'credentials.yml' + +- name: 'Tests around profiles' + include_tasks: 'profiles.yml' + +- name: 'Tests around endpoints' + include_tasks: 'endpoints.yml' + +- name: 'Tests around CA Bundles' + include_tasks: 'ca_bundle.yml' diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/profiles.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/profiles.yml new file mode 100644 index 00000000000..bd54edb1cb4 --- /dev/null +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/profiles.yml @@ -0,0 +1,57 @@ +--- +################################################################################## +# Tests using profiles instead of directly consuming credentials + +- name: 'Test basic operation using profile (simple-parameters)' + boto3_example: + profile: 'test_profile' + register: profile_result + +- assert: + that: + - profile_result is successful + +- name: 'Test basic operation using profile (aws-parameters)' + boto3_example: + profile: 'test_profile' + register: profile_result + +- assert: + that: + - profile_result is successful + +- name: 'Test basic operation using profile (aws-environment)' + boto3_example: + environment: + AWS_PROFILE: 'test_profile' + register: profile_result + +- assert: + that: + - profile_result is successful + +- name: 'Test basic operation using profile (aws2-environment)' + boto3_example: + environment: + AWS_DEFAULT_PROFILE: 'test_profile' + register: profile_result + +- assert: + that: + - profile_result is successful + +################################################################################## +# Tests with bad profile + +- name: 'Test with bad profile' + boto3_example: + profile: 'junk-profile' + register: bad_profile + ignore_errors: True + +- assert: + that: + - bad_profile is failed + - '"msg" in bad_profile' + - '"junk-profile" in bad_profile.msg' + - '"could not be found" in bad_profile.msg' diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/success.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/success.yml deleted file mode 100644 index e71f26bbc1c..00000000000 --- a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/success.yml +++ /dev/null @@ -1,118 +0,0 @@ ---- -################################################################################## -# Tests using standard credential parameters - -- name: 'Test basic operation using simple credentials (simple-parameters)' - boto3_example: - region: '{{ aws_region }}' - access_key: '{{ aws_access_key }}' - secret_key: '{{ aws_secret_key }}' - security_token: '{{ security_token }}' - register: credential_result - -- name: 'Test basic operation using simple credentials (aws-parameters)' - boto3_example: - aws_region: '{{ aws_region }}' - aws_access_key: '{{ aws_access_key }}' - aws_secret_key: '{{ aws_secret_key }}' - aws_security_token: '{{ security_token }}' - register: credential_result - -- name: 'Test basic operation using simple credentials (ec2-parameters)' - boto3_example: - ec2_region: '{{ aws_region }}' - ec2_access_key: '{{ aws_access_key }}' - ec2_secret_key: '{{ aws_secret_key }}' - access_token: '{{ security_token }}' - register: credential_result - -################################################################################## -# Tests using standard credentials from environment variables - -- name: 'Test basic operation using simple credentials (aws-environment)' - boto3_example: - environment: - AWS_REGION: '{{ aws_region }}' - AWS_ACCESS_KEY_ID: '{{ aws_access_key }}' - AWS_SECRET_ACCESS_KEY: '{{ aws_secret_key }}' - AWS_SECURITY_TOKEN: '{{ security_token }}' - register: credential_result - -- name: 'Test basic operation using simple credentials (aws2-environment)' - boto3_example: - environment: - AWS_DEFAULT_REGION: '{{ aws_region }}' - AWS_ACCESS_KEY: '{{ aws_access_key }}' - AWS_SECRET_KEY: '{{ aws_secret_key }}' - AWS_SESSION_TOKEN: '{{ security_token }}' - register: credential_result - -- name: 'Test basic operation using simple credentials (ec2-environment)' - boto3_example: - environment: - EC2_REGION: '{{ aws_region }}' - EC2_ACCESS_KEY: '{{ aws_access_key }}' - EC2_SECRET_KEY: '{{ aws_secret_key }}' - EC2_SECURITY_TOKEN: '{{ security_token }}' - register: credential_result - -################################################################################## -# Tests using profiles instead of directly consuming credentials - -- name: 'Test basic operation using profile (simple-parameters)' - boto3_example: - profile: 'test_profile' - register: profile_result - -- name: 'Test basic operation using profile (aws-parameters)' - boto3_example: - profile: 'test_profile' - register: profile_result - -- name: 'Test basic operation using profile (aws-environment)' - boto3_example: - environment: - AWS_PROFILE: 'test_profile' - register: profile_result - -- name: 'Test basic operation using profile (aws2-environment)' - boto3_example: - environment: - AWS_DEFAULT_PROFILE: 'test_profile' - register: profile_result - -################################################################################## -# Tests using profiles instead of directly consuming credentials - -- name: 'Test basic operation using standard endpoint (aws-parameters)' - boto3_example: - region: '{{ aws_region }}' - aws_endpoint_url: 'https://ec2.{{ aws_region }}.amazonaws.com' - aws_access_key: '{{ aws_access_key }}' - aws_secret_key: '{{ aws_secret_key }}' - aws_security_token: '{{ security_token }}' - register: standard_endpoint_result - -- name: 'Check that we connected to the standard endpoint' - assert: - that: - - standard_endpoint_result is successful - - '"ec2:DescribeImages" in standard_endpoint_result.resource_actions' - -# The FIPS endpoints aren't available in every region, this will trigger errors -# outside of: [ us-east-1, us-east-2, us-west-1, us-west-2 ] - -- name: 'Test basic operation using FIPS endpoint (aws-parameters)' - boto3_example: - region: '{{ aws_region }}' - aws_endpoint_url: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' - aws_access_key: '{{ aws_access_key }}' - aws_secret_key: '{{ aws_secret_key }}' - aws_security_token: '{{ security_token }}' - register: fips_endpoint_result - -- name: 'Check that we connected to the FIPS endpoint' - assert: - that: - - fips_endpoint_result is successful - - '"ec2-fips:DescribeImages" in fips_endpoint_result.resource_actions' From dbc1ba0e403bd7990bc6ec2e74d9ec2fbf60dfac Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sat, 18 Jul 2020 19:58:16 +0200 Subject: [PATCH 08/11] Add a note about SSL certs and where they're read from --- plugins/doc_fragments/aws.py | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/doc_fragments/aws.py b/plugins/doc_fragments/aws.py index bd40d124a53..b6b9396c297 100644 --- a/plugins/doc_fragments/aws.py +++ b/plugins/doc_fragments/aws.py @@ -45,6 +45,7 @@ class ModuleDocFragment(object): description: - The location of a CA Bundle to use when validating SSL certificates. - Only used for boto3 based modules. + - Note: The CA Bundle is read 'module' side and may need to be explicitly copied from the controller if not run locally. type: path validate_certs: description: From c8421671bedc2cbffacd9e977028ed64c4c896a1 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sat, 18 Jul 2020 20:14:32 +0200 Subject: [PATCH 09/11] Add quotes so that the : doesn't result in the string being converted to a dict --- plugins/doc_fragments/aws.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/doc_fragments/aws.py b/plugins/doc_fragments/aws.py index b6b9396c297..1b797904711 100644 --- a/plugins/doc_fragments/aws.py +++ b/plugins/doc_fragments/aws.py @@ -43,9 +43,9 @@ class ModuleDocFragment(object): aliases: [ aws_security_token, access_token ] aws_ca_bundle: description: - - The location of a CA Bundle to use when validating SSL certificates. - - Only used for boto3 based modules. - - Note: The CA Bundle is read 'module' side and may need to be explicitly copied from the controller if not run locally. + - "The location of a CA Bundle to use when validating SSL certificates." + - "Only used for boto3 based modules." + - "Note: The CA Bundle is read 'module' side and may need to be explicitly copied from the controller if not run locally." type: path validate_certs: description: From a472dc56f16cae34bdfbae4306df9e439c2e2344 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Tue, 21 Jul 2020 14:32:09 +0200 Subject: [PATCH 10/11] Add changelog ci_complete --- changelogs/fragments/99-awsmodule.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/fragments/99-awsmodule.yml diff --git a/changelogs/fragments/99-awsmodule.yml b/changelogs/fragments/99-awsmodule.yml new file mode 100644 index 00000000000..c7e064ec4bc --- /dev/null +++ b/changelogs/fragments/99-awsmodule.yml @@ -0,0 +1,4 @@ +minor_changes: +- 'Add support for `aws_ca_bundle` to boto3 based AWS modules' +- 'Add `aws_security_token`, `aws_endpoint_url` and `endpoint_url` aliases to improve AWS module parameter naming consistency.' +- 'Add support for configuring boto3 profiles using `AWS_PROFILE` and `AWS_DEFAULT_PROFILE`' From 00f9100ee1af3b51a719f3f02e4cbc228cb4d127 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Thu, 30 Jul 2020 21:09:56 +0200 Subject: [PATCH 11/11] Hard code us-east-1 as FIPS endpoint - FIPS endpoints are only available in certain regions --- .../roles/ansible_aws_module/tasks/endpoints.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/endpoints.yml b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/endpoints.yml index 03476147858..dac9630a1f2 100644 --- a/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/endpoints.yml +++ b/tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/endpoints.yml @@ -23,7 +23,7 @@ - name: 'Test basic operation using FIPS endpoint (aws-parameters)' boto3_example: region: '{{ aws_region }}' - aws_endpoint_url: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + aws_endpoint_url: 'https://ec2-fips.us-east-1.amazonaws.com' aws_access_key: '{{ aws_access_key }}' aws_secret_key: '{{ aws_secret_key }}' aws_security_token: '{{ security_token }}' @@ -38,7 +38,7 @@ - name: 'Test basic operation using FIPS endpoint (aws-parameters)' boto3_example: region: '{{ aws_region }}' - endpoint_url: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + endpoint_url: 'https://ec2-fips.us-east-1.amazonaws.com' aws_access_key: '{{ aws_access_key }}' aws_secret_key: '{{ aws_secret_key }}' aws_security_token: '{{ security_token }}' @@ -53,7 +53,7 @@ - name: 'Test basic operation using FIPS endpoint (aws-parameters)' boto3_example: region: '{{ aws_region }}' - ec2_url: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + ec2_url: 'https://ec2-fips.us-east-1.amazonaws.com' aws_access_key: '{{ aws_access_key }}' aws_secret_key: '{{ aws_secret_key }}' aws_security_token: '{{ security_token }}' @@ -75,7 +75,7 @@ aws_secret_key: '{{ aws_secret_key }}' aws_security_token: '{{ security_token }}' environment: - AWS_URL: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + AWS_URL: 'https://ec2-fips.us-east-1.amazonaws.com' register: fips_endpoint_result - name: 'Check that we connected to the FIPS endpoint' @@ -91,7 +91,7 @@ aws_secret_key: '{{ aws_secret_key }}' aws_security_token: '{{ security_token }}' environment: - EC2_URL: 'https://ec2-fips.{{ aws_region }}.amazonaws.com' + EC2_URL: 'https://ec2-fips.us-east-1.amazonaws.com' register: fips_endpoint_result - name: 'Check that we connected to the FIPS endpoint'