Add support for configuring the CA bundle used for validating the endpoint certs

changelogs/fragments/99-awsmodule.yml

@@ -0,0 +1,4 @@
+minor_changes:
+- 'Add support for `aws_ca_bundle` to boto3 based AWS modules'
+- 'Add `aws_security_token`, `aws_endpoint_url` and `endpoint_url` aliases to improve AWS module parameter naming consistency.'
+- 'Add support for configuring boto3 profiles using `AWS_PROFILE` and `AWS_DEFAULT_PROFILE`'

plugins/doc_fragments/aws.py

@@ -25,6 +25,7 @@ class ModuleDocFragment(object):
         Ignored for modules where region is required. Must be specified for all other modules if region is not used.
         If not set then the value of the EC2_URL environment variable, if any, is used.
     type: str
+    aliases: [ aws_endpoint_url, endpoint_url ]
   aws_secret_key:
     description:
       - AWS secret key. If not set then the value of the AWS_SECRET_ACCESS_KEY, AWS_SECRET_KEY, or EC2_SECRET_KEY environment variable is used.
@@ -39,7 +40,13 @@ class ModuleDocFragment(object):
     description:
       - AWS STS security token. If not set then the value of the AWS_SECURITY_TOKEN or EC2_SECURITY_TOKEN environment variable is used.
     type: str
-    aliases: [ access_token ]
+    aliases: [ aws_security_token, access_token ]
+  aws_ca_bundle:
+    description:
+      - "The location of a CA Bundle to use when validating SSL certificates."
+      - "Only used for boto3 based modules."
+      - "Note: The CA Bundle is read 'module' side and may need to be explicitly copied from the controller if not run locally."
+    type: path
   validate_certs:
     description:
       - When set to "no", SSL certificates will not be validated for boto versions >= 2.6.0.
@@ -49,6 +56,7 @@ class ModuleDocFragment(object):
     description:
       - Uses a boto profile. Only works with boto >= 2.24.0.
     type: str
+    aliases: [ aws_profile ]
   aws_config:
     description:
       - A dictionary to modify the botocore configuration.
@@ -65,7 +73,9 @@ class ModuleDocFragment(object):
     C(AWS_ACCESS_KEY_ID) or C(AWS_ACCESS_KEY) or C(EC2_ACCESS_KEY),
     C(AWS_SECRET_ACCESS_KEY) or C(AWS_SECRET_KEY) or C(EC2_SECRET_KEY),
     C(AWS_SECURITY_TOKEN) or C(EC2_SECURITY_TOKEN),
-    C(AWS_REGION) or C(EC2_REGION)
+    C(AWS_REGION) or C(EC2_REGION),
+    C(AWS_PROFILE) or C(AWS_DEFAULT_PROFILE),
+    C(AWS_CA_BUNDLE)
   - Ansible uses the boto configuration file (typically ~/.boto) if no
     credentials are provided. See https://boto.readthedocs.io/en/latest/boto_config_tut.html
   - C(AWS_REGION) or C(EC2_REGION) can be typically be used to specify the

plugins/module_utils/ec2.py

@@ -189,12 +189,13 @@ def boto_exception(err):
 def aws_common_argument_spec():
     return dict(
         debug_botocore_endpoint_logs=dict(fallback=(env_fallback, ['ANSIBLE_DEBUG_BOTOCORE_LOGS']), default=False, type='bool'),
-        ec2_url=dict(),
-        aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True),
+        ec2_url=dict(aliases=['aws_endpoint_url', 'endpoint_url']),
         aws_access_key=dict(aliases=['ec2_access_key', 'access_key']),
+        aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True),
+        security_token=dict(aliases=['access_token', 'aws_security_token'], no_log=True),
         validate_certs=dict(default=True, type='bool'),
-        security_token=dict(aliases=['access_token'], no_log=True),
-        profile=dict(),
+        aws_ca_bundle=dict(type='path'),
+        profile=dict(aliases=['aws_profile']),
         aws_config=dict(type='dict'),
     )
 
@@ -254,6 +255,7 @@ def get_aws_connection_info(module, boto3=False):
     region = get_aws_region(module, boto3)
     profile_name = module.params.get('profile')
     validate_certs = module.params.get('validate_certs')
+    ca_bundle = module.params.get('aws_ca_bundle')
     config = module.params.get('aws_config')
 
     if not ec2_url:
@@ -307,11 +309,24 @@ def get_aws_connection_info(module, boto3=False):
             # in case secret_token came in as empty string
             security_token = None
 
+    if not ca_bundle:
+        if os.environ.get('AWS_CA_BUNDLE'):
+            ca_bundle = os.environ.get('AWS_CA_BUNDLE')
+
+    if not profile_name:
+        if os.environ.get('AWS_PROFILE'):
+            profile_name = os.environ.get('AWS_PROFILE')
+        if os.environ.get('AWS_DEFAULT_PROFILE'):
+            profile_name = os.environ.get('AWS_DEFAULT_PROFILE')
+
     if HAS_BOTO3 and boto3:
         boto_params = dict(aws_access_key_id=access_key,
                            aws_secret_access_key=secret_key,
                            aws_session_token=security_token)
-        boto_params['verify'] = validate_certs
+        if validate_certs and ca_bundle:
+            boto_params['verify'] = ca_bundle
+        else:
+            boto_params['verify'] = validate_certs
 
         if profile_name:
             boto_params = dict(aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None)

tests/integration/targets/ansible_aws_module/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+shippable/aws/group2

tests/integration/targets/ansible_aws_module/inventory

@@ -0,0 +1,6 @@
+[tests]
+localhost
+
+[all:vars]
+ansible_connection=local
+ansible_python_interpreter="{{ ansible_playbook_python }}"

tests/integration/targets/ansible_aws_module/main.yml

@@ -0,0 +1,7 @@
+- hosts: all
+  gather_facts: no
+  collections:
+  - community.aws
+  - amazon.aws
+  roles:
+  - 'ansible_aws_module'

tests/integration/targets/ansible_aws_module/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - prepare_tests
+  - setup_ec2

tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/amazonroot.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF
+ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6
+b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL
+MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv
+b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj
+ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM
+9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw
+IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6
+VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L
+93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm
+jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC
+AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA
+A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI
+U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs
+N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv
+o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU
+5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy
+rqXRfboQnoZsG4q5WTP468SQvvG5
+-----END CERTIFICATE-----

tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/files/isrg-x1.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/library/boto3_example.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+# Copyright (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# A bare-minimum Ansible Module based on AnsibleAWSModule used for testing some
+# of the core behaviour around AWS/Boto3 connection details
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+try:
+    from botocore.exceptions import BotoCoreError, ClientError
+except ImportError:
+    pass  # Handled by AnsibleAWSModule
+
+from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict
+
+
+def main():
+    module = AnsibleAWSModule(
+        argument_spec={},
+        supports_check_mode=True,
+    )
+
+    client = module.client('ec2')
+
+    filters = ansible_dict_to_boto3_filter_list({'name': 'amzn2-ami-hvm-2.0.202006*-x86_64-gp2'})
+
+    try:
+        images = client.describe_images(ImageIds=[], Filters=filters, Owners=['amazon'], ExecutableUsers=[])
+    except (BotoCoreError, ClientError) as e:
+        module.fail_json_aws(e, msg='Fail JSON AWS')
+
+    # Return something, just because we can.
+    module.exit_json(
+        changed=False,
+        **camel_dict_to_snake_dict(images))
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/meta/main.yml

@@ -0,0 +1,5 @@
+dependencies:
+  - prepare_tests
+  - setup_ec2
+collections:
+  - amazon.aws

tests/integration/targets/ansible_aws_module/roles/ansible_aws_module/tasks/ca_bundle.yml

@@ -0,0 +1,158 @@
+---
+- name: 'Create temporary location for CA files'
+  tempfile:
+    state: directory
+    suffix: 'test-CAs'
+  register: ca_tmp
+
+- name: 'Ensure we have Amazons root CA available to us'
+  copy:
+    src: 'amazonroot.pem'
+    dest: '{{ ca_tmp.path }}/amazonroot.pem'
+
+- name: 'Ensure we have a another CA (ISRG-X1) bundle available to us'
+  copy:
+    src: 'isrg-x1.pem'
+    dest: '{{ ca_tmp.path }}/isrg-x1.pem'
+
+##################################################################################
+# Test disabling cert validation (make sure we don't error)
+
+- name: 'Test basic operation using default CA bundle (no validation) - parameter'
+  boto3_example:
+    region: '{{ aws_region }}'
+    access_key: '{{ aws_access_key }}'
+    secret_key: '{{ aws_secret_key }}'
+    security_token: '{{ security_token }}'
+    validate_certs: False
+  register: default_bundle_result
+
+- assert:
+    that:
+    - default_bundle_result is successful
+
+##################################################################################
+# Tests using Amazon's CA (the one the endpoint certs should be signed with)
+
+- name: 'Test basic operation using Amazons root CA - parameter'
+  boto3_example:
+    region: '{{ aws_region }}'
+    access_key: '{{ aws_access_key }}'
+    secret_key: '{{ aws_secret_key }}'
+    security_token: '{{ security_token }}'
+    aws_ca_bundle: '{{ ca_tmp.path }}/amazonroot.pem'
+  register: amazon_ca_result
+
+- assert:
+    that:
+    - amazon_ca_result is successful
+
+- name: 'Test basic operation using Amazons root CA - environment'
+  boto3_example:
+    region: '{{ aws_region }}'
+    access_key: '{{ aws_access_key }}'
+    secret_key: '{{ aws_secret_key }}'
+    security_token: '{{ security_token }}'
+  environment:
+    AWS_CA_BUNDLE: '{{ ca_tmp.path }}/amazonroot.pem'
+  register: amazon_ca_result
+
+- assert:
+    that:
+    - amazon_ca_result is successful
+
+- name: 'Test basic operation using Amazons root CA (no validation) - parameter'
+  boto3_example:
+    region: '{{ aws_region }}'
+    access_key: '{{ aws_access_key }}'
+    secret_key: '{{ aws_secret_key }}'
+    security_token: '{{ security_token }}'
+    aws_ca_bundle: '{{ ca_tmp.path }}/amazonroot.pem'
+    validate_certs: False
+  register: amazon_ca_result
+
+- assert:
+    that:
+    - amazon_ca_result is successful
+
+- name: 'Test basic operation using Amazons root CA (no validation) - environment'
+  boto3_example:
+    region: '{{ aws_region }}'
+    access_key: '{{ aws_access_key }}'
+    secret_key: '{{ aws_secret_key }}'
+    security_token: '{{ security_token }}'
+    validate_certs: False
+  environment:
+    AWS_CA_BUNDLE: '{{ ca_tmp.path }}/amazonroot.pem'
+  register: amazon_ca_result
+
+- assert:
+    that:
+    - amazon_ca_result is successful
+
+##################################################################################
+# Tests using ISRG's CA (one that the endpoint certs *aren't* signed with)
+
+- name: 'Test basic operation using a different CA - parameter'
+  boto3_example:
+    region: '{{ aws_region }}'
+    access_key: '{{ aws_access_key }}'
+    secret_key: '{{ aws_secret_key }}'
+    security_token: '{{ security_token }}'
+    aws_ca_bundle: '{{ ca_tmp.path }}/isrg-x1.pem'
+  register: isrg_ca_result
+  ignore_errors: yes
+
+- assert:
+    that:
+    - isrg_ca_result is failed
+    # Caught when we try to do something, and passed to fail_json_aws
+    - '"CERTIFICATE_VERIFY_FAILED" in isrg_ca_result.msg'
+    - '"Fail JSON AWS" in isrg_ca_result.msg'
+
+- name: 'Test basic operation using a different CA - environment'
+  boto3_example:
+    region: '{{ aws_region }}'
+    access_key: '{{ aws_access_key }}'
+    secret_key: '{{ aws_secret_key }}'
+    security_token: '{{ security_token }}'
+  environment:
+    AWS_CA_BUNDLE: '{{ ca_tmp.path }}/isrg-x1.pem'
+  register: isrg_ca_result
+  ignore_errors: yes
+
+- assert:
+    that:
+    - isrg_ca_result is failed
+    # Caught when we try to do something, and passed to fail_json_aws
+    - '"CERTIFICATE_VERIFY_FAILED" in isrg_ca_result.msg'
+    - '"Fail JSON AWS" in isrg_ca_result.msg'
+
+- name: 'Test basic operation using a different CA (no validation) - parameter'
+  boto3_example:
+    region: '{{ aws_region }}'
+    access_key: '{{ aws_access_key }}'
+    secret_key: '{{ aws_secret_key }}'
+    security_token: '{{ security_token }}'
+    aws_ca_bundle: '{{ ca_tmp.path }}/isrg-x1.pem'
+    validate_certs: False
+  register: isrg_ca_result
+
+- assert:
+    that:
+    - isrg_ca_result is successful
+
+- name: 'Test basic operation using a different CA (no validation) - environment'
+  boto3_example:
+    region: '{{ aws_region }}'
+    access_key: '{{ aws_access_key }}'
+    secret_key: '{{ aws_secret_key }}'
+    security_token: '{{ security_token }}'
+    validate_certs: False
+  environment:
+    AWS_CA_BUNDLE: '{{ ca_tmp.path }}/isrg-x1.pem'
+  register: isrg_ca_result
+
+- assert:
+    that:
+    - isrg_ca_result is successful