diff --git a/.gitignore b/.gitignore index ece750efc1..ae57b2eea5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ build/ .DS_Store .tox/ *.iml +.coverage +assets/ +report.html diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a878c16168..9c3fc5df58 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ CHANGELOG ========= +2.1.1 +===== +* China regions `cn-north-1` and `cn-northwest-1` + 2.1.0 ===== * RAID support diff --git a/amis.txt b/amis.txt index e16a10d29f..fd5e824cfd 100644 --- a/amis.txt +++ b/amis.txt @@ -1,96 +1,100 @@ # alinux -ap-northeast-1: ami-03780c8327e4be0ac -ap-northeast-2: ami-01eaa2848f762416f -ap-northeast-3: ami-05625a3c29c8d158d -ap-south-1: ami-0cb4b6a940244dd2f -ap-southeast-1: ami-099312510208cc0d3 -ap-southeast-2: ami-04c142f780a8426c2 -ca-central-1: ami-0907f9cad7dbf65f6 -eu-central-1: ami-0595a707dc47e9671 -eu-north-1: ami-09b05710227925ceb -eu-west-1: ami-0f0865f124c533e6f -eu-west-2: ami-08c01fbf6cc43e166 -eu-west-3: ami-033bcfc19853275cb -sa-east-1: ami-0ffb13b9401cc9f43 -us-east-1: ami-025f4816964498a18 -us-east-2: ami-0381cb7486cdc973f -us-gov-east-1: ami-08e554700cf18e130 -us-gov-west-1: ami-a5ee8fc4 -us-west-1: ami-0f7431c55ad80bfce -us-west-2: ami-0bdc62a7fa9026972 +ap-northeast-1: ami-0ac1d0c35dc3b2a97 +ap-northeast-2: ami-0dddb89f66485f828 +ap-northeast-3: ami-0dbb09bfada65298f +ap-south-1: ami-0ed1bb8f2ab0edffb +ap-southeast-1: ami-058bcd8377aba9bef +ap-southeast-2: ami-012557ce9426ef1a0 +ca-central-1: ami-059dda8ee9af7c20c +cn-north-1: ami-00237da8e056b0936 +cn-northwest-1: ami-0974fc483e449f5ee +eu-central-1: ami-09cff6787920e967c +eu-north-1: ami-086f4f382fb1119f0 +eu-west-1: ami-080a7d5c75253bd1b +eu-west-2: ami-0b284f1028a743865 +eu-west-3: ami-099197e40d0c5de6e +sa-east-1: ami-02e542e4935ff9647 +us-east-1: ami-0cd2dd3198972a68c +us-east-2: ami-057e70f0fbb007ab6 +us-gov-east-1: ami-022f96b137a63c9ff +us-gov-west-1: ami-fefb989f +us-west-1: ami-09686d4090e35a702 +us-west-2: ami-0c588cdc9e91b0db3 # centos6 -ap-northeast-1: ami-07d83b268e25e0953 -ap-northeast-2: ami-044f3ad0badddf405 -ap-northeast-3: ami-07261626f0608a9fc -ap-south-1: ami-007862493aa8721a7 -ap-southeast-1: ami-069361480706f302e -ap-southeast-2: ami-06e0db8bdac11c698 -ca-central-1: ami-00875bb416baf6556 -eu-central-1: ami-07f13c4704a54b165 -eu-north-1: ami-03e7f9bfdcdfe6eca -eu-west-1: ami-0080ba82e12f317b9 -eu-west-2: ami-0dfae4fa33e540f25 -eu-west-3: ami-0fae6489cef475f35 -sa-east-1: ami-03a2cb750e02692c6 -us-east-1: ami-09d4b0433c3d08284 -us-east-2: ami-0b8ba3e22e08e3870 -us-west-1: ami-06bd36cf904ec0f9a -us-west-2: ami-084bb82c022317cc5 +ap-northeast-1: ami-003cfe6266cadd576 +ap-northeast-2: ami-032b3f2a4f1ac91a0 +ap-northeast-3: ami-0c4de7aece7b2db33 +ap-south-1: ami-0581583c3d7507d9e +ap-southeast-1: ami-0d00309b80b772532 +ap-southeast-2: ami-00ecf7e455945e8bc +ca-central-1: ami-0416d41ed6dbc0bd7 +eu-central-1: ami-0b1f52047bb2b7f83 +eu-north-1: ami-04fc976e8108996e6 +eu-west-1: ami-073f1f5db6cfdd3d1 +eu-west-2: ami-0ace56d1d9c1aa466 +eu-west-3: ami-0d377bccde07c887d +sa-east-1: ami-0b1c8f6aad337d5b6 +us-east-1: ami-0919d912e0e33d247 +us-east-2: ami-0bbd43b2b8991cdae +us-west-1: ami-0ac1cf1e68288fa36 +us-west-2: ami-08b28682da5721f5b # centos7 -ap-northeast-1: ami-03043938e7fc33ba9 -ap-northeast-2: ami-07ee16085dc31f6ed -ap-northeast-3: ami-01116deb7081a4d93 -ap-south-1: ami-0823edb0c94157ff4 -ap-southeast-1: ami-074acbf20aa22df0c -ap-southeast-2: ami-055dd6211ddf02af6 -ca-central-1: ami-052fb2fbfaef380d3 -eu-central-1: ami-05af6f87fc63e3d71 -eu-north-1: ami-0e933380e094ab102 -eu-west-1: ami-0e249c302ebb32aef -eu-west-2: ami-0836360ea1ef8e1a9 -eu-west-3: ami-0b8e45374e5e2f6ec -sa-east-1: ami-091c8ba9e3b0a2b6b -us-east-1: ami-0ed9fbb6accaa70ba -us-east-2: ami-059415e43c0827abf -us-west-1: ami-02ca1c89adc7b29f0 -us-west-2: ami-03abc3d448a987e73 +ap-northeast-1: ami-0c0049e4eeb0ef1ac +ap-northeast-2: ami-00f8f46a043a04530 +ap-northeast-3: ami-0043e5e12872a00d4 +ap-south-1: ami-03dd063b05c3082f1 +ap-southeast-1: ami-00fcdc55bd29f691e +ap-southeast-2: ami-0baa1787fd7a71950 +ca-central-1: ami-0c2a8b54dfd0c405f +eu-central-1: ami-00b3f34240b6021dd +eu-north-1: ami-07ac74edc9e96b343 +eu-west-1: ami-0879d97613ba8075a +eu-west-2: ami-06aa0ca6f28c29fdb +eu-west-3: ami-0b28dcaf2b4d00d87 +sa-east-1: ami-00a570d9536621660 +us-east-1: ami-09d092c1b054832df +us-east-2: ami-0e0cda1cdec7fcecf +us-west-1: ami-084c065e503e74449 +us-west-2: ami-070345cb145f2d2f0 # ubuntu1404 -ap-northeast-1: ami-08a720b6bbe06dc15 -ap-northeast-2: ami-03595544457ee270b -ap-northeast-3: ami-0f4afbf455ad42c20 -ap-south-1: ami-0821b7c6568999205 -ap-southeast-1: ami-0563d0646a1cc2dde -ap-southeast-2: ami-08209d8fa90d40e5a -ca-central-1: ami-0b05531dac34df70a -eu-central-1: ami-0a2c2f9760d34f596 -eu-north-1: ami-04b0fa5a3509e4569 -eu-west-1: ami-06e7f63766bac032b -eu-west-2: ami-0bf0e4815ea7c3f08 -eu-west-3: ami-0038fece5c9fdcf52 -sa-east-1: ami-05997e9be14a0f785 -us-east-1: ami-09ae02fc417765f4c -us-east-2: ami-0121231f2883a268e -us-gov-east-1: ami-0b621c3f09e5e06c4 -us-gov-west-1: ami-50e98831 -us-west-1: ami-0e30c804ef45d3819 -us-west-2: ami-0440f447631cb8f6c +ap-northeast-1: ami-053a84278e34b7a59 +ap-northeast-2: ami-03d314c5ac10beb53 +ap-northeast-3: ami-070f9069b20a797ec +ap-south-1: ami-0641d7ca885530589 +ap-southeast-1: ami-0e3de99412375e882 +ap-southeast-2: ami-09eae4580e8fc835a +ca-central-1: ami-08aeb7a57f73b58ab +cn-north-1: ami-00f2cae5406fb3fce +eu-central-1: ami-0b24a435216670b4a +eu-north-1: ami-0921b515f8ed512c3 +eu-west-1: ami-076fbdec21cd5c940 +eu-west-2: ami-0fb01b5b56bc27509 +eu-west-3: ami-0ea8b79c622d0a4a2 +sa-east-1: ami-00cf3910c959e9fd5 +us-east-1: ami-095016c5ff0ab7ae6 +us-east-2: ami-02a21f90af8a453f0 +us-gov-east-1: ami-09274e2587c4154f2 +us-gov-west-1: ami-6cf0930d +us-west-1: ami-099ebeb456dbf0646 +us-west-2: ami-05ddc7ec0edb9069f # ubuntu1604 -ap-northeast-1: ami-00bd60c479d9b05f2 -ap-northeast-2: ami-00317e3a1c4d47b23 -ap-northeast-3: ami-07942e464687e23cd -ap-south-1: ami-0fa6a7a4decdae641 -ap-southeast-1: ami-00db94a0965a886a6 -ap-southeast-2: ami-05c16d76720e55964 -ca-central-1: ami-02740f68858b9fc7a -eu-central-1: ami-003cc41d49855e1ef -eu-north-1: ami-0ceb6b8e665efdb87 -eu-west-1: ami-02fbb78584905c216 -eu-west-2: ami-090cb83529f72bd53 -eu-west-3: ami-03fcec4562b3cef3c -sa-east-1: ami-0ba286d64312d26f5 -us-east-1: ami-057dc350619a0bf58 -us-east-2: ami-04c283d05058f6894 -us-gov-east-1: ami-083ba3c9a5c412c8f -us-gov-west-1: ami-54f19035 -us-west-1: ami-0d1f81f40426d7c25 -us-west-2: ami-0f7c3a7a9e2f3d6a3 +ap-northeast-1: ami-026669cfcef23b3de +ap-northeast-2: ami-048214d1413ed8462 +ap-northeast-3: ami-038900edb73cb9496 +ap-south-1: ami-0cc424ec58256ea88 +ap-southeast-1: ami-059ba95190db36590 +ap-southeast-2: ami-04df2433ab61d3f37 +ca-central-1: ami-039a1b0ada060b5ce +cn-north-1: ami-072046713a0458796 +eu-central-1: ami-0d816068d1164f4d2 +eu-north-1: ami-046c32486a9abf742 +eu-west-1: ami-0f641e63ebaf647b1 +eu-west-2: ami-067c1c0157477c166 +eu-west-3: ami-0102caf8c6ec0768a +sa-east-1: ami-0ef5c70aec338bcfb +us-east-1: ami-098c8e582ca818cff +us-east-2: ami-055279b0b09d12a71 +us-gov-east-1: ami-0b664ce8c427b77eb +us-gov-west-1: ami-5ef6953f +us-west-1: ami-0db5e85c0b1ce2c20 +us-west-2: ami-02393fa61ac61547a diff --git a/cli/pcluster/cfnconfig.py b/cli/pcluster/cfnconfig.py index 10d5f7310e..fd24e27bba 100644 --- a/cli/pcluster/cfnconfig.py +++ b/cli/pcluster/cfnconfig.py @@ -487,16 +487,12 @@ def __init__(self, args): # noqa: C901 FIXME!!! self.region, self.aws_access_key_id, self.aws_secret_access_key, "URL", self.template_url ) except configparser.NoOptionError: - if self.region == "us-east-1": - self.template_url = ( - "https://s3.amazonaws.com/%s-aws-parallelcluster/templates/aws-parallelcluster-%s.cfn.json" - % (self.region, self.version) - ) - else: - self.template_url = ( - "https://s3.%s.amazonaws.com/%s-aws-parallelcluster/templates/" - "aws-parallelcluster-%s.cfn.json" % (self.region, self.region, self.version) - ) + s3_suffix = ".cn" if self.region.startswith("cn") else "" + self.template_url = ( + "https://s3.%s.amazonaws.com%s/%s-aws-parallelcluster/templates/" + "aws-parallelcluster-%s.cfn.json" % (self.region, s3_suffix, self.region, self.version) + ) + except AttributeError: pass diff --git a/cli/pcluster/config_sanity.py b/cli/pcluster/config_sanity.py index 7ce82f23b2..5835edf567 100644 --- a/cli/pcluster/config_sanity.py +++ b/cli/pcluster/config_sanity.py @@ -413,7 +413,7 @@ def check_resource( # noqa: C901 FIXME!!! # Batch Parameters elif resource_type == "AWSBatch_Parameters": # Check region - if region in ["us-gov-west-1", "us-gov-east-1", "eu-west-3", "ap-northeast-3"]: + if region in ["ap-northeast-3", "eu-north-1", "cn-north-1", "cn-northwest-1", "us-gov-east-1", "us-gov-west-1"]: print("ERROR: %s region is not supported with awsbatch" % region) sys.exit(1) diff --git a/cli/pcluster/easyconfig.py b/cli/pcluster/easyconfig.py index 617699c562..ec7cfc82c5 100644 --- a/cli/pcluster/easyconfig.py +++ b/cli/pcluster/easyconfig.py @@ -29,7 +29,7 @@ from . import cfnconfig logger = logging.getLogger("pcluster.pcluster") -unsupported_regions = ["ap-northeast-3", "cn-north-1", "cn-northwest-1"] +unsupported_regions = ["ap-northeast-3"] def handle_client_exception(func): diff --git a/cli/pcluster/pcluster.py b/cli/pcluster/pcluster.py index 4d8d3eb2a8..c554a26856 100644 --- a/cli/pcluster/pcluster.py +++ b/cli/pcluster/pcluster.py @@ -749,11 +749,10 @@ def get_cookbook_url(config, tmpdir): return config.args.custom_ami_cookbook cookbook_version = get_cookbook_version(config, tmpdir) - if config.region == "us-east-1": - return "https://s3.amazonaws.com/%s-aws-parallelcluster/cookbooks/%s.tgz" % (config.region, cookbook_version) - - return "https://s3.%s.amazonaws.com/%s-aws-parallelcluster/cookbooks/%s.tgz" % ( + s3_suffix = ".cn" if config.region.startswith("cn") else "" + return "https://s3.%s.amazonaws.com%s/%s-aws-parallelcluster/cookbooks/%s.tgz" % ( config.region, + s3_suffix, config.region, cookbook_version, ) diff --git a/cli/pcluster/resources/batch/docker/upload-docker-images.sh b/cli/pcluster/resources/batch/docker/upload-docker-images.sh index fb7a40adb3..5ec4d85bcd 100755 --- a/cli/pcluster/resources/batch/docker/upload-docker-images.sh +++ b/cli/pcluster/resources/batch/docker/upload-docker-images.sh @@ -4,8 +4,12 @@ set -eu push_docker_image() { local image=$1 echo "Uploading image image" - docker tag "${IMAGE_REPO_NAME}:${image}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${IMAGE_REPO_NAME}:${image}" - docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${IMAGE_REPO_NAME}:${image}" + S3_SUFFIX="" + if [[ ${AWS_REGION} == cn-* ]]; then + S3_SUFFIX=".cn" + fi + docker tag "${IMAGE_REPO_NAME}:${image}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com${S3_SUFFIX}/${IMAGE_REPO_NAME}:${image}" + docker push "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com${S3_SUFFIX}/${IMAGE_REPO_NAME}:${image}" } if [ -z "${IMAGE}" ]; then diff --git a/cli/setup.py b/cli/setup.py index 8f43952bd8..1c1f14a213 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -21,7 +21,7 @@ def readme(): return f.read() -VERSION = "2.1.0" +VERSION = "2.1.1" REQUIRES = ["boto3>=1.9.48", "awscli>=1.11.175", "future>=0.16.0", "tabulate>=0.8.2"] if sys.version_info[:2] == (2, 6): diff --git a/cloudformation/aws-parallelcluster.cfn.json b/cloudformation/aws-parallelcluster.cfn.json index a5bd5263d8..aa8fc0d266 100644 --- a/cloudformation/aws-parallelcluster.cfn.json +++ b/cloudformation/aws-parallelcluster.cfn.json @@ -1,6 +1,6 @@ { "AWSTemplateFormatVersion": "2010-09-09", - "Description": "AWS ParallelCluster Template. Version: aws-parallelcluster-2.1.0", + "Description": "AWS ParallelCluster Template. Version: aws-parallelcluster-2.1.1", "Metadata": { "AWS::CloudFormation::Interface": { "ParameterGroups": [ @@ -1077,12 +1077,27 @@ "aws-us-gov" ] }, + "ChinaRegion": { + "Fn::Equals": [ + { + "Ref": "AWS::Partition" + }, + "aws-cn" + ] + }, "CreateLaunchTemplate": { "Fn::And": [ { "Fn::Not": [ { - "Condition": "GovCloudRegion" + "Fn::Or": [ + { + "Condition": "ChinaRegion" + }, + { + "Condition": "GovCloudRegion" + } + ] } ] }, @@ -1094,7 +1109,14 @@ "CreateLaunchConfig": { "Fn::And": [ { - "Condition": "GovCloudRegion" + "Fn::Or": [ + { + "Condition": "ChinaRegion" + }, + { + "Condition": "GovCloudRegion" + } + ] }, { "Condition": "CreateComputeFleet" @@ -1149,133 +1171,141 @@ "Mappings": { "AWSRegionOS2AMI": { "ap-northeast-1": { - "alinux": "ami-03780c8327e4be0ac", - "centos6": "ami-07d83b268e25e0953", - "centos7": "ami-03043938e7fc33ba9", - "ubuntu1404": "ami-08a720b6bbe06dc15", - "ubuntu1604": "ami-00bd60c479d9b05f2" + "alinux": "ami-0ac1d0c35dc3b2a97", + "centos6": "ami-003cfe6266cadd576", + "centos7": "ami-0c0049e4eeb0ef1ac", + "ubuntu1404": "ami-053a84278e34b7a59", + "ubuntu1604": "ami-026669cfcef23b3de" }, "ap-northeast-2": { - "alinux": "ami-01eaa2848f762416f", - "centos6": "ami-044f3ad0badddf405", - "centos7": "ami-07ee16085dc31f6ed", - "ubuntu1404": "ami-03595544457ee270b", - "ubuntu1604": "ami-00317e3a1c4d47b23" + "alinux": "ami-0dddb89f66485f828", + "centos6": "ami-032b3f2a4f1ac91a0", + "centos7": "ami-00f8f46a043a04530", + "ubuntu1404": "ami-03d314c5ac10beb53", + "ubuntu1604": "ami-048214d1413ed8462" }, "ap-northeast-3": { - "alinux": "ami-05625a3c29c8d158d", - "centos6": "ami-07261626f0608a9fc", - "centos7": "ami-01116deb7081a4d93", - "ubuntu1404": "ami-0f4afbf455ad42c20", - "ubuntu1604": "ami-07942e464687e23cd" + "alinux": "ami-0dbb09bfada65298f", + "centos6": "ami-0c4de7aece7b2db33", + "centos7": "ami-0043e5e12872a00d4", + "ubuntu1404": "ami-070f9069b20a797ec", + "ubuntu1604": "ami-038900edb73cb9496" }, "ap-south-1": { - "alinux": "ami-0cb4b6a940244dd2f", - "centos6": "ami-007862493aa8721a7", - "centos7": "ami-0823edb0c94157ff4", - "ubuntu1404": "ami-0821b7c6568999205", - "ubuntu1604": "ami-0fa6a7a4decdae641" + "alinux": "ami-0ed1bb8f2ab0edffb", + "centos6": "ami-0581583c3d7507d9e", + "centos7": "ami-03dd063b05c3082f1", + "ubuntu1404": "ami-0641d7ca885530589", + "ubuntu1604": "ami-0cc424ec58256ea88" }, "ap-southeast-1": { - "alinux": "ami-099312510208cc0d3", - "centos6": "ami-069361480706f302e", - "centos7": "ami-074acbf20aa22df0c", - "ubuntu1404": "ami-0563d0646a1cc2dde", - "ubuntu1604": "ami-00db94a0965a886a6" + "alinux": "ami-058bcd8377aba9bef", + "centos6": "ami-0d00309b80b772532", + "centos7": "ami-00fcdc55bd29f691e", + "ubuntu1404": "ami-0e3de99412375e882", + "ubuntu1604": "ami-059ba95190db36590" }, "ap-southeast-2": { - "alinux": "ami-04c142f780a8426c2", - "centos6": "ami-06e0db8bdac11c698", - "centos7": "ami-055dd6211ddf02af6", - "ubuntu1404": "ami-08209d8fa90d40e5a", - "ubuntu1604": "ami-05c16d76720e55964" + "alinux": "ami-012557ce9426ef1a0", + "centos6": "ami-00ecf7e455945e8bc", + "centos7": "ami-0baa1787fd7a71950", + "ubuntu1404": "ami-09eae4580e8fc835a", + "ubuntu1604": "ami-04df2433ab61d3f37" }, "ca-central-1": { - "alinux": "ami-0907f9cad7dbf65f6", - "centos6": "ami-00875bb416baf6556", - "centos7": "ami-052fb2fbfaef380d3", - "ubuntu1404": "ami-0b05531dac34df70a", - "ubuntu1604": "ami-02740f68858b9fc7a" + "alinux": "ami-059dda8ee9af7c20c", + "centos6": "ami-0416d41ed6dbc0bd7", + "centos7": "ami-0c2a8b54dfd0c405f", + "ubuntu1404": "ami-08aeb7a57f73b58ab", + "ubuntu1604": "ami-039a1b0ada060b5ce" + }, + "cn-north-1": { + "alinux": "ami-00237da8e056b0936", + "ubuntu1404": "ami-00f2cae5406fb3fce", + "ubuntu1604": "ami-072046713a0458796" + }, + "cn-northwest-1": { + "alinux": "ami-0974fc483e449f5ee" }, "eu-central-1": { - "alinux": "ami-0595a707dc47e9671", - "centos6": "ami-07f13c4704a54b165", - "centos7": "ami-05af6f87fc63e3d71", - "ubuntu1404": "ami-0a2c2f9760d34f596", - "ubuntu1604": "ami-003cc41d49855e1ef" + "alinux": "ami-09cff6787920e967c", + "centos6": "ami-0b1f52047bb2b7f83", + "centos7": "ami-00b3f34240b6021dd", + "ubuntu1404": "ami-0b24a435216670b4a", + "ubuntu1604": "ami-0d816068d1164f4d2" }, "eu-north-1": { - "alinux": "ami-09b05710227925ceb", - "centos6": "ami-03e7f9bfdcdfe6eca", - "centos7": "ami-0e933380e094ab102", - "ubuntu1404": "ami-04b0fa5a3509e4569", - "ubuntu1604": "ami-0ceb6b8e665efdb87" + "alinux": "ami-086f4f382fb1119f0", + "centos6": "ami-04fc976e8108996e6", + "centos7": "ami-07ac74edc9e96b343", + "ubuntu1404": "ami-0921b515f8ed512c3", + "ubuntu1604": "ami-046c32486a9abf742" }, "eu-west-1": { - "alinux": "ami-0f0865f124c533e6f", - "centos6": "ami-0080ba82e12f317b9", - "centos7": "ami-0e249c302ebb32aef", - "ubuntu1404": "ami-06e7f63766bac032b", - "ubuntu1604": "ami-02fbb78584905c216" + "alinux": "ami-080a7d5c75253bd1b", + "centos6": "ami-073f1f5db6cfdd3d1", + "centos7": "ami-0879d97613ba8075a", + "ubuntu1404": "ami-076fbdec21cd5c940", + "ubuntu1604": "ami-0f641e63ebaf647b1" }, "eu-west-2": { - "alinux": "ami-08c01fbf6cc43e166", - "centos6": "ami-0dfae4fa33e540f25", - "centos7": "ami-0836360ea1ef8e1a9", - "ubuntu1404": "ami-0bf0e4815ea7c3f08", - "ubuntu1604": "ami-090cb83529f72bd53" + "alinux": "ami-0b284f1028a743865", + "centos6": "ami-0ace56d1d9c1aa466", + "centos7": "ami-06aa0ca6f28c29fdb", + "ubuntu1404": "ami-0fb01b5b56bc27509", + "ubuntu1604": "ami-067c1c0157477c166" }, "eu-west-3": { - "alinux": "ami-033bcfc19853275cb", - "centos6": "ami-0fae6489cef475f35", - "centos7": "ami-0b8e45374e5e2f6ec", - "ubuntu1404": "ami-0038fece5c9fdcf52", - "ubuntu1604": "ami-03fcec4562b3cef3c" + "alinux": "ami-099197e40d0c5de6e", + "centos6": "ami-0d377bccde07c887d", + "centos7": "ami-0b28dcaf2b4d00d87", + "ubuntu1404": "ami-0ea8b79c622d0a4a2", + "ubuntu1604": "ami-0102caf8c6ec0768a" }, "sa-east-1": { - "alinux": "ami-0ffb13b9401cc9f43", - "centos6": "ami-03a2cb750e02692c6", - "centos7": "ami-091c8ba9e3b0a2b6b", - "ubuntu1404": "ami-05997e9be14a0f785", - "ubuntu1604": "ami-0ba286d64312d26f5" + "alinux": "ami-02e542e4935ff9647", + "centos6": "ami-0b1c8f6aad337d5b6", + "centos7": "ami-00a570d9536621660", + "ubuntu1404": "ami-00cf3910c959e9fd5", + "ubuntu1604": "ami-0ef5c70aec338bcfb" }, "us-east-1": { - "alinux": "ami-025f4816964498a18", - "centos6": "ami-09d4b0433c3d08284", - "centos7": "ami-0ed9fbb6accaa70ba", - "ubuntu1404": "ami-09ae02fc417765f4c", - "ubuntu1604": "ami-057dc350619a0bf58" + "alinux": "ami-0cd2dd3198972a68c", + "centos6": "ami-0919d912e0e33d247", + "centos7": "ami-09d092c1b054832df", + "ubuntu1404": "ami-095016c5ff0ab7ae6", + "ubuntu1604": "ami-098c8e582ca818cff" }, "us-east-2": { - "alinux": "ami-0381cb7486cdc973f", - "centos6": "ami-0b8ba3e22e08e3870", - "centos7": "ami-059415e43c0827abf", - "ubuntu1404": "ami-0121231f2883a268e", - "ubuntu1604": "ami-04c283d05058f6894" + "alinux": "ami-057e70f0fbb007ab6", + "centos6": "ami-0bbd43b2b8991cdae", + "centos7": "ami-0e0cda1cdec7fcecf", + "ubuntu1404": "ami-02a21f90af8a453f0", + "ubuntu1604": "ami-055279b0b09d12a71" }, "us-gov-east-1": { - "alinux": "ami-08e554700cf18e130", - "ubuntu1404": "ami-0b621c3f09e5e06c4", - "ubuntu1604": "ami-083ba3c9a5c412c8f" + "alinux": "ami-022f96b137a63c9ff", + "ubuntu1404": "ami-09274e2587c4154f2", + "ubuntu1604": "ami-0b664ce8c427b77eb" }, "us-gov-west-1": { - "alinux": "ami-a5ee8fc4", - "ubuntu1404": "ami-50e98831", - "ubuntu1604": "ami-54f19035" + "alinux": "ami-fefb989f", + "ubuntu1404": "ami-6cf0930d", + "ubuntu1604": "ami-5ef6953f" }, "us-west-1": { - "alinux": "ami-0f7431c55ad80bfce", - "centos6": "ami-06bd36cf904ec0f9a", - "centos7": "ami-02ca1c89adc7b29f0", - "ubuntu1404": "ami-0e30c804ef45d3819", - "ubuntu1604": "ami-0d1f81f40426d7c25" + "alinux": "ami-09686d4090e35a702", + "centos6": "ami-0ac1cf1e68288fa36", + "centos7": "ami-084c065e503e74449", + "ubuntu1404": "ami-099ebeb456dbf0646", + "ubuntu1604": "ami-0db5e85c0b1ce2c20" }, "us-west-2": { - "alinux": "ami-0bdc62a7fa9026972", - "centos6": "ami-084bb82c022317cc5", - "centos7": "ami-03abc3d448a987e73", - "ubuntu1404": "ami-0440f447631cb8f6c", - "ubuntu1604": "ami-0f7c3a7a9e2f3d6a3" + "alinux": "ami-0c588cdc9e91b0db3", + "centos6": "ami-08b28682da5721f5b", + "centos7": "ami-070345cb145f2d2f0", + "ubuntu1404": "ami-05ddc7ec0edb9069f", + "ubuntu1604": "ami-02393fa61ac61547a" } }, "OSFeatures": { @@ -1300,10 +1330,21 @@ "RootDevice": "/dev/sda1" } }, + "Partition2Url": { + "aws": { + "url": "amazonaws.com" + }, + "aws-us-gov": { + "url": "amazonaws.com" + }, + "aws-cn": { + "url": "amazonaws.com.cn" + } + }, "PackagesVersions": { "default": { - "parallelcluster": "2.1.0", - "cookbook": "aws-parallelcluster-cookbook-2.1.0", + "parallelcluster": "2.1.1", + "cookbook": "aws-parallelcluster-cookbook-2.1.1", "chef": "14.2.0", "ridley": "5.1.1", "berkshelf": "7.0.4", @@ -1336,15 +1377,15 @@ }, "TemplateURL": { "Fn::Sub": [ - "https://${s3_domain}/${AWS::Region}-aws-parallelcluster/templates/efs-substack-${version}.cfn.json", + "https://s3.${AWS::Region}.${s3_url}/${AWS::Region}-aws-parallelcluster/templates/efs-substack-${version}.cfn.json", { - "s3_domain": { - "Fn::If": [ - "GovCloudRegion", + "s3_url": { + "Fn::FindInMap": [ + "Partition2Url", { - "Fn::Sub": "s3-${AWS::Region}.amazonaws.com" + "Ref": "AWS::Partition" }, - "s3.amazonaws.com" + "url" ] }, "version": { @@ -1457,7 +1498,22 @@ "Effect": "Allow", "Principal": { "Service": [ - "ec2.amazonaws.com" + { + "Fn::Sub": [ + "ec2.${s3_url}", + { + "s3_url": { + "Fn::FindInMap": [ + "Partition2Url", + { + "Ref": "AWS::Partition" + }, + "url" + ] + } + } + ] + } ] }, "Action": [ @@ -2053,13 +2109,13 @@ " if [ \"${apt}\" == \"0\" ]; then\n", " apt-cache search build-essential; apt-get clean; apt-get update; apt-get -y install build-essential curl wget jq\n", " fi\n", - " which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz)\n", + " which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://s3.${_region}.${s3_url}/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz)\n", " mkdir -p /etc/chef && chown -R root:root /etc/chef\n", - " curl -L https://www.chef.io/chef/install.sh | bash -s -- -v ${chef_version}\n", + " curl --retry 3 -L https://www.chef.io/chef/install.sh | bash -s -- -v ${chef_version}\n", " /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:${ridley_version} berkshelf:${berkshelf_version}\n", - " curl -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${cookbook_url}\n", - " curl -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.date ${cookbook_url}.date\n", - " curl -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.md5 ${cookbook_url}.md5\n", + " curl --retry 3 -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${cookbook_url}\n", + " curl --retry 3 -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.date ${cookbook_url}.date\n", + " curl --retry 3 -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.md5 ${cookbook_url}.md5\n", " mkdir /opt/parallelcluster && echo ${parallelcluster_version} | tee /opt/parallelcluster/.bootstrapped\n", "}\n", "proxy=", @@ -2085,27 +2141,26 @@ " proxy_args=\"\"\n", " touch /tmp/proxy.sh\n", "fi\n", - "if [ \"${custom_cookbook}\" != \"NONE\" ]; then\n", - " cookbook_url=${custom_cookbook}\n", - "else\n", - " if [ \"", - { - "Ref": "AWS::Region" - }, - "\" == \"us-east-1\" ]; then\n", - " s3_prefix=s3\n", - " else\n", - " s3_prefix=s3-", + " export _region=", { "Ref": "AWS::Region" }, "\n", - " fi\n", - " cookbook_url=https://${s3_prefix}.amazonaws.com/", + "s3_url=", { - "Ref": "AWS::Region" + "Fn::FindInMap": [ + "Partition2Url", + { + "Ref": "AWS::Partition" + }, + "url" + ] }, - "-aws-parallelcluster/cookbooks/", + "\n", + "if [ \"${custom_cookbook}\" != \"NONE\" ]; then\n", + " cookbook_url=${custom_cookbook}\n", + "else\n", + " cookbook_url=https://s3.${_region}.${s3_url}/${_region}-aws-parallelcluster/cookbooks/", { "Fn::FindInMap": [ "PackagesVersions", @@ -2171,7 +2226,7 @@ "fi\n", "mkdir /tmp/cookbooks\n", "cd /tmp/cookbooks\n", - "curl -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz -z \"$(cat /etc/chef/aws-parallelcluster-cookbook.tgz.date)\" ${cookbook_url}\n", + "curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz -z \"$(cat /etc/chef/aws-parallelcluster-cookbook.tgz.date)\" ${cookbook_url}\n", "tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz\n", "cd /tmp\n", "# Call CloudFormation\n", @@ -2381,10 +2436,25 @@ "getCookbooks": { "commands": { "berk": { - "command": ". /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks --delete; done ", + "command": "if [ ! -f /opt/parallelcluster/.bootstrapped -o \"$(cat /opt/parallelcluster/.bootstrapped)\" != \"$parallelcluster_version\" ]; then . /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks --delete; done; fi", "cwd": "/tmp/cookbooks", "env": { - "HOME": "/tmp" + "HOME": "/tmp", + "parallelcluster_version": { + "Fn::Join": [ + "", + [ + "aws-parallelcluster-", + { + "Fn::FindInMap": [ + "PackagesVersions", + "default", + "parallelcluster" + ] + } + ] + ] + } } } } @@ -2464,7 +2534,7 @@ ], "LaunchConfigurationName": { "Fn::If": [ - "GovCloudRegion", + "CreateLaunchConfig", { "Ref": "ComputeServerLaunchConfig" }, @@ -2869,13 +2939,13 @@ " if [ \"${apt}\" == \"0\" ]; then\n", " apt-cache search build-essential; apt-get clean; apt-get update; apt-get -y install build-essential curl wget jq\n", " fi\n", - " which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz)\n", + " which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://s3.${_region}.${s3_url}/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz)\n", " mkdir -p /etc/chef && chown -R root:root /etc/chef\n", - " curl -L https://www.chef.io/chef/install.sh | bash -s -- -v ${chef_version}\n", + " curl --retry 3 -L https://www.chef.io/chef/install.sh | bash -s -- -v ${chef_version}\n", " /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:${ridley_version} berkshelf:${berkshelf_version}\n", - " curl -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${cookbook_url}\n", - " curl -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.date ${cookbook_url}.date\n", - " curl -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.md5 ${cookbook_url}.md5\n", + " curl --retry 3 -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${cookbook_url}\n", + " curl --retry 3 -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.date ${cookbook_url}.date\n", + " curl --retry 3 -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.md5 ${cookbook_url}.md5\n", " mkdir /opt/parallelcluster && echo ${parallelcluster_version} | tee /opt/parallelcluster/.bootstrapped\n", "}\n", "proxy=", @@ -2901,27 +2971,26 @@ " proxy_args=\"\"\n", " touch /tmp/proxy.sh\n", "fi\n", - "if [ \"${custom_cookbook}\" != \"NONE\" ]; then\n", - " cookbook_url=${custom_cookbook}\n", - "else\n", - " if [ \"", - { - "Ref": "AWS::Region" - }, - "\" == \"us-east-1\" ]; then\n", - " s3_prefix=s3\n", - " else\n", - " s3_prefix=s3-", + " _region=", { "Ref": "AWS::Region" }, "\n", - " fi\n", - " cookbook_url=https://${s3_prefix}.amazonaws.com/", + "s3_url=", { - "Ref": "AWS::Region" + "Fn::FindInMap": [ + "Partition2Url", + { + "Ref": "AWS::Partition" + }, + "url" + ] }, - "-aws-parallelcluster/cookbooks/", + "\n", + "if [ \"${custom_cookbook}\" != \"NONE\" ]; then\n", + " cookbook_url=${custom_cookbook}\n", + "else\n", + " cookbook_url=https://s3.${_region}.${s3_url}/${_region}-aws-parallelcluster/cookbooks/", { "Fn::FindInMap": [ "PackagesVersions", @@ -2987,7 +3056,7 @@ "fi\n", "mkdir /tmp/cookbooks\n", "cd /tmp/cookbooks\n", - "curl -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz -z \"$(cat /etc/chef/aws-parallelcluster-cookbook.tgz.date)\" ${cookbook_url}\n", + "curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz -z \"$(cat /etc/chef/aws-parallelcluster-cookbook.tgz.date)\" ${cookbook_url}\n", "tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz\n", "cd /tmp\n", "# Call CloudFormation\n", @@ -3172,10 +3241,25 @@ "getCookbooks": { "commands": { "berk": { - "command": ". /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks --delete; done ", + "command": "if [ ! -f /opt/parallelcluster/.bootstrapped -o \"$(cat /opt/parallelcluster/.bootstrapped)\" != \"$parallelcluster_version\" ]; then . /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks --delete; done; fi", "cwd": "/tmp/cookbooks", "env": { - "HOME": "/tmp" + "HOME": "/tmp", + "parallelcluster_version": { + "Fn::Join": [ + "", + [ + "aws-parallelcluster-", + { + "Fn::FindInMap": [ + "PackagesVersions", + "default", + "parallelcluster" + ] + } + ] + ] + } } } } @@ -3526,13 +3610,13 @@ " if [ \"${apt}\" == \"0\" ]; then\n", " apt-cache search build-essential; apt-get clean; apt-get update; apt-get -y install build-essential curl wget jq\n", " fi\n", - " which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz)\n", + " which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://s3.${_region}.${s3_url}/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz)\n", " mkdir -p /etc/chef && chown -R root:root /etc/chef\n", - " curl -L https://www.chef.io/chef/install.sh | bash -s -- -v ${chef_version}\n", + " curl --retry 3 -L https://www.chef.io/chef/install.sh | bash -s -- -v ${chef_version}\n", " /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:${ridley_version} berkshelf:${berkshelf_version}\n", - " curl -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${cookbook_url}\n", - " curl -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.date ${cookbook_url}.date\n", - " curl -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.md5 ${cookbook_url}.md5\n", + " curl --retry 3 -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz ${cookbook_url}\n", + " curl --retry 3 -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.date ${cookbook_url}.date\n", + " curl --retry 3 -s -L -o /etc/chef/aws-parallelcluster-cookbook.tgz.md5 ${cookbook_url}.md5\n", " mkdir /opt/parallelcluster && echo ${parallelcluster_version} | tee /opt/parallelcluster/.bootstrapped\n", "}\n", "proxy=", @@ -3558,27 +3642,26 @@ " proxy_args=\"\"\n", " touch /tmp/proxy.sh\n", "fi\n", - "if [ \"${custom_cookbook}\" != \"NONE\" ]; then\n", - " cookbook_url=${custom_cookbook}\n", - "else\n", - " if [ \"", - { - "Ref": "AWS::Region" - }, - "\" == \"us-east-1\" ]; then\n", - " s3_prefix=s3\n", - " else\n", - " s3_prefix=s3-", + " export _region=", { "Ref": "AWS::Region" }, "\n", - " fi\n", - " cookbook_url=https://${s3_prefix}.amazonaws.com/", + "s3_url=", { - "Ref": "AWS::Region" + "Fn::FindInMap": [ + "Partition2Url", + { + "Ref": "AWS::Partition" + }, + "url" + ] }, - "-aws-parallelcluster/cookbooks/", + "\n", + "if [ \"${custom_cookbook}\" != \"NONE\" ]; then\n", + " cookbook_url=${custom_cookbook}\n", + "else\n", + " cookbook_url=https://s3.${_region}.${s3_url}/${_region}-aws-parallelcluster/cookbooks/", { "Fn::FindInMap": [ "PackagesVersions", @@ -3644,7 +3727,7 @@ "fi\n", "mkdir /tmp/cookbooks\n", "cd /tmp/cookbooks\n", - "curl -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz -z \"$(cat /etc/chef/aws-parallelcluster-cookbook.tgz.date)\" ${cookbook_url}\n", + "curl --retry 3 -v -L -o /etc/chef/aws-parallelcluster-cookbook.tgz -z \"$(cat /etc/chef/aws-parallelcluster-cookbook.tgz.date)\" ${cookbook_url}\n", "tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz\n", "cd /tmp\n", "# Call CloudFormation\n", @@ -3831,10 +3914,25 @@ "getCookbooks": { "commands": { "berk": { - "command": ". /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks --delete; done ", + "command": "if [ ! -f /opt/parallelcluster/.bootstrapped -o \"$(cat /opt/parallelcluster/.bootstrapped)\" != \"$parallelcluster_version\" ]; then . /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks --delete; done; fi", "cwd": "/tmp/cookbooks", "env": { - "HOME": "/tmp" + "HOME": "/tmp", + "parallelcluster_version": { + "Fn::Join": [ + "", + [ + "aws-parallelcluster-", + { + "Fn::FindInMap": [ + "PackagesVersions", + "default", + "parallelcluster" + ] + } + ] + ] + } } } } @@ -4098,6 +4196,15 @@ } ] }, + "S3Url": { + "Fn::FindInMap": [ + "Partition2Url", + { + "Ref": "AWS::Partition" + }, + "url" + ] + }, "FileSystemTags": { "Fn::Sub": [ "efs=${efs}, multiebs=${NumberOfEBSVol}, raid=${raid}", @@ -4268,15 +4375,15 @@ }, { "Fn::Sub": [ - "https://${s3_domain}/${AWS::Region}-aws-parallelcluster/templates/batch-substack-${version}.cfn.json", + "https://s3.${AWS::Region}.${s3_url}/${AWS::Region}-aws-parallelcluster/templates/batch-substack-${version}.cfn.json", { - "s3_domain": { - "Fn::If": [ - "GovCloudRegion", + "s3_url": { + "Fn::FindInMap": [ + "Partition2Url", { - "Fn::Sub": "s3-${AWS::Region}.amazonaws.com" + "Ref": "AWS::Partition" }, - "s3.amazonaws.com" + "url" ] }, "version": { @@ -4328,7 +4435,22 @@ "Effect": "Allow", "Principal": { "Service": [ - "lambda.amazonaws.com" + { + "Fn::Sub": [ + "lambda.${s3_url}", + { + "s3_url": { + "Fn::FindInMap": [ + "Partition2Url", + { + "Ref": "AWS::Partition" + }, + "url" + ] + } + } + ] + } ] } } @@ -4438,15 +4560,15 @@ }, "TemplateURL": { "Fn::Sub": [ - "https://${s3_domain}/${AWS::Region}-aws-parallelcluster/templates/raid-substack-${version}.cfn.json", + "https://s3.${AWS::Region}.${s3_url}/${AWS::Region}-aws-parallelcluster/templates/raid-substack-${version}.cfn.json", { - "s3_domain": { - "Fn::If": [ - "GovCloudRegion", + "s3_url": { + "Fn::FindInMap": [ + "Partition2Url", { - "Fn::Sub": "s3-${AWS::Region}.amazonaws.com" + "Ref": "AWS::Partition" }, - "s3.amazonaws.com" + "url" ] }, "version": { @@ -4496,15 +4618,15 @@ }, "TemplateURL": { "Fn::Sub": [ - "https://${s3_domain}/${AWS::Region}-aws-parallelcluster/templates/ebs-substack-${version}.cfn.json", + "https://s3.${AWS::Region}.${s3_url}/${AWS::Region}-aws-parallelcluster/templates/ebs-substack-${version}.cfn.json", { - "s3_domain": { - "Fn::If": [ - "GovCloudRegion", + "s3_url": { + "Fn::FindInMap": [ + "Partition2Url", { - "Fn::Sub": "s3-${AWS::Region}.amazonaws.com" + "Ref": "AWS::Partition" }, - "s3.amazonaws.com" + "url" ] }, "version": { diff --git a/cloudformation/batch-substack.cfn.json b/cloudformation/batch-substack.cfn.json index 3678879c3b..43de764753 100644 --- a/cloudformation/batch-substack.cfn.json +++ b/cloudformation/batch-substack.cfn.json @@ -73,6 +73,10 @@ "FileSystemTags": { "Description": "efs=true/false, multiebs=[1-5]", "Type": "String" + }, + "S3Url": { + "Description": "amazonaws.com or amazonaws.com.cn", + "Type": "String" } }, "Conditions": { @@ -106,7 +110,9 @@ "Sid": "", "Effect": "Allow", "Principal": { - "Service": "ec2.amazonaws.com" + "Service": { + "Fn::Sub": "ec2.${S3Url}" + } }, "Action": "sts:AssumeRole" } @@ -126,7 +132,9 @@ { "Effect": "Allow", "Principal": { - "Service": "ecs-tasks.amazonaws.com" + "Service": { + "Fn::Sub": "ecs-tasks.${S3Url}" + } }, "Action": "sts:AssumeRole" } @@ -189,7 +197,9 @@ { "Effect": "Allow", "Principal": { - "Service": "batch.amazonaws.com" + "Service": { + "Fn::Sub": "batch.${S3Url}" + } }, "Action": "sts:AssumeRole" } @@ -346,7 +356,9 @@ { "Effect": "Allow", "Principal": { - "Service": "spotfleet.amazonaws.com" + "Service": { + "Fn::Sub": "spotfleet.${S3Url}" + } }, "Action": "sts:AssumeRole" } @@ -458,7 +470,7 @@ ] }, "Image": { - "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${DockerImagesRepo}:${OS}" + "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${S3Url}/${DockerImagesRepo}:${OS}" }, "Vcpus": 1, "Memory": 512, @@ -516,7 +528,7 @@ "TargetNodes": "0:", "Container": { "Image": { - "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${DockerImagesRepo}:${OS}" + "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${S3Url}/${DockerImagesRepo}:${OS}" }, "JobRoleArn": { "Fn::GetAtt": [ @@ -707,7 +719,9 @@ "Effect": "Allow", "Principal": { "Service": [ - "codebuild.amazonaws.com" + { + "Fn::Sub": "codebuild.${S3Url}" + } ] } } @@ -768,7 +782,9 @@ "Effect": "Allow", "Principal": { "Service": [ - "lambda.amazonaws.com" + { + "Fn::Sub": "lambda.${S3Url}" + } ] } } @@ -860,7 +876,9 @@ "Effect": "Allow", "Principal": { "Service": [ - "lambda.amazonaws.com" + { + "Fn::Sub": "lambda.${S3Url}" + } ] } } @@ -936,7 +954,9 @@ "Arn" ] }, - "Principal": "events.amazonaws.com", + "Principal": { + "Fn::Sub": "events.${S3Url}" + }, "SourceArn": { "Fn::GetAtt": [ "CodeBuildNotificationRule", @@ -1029,7 +1049,9 @@ "Effect": "Allow", "Principal": { "Service": [ - "lambda.amazonaws.com" + { + "Fn::Sub": "lambda.${S3Url}" + } ] } } diff --git a/docs/conf.py b/docs/conf.py index 90643e7a6b..31863370e7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,9 +51,9 @@ # built documents. # # The short X.Y version. -version = '2.0' +version = '2.1' # The full version, including alpha/beta/rc tags. -release = '2.1.0' +release = '2.1.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/configuration.rst b/docs/configuration.rst index c80bd29ff8..c047852b2f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -331,9 +331,11 @@ placement """"""""" Cluster placement logic. This enables the whole cluster or only compute to use the placement group. +Can be ``cluster`` or ``compute``. + This does not apply to awsbatch. -Defaults to cluster. :: +Defaults to ``cluster``. :: placement = cluster @@ -388,6 +390,18 @@ Defaults to alinux. Available options are: alinux, centos6, centos7, ubuntu1404 Note: The base_os determines the username used to log into the cluster. +Supported OS's by region. Note that commercial is all supported regions such as us-east-1, us-west-2 etc. :: + + ============== ====== ============ ============ ============= ============ + region alinux centos6 centos7 ubuntu1404 ubuntu1604 + ============== ====== ============ ============ ============= ============ + commercial True True True True True + us-gov-west-1 True False False True True + us-gov-east-1 True False False True True + cn-north-1 True False False True True + cn-northwest-1 True False False False False + ============== ====== ============ ============ ============= ============ + * CentOS 6 & 7: ``centos`` * Ubuntu: ``ubuntu`` * Amazon Linux: ``ec2-user`` :: diff --git a/docs/tutorials/code_samples/batch_mpi/submit_mpi.sh b/docs/tutorials/code_samples/batch_mpi/submit_mpi.sh index fd182f0c7f..9e2db5338b 100644 --- a/docs/tutorials/code_samples/batch_mpi/submit_mpi.sh +++ b/docs/tutorials/code_samples/batch_mpi/submit_mpi.sh @@ -2,16 +2,34 @@ echo "ip container: $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)" echo "ip host: $(curl -s "http://169.254.169.254/latest/meta-data/local-ipv4")" +# get shared dir +IFS=',' _shared_dirs=(${PCLUSTER_SHARED_DIRS}) +_shared_dir=${_shared_dirs[0]} +_job_dir="${_shared_dir}/${AWS_BATCH_JOB_ID%#*}-${AWS_BATCH_JOB_ATTEMPT}" +_exit_code_file="${_job_dir}/batch-exit-code" + if [[ "${AWS_BATCH_JOB_NODE_INDEX}" -eq "${AWS_BATCH_JOB_MAIN_NODE_INDEX}" ]]; then + echo "Hello I'm the main node $(hostname)! I run the mpi job!" + + mkdir -p "${_job_dir}" + echo "Compiling..." - /usr/lib64/openmpi/bin/mpicc -o /shared/mpi_hello_world /shared/mpi_hello_world.c + /usr/lib64/openmpi/bin/mpicc -o "${_job_dir}/mpi_hello_world" "${_shared_dir}/mpi_hello_world.c" + + echo "Running..." + /usr/lib64/openmpi/bin/mpirun --mca btl_tcp_if_include eth0 --allow-run-as-root --machinefile "${HOME}/hostfile" "${_job_dir}/mpi_hello_world" - echo "Hello I'm the main node! I run the mpi job!" - /usr/lib64/openmpi/bin/mpirun --mca btl_tcp_if_include eth0 --allow-run-as-root --machinefile "${HOME}/hostfile" /shared/mpi_hello_world + # Write exit status code + echo "0" > "${_exit_code_file}" + # Waiting for compute nodes to terminate + sleep 30 else - echo "Hello I'm a compute note! I let the main node orchestrate the mpi execution!" + echo "Hello I'm the compute node $(hostname)! I let the main node orchestrate the mpi execution!" # Since mpi orchestration happens on the main node, we need to make sure the containers representing the compute - # nodes are not terminated. A simple trick is to run an infinite sleep. - # All compute nodes will be terminated by Batch once the main node exits. - sleep infinity + # nodes are not terminated. A simple trick is to wait for a file containing the status code to be created. + # All compute nodes are terminated by Batch if the main node exits abruptly. + while [ ! -f "${_exit_code_file}" ]; do + sleep 2 + done + exit $(cat "${_exit_code_file}") fi diff --git a/tests/config_raid b/tests/config_raid index 3afc4d97fd..c96a898727 100644 --- a/tests/config_raid +++ b/tests/config_raid @@ -12,8 +12,8 @@ key_name = id_rsa vpc_settings = public maintain_initial_size = true initial_queue_size = 1 -template_url = https://s3.us-east-2.amazonaws.com/shuningc-raid/aws-parallelcluster.cfn.json -custom_chef_cookbook = https://s3.us-east-2.amazonaws.com/shuningc-raid/cookbooks/cfncluster-cookbook-2.0.2.tgz +template_url = +custom_chef_cookbook = [cluster raid2Vol] base_os = ubuntu1604 @@ -22,8 +22,8 @@ vpc_settings = public maintain_initial_size = true initial_queue_size = 1 raid_settings = rs2 -template_url = https://s3.us-east-2.amazonaws.com/shuningc-raid/aws-parallelcluster.cfn.json -custom_chef_cookbook = https://s3.us-east-2.amazonaws.com/shuningc-raid/cookbooks/cfncluster-cookbook-2.0.2.tgz +template_url = +custom_chef_cookbook = [cluster raid5Vol] base_os = ubuntu1604 @@ -32,8 +32,8 @@ vpc_settings = public maintain_initial_size = true initial_queue_size = 1 raid_settings = rs5 -template_url = https://s3.us-east-2.amazonaws.com/shuningc-raid/cfncluster.cfn.json -custom_chef_cookbook = https://s3.us-east-2.amazonaws.com/shuningc-raid/cookbooks/cfncluster-cookbook-2.0.0.tgz +template_url = +custom_chef_cookbook = [cluster raid1Vol] base_os = ubuntu1604 @@ -42,8 +42,8 @@ vpc_settings = public maintain_initial_size = true initial_queue_size = 1 raid_settings = rs1 -template_url = https://s3.us-east-2.amazonaws.com/shuningc-raid/cfncluster.cfn.json -custom_chef_cookbook = https://s3.us-east-2.amazonaws.com/shuningc-raid/cookbooks/cfncluster-cookbook-1.6.0.tgz +template_url = +custom_chef_cookbook = [cluster raid6Vol] base_os = ubuntu1604 @@ -52,8 +52,8 @@ vpc_settings = public maintain_initial_size = true initial_queue_size = 1 raid_settings = rs6 -template_url = https://s3.us-east-2.amazonaws.com/shuningc-raid/cfncluster.cfn.json -custom_chef_cookbook = https://s3.us-east-2.amazonaws.com/shuningc-raid/cookbooks/cfncluster-cookbook-1.6.0.tgz +template_url = +custom_chef_cookbook = [cluster raidIOPS] base_os = ubuntu1604 @@ -62,8 +62,8 @@ vpc_settings = public maintain_initial_size = true initial_queue_size = 1 raid_settings = rsIOPS -template_url = https://s3.us-east-2.amazonaws.com/shuningc-raid/cfncluster.cfn.json -custom_chef_cookbook = https://s3.us-east-2.amazonaws.com/shuningc-raid/cookbooks/cfncluster-cookbook-1.6.0.tgz +template_url = +custom_chef_cookbook = [cluster raidType] base_os = ubuntu1604 @@ -72,8 +72,8 @@ vpc_settings = public maintain_initial_size = true initial_queue_size = 1 raid_settings = rsT -template_url = https://s3.us-east-2.amazonaws.com/shuningc-raid/cfncluster.cfn.json -custom_chef_cookbook = https://s3.us-east-2.amazonaws.com/shuningc-raid/cookbooks/cfncluster-cookbook-1.6.0.tgz +template_url = +custom_chef_cookbook = [vpc public] vpc_id = vpc-f9cff091 diff --git a/util/batch-instance-whitelist.py b/util/batch-instance-whitelist.py index 67ae64fc67..5b64ed264f 100755 --- a/util/batch-instance-whitelist.py +++ b/util/batch-instance-whitelist.py @@ -26,12 +26,24 @@ import boto3 from botocore.exceptions import ClientError -# commercial regions unsupported by aws batch -UNSUPPORTED_REGIONS = set(["ap-northeast-3", "eu-west-3"]) - +# regions unsupported by aws batch +UNSUPPORTED_REGIONS = set( + ["ap-northeast-3", "eu-north-1", "cn-north-1", "cn-northwest-1", "us-gov-east-1", "us-gov-west-1"] +) + + +def get_all_aws_regions(partition): + if partition == "commercial": + region = "us-east-1" + elif partition == "govcloud": + region = "us-gov-west-1" + elif partition == "china": + region = "cn-north-1" + else: + print("Unsupported partition %s" % partition) + sys.exit(1) -def get_all_aws_regions(): - ec2 = boto3.client("ec2") + ec2 = boto3.client("ec2", region_name=region) return set(sorted(r.get("RegionName") for r in ec2.describe_regions().get("Regions"))) - UNSUPPORTED_REGIONS @@ -75,7 +87,7 @@ def upload_to_s3(args, region, instances): if args.dryrun == "true": print(instances) - print("Skipping upload to s3://%s/%s" % (args.bucket, key)) + print("Skipping upload to s3://%s/%s" % (bucket, key)) return try: @@ -101,6 +113,7 @@ def main(args): if __name__ == "__main__": # parse inputs parser = argparse.ArgumentParser(description="Generate a whitelist of batch instance types.") + parser.add_argument("--partition", type=str, help="commercial | china | govcloud", required=True) parser.add_argument( "--regions", type=str, @@ -114,7 +127,7 @@ def main(args): args = parser.parse_args() if args.regions == "all": - args.regions = get_all_aws_regions() + args.regions = get_all_aws_regions(args.partition) else: args.regions = args.regions.split(",") diff --git a/util/generate-ami-list.py b/util/generate-ami-list.py index 3befc7f4e0..c06485eaf1 100644 --- a/util/generate-ami-list.py +++ b/util/generate-ami-list.py @@ -20,6 +20,7 @@ import argparse import json +import sys from collections import OrderedDict import boto3 @@ -74,8 +75,8 @@ def convert_json_to_txt(amis_json): return amis_txt -def get_all_aws_regions(): - ec2 = boto3.client("ec2") +def get_all_aws_regions(region): + ec2 = boto3.client("ec2", region_name=region) return sorted(r.get("RegionName") for r in ec2.describe_regions().get("Regions")) @@ -113,9 +114,7 @@ def update_amis_txt(amis_txt_file, amis): parser.add_argument("--version", type=str, help="release version", required=True) parser.add_argument("--date", type=str, help="release date [timestamp] (e.g. 201801112350)", required=True) parser.add_argument("--txt-file", type=str, help="txt output file path", required=False, default="amis.txt") - parser.add_argument( - "--account-id", type=str, help="account id that owns the amis", required=False, default="247102896272" - ) + parser.add_argument("--partition", type=str, help="commercial | china | govcloud", required=True) parser.add_argument( "--cloudformation-template", type=str, @@ -125,9 +124,22 @@ def update_amis_txt(amis_txt_file, amis): ) args = parser.parse_args() - regions = get_all_aws_regions() - - amis_dict = get_ami_list(regions=regions, date=args.date, version=args.version, owner=args.account_id) + if args.partition == "commercial": + account_id = "247102896272" + region = "us-east-1" + elif args.partition == "govcloud": + account_id = "124026578433" + region = "us-gov-west-1" + elif args.partition == "china": + account_id = "036028979999" + region = "cn-north-1" + else: + print("Unsupported partition %s" % args.partition) + sys.exit(1) + + regions = get_all_aws_regions(region) + + amis_dict = get_ami_list(regions=regions, date=args.date, version=args.version, owner=account_id) cfn_amis = update_cfn_template(cfn_template_file=args.cloudformation_template, amis_to_update=amis_dict) diff --git a/util/upload-cfn-templates.py b/util/upload-cfn-templates.py index 850867b3c3..0ed4b412ae 100644 --- a/util/upload-cfn-templates.py +++ b/util/upload-cfn-templates.py @@ -7,11 +7,48 @@ from botocore.exceptions import ClientError -def get_all_aws_regions(): - ec2 = boto3.client("ec2") +def get_all_aws_regions(partition): + if partition == "commercial": + region = "us-east-1" + elif partition == "govcloud": + region = "us-gov-west-1" + elif partition == "china": + region = "cn-north-1" + else: + print("Unsupported partition %s" % partition) + sys.exit(1) + + ec2 = boto3.client("ec2", region_name=region) return set(sorted(r.get("RegionName") for r in ec2.describe_regions().get("Regions"))) +def put_object_to_s3(s3_client, bucket, key, region, data, template_name): + try: + object = s3_client.Object(bucket, key) + response = object.put(Body=data, ACL="public-read") + if response.get("ResponseMetadata").get("HTTPStatusCode") == 200: + print("Successfully uploaded %s to s3://%s/%s" % (template_name, bucket, key)) + except ClientError as e: + if args.createifnobucket and e.response["Error"]["Code"] == "NoSuchBucket": + print("No bucket, creating now: ") + if region == "us-east-1": + s3_client.create_bucket(Bucket=bucket) + else: + s3_client.create_bucket(Bucket=bucket, CreateBucketConfiguration={"LocationConstraint": region}) + s3_client.BucketVersioning(bucket).enable() + print("Created %s bucket. Bucket versioning is enabled, " "please enable bucket logging manually." % bucket) + b = s3_client.Bucket(bucket) + res = b.put_object(Body=data, ACL="public-read", Key=key) + print(res) + else: + print("Couldn't upload %s to bucket s3://%s/%s" % (template_name, bucket, key)) + if e.response["Error"]["Code"] == "NoSuchBucket": + print("Bucket is not present.") + else: + raise e + pass + + def upload_to_s3(args, region): s3_client = boto3.resource("s3", region_name=region) @@ -26,49 +63,23 @@ def upload_to_s3(args, region): template_name = "%s%s.cfn.json" % (template_paths, t) key = key_path + "%s-%s.cfn.json" % (t, args.version) data = open(template_name, "rb") - if not args.override: - for bucket in buckets: - try: - s3 = boto3.client("s3", region_name=region) - s3.head_object(Bucket=bucket, Key=key) - print("Error: %s already exist in bucket %s" % (key, bucket)) - sys.exit(1) - except ClientError: - pass for bucket in buckets: - if args.dryrun: - key = key_path + "%s-%s.cfn.json" % (t, args.version) - print("If dryrun is disabled, %s will be uploaded to s3://%s/%s" % (template_name, bucket, key)) + try: + s3 = boto3.client("s3", region_name=region) + s3.head_object(Bucket=bucket, Key=key) + print("Warning: %s already exist in bucket %s" % (key, bucket)) + exist = True + except ClientError: + exist = False + pass + + if (exist and args.override and not args.dryrun) or (not exist and not args.dryrun): + put_object_to_s3(s3_client, bucket, key, region, data, template_name) else: - try: - object = s3_client.Object(bucket, key) - response = object.put(Body=data, ACL="public-read") - if response.get("ResponseMetadata").get("HTTPStatusCode") == 200: - print("Successfully uploaded %s to s3://%s/%s" % (template_name, bucket, key)) - except ClientError as e: - if args.createifnobucket and e.response["Error"]["Code"] == "NoSuchBucket": - print("No bucket, creating now: ") - if region == "us-east-1": - s3_client.create_bucket(Bucket=bucket) - else: - s3_client.create_bucket( - Bucket=bucket, CreateBucketConfiguration={"LocationConstraint": region} - ) - s3_client.BucketVersioning(bucket).enable() - print( - "Created %s bucket. Bucket versioning is enabled, " - "please enable bucket logging manually." % bucket - ) - b = s3_client.Bucket(bucket) - res = b.put_object(Body=data, ACL="public-read", Key=key) - print(res) - else: - print("Couldn't upload %s to bucket s3://%s/%s" % (template_name, bucket, key)) - if e.response["Error"]["Code"] == "NoSuchBucket": - print("Bucket is not present.") - else: - raise e - pass + print( + "Not uploading %s to bucket %s, object exists %s, override is %s, dryrun is %s" + % (template_name, bucket, exist, args.override, args.dryrun) + ) def main(args): @@ -80,6 +91,7 @@ def main(args): if __name__ == "__main__": # parse inputs parser = argparse.ArgumentParser(description="Upload extra templates under /cloudformation") + parser.add_argument("--partition", type=str, help="commercial | china | govcloud", required=True) parser.add_argument( "--regions", type=str, @@ -119,7 +131,7 @@ def main(args): args.version = pkg_resources.get_distribution("aws-parallelcluster").version if args.regions == "all": - args.regions = get_all_aws_regions() + args.regions = get_all_aws_regions(args.partition) else: args.regions = args.regions.split(",") args.regions = set(args.regions) - set(args.unsupportedregions.split(",")) diff --git a/util/upload-instance-slot-map.py b/util/upload-instance-slot-map.py new file mode 100644 index 0000000000..ce865fcd29 --- /dev/null +++ b/util/upload-instance-slot-map.py @@ -0,0 +1,96 @@ +#!/usr/bin/python +# +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not +# use this file except in compliance with the License. A copy of the License +# is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, express or implied. See the License for the specific language +# governing permissions and limitations under the License. +# +# +# Upload instance slot map +# +# usage: ./upload-instance-slot-map.py --partition [--instance-details ] + +import argparse +import json +import sys + +import boto3 +from botocore.exceptions import ClientError + + +def dump_instances(instance_details): + with open(instance_details) as data_file: + data = json.load(data_file) + instances = {} + for sku, product in data.get("products").iteritems(): + if product.get("productFamily") == "Compute Instance": + instance = product.get("attributes") + instances[instance.get("instanceType")] = {"vcpus": instance.get("vcpu")} + print(json.dumps(instances)) + json.dump(instances, open("instances.json", "w")) + + +def get_all_aws_regions(region): + ec2 = boto3.client("ec2", region_name=region) + return sorted(r.get("RegionName") for r in ec2.describe_regions().get("Regions")) + + +def upload(regions): + for region in regions: + bucket_name = region + "-aws-parallelcluster" + print(bucket_name) + try: + s3 = boto3.resource("s3", region_name=region) + s3.meta.client.head_bucket(Bucket=bucket_name) + except ClientError as e: + # If a client error is thrown, then check that it was a 404 error. + # If it was a 404 error, then the bucket does not exist. + error_code = int(e.response["Error"]["Code"]) + if error_code == 404: + print("Bucket %s does not exist", bucket_name) + continue + raise + + bucket = s3.Bucket(bucket_name) + bucket.upload_file("instances.json", "instances/instances.json") + object_acl = s3.ObjectAcl(bucket_name, "instances/instances.json") + object_acl.put(ACL="public-read") + + +if __name__ == "__main__": + # parse inputs + parser = argparse.ArgumentParser(description="Upload instance slot map") + + parser.add_argument("--partition", type=str, help="commercial | china | govcloud", required=True) + parser.add_argument( + "--instance-details", + type=str, + help="path to cloudfomation template", + required=False, + default="instance-details.json", + ) + args = parser.parse_args() + + if args.partition == "commercial": + region = "us-east-1" + elif args.partition == "govcloud": + region = "us-gov-west-1" + elif args.partition == "china": + region = "cn-north-1" + else: + print("Unsupported partition %s" % args.partition) + sys.exit(1) + + dump_instances(args.instance_details) + + regions = get_all_aws_regions(region) + + upload(regions)