From a0be16d1a90d46a281e628afbbecb349103cf3ce Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Tue, 27 Aug 2019 18:57:52 +0300 Subject: [PATCH] Adding Cloudformation templates --- cloudformation/Jenkinsfile | 66 ++ cloudformation/README.md | 36 + cloudformation/conftest.py | 5 + cloudformation/pytest.ini | 3 + cloudformation/requirements.txt | 5 + cloudformation/scylla.yaml | 1134 +++++++++++++++++++++++++++++ cloudformation/scylla.yaml.j2 | 333 +++++++++ cloudformation/test_scylla_cfn.py | 137 ++++ 8 files changed, 1719 insertions(+) create mode 100644 cloudformation/Jenkinsfile create mode 100644 cloudformation/README.md create mode 100644 cloudformation/conftest.py create mode 100644 cloudformation/pytest.ini create mode 100644 cloudformation/requirements.txt create mode 100644 cloudformation/scylla.yaml create mode 100644 cloudformation/scylla.yaml.j2 create mode 100644 cloudformation/test_scylla_cfn.py diff --git a/cloudformation/Jenkinsfile b/cloudformation/Jenkinsfile new file mode 100644 index 0000000..2b89ac6 --- /dev/null +++ b/cloudformation/Jenkinsfile @@ -0,0 +1,66 @@ +#!groovy +pipeline { + agent { + label { + label "aws-sct-builders-eu-west-1" + } + } + options { + timestamps() + disableConcurrentBuilds() + timeout([time: 30, unit: 'MINUTES']) + buildDiscarder(logRotator(numToKeepStr: '20')) + } + stages { + stage('Checkout') { + steps { + checkout scm + } + } + stage('create virtualenv') { + steps { + script { + wrap([$class: 'BuildUser']) { + dir('cloudformation') { + sh """ + #!/bin/bash + set -xe + + python3 -m venv .venv3 + source .venv3/bin/activate + + pip install --upgrade pip + pip install -r requirements.txt + """ + } + } + } + } + } + stage('test') { + steps { + script { + wrap([$class: 'BuildUser']) { + dir('cloudformation') { + sh """ + #!/bin/bash + set -xe + + source .venv3/bin/activate + + # regenerate the teample + jinja2 scylla.yaml.j2 > scylla.yaml + + # lint the template + cfn-lint --template scylla.yaml --region us-east-1 + + # running the test + pytest --log-cli-level info + """ + } + } + } + } + } + } +} diff --git a/cloudformation/README.md b/cloudformation/README.md new file mode 100644 index 0000000..bc1ab86 --- /dev/null +++ b/cloudformation/README.md @@ -0,0 +1,36 @@ +## Installation +``` +python3 -m venv .venv3 +source .venv3/bin/activate +install -r requirements.txt +``` + +## Testing +``` bash +# regenerate the teample +jinja2 scylla.yaml.j2 > scylla.yaml + +# lint the template +cfn-lint --template scylla.yaml --region us-east-1 + +# running the test +pytest --log-cli-level info +``` + +# Using the template +``` +# run the template from command line +aws cloudformation create-stack --region eu-west-1 --stack-name fruch-test-05 --template-body file://scylla.yaml \ + --parameters ParameterKey=KeyName,ParameterValue=scylla-qa-ec2 \ + ParameterKey=InstanceType,ParameterValue=i3.large \ + ParameterKey=AvailabilityZone,ParameterValue=eu-west-1a \ + ParameterKey=ClusterName,ParameterValue=fruch \ + ParameterKey=InstanceCount,ParameterValue=3 \ + ParameterKey=ScyllaAmi,ParameterValue=ami-0ececa5cacea302a8 + +``` +Example of link to start the cloudforamtion: + +https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/create/review?templateURL=https://s3-eu-west-1.amazonaws.com/cf-templates-1jy8um4tbzwit-eu-west-1/2019241R3e-scylla.templateenk889k0zz&stackName=fruch-test¶m_ScyllaAmi=ami-0ececa5cacea302a8 + + diff --git a/cloudformation/conftest.py b/cloudformation/conftest.py new file mode 100644 index 0000000..24e78c4 --- /dev/null +++ b/cloudformation/conftest.py @@ -0,0 +1,5 @@ +def pytest_addoption(parser): + parser.addoption("--keep-cfn", action="store_true", default=False) + parser.addoption("--stack-name", action="store", default=None) + parser.addoption("--region", action="store", default="us-east-1") + parser.addoption("--ami", action="store", default="ami-0b48ed57ebc0d5f06") diff --git a/cloudformation/pytest.ini b/cloudformation/pytest.ini new file mode 100644 index 0000000..96735eb --- /dev/null +++ b/cloudformation/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/cloudformation/requirements.txt b/cloudformation/requirements.txt new file mode 100644 index 0000000..3d40740 --- /dev/null +++ b/cloudformation/requirements.txt @@ -0,0 +1,5 @@ +cfn-lint==0.23.4 +boto3==1.9.216 +pytest==5.1.1 +Jinja2==2.10.1 +jinja2-cli==0.7.0 \ No newline at end of file diff --git a/cloudformation/scylla.yaml b/cloudformation/scylla.yaml new file mode 100644 index 0000000..3ec6ad5 --- /dev/null +++ b/cloudformation/scylla.yaml @@ -0,0 +1,1134 @@ + +AWSTemplateFormatVersion: 2010-09-09 +Description: >- + AWS CloudFormation Scylla Sample Template: This would create a new Scylla Cluster + Including it's own VPC and subnet, Elastic IPs are used for accessing those node publicly + + Use `SGUser` security group to enable access from outside to this cluster + By default only SSH port is out to the outside world. + + Caution: password authentication isn't enabled by default + +Parameters: + ClusterName: + Type: String + + InstanceCount: + Description: Number of EC2 instances (must be between 1 and 10). + Type: Number + Default: 1 + MinValue: 1 + MaxValue: 10 + ConstraintDescription: Must be a number between 1 and 10. + + ScyllaAmi: + Type: 'AWS::EC2::Image::Id' + ConstraintDescription: must be a valid scylla AMI in this region. + + InstanceType: + Description: EC2 instance type + Type: String + Default: i3.large + AllowedValues: + - i3.large + - i3.xlarge + - i3.2xlarge + - i3.4xlarge + - i3.8xlarge + - i3.16xlarge + - i3en.large + - i3en.xlarge + - i3en.2xlarge + - i3en.3xlarge + - i3en.6xlarge + - i3en.12xlarge + - i3en.24xlarge + - i3en.metal + ConstraintDescription: must be a valid EC2 instance type. + + AvailabilityZone: + Type: 'AWS::EC2::AvailabilityZone::Name' + ConstraintDescription: must be the name of available AvailabilityZone. + + KeyName: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instances + Type: 'AWS::EC2::KeyPair::KeyName' + ConstraintDescription: must be the name of an existing EC2 KeyPair. + +# Those conditions would be used to enable nodes based on InstanceCount parameter +Conditions: + Launch1: !Equals [1, 1] + Launch2: !Not [!Equals [1, !Ref InstanceCount]] + Launch3: !And + - !Not [!Equals [1, !Ref InstanceCount]] + - !Not [!Equals [2, !Ref InstanceCount]] + Launch4: !And + - !Not [!Equals [1, !Ref InstanceCount]] + - !Not [!Equals [2, !Ref InstanceCount]] + - !Not [!Equals [3, !Ref InstanceCount]] + Launch5: !And + - !Not [!Equals [1, !Ref InstanceCount]] + - !Not [!Equals [2, !Ref InstanceCount]] + - !Not [!Equals [3, !Ref InstanceCount]] + - !Not [!Equals [4, !Ref InstanceCount]] + Launch6: !And + - !Not [!Equals [1, !Ref InstanceCount]] + - !Not [!Equals [2, !Ref InstanceCount]] + - !Not [!Equals [3, !Ref InstanceCount]] + - !Not [!Equals [4, !Ref InstanceCount]] + - !Not [!Equals [5, !Ref InstanceCount]] + Launch7: !And + - !Not [!Equals [1, !Ref InstanceCount]] + - !Not [!Equals [2, !Ref InstanceCount]] + - !Not [!Equals [3, !Ref InstanceCount]] + - !Not [!Equals [4, !Ref InstanceCount]] + - !Not [!Equals [5, !Ref InstanceCount]] + - !Not [!Equals [6, !Ref InstanceCount]] + Launch8: !And + - !Not [!Equals [1, !Ref InstanceCount]] + - !Not [!Equals [2, !Ref InstanceCount]] + - !Not [!Equals [3, !Ref InstanceCount]] + - !Not [!Equals [4, !Ref InstanceCount]] + - !Not [!Equals [5, !Ref InstanceCount]] + - !Not [!Equals [6, !Ref InstanceCount]] + - !Not [!Equals [7, !Ref InstanceCount]] + Launch9: !And + - !Not [!Equals [1, !Ref InstanceCount]] + - !Not [!Equals [2, !Ref InstanceCount]] + - !Not [!Equals [3, !Ref InstanceCount]] + - !Not [!Equals [4, !Ref InstanceCount]] + - !Not [!Equals [5, !Ref InstanceCount]] + - !Not [!Equals [6, !Ref InstanceCount]] + - !Not [!Equals [7, !Ref InstanceCount]] + - !Not [!Equals [8, !Ref InstanceCount]] + Launch10: !Equals [10, !Ref InstanceCount] + +Resources: + GatewayAttachment: + Type: 'AWS::EC2::VPCGatewayAttachment' + Properties: + InternetGatewayId: !Ref InternetGateway + VpcId: !Ref VPC + + InternetGateway: + Type: 'AWS::EC2::InternetGateway' + Properties: + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Gateway' + - Key: ClusterName + Value: !Ref ClusterName + Node0: + Condition: Launch1 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-1' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node0 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node0ETH0 + DeviceIndex: '0' + Node0ETH0: + Condition: Launch1 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.0.11 + SubnetId: !Ref Subnet + Node0ElasticIP: + Condition: Launch1 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node0ElasticIPAssociation: + Condition: Launch1 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node0 + Properties: + AllocationId: !GetAtt + - Node0ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node0ETH0 + PrivateIpAddress: 172.31.0.11 + Node1: + Condition: Launch2 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-2' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node1 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node1ETH0 + DeviceIndex: '0' + Node1ETH0: + Condition: Launch2 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.1.11 + SubnetId: !Ref Subnet + Node1ElasticIP: + Condition: Launch2 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node1ElasticIPAssociation: + Condition: Launch2 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node1 + Properties: + AllocationId: !GetAtt + - Node1ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node1ETH0 + PrivateIpAddress: 172.31.1.11 + Node2: + Condition: Launch3 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-3' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node2 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node2ETH0 + DeviceIndex: '0' + Node2ETH0: + Condition: Launch3 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.2.11 + SubnetId: !Ref Subnet + Node2ElasticIP: + Condition: Launch3 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node2ElasticIPAssociation: + Condition: Launch3 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node2 + Properties: + AllocationId: !GetAtt + - Node2ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node2ETH0 + PrivateIpAddress: 172.31.2.11 + Node3: + Condition: Launch4 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-4' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node3 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node3ETH0 + DeviceIndex: '0' + Node3ETH0: + Condition: Launch4 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.3.11 + SubnetId: !Ref Subnet + Node3ElasticIP: + Condition: Launch4 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node3ElasticIPAssociation: + Condition: Launch4 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node3 + Properties: + AllocationId: !GetAtt + - Node3ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node3ETH0 + PrivateIpAddress: 172.31.3.11 + Node4: + Condition: Launch5 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-5' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node4 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node4ETH0 + DeviceIndex: '0' + Node4ETH0: + Condition: Launch5 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.4.11 + SubnetId: !Ref Subnet + Node4ElasticIP: + Condition: Launch5 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node4ElasticIPAssociation: + Condition: Launch5 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node4 + Properties: + AllocationId: !GetAtt + - Node4ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node4ETH0 + PrivateIpAddress: 172.31.4.11 + Node5: + Condition: Launch6 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-6' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node5 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node5ETH0 + DeviceIndex: '0' + Node5ETH0: + Condition: Launch6 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.5.11 + SubnetId: !Ref Subnet + Node5ElasticIP: + Condition: Launch6 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node5ElasticIPAssociation: + Condition: Launch6 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node5 + Properties: + AllocationId: !GetAtt + - Node5ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node5ETH0 + PrivateIpAddress: 172.31.5.11 + Node6: + Condition: Launch7 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-7' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node6 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node6ETH0 + DeviceIndex: '0' + Node6ETH0: + Condition: Launch7 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.6.11 + SubnetId: !Ref Subnet + Node6ElasticIP: + Condition: Launch7 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node6ElasticIPAssociation: + Condition: Launch7 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node6 + Properties: + AllocationId: !GetAtt + - Node6ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node6ETH0 + PrivateIpAddress: 172.31.6.11 + Node7: + Condition: Launch8 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-8' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node7 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node7ETH0 + DeviceIndex: '0' + Node7ETH0: + Condition: Launch8 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.7.11 + SubnetId: !Ref Subnet + Node7ElasticIP: + Condition: Launch8 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node7ElasticIPAssociation: + Condition: Launch8 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node7 + Properties: + AllocationId: !GetAtt + - Node7ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node7ETH0 + PrivateIpAddress: 172.31.7.11 + Node8: + Condition: Launch9 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-9' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node8 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node8ETH0 + DeviceIndex: '0' + Node8ETH0: + Condition: Launch9 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.8.11 + SubnetId: !Ref Subnet + Node8ElasticIP: + Condition: Launch9 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node8ElasticIPAssociation: + Condition: Launch9 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node8 + Properties: + AllocationId: !GetAtt + - Node8ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node8ETH0 + PrivateIpAddress: 172.31.8.11 + Node9: + Condition: Launch10 + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-10' + - Key: ClusterName + Value: !Ref ClusterName + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node9 --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node9ETH0 + DeviceIndex: '0' + Node9ETH0: + Condition: Launch10 + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: 172.31.9.11 + SubnetId: !Ref Subnet + Node9ElasticIP: + Condition: Launch10 + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node9ElasticIPAssociation: + Condition: Launch10 + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node9 + Properties: + AllocationId: !GetAtt + - Node9ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node9ETH0 + PrivateIpAddress: 172.31.9.11 + Route: + Type: 'AWS::EC2::Route' + DependsOn: GatewayAttachment + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + RouteTableId: !Ref RouteTable + RouteTable: + Type: 'AWS::EC2::RouteTable' + Properties: + Tags: + - Key: Name + Value: !Sub '${ClusterName}-RT' + - Key: ClusterName + Value: !Ref ClusterName + + VpcId: !Ref VPC + SGAdmin: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: Security group for the admin + Tags: + - Key: Name + Value: !Sub '${ClusterName}-SGAdmin' + - Key: ClusterName + Value: !Ref ClusterName + SecurityGroupIngress: + - CidrIp: 0.0.0.0/0 + FromPort: 22 + ToPort: 22 + IpProtocol: tcp + - CidrIpv6: ::/0 + FromPort: 22 + ToPort: 22 + IpProtocol: tcp + VpcId: !Ref VPC + + SGCluster: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: Security group for the cluster + Tags: + - Key: Name + Value: !Sub '${ClusterName}-SGCluster' + - Key: ClusterName + Value: !Ref ClusterName + SecurityGroupIngress: + - CidrIp: 172.31.0.0/16 + FromPort: 22 + ToPort: 22 + IpProtocol: '-1' + VpcId: !Ref VPC + + SGUser: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: Security group for the user CQL access + Tags: + - Key: Name + Value: !Sub '${ClusterName}-SGUser' + - Key: ClusterName + Value: !Ref ClusterName + VpcId: !Ref VPC + + Subnet: + Type: 'AWS::EC2::Subnet' + Properties: + AvailabilityZone: !Ref AvailabilityZone + CidrBlock: 172.31.0.0/16 + MapPublicIpOnLaunch: true + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Subnet' + - Key: ClusterName + Value: !Ref ClusterName + VpcId: !Ref VPC + + SubnetRouteTableAssociation: + Type: 'AWS::EC2::SubnetRouteTableAssociation' + Properties: + RouteTableId: !Ref RouteTable + SubnetId: !Ref Subnet + + VPC: + Type: 'AWS::EC2::VPC' + Properties: + CidrBlock: 172.31.0.0/16 + Tags: + - Key: Name + Value: !Sub '${ClusterName}-VPC' + - Key: ClusterName + Value: !Ref ClusterName + +Outputs: + Node0: + Condition: Launch1 + Value: !Ref Node0 + Node0PrivateDnsName: + Condition: Launch1 + Value: !GetAtt + - Node0 + - PrivateDnsName + Node0PrivateIp: + Condition: Launch1 + Value: !GetAtt + - Node0 + - PrivateIp + Node0PublicIp: + Condition: Launch1 + Value: !GetAtt + - Node0 + - PublicIp + Node1: + Condition: Launch2 + Value: !Ref Node1 + Node1PrivateDnsName: + Condition: Launch2 + Value: !GetAtt + - Node1 + - PrivateDnsName + Node1PrivateIp: + Condition: Launch2 + Value: !GetAtt + - Node1 + - PrivateIp + Node1PublicIp: + Condition: Launch2 + Value: !GetAtt + - Node1 + - PublicIp + Node2: + Condition: Launch3 + Value: !Ref Node2 + Node2PrivateDnsName: + Condition: Launch3 + Value: !GetAtt + - Node2 + - PrivateDnsName + Node2PrivateIp: + Condition: Launch3 + Value: !GetAtt + - Node2 + - PrivateIp + Node2PublicIp: + Condition: Launch3 + Value: !GetAtt + - Node2 + - PublicIp + Node3: + Condition: Launch4 + Value: !Ref Node3 + Node3PrivateDnsName: + Condition: Launch4 + Value: !GetAtt + - Node3 + - PrivateDnsName + Node3PrivateIp: + Condition: Launch4 + Value: !GetAtt + - Node3 + - PrivateIp + Node3PublicIp: + Condition: Launch4 + Value: !GetAtt + - Node3 + - PublicIp + Node4: + Condition: Launch5 + Value: !Ref Node4 + Node4PrivateDnsName: + Condition: Launch5 + Value: !GetAtt + - Node4 + - PrivateDnsName + Node4PrivateIp: + Condition: Launch5 + Value: !GetAtt + - Node4 + - PrivateIp + Node4PublicIp: + Condition: Launch5 + Value: !GetAtt + - Node4 + - PublicIp + Node5: + Condition: Launch6 + Value: !Ref Node5 + Node5PrivateDnsName: + Condition: Launch6 + Value: !GetAtt + - Node5 + - PrivateDnsName + Node5PrivateIp: + Condition: Launch6 + Value: !GetAtt + - Node5 + - PrivateIp + Node5PublicIp: + Condition: Launch6 + Value: !GetAtt + - Node5 + - PublicIp + Node6: + Condition: Launch7 + Value: !Ref Node6 + Node6PrivateDnsName: + Condition: Launch7 + Value: !GetAtt + - Node6 + - PrivateDnsName + Node6PrivateIp: + Condition: Launch7 + Value: !GetAtt + - Node6 + - PrivateIp + Node6PublicIp: + Condition: Launch7 + Value: !GetAtt + - Node6 + - PublicIp + Node7: + Condition: Launch8 + Value: !Ref Node7 + Node7PrivateDnsName: + Condition: Launch8 + Value: !GetAtt + - Node7 + - PrivateDnsName + Node7PrivateIp: + Condition: Launch8 + Value: !GetAtt + - Node7 + - PrivateIp + Node7PublicIp: + Condition: Launch8 + Value: !GetAtt + - Node7 + - PublicIp + Node8: + Condition: Launch9 + Value: !Ref Node8 + Node8PrivateDnsName: + Condition: Launch9 + Value: !GetAtt + - Node8 + - PrivateDnsName + Node8PrivateIp: + Condition: Launch9 + Value: !GetAtt + - Node8 + - PrivateIp + Node8PublicIp: + Condition: Launch9 + Value: !GetAtt + - Node8 + - PublicIp + Node9: + Condition: Launch10 + Value: !Ref Node9 + Node9PrivateDnsName: + Condition: Launch10 + Value: !GetAtt + - Node9 + - PrivateDnsName + Node9PrivateIp: + Condition: Launch10 + Value: !GetAtt + - Node9 + - PrivateIp + Node9PublicIp: + Condition: Launch10 + Value: !GetAtt + - Node9 + - PublicIp + SGAdmin: + Value: !Ref SGAdmin + SGCluster: + Value: !Ref SGCluster + SGUser: + Value: !Ref SGUser + Subnet: + Value: !Ref Subnet + VPC: + Value: !Ref VPC \ No newline at end of file diff --git a/cloudformation/scylla.yaml.j2 b/cloudformation/scylla.yaml.j2 new file mode 100644 index 0000000..e1c5260 --- /dev/null +++ b/cloudformation/scylla.yaml.j2 @@ -0,0 +1,333 @@ +{%- set total_num_node = 10 %} +{%- set ip_address_list = [] %} +{%- for node_index in range(total_num_node) %} + {%- do ip_address_list.append("172.31.%d.11" % node_index) %} +{%- endfor -%} +{%- set new_ami_boot = True %} +AWSTemplateFormatVersion: 2010-09-09 +Description: >- + AWS CloudFormation Scylla Sample Template: This would create a new Scylla Cluster + Including it's own VPC and subnet, Elastic IPs are used for accessing those node publicly + + Use `SGUser` security group to enable access from outside to this cluster + By default only SSH port is out to the outside world. + + Caution: password authentication isn't enabled by default + +Parameters: + ClusterName: + Type: String + + InstanceCount: + Description: Number of EC2 instances (must be between 1 and {{ total_num_node }}). + Type: Number + Default: 1 + MinValue: 1 + MaxValue: {{ total_num_node }} + ConstraintDescription: Must be a number between 1 and {{ total_num_node }}. + + ScyllaAmi: + Type: 'AWS::EC2::Image::Id' + ConstraintDescription: must be a valid scylla AMI in this region. + + InstanceType: + Description: EC2 instance type + Type: String + Default: i3.large + AllowedValues: + - i3.large + - i3.xlarge + - i3.2xlarge + - i3.4xlarge + - i3.8xlarge + - i3.16xlarge + - i3en.large + - i3en.xlarge + - i3en.2xlarge + - i3en.3xlarge + - i3en.6xlarge + - i3en.12xlarge + - i3en.24xlarge + - i3en.metal + ConstraintDescription: must be a valid EC2 instance type. + + AvailabilityZone: + Type: 'AWS::EC2::AvailabilityZone::Name' + ConstraintDescription: must be the name of available AvailabilityZone. + + KeyName: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instances + Type: 'AWS::EC2::KeyPair::KeyName' + ConstraintDescription: must be the name of an existing EC2 KeyPair. + +# Those conditions would be used to enable nodes based on InstanceCount parameter +Conditions: + Launch1: !Equals [1, 1] + Launch2: !Not [!Equals [1, !Ref InstanceCount]] +{%- for node_index in range(3, total_num_node) %} + Launch{{ node_index }}: !And +{%- for i in range(1, node_index) %} + - !Not [!Equals [{{ i }}, !Ref InstanceCount]] +{%- endfor %} +{%- endfor %} + Launch{{ total_num_node }}: !Equals [{{ total_num_node }}, !Ref InstanceCount] + +Resources: + GatewayAttachment: + Type: 'AWS::EC2::VPCGatewayAttachment' + Properties: + InternetGatewayId: !Ref InternetGateway + VpcId: !Ref VPC + + InternetGateway: + Type: 'AWS::EC2::InternetGateway' + Properties: + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Gateway' + - Key: ClusterName + Value: !Ref ClusterName + +{%- for node_index in range(total_num_node) %} + Node{{ node_index }}: + Condition: Launch{{ node_index + 1 }} + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT10M + DependsOn: SGCluster + Properties: + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + DeleteOnTermination: true + VolumeSize: 50 + ImageId: !Ref ScyllaAmi + InstanceType: !Ref InstanceType + KeyName: !Ref KeyName + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Node-{{ node_index + 1 }}' + - Key: ClusterName + Value: !Ref ClusterName +{%- if new_ami_boot %} + UserData: !Base64 + 'Fn::Join': + - '' + - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - '"}]}]' + - !Sub ', "cluster_name": "${ClusterName}", ' + - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch"}, ' + - '"post_configuration_script" : "' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node{{ node_index }} --region ${AWS::Region} --stack ${AWS::StackName} + - '"}' +{%- else %} + UserData: !Base64 + 'Fn::Join': + - '' + - - !Sub '--clustername ${ClusterName}' + - |+ + + - '--base64postscript ' + - !Base64 + 'Fn::Join': + - '' + - - !Sub | + #!/bin/bash -ex + + sudo yum -y install epel-release + sudo yum -y install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm + ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap + + trap '/opt/aws/bin/cfn-signal --exit-code 1 --resource Node{{ node_index }} --region ${AWS::Region} --stack ${AWS::StackName}' ERR + + export PUBLIC_IP=${Node2ElasticIP} + export PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) + sed -i "s|# broadcast_address: 1.2.3.4|broadcast_address: $PRIVATE_IP|g" /etc/scylla/scylla.yaml + - |+ + + - export SEEDS= + - !Join + - ',' + - - !If [Launch1, '172.31.0.11', !Ref "AWS::NoValue"] + - !If [Launch2, '172.31.1.11', !Ref "AWS::NoValue"] + - !If [Launch3, '172.31.2.11', !Ref "AWS::NoValue"] + - |+ + + - !Sub | + sed -i "s| - seeds: \"$PRIVATE_IP\"| - seeds: \"$SEEDS\"|g" /etc/scylla/scylla.yaml + /opt/aws/bin/cfn-signal --exit-code 0 --resource Node{{ node_index }} --region ${AWS::Region} --stack ${AWS::StackName} + - |+ + + - |+ + + - | + --totalnodes 1 --reflector example.com +{%- endif %} + NetworkInterfaces: + - NetworkInterfaceId: !Ref Node{{ node_index }}ETH0 + DeviceIndex: '0' + Node{{ node_index }}ETH0: + Condition: Launch{{ node_index + 1 }} + Type: 'AWS::EC2::NetworkInterface' + Properties: + GroupSet: + - !Ref SGAdmin + - !Ref SGCluster + - !Ref SGUser + PrivateIpAddress: {{ ip_address_list[node_index] }} + SubnetId: !Ref Subnet + Node{{ node_index }}ElasticIP: + Condition: Launch{{ node_index + 1 }} + Type: 'AWS::EC2::EIP' + Properties: + Domain: vpc + Node{{ node_index }}ElasticIPAssociation: + Condition: Launch{{ node_index + 1 }} + Type: 'AWS::EC2::EIPAssociation' + DependsOn: Node{{ node_index }} + Properties: + AllocationId: !GetAtt + - Node{{ node_index }}ElasticIP + - AllocationId + NetworkInterfaceId: !Ref Node{{ node_index }}ETH0 + PrivateIpAddress: {{ ip_address_list[node_index] }} +{%- endfor %} + Route: + Type: 'AWS::EC2::Route' + DependsOn: GatewayAttachment + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + RouteTableId: !Ref RouteTable + RouteTable: + Type: 'AWS::EC2::RouteTable' + Properties: + Tags: + - Key: Name + Value: !Sub '${ClusterName}-RT' + - Key: ClusterName + Value: !Ref ClusterName + + VpcId: !Ref VPC + SGAdmin: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: Security group for the admin + Tags: + - Key: Name + Value: !Sub '${ClusterName}-SGAdmin' + - Key: ClusterName + Value: !Ref ClusterName + SecurityGroupIngress: + - CidrIp: 0.0.0.0/0 + FromPort: 22 + ToPort: 22 + IpProtocol: tcp + - CidrIpv6: ::/0 + FromPort: 22 + ToPort: 22 + IpProtocol: tcp + VpcId: !Ref VPC + + SGCluster: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: Security group for the cluster + Tags: + - Key: Name + Value: !Sub '${ClusterName}-SGCluster' + - Key: ClusterName + Value: !Ref ClusterName + SecurityGroupIngress: + - CidrIp: 172.31.0.0/16 + FromPort: 22 + ToPort: 22 + IpProtocol: '-1' + VpcId: !Ref VPC + + SGUser: + Type: 'AWS::EC2::SecurityGroup' + Properties: + GroupDescription: Security group for the user CQL access + Tags: + - Key: Name + Value: !Sub '${ClusterName}-SGUser' + - Key: ClusterName + Value: !Ref ClusterName + VpcId: !Ref VPC + + Subnet: + Type: 'AWS::EC2::Subnet' + Properties: + AvailabilityZone: !Ref AvailabilityZone + CidrBlock: 172.31.0.0/16 + MapPublicIpOnLaunch: true + Tags: + - Key: Name + Value: !Sub '${ClusterName}-Subnet' + - Key: ClusterName + Value: !Ref ClusterName + VpcId: !Ref VPC + + SubnetRouteTableAssociation: + Type: 'AWS::EC2::SubnetRouteTableAssociation' + Properties: + RouteTableId: !Ref RouteTable + SubnetId: !Ref Subnet + + VPC: + Type: 'AWS::EC2::VPC' + Properties: + CidrBlock: 172.31.0.0/16 + Tags: + - Key: Name + Value: !Sub '${ClusterName}-VPC' + - Key: ClusterName + Value: !Ref ClusterName + +Outputs: +{%- for node_index in range(total_num_node) %} + Node{{ node_index }}: + Condition: Launch{{ node_index + 1 }} + Value: !Ref Node{{ node_index }} + Node{{ node_index }}PrivateDnsName: + Condition: Launch{{ node_index + 1 }} + Value: !GetAtt + - Node{{ node_index }} + - PrivateDnsName + Node{{ node_index }}PrivateIp: + Condition: Launch{{ node_index + 1 }} + Value: !GetAtt + - Node{{ node_index }} + - PrivateIp + Node{{ node_index }}PublicIp: + Condition: Launch{{ node_index + 1 }} + Value: !GetAtt + - Node{{ node_index }} + - PublicIp +{%- endfor %} + SGAdmin: + Value: !Ref SGAdmin + SGCluster: + Value: !Ref SGCluster + SGUser: + Value: !Ref SGUser + Subnet: + Value: !Ref Subnet + VPC: + Value: !Ref VPC \ No newline at end of file diff --git a/cloudformation/test_scylla_cfn.py b/cloudformation/test_scylla_cfn.py new file mode 100644 index 0000000..3cd95e6 --- /dev/null +++ b/cloudformation/test_scylla_cfn.py @@ -0,0 +1,137 @@ +import pprint +import logging +import subprocess +import uuid + +import pytest +import boto3 +from botocore.errorfactory import ClientError + + +def run_on_node(node_ip, cmd): + output = subprocess.run( + [ + "/bin/bash", + "-c", + f'ssh -i ~/.ssh/scylla-qa-ec2 -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" centos@{node_ip} {cmd}', + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + logging.info(output.stderr.decode("utf-8").strip()) + return output.stdout.decode("utf-8").strip() + + +@pytest.fixture(scope="session") +def cfn_scylla_cluster(request): + region = request.config.getoption("--region") + client = boto3.client("cloudformation", region_name=region) + if request.config.getoption("--stack-name"): + name = request.config.getoption("--stack-name") + else: + rand_suffix = uuid.uuid4().hex[:6] + name = f"scylla-cfn-test-{rand_suffix}" + + try: + response = client.create_stack( + StackName=name, + TemplateBody=open("scylla.yaml").read(), + Parameters=[ + {"ParameterKey": "KeyName", "ParameterValue": "scylla-qa-ec2"}, + {"ParameterKey": "InstanceType", "ParameterValue": "i3.large"}, + {"ParameterKey": "AvailabilityZone", "ParameterValue": "us-east-1a"}, + {"ParameterKey": "ClusterName", "ParameterValue": name}, + {"ParameterKey": "InstanceCount", "ParameterValue": "3"}, + { + "ParameterKey": "ScyllaAmi", + "ParameterValue": request.config.getoption("--ami"), + }, + ], + ) + logging.info(pprint.pformat(response)) + + except ClientError as ex: + logging.info(ex) + + logging.info(f"waiting for cloudformation [{name}] to complete") + + waiter = client.get_waiter("stack_create_complete") + response = waiter.wait(StackName=name) + + resources = client.list_stack_resources(StackName=name) + + outputs = client.describe_stacks(StackName=name) + logging.info(pprint.pformat(outputs)) + outputs = { + item["OutputKey"]: item["OutputValue"] + for item in outputs["Stacks"][0]["Outputs"] + } + + yield resources, outputs + + if not request.session.testsfailed and not request.config.getoption("--keep-cfn"): + response = client.delete_stack(StackName=name) + logging.info(response) + + waiter = client.get_waiter("stack_delete_complete") + response = waiter.wait(StackName=name) + logging.info(response) + + +def wait_nodes_ready(resources, region): + ec2 = boto3.client("ec2", region_name=region) + ok_waiter = ec2.get_waiter("instance_status_ok") + + instances = [ + r["PhysicalResourceId"] + for r in resources["StackResourceSummaries"] + if r["ResourceType"] == "AWS::EC2::Instance" + ] + + logging.debug(pprint.pformat(instances)) + + response = ok_waiter.wait(InstanceIds=instances) + logging.info(pprint.pformat(response)) + + +def test_cluster_up(request, cfn_scylla_cluster): + resources, outputs = cfn_scylla_cluster + region = request.config.getoption("--region") + wait_nodes_ready(resources, region) + + +def test_connect(cfn_scylla_cluster): + resources, outputs = cfn_scylla_cluster + nodes_ip_addresses = [v for k, v in outputs.items() if "PublicIp" in k] + + for node_ip in nodes_ip_addresses: + logging.info(f"connecting to node {node_ip}") + output = run_on_node(node_ip, "scylla --version") + logging.info(output) + + +def test_nodetool_status(cfn_scylla_cluster): + resources, outputs = cfn_scylla_cluster + nodes_ip_addresses = [v for k, v in outputs.items() if "PublicIp" in k] + node_private_ips = [v for k, v in outputs.items() if "PrivateIp" in k] + + for node_ip in nodes_ip_addresses: + logging.info(f"running nodetool on node {node_ip}") + output = run_on_node(node_ip, "nodetool status") + logging.info(output) + for private_ip in node_private_ips: + assert f"UN {private_ip}" in output + + +def test_cassandra_stress(cfn_scylla_cluster): + resources, outputs = cfn_scylla_cluster + nodes_ip_addresses = [v for k, v in outputs.items() if "PublicIp" in k] + + for node_ip in nodes_ip_addresses: + logging.info(f"running c-s to node {node_ip}") + output = run_on_node( + node_ip, "cassandra-stress write n=40000 -rate threads=40 -node 172.31.0.11" + ) + logging.info(output) + assert "Total errors : 0 [WRITE: 0]" in output