From 487ba07c84387d3f107f1b752650bc5732970a9a Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 12:42:11 -0800 Subject: [PATCH 01/11] echo all ENVS in jenkins --- Jenkinsfile | 1 + requirements_dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 5bf61498..c9420db2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,6 +17,7 @@ node { stage("Build Test Image") { sh "docker build -f Dockerfile-test -t ${img_tag} ." + echo sh(returnStdout: true, script: 'env') } stage("Run Tests") { diff --git a/requirements_dev.txt b/requirements_dev.txt index 82dc4183..7d85f3ef 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -11,3 +11,4 @@ PyYAML pytest pytest-runner pytest-cov +twine From df9e808c4febaabb69e1d407723e698858f0fe5c Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 12:42:26 -0800 Subject: [PATCH 02/11] =?UTF-8?q?Bump=20version:=200.1.2=20=E2=86=92=200.1?= =?UTF-8?q?.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.cfg | 2 +- setup.py | 2 +- stackformation/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index a8c053d9..2090e7ad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.2 +current_version = 0.1.3 commit = True tag = True diff --git a/setup.py b/setup.py index cfa56a12..a74a8960 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( name='jh-stackformation', - version='0.1.2', + version='0.1.3', description="AWS CloudFormation framework", long_description=readme + '\n\n' + history, author="John Hardy", diff --git a/stackformation/__init__.py b/stackformation/__init__.py index ab296726..88a7b192 100644 --- a/stackformation/__init__.py +++ b/stackformation/__init__.py @@ -17,7 +17,7 @@ __author__ = """John Hardy""" __email__ = 'john@johnchardy.com' -__version__ = '0.1.2' +__version__ = '0.1.3' logger = logging.getLogger(__name__) From 503f25e5c1b3d0d1d47198ba07e5beffc4745695 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 13:32:14 -0800 Subject: [PATCH 03/11] Add self-reference security group --- stackformation/aws/vpc.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/stackformation/aws/vpc.py b/stackformation/aws/vpc.py index 7a64abc8..2f36a7d7 100644 --- a/stackformation/aws/vpc.py +++ b/stackformation/aws/vpc.py @@ -36,6 +36,33 @@ def output_security_group(self): self.name ) + +class SelfReferenceSecurityGroup(SecurityGroup): + + def __init__(self): + name = "SelfReferenceSecurityGroup" + super(SelfReferenceSecurityGroup, self).__init__(name) + + def _build_security_group(self, t, vpc): + + sg = t.add_resource(ec2.SecurityGroup( + '{}'.format(self.name), + GroupDescription="{} Self Reference Security Group".format(self.stack.get_stack_name()), + GroupName="{} {}".format(self.stack.get_stack_name(), self.name), + VpcId=Ref(vpc), + SecurityGroupIngress=[] + )) + + t.add_resource(ec2.SecurityGroupIngress( + '{}Ingress'.format(self.name), + ToPort='-1', + FromPort='-1', + IpProtocol='-1', + SourceSecurityGroupId=Ref(sg), + GroupId=Ref(sg), + )) + return sg + class SSHSecurityGroup(SecurityGroup): def __init__(self, name="SSH"): From ea20604ec5705fd59fcc02def06d6be44a54d14a Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 15:11:47 -0800 Subject: [PATCH 04/11] pause for bumptest --- setup.cfg | 1 + stackformation/__init__.py | 20 +++++++++++++++----- stackformation/cli.py | 34 +++++++++++++++++++++++++++++++++- stackformation/utils.py | 29 +++++++++++++++++++++++++++-- 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2090e7ad..ac056b1d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,7 @@ current_version = 0.1.3 commit = True tag = True +tag_name = "{new_version}" [bumpversion:file:setup.py] search = version='{current_version}' diff --git a/stackformation/__init__.py b/stackformation/__init__.py index 88a7b192..fb77ff5d 100644 --- a/stackformation/__init__.py +++ b/stackformation/__init__.py @@ -290,6 +290,8 @@ def list_amis(self): return amis + + class SoloStack(): pass @@ -370,10 +372,10 @@ def parse_template_components(self, env, context): def get_stack_name(self): return "{}{}{}{}".format( - ''.join([i.capitalize() for i in self.infra.prefix]), - self.infra.name.capitalize(), - self.stack_name.capitalize(), - self.name.capitalize() + ''.join([utils.ucfirst(i) for i in self.infra.prefix]), + utils.ucfirst(self.infra.name), + utils.ucfirst(self.stack_name), + utils.ucfirst(self.name) ) def get_remote_stack_name(self): @@ -656,7 +658,15 @@ def find_class_in_list(self, ls, clazz, name=None): return None - # def get_dependent_stacks(self, infra): + def review(self, infra): + + # get stack dependencies + deps = infra.get_dependent_stacks(self) + + cf = self.infra.boto_session.client('cloudformation') + + info = cf.describe_stacks(StackName=self.get_remote_stack_name()) + print(info['Stacks'][0]['Parameters']) def build_template(self): raise NotImplementedError("Must implement method to extend Stack") diff --git a/stackformation/cli.py b/stackformation/cli.py index 47c53620..70f5dd56 100644 --- a/stackformation/cli.py +++ b/stackformation/cli.py @@ -36,12 +36,23 @@ def build(): @stacks.command(name='list') -def list_stack(): +@click.argument('selector', nargs=-1) +def list_stack(selector): + + selector = list(selector) infra = load_infra_file() stacks = infra.list_stacks() + results = [] + + for stack in stacks: + if match_stack(selector, stack): + results.append(stack) + + stacks = results + for stack in stacks: click.echo("{}Stack:{} {} {}[{}]{}".format( Style.BRIGHT, @@ -58,6 +69,25 @@ def list_stack(): Style.RESET_ALL )) +@stacks.command(help='Deploy stacks') +@click.argument('selector', nargs=-1) +def review(selector): + + selector = list(selector) + + infra = load_infra_file() + + stacks = infra.list_stacks() + + results = [] + + for stack in stacks: + if match_stack(selector, stack): + results.append(stack) + + for stack in results: + stack.review(infra) + @stacks.command(help='Deploy stacks') @click.argument('selector', nargs=-1) @@ -154,6 +184,8 @@ def load_configuration(): if HOME is None: raise Exception("$HOME environment variable needs to be set to save configuration") + + def jinja_env(): path = os.path.dirname(os.path.realpath(__file__)) diff --git a/stackformation/utils.py b/stackformation/utils.py index b4ec6d58..dd27f0dd 100644 --- a/stackformation/utils.py +++ b/stackformation/utils.py @@ -36,10 +36,12 @@ def match_stack(selector, stack): if not isinstance(selector, list): selector = selector.split(' ') + selector = [i.lower() for i in selector] + pos = [] neg = [] - sn = stack.get_stack_name() - rn = stack.get_remote_stack_name() + sn = stack.get_stack_name().lower() + rn = stack.get_remote_stack_name().lower() result = False for s in selector: @@ -57,3 +59,26 @@ def match_stack(selector, stack): result = False return result + + +def ucfirst(word): + """Uppercase the first letter in a string + and error if string starts with an digit + + Args: + word (str): the word + + Raises: + Exception + + Returns: + (str) + + """ + if len(word) <= 0: + return "" + if not re.match('^[a-zA-Z]', word): + raise Exception("{} Cannot begin with a digit".format(word)) + ls = list(word) + ls[0] = ls[0].upper() + return ''.join(ls) From b19f74b663cb0d4271d08a72466976f4ea11e9fd Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 15:11:57 -0800 Subject: [PATCH 05/11] add new files for bumptest --- stackformation/aws/cloudfront.py | 0 stackformation/aws/templates/slack-lambda-notification.py.j2 | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 stackformation/aws/cloudfront.py create mode 100644 stackformation/aws/templates/slack-lambda-notification.py.j2 diff --git a/stackformation/aws/cloudfront.py b/stackformation/aws/cloudfront.py new file mode 100644 index 00000000..e69de29b diff --git a/stackformation/aws/templates/slack-lambda-notification.py.j2 b/stackformation/aws/templates/slack-lambda-notification.py.j2 new file mode 100644 index 00000000..e69de29b From 3856a44d28e82a15f44e5781521e3d8944683672 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 15:13:58 -0800 Subject: [PATCH 06/11] update bump tag template --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ac056b1d..208496c3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ current_version = 0.1.3 commit = True tag = True -tag_name = "{new_version}" +tag_name = {new_version} [bumpversion:file:setup.py] search = version='{current_version}' From d6b569f50b604c4ee0972dd4a2a7b10d054410d5 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 15:21:17 -0800 Subject: [PATCH 07/11] update jenksins to push tag --- Jenkinsfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index c9420db2..70ee3327 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,6 +37,25 @@ node { } } + if (env.TAG_NAME) { + stage("Publish To PyPi") { + + echo "Cleaning" + sh "docker run --rm -v ${env.WORKSPACE}:${env.WORKSPACE} -w ${env.WORKSPACE} ${img_tag} make clean" + echo "Build DIST Package" + sh "docker run --rm -v ${env.WORKSPACE}:${env.WORKSPACE} -w ${env.WORKSPACE} ${img_tag} python3 setup.py dist" + + withCredentials([usernamePassword(credentialsId: 'ibejohn818PyPi', passwordVariable: 'PYPIPASSWD', usernameVariable: 'PYPIUSER')]) { + echo "Send to PyPi" + + def dist_name = "jh-stackformation-${env.TAG_NAME}.tar.gz" + + sh "docker run --rm -v ${env.WORKSPACE}:${env.WORKSPACE} -w ${env.WORKSPACE} ${img_tag} twine upload dist/${dist_name} -u ${env.PYPIUSER} -p ${env.PYPIPASSWD}" + } + + } + } + } catch(Exception err) { currentBuild.result = "FAILURE" } finally { From 29cdcbb145940af38c2cb10b3d52b1d5f0db62cf Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 15:28:16 -0800 Subject: [PATCH 08/11] fix tests after camel case updates --- tests/test_aws_ec2.py | 8 ++++---- tests/test_aws_vpc.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_aws_ec2.py b/tests/test_aws_ec2.py index 3be7b68f..7ea8ab0e 100644 --- a/tests/test_aws_ec2.py +++ b/tests/test_aws_ec2.py @@ -41,13 +41,13 @@ def test_ec2_stack(infra): inst = t.resources['WebEC2Instance'].to_dict() - assert ec2_stack.output_instance() == "ProdTestWebEc2WebEC2Instance" + assert ec2_stack.output_instance() == "ProdTestWebEC2WebEC2Instance" assert inst['Properties']['KeyName'] == 'testkey' - assert inst['Properties']['NetworkInterfaces'][0]['SubnetId'] == {'Ref': 'ProdTestVpcPublicSubnet1'} + assert inst['Properties']['NetworkInterfaces'][0]['SubnetId'] == {'Ref': 'ProdTestVPCPublicSubnet1'} - assert inst['Properties']['NetworkInterfaces'][0]['GroupSet'][0] == {'Ref': 'ProdTestVpcSSHSecurityGroup'} + assert inst['Properties']['NetworkInterfaces'][0]['GroupSet'][0] == {'Ref': 'ProdTestVPCSSHSecurityGroup'} ec2_stack.private_subnet = True @@ -55,4 +55,4 @@ def test_ec2_stack(infra): inst = t.resources['WebEC2Instance'].to_dict() - assert inst['Properties']['NetworkInterfaces'][0]['SubnetId'] == {'Ref': 'ProdTestVpcPrivateSubnet1'} + assert inst['Properties']['NetworkInterfaces'][0]['SubnetId'] == {'Ref': 'ProdTestVPCPrivateSubnet1'} diff --git a/tests/test_aws_vpc.py b/tests/test_aws_vpc.py index d1793fb9..b4e34313 100644 --- a/tests/test_aws_vpc.py +++ b/tests/test_aws_vpc.py @@ -27,10 +27,10 @@ def test_vpc(prod_infra): assert len(vpc_stack.output_azs()) == 3 assert len(vpc_stack.output_private_subnets()) == 3 assert len(vpc_stack.output_public_subnets()) == 3 - assert vpc_stack.output_vpc() == "ProdTestVpcVpcId" - assert vpc_stack.output_public_routetable() == "ProdTestVpcPublicRouteTable" - assert vpc_stack.output_private_routetable() == "ProdTestVpcPrivateRouteTable" - assert vpc_stack.output_default_acl_table() == "ProdTestVpcDefaultAclTable" + assert vpc_stack.output_vpc() == "ProdTestVPCVpcId" + assert vpc_stack.output_public_routetable() == "ProdTestVPCPublicRouteTable" + assert vpc_stack.output_private_routetable() == "ProdTestVPCPrivateRouteTable" + assert vpc_stack.output_default_acl_table() == "ProdTestVPCDefaultAclTable" def test_base_sec_group(prod_infra): @@ -103,7 +103,7 @@ def test_ssh_sec_group(prod_infra): assert sg_dict['Properties']['SecurityGroupIngress'][0]['FromPort'] == 22 assert sg_dict['Properties']['SecurityGroupIngress'][0]['CidrIp'] == '1.2.3.4/5' - assert ssh_sg.output_security_group() == "ProdTestVpcSSHSecurityGroup" + assert ssh_sg.output_security_group() == "ProdTestVPCSSHSecurityGroup" def test_web_sec_group(prod_infra): @@ -126,7 +126,7 @@ def test_web_sec_group(prod_infra): assert sg['Properties']['SecurityGroupIngress'][1]['FromPort'] == 443 assert sg['Properties']['SecurityGroupIngress'][1]['CidrIp'] == '0.0.0.0/0' - assert web_sg.output_security_group() == "ProdTestVpcWebSecurityGroup" + assert web_sg.output_security_group() == "ProdTestVPCWebSecurityGroup" def test_all_ports_sec_group(prod_infra): From 00bb5b2b179c280e3d7aac7dbfa5298f508ea0fc Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 15:30:40 -0800 Subject: [PATCH 09/11] =?UTF-8?q?Bump=20version:=200.1.3=20=E2=86=92=200.1?= =?UTF-8?q?.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.cfg | 2 +- setup.py | 2 +- stackformation/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 208496c3..b27afd9b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.3 +current_version = 0.1.4 commit = True tag = True tag_name = {new_version} diff --git a/setup.py b/setup.py index a74a8960..b70602e7 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( name='jh-stackformation', - version='0.1.3', + version='0.1.4', description="AWS CloudFormation framework", long_description=readme + '\n\n' + history, author="John Hardy", diff --git a/stackformation/__init__.py b/stackformation/__init__.py index fb77ff5d..23218ddb 100644 --- a/stackformation/__init__.py +++ b/stackformation/__init__.py @@ -17,7 +17,7 @@ __author__ = """John Hardy""" __email__ = 'john@johnchardy.com' -__version__ = '0.1.3' +__version__ = '0.1.4' logger = logging.getLogger(__name__) From 382151a8ba55ba3d5c1614ddc99119c1a4ba9607 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 15:35:46 -0800 Subject: [PATCH 10/11] update dist build --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 70ee3327..f4cdac7c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -43,7 +43,7 @@ node { echo "Cleaning" sh "docker run --rm -v ${env.WORKSPACE}:${env.WORKSPACE} -w ${env.WORKSPACE} ${img_tag} make clean" echo "Build DIST Package" - sh "docker run --rm -v ${env.WORKSPACE}:${env.WORKSPACE} -w ${env.WORKSPACE} ${img_tag} python3 setup.py dist" + sh "docker run --rm -v ${env.WORKSPACE}:${env.WORKSPACE} -w ${env.WORKSPACE} ${img_tag} python3 setup.py sdist" withCredentials([usernamePassword(credentialsId: 'ibejohn818PyPi', passwordVariable: 'PYPIPASSWD', usernameVariable: 'PYPIUSER')]) { echo "Send to PyPi" From 5a714b932db1bf6902132428d650729041bafe1f Mon Sep 17 00:00:00 2001 From: John Date: Wed, 27 Dec 2017 15:35:55 -0800 Subject: [PATCH 11/11] =?UTF-8?q?Bump=20version:=200.1.4=20=E2=86=92=200.1?= =?UTF-8?q?.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.cfg | 2 +- setup.py | 2 +- stackformation/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index b27afd9b..9c9b5629 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.4 +current_version = 0.1.5 commit = True tag = True tag_name = {new_version} diff --git a/setup.py b/setup.py index b70602e7..1786f484 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( name='jh-stackformation', - version='0.1.4', + version='0.1.5', description="AWS CloudFormation framework", long_description=readme + '\n\n' + history, author="John Hardy", diff --git a/stackformation/__init__.py b/stackformation/__init__.py index 23218ddb..a193e602 100644 --- a/stackformation/__init__.py +++ b/stackformation/__init__.py @@ -17,7 +17,7 @@ __author__ = """John Hardy""" __email__ = 'john@johnchardy.com' -__version__ = '0.1.4' +__version__ = '0.1.5' logger = logging.getLogger(__name__)