diff --git a/chalice/awsclient.py b/chalice/awsclient.py index 32af99785f..1ecd21f941 100644 --- a/chalice/awsclient.py +++ b/chalice/awsclient.py @@ -80,8 +80,8 @@ def create_function(self, function_name, role_arn, zip_contents): return response['FunctionArn'] def update_function_code(self, function_name, zip_contents): - # type: (str, str) -> None - self._client('lambda').update_function_code( + # type: (str, str) -> Dict[str, Any] + return self._client('lambda').update_function_code( FunctionName=function_name, ZipFile=zip_contents) def get_role_arn_for_name(self, name): @@ -138,6 +138,17 @@ def get_rest_api_id(self, name): return api['id'] return None + def rest_api_exists(self, rest_api_id): + # type: (str) -> bool + """Check if an an API Gateway REST API exists.""" + try: + self._client('apigateway').get_rest_api(restApiId=rest_api_id) + return True + except botocore.exceptions.ClientError as e: + if e['Code'] == 'NotFoundException': + return False + raise + def import_rest_api(self, swagger_document): # type: (Dict[str, Any]) -> str client = self._client('apigateway') diff --git a/chalice/cli/__init__.py b/chalice/cli/__init__.py index 482f114fe1..75b2310bb7 100644 --- a/chalice/cli/__init__.py +++ b/chalice/cli/__init__.py @@ -22,6 +22,8 @@ from chalice.deploy import deployer from chalice.logs import LogRetriever from chalice.package import create_app_packager +from chalice.utils import record_deployed_values + TEMPLATE_APP = """\ from chalice import Chalice @@ -168,13 +170,20 @@ def local(ctx, port=8000): def deploy(ctx, autogen_policy, profile, stage): # type: (click.Context, bool, str, str) -> None config = create_config_obj( - ctx, stage_name=stage, autogen_policy=autogen_policy, + # Note: stage_name is not the same thing as the chalice stage. + # This is a legacy artifact that just means "API gateway stage", + # or for our purposes, the URL prefix. + ctx, stage_name='dev', autogen_policy=autogen_policy, profile=profile) + if stage is None: + stage = 'dev' session = create_botocore_session(profile=config.profile, debug=ctx.obj['debug']) d = deployer.create_default_deployer(session=session, prompter=click) try: - d.deploy(config) + deployed_values = d.deploy(config, stage_name=stage) + record_deployed_values(deployed_values, os.path.join( + config.project_dir, '.chalice', 'deployed.json')) except botocore.exceptions.NoRegionError: e = click.ClickException("No region configured. " "Either export the AWS_DEFAULT_REGION " diff --git a/chalice/config.py b/chalice/config.py index 1f117004ba..a7394ad2b9 100644 --- a/chalice/config.py +++ b/chalice/config.py @@ -1,4 +1,7 @@ -from typing import Dict, Any # noqa +import os +import json + +from typing import Dict, Any, Optional # noqa from chalice.app import Chalice # noqa StrMap = Dict[str, Any] @@ -29,11 +32,6 @@ def create(cls, **kwargs): # type: (**Any) -> Config return cls(user_provided_params=kwargs.copy()) - @property - def lambda_arn(self): - # type: () -> str - return self._chain_lookup('lambda_arn') - @property def profile(self): # type: () -> str @@ -97,3 +95,55 @@ def _chain_lookup(self, name): for cfg_dict in all_dicts: if isinstance(cfg_dict, dict) and cfg_dict.get(name) is not None: return cfg_dict[name] + + @property + def lambda_arn(self): + # type: () -> str + return self._chain_lookup('lambda_arn') + + def deployed_resources(self, stage_name): + # type: (str) -> Optional[DeployedResources] + """Return resources associated with a given stage. + + If a deployment to a given stage has never happened, + this method will return a value of None. + + """ + # This is arguably the wrong level of abstraction. + # We might be able to move this elsewhere. + deployed_file = os.path.join(self.project_dir, '.chalice', + 'deployed.json') + if not os.path.isfile(deployed_file): + return None + with open(deployed_file, 'r') as f: + data = json.load(f) + if stage_name not in data: + return None + return DeployedResources.from_dict(data[stage_name]) + + +class DeployedResources(object): + def __init__(self, backend, api_handler_arn, + api_handler_name, rest_api_id, api_gateway_stage, + region, chalice_version): + # type: (str, str, str, str, str, str, str) -> None + self.backend = backend + self.api_handler_arn = api_handler_arn + self.api_handler_name = api_handler_name + self.rest_api_id = rest_api_id + self.api_gateway_stage = api_gateway_stage + self.region = region + self.chalice_version = chalice_version + + @classmethod + def from_dict(cls, data): + # type: (Dict[str, str]) -> DeployedResources + return cls( + data['backend'], + data['api_handler_arn'], + data['api_handler_name'], + data['rest_api_id'], + data['api_gateway_stage'], + data['region'], + data['chalice_version'], + ) diff --git a/chalice/deploy/deployer.py b/chalice/deploy/deployer.py index 3dd12f336a..fe918c0c16 100644 --- a/chalice/deploy/deployer.py +++ b/chalice/deploy/deployer.py @@ -8,12 +8,13 @@ import uuid import botocore.session # noqa -from typing import Any, Tuple, Callable, List, Dict # noqa +from typing import Any, Tuple, Callable, List, Dict, Optional # noqa from chalice import app # noqa +from chalice import __version__ as chalice_version from chalice import policy from chalice.awsclient import TypedAWSClient -from chalice.config import Config # noqa +from chalice.config import Config, DeployedResources # noqa from chalice.deploy.packager import LambdaDeploymentPackager from chalice.deploy.swagger import SwaggerGenerator from chalice.utils import OSUtils @@ -132,16 +133,19 @@ def confirm(self, text, default=False, abort=False): class Deployer(object): + + BACKEND_NAME = 'api' + def __init__(self, apigateway_deploy, lambda_deploy): # type: (APIGatewayDeployer, LambdaDeployer) -> None self._apigateway_deploy = apigateway_deploy self._lambda_deploy = lambda_deploy - def deploy(self, config): - # type: (Config) -> Tuple[str, str, str] + def deploy(self, config, stage_name='dev'): + # type: (Config, str) -> Dict[str, Any] """Deploy chalice application to AWS. - :type config: dict + :type config: Config :param config: A dictionary of config values including: * project_dir - The directory containing the project @@ -150,83 +154,25 @@ def deploy(self, config): """ validate_configuration(config) - self._lambda_deploy.deploy(config) - rest_api_id, region_name, stage = self._apigateway_deploy.deploy( - config) + existing_resources = config.deployed_resources(stage_name) + deployed_values = self._lambda_deploy.deploy( + config, existing_resources, stage_name) + rest_api_id, region_name, apig_stage = self._apigateway_deploy.deploy( + config, existing_resources) print ( "https://{api_id}.execute-api.{region}.amazonaws.com/{stage}/" - .format(api_id=rest_api_id, region=region_name, stage=stage) + .format(api_id=rest_api_id, region=region_name, stage=apig_stage) ) - return rest_api_id, region_name, stage - - -class ApplicationPolicyHandler(object): - """Manages the IAM policy for an application.""" - - _EMPTY_POLICY = { - 'Version': '2012-10-17', - 'Statement': [], - } - - def __init__(self, osutils): - # type: (OSUtils) -> None - self._osutils = osutils - - def generate_policy_from_app_source(self, project_dir, autogen_policy): - # type: (str, bool) -> Dict[str, Any] - """Generate a policy from application source code. - - If the ``autogen_policy`` value is set to false, then - the .chalice/policy.json file will be used instead of generating - the policy from the source code. - - """ - if autogen_policy: - app_policy = self._do_generate_from_source(project_dir) - else: - app_policy = self.load_last_policy(project_dir) - return app_policy - - def _do_generate_from_source(self, project_dir): - # type: (str) -> Dict[str, Any] - app_py = os.path.join(project_dir, 'app.py') - assert self._osutils.file_exists(app_py) - app_source = self._osutils.get_file_contents(app_py, binary=False) - app_policy = policy.policy_from_source_code(app_source) - app_policy['Statement'].append(CLOUDWATCH_LOGS) - return app_policy - - def load_last_policy(self, project_dir): - # type: (str) -> Dict[str, Any] - """Load the last recorded policy file for the app. - - Whenever a policy is generated, the file is written to - .chalice/policy.json. This method will load that file - and return the IAM policy. - - If the file does not exist, an empty policy is returned. - - """ - policy_file = self._app_policy_file(project_dir) - if not self._osutils.file_exists(policy_file): - return self._EMPTY_POLICY - return json.loads( - self._osutils.get_file_contents(policy_file, binary=False) - ) - - def record_policy(self, project_dir, policy): - # type: (str, Dict[str, Any]) -> None - policy_file = self._app_policy_file(project_dir) - self._osutils.set_file_contents( - policy_file, - json.dumps(policy, indent=2, separators=(',', ': ')), - binary=False - ) - - def _app_policy_file(self, project_dir): - # type: (str) -> str - policy_file = os.path.join(project_dir, '.chalice', 'policy.json') - return policy_file + deployed_values.update({ + 'rest_api_id': rest_api_id, + 'region': region_name, + 'api_gateway_stage': apig_stage, + 'backend': self.BACKEND_NAME, + 'chalice_version': chalice_version, + }) + return { + stage_name: deployed_values + } class LambdaDeployer(object): @@ -244,34 +190,42 @@ def __init__(self, self._osutils = osutils self._app_policy = app_policy - def deploy(self, config): - # type: (Config) -> None - app_name = config.app_name - if self._aws_client.lambda_function_exists(app_name): - self._get_or_create_lambda_role_arn(config) - self._update_lambda_function(config) + def deploy(self, config, existing_resources, stage_name): + # type: (Config, Optional[DeployedResources], str) -> Dict[str, str] + function_name = '%s-%s' % (config.app_name, stage_name) + deployed_values = {'api_handler_name': function_name} + if existing_resources is not None and \ + self._aws_client.lambda_function_exists( + existing_resources.api_handler_name): + self._get_or_create_lambda_role_arn( + config, existing_resources.api_handler_name) + self._update_lambda_function( + config, existing_resources.api_handler_name) + function_arn = existing_resources.api_handler_arn else: - function_arn = self._first_time_lambda_create(config) + function_arn = self._first_time_lambda_create( + config, function_name) # Record the lambda_arn for later use. config.config_from_disk['lambda_arn'] = function_arn self._write_config_to_disk(config) - print "Lambda deploy done." + deployed_values['api_handler_arn'] = function_arn + return deployed_values - def _get_or_create_lambda_role_arn(self, config): - # type: (Config) -> str + def _get_or_create_lambda_role_arn(self, config, role_name): + # type: (Config, str) -> str if not config.manage_iam_role: # We've already validated the config, so we know # if manage_iam_role==False, then they've provided a # an iam_role_arn. return config.iam_role_arn - app_name = config.app_name try: - role_arn = self._aws_client.get_role_arn_for_name(app_name) - self._update_role_with_latest_policy(app_name, config) + # We're using the lambda function_name as the role_name. + role_arn = self._aws_client.get_role_arn_for_name(role_name) + self._update_role_with_latest_policy(role_name, config) except ValueError: print "Creating role" - role_arn = self._create_role_from_source_code(config) + role_arn = self._create_role_from_source_code(config, role_name) return role_arn def _update_role_with_latest_policy(self, app_name, config): @@ -301,23 +255,22 @@ def _update_role_with_latest_policy(self, app_name, config): policy_document=app_policy) self._app_policy.record_policy(config.project_dir, app_policy) - def _first_time_lambda_create(self, config): - # type: (Config) -> str + def _first_time_lambda_create(self, config, function_name): + # type: (Config, str) -> str # Creates a lambda function and returns the # function arn. # First we need to create a deployment package. print "Initial creation of lambda function." - app_name = config.app_name - role_arn = self._get_or_create_lambda_role_arn(config) + role_arn = self._get_or_create_lambda_role_arn(config, function_name) zip_filename = self._packager.create_deployment_package( config.project_dir) with open(zip_filename, 'rb') as f: zip_contents = f.read() return self._aws_client.create_function( - app_name, role_arn, zip_contents) + function_name, role_arn, zip_contents) - def _update_lambda_function(self, config): - # type: (Config) -> None + def _update_lambda_function(self, config, lambda_name): + # type: (Config, str) -> None print "Updating lambda function..." project_dir = config.project_dir packager = self._packager @@ -332,8 +285,7 @@ def _update_lambda_function(self, config): zip_contents = self._osutils.get_file_contents( deployment_package_filename, binary=True) print "Sending changes to lambda." - self._aws_client.update_function_code(config.app_name, - zip_contents) + self._aws_client.update_function_code(lambda_name, zip_contents) def _write_config_to_disk(self, config): # type: (Config) -> None @@ -342,9 +294,8 @@ def _write_config_to_disk(self, config): with open(config_filename, 'w') as f: f.write(json.dumps(config.config_from_disk, indent=2)) - def _create_role_from_source_code(self, config): - # type: (Config) -> str - app_name = config.app_name + def _create_role_from_source_code(self, config, role_name): + # type: (Config, str) -> str app_policy = self._app_policy.generate_policy_from_app_source( config.project_dir, config.autogen_policy) if len(app_policy['Statement']) > 1: @@ -353,7 +304,7 @@ def _create_role_from_source_code(self, config): self._prompter.confirm("Would you like to continue? ", default=True, abort=True) role_arn = self._aws_client.create_role( - name=app_name, + name=role_name, trust_policy=LAMBDA_TRUST_POLICY, policy=app_policy ) @@ -366,22 +317,29 @@ def __init__(self, aws_client): # type: (TypedAWSClient) -> None self._aws_client = aws_client - def deploy(self, config): - # type: (Config) -> Tuple[str, str, str] - app_name = config.app_name - rest_api_id = self._aws_client.get_rest_api_id(app_name) - if rest_api_id is None: - print "Initiating first time deployment..." - return self._first_time_deploy(config) - else: + def deploy(self, config, existing_resources): + # type: (Config, Optional[DeployedResources]) -> Tuple[str, str, str] + if existing_resources is not None and \ + self._aws_client.rest_api_exists( + existing_resources.rest_api_id): print "API Gateway rest API already found." + rest_api_id = existing_resources.rest_api_id return self._create_resources_for_api(config, rest_api_id) + else: + print "Initiating first time deployment..." + return self._first_time_deploy(config) def _first_time_deploy(self, config): # type: (Config) -> Tuple[str, str, str] generator = SwaggerGenerator(self._aws_client.region_name, config.lambda_arn) swagger_doc = generator.generate_swagger(config.chalice_app) + # The swagger_doc that's generated will contain the "name" which is + # used to set the name for the restAPI. API Gateway allows you + # to have multiple restAPIs with the same name, they'll have + # different restAPI ids. It might be worth creating unique names + # for each rest API, but that would require injecting chalice stage + # information into the swagger generator. rest_api_id = self._aws_client.import_rest_api(swagger_doc) stage = config.stage or 'dev' self._deploy_api_to_stage(rest_api_id, stage, config) @@ -408,3 +366,72 @@ def _deploy_api_to_stage(self, rest_api_id, stage, config): rest_api_id, str(uuid.uuid4()), ) + + +class ApplicationPolicyHandler(object): + """Manages the IAM policy for an application.""" + + _EMPTY_POLICY = { + 'Version': '2012-10-17', + 'Statement': [], + } + + def __init__(self, osutils): + # type: (OSUtils) -> None + self._osutils = osutils + + def generate_policy_from_app_source(self, project_dir, autogen_policy): + # type: (str, bool) -> Dict[str, Any] + """Generate a policy from application source code. + + If the ``autogen_policy`` value is set to false, then + the .chalice/policy.json file will be used instead of generating + the policy from the source code. + + """ + if autogen_policy: + app_policy = self._do_generate_from_source(project_dir) + else: + app_policy = self.load_last_policy(project_dir) + return app_policy + + def _do_generate_from_source(self, project_dir): + # type: (str) -> Dict[str, Any] + app_py = os.path.join(project_dir, 'app.py') + assert self._osutils.file_exists(app_py) + app_source = self._osutils.get_file_contents(app_py, binary=False) + app_policy = policy.policy_from_source_code(app_source) + app_policy['Statement'].append(CLOUDWATCH_LOGS) + return app_policy + + def load_last_policy(self, project_dir): + # type: (str) -> Dict[str, Any] + """Load the last recorded policy file for the app. + + Whenever a policy is generated, the file is written to + .chalice/policy.json. This method will load that file + and return the IAM policy. + + If the file does not exist, an empty policy is returned. + + """ + policy_file = self._app_policy_file(project_dir) + if not self._osutils.file_exists(policy_file): + return self._EMPTY_POLICY + return json.loads( + self._osutils.get_file_contents(policy_file, binary=False) + ) + + def record_policy(self, project_dir, policy): + # type: (str, Dict[str, Any]) -> None + policy_file = self._app_policy_file(project_dir) + self._osutils.set_file_contents( + policy_file, + json.dumps(policy, indent=2, separators=(',', ': ')), + binary=False + ) + + def _app_policy_file(self, project_dir): + # type: (str) -> str + policy_file = os.path.join(project_dir, '.chalice', 'policy.json') + return policy_file diff --git a/chalice/utils.py b/chalice/utils.py index 3cc5122560..c7cbb83c84 100644 --- a/chalice/utils.py +++ b/chalice/utils.py @@ -1,6 +1,23 @@ import os +import json -from typing import IO # noqa +from typing import IO, Dict, Any # noqa + + +def record_deployed_values(deployed_values, filename): + # type: (Dict[str, str], str) -> None + """Record deployed values to a JSON file. + + This allows subsequent deploys to lookup previously deployed values. + + """ + final_values = {} # type: Dict[str, Any] + if os.path.isfile(filename): + with open(filename, 'r') as f: + final_values = json.load(f) + final_values.update(deployed_values) + with open(filename, 'wb') as f: + f.write(json.dumps(final_values, indent=2, separators=(',', ': '))) class OSUtils(object): diff --git a/tests/functional/test_utils.py b/tests/functional/test_utils.py new file mode 100644 index 0000000000..5091514011 --- /dev/null +++ b/tests/functional/test_utils.py @@ -0,0 +1,10 @@ +import json + +from chalice import utils + + +def test_can_write_recorded_values(tmpdir): + filename = str(tmpdir.join('deployed.json')) + utils.record_deployed_values({'deployed': 'foo'}, filename) + with open(filename, 'r') as f: + assert json.load(f) == {'deployed': 'foo'} diff --git a/tests/unit/deploy/test_deployer.py b/tests/unit/deploy/test_deployer.py index f2119794ff..05ca8ae2e4 100644 --- a/tests/unit/deploy/test_deployer.py +++ b/tests/unit/deploy/test_deployer.py @@ -7,9 +7,10 @@ from botocore.stub import Stubber from pytest import fixture +from chalice import __version__ as chalice_version from chalice.app import Chalice from chalice.awsclient import TypedAWSClient -from chalice.config import Config +from chalice.config import Config, DeployedResources from chalice.deploy.deployer import APIGatewayDeployer from chalice.deploy.deployer import ApplicationPolicyHandler from chalice.deploy.deployer import Deployer @@ -92,7 +93,7 @@ def test_api_gateway_deployer_initial_deploy(config_obj): aws_client.import_rest_api.return_value = 'rest-api-id' d = APIGatewayDeployer(aws_client) - d.deploy(config_obj) + d.deploy(config_obj, None) # mock.ANY because we don't want to test the contents of the swagger # doc. That's tested exhaustively elsewhere. @@ -114,10 +115,12 @@ def test_api_gateway_deployer_redeploy_api(config_obj): # The rest_api_id does not exist which will trigger # the initial import - aws_client.get_rest_api_id.return_value = 'existing-id' + deployed = DeployedResources( + None, None, None, 'existing-id', 'dev', None, None) + aws_client.rest_api_exists.return_value = True d = APIGatewayDeployer(aws_client) - d.deploy(config_obj) + d.deploy(config_obj, deployed) aws_client.update_api_from_swagger.assert_called_with('existing-id', mock.ANY) @@ -202,14 +205,43 @@ def test_can_deploy_apig_and_lambda(sample_app): lambda_deploy = mock.Mock(spec=LambdaDeployer) apig_deploy = mock.Mock(spec=APIGatewayDeployer) + lambda_deploy.deploy.return_value = { + 'api_handler_name': 'lambda_function', + 'api_handler_arn': 'my_lambda_arn', + } apig_deploy.deploy.return_value = ('api_id', 'region', 'stage') d = Deployer(apig_deploy, lambda_deploy) - cfg = Config({'chalice_app': sample_app}) - result = d.deploy(cfg) - assert result == ('api_id', 'region', 'stage') - lambda_deploy.deploy.assert_called_with(cfg) - apig_deploy.deploy.assert_called_with(cfg) + cfg = Config({'chalice_app': sample_app, 'project_dir': '.'}) + d.deploy(cfg) + lambda_deploy.deploy.assert_called_with(cfg, None, 'dev') + apig_deploy.deploy.assert_called_with(cfg, None) + + +def test_deployer_returns_deployed_resources(sample_app): + cfg = Config({'chalice_app': sample_app, 'project_dir': '.'}) + lambda_deploy = mock.Mock(spec=LambdaDeployer) + apig_deploy = mock.Mock(spec=APIGatewayDeployer) + + apig_deploy.deploy.return_value = ('api_id', 'region', 'stage') + lambda_deploy.deploy.return_value = { + 'api_handler_name': 'lambda_function', + 'api_handler_arn': 'my_lambda_arn', + } + + d = Deployer(apig_deploy, lambda_deploy) + deployed_values = d.deploy(cfg) + assert deployed_values == { + 'dev': { + 'backend': 'api', + 'api_handler_arn': 'my_lambda_arn', + 'api_handler_name': 'lambda_function', + 'rest_api_id': 'api_id', + 'api_gateway_stage': 'stage', + 'region': 'region', + 'chalice_version': chalice_version, + } + } def test_noprompt_always_returns_default(): @@ -228,6 +260,7 @@ def test_lambda_deployer_repeated_deploy(app_policy, sample_app): packager.deployment_package_filename.return_value = 'packages.zip' # Given the lambda function already exists: aws_client.lambda_function_exists.return_value = True + aws_client.update_function_code.return_value = {"FunctionArn": "myarn"} # And given we don't want chalice to manage our iam role for the lambda # function: cfg = Config({'chalice_app': sample_app, 'manage_iam_role': False, @@ -236,7 +269,11 @@ def test_lambda_deployer_repeated_deploy(app_policy, sample_app): d = LambdaDeployer(aws_client, packager, None, osutils, app_policy) # Doing a lambda deploy: - d.deploy(cfg) + lambda_function_name = 'lambda_function_name' + deployed = DeployedResources( + 'api', 'api_handler_arn', lambda_function_name, + None, 'dev', None, None) + d.deploy(cfg, deployed, 'dev') # Should result in injecting the latest app code. packager.inject_latest_app.assert_called_with('packages.zip', @@ -244,7 +281,7 @@ def test_lambda_deployer_repeated_deploy(app_policy, sample_app): # And should result in the lambda function being updated with the API. aws_client.update_function_code.assert_called_with( - 'appname', 'package contents') + lambda_function_name, 'package contents') def test_cant_have_options_with_cors(sample_app):