From f5488c969125939b449aa2bee35812bab20b14f8 Mon Sep 17 00:00:00 2001 From: Sam Liu Date: Fri, 24 Feb 2023 15:13:09 -0800 Subject: [PATCH 1/2] chore: Remove type: ignore in samtranslator/translator --- .../feature_toggle/feature_toggle.py | 10 ++++- samtranslator/metrics/method_decorator.py | 2 +- samtranslator/metrics/metrics.py | 10 +++-- samtranslator/parser/parser.py | 4 +- .../application/serverless_app_plugin.py | 37 +++++++++------ .../policies/policy_templates_plugin.py | 3 +- samtranslator/region_configuration.py | 2 +- samtranslator/sdk/parameter.py | 5 ++- samtranslator/translator/arn_generator.py | 2 +- .../translator/managed_policy_translator.py | 12 ++--- samtranslator/translator/transform.py | 15 +++++-- samtranslator/translator/translator.py | 45 ++++++++++--------- samtranslator/utils/py27hash_fix.py | 4 +- 13 files changed, 92 insertions(+), 59 deletions(-) diff --git a/samtranslator/feature_toggle/feature_toggle.py b/samtranslator/feature_toggle/feature_toggle.py index 8973dd3a1..ecedf32e0 100644 --- a/samtranslator/feature_toggle/feature_toggle.py +++ b/samtranslator/feature_toggle/feature_toggle.py @@ -1,7 +1,7 @@ import json import logging from abc import ABC, abstractmethod -from typing import Any, Dict, cast +from typing import Any, Dict, Optional, cast import boto3 from botocore.config import Config @@ -28,7 +28,13 @@ class FeatureToggle: "account-percentile": SimpleAccountPercentileDialup, } - def __init__(self, config_provider, stage, account_id, region): # type: ignore[no-untyped-def] + def __init__( + self, + config_provider: "FeatureToggleConfigProvider", + stage: Optional[str], + account_id: Optional[str], + region: Optional[str], + ) -> None: self.feature_config = config_provider.config self.stage = stage self.account_id = account_id diff --git a/samtranslator/metrics/method_decorator.py b/samtranslator/metrics/method_decorator.py index 802e7169f..0bba003df 100644 --- a/samtranslator/metrics/method_decorator.py +++ b/samtranslator/metrics/method_decorator.py @@ -23,7 +23,7 @@ class MetricsMethodWrapperSingleton: This singleton will be alive until lambda receives shutdown event """ - _DUMMY_INSTANCE = Metrics("ServerlessTransform", DummyMetricsPublisher()) # type: ignore[no-untyped-call, no-untyped-call] + _DUMMY_INSTANCE = Metrics("ServerlessTransform", DummyMetricsPublisher()) _METRICS_INSTANCE = _DUMMY_INSTANCE @staticmethod diff --git a/samtranslator/metrics/metrics.py b/samtranslator/metrics/metrics.py index c228f8f26..582dea5e7 100644 --- a/samtranslator/metrics/metrics.py +++ b/samtranslator/metrics/metrics.py @@ -4,7 +4,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime -from typing import Any, Dict +from typing import Any, Dict, List, Optional LOG = logging.getLogger(__name__) @@ -13,7 +13,7 @@ class MetricsPublisher(ABC): """Interface for all MetricPublishers""" @abstractmethod - def publish(self, namespace, metrics): # type: ignore[no-untyped-def] + def publish(self, namespace: str, metrics: List["MetricDatum"]) -> None: """ Abstract method to publish all metrics to CloudWatch @@ -117,7 +117,9 @@ def get_metric_data(self) -> Dict[str, Any]: class Metrics: - def __init__(self, namespace="ServerlessTransform", metrics_publisher=None): # type: ignore[no-untyped-def] + def __init__( + self, namespace: str = "ServerlessTransform", metrics_publisher: Optional[MetricsPublisher] = None + ) -> None: """ Constructor @@ -125,7 +127,7 @@ def __init__(self, namespace="ServerlessTransform", metrics_publisher=None): # :param metrics_publisher: publisher to publish all metrics """ self.metrics_publisher = metrics_publisher if metrics_publisher else DummyMetricsPublisher() - self.metrics_cache = {} + self.metrics_cache: Dict[str, List[MetricDatum]] = {} self.namespace = namespace def __del__(self) -> None: diff --git a/samtranslator/parser/parser.py b/samtranslator/parser/parser.py index f1562f7bf..9d5f75e2e 100644 --- a/samtranslator/parser/parser.py +++ b/samtranslator/parser/parser.py @@ -1,4 +1,5 @@ import logging +from typing import Any, Dict from samtranslator.model.exceptions import ( InvalidDocumentException, @@ -6,6 +7,7 @@ InvalidTemplateException, ) from samtranslator.plugins import LifeCycleEvents +from samtranslator.plugins.sam_plugins import SamPlugins from samtranslator.public.sdk.template import SamTemplate from samtranslator.validator.validator import SamTemplateValidator from samtranslator.validator.value_validator import sam_expect @@ -17,7 +19,7 @@ class Parser: def __init__(self) -> None: pass - def parse(self, sam_template, parameter_values, sam_plugins): # type: ignore[no-untyped-def] + def parse(self, sam_template: Dict[str, Any], parameter_values: Dict[str, Any], sam_plugins: SamPlugins) -> None: self._validate(sam_template, parameter_values) # type: ignore[no-untyped-call] sam_plugins.act(LifeCycleEvents.before_transform_template, sam_template) diff --git a/samtranslator/plugins/application/serverless_app_plugin.py b/samtranslator/plugins/application/serverless_app_plugin.py index 095c19940..b6b2f08fa 100644 --- a/samtranslator/plugins/application/serverless_app_plugin.py +++ b/samtranslator/plugins/application/serverless_app_plugin.py @@ -2,9 +2,10 @@ import json import logging from time import sleep -from typing import Any, Dict, Tuple +from typing import Any, Dict, List, Optional, Tuple import boto3 +from botocore.client import BaseClient from botocore.config import Config from botocore.exceptions import ClientError, EndpointConnectionError @@ -50,7 +51,13 @@ class ServerlessAppPlugin(BasePlugin): LOCATION_KEY = "Location" TEMPLATE_URL_KEY = "TemplateUrl" - def __init__(self, sar_client=None, wait_for_template_active_status=False, validate_only=False, parameters=None): # type: ignore[no-untyped-def] + def __init__( + self, + sar_client: Optional[BaseClient] = None, + wait_for_template_active_status: bool = False, + validate_only: bool = False, + parameters: Optional[Dict[str, Any]] = None, + ): """ Initialize the plugin. @@ -62,9 +69,9 @@ def __init__(self, sar_client=None, wait_for_template_active_status=False, valid super().__init__() if parameters is None: parameters = {} - self._applications = {} - self._in_progress_templates = [] - self._sar_client = sar_client + self._applications: Dict[Tuple[str, str], Any] = {} + self._in_progress_templates: List[Tuple[str, str]] = [] + self.__sar_client = sar_client self._wait_for_template_active_status = wait_for_template_active_status self._validate_only = validate_only self._parameters = parameters @@ -75,6 +82,15 @@ def __init__(self, sar_client=None, wait_for_template_active_status=False, valid message = "Cannot set both validate_only and wait_for_template_active_status flags to True." raise InvalidPluginException(ServerlessAppPlugin.__name__, message) + @property + def _sar_client(self) -> BaseClient: + # Lazy initialization of the client- create it when it is needed + if not self.__sar_client: + # a SAR call could take a while to finish, leaving the read_timeout default (60s). + client_config = Config(connect_timeout=BOTO3_CONNECT_TIMEOUT) + self.__sar_client = boto3.client("serverlessrepo", config=client_config) + return self.__sar_client + @staticmethod def _make_app_key(app_id: Any, semver: Any) -> Tuple[str, str]: """Generate a key that is always hashable.""" @@ -127,11 +143,6 @@ def on_before_transform_template(self, template_dict): # type: ignore[no-untype raise InvalidResourceException( logical_id, "Serverless Application Repository is not available in this region." ) - # Lazy initialization of the client- create it when it is needed - if not self._sar_client: - # a SAR call could take a while to finish, leaving the read_timeout default (60s). - client_config = Config(connect_timeout=BOTO3_CONNECT_TIMEOUT) - self._sar_client = boto3.client("serverlessrepo", config=client_config) self._make_service_call_with_retry(service_call, app_id, semver, key, logical_id) # type: ignore[no-untyped-call] except InvalidResourceException as e: # Catch all InvalidResourceExceptions, raise those in the before_resource_transform target. @@ -421,17 +432,17 @@ def _resource_is_supported(self, resource_type): # type: ignore[no-untyped-def] return resource_type == self.SUPPORTED_RESOURCE_TYPE def _get_application(self, app_id, semver): # type: ignore[no-untyped-def] - return self._sar_client.get_application( + return self._sar_client.get_application( # type: ignore[attr-defined] ApplicationId=self._sanitize_sar_str_param(app_id), SemanticVersion=self._sanitize_sar_str_param(semver) # type: ignore[no-untyped-call] ) def _create_cfn_template(self, app_id, semver): # type: ignore[no-untyped-def] - return self._sar_client.create_cloud_formation_template( + return self._sar_client.create_cloud_formation_template( # type: ignore[attr-defined] ApplicationId=self._sanitize_sar_str_param(app_id), SemanticVersion=self._sanitize_sar_str_param(semver) # type: ignore[no-untyped-call] ) def _get_cfn_template(self, app_id, template_id): # type: ignore[no-untyped-def] - return self._sar_client.get_cloud_formation_template( + return self._sar_client.get_cloud_formation_template( # type: ignore[attr-defined] ApplicationId=self._sanitize_sar_str_param(app_id), # type: ignore[no-untyped-call] TemplateId=self._sanitize_sar_str_param(template_id), # type: ignore[no-untyped-call] ) diff --git a/samtranslator/plugins/policies/policy_templates_plugin.py b/samtranslator/plugins/policies/policy_templates_plugin.py index 5bf2eeb46..e614644a5 100644 --- a/samtranslator/plugins/policies/policy_templates_plugin.py +++ b/samtranslator/plugins/policies/policy_templates_plugin.py @@ -4,6 +4,7 @@ from samtranslator.model.resource_policies import PolicyTypes, ResourcePolicies from samtranslator.plugins import BasePlugin from samtranslator.policy_template_processor.exceptions import InsufficientParameterValues, InvalidParameterValues +from samtranslator.policy_template_processor.processor import PolicyTemplatesProcessor class PolicyTemplatesForResourcePlugin(BasePlugin): @@ -17,7 +18,7 @@ class PolicyTemplatesForResourcePlugin(BasePlugin): _plugin_name = "" SUPPORTED_RESOURCE_TYPE = {"AWS::Serverless::Function", "AWS::Serverless::StateMachine"} - def __init__(self, policy_template_processor): # type: ignore[no-untyped-def] + def __init__(self, policy_template_processor: PolicyTemplatesProcessor) -> None: """ Initialize the plugin. diff --git a/samtranslator/region_configuration.py b/samtranslator/region_configuration.py index f8ba03e4b..d6d75fc4a 100644 --- a/samtranslator/region_configuration.py +++ b/samtranslator/region_configuration.py @@ -47,7 +47,7 @@ def is_service_supported(cls, service, region=None): # type: ignore[no-untyped- # need to handle when region is None so that it won't break if region is None: if ArnGenerator.BOTO_SESSION_REGION_NAME is not None: - region = ArnGenerator.BOTO_SESSION_REGION_NAME # type: ignore[unreachable] + region = ArnGenerator.BOTO_SESSION_REGION_NAME else: raise NoRegionFound("AWS Region cannot be found") diff --git a/samtranslator/sdk/parameter.py b/samtranslator/sdk/parameter.py index 2458bcfcf..50f39108b 100644 --- a/samtranslator/sdk/parameter.py +++ b/samtranslator/sdk/parameter.py @@ -1,7 +1,8 @@ import copy -from typing import Any, Dict +from typing import Any, Dict, Optional import boto3 +from boto3 import Session from samtranslator.translator.arn_generator import ArnGenerator, NoRegionFound @@ -64,7 +65,7 @@ def add_default_parameter_values(self, sam_template: Dict[str, Any]) -> Any: return None - def add_pseudo_parameter_values(self, session=None): # type: ignore[no-untyped-def] + def add_pseudo_parameter_values(self, session: Optional[Session] = None) -> None: """ Add pseudo parameter values :return: parameter values that have pseudo parameter in it diff --git a/samtranslator/translator/arn_generator.py b/samtranslator/translator/arn_generator.py index b60f770ca..445b26762 100644 --- a/samtranslator/translator/arn_generator.py +++ b/samtranslator/translator/arn_generator.py @@ -30,7 +30,7 @@ def _region_to_partition(region: str) -> str: class ArnGenerator: - BOTO_SESSION_REGION_NAME = None + BOTO_SESSION_REGION_NAME: Optional[str] = None @classmethod def generate_arn( diff --git a/samtranslator/translator/managed_policy_translator.py b/samtranslator/translator/managed_policy_translator.py index 76b31e47a..c93ab122d 100644 --- a/samtranslator/translator/managed_policy_translator.py +++ b/samtranslator/translator/managed_policy_translator.py @@ -1,5 +1,7 @@ import logging -from typing import Dict, cast +from typing import Dict, Optional, cast + +from botocore.client import BaseClient from samtranslator.metrics.method_decorator import cw_timer @@ -7,13 +9,13 @@ class ManagedPolicyLoader: - def __init__(self, iam_client): # type: ignore[no-untyped-def] + def __init__(self, iam_client: BaseClient) -> None: self._iam_client = iam_client - self._policy_map = None + self._policy_map: Optional[Dict[str, str]] = None self.max_items = 1000 @cw_timer(prefix="External", name="IAM") - def _load_policies_from_iam(self): # type: ignore[no-untyped-def] + def _load_policies_from_iam(self) -> None: LOG.info("Loading policies from IAM...") paginator = self._iam_client.get_paginator("list_policies") @@ -23,7 +25,7 @@ def _load_policies_from_iam(self): # type: ignore[no-untyped-def] # Note(jfuss): boto3 PaginationConfig MaxItems does not control the number of items returned from the API # call. This is actually controlled by PageSize. page_iterator = paginator.paginate(Scope="AWS", PaginationConfig={"PageSize": self.max_items}) - name_to_arn_map = {} # type: ignore[var-annotated] + name_to_arn_map: Dict[str, str] = {} for page in page_iterator: name_to_arn_map.update(map(lambda x: (x["PolicyName"], x["Arn"]), page["Policies"])) diff --git a/samtranslator/translator/transform.py b/samtranslator/translator/transform.py index 2647a46be..34ec50709 100644 --- a/samtranslator/translator/transform.py +++ b/samtranslator/translator/transform.py @@ -1,13 +1,20 @@ from functools import lru_cache -from typing import Dict +from typing import Any, Dict, Optional +from samtranslator.feature_toggle.feature_toggle import FeatureToggle from samtranslator.parser.parser import Parser from samtranslator.translator.managed_policy_translator import ManagedPolicyLoader from samtranslator.translator.translator import Translator from samtranslator.utils.py27hash_fix import to_py27_compatible_template, undo_mark_unicode_str_in_template -def transform(input_fragment, parameter_values, managed_policy_loader: ManagedPolicyLoader, feature_toggle=None, passthrough_metadata=False): # type: ignore[no-untyped-def] +def transform( + input_fragment: Dict[str, Any], + parameter_values: Dict[str, Any], + managed_policy_loader: ManagedPolicyLoader, + feature_toggle: Optional[FeatureToggle] = None, + passthrough_metadata: Optional[bool] = False, +) -> Dict[str, Any]: """Translates the SAM manifest provided in the and returns the translation to CloudFormation. :param dict input_fragment: the SAM template to transform @@ -18,7 +25,7 @@ def transform(input_fragment, parameter_values, managed_policy_loader: ManagedPo sam_parser = Parser() to_py27_compatible_template(input_fragment, parameter_values) - translator = Translator( # type: ignore[no-untyped-call] + translator = Translator( None, sam_parser, ) @@ -34,4 +41,4 @@ def get_managed_policy_map() -> Dict[str, str]: passthrough_metadata=passthrough_metadata, get_managed_policy_map=get_managed_policy_map, ) - return undo_mark_unicode_str_in_template(transformed) # type: ignore[no-untyped-call] + return undo_mark_unicode_str_in_template(transformed) diff --git a/samtranslator/translator/translator.py b/samtranslator/translator/translator.py index e7b034027..7c7cca06b 100644 --- a/samtranslator/translator/translator.py +++ b/samtranslator/translator/translator.py @@ -1,6 +1,8 @@ import copy from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from boto3 import Session + from samtranslator.feature_toggle.feature_toggle import ( FeatureToggle, FeatureToggleDefaultConfigProvider, @@ -16,6 +18,7 @@ from samtranslator.model.eventsources.push import Api from samtranslator.model.exceptions import ( DuplicateLogicalIdException, + ExceptionWithMessage, InvalidDocumentException, InvalidEventException, InvalidResourceException, @@ -23,7 +26,8 @@ ) from samtranslator.model.preferences.deployment_preference_collection import DeploymentPreferenceCollection from samtranslator.model.sam_resources import SamConnector -from samtranslator.plugins import LifeCycleEvents +from samtranslator.parser.parser import Parser +from samtranslator.plugins import BasePlugin, LifeCycleEvents from samtranslator.plugins.api.default_definition_body_plugin import DefaultDefinitionBodyPlugin from samtranslator.plugins.application.serverless_app_plugin import ServerlessAppPlugin from samtranslator.plugins.globals.globals_plugin import GlobalsPlugin @@ -39,14 +43,14 @@ class Translator: """Translates SAM templates into CloudFormation templates""" - def __init__( # type: ignore[no-untyped-def] + def __init__( self, - managed_policy_map, - sam_parser, - plugins=None, - boto_session=None, - metrics=None, - ): + managed_policy_map: Optional[Dict[str, str]], + sam_parser: Parser, + plugins: Optional[List[BasePlugin]] = None, + boto_session: Optional[Session] = None, + metrics: Optional[Metrics] = None, + ) -> None: """ :param dict managed_policy_map: Map of managed policy names to the ARNs :param sam_parser: Instance of a SAM Parser @@ -56,12 +60,11 @@ def __init__( # type: ignore[no-untyped-def] self.managed_policy_map = managed_policy_map self.plugins = plugins self.sam_parser = sam_parser - self.feature_toggle = None + self.feature_toggle: Optional[FeatureToggle] = None self.boto_session = boto_session - self.metrics = metrics if metrics else Metrics("ServerlessTransform", DummyMetricsPublisher()) # type: ignore[no-untyped-call, no-untyped-call] + self.metrics = metrics if metrics else Metrics("ServerlessTransform", DummyMetricsPublisher()) MetricsMethodWrapperSingleton.set_instance(self.metrics) - self._translated_resouce_mapping = {} - self.document_errors = [] + self.document_errors: List[ExceptionWithMessage] = [] if self.boto_session: ArnGenerator.BOTO_SESSION_REGION_NAME = self.boto_session.region_name @@ -100,7 +103,7 @@ def _get_function_names( def translate( # noqa: too-many-branches self, sam_template: Dict[str, Any], - parameter_values: Dict[Any, Any], + parameter_values: Dict[str, Any], feature_toggle: Optional[FeatureToggle] = None, passthrough_metadata: Optional[bool] = False, get_managed_policy_map: Optional[GetManagedPolicyMap] = None, @@ -119,16 +122,14 @@ def translate( # noqa: too-many-branches :returns: a copy of the template with SAM resources replaced with the corresponding CloudFormation, which may \ be dumped into a valid CloudFormation JSON or YAML template """ - self.feature_toggle = ( - feature_toggle - if feature_toggle - else FeatureToggle(FeatureToggleDefaultConfigProvider(), stage=None, account_id=None, region=None) # type: ignore[no-untyped-call, no-untyped-call] + self.feature_toggle = feature_toggle or FeatureToggle( + FeatureToggleDefaultConfigProvider(), stage=None, account_id=None, region=None ) self.function_names: Dict[Any, Any] = {} self.redeploy_restapi_parameters = {} sam_parameter_values = SamParameterValues(parameter_values) sam_parameter_values.add_default_parameter_values(sam_template) - sam_parameter_values.add_pseudo_parameter_values(self.boto_session) # type: ignore[no-untyped-call] + sam_parameter_values.add_pseudo_parameter_values(self.boto_session) parameter_values = sam_parameter_values.parameter_values # Create & Install plugins sam_plugins = prepare_plugins(self.plugins, parameter_values) @@ -389,7 +390,7 @@ def _get_generated_connector( return SamConnector.from_dict(connector_logical_id, connector) -def prepare_plugins(plugins: List[Any], parameters: Optional[Dict[str, Any]] = None) -> SamPlugins: +def prepare_plugins(plugins: Optional[List[BasePlugin]], parameters: Optional[Dict[str, Any]] = None) -> SamPlugins: """ Creates & returns a plugins object with the given list of plugins installed. In addition to the given plugins, we will also install a few "required" plugins that are necessary to provide complete support for SAM template spec. @@ -409,11 +410,11 @@ def prepare_plugins(plugins: List[Any], parameters: Optional[Dict[str, Any]] = N make_policy_template_for_function_plugin(), ] - plugins = plugins if plugins else [] + plugins = plugins or [] # If a ServerlessAppPlugin does not yet exist, create one and add to the beginning of the required plugins list. if not any(isinstance(plugin, ServerlessAppPlugin) for plugin in plugins): - required_plugins.insert(0, ServerlessAppPlugin(parameters=parameters)) # type: ignore[no-untyped-call] + required_plugins.insert(0, ServerlessAppPlugin(parameters=parameters)) # Execute customer's plugins first before running SAM plugins. It is very important to retain this order because # other plugins will be dependent on this ordering. @@ -448,4 +449,4 @@ def make_policy_template_for_function_plugin() -> PolicyTemplatesForResourcePlug policy_templates = PolicyTemplatesProcessor.get_default_policy_templates_json() processor = PolicyTemplatesProcessor(policy_templates) - return PolicyTemplatesForResourcePlugin(processor) # type: ignore[no-untyped-call] + return PolicyTemplatesForResourcePlugin(processor) diff --git a/samtranslator/utils/py27hash_fix.py b/samtranslator/utils/py27hash_fix.py index ef58bfc6b..6b6e1e171 100644 --- a/samtranslator/utils/py27hash_fix.py +++ b/samtranslator/utils/py27hash_fix.py @@ -96,8 +96,8 @@ def to_py27_compatible_template( # noqa: too-many-branches parameter_values[key] = _convert_to_py27_type(val) # type: ignore[no-untyped-call] -def undo_mark_unicode_str_in_template(template_dict): # type: ignore[no-untyped-def] - return json.loads(json.dumps(template_dict)) +def undo_mark_unicode_str_in_template(template_dict: Dict[str, Any]) -> Dict[str, Any]: + return cast(Dict[str, Any], json.loads(json.dumps(template_dict))) class Py27UniStr(unicode_string_type): From bea78b5a1d24d13e2439a7cfe3682d49775e4a8c Mon Sep 17 00:00:00 2001 From: Sam Liu Date: Fri, 24 Feb 2023 15:15:50 -0800 Subject: [PATCH 2/2] Remove one more type: ignore --- bin/sam-translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/sam-translate.py b/bin/sam-translate.py index 9ce8e6c60..16f04338c 100755 --- a/bin/sam-translate.py +++ b/bin/sam-translate.py @@ -106,7 +106,7 @@ def transform_template(input_file_path, output_file_path): # type: ignore[no-un sam_template = yaml_parse(f) # type: ignore[no-untyped-call] try: - cloud_formation_template = transform(sam_template, {}, ManagedPolicyLoader(iam_client)) # type: ignore[no-untyped-call] + cloud_formation_template = transform(sam_template, {}, ManagedPolicyLoader(iam_client)) cloud_formation_template_prettified = json.dumps(cloud_formation_template, indent=1) with open(output_file_path, "w") as f: