Make lookup-logic more generic (cloudtools#665)

* Rewrote Lookup-parser

The new parser will build the entire AST to support nested lookups.

* Move dependency injection of ${output} to the lookup itself

* Addressed comments

* Removed dead code

* Fix lint warnings

* Fix lint errors after master merge

* Fix lint error (unused exception)

* Add warning when using old style lookups

* Convert lookups to new style

* Reformat code to fix linting errors

stacker/config/__init__.py

@@ -234,6 +234,7 @@ def not_empty_list(value):
 class AnyType(BaseType):
     pass
 
+
 class LocalPackageSource(Model):
     source = StringType(required=True)
 

stacker/config/translators/kms.py

@@ -2,9 +2,9 @@
 from __future__ import division
 from __future__ import absolute_import
 # NOTE: The translator is going to be deprecated in favor of the lookup
-from ...lookups.handlers.kms import handler
+from ...lookups.handlers.kms import KmsLookup
 
 
 def kms_simple_constructor(loader, node):
     value = loader.construct_scalar(node)
-    return handler(value)
+    return KmsLookup.handler(value)

stacker/exceptions.py

@@ -15,17 +15,27 @@ def __init__(self, lookup, lookups, value, *args, **kwargs):
         message = (
             "Lookup: \"{}\" has non-string return value, must be only lookup "
             "present (not {}) in \"{}\""
-        ).format(lookup.raw, len(lookups), value)
+        ).format(str(lookup), len(lookups), value)
         super(InvalidLookupCombination, self).__init__(message,
                                                        *args,
                                                        **kwargs)
 
 
+class InvalidLookupConcatenation(Exception):
+    """
+    Intermediary Exception to be converted to InvalidLookupCombination once it
+    bubbles up there
+    """
+    def __init__(self, lookup, lookups, *args, **kwargs):
+        self.lookup = lookup
+        self.lookups = lookups
+        super(InvalidLookupConcatenation, self).__init__("", *args, **kwargs)
+
+
 class UnknownLookupType(Exception):
 
-    def __init__(self, lookup, *args, **kwargs):
-        self.lookup = lookup
-        message = "Unknown lookup type: \"{}\"".format(lookup.type)
+    def __init__(self, lookup_type, *args, **kwargs):
+        message = "Unknown lookup type: \"{}\"".format(lookup_type)
         super(UnknownLookupType, self).__init__(message, *args, **kwargs)
 
 
@@ -35,11 +45,22 @@ def __init__(self, variable_name, lookup, error, *args, **kwargs):
         self.lookup = lookup
         self.error = error
         message = "Couldn't resolve lookup in variable `%s`, " % variable_name
-        message += "lookup: ${%s}: " % lookup.raw
+        message += "lookup: ${%s}: " % repr(lookup)
         message += "(%s) %s" % (error.__class__, error)
         super(FailedVariableLookup, self).__init__(message, *args, **kwargs)
 
 
+class FailedLookup(Exception):
+    """
+    Intermediary Exception to be converted to FailedVariableLookup once it
+    bubbles up there
+    """
+    def __init__(self, lookup, error, *args, **kwargs):
+        self.lookup = lookup
+        self.error = error
+        super(FailedLookup, self).__init__("Failed lookup", *args, **kwargs)
+
+
 class InvalidUserdataPlaceholder(Exception):
 
     def __init__(self, blueprint_name, exception_message, *args, **kwargs):
@@ -70,6 +91,17 @@ def __init__(self, blueprint_name, variable, *args, **kwargs):
         super(UnresolvedVariable, self).__init__(message, *args, **kwargs)
 
 
+class UnresolvedVariableValue(Exception):
+    """
+    Intermediary Exception to be converted to UnresolvedVariable once it
+    bubbles up there
+    """
+    def __init__(self, lookup, *args, **kwargs):
+        self.lookup = lookup
+        super(UnresolvedVariableValue, self).__init__(
+            "Unresolved lookup", *args, **kwargs)
+
+
 class MissingVariable(Exception):
 
     def __init__(self, blueprint_name, variable_name, *args, **kwargs):

stacker/lookups/handlers/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import division
+
+
+class LookupHandler(object):
+    @classmethod
+    def handle(cls, value, context, provider):
+        """
+        Perform the actual lookup
+
+        :param value: Parameter(s) given to this lookup
+        :type value: str
+        :param context:
+        :param provider:
+        :return: Looked-up value
+        :rtype: str
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    def dependencies(cls, lookup_data):
+        """
+        Calculate any dependencies required to perform this lookup.
+
+        Note that lookup_data may not be (completely) resolved at this time.
+
+        :param lookup_data: Parameter(s) given to this lookup
+        :type lookup_data VariableValue
+        :return: Set of stack names (str) this lookup depends on
+        :rtype: set
+        """
+        del lookup_data  # unused in this implementation
+        return set()

stacker/lookups/handlers/ami.py

@@ -5,6 +5,7 @@
 import re
 import operator
 
+from . import LookupHandler
 from ...util import read_value_from_path
 
 TYPE_NAME = "ami"
@@ -19,76 +20,80 @@ def __init__(self, search_string):
         super(ImageNotFound, self).__init__(message)
 
 
-def handler(value, provider, **kwargs):
-    """Fetch the most recent AMI Id using a filter
-
-    For example:
-
-        ${ami [<region>@]owners:self,account,amazon name_regex:serverX-[0-9]+ architecture:x64,i386}
-
-        The above fetches the most recent AMI where owner is self
-        account or amazon and the ami name matches the regex described,
-        the architecture will be either x64 or i386
-
-        You can also optionally specify the region in which to perform the AMI lookup.
-
-        Valid arguments:
-
-        owners (comma delimited) REQUIRED ONCE:
-            aws_account_id | amazon | self
-
-        name_regex (a regex) REQUIRED ONCE:
-            e.g. my-ubuntu-server-[0-9]+
-
-        executable_users (comma delimited) OPTIONAL ONCE:
-            aws_account_id | amazon | self
-
-        Any other arguments specified are sent as filters to the aws api
-        For example, "architecture:x86_64" will add a filter
-    """  # noqa
-    value = read_value_from_path(value)
-
-    if "@" in value:
-        region, value = value.split("@", 1)
-    else:
-        region = provider.region
-
-    ec2 = get_session(region).client('ec2')
-
-    values = {}
-    describe_args = {}
-
-    # now find any other arguments that can be filters
-    matches = re.findall('([0-9a-zA-z_-]+:[^\s$]+)', value)
-    for match in matches:
-        k, v = match.split(':', 1)
-        values[k] = v
-
-    if not values.get('owners'):
-        raise Exception("'owners' value required when using ami")
-    owners = values.pop('owners').split(',')
-    describe_args["Owners"] = owners
-
-    if not values.get('name_regex'):
-        raise Exception("'name_regex' value required when using ami")
-    name_regex = values.pop('name_regex')
-
-    executable_users = None
-    if values.get('executable_users'):
-        executable_users = values.pop('executable_users').split(',')
-        describe_args["ExecutableUsers"] = executable_users
-
-    filters = []
-    for k, v in values.items():
-        filters.append({"Name": k, "Values": v.split(',')})
-    describe_args["Filters"] = filters
-
-    result = ec2.describe_images(**describe_args)
-
-    images = sorted(result['Images'], key=operator.itemgetter('CreationDate'),
-                    reverse=True)
-    for image in images:
-        if re.match("^%s$" % name_regex, image['Name']):
-            return image['ImageId']
-
-    raise ImageNotFound(value)
+class AmiLookup(LookupHandler):
+    @classmethod
+    def handle(cls, value, provider, **kwargs):
+        """Fetch the most recent AMI Id using a filter
+    
+        For example:
+    
+            ${ami [<region>@]owners:self,account,amazon name_regex:serverX-[0-9]+ architecture:x64,i386}
+    
+            The above fetches the most recent AMI where owner is self
+            account or amazon and the ami name matches the regex described,
+            the architecture will be either x64 or i386
+    
+            You can also optionally specify the region in which to perform the
+            AMI lookup.
+    
+            Valid arguments:
+    
+            owners (comma delimited) REQUIRED ONCE:
+                aws_account_id | amazon | self
+    
+            name_regex (a regex) REQUIRED ONCE:
+                e.g. my-ubuntu-server-[0-9]+
+    
+            executable_users (comma delimited) OPTIONAL ONCE:
+                aws_account_id | amazon | self
+    
+            Any other arguments specified are sent as filters to the aws api
+            For example, "architecture:x86_64" will add a filter
+        """  # noqa
+        value = read_value_from_path(value)
+
+        if "@" in value:
+            region, value = value.split("@", 1)
+        else:
+            region = provider.region
+
+        ec2 = get_session(region).client('ec2')
+
+        values = {}
+        describe_args = {}
+
+        # now find any other arguments that can be filters
+        matches = re.findall('([0-9a-zA-z_-]+:[^\s$]+)', value)
+        for match in matches:
+            k, v = match.split(':', 1)
+            values[k] = v
+
+        if not values.get('owners'):
+            raise Exception("'owners' value required when using ami")
+        owners = values.pop('owners').split(',')
+        describe_args["Owners"] = owners
+
+        if not values.get('name_regex'):
+            raise Exception("'name_regex' value required when using ami")
+        name_regex = values.pop('name_regex')
+
+        executable_users = None
+        if values.get('executable_users'):
+            executable_users = values.pop('executable_users').split(',')
+            describe_args["ExecutableUsers"] = executable_users
+
+        filters = []
+        for k, v in values.items():
+            filters.append({"Name": k, "Values": v.split(',')})
+        describe_args["Filters"] = filters
+
+        result = ec2.describe_images(**describe_args)
+
+        images = sorted(result['Images'],
+                        key=operator.itemgetter('CreationDate'),
+                        reverse=True)
+        for image in images:
+            if re.match("^%s$" % name_regex, image['Name']):
+                return image['ImageId']
+
+        raise ImageNotFound(value)

stacker/lookups/handlers/default.py

@@ -1,35 +1,41 @@
 from __future__ import print_function
 from __future__ import division
 from __future__ import absolute_import
+
+from . import LookupHandler
+
+
 TYPE_NAME = "default"
 
 
-def handler(value, **kwargs):
-    """Use a value from the environment or fall back to a default if the
-       environment doesn't contain the variable.
+class DefaultLookup(LookupHandler):
+    @classmethod
+    def handle(cls, value, **kwargs):
+        """Use a value from the environment or fall back to a default if the
+           environment doesn't contain the variable.
 
-    Format of value:
+        Format of value:
 
-        <env_var>::<default value>
+            <env_var>::<default value>
 
-    For example:
+        For example:
 
-        Groups: ${default app_security_groups::sg-12345,sg-67890}
+            Groups: ${default app_security_groups::sg-12345,sg-67890}
 
-    If `app_security_groups` is defined in the environment, its defined value
-    will be returned. Otherwise, `sg-12345,sg-67890` will be the returned
-    value.
+        If `app_security_groups` is defined in the environment, its defined
+        value will be returned. Otherwise, `sg-12345,sg-67890` will be the
+        returned value.
 
-    This allows defaults to be set at the config file level.
-    """
+        This allows defaults to be set at the config file level.
+        """
 
-    try:
-        env_var_name, default_val = value.split("::", 1)
-    except ValueError:
-        raise ValueError("Invalid value for default: %s. Must be in "
-                         "<env_var>::<default value> format." % value)
+        try:
+            env_var_name, default_val = value.split("::", 1)
+        except ValueError:
+            raise ValueError("Invalid value for default: %s. Must be in "
+                             "<env_var>::<default value> format." % value)
 
-    if env_var_name in kwargs['context'].environment:
-        return kwargs['context'].environment[env_var_name]
-    else:
-        return default_val
+        if env_var_name in kwargs['context'].environment:
+            return kwargs['context'].environment[env_var_name]
+        else:
+            return default_val