feat(schemas): now we pre-generate nested schemas

Into a complete, global list of schemas, with additional meta-data.

However, it's currently not complete, as $refs are missing.
There is some resemblance to to_rust_type(...), which worries me
slightly

src/mako/lib.rs.mako

@@ -4,15 +4,12 @@
 <%namespace name="mbuild" file="lib/mbuild.mako"/>\
 <%namespace name="schema" file="lib/schema.mako"/>\
 <%  
-    from util import (iter_nested_types, new_context, rust_comment, rust_doc_comment,
+    from util import (new_context, rust_comment, rust_doc_comment,
                       rust_module_doc_comment, rb_type, hub_type, mangle_ident, hub_type_params_s,
                       hub_type_bounds, rb_type_params_s)
 
-    nested_schemas = list()
-    if schemas:
-        nested_schemas = list(iter_nested_types(schemas))
     c = new_context(schemas, resources)
-    hub_type = hub_type(schemas, util.canonical_name())
+    hub_type = hub_type(c.schemas, util.canonical_name())
     ht_params = hub_type_params_s()
 %>\
 <%block filter="rust_comment">\
@@ -112,22 +109,11 @@ impl<'a, C, NC, A> ${hub_type}${ht_params}
 }
 
 
-% if schemas:
+% if c.schemas:
 // ############
 // SCHEMAS ###
 // ##########
-% for s in schemas.values():
-${schema.new(s, c)}
-% endfor
-% endif
-
-% if nested_schemas:
-// ###################
-// NESTED SCHEMAS ###
-// #################
-## some schemas are only used once and basically internal types.
-## We have to find them and process them as normal types
-% for s in nested_schemas:
+% for s in c.schemas.values():
 ${schema.new(s, c)}
 % endfor
 % endif

src/mako/lib/schema.mako

@@ -32,15 +32,14 @@ ${struct};
 <% 
     markers = schema_markers(s, c)
     traits = ['Default', 'Clone']
-    # Note: it would be great if we could put just the traits we need.
-    # to do that, we have to transitively determine what is used in responses or requests.
-    # Even though a type is not directly used as response, it might be used in a type that is.
-    # Therefore, we just put all traits, unless there are special cases.
-    traits.extend(('RustcEncodable', 'RustcDecodable'))
+    if REQUEST_MARKER_TRAIT in markers:
+        traits.append('RustcEncodable')
+    if RESPONSE_MARKER_TRAIT in markers:
+        traits.append('RustcDecodable')
+    
     ## waiting for Default: https://github.com/rust-lang/rustc-serialize/issues/71
     if s.type == 'any':
         traits.remove('Default')
-        traits.remove('RustcDecodable')
 %>\
 <%block filter="rust_doc_comment">\
 ${doc(s, c)}\
@@ -57,6 +56,7 @@ ${_new_object(s, s.items.get('properties'), c)}\
 % elif s.type == 'any':
 ## waiting for Default: https://github.com/rust-lang/rustc-serialize/issues/71
 pub struct ${s.id}(rustc_serialize::json::Json);
+
 impl Default for ${s.id} {
     fn default() -> ${s.id} {
         ${s.id}(rustc_serialize::json::Json::Null)

src/mako/lib/util.py

@@ -49,7 +49,6 @@
 INS_METHOD = 'insert'
 DEL_METHOD = 'delete'
 
-NESTED_TYPE_MARKER = 'is_nested'
 SPACES_PER_TAB = 4
 
 NESTED_TYPE_SUFFIX = 'item'
@@ -271,7 +270,6 @@ def _is_map_prop(p):
 def _assure_unique_type_name(schemas, tn):
     if tn in schemas:
         tn += 'Internal'
-        assert tn not in schemas
     return tn
 
 # map a json type to an rust type
@@ -330,7 +328,7 @@ def is_nested_type_property(t):
 
 # Return True if the schema is nested
 def is_nested_type(s):
-    return NESTED_TYPE_MARKER in s
+    return len(s.parents) > 0
 
 # convert a rust-type to something that would be taken as input of a function
 # even though our storage type is different
@@ -348,64 +346,31 @@ def activity_input_type(schemas, p):
 def is_pod_property(p):
     return 'format' in p or p.get('type','') == 'boolean'
 
-# return an iterator yielding fake-schemas that identify a nested type
-# NOTE: In case you don't understand how this algorithm really works ... me neither - THE AUTHOR
-def iter_nested_types(schemas):
-    # 'type' in t and t.type == 'object' and 'properties' in t or ('items' in t and 'properties' in t.items)
-    def iter_nested_properties(prefix, properties):
-        for pn, p in properties.iteritems():
-            if is_nested_type_property(p):
-                ns = deepcopy(p)
-                ns.id = nested_type_name(prefix, pn)
-                ns[NESTED_TYPE_MARKER] = True
-
-                # To allow us recursing arrays, we simply put items one level up
-                if 'items' in p:
-                    ns.update((k, deepcopy(v)) for k, v in p.items.iteritems())
-
-                yield ns
-                if 'properties' in ns:
-                    for np in iter_nested_properties(prefix + canonical_type_name(pn), ns.properties):
-                        yield np
-            elif _is_map_prop(p):
-                # it's a hash, check its type
-                # TODO: does this code run ? Why is there a plain prefix
-                for np in iter_nested_properties(prefix, {pn: p.additionalProperties}):
-                    yield np
-            elif 'items' in p:
-                for np in iter_nested_properties(prefix, {pn: p.items}):
-                    yield np
-            # end handle prop itself
-        # end for ach property
-    for s in schemas.values():
-        if 'properties' not in s:
-            continue
-        for np in iter_nested_properties(s.id, s.properties):
-            np.id = _assure_unique_type_name(schemas, np.id)
-            yield np
-    # end for aech schma
-
 # Return sorted type names of all markers applicable to the given schema
+# This list is transitive. Thus, if the schema is used as child of someone with a trait, it 
+# inherits this trait
 def schema_markers(s, c):
     res = set()
-
-    activities = c.sta_map.get(s.id, dict())
-    if len(activities) == 0:
-        res.add(PART_MARKER_TRAIT)
-    else:
-        # it should have at least one activity that matches it's type to qualify for the Resource trait
-        for fqan, iot in activities.iteritems():
-            if activity_name_to_type_name(activity_split(fqan)[1]).lower() == s.id.lower():
-                res.add('cmn::Resource')
-            if IO_RESPONSE in iot:
-                res.add(RESPONSE_MARKER_TRAIT)
-            if IO_REQUEST in iot:
-                res.add(REQUEST_MARKER_TRAIT)
-        # end for each activity
-    # end handle activites
-
-    if is_nested_type(s):
-        res.add(NESTED_MARKER_TRAIT)
+    ids = s['parents'] + [s.id]
+    for sid in ids:
+        activities = c.sta_map.get(sid, dict())
+        if len(activities) == 0:
+            res.add(PART_MARKER_TRAIT)
+        else:
+            # it should have at least one activity that matches it's type to qualify for the Resource trait
+            for fqan, iot in activities.iteritems():
+                if activity_name_to_type_name(activity_split(fqan)[1]).lower() == sid.lower():
+                    res.add('cmn::Resource')
+                if IO_RESPONSE in iot:
+                    res.add(RESPONSE_MARKER_TRAIT)
+                if IO_REQUEST in iot:
+                    res.add(REQUEST_MARKER_TRAIT)
+            # end for each activity
+        # end handle activites
+
+        if is_nested_type(s):
+            res.add(NESTED_MARKER_TRAIT)
+    # end for each parent ... transitively
 
     return sorted(res)
 
@@ -572,12 +537,10 @@ def build_all_params(schemas, c, m, n, npn):
 ## -- End Activity Utilities -- @}
 
 
-Context = collections.namedtuple('Context', ['sta_map', 'fqan_map', 'rta_map', 'rtc_map'])
+Context = collections.namedtuple('Context', ['sta_map', 'fqan_map', 'rta_map', 'rtc_map', 'schemas'])
 
 # return a newly build context from the given data
 def new_context(schemas, resources):
-    if not resources:
-        return Context(dict(), dict(), dict(), dict())
     # Returns (A, B) where
     # A: { SchemaTypeName -> { fqan -> ['request'|'response', ...]}
     # B: { fqan -> activity_method_data }
@@ -620,14 +583,59 @@ def build_activity_mappings(activities, res = None, fqan = None):
         return res, fqan
     # end utility
 
+    # A dict of {s.id -> schema} , with all schemas having the 'parents' key set with [s.id, ...] of all parents
+    # in order of traversal, [0] is first parent, [-1] is the root of them all
+    # current schemas - the dict will be altered ! Changing global state seems odd, but we own it !
+    def build_schema_map(schemas):
+        # 'type' in t and t.type == 'object' and 'properties' in t or ('items' in t and 'properties' in t.items)
+        PKEY = 'parents'
+        def recurse_properties(prefix, properties, parent_ids):
+            for pn, p in properties.iteritems():
+                if is_nested_type_property(p):
+                    ns = deepcopy(p)
+                    ns.id = _assure_unique_type_name(schemas, nested_type_name(prefix, pn))
+                    schemas[ns.id] = ns
+                    ns[PKEY] = parent_ids
+
+                    # To allow us recursing arrays, we simply put items one level up
+                    if 'items' in p:
+                        ns.update((k, deepcopy(v)) for k, v in p.items.iteritems())
+
+                    if 'properties' in ns:
+                        recurse_properties(prefix + canonical_type_name(pn), ns.properties, parent_ids + [ns.id])
+                elif _is_map_prop(p):
+                    # it's a hash, check its type
+                    # TODO: does this code run ? Why is there a plain prefix
+                    recurse_properties(prefix, {pn: p.additionalProperties}, parent_ids + [])
+                elif 'items' in p:
+                    # it's an array
+                    recurse_properties(prefix, {pn: p.items}, parent_ids + [])
+                # end handle prop itself
+            # end for each property
+        # end utility
+        for s in schemas.values():
+            s[PKEY] = list() # roots never have parents
+            if 'properties' not in s:
+                continue
+            recurse_properties(s.id, s.properties, [s.id])
+        # end for each schema
+        return schemas
+    # end utility
+
+    if schemas:
+        schemas = build_schema_map(schemas)
+    else:
+        schemas = dict()
+    if not resources:
+        return Context(dict(), dict(), dict(), dict(), schemas)
     sta_map, fqan_map = build_activity_mappings(resources)
     rta_map = dict()
     rtc_map = dict()
     for an in fqan_map:
         category, resource, activity = activity_split(an)
         rta_map.setdefault(resource, list()).append(activity)
         assert rtc_map.setdefault(resource, category) == category
-    return Context(sta_map, fqan_map, rta_map, rtc_map)
+    return Context(sta_map, fqan_map, rta_map, rtc_map, schemas)
 
 # Expects v to be 'v\d+', throws otherwise
 def to_api_version(v):