Auto-generate workflows in genpkg from dir (SYN-3106) (#2531)

synapse/tests/files/stormpkg/testpkg.yaml

@@ -52,13 +52,12 @@ optic:
 
     workflows:
 
-      310eb7324b5da268fb31e4cd3d74e673:
-        name: foo
+      # testpkg-foo will get inserted from the workflows/testpkg-foo.yaml file
 
-      41e3368bd094e1c1563a242bfa56bd01:
+      testpkg-bar:
         name: bar
         desc: this is an inline workflow
 
-      bfb53cbaa789f2960de003d72b6e4544:
+      testpkg-baz:
         name: baz
-        desc: this baz desc gets overwritten
+        desc: this workflow gets overwritten by the workflows/testpkg-baz.yaml file

synapse/tests/files/stormpkg/workflows/testpkg-bam.newp

@@ -0,0 +1,2 @@
+name: newp
+desc: not included

synapse/tests/files/stormpkg/workflows/baz.yaml → synapse/tests/files/stormpkg/workflows/testpkg-baz.yaml

@@ -1 +1,2 @@
+name: real-baz
 desc: this is the real baz desc

synapse/tests/files/stormpkg/workflows/foo.yaml → synapse/tests/files/stormpkg/workflows/testpkg-foo.yaml

@@ -1 +1,2 @@
+name: foo
 desc: a foo workflow

synapse/tests/test_tools_genpkg.py

@@ -96,14 +96,16 @@ async def test_tools_genpkg(self):
             self.eq(pdef['logo']['mime'], 'image/svg')
             self.eq(pdef['logo']['file'], 'c3R1ZmYK')
 
-            wflow = pdef['optic']['workflows']['310eb7324b5da268fb31e4cd3d74e673']
+            self.len(3, pdef['optic']['workflows'])
+
+            wflow = pdef['optic']['workflows']['testpkg-foo']
             self.eq(wflow, {'name': 'foo', 'desc': 'a foo workflow'})
 
-            wflow = pdef['optic']['workflows']['41e3368bd094e1c1563a242bfa56bd01']
+            wflow = pdef['optic']['workflows']['testpkg-bar']
             self.eq(wflow, {'name': 'bar', 'desc': 'this is an inline workflow'})
 
-            wflow = pdef['optic']['workflows']['bfb53cbaa789f2960de003d72b6e4544']
-            self.eq(wflow, {'name': 'baz', 'desc': 'this is the real baz desc'})
+            wflow = pdef['optic']['workflows']['testpkg-baz']
+            self.eq(wflow, {'name': 'real-baz', 'desc': 'this is the real baz desc'})
 
             # nodocs
             nodocspath = s_common.genpath(core.dirn, 'testpkg_nodocs.json')

synapse/tools/genpkg.py

@@ -2,15 +2,22 @@
 import sys
 import base64
 import asyncio
+import logging
 import argparse
 
+import regex
+
 import synapse.exc as s_exc
 import synapse.common as s_common
 import synapse.telepath as s_telepath
 
 import synapse.lib.output as s_output
 import synapse.lib.dyndeps as s_dyndeps
 
+logger = logging.getLogger(__name__)
+
+wflownamere = regex.compile(r'^([\w-]+)\.yaml$')
+
 def chopSemVer(vers):
     return tuple([int(x) for x in vers.split('.')])
 
@@ -44,6 +51,28 @@ def loadOpticFiles(pkgdef, path):
                     'file': base64.b64encode(fd.read()).decode(),
                 }
 
+def loadOpticWorkflows(pkgdef, path):
+
+    wdefs = pkgdef['optic']['workflows']
+
+    for root, dirs, files in os.walk(path):
+
+        for name in files:
+
+            match = wflownamere.match(name)
+
+            if match is None:
+                logger.warning('Skipping workflow "%s" that does not match pattern "%s"' % (name, wflownamere.pattern))
+                continue
+
+            wname = match.groups()[0]
+
+            fullname = s_common.genpath(root, name)
+            if not os.path.isfile(fullname):  # pragma: no cover
+                continue
+
+            wdefs[wname] = s_common.yamlload(fullname)
+
 def tryLoadPkgProto(fp, opticdir=None, readonly=False):
     '''
     Try to get a Storm Package prototype from disk with or without inline documentation.
@@ -157,11 +186,11 @@ def loadPkgProto(path, opticdir=None, no_docs=False, readonly=False):
             with s_common.genfile(cmd_path) as fd:
                 cmd['storm'] = fd.read().decode()
 
-    for widen, wdef in pkgdef.get('optic', {}).get('workflows', {}).items():
-        name = wdef.get('name')
-        wyaml = s_common.yamlload(protodir, 'workflows', f'{name}.yaml')
-        if wyaml is not None:
-            wdef.update(wyaml)
+    wflowdir = s_common.genpath(protodir, 'workflows')
+    if os.path.isdir(wflowdir):
+        pkgdef.setdefault('optic', {})
+        pkgdef['optic'].setdefault('workflows', {})
+        loadOpticWorkflows(pkgdef, wflowdir)
 
     if opticdir is None:
         opticdir = s_common.genpath(protodir, 'optic')