Parallel processing of templates implemented

Related to PR #119

process.go

@@ -8,11 +8,19 @@ import (
 
 	"io"
 
+	"errors"
+
 	"github.com/urfave/cli"
 )
 
 // == Direct input processing ========================================
 
+type renderResult struct {
+	err     error
+	inPath  string
+	outPath string
+}
+
 func processInputFiles(c *cli.Context, g *Gomplate) error {
 	inputs, err := readInputs(c.String("in"), c.StringSlice("file"))
 	if err != nil {
@@ -24,12 +32,18 @@ func processInputFiles(c *cli.Context, g *Gomplate) error {
 		outputs = []string{"-"}
 	}
 
+	results := make(chan *renderResult)
+	defer close(results)
+
 	for n, input := range inputs {
-		if err := renderTemplate(g, input, outputs[n]); err != nil {
-			return err
-		}
+		go func(idx int, input string) {
+			if err := renderTemplate(g, input, outputs[idx]); err != nil {
+				results <- &renderResult{err, input, outputs[idx]}
+			}
+		}(n, input)
 	}
-	return nil
+
+	return waitAndEvaluateResults(results, len(inputs))
 }
 
 // == Recursive input dir processing ======================================
@@ -40,28 +54,31 @@ func processInputDir(c *cli.Context, g *Gomplate) error {
 	if _, err := assertDir(outDir); err != nil {
 		return err
 	}
-	return processDir(g, inputDir, outDir)
+	results := make(chan *renderResult)
+	defer close(results)
+	nr := processDir(g, inputDir, outDir, results, 0)
+	return waitAndEvaluateResults(results, nr)
 }
 
-func processDir(g *Gomplate, inPath string, outPath string) error {
+func processDir(g *Gomplate, inPath string, outPath string, results chan *renderResult, nr int) int {
 	inPath = filepath.Clean(inPath)
 	outPath = filepath.Clean(outPath)
 
 	// assert tha input path exists
 	si, err := assertDir(inPath)
 	if err != nil {
-		return err
+		return reportError(err, inPath, outPath, results, nr)
 	}
 
 	// ensure output directory
 	if err = os.MkdirAll(outPath, si.Mode()); err != nil {
-		return err
+		return reportError(err, inPath, outPath, results, nr)
 	}
 
 	// read directory
 	entries, err := ioutil.ReadDir(inPath)
 	if err != nil {
-		return err
+		return reportError(err, inPath, outPath, results, nr)
 	}
 
 	// process or dive in again
@@ -70,20 +87,41 @@ func processDir(g *Gomplate, inPath string, outPath string) error {
 		nextOutPath := filepath.Join(outPath, entry.Name())
 
 		if entry.IsDir() {
-			err := processDir(g, nextInPath, nextOutPath)
-			if err != nil {
-				return err
-			}
+			nr += processDir(g, nextInPath, nextOutPath, results, 0)
 		} else {
-			inString, err := readInput(nextInPath)
-			if err != nil {
-				return err
-			}
-			if err := renderTemplate(g, inString, nextOutPath); err != nil {
-				return err
-			}
+			go (func(nextInPath string, nextOutPath string) {
+				inString, err := readInput(nextInPath)
+				if err == nil {
+					err = renderTemplate(g, inString, nextOutPath)
+				}
+				results <- &renderResult{err, nextInPath, nextOutPath}
+			})(nextInPath, nextOutPath)
+			nr++
 		}
 	}
+	return nr
+}
+
+func reportError(err error, inPath string, outPath string, results chan *renderResult, nr int) int {
+	go (func() {
+		results <- &renderResult{err, inPath, outPath}
+	})()
+	return nr + 1
+}
+
+// == Thread handling ================================================
+
+func waitAndEvaluateResults(results chan *renderResult, nr int) error {
+	errorMsg := ""
+	for i := 0; i < nr; i++ {
+		result := <-results
+		if result.err != nil {
+			errorMsg += fmt.Sprintf("   %s --> %s : %v", result.inPath, result.outPath, result.err)
+		}
+	}
+	if errorMsg != "" {
+		return errors.New("rendering of the following templates failed:\n" + errorMsg)
+	}
 	return nil
 }
 

process_test.go

@@ -45,7 +45,11 @@ func TestInputDir(t *testing.T) {
 		Sources: map[string]*Source{"config": src},
 	}
 	gomplate := NewGomplate(data, "{{", "}}")
-	err = processDir(gomplate, "test/files/input-dir/in", outDir)
+	results := make(chan *renderResult)
+	defer close(results)
+	nr := processDir(gomplate, "test/files/input-dir/in", outDir, results, 0)
+	log.Print(nr)
+	err = waitAndEvaluateResults(results, nr)
 	assert.Nil(t, err)
 
 	top, err := ioutil.ReadFile(filepath.Join(outDir, "top.txt"))