Add support for datasource directories

When the argument given with `--datasource` point to a directory, every file in this directory is processed with the plain-file logic for datasource (e.g. name of ds == filebasename, type determined by file extension).

Related to hairyhenderson#117

README.md

@@ -217,7 +217,9 @@ A few different forms are valid:
   - Create a data source named `mydata` which is read from `file.json` (in the current working directory). This form is only valid for files in the current directory.
 - `mydata.json`
   - This form infers the name from the file name (without extension). Only valid for files in the current directory.
-
+- `mydatadir/`
+  - If a plain directory is given then any plain file within the directory is treated as a data file as described above.
+
 #### Overriding the template delimiters
 
 Sometimes it's necessary to override the default template delimiters (`{{`/`}}`).

data.go

@@ -62,13 +62,15 @@ func NewData(datasourceArgs []string, headerArgs []string) *Data {
 	sources := make(map[string]*Source)
 	headers := parseHeaderArgs(headerArgs)
 	for _, v := range datasourceArgs {
-		s, err := ParseSource(v)
-		if err != nil {
-			log.Fatalf("error parsing datasource %v", err)
-			return nil
+		for _, arg := range fanOutDatasourceArgs(v) {
+			s, err := ParseSource(arg)
+			if err != nil {
+				log.Fatalf("error parsing datasource %s: %v", arg, err)
+				return nil
+			}
+			s.Header = headers[s.Alias]
+			sources[s.Alias] = s
 		}
-		s.Header = headers[s.Alias]
-		sources[s.Alias] = s
 	}
 	return &Data{
 		Sources: sources,
@@ -119,13 +121,13 @@ func ParseSource(value string) (*Source, error) {
 	)
 	parts := strings.SplitN(value, "=", 2)
 	if len(parts) == 1 {
-		f := parts[0]
-		alias = strings.SplitN(value, ".", 2)[0]
-		if path.Base(f) != f {
-			err := fmt.Errorf("Invalid datasource (%s). Must provide an alias with files not in working directory", value)
+		srcURL = absURL(value)
+		if srcURL.Scheme == "file" {
+			alias = strings.SplitN(path.Base(value), ".", 2)[0]
+		} else {
+			err := fmt.Errorf("Invalid datasource (%s). Must provide an alias name for non-file datasources", value)
 			return nil, err
 		}
-		srcURL = absURL(f)
 	} else if len(parts) == 2 {
 		alias = parts[0]
 		var err error
@@ -144,6 +146,14 @@ func ParseSource(value string) (*Source, error) {
 }
 
 func absURL(value string) *url.URL {
+	parsedURL, err := url.Parse(value)
+	if err != nil || parsedURL.Scheme == "" || !parsedURL.IsAbs() {
+		return fileToURL(value)
+	}
+	return parsedURL
+}
+
+func fileToURL(path string) *url.URL {
 	cwd, err := os.Getwd()
 	if err != nil {
 		log.Fatalf("Can't get working directory: %s", err)
@@ -153,7 +163,7 @@ func absURL(value string) *url.URL {
 		Path:   cwd + "/",
 	}
 	relURL := &url.URL{
-		Path: value,
+		Path: path,
 	}
 	return baseURL.ResolveReference(relURL)
 }
@@ -212,6 +222,26 @@ func (d *Data) ReadSource(fs vfs.Filesystem, source *Source, args ...string) ([]
 	return nil, nil
 }
 
+// If argument is a plain directory, use dir contents as datasource files
+func fanOutDatasourceArgs(arg string) []string {
+	if _, err := statDir(arg); err == nil {
+		files, err := ioutil.ReadDir(arg)
+		if err != nil {
+			log.Fatalf("error parsing datasource directory %s %v", arg, err)
+			return nil
+		}
+
+		var paths []string
+		for _, file := range files {
+			if !file.IsDir() {
+				paths = append(paths, path.Join(arg, file.Name()))
+			}
+		}
+		return paths
+	}
+	return []string{arg}
+}
+
 func readFile(source *Source, args ...string) ([]byte, error) {
 	if source.FS == nil {
 		source.FS = vfs.OS()

data_test.go

@@ -81,7 +81,8 @@ func TestParseSourceNoAlias(t *testing.T) {
 	assert.Equal(t, "foo", s.Alias)
 
 	_, err = ParseSource("../foo.json")
-	assert.Error(t, err)
+	assert.NoError(t, err)
+	assert.Equal(t, "foo", s.Alias)
 
 	_, err = ParseSource("ftp://example.com/foo.yml")
 	assert.Error(t, err)

process_test.go

@@ -9,9 +9,34 @@ import (
 
 	"log"
 
+	"path"
+
 	"github.com/stretchr/testify/assert"
 )
 
+func TestDatasourceDir(t *testing.T) {
+	outDir, err := ioutil.TempDir("test/files/datasource-dir", "out-temp-")
+	assert.Nil(t, err)
+	defer (func() {
+		if cerr := os.RemoveAll(outDir); cerr != nil {
+			log.Fatalf("Error while removing temporary directory %s : %v", outDir, cerr)
+		}
+	})()
+
+	data := NewData([]string{"test/files/datasource-dir/ds"}, []string{})
+	gomplate := NewGomplate(data, "{{", "}}")
+	err = processInputFiles(
+		"",
+		[]string{"test/files/datasource-dir/in/test.txt"},
+		[]string{path.Join(outDir, "out.txt")},
+		gomplate)
+	assert.NoError(t, err)
+
+	out, err := ioutil.ReadFile(filepath.Join(outDir, "out.txt"))
+	assert.NoError(t, err)
+	assert.Equal(t, "eins-deux", string(out))
+}
+
 func TestReadInput(t *testing.T) {
 	actual, err := readInputs("foo", nil)
 	assert.Nil(t, err)

test/files/datasource-dir/ds/french.json

@@ -0,0 +1,4 @@
+{
+  "one": "un",
+  "two": "deux"
+}

test/files/datasource-dir/ds/german.yml

@@ -0,0 +1,2 @@
+one: eins
+two: zwei

test/files/datasource-dir/in/test.txt

@@ -0,0 +1 @@
+{{ (datasource "german").one }}-{{ (datasource "french").two }}