readmegen: A utility for generating readme from specifications (#115)

* add utility for generating readme from specifications

* allow readme to define readmegen tags

* restructure main

* take data from connector.yaml

* improve readmegen util.Generate by introducing options pattern

* fix linter errors

* fix tests

* Combine redmegen and specgen into conn-sdk-cli (#246)

* combine redmegen and specgen into conn-sdk-cli

* make tidy-all

* update year in header

* delete obsolete code

* reword comment

* add end-to-end test, add nicer output for zero values

Makefile

@@ -4,8 +4,8 @@ test:
 	echo
 	echo "Running integration tests..."
 	echo
-	cd specgen/specgen/tests/parse_specs/ && go test $(GOTEST_FLAGS) -race ./...
-	cd specgen/specgen/tests/write_and_combine/ && go test $(GOTEST_FLAGS) -race ./...
+	cd conn-sdk-cli/specgen/tests/parse_specs/ && go test $(GOTEST_FLAGS) -race ./...
+	cd conn-sdk-cli/specgen/tests/write_and_combine/ && go test $(GOTEST_FLAGS) -race ./...
 
 .PHONY: fmt
 fmt:
@@ -27,20 +27,21 @@ install-tools:
 
 .PHONY: tidy-all
 tidy-all:
+	go mod tidy
 	@echo "Tidying up module in parse_specs directory"
-	@(cd specgen/specgen/tests/parse_specs && go mod tidy)
+	@(cd conn-sdk-cli/specgen/tests/parse_specs && go mod tidy)
 	@echo "Tidying up subdirectories..."
-	@for dir in specgen/specgen/tests/parse_specs/*/; do \
+	@for dir in conn-sdk-cli/specgen/tests/parse_specs/*/; do \
 		if [ -f "$$dir/go.mod" ]; then \
 			echo "Processing directory: $$dir"; \
 			(cd "$$dir" && go mod tidy) || exit 1; \
 		fi \
 	done
 
 	@echo "Tidying up module in write_and_combine directory"
-	@(cd specgen/specgen/tests/write_and_combine && go mod tidy)
+	@(cd conn-sdk-cli/specgen/tests/write_and_combine && go mod tidy)
 	@echo "Tidying up subdirectories..."
-	@for dir in specgen/specgen/tests/write_and_combine/*/; do \
+	@for dir in conn-sdk-cli/specgen/tests/write_and_combine/*/; do \
 		if [ -f "$$dir/go.mod" ]; then \
 			echo "Processing directory: $$dir"; \
 			(cd "$$dir" && go mod tidy) || exit 1; \

conn-sdk-cli/main.go

@@ -0,0 +1,72 @@
+// Copyright © 2025 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/conduitio/conduit-connector-sdk/conn-sdk-cli/readmegen"
+	"github.com/conduitio/conduit-connector-sdk/conn-sdk-cli/specgen"
+	"github.com/spf13/cobra"
+)
+
+func main() {
+	cmdRoot := &cobra.Command{
+		Use:   "conn-sdk-cli",
+		Short: "Tooling around generating connector related files",
+	}
+
+	cmdReadmegen := &cobra.Command{
+		Use:   "readmegen",
+		Short: "Generate README for connector",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			specifications := cmd.Flag("specifications").Value.String()
+			readme := cmd.Flag("readme").Value.String()
+			write, _ := cmd.Flags().GetBool("write")
+
+			return readmegen.NewCommand(specifications, readme, write).Execute(cmd.Context())
+		},
+	}
+	cmdReadmegen.Flags().StringP("specifications", "s", "./connector.yaml", "path to the connector.yaml file")
+	cmdReadmegen.Flags().StringP("readme", "r", "./README.md", "path to the README.md file")
+	cmdReadmegen.Flags().BoolP("write", "w", false, "Overwrite readme file instead of printing to stdout")
+
+	cmdSpecgen := &cobra.Command{
+		Use:   "specgen",
+		Short: "Generate specification files for connector",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			output := cmd.Flag("output").Value.String()
+			path := cmd.Flag("path").Value.String()
+
+			return specgen.NewCommand(output, path).Execute(cmd.Context())
+		},
+	}
+	cmdSpecgen.Flags().StringP("output", "o", "connector.yaml", "name of the output file")
+	cmdSpecgen.Flags().StringP("package", "p", ".", "path to the package that contains the Connector variable")
+
+	cmdRoot.AddCommand(
+		cmdReadmegen,
+		cmdSpecgen,
+	)
+	cmdRoot.CompletionOptions.DisableDefaultCmd = true
+
+	if err := cmdRoot.Execute(); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
+		os.Exit(1)
+	}
+}

conn-sdk-cli/readmegen/cmd.go

@@ -0,0 +1,86 @@
+// Copyright © 2025 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package readmegen
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/conduitio/yaml/v3"
+)
+
+type Command struct {
+	specificationsFile string
+	readmeFile         string
+	write              bool
+}
+
+func NewCommand(specificationsFile, readmeFile string, write bool) *Command {
+	return &Command{
+		specificationsFile: specificationsFile,
+		readmeFile:         readmeFile,
+		write:              write,
+	}
+}
+
+func (cmd *Command) Execute(_ context.Context) error {
+	specs, err := cmd.readSpecs(cmd.specificationsFile)
+	if err != nil {
+		return err
+	}
+
+	buf := new(bytes.Buffer)
+	var out io.Writer = os.Stdout
+	if cmd.write {
+		// Write to buffer and then flush to file
+		out = buf
+	}
+
+	opts := GenerateOptions{
+		Data:       specs,
+		ReadmePath: cmd.readmeFile,
+		Out:        out,
+	}
+
+	err = Generate(opts)
+	if err != nil {
+		return fmt.Errorf("failed to generate readme: %w", err)
+	}
+
+	if cmd.write {
+		err = os.WriteFile(cmd.readmeFile, buf.Bytes(), 0o644)
+		if err != nil {
+			return fmt.Errorf("failed to write %s: %w", cmd.readmeFile, err)
+		}
+	}
+	return nil
+}
+
+func (*Command) readSpecs(path string) (map[string]any, error) {
+	specsRaw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read specifications from %v: %w", path, err)
+	}
+
+	var data map[string]any
+	err = yaml.Unmarshal(specsRaw, &data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal specifications: %w", err)
+	}
+	return data, nil
+}

conn-sdk-cli/readmegen/generate.go

@@ -0,0 +1,201 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package readmegen
+
+import (
+	"embed"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"strings"
+	"text/template"
+
+	"github.com/Masterminds/sprig/v3"
+	"github.com/conduitio/conduit-commons/config"
+)
+
+//go:embed templates/*
+var templates embed.FS
+
+type GenerateOptions struct {
+	// Required fields
+	Data       any       // The data to use in template generation
+	ReadmePath string    // Path to the readme template file
+	Out        io.Writer // Where to write the generated output
+
+	// Optional fields
+	FuncMap     template.FuncMap // Custom function map to merge with default functions
+	TemplatesFS fs.FS            // Custom templates directory to merge with default templates
+}
+
+func (opts GenerateOptions) validate() error {
+	// Validate required fields
+	if opts.Data == nil {
+		return fmt.Errorf("field Data is required")
+	}
+	if opts.ReadmePath == "" {
+		return fmt.Errorf("field ReadmePath is required")
+	}
+	if opts.Out == nil {
+		return fmt.Errorf("field Out is required")
+	}
+	return nil
+}
+
+func Generate(opts GenerateOptions) error {
+	if err := opts.validate(); err != nil {
+		return fmt.Errorf("invalid options: %w", err)
+	}
+
+	readme, err := os.ReadFile(opts.ReadmePath)
+	if err != nil {
+		return fmt.Errorf("could not read readme file %v: %w", opts.ReadmePath, err)
+	}
+	readmeTmpl, err := Preprocess(string(readme))
+	if err != nil {
+		return fmt.Errorf("could not preprocess readme file %v: %w", opts.ReadmePath, err)
+	}
+
+	t := template.New("readme").Funcs(funcMap).Funcs(sprig.FuncMap())
+	if opts.FuncMap != nil {
+		t = t.Funcs(opts.FuncMap)
+	}
+
+	t = template.Must(t.ParseFS(templates, "templates/*.tmpl"))
+	if opts.TemplatesFS != nil {
+		t = template.Must(t.ParseFS(opts.TemplatesFS, "*.tmpl"))
+	}
+
+	t = template.Must(t.Parse(readmeTmpl))
+
+	return t.Execute(opts.Out, opts.Data)
+}
+
+var funcMap = template.FuncMap{
+	"formatCommentYAML": formatCommentYAML,
+	"formatValueYAML":   formatValueYAML,
+	"zeroValueForType":  zeroValueForType,
+	"args":              args,
+}
+
+func args(kvs ...any) (map[string]any, error) {
+	if len(kvs)%2 != 0 {
+		return nil, errors.New("args requires even number of arguments")
+	}
+	m := make(map[string]any)
+	for i := 0; i < len(kvs); i += 2 {
+		s, ok := kvs[i].(string)
+		if !ok {
+			return nil, errors.New("even args must be strings")
+		}
+		m[s] = kvs[i+1]
+	}
+	return m, nil
+}
+
+// formatCommentYAML takes a markdown text and formats it as a comment in a YAML
+// file. The comment is prefixed with the given indent level and "# ". The lines
+// are wrapped at 80 characters.
+func formatCommentYAML(text string, indent int) string {
+	const (
+		prefix     = "# "
+		lineLen    = 80
+		tmpNewLine = "〠"
+	)
+
+	// remove markdown new lines
+	text = strings.ReplaceAll(text, "\n\n", tmpNewLine)
+	text = strings.ReplaceAll(text, "\n", " ")
+	text = strings.ReplaceAll(text, tmpNewLine, "\n")
+
+	comment := formatMultiline(text, strings.Repeat(" ", indent)+prefix, lineLen)
+	// remove first indent and last new line
+	comment = comment[indent : len(comment)-1]
+	return comment
+}
+
+func formatValueYAML(value string, indent int) string {
+	switch {
+	case value == "":
+		return `""`
+	case strings.Contains(value, "\n"):
+		// specifically used in the javascript processor
+		formattedValue := formatMultiline(value, strings.Repeat(" ", indent), 10000)
+		return fmt.Sprintf("|\n%s", formattedValue)
+	default:
+		return fmt.Sprintf(`"%s"`, value)
+	}
+}
+
+func formatMultiline(
+	input string,
+	prefix string,
+	maxLineLen int,
+) string {
+	textLen := maxLineLen - len(prefix)
+
+	// split the input into lines of length textLen
+	lines := strings.Split(input, "\n")
+	var formattedLines []string
+	for _, line := range lines {
+		if len(line) <= textLen {
+			formattedLines = append(formattedLines, line)
+			continue
+		}
+
+		// split the line into multiple lines, don't break words
+		words := strings.Fields(line)
+		var formattedLine string
+		for _, word := range words {
+			if len(formattedLine)+len(word) > textLen {
+				formattedLines = append(formattedLines, formattedLine[1:])
+				formattedLine = ""
+			}
+			formattedLine += " " + word
+		}
+		if formattedLine != "" {
+			formattedLines = append(formattedLines, formattedLine[1:])
+		}
+	}
+
+	// combine lines including indent and prefix
+	var formatted string
+	for _, line := range formattedLines {
+		formatted += prefix + line + "\n"
+	}
+
+	return formatted
+}
+
+func zeroValueForType(t string) string {
+	switch t {
+	case config.ParameterTypeString.String():
+		return ""
+	case config.ParameterTypeInt.String():
+		return "0"
+	case config.ParameterTypeFloat.String():
+		return "0.0"
+	case config.ParameterTypeBool.String():
+		return "false"
+	case config.ParameterTypeFile.String():
+		return ""
+	case config.ParameterTypeDuration.String():
+		return "0s"
+	default:
+		return ""
+	}
+}