Expose component configuration schemas (#2210)

* POC/Proposal for exposing configuration schemas

Issue: https://github.com/open-telemetry/opentelemetry-collector/issues/1995

This change proposes a way to generate metadata for the configuration structs
used by Collector components. The generated metadata could be used for
documentation and tooling.

The way it works is, given a component, its configuration struct is found, and
its field names, types, tags, default values, and comments are retrieved and
put into a struct. This is done recursively, but respecting "squashed" fields,
and any fields omitted with a "-" tag. The final struct is marshaled to a yaml
file in the directory of the configuration type.

Also included are a simple command line interface and an example metadata
file. The CLI lets users create a metadata file for either a given component or
for all registered components.

Tests have not been included as this change is meant to be exploratory.

* Use a map rather than an array for fields

* Expose component configuration schemas

Issue: https://github.com/open-telemetry/opentelemetry-collector/issues/1995

This change adds a command line utility to generate schema files for each of
the configuration structs used by Collector components. The intent is for these
generated files to be used by documentation utilities and tooling.

The way it works is, given a component, its default configuration struct is
retrieved and introspected, producing field names, types, default values, and
comments. Those are recursively put into a graph of structs, respecting
"squashed" fields and any fields omitted with a "-" tag. Finally, the struct is
marshaled to a yaml file in the directory of the configuration type.

The CLI lets users create a metadata file for either one component or for all
registered components.

For reference, preliminary feedback was gathered here #2169

* Create a local go module, don't panic on mapstructure errors

* Move code to internal, exclude cfgschema from Makefile

* Increase test coverage

* Increase test coverage

* Fix lint

* Reorg so schemagen can be used from contrib

* Don't export stuff that no longer needs to be exported

Makefile

@@ -4,6 +4,7 @@ include ./Makefile.Common
 ALL_SRC := $(shell find . -name '*.go' \
 							-not -path './cmd/issuegenerator/*' \
 							-not -path './cmd/mdatagen/*' \
+							-not -path './cmd/schemagen/*' \
 							-not -path './internal/tools/*' \
 							-not -path './examples/demo/app/*' \
 							-not -path './internal/data/opentelemetry-proto-gen/*' \

cmd/schemagen/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

cmd/schemagen/go.mod

@@ -0,0 +1,10 @@
+module go.opentelemetry.io/collector/cmd/schemagen
+
+go 1.15
+
+require (
+	github.com/fatih/structtag v1.2.0
+	github.com/stretchr/testify v1.6.1
+	go.opentelemetry.io/collector v0.15.0
+	gopkg.in/yaml.v2 v2.4.0
+)

cmd/schemagen/main.go

@@ -0,0 +1,28 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"go.opentelemetry.io/collector/cmd/schemagen/schemagen"
+	"go.opentelemetry.io/collector/service/defaultcomponents"
+)
+
+func main() {
+	components, err := defaultcomponents.Components()
+	if err != nil {
+		panic(err)
+	}
+	schemagen.CLI(components)
+}

cmd/schemagen/schemagen/cli.go

@@ -0,0 +1,65 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schemagen
+
+import (
+	"flag"
+	"fmt"
+
+	"go.opentelemetry.io/collector/component"
+)
+
+func CLI(c component.Factories) {
+	prepUsage()
+
+	e, componentType, componentName := parseArgs()
+	e.yamlFilename = yamlFilename
+
+	switch {
+	case componentType == "all":
+		createAllSchemaFiles(c, e)
+	case componentType != "" && componentName != "":
+		createSingleSchemaFile(
+			c,
+			componentType,
+			componentName,
+			e,
+		)
+	default:
+		flag.Usage()
+	}
+}
+
+func prepUsage() {
+	const usage = `cfgschema all
+cfgschema <componentType> <componentName>
+
+options
+`
+	flag.Usage = func() {
+		_, _ = fmt.Fprint(flag.CommandLine.Output(), usage)
+		flag.PrintDefaults()
+	}
+}
+
+func parseArgs() (env, string, string) {
+	e := env{}
+	flag.StringVar(&e.srcRoot, "s", defaultSrcRoot, "collector source root")
+	flag.StringVar(&e.moduleName, "m", defaultModule, "module name")
+	flag.Parse()
+	componentType := flag.Arg(0)
+	componentName := flag.Arg(1)
+	return e, componentType, componentName
+}

cmd/schemagen/schemagen/cli_all.go

@@ -0,0 +1,45 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schemagen
+
+import (
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/configmodels"
+)
+
+// createAllSchemaFiles creates config yaml schema files for all registered components
+func createAllSchemaFiles(components component.Factories, env env) {
+	cfgs := getAllConfigs(components)
+	for _, cfg := range cfgs {
+		createSchemaFile(cfg, env)
+	}
+}
+
+func getAllConfigs(components component.Factories) []configmodels.NamedEntity {
+	var cfgs []configmodels.NamedEntity
+	for _, f := range components.Receivers {
+		cfgs = append(cfgs, f.CreateDefaultConfig())
+	}
+	for _, f := range components.Extensions {
+		cfgs = append(cfgs, f.CreateDefaultConfig())
+	}
+	for _, f := range components.Processors {
+		cfgs = append(cfgs, f.CreateDefaultConfig())
+	}
+	for _, f := range components.Exporters {
+		cfgs = append(cfgs, f.CreateDefaultConfig())
+	}
+	return cfgs
+}

cmd/schemagen/schemagen/cli_all_test.go

@@ -0,0 +1,49 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schemagen
+
+import (
+	"io/ioutil"
+	"path"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
+)
+
+func TestGetAllConfigs(t *testing.T) {
+	cfgs := getAllConfigs(testComponents())
+	require.NotNil(t, cfgs)
+}
+
+func TestCreateAllSchemaFiles(t *testing.T) {
+	e := testEnv()
+	tempDir := t.TempDir()
+	e.yamlFilename = func(t reflect.Type, e env) string {
+		return path.Join(tempDir, t.String()+".yaml")
+	}
+	createAllSchemaFiles(testComponents(), e)
+	fileInfos, err := ioutil.ReadDir(tempDir)
+	require.NoError(t, err)
+	require.NotNil(t, fileInfos)
+	file, err := ioutil.ReadFile(path.Join(tempDir, "otlpexporter.Config.yaml"))
+	require.NoError(t, err)
+	fld := field{}
+	err = yaml.Unmarshal(file, &fld)
+	require.NoError(t, err)
+	require.Equal(t, "*otlpexporter.Config", fld.Type)
+	require.NotNil(t, fld.Fields)
+}

cmd/schemagen/schemagen/cli_single.go

@@ -0,0 +1,64 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schemagen
+
+import (
+	"fmt"
+	"os"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/configmodels"
+)
+
+// createSingleSchemaFile creates a config schema yaml file for a single component
+func createSingleSchemaFile(components component.Factories, componentType, componentName string, env env) {
+	cfg, err := getConfig(components, componentType, componentName)
+	if err != nil {
+		println(err.Error())
+		os.Exit(1)
+	}
+	createSchemaFile(cfg, env)
+}
+
+func getConfig(components component.Factories, componentType, componentName string) (configmodels.NamedEntity, error) {
+	t := configmodels.Type(componentName)
+	switch componentType {
+	case "receiver":
+		c := components.Receivers[t]
+		if c == nil {
+			return nil, fmt.Errorf("unknown receiver name %q", componentName)
+		}
+		return c.CreateDefaultConfig(), nil
+	case "processor":
+		c := components.Processors[t]
+		if c == nil {
+			return nil, fmt.Errorf("unknown processor name %q", componentName)
+		}
+		return c.CreateDefaultConfig(), nil
+	case "exporter":
+		c := components.Exporters[t]
+		if c == nil {
+			return nil, fmt.Errorf("unknown exporter name %q", componentName)
+		}
+		return c.CreateDefaultConfig(), nil
+	case "extension":
+		c := components.Extensions[t]
+		if c == nil {
+			return nil, fmt.Errorf("unknown extension name %q", componentName)
+		}
+		return c.CreateDefaultConfig(), nil
+	}
+	return nil, fmt.Errorf("unknown component type %q", componentType)
+}

cmd/schemagen/schemagen/cli_single_test.go

@@ -0,0 +1,95 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schemagen
+
+import (
+	"io/ioutil"
+	"path"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/service/defaultcomponents"
+)
+
+func TestCreateReceiverConfig(t *testing.T) {
+	cfg, err := getConfig(testComponents(), "receiver", "otlp")
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+}
+
+func TestCreateProcesorConfig(t *testing.T) {
+	cfg, err := getConfig(testComponents(), "processor", "filter")
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+}
+
+func TestGetConfig(t *testing.T) {
+	tests := []struct {
+		name          string
+		componentType string
+	}{
+		{
+			name:          "otlp",
+			componentType: "receiver",
+		},
+		{
+			name:          "filter",
+			componentType: "processor",
+		},
+		{
+			name:          "otlp",
+			componentType: "exporter",
+		},
+		{
+			name:          "zpages",
+			componentType: "extension",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cfg, err := getConfig(testComponents(), test.componentType, test.name)
+			require.NoError(t, err)
+			require.NotNil(t, cfg)
+		})
+	}
+}
+
+func TestCreateSingleSchemaFile(t *testing.T) {
+	e := testEnv()
+	tempDir := t.TempDir()
+	e.yamlFilename = func(reflect.Type, env) string {
+		return path.Join(tempDir, schemaFilename)
+	}
+	createSingleSchemaFile(testComponents(), "exporter", "otlp", e)
+	file, err := ioutil.ReadFile(path.Join(tempDir, schemaFilename))
+	require.NoError(t, err)
+	fld := field{}
+	err = yaml.Unmarshal(file, &fld)
+	require.NoError(t, err)
+	require.Equal(t, "*otlpexporter.Config", fld.Type)
+	require.NotNil(t, fld.Fields)
+}
+
+func testComponents() component.Factories {
+	components, err := defaultcomponents.Components()
+	if err != nil {
+		panic(err)
+	}
+	return components
+}