Merge pull request #194 from ulucinar/families

Generate per-service providers

pkg/config/provider.go

@@ -89,6 +89,13 @@ type Provider struct {
 	// can add "aws_waf.*" to the list.
 	SkipList []string
 
+	// MainTemplate is the template string to be used to render the
+	// provider subpackage main program. If this is set, the generated provider
+	// is broken up into subpackage families partitioned across the API groups.
+	// A monolithic provider is also generated to
+	// ensure backwards-compatibility.
+	MainTemplate string
+
 	// skippedResourceNames is a list of Terraform resource names
 	// available in the Terraform provider schema, but
 	// not in the include list or in the skip list, meaning that
@@ -183,6 +190,12 @@ func WithFeaturesPackage(s string) ProviderOption {
 	}
 }
 
+func WithMainTemplate(template string) ProviderOption {
+	return func(p *Provider) {
+		p.MainTemplate = template
+	}
+}
+
 // NewProvider builds and returns a new Provider from provider
 // tfjson schema, that is generated using Terraform CLI with:
 // `terraform providers schema --json`

pkg/pipeline/run.go

@@ -22,6 +22,14 @@ type terraformedInput struct {
 	ParametersTypeName string
 }
 
+const (
+	// TODO: we should be careful that there may also exist short groups with
+	// these names. We can consider making these configurable by the provider
+	// maintainer.
+	configPackageName   = "config"
+	monolithPackageName = "monolith"
+)
+
 // Run runs the Upjet code generation pipelines.
 func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 	// Note(turkenh): nolint reasoning - this is the main function of the code
@@ -56,9 +64,11 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 		apiVersionPkgList = append(apiVersionPkgList, filepath.Join(pc.ModulePath, p))
 	}
 	// Add ProviderConfig controller package to the list of controller packages.
-	controllerPkgList := make([]string, 0)
+	controllerPkgMap := make(map[string][]string)
 	for _, p := range pc.BasePackages.Controller {
-		controllerPkgList = append(controllerPkgList, filepath.Join(pc.ModulePath, p))
+		path := filepath.Join(pc.ModulePath, p)
+		controllerPkgMap[configPackageName] = append(controllerPkgMap[configPackageName], path)
+		controllerPkgMap[monolithPackageName] = append(controllerPkgMap[monolithPackageName], path)
 	}
 	count := 0
 	for group, versions := range resourcesGroups {
@@ -87,7 +97,9 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 				if err != nil {
 					panic(errors.Wrapf(err, "cannot generate controller for resource %s", name))
 				}
-				controllerPkgList = append(controllerPkgList, ctrlPkgPath)
+				sGroup := strings.Split(group, ".")[0]
+				controllerPkgMap[sGroup] = append(controllerPkgMap[sGroup], ctrlPkgPath)
+				controllerPkgMap[monolithPackageName] = append(controllerPkgMap[monolithPackageName], ctrlPkgPath)
 				if err := exampleGen.Generate(group, version, resources[name]); err != nil {
 					panic(errors.Wrapf(err, "cannot generate example manifest for resource %s", name))
 				}
@@ -112,7 +124,9 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 	if err := NewRegisterGenerator(rootDir, pc.ModulePath).Generate(apiVersionPkgList); err != nil {
 		panic(errors.Wrap(err, "cannot generate register file"))
 	}
-	if err := NewSetupGenerator(rootDir, pc.ModulePath).Generate(controllerPkgList); err != nil {
+	// Generate the provider,
+	// i.e. the setup function and optionally the provider's main program.
+	if err := NewProviderGenerator(rootDir, pc.ModulePath).Generate(controllerPkgMap, pc.MainTemplate); err != nil {
 		panic(errors.Wrap(err, "cannot generate setup file"))
 	}
 

pkg/pipeline/setup.go

@@ -5,35 +5,85 @@ Copyright 2021 Upbound Inc.
 package pipeline
 
 import (
+	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 	"sort"
+	"text/template"
 
 	"github.com/muvaf/typewriter/pkg/wrapper"
 	"github.com/pkg/errors"
 
 	"github.com/upbound/upjet/pkg/pipeline/templates"
 )
 
-// NewSetupGenerator returns a new SetupGenerator.
-func NewSetupGenerator(rootDir, modulePath string) *SetupGenerator {
-	return &SetupGenerator{
+// NewProviderGenerator returns a new ProviderGenerator.
+func NewProviderGenerator(rootDir, modulePath string) *ProviderGenerator {
+	return &ProviderGenerator{
+		ProviderPath:       filepath.Join(rootDir, "cmd", "provider"),
 		LocalDirectoryPath: filepath.Join(rootDir, "internal", "controller"),
 		LicenseHeaderPath:  filepath.Join(rootDir, "hack", "boilerplate.go.txt"),
 		ModulePath:         modulePath,
 	}
 }
 
-// SetupGenerator generates controller setup file.
-type SetupGenerator struct {
+// ProviderGenerator generates controller setup file.
+type ProviderGenerator struct {
+	ProviderPath       string
 	LocalDirectoryPath string
 	LicenseHeaderPath  string
 	ModulePath         string
 }
 
-// Generate writes the setup file with the content produced using given
-// list of version packages.
-func (sg *SetupGenerator) Generate(versionPkgList []string) error {
+// Generate writes the setup file and the corresponding provider main file
+// using the given list of version packages.
+func (sg *ProviderGenerator) Generate(versionPkgMap map[string][]string, mainTemplate string) error {
+	var t *template.Template
+	if len(mainTemplate) != 0 {
+		tmpl, err := template.New("main").Parse(mainTemplate)
+		if err != nil {
+			return errors.Wrap(err, "failed to parse the provider main program template")
+		}
+		t = tmpl
+	}
+	if t == nil {
+		return errors.Wrap(sg.generate("", versionPkgMap[monolithPackageName]), "failed to generate the controller setup file")
+	}
+	for g, versionPkgList := range versionPkgMap {
+		if err := sg.generate(g, versionPkgList); err != nil {
+			return errors.Wrapf(err, "failed to generate the controller setup file for group: %s", g)
+		}
+		if err := generateProviderMain(sg.ProviderPath, g, t); err != nil {
+			return errors.Wrapf(err, "failed to write main program for group: %s", g)
+		}
+	}
+	return nil
+}
+
+func generateProviderMain(providerPath, group string, t *template.Template) error {
+	f := filepath.Join(providerPath, group)
+	if err := os.MkdirAll(f, 0750); err != nil {
+		return errors.Wrapf(err, "failed to mkdir provider main program path: %s", f)
+	}
+	m, err := os.OpenFile(filepath.Join(filepath.Clean(f), "zz_main.go"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return errors.Wrap(err, "failed to open provider main program file")
+	}
+	defer func() {
+		if err := m.Close(); err != nil {
+			log.Fatalf("Failed to close the templated main %q: %s", f, err.Error())
+		}
+	}()
+	if err := t.Execute(m, map[string]any{
+		"Group": group,
+	}); err != nil {
+		return errors.Wrap(err, "failed to execute provider main program template")
+	}
+	return nil
+}
+
+func (sg *ProviderGenerator) generate(group string, versionPkgList []string) error {
 	setupFile := wrapper.NewFile(filepath.Join(sg.ModulePath, "apis"), "apis", templates.SetupTemplate,
 		wrapper.WithGenStatement(GenStatement),
 		wrapper.WithHeaderPath(sg.LicenseHeaderPath),
@@ -43,9 +93,19 @@ func (sg *SetupGenerator) Generate(versionPkgList []string) error {
 	for i, pkgPath := range versionPkgList {
 		aliases[i] = setupFile.Imports.UsePackage(pkgPath)
 	}
+	g := ""
+	if len(group) != 0 {
+		g = "_" + group
+	}
 	vars := map[string]any{
 		"Aliases": aliases,
+		"Group":   g,
+	}
+	filePath := ""
+	if len(group) == 0 {
+		filePath = filepath.Join(sg.LocalDirectoryPath, "zz_setup.go")
+	} else {
+		filePath = filepath.Join(sg.LocalDirectoryPath, fmt.Sprintf("zz_%s_setup.go", group))
 	}
-	filePath := filepath.Join(sg.LocalDirectoryPath, "zz_setup.go")
 	return errors.Wrap(setupFile.Write(filePath, vars, os.ModePerm), "cannot write setup file")
 }

pkg/pipeline/templates/setup.go.tmpl

@@ -12,9 +12,9 @@ import (
 	{{ .Imports }}
 )
 
-// Setup creates all controllers with the supplied logger and adds them to
+// Setup{{ .Group }} creates all controllers with the supplied logger and adds them to
 // the supplied manager.
-func Setup(mgr ctrl.Manager, o controller.Options) error {
+func Setup{{ .Group }}(mgr ctrl.Manager, o controller.Options) error {
 	for _, setup := range []func(ctrl.Manager, controller.Options) error{
 		{{- range $alias := .Aliases }}
 		{{ $alias }}Setup,