add buildkit and more

.gitignore

@@ -1,7 +1,9 @@
 test
 dist/
 bin/
+pkg/
 .idea/
+playground/
 
 *.exe
 *.zip

Makefile

@@ -1,5 +1,8 @@
 
 .PHONY: barbe
 barbe:
-	cd cli && CGO_ENABLED=0 go build -o barbe -ldflags '-s -w' main.go
-	echo "built at `pwd`/cli/barbe"
+	BARBE_RELEASE=1 ./build.sh
+
+.PHONY: barbe-dev
+barbe-dev:
+	BARBE_DEV=1 BARBE_RELEASE=1 ./build.sh

README.md

@@ -36,7 +36,7 @@ These definitions are written into templates that generate all the files needed
   <img src="./readme_img_1.png" width="400" />
 </p>
 
-This creates a lot of attracting advantages:
+This approach has lot of advantages:
 - Drastically reduce boilerplate of any toolchain
 - Use a toolchain's best practices by default, forever
 - Reduce cost of changing tooling and cloud platforms
@@ -58,7 +58,7 @@ each one can manipulate and create more syntax tokens, Barbe then formats the re
 You specify the templates you want using a simple URL, kind of like a `FROM` statement in a Dockerfile.
 This allows your configuration file to stay simple, but still harness a world a complexity that the template makers prepared for you.
 
-Templates can also manipulate the syntax tokens it receives using dark magic, a simple reference like `cloudformation.MyBucket.Id` in your Terraform file could be turned
+Templates can also manipulate the syntax tokens it receives using dark magic, a simple reference like `cloudformation("my-stack").output.MyBucketName` in your Terraform file can be turned
 into a concrete value gotten from your Cloudformation stack, without you lifting a finger.
 
 #### An imaginary example project

build.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+# https://github.com/hashicorp/terraform/blob/main/scripts/build.sh
+# This script builds the application from source for multiple platforms.
+
+# Determine the arch/os combos we're building for
+XC_ARCH=${XC_ARCH:-"386 amd64 arm"}
+XC_OS=${XC_OS:-linux darwin windows freebsd}
+XC_EXCLUDE_OSARCH=""# !darwin/arm !darwin/386
+
+# Delete the old dir
+echo "==> Removing old directory..."
+rm -f bin/*
+rm -rf pkg/*
+mkdir -p bin/
+
+# If its dev mode, only build for ourself
+if [[ -n "${BARBE_DEV}" ]]; then
+    XC_OS=$(go env GOOS)
+    XC_ARCH=$(go env GOARCH)
+fi
+
+if ! which gox > /dev/null; then
+    echo "==> Installing gox..."
+    go install github.com/mitchellh/gox
+fi
+
+# Instruct gox to build statically linked binaries
+export CGO_ENABLED=0
+
+# Set module download mode to readonly to not implicitly update go.mod
+export GOFLAGS="-mod=readonly"
+
+# In release mode we don't want debug information in the binary
+if [[ -n "${BARBE_RELEASE}" ]]; then
+    LD_FLAGS="-s -w"
+fi
+
+# Ensure all remote modules are downloaded and cached before build so that
+# the concurrent builds launched by gox won't race to redundantly download them.
+go mod download
+
+# Build!
+echo "==> Building..."
+gox  \
+    -os="${XC_OS}" \
+    -arch="${XC_ARCH}" \
+    -osarch="${XC_EXCLUDE_OSARCH}" \
+    -ldflags "${LD_FLAGS}" \
+    -output "pkg/{{.OS}}_{{.Arch}}/barbe" \
+    ./cli
+
+# Move all the compiled things to the $GOPATH/bin
+#GOPATH=${GOPATH:-$(go env GOPATH)}
+#case $(uname) in
+#    CYGWIN*)
+#        GOPATH="$(cygpath $GOPATH)"
+#        ;;
+#esac
+#OLDIFS=$IFS
+#IFS=: MAIN_GOPATH=($GOPATH)
+#IFS=$OLDIFS
+
+# Create GOPATH/bin if it's doesn't exists
+#if [ ! -d $MAIN_GOPATH/bin ]; then
+#    echo "==> Creating GOPATH/bin directory..."
+#    mkdir -p $MAIN_GOPATH/bin
+#fi
+
+# Copy our OS/Arch to the bin/ directory
+DEV_PLATFORM="./pkg/$(go env GOOS)_$(go env GOARCH)"
+if [[ -d "${DEV_PLATFORM}" ]]; then
+    for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f); do
+        cp ${F} bin/
+        # cp ${F} ${MAIN_GOPATH}/bin/
+    done
+fi
+
+if [ "${BARBE_DEV}x" = "x" ]; then
+    # Zip and copy to the dist dir
+    echo "==> Packaging..."
+    for PLATFORM in $(find ./pkg -mindepth 1 -maxdepth 1 -type d); do
+        OSARCH=$(basename ${PLATFORM})
+        echo "--> ${OSARCH}"
+
+        pushd $PLATFORM >/dev/null 2>&1
+        zip ../${OSARCH}.zip ./*
+        popd >/dev/null 2>&1
+    done
+fi
+
+# Done!
+echo
+echo "==> Results:"
+ls -hl bin/

cli/cmd/apply.go

@@ -0,0 +1,87 @@
+package cmd
+
+import (
+	"barbe/cli/logger"
+	"barbe/core"
+	"barbe/core/chown_util"
+	"context"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+var applyCmd = &cobra.Command{
+	Use:     "apply [GLOB...]",
+	Short:   "Generate files based on the given configuration, and execute all the appliers that will deploy the generated files",
+	Args:    cobra.ArbitraryArgs,
+	Example: "barbe apply config.hcl --output dist\nbarbe apply **/*.hcl --output dist",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := viper.BindPFlags(cmd.Flags()); err != nil {
+			panic(err)
+		}
+
+		lg := logger.New()
+		ctx := lg.WithContext(cmd.Context())
+
+		if len(args) == 0 {
+			args = []string{"*.hcl"}
+		}
+		log.Ctx(ctx).Debug().Msgf("running with args: %v", args)
+
+		allFiles := make([]core.FileDescription, 0)
+		for _, arg := range args {
+			matches, err := glob(arg)
+			if err != nil {
+				log.Ctx(ctx).Fatal().Err(err).Msg("glob matching failed")
+			}
+			for _, match := range matches {
+				fileContent, err := os.ReadFile(match)
+				if err != nil {
+					log.Ctx(ctx).Error().Err(err).Msg("reading file failed")
+					continue
+				}
+				allFiles = append(allFiles, core.FileDescription{
+					Name:    match,
+					Content: fileContent,
+				})
+			}
+		}
+
+		grouped := groupFilesByDirectory(dedup(allFiles))
+		for dir, files := range grouped {
+			log.Ctx(ctx).Debug().Msg("executing maker for directory: '" + dir + "'")
+
+			fileNames := make([]string, 0, len(files))
+			for _, file := range files {
+				fileNames = append(fileNames, file.Name)
+			}
+			log.Ctx(ctx).Debug().Msg("with files: [" + strings.Join(fileNames, ", ") + "]")
+
+			maker := makeMaker(path.Join(viper.GetString("output"), dir))
+			innerCtx := context.WithValue(ctx, "maker", maker)
+
+			err := os.MkdirAll(maker.OutputDir, 0755)
+			if err != nil {
+				log.Ctx(innerCtx).Fatal().Err(err).Msg("failed to create output directory")
+			}
+			chown_util.TryRectifyRootFiles(innerCtx, []string{maker.OutputDir})
+
+			err = maker.Make(innerCtx, files, true)
+			if err != nil {
+				log.Ctx(innerCtx).Fatal().Err(err).Msg("generation failed")
+			}
+
+			allPaths := make([]string, 0)
+			filepath.WalkDir(maker.OutputDir, func(path string, d fs.DirEntry, err error) error {
+				allPaths = append(allPaths, path)
+				return nil
+			})
+			chown_util.TryRectifyRootFiles(innerCtx, allPaths)
+		}
+	},
+}

cli/cmd/generate.go

@@ -1,30 +1,35 @@
 package cmd
 
 import (
-	"context"
-	"github.com/rs/zerolog/log"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"os"
-	"path"
-	"path/filepath"
 	"barbe/cli/logger"
 	"barbe/core"
+	"barbe/core/aws_session_provider"
+	"barbe/core/buildkit_runner"
+	"barbe/core/chown_util"
 	"barbe/core/hcl_parser"
+	"barbe/core/json_parser"
 	"barbe/core/jsonnet_templater"
 	"barbe/core/raw_file"
-	"barbe/core/s3_bucket_creator"
 	"barbe/core/simplifier_transform"
 	"barbe/core/terraform_fmt"
 	"barbe/core/traversal_manipulator"
 	"barbe/core/zipper_fmt"
+	"context"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
 	"strings"
 )
 
 var generateCmd = &cobra.Command{
-	Use:   "generate [GLOB...]",
-	Short: "Generate files out of abstracted templates",
-	Args:  cobra.ArbitraryArgs,
+	Use:     "generate [GLOB...]",
+	Short:   "Generate files based on the given configuration",
+	Args:    cobra.ArbitraryArgs,
+	Example: "barbe generate config.hcl --output dist\nbarbe generate **/*.hcl --output dist",
 	Run: func(cmd *cobra.Command, args []string) {
 		if err := viper.BindPFlags(cmd.Flags()); err != nil {
 			panic(err)
@@ -74,11 +79,19 @@ var generateCmd = &cobra.Command{
 			if err != nil {
 				log.Ctx(innerCtx).Fatal().Err(err).Msg("failed to create output directory")
 			}
+			chown_util.TryRectifyRootFiles(innerCtx, []string{maker.OutputDir})
 
-			err = maker.Make(innerCtx, files)
+			err = maker.Make(innerCtx, files, false)
 			if err != nil {
 				log.Ctx(innerCtx).Fatal().Err(err).Msg("generation failed")
 			}
+
+			allPaths := make([]string, 0)
+			filepath.WalkDir(maker.OutputDir, func(path string, d fs.DirEntry, err error) error {
+				allPaths = append(allPaths, path)
+				return nil
+			})
+			chown_util.TryRectifyRootFiles(innerCtx, allPaths)
 		}
 	},
 }
@@ -160,6 +173,7 @@ func makeMaker(dir string) *core.Maker {
 		OutputDir: dir,
 		Parsers: []core.Parser{
 			hcl_parser.HclParser{},
+			json_parser.JsonParser{},
 		},
 		PreTransformers: []core.Transformer{
 			simplifier_transform.SimplifierTransformer{},
@@ -170,16 +184,20 @@ func makeMaker(dir string) *core.Maker {
 			jsonnet_templater.JsonnetTemplater{},
 		},
 		Transformers: []core.Transformer{
-			//the simplifier being first is very important, it simplify syntax that is equivalent
+			//the simplifier being first is very important, it simplifies syntax that is equivalent
 			//to make it a lot easier for the transformers to work with
 			simplifier_transform.SimplifierTransformer{},
 			traversal_manipulator.TraversalManipulator{},
+			aws_session_provider.AwsSessionProviderTransformer{},
+			buildkit_runner.BuildkitRunner{},
 		},
 		Formatters: []core.Formatter{
 			terraform_fmt.TerraformFormatter{},
 			zipper_fmt.ZipperFormatter{},
 			raw_file.RawFileFormatter{},
-			s3_bucket_creator.S3BucketCreator{},
+		},
+		Appliers: []core.Applier{
+			buildkit_runner.BuildkitRunner{},
 		},
 	}
 }

cli/cmd/root.go

@@ -9,7 +9,7 @@ import (
 
 var rootCmd = &cobra.Command{
 	Use:   "barbe",
-	Short: "A programmable abstraction layer for any configuration language",
+	Short: "A programmable syntax manipulation engine",
 }
 
 func init() {
@@ -24,6 +24,7 @@ func init() {
 	rootCmd.AddCommand(
 		versionCmd,
 		generateCmd,
+		applyCmd,
 	)
 	rootCmd.CompletionOptions.HiddenDefaultCmd = true
 

cli/logger/logger.go

@@ -21,8 +21,6 @@ func New() zerolog.Logger {
 
 	if !jsonLogs() {
 		logger = logger.Output(&PlainOutput{Out: colorable.NewColorableStderr()})
-	} else {
-		logger = logger.With().Timestamp().Caller().Logger()
 	}
 
 	level := viper.GetString("log-level")