Skip to content

Commit

Permalink
refactor: Remove the need for the flux_test_gen files (#4715)
Browse files Browse the repository at this point in the history 
* refactor: Discover the flux test files at test time instead of generating

* chore: Remove the flux_test_gen files

We can discover and compile these files at test time instead of relying on the generate step

* chore: Add an explanatory comment for packages.go

* chore: make staticcheck
  • Loading branch information
Markus Westerlind authored May 5, 2022
1 parent df7c5ac commit 8505b7e
Show file tree
Hide file tree
Showing 54 changed files with 142 additions and 659,824 deletions.
283 changes: 8 additions & 275 deletions internal/cmd/builtin/cmd/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ import (
"os"
"path"
"path/filepath"
"reflect"
"strings"
"time"

"github.com/dave/jennifer/jen"
"github.com/influxdata/flux/ast"
Expand Down Expand Up @@ -53,7 +51,7 @@ func generate(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
var goPackages, testPackages []string
var goPackages []string
err = walkDirs(rootDir, func(dir string) error {
// Determine the absolute flux package path
fluxPath, err := filepath.Rel(rootDir, dir)
Expand All @@ -75,32 +73,26 @@ func generate(cmd *cobra.Command, args []string) error {
pkg.Path = fluxPath
}

var fluxPkg, testPkg *ast.Package
var fluxPkg *ast.Package
switch len(pkgs) {
case 0:
return nil
case 1:
for k, v := range pkgs {
if strings.HasSuffix(k, "_test") {
testPkg = v
} else {
if !strings.HasSuffix(k, "_test") {
fluxPkg = v
}
}
case 2:
for k, v := range pkgs {
if strings.HasSuffix(k, "_test") {
testPkg = v
continue
}
fluxPkg = v
}
if fluxPkg == nil {
return fmt.Errorf("cannot have two Flux test packages in the same directory")
}
if testPkg == nil {
return fmt.Errorf("cannot have two distinct non-test Flux packages in the same directory")
}
default:
keys := make([]string, 0, len(pkgs))
for k := range pkgs {
Expand All @@ -125,56 +117,19 @@ func generate(cmd *cobra.Command, args []string) error {
return err
}
}
if testPkg != nil {
// Strip out test files with the testcase statement.
validFiles := []*ast.File{}
for _, file := range testPkg.Files {
valid := true
for _, item := range file.Body {
if _, ok := item.(*ast.TestCaseStatement); ok {
valid = false
}
}
if valid {
validFiles = append(validFiles, file)
}
}
if len(validFiles) < len(testPkg.Files) {
testPkg.Files = validFiles
}

if ast.Check(testPkg) > 0 {
return errors.Wrapf(ast.GetError(testPkg), codes.Inherit, "failed to parse test package %q", testPkg.Package)
}
// Validate test package file use _test.flux suffix for the file name
for _, f := range testPkg.Files {
if !strings.HasSuffix(f.Name, "_test.flux") {
return fmt.Errorf("flux test files must use the _test.flux suffix in their file name, found %q", path.Join(dir, f.Name))
}
}
// Track go import path
importPath := path.Join(pkgName, dir)
if importPath != pkgName {
testPackages = append(testPackages, importPath)
}
// Generate test AST file using non *_test package name since this is Go code that needs to be part of the normal build.
if err := generateTestASTFile(dir, strings.TrimSuffix(testPkg.Package, "_test"), []*ast.Package{testPkg}); err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}

if err := generateTestPkgList(testPackages); err != nil {
return err
}

// Write the import file
f := jen.NewFile(path.Base(pkgName))
f.HeaderComment("// DO NOT EDIT: This file is autogenerated via the builtin command.")
f.HeaderComment(`// DO NOT EDIT: This file is autogenerated via the builtin command.
//
// The imports in this file ensures that all the init functions runs and registers
// the builtins for the flux runtime
`)
f.Anon(goPackages...)
return f.Save(filepath.Join(rootDir, importFile))
}
Expand All @@ -190,56 +145,6 @@ func generateFluxASTFile(dir string, pkg *ast.Package) error {
return file.Save(filepath.Join(dir, "flux_gen.go"))
}

func generateTestPkgList(imports []string) error {
stmts := make([]jen.Code, len(imports)+2)
// var pkgs []*ast.Package
stmts[0] = jen.
Var().
Id("pkgs").
Index().
Op("*").
Qual("github.com/influxdata/flux/ast", "Package")

for i, path := range imports {
// pkgs = append(pkgs, <imported_package>.FluxTestPackages...)
stmts[i+1] = jen.Id("pkgs").Op("=").Id("append").Call(
jen.Id("pkgs"), jen.Qual(path, "FluxTestPackages").Op("..."),
)
}

// return pkgs
stmts[len(stmts)-1] = jen.Return(jen.Id("pkgs"))

file := jen.NewFile(path.Base(pkgName))
file.HeaderComment("// DO NOT EDIT: This file is autogenerated via the builtin command.")
// var FluxTestPackages = func() []*ast.Package {
// statements ...
// }
file.
Var().
Id("FluxTestPackages").
Op("=").
Func().
Params().
Index().
Op("*").
Qual("github.com/influxdata/flux/ast", "Package").
Block(stmts...).
Call()
return file.Save(filepath.Join(rootDir, "test_packages.go"))
}

func generateTestASTFile(dir, pkg string, pkgs []*ast.Package) error {
file := jen.NewFile(pkg)
file.HeaderComment("// DO NOT EDIT: This file is autogenerated via the builtin command.")
v, err := constructValue(reflect.ValueOf(pkgs))
if err != nil {
return err
}
file.Var().Id("FluxTestPackages").Op("=").Add(v)
return file.Save(filepath.Join(dir, "flux_test_gen.go"))
}

func readIgnoreFile(fn string) ([]string, error) {
f, err := os.Open(fn)
if err != nil {
Expand Down Expand Up @@ -284,175 +189,3 @@ func walkDirs(path string, f func(dir string) error) error {
}
return nil
}

// indirectType returns a code statement that represents the type expression
// for the given type.
func indirectType(typ reflect.Type) *jen.Statement {
switch typ.Kind() {
case reflect.Map:
c := jen.Index(indirectType(typ.Key()))
c.Add(indirectType(typ.Elem()))
return c
case reflect.Ptr:
c := jen.Op("*")
c.Add(indirectType(typ.Elem()))
return c
case reflect.Array, reflect.Slice:
c := jen.Index()
c.Add(indirectType(typ.Elem()))
return c
default:
return jen.Qual(typ.PkgPath(), typ.Name())
}
}

// constructValue returns a Code value for the given value.
func constructValue(v reflect.Value) (jen.Code, error) {
switch v.Kind() {
case reflect.Array:
s := indirectType(v.Type())
values := make([]jen.Code, v.Len())
for i := 0; i < v.Len(); i++ {
val, err := constructValue(v.Index(i))
if err != nil {
return nil, err
}
values[i] = val
}
s.Values(values...)
return s, nil
case reflect.Slice:
if v.IsNil() {
return jen.Nil(), nil
}
s := indirectType(v.Type())
values := make([]jen.Code, v.Len())
for i := 0; i < v.Len(); i++ {
val, err := constructValue(v.Index(i))
if err != nil {
return nil, err
}
values[i] = val
}
s.Values(values...)
return s, nil
case reflect.Interface:
if v.IsNil() {
return jen.Nil(), nil
}
return constructValue(v.Elem())
case reflect.Ptr:
if v.IsNil() {
return jen.Nil(), nil
}
s := jen.Op("&")
val, err := constructValue(reflect.Indirect(v))
if err != nil {
return nil, err
}
return s.Add(val), nil
case reflect.Map:
if v.IsNil() {
return jen.Nil(), nil
}
s := indirectType(v.Type())
keys := v.MapKeys()
values := make(jen.Dict, v.Len())
for _, k := range keys {
key, err := constructValue(k)
if err != nil {
return nil, err
}
val, err := constructValue(v.MapIndex(k))
if err != nil {
return nil, err
}
values[key] = val
}
s.Values(values)
return s, nil
case reflect.Struct:
switch v.Type().Name() {
case "DateTimeLiteral":
lit := v.Interface().(ast.DateTimeLiteral)
fmtTime := lit.Value.Format(time.RFC3339Nano)
return constructStructValue(v, map[string]*jen.Statement{
"Value": jen.Qual("github.com/influxdata/flux/internal/parser", "MustParseTime").Call(jen.Lit(fmtTime)),
})
case "RegexpLiteral":
lit := v.Interface().(ast.RegexpLiteral)
regexString := lit.Value.String()
return constructStructValue(v, map[string]*jen.Statement{
"Value": jen.Qual("regexp", "MustCompile").Call(jen.Lit(regexString)),
})
}
return constructStructValue(v, nil)
case reflect.Bool,
reflect.Int,
reflect.Int8,
reflect.Int16,
reflect.Int32,
reflect.Int64,
reflect.Uint,
reflect.Uint8,
reflect.Uint16,
reflect.Uint32,
reflect.Uint64,
reflect.Uintptr,
reflect.Float32,
reflect.Float64,
reflect.Complex64,
reflect.Complex128,
reflect.String:
typ := types[v.Kind()]
cv := v.Convert(typ)
return jen.Lit(cv.Interface()), nil
default:
return nil, fmt.Errorf("unsupport value kind %v", v.Kind())
}
}

func constructStructValue(v reflect.Value, replace map[string]*jen.Statement) (*jen.Statement, error) {
typ := v.Type()
s := indirectType(typ)
values := make(jen.Dict, v.NumField())
for i := 0; i < v.NumField(); i++ {
field := v.Field(i)
name := typ.Field(i).Name
if !field.CanInterface() {
// Ignore private fields
continue
}
if s, ok := replace[name]; ok {
values[jen.Id(name)] = s
continue
}
val, err := constructValue(field)
if err != nil {
return nil, err
}
values[jen.Id(name)] = val
}
return s.Values(values), nil
}

// types is map of reflect.Kind to reflect.Type for the primitive types
var types = map[reflect.Kind]reflect.Type{
reflect.Bool: reflect.TypeOf(false),
reflect.Int: reflect.TypeOf(int(0)),
reflect.Int8: reflect.TypeOf(int8(0)),
reflect.Int16: reflect.TypeOf(int16(0)),
reflect.Int32: reflect.TypeOf(int32(0)),
reflect.Int64: reflect.TypeOf(int64(0)),
reflect.Uint: reflect.TypeOf(uint(0)),
reflect.Uint8: reflect.TypeOf(uint8(0)),
reflect.Uint16: reflect.TypeOf(uint16(0)),
reflect.Uint32: reflect.TypeOf(uint32(0)),
reflect.Uint64: reflect.TypeOf(uint64(0)),
reflect.Uintptr: reflect.TypeOf(uintptr(0)),
reflect.Float32: reflect.TypeOf(float32(0)),
reflect.Float64: reflect.TypeOf(float64(0)),
reflect.Complex64: reflect.TypeOf(complex64(0)),
reflect.Complex128: reflect.TypeOf(complex128(0)),
reflect.String: reflect.TypeOf(""),
}
Loading

0 comments on commit 8505b7e

Please sign in to comment.