diff --git a/ast/edit/option_editor_test.go b/ast/edit/option_editor_test.go index 946379027e..aa1695a03d 100644 --- a/ast/edit/option_editor_test.go +++ b/ast/edit/option_editor_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/influxdata/flux/ast" + "github.com/influxdata/flux/ast/astutil" "github.com/influxdata/flux/ast/edit" "github.com/influxdata/flux/parser" "github.com/influxdata/flux/values" @@ -21,7 +22,7 @@ func TestEditor(t *testing.T) { { name: "no_option", in: `from(bucket: "test") - |> range(start: 2018-05-23T13:09:22.885021542Z)`, + |> range(start: 2018-05-23T13:09:22.885021542Z)`, unchanged: true, edit: func(node ast.Node) (bool, error) { return edit.Option(node, "from", nil) @@ -58,20 +59,20 @@ option bar = 42`, { name: "updates_object", in: `option foo = 1 -option task = { - name: "bar", - every: 1m, - delay: 1m, - cron: "20 * * *", - retry: 5, + option task = { + name: "bar", + every: 1m, + delay: 1m, + cron: "20 * * *", + retry: 5, }`, edited: `option foo = 1 option task = { - name: "bar", - every: 2h3m10s, - delay: 42m, - cron: "buz", - retry: 10, + name: "bar", + every: 2h3m10s, + delay: 42m, + cron: "buz", + retry: 10, }`, edit: func(node ast.Node) (bool, error) { every, err := parser.ParseDuration("2h3m10s") @@ -94,19 +95,19 @@ option task = { name: "error_key_not_found", in: `option foo = 1 option task = { - name: "bar", - every: 1m, - delay: 1m, - cron: "20 * * *", - retry: 5, + name: "bar", + every: 1m, + delay: 1m, + cron: "20 * * *", + retry: 5, }`, edited: `option foo = 1 option task = { - name: "bar", - every: 2h, - delay: 1m, - cron: "20 * * *", - retry: 5, - foo: "foo", + name: "bar", + every: 2h, + delay: 1m, + cron: "20 * * *", + retry: 5, + foo: "foo", }`, errorWanted: false, edit: func(node ast.Node) (bool, error) { @@ -211,10 +212,9 @@ option task = { }, }, { - name: "sets_option_to_function", - in: `option now = "edit me"`, - edited: `option now = () => - (2018-12-03T20:52:48.464942Z)`, + name: "sets_option_to_function", + in: `option now = "edit me"`, + edited: `option now = () => 2018-12-03T20:52:48.464942Z`, edit: func(node ast.Node) (bool, error) { t, err := values.ParseTime("2018-12-03T20:52:48.464942000Z") if err != nil { @@ -256,7 +256,10 @@ option task = { t.Fatal("unexpected option edit") } - out := ast.Format(p.Files[0]) + out, err := astutil.Format(p.Files[0]) + if err != nil { + t.Fatalf("got unexpected error from formatter: %s", err) + } if out != tc.edited { t.Errorf("\nexpected:\n%s\nedited:\n%s\n", tc.edited, out) diff --git a/ast/edit/testcase_test.go b/ast/edit/testcase_test.go index 3facb07fda..f5954457d9 100644 --- a/ast/edit/testcase_test.go +++ b/ast/edit/testcase_test.go @@ -11,6 +11,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/influxdata/flux/ast" "github.com/influxdata/flux/ast/asttest" + "github.com/influxdata/flux/ast/astutil" "github.com/influxdata/flux/ast/edit" "github.com/influxdata/flux/dependencies/filesystem" "github.com/influxdata/flux/parser" @@ -126,7 +127,11 @@ testcase b extends "flux/a/a_test" { files := make([]string, len(pkgs[0].Files)) for i, file := range pkgs[0].Files { - files[i] = ast.Format(file) + fileStr, err := astutil.Format(file) + if err != nil { + t.Fatalf("unexpected error from formatter: %s", err) + } + files[i] = fileStr } want := []string{ @@ -135,16 +140,16 @@ testcase b extends "flux/a/a_test" { import "testing/assert" -a_test = () => { - want = 4 - a = () => { - assert.equal(want: want, got: 2 + 2) +a_test = (() => { + want = 4 + a = () => { + assert.equal(want: want, got: 2 + 2) - return {} - } + return {} + } - return {want, a} -}()`, + return {want, a} +})()`, `package main diff --git a/ast/format.go b/ast/format.go deleted file mode 100644 index 60d97e230b..0000000000 --- a/ast/format.go +++ /dev/null @@ -1,817 +0,0 @@ -package ast - -import ( - "fmt" - "strconv" - "strings" - "time" -) - -//Returns a valid script for a given AST rooted at node `n`. -//Formatting rules: -// - In a list of statements, if two statements are of a different type -// (e.g. an `OptionStatement` followed by an `ExpressionStatement`), they are separated by a double newline. -// - In a function call (or object definition), if the arguments (or properties) are more than 3, -// they are split into multiple lines. -func Format(n Node) string { - f := &formatter{new(strings.Builder), 0} - f.formatNode(n) - return f.get() -} - -type formatter struct { - *strings.Builder - indentation int -} - -func (f *formatter) get() string { - return f.String() -} - -func (f *formatter) writeString(s string) { - // `strings.Builder`'s methods never return a non-nil error, - // so it is safe to ignore it. - f.WriteString(s) -} - -func (f *formatter) writeRune(r rune) { - f.WriteRune(r) -} - -func (f *formatter) writeIndent() { - for i := 0; i < f.indentation; i++ { - f.writeRune('\t') - } -} - -func (f *formatter) indent() { - f.indentation++ -} - -func (f *formatter) unIndent() { - f.indentation-- -} - -func (f *formatter) setIndent(i int) { - f.indentation = i -} -func (f *formatter) writeComment(comment string) { - f.writeString("// ") - f.writeString(comment) - f.writeRune('\n') -} - -// Logic for handling operator precedence and parenthesis formatting. - -const ( - functionCall = 1 - member = 2 - index = 3 - - // Use offsets for operators and logical operators to ensure they are unique keys - // in the map of operators precedence. - opOffset = 100 - lopOffset = 1000 -) - -func getIntForOp(op OperatorKind) int { - return int(op) + opOffset -} - -func getIntForLOp(op LogicalOperatorKind) int { - return int(op) + lopOffset -} - -func getPrecedence(key int) int { - return opPrecedence[key] -} - -func getPrecedenceForOp(op OperatorKind) int { - return getPrecedence(getIntForOp(op)) -} - -func getPrecedenceForLOp(op LogicalOperatorKind) int { - return getPrecedence(getIntForLOp(op)) -} - -// this matches the SPEC -var opPrecedence = map[int]int{ - functionCall: 1, - member: 1, - index: 1, - // these are OperatorKinds - getIntForOp(PowerOperator): 2, - getIntForOp(MultiplicationOperator): 3, - getIntForOp(DivisionOperator): 3, - getIntForOp(ModuloOperator): 3, - getIntForOp(AdditionOperator): 4, - getIntForOp(SubtractionOperator): 4, - getIntForOp(LessThanEqualOperator): 5, - getIntForOp(LessThanOperator): 5, - getIntForOp(GreaterThanEqualOperator): 5, - getIntForOp(GreaterThanOperator): 5, - getIntForOp(StartsWithOperator): 5, - getIntForOp(InOperator): 5, - getIntForOp(NotEmptyOperator): 5, - getIntForOp(EmptyOperator): 5, - getIntForOp(EqualOperator): 5, - getIntForOp(NotEqualOperator): 5, - getIntForOp(RegexpMatchOperator): 5, - getIntForOp(NotRegexpMatchOperator): 5, - getIntForOp(NotOperator): 6, - getIntForOp(ExistsOperator): 6, - // theses are LogicalOperatorKinds: - getIntForLOp(AndOperator): 7, - getIntForLOp(OrOperator): 8, -} - -// formatChildWithParens applies the generic rule for parenthesis (not for binary expressions). -func (f *formatter) formatChildWithParens(parent, child Node) { - f.formatLeftChildWithParens(parent, child) -} - -// formatLeftChildWithParens applies the generic rule for parenthesis to the left child of a binary expression. -func (f *formatter) formatLeftChildWithParens(parent, child Node) { - pvp, pvc := getPrecedences(parent, child) - if needsParenthesis(pvp, pvc, false) { - f.formatNodeWithParens(child) - } else { - f.formatNode(child) - } -} - -// formatRightChildWithParens applies the generic rule for parenthesis to the right child of a binary expression. -func (f *formatter) formatRightChildWithParens(parent, child Node) { - pvp, pvc := getPrecedences(parent, child) - if needsParenthesis(pvp, pvc, true) { - f.formatNodeWithParens(child) - } else { - f.formatNode(child) - } -} - -func getPrecedences(parent, child Node) (int, int) { - var pvp, pvc int - switch parent := parent.(type) { - case *BinaryExpression: - pvp = getPrecedenceForOp(parent.Operator) - case *LogicalExpression: - pvp = getPrecedenceForLOp(parent.Operator) - case *UnaryExpression: - pvp = getPrecedenceForOp(parent.Operator) - case *CallExpression: - pvp = getPrecedence(functionCall) - case *MemberExpression: - pvp = getPrecedence(member) - case *IndexExpression: - pvp = getPrecedence(index) - case *ParenExpression: - return getPrecedences(parent.Expression, child) - } - - switch child := child.(type) { - case *BinaryExpression: - pvc = getPrecedenceForOp(child.Operator) - case *LogicalExpression: - pvc = getPrecedenceForLOp(child.Operator) - case *UnaryExpression: - pvc = getPrecedenceForOp(child.Operator) - case *CallExpression: - pvc = getPrecedence(functionCall) - case *MemberExpression: - pvc = getPrecedence(member) - case *IndexExpression: - pvc = getPrecedence(index) - case *ParenExpression: - return getPrecedences(parent, child.Expression) - } - - return pvp, pvc -} - -// About parenthesis: -// We need parenthesis if a child node has lower precedence (bigger value) than its parent node. -// The same stands for the left child of a binary expression; while, for the right child, we need parenthesis if its -// precedence is lower or equal then its parent's. -// -// To explain parenthesis logic, we must to understand how the parser generates the AST. -// (A) - The parser always puts lower precedence operators at the root of the AST. -// (B) - When there are multiple operators with the same precedence, the right-most expression is at root. -// (C) - When there are parenthesis, instead, the parser recursively generates a AST for the expression contained -// in the parenthesis, and makes it the right child. -// So, when formatting: -// - if we encounter a child with lower precedence on the left, this means it requires parenthesis, because, for sure, -// the parser detected parenthesis to break (A); -// - if we encounter a child with higher or equal precedence on the left, it doesn't need parenthesis, because -// that was the natural parsing order of elements (see (B)); -// - if we encounter a child with lower or equal precedence on the right, it requires parenthesis, otherwise, it -// would have been at root (see (C)). -func needsParenthesis(pvp, pvc int, isRight bool) bool { - // If one of the precedence values is invalid, then we shouldn't apply any parenthesis. - par := !(pvc == 0 || pvp == 0) - par = par && ((!isRight && pvc > pvp) || (isRight && pvc >= pvp)) - return par -} - -func (f *formatter) formatNodeWithParens(node Node) { - f.writeRune('(') - f.formatNode(node) - f.writeRune(')') -} - -func (f *formatter) formatPackage(n *Package) { - f.formatPackageClause(&PackageClause{ - Name: &Identifier{Name: n.Package}, - }) - for i, file := range n.Files { - if i != 0 { - f.writeRune('\n') - f.writeRune('\n') - } - if len(file.Name) > 0 { - f.writeComment(file.Name) - } - f.formatFile(file, false) - } -} - -func (f *formatter) formatFile(n *File, includePkg bool) { - sep := '\n' - - if includePkg && n.Package != nil && n.Package.Name != nil && n.Package.Name.Name != "" { - f.writeIndent() - f.formatNode(n.Package) - - if len(n.Imports) > 0 || len(n.Body) > 0 { - f.writeRune(sep) - f.writeRune(sep) - } - } - - for i, imp := range n.Imports { - if i != 0 { - f.writeRune(sep) - } - - f.writeIndent() - f.formatNode(imp) - } - - if len(n.Imports) > 0 && len(n.Body) > 0 { - f.writeRune(sep) - f.writeRune(sep) - } - - for i, c := range n.Body { - if i != 0 { - f.writeRune(sep) - - // separate different statements with double newline - if n.Body[i-1].Type() != n.Body[i].Type() { - f.writeRune(sep) - } - } - - f.writeIndent() - f.formatNode(c) - } -} - -func (f *formatter) formatBlock(n *Block) { - f.writeRune('{') - - sep := '\n' - if len(n.Body) > 0 { - f.indent() - } - - for i, c := range n.Body { - f.writeRune(sep) - - if i != 0 { - // separate different statements with double newline - if n.Body[i-1].Type() != n.Body[i].Type() { - f.writeRune(sep) - } - } - - f.writeIndent() - f.formatNode(c) - } - - if len(n.Body) > 0 { - f.writeRune(sep) - f.unIndent() - f.writeIndent() - } - - f.writeRune('}') -} - -func (f *formatter) formatPackageClause(n *PackageClause) { - f.writeString("package ") - f.formatNode(n.Name) - f.writeRune('\n') -} - -func (f *formatter) formatImportDeclaration(n *ImportDeclaration) { - f.writeString("import ") - - if n.As != nil && len(n.As.Name) > 0 { - f.formatNode(n.As) - f.writeRune(' ') - } - - f.formatNode(n.Path) -} - -func (f *formatter) formatExpressionStatement(n *ExpressionStatement) { - f.formatNode(n.Expression) -} - -func (f *formatter) formatReturnStatement(n *ReturnStatement) { - f.writeString("return ") - f.formatNode(n.Argument) -} - -func (f *formatter) formatOptionStatement(n *OptionStatement) { - f.writeString("option ") - f.formatNode(n.Assignment) -} - -func (f *formatter) formatTestStatement(n *TestStatement) { - f.writeString("test ") - f.formatNode(n.Assignment) -} - -func (f *formatter) formatTestCaseStatement(n *TestCaseStatement) { - f.writeString("testcase ") - f.formatNode(n.ID) - if n.Extends != nil { - f.writeString(" extends ") - f.formatNode(n.Extends) - } - f.writeRune(' ') - f.formatNode(n.Block) -} - -func (f *formatter) formatVariableAssignment(n *VariableAssignment) { - f.formatNode(n.ID) - f.writeString(" = ") - f.formatNode(n.Init) -} - -func (f *formatter) formatMemberAssignment(n *MemberAssignment) { - f.formatNode(n.Member) - f.writeString(" = ") - f.formatNode(n.Init) -} - -func (f *formatter) formatArrayExpression(n *ArrayExpression) { - f.writeRune('[') - - sep := ", " - for i, c := range n.Elements { - if i != 0 { - f.writeString(sep) - } - - f.formatNode(c) - } - - f.writeRune(']') -} - -func (f *formatter) formatDictExpression(n *DictExpression) { - multiline := len(n.Elements) > 3 - - f.writeRune('[') - if multiline { - f.writeString("\n") - f.indent() - } - - sep := ", " - if multiline { - sep = ",\n" - } - - if len(n.Elements) > 0 { - for _, elem := range n.Elements { - f.formatNode(elem.Key) - f.writeString(": ") - f.formatNode(elem.Val) - f.writeString(sep) - } - } else { - f.writeString(":") - } - - if multiline { - f.unIndent() - } - f.writeRune(']') -} - -func (f *formatter) formatFunctionExpression(n *FunctionExpression) { - f.writeRune('(') - - sep := ", " - for i, c := range n.Params { - if i != 0 { - f.writeString(sep) - } - - // treat properties differently than in general case - f.formatFunctionArgument(c) - } - - f.writeString(") =>") - - // must wrap body with parenthesis in order to discriminate between: - // - returning an object: (x) => ({foo: x}) - // - and block statements: - // (x) => { - // return x + 1 - // } - _, block := n.Body.(*Block) - if !block { - f.writeRune('\n') - f.indent() - f.writeIndent() - f.writeRune('(') - } else { - f.writeRune(' ') - } - - f.formatNode(n.Body) - if !block { - f.writeRune(')') - } -} - -func (f *formatter) formatUnaryExpression(n *UnaryExpression) { - f.writeString(n.Operator.String()) - if n.Operator != SubtractionOperator && - n.Operator != AdditionOperator { - f.WriteRune(' ') - } - f.formatChildWithParens(n, n.Argument) -} - -func (f *formatter) formatBinaryExpression(n *BinaryExpression) { - f.formatBinary(n.Operator.String(), n, n.Left, n.Right) -} - -func (f *formatter) formatLogicalExpression(n *LogicalExpression) { - f.formatBinary(n.Operator.String(), n, n.Left, n.Right) -} - -func (f *formatter) formatBinary(op string, parent, left, right Node) { - f.formatLeftChildWithParens(parent, left) - f.writeRune(' ') - f.writeString(op) - f.writeRune(' ') - f.formatRightChildWithParens(parent, right) -} - -func (f *formatter) formatCallExpression(n *CallExpression) { - f.formatChildWithParens(n, n.Callee) - f.writeRune('(') - - sep := ", " - for i, c := range n.Arguments { - if i != 0 { - f.writeString(sep) - } - - // treat ObjectExpression as argument in a special way - // (an object as argument doesn't need braces) - if oe, ok := c.(*ObjectExpression); ok { - f.formatObjectExpressionAsFunctionArgument(oe) - } else { - f.formatNode(c) - } - } - - f.writeRune(')') -} - -func (f *formatter) formatPipeExpression(n *PipeExpression) { - f.formatNode(n.Argument) - f.writeRune('\n') - f.indent() - f.writeIndent() - f.writeString("|> ") - f.formatNode(n.Call) -} - -func (f *formatter) formatConditionalExpression(n *ConditionalExpression) { - f.writeString("if ") - f.formatNode(n.Test) - f.writeString(" then ") - f.formatNode(n.Consequent) - f.writeString(" else ") - f.formatNode(n.Alternate) -} - -func (f *formatter) formatMemberExpression(n *MemberExpression) { - f.formatChildWithParens(n, n.Object) - - if _, ok := n.Property.(*StringLiteral); ok { - f.writeRune('[') - f.formatNode(n.Property) - f.writeRune(']') - } else { - f.writeRune('.') - f.formatNode(n.Property) - } -} - -func (f *formatter) formatIndexExpression(n *IndexExpression) { - f.formatChildWithParens(n, n.Array) - f.writeRune('[') - f.formatNode(n.Index) - f.writeRune(']') -} - -func (f *formatter) formatObjectExpression(n *ObjectExpression) { - f.formatObjectExpressionBraces(n, true) -} - -func (f *formatter) formatObjectExpressionAsFunctionArgument(n *ObjectExpression) { - // not called from formatNode, need to save indentation - i := f.indentation - f.formatObjectExpressionBraces(n, false) - f.setIndent(i) -} - -func (f *formatter) formatObjectExpressionBraces(n *ObjectExpression, braces bool) { - multiline := len(n.Properties) > 3 - - if braces { - f.writeRune('{') - } - - if n.With != nil { - f.formatIdentifier(n.With) - f.writeString(" with ") - } - - if multiline { - f.writeRune('\n') - f.indent() - f.writeIndent() - } - - var sep string - if multiline { - sep = ",\n" - } else { - sep = ", " - } - - for i, c := range n.Properties { - if i != 0 { - f.writeString(sep) - - if multiline { - f.writeIndent() - } - } - - f.formatNode(c) - } - - if multiline { - f.writeString(sep) - f.unIndent() - f.writeIndent() - } - - if braces { - f.writeRune('}') - } -} - -func (f *formatter) formatProperty(n *Property) { - f.formatNode(n.Key) - if n.Value != nil { - f.writeString(": ") - f.formatNode(n.Value) - } -} - -func (f *formatter) formatFunctionArgument(n *Property) { - if n.Value == nil { - f.formatNode(n.Key) - return - } - - f.formatNode(n.Key) - f.writeRune('=') - f.formatNode(n.Value) -} - -func (f *formatter) formatIdentifier(n *Identifier) { - f.writeString(n.Name) -} - -func (f *formatter) formatStringExpression(n *StringExpression) { - f.writeRune('"') - for _, p := range n.Parts { - f.formatStringExpressionPart(p) - } - f.writeRune('"') -} - -func (f *formatter) formatStringExpressionPart(n StringExpressionPart) { - switch p := n.(type) { - case *TextPart: - f.formatTextPart(p) - case *InterpolatedPart: - f.formatInterpolatedPart(p) - } -} - -func (f *formatter) formatTextPart(n *TextPart) { - f.writeString(escapeStr(n.Value)) -} - -func (f *formatter) formatInterpolatedPart(n *InterpolatedPart) { - f.writeString("${") - f.formatNode(n.Expression) - f.writeString("}") -} - -func (f *formatter) formatParenExpression(n *ParenExpression) { - f.formatNode(n.Expression) -} - -func (f *formatter) formatStringLiteral(n *StringLiteral) { - if n.Loc != nil && n.Loc.Source != "" { - // Preserve the exact literal if we have it - f.writeString(n.Loc.Source) - return - } - // Write out escaped string value - f.writeRune('"') - f.writeString(escapeStr(n.Value)) - f.writeRune('"') -} - -func escapeStr(s string) string { - if !strings.ContainsAny(s, `"\`) { - return s - } - var builder strings.Builder - // Allocate for worst case where every rune needs to be escaped. - builder.Grow(len(s) * 2) - for _, r := range s { - switch r { - case '"', '\\': - builder.WriteRune('\\') - } - builder.WriteRune(r) - } - return builder.String() -} - -func (f *formatter) formatBooleanLiteral(n *BooleanLiteral) { - f.writeString(strconv.FormatBool(n.Value)) -} - -func (f *formatter) formatDateTimeLiteral(n *DateTimeLiteral) { - f.writeString(n.Value.Format(time.RFC3339Nano)) -} - -func (f *formatter) formatDurationLiteral(n *DurationLiteral) { - formatDuration := func(d Duration) { - f.writeString(strconv.FormatInt(d.Magnitude, 10)) - f.writeString(d.Unit) - } - - for _, d := range n.Values { - formatDuration(d) - } -} - -func (f *formatter) formatFloatLiteral(n *FloatLiteral) { - sf := strconv.FormatFloat(n.Value, 'f', -1, 64) - - if !strings.Contains(sf, ".") { - sf += ".0" // force to make it a float - } - - f.writeString(sf) -} - -func (f *formatter) formatIntegerLiteral(n *IntegerLiteral) { - f.writeString(strconv.FormatInt(n.Value, 10)) -} - -func (f *formatter) formatUnsignedIntegerLiteral(n *UnsignedIntegerLiteral) { - f.writeString(strconv.FormatUint(n.Value, 10)) -} - -func (f *formatter) formatPipeLiteral(_ *PipeLiteral) { - f.writeString("<-") -} - -func (f *formatter) formatRegexpLiteral(n *RegexpLiteral) { - f.writeRune('/') - f.writeString(strings.Replace(n.Value.String(), "/", "\\/", -1)) - f.writeRune('/') -} - -func (f *formatter) formatNode(n Node) { - //save current indentation - currInd := f.indentation - - switch n := n.(type) { - case *Package: - f.formatPackage(n) - case *File: - f.formatFile(n, true) - case *Block: - f.formatBlock(n) - case *PackageClause: - f.formatPackageClause(n) - case *ImportDeclaration: - f.formatImportDeclaration(n) - case *OptionStatement: - f.formatOptionStatement(n) - case *TestStatement: - f.formatTestStatement(n) - case *TestCaseStatement: - f.formatTestCaseStatement(n) - case *ExpressionStatement: - f.formatExpressionStatement(n) - case *ReturnStatement: - f.formatReturnStatement(n) - case *VariableAssignment: - f.formatVariableAssignment(n) - case *MemberAssignment: - f.formatMemberAssignment(n) - case *CallExpression: - f.formatCallExpression(n) - case *PipeExpression: - f.formatPipeExpression(n) - case *MemberExpression: - f.formatMemberExpression(n) - case *IndexExpression: - f.formatIndexExpression(n) - case *BinaryExpression: - f.formatBinaryExpression(n) - case *UnaryExpression: - f.formatUnaryExpression(n) - case *LogicalExpression: - f.formatLogicalExpression(n) - case *ObjectExpression: - f.formatObjectExpression(n) - case *ConditionalExpression: - f.formatConditionalExpression(n) - case *ArrayExpression: - f.formatArrayExpression(n) - case *DictExpression: - f.formatDictExpression(n) - case *Identifier: - f.formatIdentifier(n) - case *PipeLiteral: - f.formatPipeLiteral(n) - case *StringExpression: - f.formatStringExpression(n) - case *TextPart: - f.formatTextPart(n) - case *InterpolatedPart: - f.formatInterpolatedPart(n) - case *ParenExpression: - f.formatParenExpression(n) - case *StringLiteral: - f.formatStringLiteral(n) - case *BooleanLiteral: - f.formatBooleanLiteral(n) - case *FloatLiteral: - f.formatFloatLiteral(n) - case *IntegerLiteral: - f.formatIntegerLiteral(n) - case *UnsignedIntegerLiteral: - f.formatUnsignedIntegerLiteral(n) - case *RegexpLiteral: - f.formatRegexpLiteral(n) - case *DurationLiteral: - f.formatDurationLiteral(n) - case *DateTimeLiteral: - f.formatDateTimeLiteral(n) - case *FunctionExpression: - f.formatFunctionExpression(n) - case *Property: - f.formatProperty(n) - default: - // If we were able not to find the type, than this switch is wrong - panic(fmt.Errorf("unknown type %q", n.Type())) - } - - // reset indentation - f.setIndent(currInd) -} diff --git a/ast/format_test.go b/ast/format_test.go deleted file mode 100644 index 303c906cc5..0000000000 --- a/ast/format_test.go +++ /dev/null @@ -1,543 +0,0 @@ -package ast_test - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/influxdata/flux/ast" - "github.com/influxdata/flux/parser" -) - -type formatTestCase struct { - name string - script string - shouldFail bool -} - -// formatTestHelper tests that a raw script has valid syntax and -// that it has the same value if parsed and then formatted. -func formatTestHelper(t *testing.T, testCases []formatTestCase) { - t.Helper() - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - pkg := parser.ParseSource(tc.script) - if ast.Check(pkg) > 0 { - err := ast.GetError(pkg) - t.Fatalf("source has bad syntax: %s\n%s", err, tc.script) - } - - stringResult := ast.Format(pkg.Files[0]) - - if tc.script != stringResult { - if !tc.shouldFail { - t.Errorf("unexpected output: -want/+got:\n %s", cmp.Diff(tc.script, stringResult)) - } - } - }) - } -} - -func TestFormat_Nodes(t *testing.T) { - testCases := []formatTestCase{ - { - name: "string interpolation", - script: `"a + b = ${a + b}"`, - }, - { - name: "string interpolation with quotes", - script: `"a + b = ${a + b}; \"quoted string\""`, - }, - { - name: "string interpolation wrapped in quotes", - script: `"\"a + b = ${a + b}; c\""`, - }, - { - name: "string interpolation with nested strings", - script: `"\"a + b = ${a + b}; \\\"quoted string\\\"\""`, - }, - { - name: "dict empty", - script: `codes = [:]`, - }, - { - name: "binary_op", - script: `1 + 1 - 2`, - }, - { - name: "binary_op 2", - script: `2 ^ 4`, - }, - { - name: "arrow_fn", - script: `(r) => - (r.user == "user1")`, - }, - { - name: "fn_decl", - script: `add = (a, b) => - (a + b)`, - }, - { - name: "fn_call", - script: `add(a: 1, b: 2)`, - }, - { - name: "object", - script: `{a: 1, b: {c: 11, d: 12}}`, - }, - { - name: "object with", - script: `{foo with a: 1, b: {c: 11, d: 12}}`, - }, - { - name: "implicit key object literal", - script: `{a, b, c}`, - }, - { - name: "object with string literal keys", - script: `{"a": 1, "b": 2}`, - }, - { - name: "object with mixed keys", - script: `{"a": 1, b: 2}`, - }, - { - name: "member ident", - script: `object.property`, - }, - { - name: "member string literal", - script: `object["property"]`, - }, - { - name: "array", - script: `a = [1, 2, 3] - -a[i]`, - }, - { - name: "array_expr", - script: `a[i + 1]`, - }, - { - name: "float", - script: `0.1`, - }, - { - name: "duration", - script: `365d`, - }, - { - name: "duration_multiple", - script: `1d1m1s`, - }, - { - name: "time", - script: `2018-05-22T19:53:00Z`, - }, - { - name: "regexp", - script: `/^\w+@[a-zA-Z_]+?\.[a-zA-Z]{2,3}$/`, - }, - { - name: "regexp_escape", - script: `/^http:\/\/\w+\.com$/`, - }, - { - name: "return", - script: `return 42`, - }, - { - name: "option", - script: `option foo = {a: 1}`, - }, - { - name: "qualified option", - script: `option alert.state = "Warning"`, - }, - { - name: "test statement", - script: `test mean = {want: 0, got: 0}`, - }, - { - name: "conditional", - script: "if a then b else c", - }, - { - name: "conditional with more complex expressions", - script: `if not a or b and c then 2 / (3 * 2) else obj.a(par: "foo")`, - }, - { - name: "nil_value_as_default", - script: `foo = (arg=[]) => - (1)`, - }, - { - name: "non_nil_value_as_default", - script: `foo = (arg=[1, 2]) => - (1)`, - }, - { - name: "block", - script: `foo = () => { - foo(f: 1) - 1 + 1 -}`, - }, - { - name: "string", - script: `"foo"`, - }, - { - name: "string multiline", - script: `"this is -a string -with multiple lines"`, - }, - { - name: "string with escape", - script: `"foo \\ \" \r\n"`, - }, - { - name: "string with byte value", - script: `"\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"`, - }, - { - name: "package", - script: "package foo\n", - }, - { - name: "imports", - script: `import "path/foo" -import bar "path/bar"`, - }, - { - name: "no_package", - script: `import foo "path/foo" - -foo.from(bucket: "testdb") - |> range(start: 2018-05-20T19:53:26Z)`, - }, - { - name: "no_import", - script: `package foo - - -from(bucket: "testdb") - |> range(start: 2018-05-20T19:53:26Z)`, - }, - { - name: "package_import", - script: `package foo - - -import "path/foo" -import bar "path/bar" - -from(bucket: "testdb") - |> range(start: 2018-05-20T19:53:26Z)`, - }, - { - name: "simple", - script: `from(bucket: "testdb") - |> range(start: 2018-05-20T19:53:26Z) - |> filter(fn: (r) => - (r.name =~ /.*0/)) - |> group(by: ["_measurement", "_start"]) - |> map(fn: (r) => - ({_time: r._time, io_time: r._value}))`, - }, - { - name: "medium", - script: `from(bucket: "testdb") - |> range(start: 2018-05-20T19:53:26Z) - |> filter(fn: (r) => - (r.name =~ /.*0/)) - |> group(by: ["_measurement", "_start"]) - |> map(fn: (r) => - ({_time: r._time, io_time: r._value}))`, - }, - { - name: "complex", - script: `left = from(bucket: "test") - |> range(start: 2018-05-22T19:53:00Z, stop: 2018-05-22T19:55:00Z) - |> drop(columns: ["_start", "_stop"]) - |> filter(fn: (r) => - (r.user == "user1")) - |> group(by: ["user"]) -right = from(bucket: "test") - |> range(start: 2018-05-22T19:53:00Z, stop: 2018-05-22T19:55:00Z) - |> drop(columns: ["_start", "_stop"]) - |> filter(fn: (r) => - (r.user == "user2")) - |> group(by: ["_measurement"]) - -join(tables: {left: left, right: right}, on: ["_time", "_measurement"])`, - }, - { - name: "option", - script: `option task = { - name: "foo", - every: 1h, - delay: 10m, - cron: "02***", - retry: 5, -} - -from(bucket: "test") - |> range(start: 2018-05-22T19:53:26Z) - |> window(every: task.every) - |> group(by: ["_field", "host"]) - |> sum() - |> to(bucket: "test", tagColumns: ["host", "_field"])`, - }, - { - name: "functions", - script: `foo = () => - (from(bucket: "testdb")) -bar = (x=<-) => - (x - |> filter(fn: (r) => - (r.name =~ /.*0/))) -baz = (y=<-) => - (y - |> map(fn: (r) => - ({_time: r._time, io_time: r._value}))) - -foo() - |> bar() - |> baz()`, - }, - { - name: "multi_indent", - script: `_sortLimit = (n, desc, columns=["_value"], tables=<-) => - (tables - |> sort(columns: columns, desc: desc) - |> limit(n: n)) -_highestOrLowest = (n, _sortLimit, reducer, columns=["_value"], by, tables=<-) => - (tables - |> group(by: by) - |> reducer() - |> group(none: true) - |> _sortLimit(n: n, columns: columns)) -highestAverage = (n, columns=["_value"], by, tables=<-) => - (tables - |> _highestOrLowest( - n: n, - columns: columns, - by: by, - reducer: (tables=<-) => - (tables - |> mean(columns: [columns[0]])), - _sortLimit: top, - ))`, - }, - } - - formatTestHelper(t, testCases) -} - -func TestFormat_Associativity(t *testing.T) { - testCases := []formatTestCase{ - { - name: "math no pars", - script: `a * b + c / d - e * f`, - }, - { - name: "math with pars", - script: `(a * b + c / d - e) * f`, - }, - { - name: "minus before parens", - script: `r._value - (1 * 2 + 4 / 6 - 10)`, - }, - { - name: "plus with unintended parens 1", - script: `(1 + 2) + 3`, - shouldFail: true, - }, - { - name: "plus with unintended parens 2", - script: `1 + (2 + 3)`, - shouldFail: true, - }, - { - name: "minus with unintended parens", - script: `(1 - 2) - 3`, - shouldFail: true, - }, - { - name: "minus with intended parens", - script: `1 - (2 - 3)`, - }, - { - name: "minus no parens", - script: `1 - 2 - 3`, - }, - { - name: "div with parens", - script: `1 / (2 * 3)`, - }, - { - name: "div no parens", - script: `1 / 2 * 3`, - }, - { - name: "div with unintended parens", - script: `(1 / 2) * 3`, - shouldFail: true, - }, - { - name: "math with more pars", - script: `(a * (b + c) / d / e * (f + g) - h) * i * j / (k + l)`, - }, - { - name: "logic", - script: `a or b and c`, - }, - { - name: "logic with pars", - script: `(a or b) and c`, - }, - { - name: "logic with comparison", - script: `a == 0 or b != 1 and c > 2`, - }, - { - name: "logic with comparison with pars", - script: `(a == 0 or b != 1) and c > 2`, - }, - { - name: "logic and math", - script: `a * b + c * d != 0 or not e == 1 and f == g`, - }, - { - name: "logic and math with pars", - script: `(a * (b + c) * d != 0 or not e == 1) and f == g`, - }, - { - name: "unary", - script: `not b and c`, - }, - { - name: "unary with pars", - script: `not (b and c) and exists d or exists (e and f)`, - }, - { - name: "unary negative duration", - script: `-30s`, - }, - { - name: "unary positive duration", - script: `+30s`, - }, - { - name: "function call with pars", - script: `(a + b * c == 0)(foo: "bar")`, - }, - { - name: "member with pars", - script: `((a + b) * c)._value`, - }, - { - name: "index with pars", - script: `((a - b) / (c + d))[3]`, - }, - { - name: "misc", - script: `foo = (a) => - ((bar or buz)(arg: a + 1) + (a / (b + c))[42]) - -foo(a: (obj1 and obj2 or obj3).idk)`, - }, - } - - formatTestHelper(t, testCases) -} - -func TestFormat_Raw(t *testing.T) { - testCases := []struct { - name string - node ast.Node - script string - }{ - { - name: "string escape", - node: &ast.StringLiteral{ - Value: "foo \\ \" \r\n", - }, - script: "\"foo \\\\ \\\" \r\n\"", - }, - { - name: "package multiple files", - node: &ast.Package{ - Package: "foo", - Files: []*ast.File{ - { - Name: "a.flux", - Package: &ast.PackageClause{ - Name: &ast.Identifier{Name: "foo"}, - }, - Body: []ast.Statement{ - &ast.VariableAssignment{ - ID: &ast.Identifier{Name: "a"}, - Init: &ast.IntegerLiteral{Value: 1}, - }, - }, - }, - { - Name: "b.flux", - Package: &ast.PackageClause{ - Name: &ast.Identifier{Name: "foo"}, - }, - Body: []ast.Statement{ - &ast.VariableAssignment{ - ID: &ast.Identifier{Name: "b"}, - Init: &ast.IntegerLiteral{Value: 2}, - }, - }, - }, - }, - }, - script: `package foo -// a.flux -a = 1 - -// b.flux -b = 2`, - }, - { - name: "package file no name", - node: &ast.Package{ - Package: "foo", - Files: []*ast.File{ - { - Package: &ast.PackageClause{ - Name: &ast.Identifier{Name: "foo"}, - }, - Body: []ast.Statement{ - &ast.VariableAssignment{ - ID: &ast.Identifier{Name: "a"}, - Init: &ast.IntegerLiteral{Value: 1}, - }, - }, - }, - }, - }, - script: `package foo -a = 1`, - }, - } - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - got := ast.Format(tc.node) - if tc.script != got { - t.Errorf("unexpected output: -want/+got:\n %s", cmp.Diff(tc.script, got)) - } - }) - } -} diff --git a/dependencies/influxdb/http.go b/dependencies/influxdb/http.go index 6de1081169..4a236f7256 100644 --- a/dependencies/influxdb/http.go +++ b/dependencies/influxdb/http.go @@ -16,6 +16,7 @@ import ( "github.com/apache/arrow/go/arrow/memory" "github.com/influxdata/flux" "github.com/influxdata/flux/ast" + "github.com/influxdata/flux/ast/astutil" "github.com/influxdata/flux/codes" "github.com/influxdata/flux/csv" "github.com/influxdata/flux/dependencies/http" @@ -260,7 +261,11 @@ func (h *HttpClient) newRequestBody(file *ast.File, now time.Time) ([]byte, erro } `json:"dialect"` Now time.Time `json:"now"` } - req.Query = ast.Format(file) + query, err := astutil.Format(file) + if err != nil { + return nil, err + } + req.Query = query req.Dialect.Header = true req.Dialect.DateTimeFormat = "RFC3339Nano" req.Dialect.Annotations = []string{"group", "datatype", "default"} diff --git a/internal/cmd/test_rewriter/add_measurement.go b/internal/cmd/test_rewriter/add_measurement.go deleted file mode 100644 index c3cd551118..0000000000 --- a/internal/cmd/test_rewriter/add_measurement.go +++ /dev/null @@ -1,158 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "strings" - - "github.com/influxdata/flux" - "github.com/influxdata/flux/ast" - "github.com/influxdata/flux/codes" - "github.com/influxdata/flux/csv" - "github.com/influxdata/flux/internal/errors" - "github.com/spf13/cobra" -) - -var addMeasurementCmd = &cobra.Command{ - Use: "add-measurement [test files...]", - Short: "Update test inData and outData to have a _measurement column", - RunE: addMeasurementE, - Args: cobra.MinimumNArgs(1), -} - -func init() { - rootCmd.AddCommand(addMeasurementCmd) - addMeasurementCmd.Flags().StringVar(&flagMeasurementName, "measurement-name", "m", "value to populate new column") -} - -var ( - flagMeasurementName string -) - -func addMeasurementE(cmd *cobra.Command, args []string) error { - return doSubCommand(addMeasurementColumn, args) -} - -func addMeasurementColumn(fileName string) error { - astPkg, err := getFileAST(fileName) - if err != nil { - return err - } - - df := &dataFinder{ - dataStmts: make(map[string]*ast.VariableAssignment), - } - ast.Walk(df, astPkg) - - if _, ok := df.dataStmts["inData"]; !ok { - fmt.Printf(" No inData; skipping\n") - return nil - } - - for _, a := range df.dataStmts { - lack, err := csvLacksMeasurementColumn(a) - if err != nil { - return err - } - if !lack { - fmt.Printf(" %v has _measurement, skipping\n", a.ID.Name) - return nil - } - } - - var q string - { - var sb strings.Builder - sb.WriteString(` -import "csv" -import "experimental" -`) - for _, a := range df.dataStmts { - sb.WriteString(ast.Format(a) + "\n") - sb.WriteString(fmt.Sprintf(` -csv.from(csv: %v) - |> experimental.set(o: {_measurement: "%v"}) - |> experimental.group(mode: "extend", columns: ["_measurement"]) - |> yield(name: "%v") -`, a.ID.Name, flagMeasurementName, a.ID.Name)) - } - q = sb.String() - } - - ri, err := runQuery(q) - if err != nil { - return err - } - defer ri.Release() - - for ri.More() { - r := ri.Next() - - var bb bytes.Buffer - enc := csv.NewResultEncoder(csv.DefaultEncoderConfig()) - if _, err := enc.Encode(&bb, r); err != nil { - return err - } - if err := replaceStringLitRHS(df.dataStmts[r.Name()], "\n"+bb.String()); err != nil { - return err - } - } - if ri.Err() != nil { - return ri.Err() - } - - if err := rewriteFile(fileName, astPkg); err != nil { - return nil - } - fmt.Printf(" Rewrote %s with measurement columns added.\n", fileName) - return nil -} - -func csvLacksMeasurementColumn(a *ast.VariableAssignment) (bool, error) { - bb := bytes.NewBuffer([]byte(a.Init.(*ast.StringLiteral).Value)) - dec := csv.NewResultDecoder(csv.ResultDecoderConfig{}) - r, err := dec.Decode(bb) - if err != nil { - return false, err - } - lacks := true - if err := r.Tables().Do(func(t flux.Table) error { - for _, c := range t.Cols() { - if c.Label == "_measurement" { - lacks = false - break - } - } - return nil - }); err != nil { - return false, err - } - return lacks, nil -} - -func replaceStringLitRHS(va *ast.VariableAssignment, v string) error { - sl, ok := va.Init.(*ast.StringLiteral) - if !ok { - return errors.New(codes.Invalid, "funky assignment") - } - sl.Value = v - sl.Loc = nil - return nil -} - -type dataFinder struct { - dataStmts map[string]*ast.VariableAssignment -} - -func (d *dataFinder) Visit(node ast.Node) ast.Visitor { - return d -} - -func (d *dataFinder) Done(node ast.Node) { - switch n := node.(type) { - case *ast.VariableAssignment: - if n.ID.Name == "inData" || n.ID.Name == "outData" { - d.dataStmts[n.ID.Name] = n - } - } -} diff --git a/internal/cmd/test_rewriter/add_range.go b/internal/cmd/test_rewriter/add_range.go deleted file mode 100644 index 1d8b2121f9..0000000000 --- a/internal/cmd/test_rewriter/add_range.go +++ /dev/null @@ -1,142 +0,0 @@ -package main - -import ( - "fmt" - "time" - - "github.com/influxdata/flux/ast" - "github.com/spf13/cobra" -) - -var addRangeCmd = &cobra.Command{ - Use: "add-range [test files...]", - Short: "Update tests that lack a call to range()", - RunE: addRangeE, - Args: cobra.MinimumNArgs(1), -} - -func init() { - rootCmd.AddCommand(addRangeCmd) -} - -func addRangeE(cmd *cobra.Command, args []string) error { - return doSubCommand(addRangeCall, args) -} - -func addRangeCall(fileName string) error { - astPkg, err := getFileAST(fileName) - if err != nil { - return err - } - - var rf rangeFinder - ast.Walk(&rf, astPkg) - - if rf.found { - fmt.Printf(" range found, nothing to do.\n") - return nil - } - - // add call to range to the leafiest |> we found - pe := rf.pipeExpr - if pe == nil { - fmt.Println(" no pipe expression found") - return nil - } - - if err := addRangeToPipeline(pe); err != nil { - return err - } - if err := addDropToPipeline(pe); err != nil { - return err - } - - if err := rewriteFile(fileName, astPkg); err != nil { - return nil - } - fmt.Printf(" Rewrote %s with a call to range() added.\n", fileName) - return nil -} - -func addRangeToPipeline(pe *ast.PipeExpression) error { - startTime, err := time.Parse(time.RFC3339, "1980-01-01T00:00:00.000Z") - if err != nil { - return err - } - - oldArg := pe.Argument - pe.Argument = &ast.PipeExpression{ - Argument: oldArg, - Call: &ast.CallExpression{ - Callee: &ast.Identifier{Name: "range"}, - Arguments: []ast.Expression{ - &ast.ObjectExpression{ - Properties: []*ast.Property{ - { - Key: &ast.Identifier{Name: "start"}, - Value: &ast.DateTimeLiteral{Value: startTime}, - }, - }, - }, - }, - }, - } - return nil -} - -func addDropToPipeline(pe *ast.PipeExpression) error { - oldArg := pe.Argument - pe.Argument = &ast.PipeExpression{ - Argument: oldArg, - Call: &ast.CallExpression{ - Callee: &ast.Identifier{Name: "drop"}, - Arguments: []ast.Expression{ - &ast.ObjectExpression{ - Properties: []*ast.Property{ - { - Key: &ast.Identifier{Name: "columns"}, - Value: &ast.ArrayExpression{ - Elements: []ast.Expression{ - &ast.StringLiteral{Value: "_start"}, - &ast.StringLiteral{Value: "_stop"}, - }, - }, - }, - }, - }, - }, - }, - } - return nil -} - -type rangeFinder struct { - found bool - pipeExpr *ast.PipeExpression -} - -func (r *rangeFinder) Visit(node ast.Node) ast.Visitor { - if r.found { - return nil - } - switch n := node.(type) { - case *ast.CallExpression: - if id, ok := n.Callee.(*ast.Identifier); ok { - if id.Name == "range" { - r.found = true - return nil - } - } - } - return r -} - -func (r *rangeFinder) Done(node ast.Node) { - switch n := node.(type) { - case *ast.PipeExpression: - if r.pipeExpr == nil { - r.pipeExpr = n - } - } - -} diff --git a/internal/cmd/test_rewriter/main.go b/internal/cmd/test_rewriter/main.go deleted file mode 100644 index 60915bada9..0000000000 --- a/internal/cmd/test_rewriter/main.go +++ /dev/null @@ -1,103 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io/ioutil" - "os" - - "github.com/influxdata/flux" - "github.com/influxdata/flux/ast" - "github.com/influxdata/flux/codes" - "github.com/influxdata/flux/dependencies/filesystem" - _ "github.com/influxdata/flux/fluxinit/static" - "github.com/influxdata/flux/internal/errors" - "github.com/influxdata/flux/lang" - "github.com/influxdata/flux/memory" - "github.com/influxdata/flux/parser" - "github.com/influxdata/flux/runtime" - "github.com/spf13/cobra" -) - -var rootCmd = &cobra.Command{ - Use: "test_rewriter", - Short: "A tool for fixing common problems with Flux tests by rewriting them in-place.", - Long: "A tool for fixing common problems with Flux tests by rewriting them in-place.", -} - -func init() { - rootCmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "do nothing, but show what would be done") -} - -var ( - flagDryRun = false -) - -func main() { - if err := rootCmd.Execute(); err != nil { - _, _ = fmt.Fprint(os.Stderr, err) - os.Exit(1) - } - os.Exit(0) -} - -func doSubCommand(f func(fileName string) error, args []string) error { - for _, arg := range args { - fmt.Printf("%v:\n", arg) - if err := f(arg); err != nil { - return errors.Wrap(err, codes.Inherit, arg) - } - fmt.Println() - } - return nil -} - -func getFileAST(fileName string) (*ast.Package, error) { - f, err := os.Open(fileName) - if err != nil { - return nil, err - } - defer func() { - _ = f.Close() - }() - script, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - astPkg := parser.ParseSource(string(script)) - return astPkg, nil - -} - -func runQuery(query string) (flux.ResultIterator, error) { - c := lang.FluxCompiler{ - Extern: nil, - Query: query, - } - deps := flux.NewDefaultDependencies() - deps.Deps.FilesystemService = filesystem.SystemFS - - ctx := deps.Inject(context.Background()) - program, err := c.Compile(ctx, runtime.Default) - if err != nil { - return nil, err - } - ctx = deps.Inject(ctx) - alloc := &memory.Allocator{} - qry, err := program.Start(ctx, alloc) - if err != nil { - return nil, err - } - return flux.NewResultIteratorFromQuery(qry), nil -} - -func rewriteFile(fileName string, astPkg *ast.Package) error { - newScript := ast.Format(astPkg) + "\n" - if !flagDryRun { - if err := ioutil.WriteFile(fileName, []byte(newScript), 0644); err != nil { - return err - } - } - return nil -} diff --git a/semantic/ast_test.go b/semantic/ast_test.go index 26d3c30ca9..0e73709110 100644 --- a/semantic/ast_test.go +++ b/semantic/ast_test.go @@ -5,6 +5,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/influxdata/flux/ast" + "github.com/influxdata/flux/ast/astutil" "github.com/influxdata/flux/runtime" "github.com/influxdata/flux/semantic" "github.com/influxdata/flux/semantic/semantictest" @@ -143,7 +144,13 @@ exists r.b t.Fatalf("unexpected error analyzing source: %s", err) } - got, err := runtime.AnalyzeSource(ast.Format(semantic.ToAST(want))) + toAst := semantic.ToAST(want) + fmtdAst, err := astutil.Format(toAst.(*ast.Package).Files[0]) + if err != nil { + t.Fatalf("unexpected error from formatter: %s", err) + } + + got, err := runtime.AnalyzeSource(fmtdAst) if err != nil { t.Fatalf("unexpected error analyzing generated AST: %s", err) } diff --git a/stdlib/influxdata/influxdb/cardinality_test.go b/stdlib/influxdata/influxdb/cardinality_test.go index 7f3036dcf6..53f1d7a209 100644 --- a/stdlib/influxdata/influxdb/cardinality_test.go +++ b/stdlib/influxdata/influxdb/cardinality_test.go @@ -290,14 +290,9 @@ influxdb.cardinality(bucket: "telegraf", start: 2018-05-30T09:00:00Z, stop: 2018 import influxdb "influxdata/influxdb" -influxdb.cardinality( - bucket: "telegraf", - start: 2020-10-22T09:29:00Z, - stop: 2020-10-22T09:30:00Z, - predicate: (r) => { - return r["_value"] >= 0.0 - }, -)`, +influxdb.cardinality(bucket: "telegraf", start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z, predicate: (r) => { + return r["_value"] >= 0.0 +})`, Tables: defaultTablesFn, }, }, @@ -353,14 +348,9 @@ import "math" import influxdb "influxdata/influxdb" import math "math" -influxdb.cardinality( - bucket: "telegraf", - start: 2020-10-22T09:29:00Z, - stop: 2020-10-22T09:30:00Z, - predicate: (r) => { - return r["_value"] >= math["pi"] - }, -)`, +influxdb.cardinality(bucket: "telegraf", start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z, predicate: (r) => { + return r["_value"] >= math["pi"] +})`, Tables: defaultTablesFn, }, }, diff --git a/stdlib/influxdata/influxdb/from_test.go b/stdlib/influxdata/influxdb/from_test.go index 7e458362ab..fc32fe8508 100644 --- a/stdlib/influxdata/influxdb/from_test.go +++ b/stdlib/influxdata/influxdb/from_test.go @@ -180,8 +180,7 @@ func TestFrom_Run(t *testing.T) { Query: `package main -from(bucket: "telegraf") - |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z)`, +from(bucket: "telegraf") |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z)`, Tables: defaultTablesFn, }, }, @@ -211,8 +210,7 @@ from(bucket: "telegraf") Query: `package main -from(bucketID: "1e01ac57da723035") - |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z)`, +from(bucketID: "1e01ac57da723035") |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z)`, Tables: defaultTablesFn, }, }, @@ -241,8 +239,7 @@ from(bucketID: "1e01ac57da723035") Query: `package main -from(bucket: "telegraf") - |> range(start: 2018-05-30T09:00:00Z, stop: 2018-05-30T10:00:00Z)`, +from(bucket: "telegraf") |> range(start: 2018-05-30T09:00:00Z, stop: 2018-05-30T10:00:00Z)`, Tables: defaultTablesFn, }, }, @@ -278,11 +275,9 @@ from(bucket: "telegraf") Query: `package main -from(bucket: "telegraf") - |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z) - |> filter(fn: (r) => { - return r["_value"] >= 0.0 - })`, +from(bucket: "telegraf") |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z) |> filter(fn: (r) => { + return r["_value"] >= 0.0 +})`, Tables: defaultTablesFn, }, }, @@ -319,11 +314,9 @@ from(bucket: "telegraf") Query: `package main -from(bucket: "telegraf") - |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z) - |> filter(fn: (r) => { - return r["_value"] >= 0.0 - }, onEmpty: "keep")`, +from(bucket: "telegraf") |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z) |> filter(fn: (r) => { + return r["_value"] >= 0.0 +}, onEmpty: "keep")`, Tables: defaultTablesFn, }, }, @@ -379,11 +372,9 @@ import "math" import math "math" -from(bucket: "telegraf") - |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z) - |> filter(fn: (r) => { - return r["_value"] >= math["pi"] - }, onEmpty: "keep")`, +from(bucket: "telegraf") |> range(start: 2020-10-22T09:29:00Z, stop: 2020-10-22T09:30:00Z) |> filter(fn: (r) => { + return r["_value"] >= math["pi"] +}, onEmpty: "keep")`, Tables: defaultTablesFn, }, }, diff --git a/stdlib/influxdata/influxdb/source.go b/stdlib/influxdata/influxdb/source.go index fd567db508..9b29e0fccf 100644 --- a/stdlib/influxdata/influxdb/source.go +++ b/stdlib/influxdata/influxdb/source.go @@ -11,6 +11,7 @@ import ( "github.com/influxdata/flux" "github.com/influxdata/flux/ast" + "github.com/influxdata/flux/ast/astutil" "github.com/influxdata/flux/codes" "github.com/influxdata/flux/csv" "github.com/influxdata/flux/dependencies/influxdb" @@ -159,9 +160,14 @@ func (s *source) newRequestBody() ([]byte, error) { Annotations []string `json:"annotations"` } `json:"dialect"` } + var err error // Build the query. This needs to be done first to build // up the list of imports. - req.Query = ast.Format(s.spec.BuildQuery()) + query, err := astutil.Format(s.spec.BuildQuery()) + if err != nil { + return nil, err + } + req.Query = query req.Dialect.Header = true req.Dialect.DateTimeFormat = "RFC3339Nano" req.Dialect.Annotations = []string{"group", "datatype", "default"}