From 96ecf38a908487ef6c5bb70cce71ad7a7e16d563 Mon Sep 17 00:00:00 2001 From: Johan Fylling Date: Tue, 25 Jun 2024 14:31:12 +0200 Subject: [PATCH] trace+tester: Adding local var values to trace and test report (#6815) Fixing: #2546 Signed-off-by: Johan Fylling --- .github/workflows/pull-request.yaml | 2 +- ast/compile.go | 149 ++- ast/compile_test.go | 162 +++ ast/internal/scanner/scanner.go | 17 +- ast/location/location.go | 2 + ast/parser.go | 1 + ast/policy.go | 44 +- cmd/eval.go | 10 +- cmd/eval_test.go | 457 +++++++++ cmd/test.go | 9 +- cmd/test_test.go | 1316 +++++++++++++++++++++++++ docs/content/cli.md | 2 + docs/content/policy-testing.md | 63 ++ internal/presentation/presentation.go | 19 +- internal/strings/strings.go | 8 + tester/reporter.go | 48 +- tester/reporter_test.go | 162 ++- topdown/eval.go | 21 +- topdown/lineage/lineage.go | 2 +- topdown/trace.go | 469 ++++++++- topdown/trace_test.go | 170 ++++ 21 files changed, 3061 insertions(+), 72 deletions(-) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 722032ec9f..1f9d3927a7 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -321,7 +321,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-22.04, macos-14] - version: ["1.20"] + version: ["1.21"] steps: - uses: actions/checkout@v4 - name: Download generated artifacts diff --git a/ast/compile.go b/ast/compile.go index 3010bfe8ba..d54cf93dcb 100644 --- a/ast/compile.go +++ b/ast/compile.go @@ -120,34 +120,35 @@ type Compiler struct { // Capabliities required by the modules that were compiled. Required *Capabilities - localvargen *localVarGenerator - moduleLoader ModuleLoader - ruleIndices *util.HashMap - stages []stage - maxErrs int - sorted []string // list of sorted module names - pathExists func([]string) (bool, error) - after map[string][]CompilerStageDefinition - metrics metrics.Metrics - capabilities *Capabilities // user-supplied capabilities - imports map[string][]*Import // saved imports from stripping - builtins map[string]*Builtin // universe of built-in functions - customBuiltins map[string]*Builtin // user-supplied custom built-in functions (deprecated: use capabilities) - unsafeBuiltinsMap map[string]struct{} // user-supplied set of unsafe built-ins functions to block (deprecated: use capabilities) - deprecatedBuiltinsMap map[string]struct{} // set of deprecated, but not removed, built-in functions - enablePrintStatements bool // indicates if print statements should be elided (default) - comprehensionIndices map[*Term]*ComprehensionIndex // comprehension key index - initialized bool // indicates if init() has been called - debug debug.Debug // emits debug information produced during compilation - schemaSet *SchemaSet // user-supplied schemas for input and data documents - inputType types.Type // global input type retrieved from schema set - annotationSet *AnnotationSet // hierarchical set of annotations - strict bool // enforce strict compilation checks - keepModules bool // whether to keep the unprocessed, parse modules (below) - parsedModules map[string]*Module // parsed, but otherwise unprocessed modules, kept track of when keepModules is true - useTypeCheckAnnotations bool // whether to provide annotated information (schemas) to the type checker - allowUndefinedFuncCalls bool // don't error on calls to unknown functions. - evalMode CompilerEvalMode + localvargen *localVarGenerator + moduleLoader ModuleLoader + ruleIndices *util.HashMap + stages []stage + maxErrs int + sorted []string // list of sorted module names + pathExists func([]string) (bool, error) + after map[string][]CompilerStageDefinition + metrics metrics.Metrics + capabilities *Capabilities // user-supplied capabilities + imports map[string][]*Import // saved imports from stripping + builtins map[string]*Builtin // universe of built-in functions + customBuiltins map[string]*Builtin // user-supplied custom built-in functions (deprecated: use capabilities) + unsafeBuiltinsMap map[string]struct{} // user-supplied set of unsafe built-ins functions to block (deprecated: use capabilities) + deprecatedBuiltinsMap map[string]struct{} // set of deprecated, but not removed, built-in functions + enablePrintStatements bool // indicates if print statements should be elided (default) + comprehensionIndices map[*Term]*ComprehensionIndex // comprehension key index + initialized bool // indicates if init() has been called + debug debug.Debug // emits debug information produced during compilation + schemaSet *SchemaSet // user-supplied schemas for input and data documents + inputType types.Type // global input type retrieved from schema set + annotationSet *AnnotationSet // hierarchical set of annotations + strict bool // enforce strict compilation checks + keepModules bool // whether to keep the unprocessed, parse modules (below) + parsedModules map[string]*Module // parsed, but otherwise unprocessed modules, kept track of when keepModules is true + useTypeCheckAnnotations bool // whether to provide annotated information (schemas) to the type checker + allowUndefinedFuncCalls bool // don't error on calls to unknown functions. + evalMode CompilerEvalMode // + rewriteTestRulesForTracing bool // rewrite test rules to capture dynamic values for tracing. } // CompilerStage defines the interface for stages in the compiler. @@ -346,6 +347,7 @@ func NewCompiler() *Compiler { {"CheckSafetyRuleBodies", "compile_stage_check_safety_rule_bodies", c.checkSafetyRuleBodies}, {"RewriteEquals", "compile_stage_rewrite_equals", c.rewriteEquals}, {"RewriteDynamicTerms", "compile_stage_rewrite_dynamic_terms", c.rewriteDynamicTerms}, + {"RewriteTestRulesForTracing", "compile_stage_rewrite_test_rules_for_tracing", c.rewriteTestRuleEqualities}, // must run after RewriteDynamicTerms {"CheckRecursion", "compile_stage_check_recursion", c.checkRecursion}, {"CheckTypes", "compile_stage_check_types", c.checkTypes}, // must be run after CheckRecursion {"CheckUnsafeBuiltins", "compile_state_check_unsafe_builtins", c.checkUnsafeBuiltins}, @@ -469,6 +471,13 @@ func (c *Compiler) WithEvalMode(e CompilerEvalMode) *Compiler { return c } +// WithRewriteTestRules enables rewriting test rules to capture dynamic values in local variables, +// so they can be accessed by tracing. +func (c *Compiler) WithRewriteTestRules(rewrite bool) *Compiler { + c.rewriteTestRulesForTracing = rewrite + return c +} + // ParsedModules returns the parsed, unprocessed modules from the compiler. // It is `nil` if keeping modules wasn't enabled via `WithKeepModules(true)`. // The map includes all modules loaded via the ModuleLoader, if one was used. @@ -2167,6 +2176,43 @@ func (c *Compiler) rewriteDynamicTerms() { } } +// rewriteTestRuleEqualities rewrites equality expressions in test rule bodies to create local vars for statements that would otherwise +// not have their values captured through tracing, such as refs and comprehensions not unified/assigned to a local var. +// For example, given the following module: +// +// package test +// +// p.q contains v if { +// some v in numbers.range(1, 3) +// } +// +// p.r := "foo" +// +// test_rule { +// p == { +// "q": {4, 5, 6} +// } +// } +// +// `p` in `test_rule` resolves to `data.test.p`, which won't be an entry in the virtual-cache and must therefore be calculated after-the-fact. +// If `p` isn't captured in a local var, there is no trivial way to retrieve its value for test reporting. +func (c *Compiler) rewriteTestRuleEqualities() { + if !c.rewriteTestRulesForTracing { + return + } + + f := newEqualityFactory(c.localvargen) + for _, name := range c.sorted { + mod := c.Modules[name] + WalkRules(mod, func(rule *Rule) bool { + if strings.HasPrefix(string(rule.Head.Name), "test_") { + rule.Body = rewriteTestEqualities(f, rule.Body) + } + return false + }) + } +} + func (c *Compiler) parseMetadataBlocks() { // Only parse annotations if rego.metadata built-ins are called regoMetadataCalled := false @@ -4517,6 +4563,41 @@ func rewriteEquals(x interface{}) (modified bool) { return modified } +func rewriteTestEqualities(f *equalityFactory, body Body) Body { + result := make(Body, 0, len(body)) + for _, expr := range body { + // We can't rewrite negated expressions; if the extracted term is undefined, evaluation would fail before + // reaching the negation check. + if !expr.Negated && !expr.Generated { + switch { + case expr.IsEquality(): + terms := expr.Terms.([]*Term) + result, terms[1] = rewriteDynamicsShallow(expr, f, terms[1], result) + result, terms[2] = rewriteDynamicsShallow(expr, f, terms[2], result) + case expr.IsEvery(): + // We rewrite equalities inside of every-bodies as a fail here will be the cause of the test-rule fail. + // Failures inside other expressions with closures, such as comprehensions, won't cause the test-rule to fail, so we skip those. + every := expr.Terms.(*Every) + every.Body = rewriteTestEqualities(f, every.Body) + } + } + result = appendExpr(result, expr) + } + return result +} + +func rewriteDynamicsShallow(original *Expr, f *equalityFactory, term *Term, result Body) (Body, *Term) { + switch term.Value.(type) { + case Ref, *ArrayComprehension, *SetComprehension, *ObjectComprehension: + generated := f.Generate(term) + generated.With = original.With + result.Append(generated) + connectGeneratedExprs(original, generated) + return result, result[len(result)-1].Operand(0) + } + return result, term +} + // rewriteDynamics will rewrite the body so that dynamic terms (i.e., refs and // comprehensions) are bound to vars earlier in the query. This translation // results in eager evaluation. @@ -4608,6 +4689,7 @@ func rewriteDynamicsOne(original *Expr, f *equalityFactory, term *Term, result B generated := f.Generate(term) generated.With = original.With result.Append(generated) + connectGeneratedExprs(original, generated) return result, result[len(result)-1].Operand(0) case *Array: for i := 0; i < v.Len(); i++ { @@ -4636,16 +4718,19 @@ func rewriteDynamicsOne(original *Expr, f *equalityFactory, term *Term, result B var extra *Expr v.Body, extra = rewriteDynamicsComprehensionBody(original, f, v.Body, term) result.Append(extra) + connectGeneratedExprs(original, extra) return result, result[len(result)-1].Operand(0) case *SetComprehension: var extra *Expr v.Body, extra = rewriteDynamicsComprehensionBody(original, f, v.Body, term) result.Append(extra) + connectGeneratedExprs(original, extra) return result, result[len(result)-1].Operand(0) case *ObjectComprehension: var extra *Expr v.Body, extra = rewriteDynamicsComprehensionBody(original, f, v.Body, term) result.Append(extra) + connectGeneratedExprs(original, extra) return result, result[len(result)-1].Operand(0) } return result, term @@ -4713,6 +4798,7 @@ func expandExpr(gen *localVarGenerator, expr *Expr) (result []*Expr) { for i := 1; i < len(terms); i++ { var extras []*Expr extras, terms[i] = expandExprTerm(gen, terms[i]) + connectGeneratedExprs(expr, extras...) if len(expr.With) > 0 { for i := range extras { extras[i].With = expr.With @@ -4740,6 +4826,13 @@ func expandExpr(gen *localVarGenerator, expr *Expr) (result []*Expr) { return } +func connectGeneratedExprs(parent *Expr, children ...*Expr) { + for _, child := range children { + child.generatedFrom = parent + parent.generates = append(parent.generates, child) + } +} + func expandExprTerm(gen *localVarGenerator, term *Term) (support []*Expr, output *Term) { output = term switch v := term.Value.(type) { diff --git a/ast/compile_test.go b/ast/compile_test.go index 069f4149b6..0f87795305 100644 --- a/ast/compile_test.go +++ b/ast/compile_test.go @@ -10679,3 +10679,165 @@ deny { t.Fatal(c.Errors) } } + +func TestCompilerRewriteTestRulesForTracing(t *testing.T) { + tests := []struct { + note string + rewrite bool + module string + exp string + }{ + { + note: "ref comparison, no rewrite", + module: `package test +import rego.v1 + +a := 1 +b := 2 + +test_something if { + a == b +}`, + exp: `package test + +a := 1 { true } +b := 2 { true } + +test_something = true { + data.test.a = data.test.b +}`, + }, + { + note: "ref comparison, rewrite", + rewrite: true, + module: `package test +import rego.v1 + +a := 1 +b := 2 + +test_something if { + a == b +}`, + // When the test fails on '__local0__ = __local1__', the values for 'a' and 'b' are captured in local bindings, + // accessible by the tracer. + exp: `package test + +a := 1 { true } +b := 2 { true } + +test_something = true { + __local0__ = data.test.a + __local1__ = data.test.b + __local0__ = __local1__ +}`, + }, + { + note: "ref comparison, not-stmt, rewrite", + rewrite: true, + module: `package test +import rego.v1 + +a := 1 +b := 2 + +test_something if { + not a == b +}`, + // We don't break out local vars from a not-stmt, as that would change the semantics of the rule. + exp: `package test + +a := 1 { true } +b := 2 { true } + +test_something = true { + not data.test.a = data.test.b +}`, + }, + { + note: "ref comparison, inside every-stmt, no rewrite", + module: `package test +import rego.v1 + +a := 1 +b := 2 +l := [1, 2, 3] + +test_something if { + every x in l { + a < b + x + } +}`, + exp: `package test +import future.keywords + +a := 1 { true } +b := 2 { true } +l := [1, 2, 3] { true } + +test_something = true { + __local2__ = data.test.l + every __local0__, __local1__ in __local2__ { + __local4__ = data.test.b + plus(__local4__, __local1__, __local3__) + __local5__ = data.test.a + lt(__local5__, __local3__) + } +}`, + }, + { + note: "ref comparison, inside every-stmt, rewrite", + rewrite: true, + module: `package test +import rego.v1 + +a := 1 +b := 2 +l := [1, 2, 3] + +test_something if { + every x in l { + a < b + x + } +}`, + // When tests contain an 'every' statement, we're interested in the circumstances that made the every fail, + // so it's body is rewritten. + exp: `package test +import future.keywords + +a := 1 { true } +b := 2 { true } +l := [1, 2, 3] { true } + +test_something = true { + __local2__ = data.test.l; + every __local0__, __local1__ in __local2__ { + __local4__ = data.test.b + plus(__local4__, __local1__, __local3__) + __local5__ = data.test.a + lt(__local5__, __local3__) + } +}`, + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + ms := map[string]string{ + "test.rego": tc.module, + } + c := getCompilerWithParsedModules(ms). + WithRewriteTestRules(tc.rewrite) + + compileStages(c, c.rewriteTestRuleEqualities) + assertNotFailed(t, c) + + result := c.Modules["test.rego"] + exp := MustParseModule(tc.exp) + exp.Imports = nil // We strip the imports since the compiler will too + if result.Compare(exp) != 0 { + t.Fatalf("\nExpected:\n\n%v\n\nGot:\n\n%v", exp, result) + } + }) + } +} diff --git a/ast/internal/scanner/scanner.go b/ast/internal/scanner/scanner.go index 97ee8bde01..a0200ac18d 100644 --- a/ast/internal/scanner/scanner.go +++ b/ast/internal/scanner/scanner.go @@ -26,6 +26,7 @@ type Scanner struct { width int errors []Error keywords map[string]tokens.Token + tabs []int regoV1Compatible bool } @@ -37,10 +38,11 @@ type Error struct { // Position represents a point in the scanned source code. type Position struct { - Offset int // start offset in bytes - End int // end offset in bytes - Row int // line number computed in bytes - Col int // column number computed in bytes + Offset int // start offset in bytes + End int // end offset in bytes + Row int // line number computed in bytes + Col int // column number computed in bytes + Tabs []int // positions of any tabs preceding Col } // New returns an initialized scanner that will scan @@ -60,6 +62,7 @@ func New(r io.Reader) (*Scanner, error) { curr: -1, width: 0, keywords: tokens.Keywords(), + tabs: []int{}, } s.next() @@ -156,7 +159,7 @@ func (s *Scanner) WithoutKeywords(kws map[string]tokens.Token) (*Scanner, map[st // for any errors before using the other values. func (s *Scanner) Scan() (tokens.Token, Position, string, []Error) { - pos := Position{Offset: s.offset - s.width, Row: s.row, Col: s.col} + pos := Position{Offset: s.offset - s.width, Row: s.row, Col: s.col, Tabs: s.tabs} var tok tokens.Token var lit string @@ -410,8 +413,12 @@ func (s *Scanner) next() { if s.curr == '\n' { s.row++ s.col = 0 + s.tabs = []int{} } else { s.col++ + if s.curr == '\t' { + s.tabs = append(s.tabs, s.col) + } } } diff --git a/ast/location/location.go b/ast/location/location.go index 309351a1ed..92226df3f0 100644 --- a/ast/location/location.go +++ b/ast/location/location.go @@ -20,6 +20,8 @@ type Location struct { // JSONOptions specifies options for marshaling and unmarshalling of locations JSONOptions astJSON.Options + + Tabs []int `json:"-"` // The column offsets of tabs in the source. } // NewLocation returns a new Location object. diff --git a/ast/parser.go b/ast/parser.go index 15001ad403..388e5e5926 100644 --- a/ast/parser.go +++ b/ast/parser.go @@ -2130,6 +2130,7 @@ func (p *Parser) doScan(skipws bool) { p.s.loc.Col = pos.Col p.s.loc.Offset = pos.Offset p.s.loc.Text = p.s.Text(pos.Offset, pos.End) + p.s.loc.Tabs = pos.Tabs for _, err := range errs { p.error(p.s.Loc(), err.Message) diff --git a/ast/policy.go b/ast/policy.go index 505b7abd7f..d8e6fa3bc4 100644 --- a/ast/policy.go +++ b/ast/policy.go @@ -233,7 +233,9 @@ type ( Negated bool `json:"negated,omitempty"` Location *Location `json:"location,omitempty"` - jsonOptions astJSON.Options + jsonOptions astJSON.Options + generatedFrom *Expr + generates []*Expr } // SomeDecl represents a variable declaration statement. The symbols are variables. @@ -1593,6 +1595,46 @@ func NewBuiltinExpr(terms ...*Term) *Expr { return &Expr{Terms: terms} } +func (expr *Expr) CogeneratedExprs() []*Expr { + visited := map[*Expr]struct{}{} + visitCogeneratedExprs(expr, func(e *Expr) bool { + if expr.Equal(e) { + return true + } + if _, ok := visited[e]; ok { + return true + } + visited[e] = struct{}{} + return false + }) + + result := make([]*Expr, 0, len(visited)) + for e := range visited { + result = append(result, e) + } + return result +} + +func (expr *Expr) BaseCogeneratedExpr() *Expr { + if expr.generatedFrom == nil { + return expr + } + return expr.generatedFrom.BaseCogeneratedExpr() +} + +func visitCogeneratedExprs(expr *Expr, f func(*Expr) bool) { + if parent := expr.generatedFrom; parent != nil { + if stop := f(parent); !stop { + visitCogeneratedExprs(parent, f) + } + } + for _, child := range expr.generates { + if stop := f(child); !stop { + visitCogeneratedExprs(child, f) + } + } +} + func (d *SomeDecl) String() string { if call, ok := d.Symbols[0].Value.(Call); ok { if len(call) == 4 { diff --git a/cmd/eval.go b/cmd/eval.go index 21569b0d5d..b10bf6459d 100644 --- a/cmd/eval.go +++ b/cmd/eval.go @@ -72,6 +72,7 @@ type evalCommandParams struct { entrypoints repeatedStringFlag strict bool v1Compatible bool + traceVarValues bool } func newEvalCommandParams() evalCommandParams { @@ -307,9 +308,9 @@ access. evalCommand.Flags().VarP(¶ms.prettyLimit, "pretty-limit", "", "set limit after which pretty output gets truncated") evalCommand.Flags().BoolVarP(¶ms.failDefined, "fail-defined", "", false, "exits with non-zero exit code on defined/non-empty result and errors") evalCommand.Flags().DurationVar(¶ms.timeout, "timeout", 0, "set eval timeout (default unlimited)") - evalCommand.Flags().IntVarP(¶ms.optimizationLevel, "optimize", "O", 0, "set optimization level") evalCommand.Flags().VarP(¶ms.entrypoints, "entrypoint", "e", "set slash separated entrypoint path") + evalCommand.Flags().BoolVar(¶ms.traceVarValues, "var-values", false, "show local variable values in pretty trace output") // Shared flags addCapabilitiesFlag(evalCommand.Flags(), params.capabilities) @@ -398,7 +399,12 @@ func eval(args []string, params evalCommandParams, w io.Writer) (bool, error) { case evalValuesOutput: err = pr.Values(w, result) case evalPrettyOutput: - err = pr.Pretty(w, result) + err = pr.PrettyWithOptions(w, result, pr.PrettyOptions{ + TraceOpts: topdown.PrettyTraceOptions{ + Locations: true, + ExprVariables: ectx.params.traceVarValues, + }, + }) case evalSourceOutput: err = pr.Source(w, result) case evalRawOutput: diff --git a/cmd/eval_test.go b/cmd/eval_test.go index 7292817797..5e4ea7ef55 100755 --- a/cmd/eval_test.go +++ b/cmd/eval_test.go @@ -1211,6 +1211,463 @@ func TestEvalDebugTraceJSONOutput(t *testing.T) { } } +func TestEvalPrettyTrace(t *testing.T) { + tests := []struct { + note string + query string + includeVars bool + files map[string]string + expected string + }{ + { + note: "simple without vars", + query: "data.test.p", + includeVars: false, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +p if { + x := 1 + y := 2 + z := 3 + x == z - y +} +`, + }, + expected: `%SKIP_LINE% +query:1 %.*% Enter data.test.p = _ +query:1 %.*% | Eval data.test.p = _ +query:1 %.*% | Index data.test.p (matched 1 rule, early exit) +%.*%/test.rego:4 | Enter data.test.p +%.*%/test.rego:5 | | Eval x = 1 +%.*%/test.rego:6 | | Eval y = 2 +%.*%/test.rego:7 | | Eval z = 3 +%.*%/test.rego:8 | | Eval minus(z, y, __local3__) +%.*%/test.rego:8 | | Eval x = __local3__ +%.*%/test.rego:4 | | Exit data.test.p early +query:1 %.*% | Exit data.test.p = _ +query:1 %.*% Redo data.test.p = _ +query:1 %.*% | Redo data.test.p = _ +%.*%/test.rego:4 | Redo data.test.p +%.*%/test.rego:8 | | Redo x = __local3__ +%.*%/test.rego:8 | | Redo minus(z, y, __local3__) +%.*%/test.rego:7 | | Redo z = 3 +%.*%/test.rego:6 | | Redo y = 2 +%.*%/test.rego:5 | | Redo x = 1 +true +`, + }, + { + note: "simple with vars", + query: "data.test.p", + includeVars: true, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +p if { + x := 1 + y := 2 + z := 3 + x == z - y +} +`, + }, + expected: `%SKIP_LINE% +query:1 %.*% Enter data.test.p = _ {} +query:1 %.*% | Eval data.test.p = _ {} +query:1 %.*% | Index data.test.p (matched 1 rule, early exit) {} +%.*%/test.rego:4 | Enter data.test.p {} +%.*%/test.rego:5 | | Eval x = 1 {} +%.*%/test.rego:6 | | Eval y = 2 {} +%.*%/test.rego:7 | | Eval z = 3 {} +%.*%/test.rego:8 | | Eval minus(z, y, __local3__) {y: 2, z: 3} +%.*%/test.rego:8 | | Eval x = __local3__ {__local3__: 1, x: 1} +%.*%/test.rego:4 | | Exit data.test.p early {} +query:1 %.*% | Exit data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% Redo data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% | Redo data.test.p = _ {_: true, data.test.p: true} +%.*%/test.rego:4 | Redo data.test.p {} +%.*%/test.rego:8 | | Redo x = __local3__ {__local3__: 1, x: 1} +%.*%/test.rego:8 | | Redo minus(z, y, __local3__) {__local3__: 1, y: 2, z: 3} +%.*%/test.rego:7 | | Redo z = 3 {z: 3} +%.*%/test.rego:6 | | Redo y = 2 {y: 2} +%.*%/test.rego:5 | | Redo x = 1 {x: 1} +true +`, + }, + { + note: "large var", + query: "data.test.p", + includeVars: true, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +v := { + "foo": ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"], + "bar": ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"], + "baz": ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"], + "qux": ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"], + } + +p if { + x := v + + x.foo[_] == "a" +} +`, + }, + expected: `%SKIP_LINE% +query:1 %.*% Enter data.test.p = _ {} +query:1 %.*% | Eval data.test.p = _ {} +query:1 %.*% | Index data.test.p (matched 1 rule, early exit) {} +%.*%/test.rego:11 | Enter data.test.p {} +%.*%/test.rego:12 | | Eval x = data.test.v {} +%.*%/test.rego:12 | | Index data.test.v (matched 1 rule, early exit) {} +%.*%/test.rego:4 | | Enter data.test.v {} +%.*%/test.rego:4 | | | Eval true {} +%.*%/test.rego:4 | | | Exit data.test.v early {} +%.*%/test.rego:14 | | Eval x.foo[_] = "a" {x: {"bar": ["a", "b", "c", "d", ...} +%.*%/test.rego:11 | | Exit data.test.p early {} +query:1 %.*% | Exit data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% Redo data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% | Redo data.test.p = _ {_: true, data.test.p: true} +%.*%/test.rego:11 | Redo data.test.p {} +%.*%/test.rego:14 | | Redo x.foo[_] = "a" {_: 0, x: {"bar": ["a", "b", "c", "d", ...} +%.*%/test.rego:12 | | Redo x = data.test.v {data.test.v: {"bar": ["a", "b", "c", "d", ..., x: {"bar": ["a", "b", "c", "d", ...} +%.*%/test.rego:4 | | | Redo true {} +true +`, + }, + { + note: "func call", + query: "data.test.p", + includeVars: true, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +p if { + x := 1 + y := 2 + z := 3 + z == f(x, y) +} + +f(a, b) := c if { + c := a + b +} +`, + }, + expected: `%SKIP_LINE% +query:1 %.*% Enter data.test.p = _ {} +query:1 %.*% | Eval data.test.p = _ {} +query:1 %.*% | Index data.test.p (matched 1 rule, early exit) {} +%.*%/test.rego:4 | Enter data.test.p {} +%.*%/test.rego:5 | | Eval x = 1 {} +%.*%/test.rego:6 | | Eval y = 2 {} +%.*%/test.rego:7 | | Eval z = 3 {} +%.*%/test.rego:8 | | Eval data.test.f(x, y, __local6__) {x: 1, y: 2} +%.*%/test.rego:8 | | Index data.test.f (matched 1 rule) {x: 1, y: 2} +%.*%/test.rego:11 | | Enter data.test.f {} +%.*%/test.rego:12 | | | Eval plus(a, b, __local7__) {a: 1, b: 2} +%.*%/test.rego:12 | | | Eval c = __local7__ {__local7__: 3} +%.*%/test.rego:11 | | | Exit data.test.f {a: 1, b: 2, c: 3} +%.*%/test.rego:8 | | Eval z = __local6__ {__local6__: 3, z: 3} +%.*%/test.rego:4 | | Exit data.test.p early {} +query:1 %.*% | Exit data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% Redo data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% | Redo data.test.p = _ {_: true, data.test.p: true} +%.*%/test.rego:4 | Redo data.test.p {} +%.*%/test.rego:8 | | Redo z = __local6__ {__local6__: 3, z: 3} +%.*%/test.rego:8 | | Redo data.test.f(x, y, __local6__) {__local6__: 3, x: 1, y: 2} +%.*%/test.rego:12 | | | Redo c = __local7__ {__local7__: 3, c: 3} +%.*%/test.rego:12 | | | Redo plus(a, b, __local7__) {__local7__: 3, a: 1, b: 2} +%.*%/test.rego:7 | | Redo z = 3 {z: 3} +%.*%/test.rego:6 | | Redo y = 2 {y: 2} +%.*%/test.rego:5 | | Redo x = 1 {x: 1} +true +`, + }, + { + note: "every", + query: "data.test.p", + includeVars: true, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +p if { + l := ["a", "b", "c"] + every x in l { + count(x) == 1 + } +} + +f(a, b) := c if { + c := a + b +} +`, + }, + expected: `%SKIP_LINE% +query:1 %.*% Enter data.test.p = _ {} +query:1 %.*% | Eval data.test.p = _ {} +query:1 %.*% | Index data.test.p (matched 1 rule, early exit) {} +%.*%/test.rego:4 | Enter data.test.p {} +%.*%/test.rego:5 | | Eval l = ["a", "b", "c"] {} +%.*%/test.rego:6 | | Eval __local6__ = l {l: ["a", "b", "c"]} +%.*%/test.rego:6 | | Eval every x in __local6__ { count(x, __local7__); __local7__ = 1 } {__local6__: ["a", "b", "c"]} +%.*%/test.rego:6 | | Enter every x in __local6__ { count(x, __local7__); __local7__ = 1 } {__local6__: ["a", "b", "c"]} +%.*%/test.rego:6 | | | Eval __local6__[__local1__] = x {__local6__: ["a", "b", "c"]} +%.*%/test.rego:7 | | | Enter count(x, __local7__); __local7__ = 1 {x: "a"} +%.*%/test.rego:7 | | | | Eval count(x, __local7__) {x: "a"} +%.*%/test.rego:7 | | | | Eval __local7__ = 1 {__local7__: 1} +%.*%/test.rego:7 | | | | Exit count(x, __local7__); __local7__ = 1 early {__local7__: 1, x: "a"} +%.*%/test.rego:7 | | | Redo count(x, __local7__); __local7__ = 1 {__local7__: 1, x: "a"} +%.*%/test.rego:7 | | | | Redo __local7__ = 1 {__local7__: 1} +%.*%/test.rego:7 | | | | Redo count(x, __local7__) {__local7__: 1, x: "a"} +%.*%/test.rego:6 | | | Redo every x in __local6__ { count(x, __local7__); __local7__ = 1 } {__local1__: 0, __local6__: ["a", "b", "c"], x: "a"} +%.*%/test.rego:6 | | | Redo __local6__[__local1__] = x {__local1__: 0, __local6__: ["a", "b", "c"], x: "a"} +%.*%/test.rego:7 | | | Enter count(x, __local7__); __local7__ = 1 {x: "b"} +%.*%/test.rego:7 | | | | Eval count(x, __local7__) {x: "b"} +%.*%/test.rego:7 | | | | Eval __local7__ = 1 {__local7__: 1} +%.*%/test.rego:7 | | | | Exit count(x, __local7__); __local7__ = 1 early {__local7__: 1, x: "b"} +%.*%/test.rego:7 | | | Redo count(x, __local7__); __local7__ = 1 {__local7__: 1, x: "b"} +%.*%/test.rego:7 | | | | Redo __local7__ = 1 {__local7__: 1} +%.*%/test.rego:7 | | | | Redo count(x, __local7__) {__local7__: 1, x: "b"} +%.*%/test.rego:6 | | | Redo every x in __local6__ { count(x, __local7__); __local7__ = 1 } {__local1__: 1, __local6__: ["a", "b", "c"], x: "b"} +%.*%/test.rego:6 | | | Redo __local6__[__local1__] = x {__local1__: 1, __local6__: ["a", "b", "c"], x: "b"} +%.*%/test.rego:7 | | | Enter count(x, __local7__); __local7__ = 1 {x: "c"} +%.*%/test.rego:7 | | | | Eval count(x, __local7__) {x: "c"} +%.*%/test.rego:7 | | | | Eval __local7__ = 1 {__local7__: 1} +%.*%/test.rego:7 | | | | Exit count(x, __local7__); __local7__ = 1 early {__local7__: 1, x: "c"} +%.*%/test.rego:7 | | | Redo count(x, __local7__); __local7__ = 1 {__local7__: 1, x: "c"} +%.*%/test.rego:7 | | | | Redo __local7__ = 1 {__local7__: 1} +%.*%/test.rego:7 | | | | Redo count(x, __local7__) {__local7__: 1, x: "c"} +%.*%/test.rego:6 | | | Redo every x in __local6__ { count(x, __local7__); __local7__ = 1 } {__local1__: 2, __local6__: ["a", "b", "c"], x: "c"} +%.*%/test.rego:6 | | | Redo __local6__[__local1__] = x {__local1__: 2, __local6__: ["a", "b", "c"], x: "c"} +%.*%/test.rego:4 | | Exit data.test.p early {} +query:1 %.*% | Exit data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% Redo data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% | Redo data.test.p = _ {_: true, data.test.p: true} +%.*%/test.rego:4 | Redo data.test.p {} +%.*%/test.rego:6 | | Redo every x in __local6__ { count(x, __local7__); __local7__ = 1 } {__local6__: ["a", "b", "c"]} +%.*%/test.rego:6 | | | Exit every x in __local6__ { count(x, __local7__); __local7__ = 1 } {__local6__: ["a", "b", "c"]} +%.*%/test.rego:6 | | Redo __local6__ = l {__local6__: ["a", "b", "c"], l: ["a", "b", "c"]} +%.*%/test.rego:5 | | Redo l = ["a", "b", "c"] {l: ["a", "b", "c"]} +true +`, + }, + { + note: "rule value", + query: "data.test.p", + includeVars: true, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +a := 1 + +p if { + a + 1 == 2 + a + 2 == 3 +} +`, + }, + expected: `%SKIP_LINE% +query:1 %.*% Enter data.test.p = _ {} +query:1 %.*% | Eval data.test.p = _ {} +query:1 %.*% | Index data.test.p (matched 1 rule, early exit) {} +%.*%/test.rego:6 | Enter data.test.p {} +%.*%/test.rego:7 | | Eval __local2__ = data.test.a {} +%.*%/test.rego:7 | | Index data.test.a (matched 1 rule, early exit) {} +%.*%/test.rego:4 | | Enter data.test.a {} +%.*%/test.rego:4 | | | Eval true {} +%.*%/test.rego:4 | | | Exit data.test.a early {} +%.*%/test.rego:7 | | Eval plus(__local2__, 1, __local0__) {__local2__: 1} +%.*%/test.rego:7 | | Eval __local0__ = 2 {__local0__: 2} +%.*%/test.rego:8 | | Eval __local3__ = data.test.a {data.test.a: 1} +%.*%/test.rego:8 | | Index data.test.a (matched 1 rule, early exit) {data.test.a: 1} +%.*%/test.rego:8 | | Eval plus(__local3__, 2, __local1__) {__local3__: 1} +%.*%/test.rego:8 | | Eval __local1__ = 3 {__local1__: 3} +%.*%/test.rego:6 | | Exit data.test.p early {} +query:1 %.*% | Exit data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% Redo data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% | Redo data.test.p = _ {_: true, data.test.p: true} +%.*%/test.rego:6 | Redo data.test.p {} +%.*%/test.rego:8 | | Redo __local1__ = 3 {__local1__: 3} +%.*%/test.rego:8 | | Redo plus(__local3__, 2, __local1__) {__local1__: 3, __local3__: 1} +%.*%/test.rego:8 | | Redo __local3__ = data.test.a {__local3__: 1, data.test.a: 1} +%.*%/test.rego:7 | | Redo __local0__ = 2 {__local0__: 2} +%.*%/test.rego:7 | | Redo plus(__local2__, 1, __local0__) {__local0__: 2, __local2__: 1} +%.*%/test.rego:7 | | Redo __local2__ = data.test.a {__local2__: 1, data.test.a: 1} +%.*%/test.rego:4 | | | Redo true {} +true +`, + }, + { + note: "input values", + query: "data.test.p", + includeVars: true, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +p if { + input.x == 1 + input.x + input.y == input.z +} +`, + "input.json": `{ + "x": 1, + "y": 2, + "z": 3 +}`, + }, + expected: `%SKIP_LINE% +query:1 %.*% Enter data.test.p = _ {} +query:1 %.*% | Eval data.test.p = _ {} +query:1 %.*% | Index data.test.p (matched 1 rule, early exit) {} +%.*%/test.rego:4 | Enter data.test.p {} +%.*%/test.rego:5 | | Eval input.x = 1 {} +%.*%/test.rego:6 | | Eval __local1__ = input.x {} +%.*%/test.rego:6 | | Eval __local2__ = input.y {} +%.*%/test.rego:6 | | Eval plus(__local1__, __local2__, __local0__) {__local1__: 1, __local2__: 2} +%.*%/test.rego:6 | | Eval __local0__ = input.z {__local0__: 3} +%.*%/test.rego:4 | | Exit data.test.p early {} +query:1 %.*% | Exit data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% Redo data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% | Redo data.test.p = _ {_: true, data.test.p: true} +%.*%/test.rego:4 | Redo data.test.p {} +%.*%/test.rego:6 | | Redo __local0__ = input.z {__local0__: 3} +%.*%/test.rego:6 | | Redo plus(__local1__, __local2__, __local0__) {__local0__: 3, __local1__: 1, __local2__: 2} +%.*%/test.rego:6 | | Redo __local2__ = input.y {__local2__: 2} +%.*%/test.rego:6 | | Redo __local1__ = input.x {__local1__: 1} +%.*%/test.rego:5 | | Redo input.x = 1 {} +true +`, + }, + { + note: "data values", + query: "data.test.p", + includeVars: true, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +p if { + data.x == 1 + data.x + data.y == data.z +} +`, + "data.json": `{ + "x": 1, + "y": 2, + "z": 3 +}`, + }, + expected: `%SKIP_LINE% +query:1 %.*% Enter data.test.p = _ {} +query:1 %.*% | Eval data.test.p = _ {} +query:1 %.*% | Index data.test.p (matched 1 rule, early exit) {} +%.*%/test.rego:4 | Enter data.test.p {} +%.*%/test.rego:5 | | Eval data.x = 1 {} +%.*%/test.rego:6 | | Eval __local1__ = data.x {} +%.*%/test.rego:6 | | Eval __local2__ = data.y {} +%.*%/test.rego:6 | | Eval plus(__local1__, __local2__, __local0__) {__local1__: 1, __local2__: 2} +%.*%/test.rego:6 | | Eval __local0__ = data.z {__local0__: 3} +%.*%/test.rego:4 | | Exit data.test.p early {} +query:1 %.*% | Exit data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% Redo data.test.p = _ {_: true, data.test.p: true} +query:1 %.*% | Redo data.test.p = _ {_: true, data.test.p: true} +%.*%/test.rego:4 | Redo data.test.p {} +%.*%/test.rego:6 | | Redo __local0__ = data.z {__local0__: 3} +%.*%/test.rego:6 | | Redo plus(__local1__, __local2__, __local0__) {__local0__: 3, __local1__: 1, __local2__: 2} +%.*%/test.rego:6 | | Redo __local2__ = data.y {__local2__: 2} +%.*%/test.rego:6 | | Redo __local1__ = data.x {__local1__: 1} +%.*%/test.rego:5 | | Redo data.x = 1 {} +true +`, + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + var buf bytes.Buffer + + test.WithTempFS(tc.files, func(path string) { + params := newEvalCommandParams() + _ = params.bundlePaths.Set(path) + inputFile := filepath.Join(path, "input.json") + if _, err := os.Stat(inputFile); err == nil { + params.inputPath = inputFile + } + _ = params.outputFormat.Set(evalPrettyOutput) + _ = params.explain.Set(explainModeFull) + params.traceVarValues = tc.includeVars + params.disableIndexing = true + _ = params.bundlePaths.Set(path) + + _, err := eval([]string{tc.query}, params, &buf) + if err != nil { + t.Fatalf("Unexpected error: %s\n\n%s", err, buf.String()) + } + }) + + actual := buf.String() + if !stringsMatch(t, tc.expected, actual) { + t.Fatalf("Expected:\n\n%v\n\nGot:\n\n%v", tc.expected, actual) + } + }) + } +} + +func stringsMatch(t *testing.T, expected, actual string) bool { + t.Helper() + + var expectedLines []string + for _, l := range strings.Split(expected, "\n") { + if !strings.Contains(l, "%SKIP_LINE%") { + expectedLines = append(expectedLines, l) + } + } + + actualLines := strings.Split(actual, "\n") + + if len(expectedLines) != len(actualLines) { + t.Errorf("Expected %d lines but got %d", len(expectedLines), len(actualLines)) + return false + } + + for i, expectedLine := range expectedLines { + actualLine := actualLines[i] + + expectedParts := strings.Split(expectedLine, "%.*%") + if len(expectedParts) == 1 { + if expectedLine != actualLine { + t.Errorf("Mismatch on line %d. Expected:\n\n%s\n\nGot:\n\n%s", i, expectedLine, actualLine) + return false + } + } else if len(expectedParts) == 2 { + if !strings.HasPrefix(actualLine, expectedParts[0]) { + t.Errorf("Expected line %d to start with:\n\n%s\n\nbut got:\n\n%s", i, expectedParts[0], actualLine) + return false + } + if !strings.HasSuffix(actualLine, expectedParts[1]) { + t.Errorf("Expected line %d to end with:\n\n%s\n\nbut got:\n\n%s", i, expectedParts[1], actualLine) + return false + } + } else { + t.Fatalf("At most one .* is allowed per line but found %d on line %d:\n\n%s", len(expectedParts)-1, i, expectedLine) + return false + } + } + + return true +} + func TestResetExprLocations(t *testing.T) { // Make sure no panic if passed nil. diff --git a/cmd/test.go b/cmd/test.go index b50176f3e8..be266ab29d 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -62,6 +62,7 @@ type testCommandParams struct { output io.Writer errOutput io.Writer v1Compatible bool + varValues bool } func newTestCommandParams() testCommandParams { @@ -348,7 +349,8 @@ func compileAndSetupTests(ctx context.Context, testParams testCommandParams, sto WithEnablePrintStatements(!testParams.benchmark). WithCapabilities(capabilities). WithSchemas(schemaSet). - WithUseTypeCheckAnnotations(true) + WithUseTypeCheckAnnotations(true). + WithRewriteTestRules(testParams.varValues) info, err := runtime.Term(runtime.Params{}) if err != nil { @@ -384,7 +386,7 @@ func compileAndSetupTests(ctx context.Context, testParams testCommandParams, sto SetCompiler(compiler). SetStore(store). CapturePrintOutput(true). - EnableTracing(testParams.verbose). + EnableTracing(testParams.verbose || testParams.varValues). SetCoverageQueryTracer(coverTracer). SetRuntime(info). SetModules(modules). @@ -413,6 +415,8 @@ func compileAndSetupTests(ctx context.Context, testParams testCommandParams, sto BenchmarkResults: testParams.benchmark, BenchMarkShowAllocations: testParams.benchMem, BenchMarkGoBenchFormat: goBench, + FailureLine: testParams.varValues, + LocalVars: testParams.varValues, } } } else { @@ -536,6 +540,7 @@ recommended as some updates might cause them to be dropped by OPA. testCommand.Flags().BoolVar(&testParams.benchmark, "bench", false, "benchmark the unit tests") testCommand.Flags().StringVarP(&testParams.runRegex, "run", "r", "", "run only test cases matching the regular expression.") testCommand.Flags().BoolVarP(&testParams.watch, "watch", "w", false, "watch command line files for changes") + testCommand.Flags().BoolVar(&testParams.varValues, "var-values", false, "show local variable values in test output") // Shared flags addBundleModeFlag(testCommand.Flags(), &testParams.bundleMode, false) diff --git a/cmd/test_test.go b/cmd/test_test.go index de906a6e0e..5c9bad9123 100644 --- a/cmd/test_test.go +++ b/cmd/test_test.go @@ -221,6 +221,1322 @@ func failTrace(t *testing.T) []*topdown.Event { return *tracer } +func TestPrettyTraceWithLocalVars(t *testing.T) { + tests := []struct { + note string + includeVars bool + files map[string]string + expected string + }{ + { + note: "without vars", + includeVars: false, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +test_p if { + x := 1 + y := 2 + z := 3 + x == z + y +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%.*%) + + query:1 %.*% Enter data.test.test_p = _ + query:1 %.*% | Eval data.test.test_p = _ + query:1 %.*% | Index data.test.test_p (matched 1 rule, early exit) + %.*%/test.rego:4 | Enter data.test.test_p + %.*%/test.rego:5 | | Eval x = 1 + %.*%/test.rego:6 | | Eval y = 2 + %.*%/test.rego:7 | | Eval z = 3 + %.*%/test.rego:8 | | Eval plus(z, y, __local3__) + %.*%/test.rego:8 | | Eval x = __local3__ + %.*%/test.rego:8 | | Fail x = __local3__ + %.*%/test.rego:8 | | Redo plus(z, y, __local3__) + %.*%/test.rego:7 | | Redo z = 3 + %.*%/test.rego:6 | | Redo y = 2 + %.*%/test.rego:5 | | Redo x = 1 + query:1 %.*% | Fail data.test.test_p = _ + +SUMMARY +-------------------------------------------------------------------------------- +%.*%/test.rego: +data.test.test_p: FAIL (%.*%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "with vars", + includeVars: true, + files: map[string]string{ + "test.rego": `package test +import rego.v1 + +test_p if { + x := 1 + y := 2 + z := 3 + x == z + y +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%.*%) + + query:1 %.*% Enter data.test.test_p = _ {} + query:1 %.*% | Eval data.test.test_p = _ {} + query:1 %.*% | Index data.test.test_p (matched 1 rule, early exit) {} + %.*%/test.rego:4 | Enter data.test.test_p {} + %.*%/test.rego:5 | | Eval x = 1 {} + %.*%/test.rego:6 | | Eval y = 2 {} + %.*%/test.rego:7 | | Eval z = 3 {} + %.*%/test.rego:8 | | Eval plus(z, y, __local3__) {y: 2, z: 3} + %.*%/test.rego:8 | | Eval x = __local3__ {__local3__: 5, x: 1} + %.*%/test.rego:8 | | Fail x = __local3__ {__local3__: 5, x: 1} + %.*%/test.rego:8 | | Redo plus(z, y, __local3__) {__local3__: 5, y: 2, z: 3} + %.*%/test.rego:7 | | Redo z = 3 {z: 3} + %.*%/test.rego:6 | | Redo y = 2 {y: 2} + %.*%/test.rego:5 | | Redo x = 1 {x: 1} + query:1 %.*% | Fail data.test.test_p = _ {} + + %.*%/test.rego:8: + x == z + y + | | | + | | 2 + | z + y: 5 + | z: 3 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%.*%/test.rego: +data.test.test_p: FAIL (%.*%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + } + + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + test.WithTempFS(tc.files, func(root string) { + buf := new(bytes.Buffer) + testParams := newTestCommandParams() + testParams.count = 1 + testParams.output = buf + testParams.errOutput = io.Discard + testParams.bundleMode = true + testParams.verbose = true + testParams.varValues = tc.includeVars + _ = testParams.explain.Set(explainModeFull) + + _, err := opaTest([]string{root}, testParams) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + actual := buf.String() + if !stringsMatch(t, tc.expected, actual) { + t.Fatalf("Expected:\n\n%v\n\nGot:\n\n%v", tc.expected, actual) + } + }) + }) + } +} + +func TestFailVarValues(t *testing.T) { + tests := []struct { + note string + files map[string]string + expected string + }{ + { + note: "simple", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + x := 1 + y := 2 + z := 3 + x == y + z +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + x == y + z + | | | + | | 3 + | y + z: 5 + | y: 2 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "simple (not)", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + x := 5 + y := 2 + z := 3 + not x == y + z +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + not x == y + z + | | | + | | 3 + | y + z: 5 + | y: 2 + 5 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "array", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + x := 1 + y := [1, 2, 3] + z := 3 + x == y[2] + z +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + x == y[2] + z + | | | + | | 3 + | y[2] + z: 6 + | y[2]: 3 + | y: [1, 2, 3] + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "array, var key", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + x := 1 + y := [1, 2, 3] + z := 3 + i := 2 + x == y[i] + z +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:9: + x == y[i] + z + | | | | + | | | 3 + | | 2 + | y[i] + z: 6 + | y[i]: 3 + | y: [1, 2, 3] + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "array containing vars", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + x := 1 + y := 2 + z := 3 + [x, y, z] == [4, 5, 6] +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + [x, y, z] == [4, 5, 6] + | | | + | | 3 + | 2 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "array containing refs", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +a := 1 + +b := 2 + +test_foo if { + [a, data.test.b, data.c] == [4, 5, 6] +} +`, + "data.json": `{"c": 3}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:9: + [a, data.test.b, data.c] == [4, 5, 6] + | | | + | | 3 + | 2 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "array containing refs, undefined", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +a := 1 + +b := data.b + +test_foo if { + [a, b, data.c] == [4, 5, 6] +} +`, + "data.json": `{"c": 3}`, + }, + // Note: each dynamic array element is broken out into a separate "co-expression" by the compiler. + // Since we failed on the 2nd element (b), we don't have value for the 3rd element (data.c). + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:9: + [a, b, data.c] == [4, 5, 6] + | | + | undefined + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "nested collections containing vars", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + x := 1 + y := 2 + z := 3 + [x, {y, {"a": z}}] == [4, {5, {"a": 6}}] +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + [x, {y, {"a": z}}] == [4, {5, {"a": 6}}] + | | | + | | 3 + | 2 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "single line expression containing tabs", + files: map[string]string{ + "/test.rego": `package test + import rego.v1 + + test_foo if { + x := 1 + y := 2 + z := 3 + x == y + z + } +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + x == y + z + | | | + | | 3 + | y + z: 5 + | y: 2 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "single line expression containing tabs #2", + files: map[string]string{ + "/test.rego": `package test + import rego.v1 + + test_foo if { + x := 1 + y := 2 + z := 3 + x == y + z + } +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + x == y + z + | | | + | | 3 + | y + z: 5 + | y: 2 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "multi-line expression containing tabs", + files: map[string]string{ + "/test.rego": `package test + import rego.v1 + + test_foo if { + x := 1 + y := 2 + z := 3 + obj := { + "foo_": 1, + "bar__": 42, + "baz": 3, + } + obj == { + "foo_": x, + "bar__": y, + "baz": z, + } + } +`, + }, + // We can't deal with tabs in a consistent manner when they occur on multiple lines + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:13: + obj == { + "foo_": x, + "bar__": y, + "baz": z, + } + + Where: + + obj: {"bar__": 42, "baz": 3, "foo_": 1} + x: 1 + y: 2 + z: 3 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "composite rule", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +p contains v if { + some v in numbers.range(1, 3) +} + +test_p if { + p == {4, 5, 6} +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%TIME%) + + %ROOT%/test.rego:9: + p == {4, 5, 6} + | + {1, 2, 3} + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_p: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "composite rule with ref-head", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +p.q contains v if { + some v in numbers.range(1, 3) +} + +test_p if { + p.q == {4, 5, 6} +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%TIME%) + + %ROOT%/test.rego:9: + p.q == {4, 5, 6} + | + {1, 2, 3} + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_p: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "composite rule with ref-head, partial ref", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +p.q contains v if { + some v in numbers.range(1, 3) +} + +test_p if { + p == { + "q": {4, 5, 6} + } +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%TIME%) + + %ROOT%/test.rego:9: + p == { + "q": {4, 5, 6} + } + | + {"q": {1, 2, 3}} + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_p: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "composite rules with ref-head, composite value", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +p.q contains v if { + some v in numbers.range(1, 3) +} + +p.r := "foo" + +test_p if { + p == { + "q": {4, 5, 6}, + "r": "bar" + } +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%TIME%) + + %ROOT%/test.rego:11: + p == { + "q": {4, 5, 6}, + "r": "bar" + } + | + {"q": {1, 2, 3}, "r": "foo"} + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_p: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "refs in different compiled sub-expressions", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +a := 1 +b := 2 +c := 3 + +test_p if { + # This expression is split into multiple final expressions by the compiler, each containing a rule ref + a == b + c +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%TIME%) + + %ROOT%/test.rego:10: + a == b + c + | | | + | | 3 + | b + c: 5 + | b: 2 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_p: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "rule not defined", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +p if { + input.x == 1 +} + +test_p if { + p with input.x as 2 +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%TIME%) + + %ROOT%/test.rego:9: + p with input.x as 2 + | + undefined + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_p: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "rule defined (not)", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +p if { + input.x == 1 +} + +test_p if { + not p with input.x as 1 +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%TIME%) + + %ROOT%/test.rego:9: + not p with input.x as 1 + | + true + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_p: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "data ref", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + y := 1 + data.x == y +} +`, + "data.json": `{"x": 2}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:6: + data.x == y + | | + | 1 + 2 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "data + virtual extent ref", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +foo.x := 1 + +test_foo if { + y := {"x": 1, "y": 42} + foo == y +} +`, + "data.json": `{"test": {"foo": {"y": 2}}}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + foo == y + | | + | {"x": 1, "y": 42} + {"x": 1, "y": 2} + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "in (array)", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + l := ["a", "b", "c"] + x := "q" + x in l +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:7: + x in l + | | + | ["a", "b", "c"] + "q" + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "in (set)", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + l := {"a", "b", "c"} + x := "q" + x in l +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:7: + x in l + | | + | {"a", "b", "c"} + "q" + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "comprehension (array)", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + l := ["a", "b", "c"] + [x | x := l[_]] == ["d", "e", "f"] +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:6: + [x | x := l[_]] == ["d", "e", "f"] + | + ["a", "b", "c"] + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "comprehension (set)", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + l := ["a"] + {x | x := l[_]} == {"b"} +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:6: + {x | x := l[_]} == {"b"} + | + {"a"} + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "comprehension (object)", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + l := ["a", "b", "c"] + {k: x | x := l[k]} == {3: "d", 4: "e", 5: "f"} +} +`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:6: + {k: x | x := l[k]} == {3: "d", 4: "e", 5: "f"} + | + {0: "a", 1: "b", 2: "c"} + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "every", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + l := [1, 2, 3] + every x in l { + x == 1 + } +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:7: + x == 1 + | + 2 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "comprehension inside every", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + l := [1, 2, 3] + every x in l { + [v | v := x] == [42] + } +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:7: + [v | v := x] == [42] + | + [1] + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "nested every", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + l := [[1, 2], [3, 4], [5, 6]] + every x in l { + every y in x { + y < 4 + } + } +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + y < 4 + | + 4 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "nested every with comprehension", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + l := [[1, 2], [3, 4], [5, 6]] + every x in l { + every y in x { + [v | v := y] == [42] + } + } +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + [v | v := y] == [42] + | + [1] + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "ref equality", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +a := 1 +b := 2 + +test_foo if { + a == b +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:8: + a == b + | | + | 2 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "ref equality (data)", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + data.a == data.b +}`, + "data.json": `{"a": 1, "b": 2}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:5: + data.a == data.b + | | + | 2 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "with, containing local vars", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +p := input.x + +test_p if { + a := 1 + p == 2 with input.x as a +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%TIME%) + + %ROOT%/test.rego:8: + p == 2 with input.x as a + | | + | 1 + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_p: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "with, containing ref", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +p := input.x + +testInput := {"x": 1} + +test_p if { + p == 2 with input as testInput +}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_p: FAIL (%TIME%) + + %ROOT%/test.rego:9: + p == 2 with input as testInput + | | + | {"x": 1} + 1 + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_p: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "negated rule ref", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +a if {true} + +test_foo if { + not a +}`, + "data.json": `{"a": true}`, + }, + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:7: + not a + | + true + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + { + note: "negated data ref", + files: map[string]string{ + "/test.rego": `package test +import rego.v1 + +test_foo if { + not data.a +}`, + "data.json": `{"a": true}`, + }, + // Because of the negated expr, the compiler will have opted out of rewriting the expression to + // capture the value of data.a in a local variable, and since data.a isn't in the local bindings + // or in the virtual cache, we don't know if it's undefined or unknown, and therefore can't report + // on a value. + expected: `FAILURES +-------------------------------------------------------------------------------- +data.test.test_foo: FAIL (%TIME%) + + %ROOT%/test.rego:5: + not data.a + +SUMMARY +-------------------------------------------------------------------------------- +%ROOT%/test.rego: +data.test.test_foo: FAIL (%TIME%) +-------------------------------------------------------------------------------- +FAIL: 1/1 +`, + }, + } + + r := regexp.MustCompile(`FAIL \(.*s\)`) + for _, tc := range tests { + t.Run(tc.note, func(t *testing.T) { + test.WithTempFS(tc.files, func(root string) { + buf := new(bytes.Buffer) + testParams := newTestCommandParams() + testParams.count = 1 + testParams.output = buf + testParams.errOutput = io.Discard + testParams.bundleMode = true + testParams.varValues = true + _ = testParams.explain.Set(explainModeFull) + + _, err := opaTest([]string{root}, testParams) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + actual := r.ReplaceAllString(buf.String(), "FAIL (%TIME%)") + expected := strings.ReplaceAll(tc.expected, "%ROOT%", root) + + if !stringsMatch(t, expected, actual) { + t.Fatalf("Expected output to be:\n\n%s\n\ngot:\n\n%s", expected, actual) + } + }) + }) + } +} + // Assert that ignore flag is correctly used when the bundle flag is activated func TestIgnoreFlag(t *testing.T) { files := map[string]string{ diff --git a/docs/content/cli.md b/docs/content/cli.md index d7a8819f43..18008b7765 100755 --- a/docs/content/cli.md +++ b/docs/content/cli.md @@ -569,6 +569,7 @@ opa eval [flags] --timeout duration set eval timeout (default unlimited) -u, --unknowns stringArray set paths to treat as unknown during partial evaluation (default [input]) --v1-compatible opt-in to OPA features and behaviors that will be enabled by default in a future OPA v1.0 release + --var-values show local variable values in pretty trace output ``` ____ @@ -1129,6 +1130,7 @@ opa test [path [...]] [flags] --threshold float set coverage threshold and exit with non-zero status if coverage is less than threshold % --timeout duration set test timeout (default 5s, 30s when benchmarking) --v1-compatible opt-in to OPA features and behaviors that will be enabled by default in a future OPA v1.0 release + --var-values show local variable values in test output -v, --verbose set verbose reporting mode -w, --watch watch command line files for changes ``` diff --git a/docs/content/policy-testing.md b/docs/content/policy-testing.md index 790e8b5849..8f57b3dcbd 100644 --- a/docs/content/policy-testing.md +++ b/docs/content/policy-testing.md @@ -119,6 +119,69 @@ PASS: 3/4 FAIL: 1/4 ``` +## Enriched Test Report With Variable Values + +Sometimes, e.g. when testing rules with complex output, it can be useful to know more about the circumstances that caused a certain expression to fail a test. +The `--var-values` flag can be used to enrich the test report with the exact expression that caused a test rule to fail, including the values of any variables or references used in the expression. + +Consider the following utility module: + +```live:example_vars:module:read_only,openable +package authz + +import rego.v1 + +allowed_actions(user) := [action | + user in data.actions[action] +] +``` + +with accompanying tests: + +```live:example_vars/test:module:read_only +package authz_test + +import data.authz +import rego.v1 + +test_allowed_actions_all_can_read if { + users := ["alice", "bob", "jane"] + r := ["alice", "bob"] + w := ["jane"] + p := {"read": r, "write": w} + + every user in users { + "read" in authz.allowed_actions(user) with data.actions as p + } +} +``` + +Exercising the tests with the `--var-values` flag: + +```console +opa test . --var-values +FAILURES +-------------------------------------------------------------------------------- +data.authz_test.test_allowed_actions_all_can_read: FAIL (904µs) + + util_test.rego:13: + "read" in authz.allowed_actions(user) with data.actions as p + | | | + | | {"read": ["alice", "bob"], "write": ["jane"]} + | "jane" + ["write"] + +SUMMARY +-------------------------------------------------------------------------------- +util_test.rego: +data.authz_test.test_allowed_actions_all_can_read: FAIL (904µs) +-------------------------------------------------------------------------------- +FAIL: 1/1 +``` + +The test failed because it expected users with __write__ permission to implicitly also have the __read__ permission, an expectation the function under test didn't meet. +By including the failing expression and its local variable assignments in the test report, we make troubleshooting easier for the developer, as it's immediately apparent what assertion and combination of parameters caused the test to fail. + ## Test Format Tests are expressed as standard Rego rules with a convention that the rule diff --git a/internal/presentation/presentation.go b/internal/presentation/presentation.go index 42fef24600..b69f93a271 100644 --- a/internal/presentation/presentation.go +++ b/internal/presentation/presentation.go @@ -282,8 +282,21 @@ func Values(w io.Writer, r Output) error { // Pretty prints all of r to w in a human-readable format. func Pretty(w io.Writer, r Output) error { + return PrettyWithOptions(w, r, PrettyOptions{ + TraceOpts: topdown.PrettyTraceOptions{ + Locations: true, + }, + }) +} + +type PrettyOptions struct { + TraceOpts topdown.PrettyTraceOptions +} + +// PrettyWithOptions prints all of r to w in a human-readable format. +func PrettyWithOptions(w io.Writer, r Output, opts PrettyOptions) error { if len(r.Explanation) > 0 { - if err := prettyExplanation(w, r.Explanation); err != nil { + if err := prettyExplanation(w, r.Explanation, opts.TraceOpts); err != nil { return err } } @@ -554,8 +567,8 @@ func prettyAggregatedProfile(w io.Writer, profile []profiler.ExprStatsAggregated return nil } -func prettyExplanation(w io.Writer, explanation []*topdown.Event) error { - topdown.PrettyTraceWithLocation(w, explanation) +func prettyExplanation(w io.Writer, explanation []*topdown.Event, opts topdown.PrettyTraceOptions) error { + topdown.PrettyTraceWithOpts(w, explanation, opts) return nil } diff --git a/internal/strings/strings.go b/internal/strings/strings.go index 76fb9ee8c8..08f3bf9182 100644 --- a/internal/strings/strings.go +++ b/internal/strings/strings.go @@ -63,6 +63,14 @@ func TruncateFilePaths(maxIdealWidth, maxWidth int, path ...string) (map[string] return result, longestLocation } +func Truncate(str string, maxWidth int) string { + if len(str) <= maxWidth { + return str + } + + return str[:maxWidth-3] + "..." +} + func getPathFromFirstSeparator(path string) string { s := filepath.Dir(path) s = strings.TrimPrefix(s, string(filepath.Separator)) diff --git a/tester/reporter.go b/tester/reporter.go index 81774d6185..c7ab9cbc97 100644 --- a/tester/reporter.go +++ b/tester/reporter.go @@ -28,6 +28,7 @@ type PrettyReporter struct { Output io.Writer Verbose bool FailureLine bool + LocalVars bool BenchmarkResults bool BenchMarkShowAllocations bool BenchMarkGoBenchFormat bool @@ -55,14 +56,41 @@ func (r PrettyReporter) Report(ch chan *Result) error { results = append(results, tr) } - if fail > 0 && r.Verbose { + if fail > 0 && (r.Verbose || r.FailureLine) { fmt.Fprintln(r.Output, "FAILURES") r.hl() for _, failure := range failures { fmt.Fprintln(r.Output, failure) - fmt.Fprintln(r.Output) - topdown.PrettyTraceWithLocation(newIndentingWriter(r.Output), failure.Trace) + if r.Verbose { + fmt.Fprintln(r.Output) + topdown.PrettyTraceWithOpts(newIndentingWriter(r.Output), failure.Trace, topdown.PrettyTraceOptions{ + Locations: true, + ExprVariables: r.LocalVars, + }) + } + + if r.FailureLine { + fmt.Fprintln(r.Output) + for i := len(failure.Trace) - 1; i >= 0; i-- { + e := failure.Trace[i] + if e.Op == topdown.FailOp && e.Location != nil && e.QueryID != 0 { + if expr, isExpr := e.Node.(*ast.Expr); isExpr { + if _, isEvery := expr.Terms.(*ast.Every); isEvery { + // We're interested in the failing expression inside the every body. + continue + } + } + _, _ = fmt.Fprintf(newIndentingWriter(r.Output), "%s:%d:\n", e.Location.File, e.Location.Row) + if err := topdown.PrettyEvent(newIndentingWriter(r.Output, 4), e, topdown.PrettyEventOpts{PrettyVars: r.LocalVars}); err != nil { + return err + } + _, _ = fmt.Fprintln(r.Output) + break + } + } + } + fmt.Fprintln(r.Output) } @@ -216,12 +244,18 @@ func (r JSONCoverageReporter) Report(ch chan *Result) error { } type indentingWriter struct { - w io.Writer + w io.Writer + indent int } -func newIndentingWriter(w io.Writer) indentingWriter { +func newIndentingWriter(w io.Writer, indent ...int) indentingWriter { + i := 2 + if len(indent) > 0 { + i = indent[0] + } return indentingWriter{ - w: w, + w: w, + indent: i, } } @@ -231,7 +265,7 @@ func (w indentingWriter) Write(bs []byte) (int, error) { indent := true for _, b := range bs { if indent { - wrote, err := w.w.Write([]byte(" ")) + wrote, err := w.w.Write([]byte(strings.Repeat(" ", w.indent))) if err != nil { return written, err } diff --git a/tester/reporter_test.go b/tester/reporter_test.go index 56d98b2d57..9a3b826d01 100644 --- a/tester/reporter_test.go +++ b/tester/reporter_test.go @@ -14,8 +14,11 @@ import ( ) func getFakeTraceEvents() []*topdown.Event { - node := ast.MustParseExpr("true = false") - return []*topdown.Event{ + return getFakeTraceEventsFor(ast.MustParseExpr("true = false")) +} + +func getFakeTraceEventsFor(node ast.Node, modifiers ...func(e *topdown.Event)) []*topdown.Event { + es := []*topdown.Event{ { Op: topdown.FailOp, Node: node, @@ -24,6 +27,12 @@ func getFakeTraceEvents() []*topdown.Event { ParentID: 0, }, } + + for _, modifier := range modifiers { + modifier(es[0]) + } + + return es } func TestPrettyReporterVerbose(t *testing.T) { @@ -99,7 +108,7 @@ func TestPrettyReporterVerbose(t *testing.T) { -------------------------------------------------------------------------------- data.foo.bar.test_corge: FAIL (0s) - query:1 | Fail true = false + query:1 | Fail true = false SUMMARY -------------------------------------------------------------------------------- @@ -134,6 +143,153 @@ ERROR: 1/6 } } +func TestPrettyReporterFailureLine(t *testing.T) { + var buf bytes.Buffer + + // supply fake trace events to verify that traces are suppressed without verbose + // flag. + ts := []*Result{ + { + Package: "data.foo.bar", + Name: "test_baz", + Trace: getFakeTraceEvents(), + Location: &ast.Location{ + File: "policy1.rego", + }, + }, + { + Package: "data.foo.bar", + Name: "test_qux", + Error: fmt.Errorf("some err"), + Trace: getFakeTraceEvents(), + Location: &ast.Location{ + File: "policy1.rego", + }, + }, + { + Package: "data.foo.bar", + Name: "test_corge", + Fail: true, + Trace: getFakeTraceEventsFor( + ast.MustParseExpr("x == y + z"), + func(e *topdown.Event) { + // QueryID == 0 is not pretty-printed, as this is the base query to eval the test rule; not the test rule itself. + e.QueryID = 1 + }, + func(e *topdown.Event) { + e.Location.File = "policy1.rego" + e.Location.Row = 5 + }, + func(e *topdown.Event) { + e.Locals = ast.NewValueMap() + e.Locals.Put(ast.Var("x"), ast.Number("1")) + e.Locals.Put(ast.Var("y"), ast.Number("2")) + e.Locals.Put(ast.Var("z"), ast.Number("3")) + }, + func(e *topdown.Event) { + e.LocalMetadata = map[ast.Var]topdown.VarMetadata{ + "x": {Name: "x"}, + "y": {Name: "y"}, + "z": {Name: "z"}, + } + }), + Location: &ast.Location{ + File: "policy1.rego", + }, + }, + { + Package: "data.foo.bar", + Name: "todo_test_qux", + Skip: true, + Trace: nil, + Location: &ast.Location{ + File: "policy1.rego", + }, + }, + { + Package: "data.foo.bar", + Name: "test_contains_print_pass", + Output: []byte("fake print output\n"), + Location: &ast.Location{ + File: "policy1.rego", + }, + }, + { + Package: "data.foo.bar", + Name: "test_contains_print_fail", + Fail: true, + Output: []byte("fake print output2\n"), + Location: &ast.Location{ + File: "policy2.rego", + }, + }, + { + Package: "data.foo.baz", + Name: "p.q.r.test_quz", + Fail: true, + Trace: getFakeTraceEvents(), + Location: &ast.Location{ + File: "policy3.rego", + }, + }, + } + + r := PrettyReporter{ + Output: &buf, + Verbose: false, + FailureLine: true, + LocalVars: true, + } + ch := resultsChan(ts) + if err := r.Report(ch); err != nil { + t.Fatal(err) + } + + exp := `FAILURES +-------------------------------------------------------------------------------- +data.foo.bar.test_corge: FAIL (0s) + + policy1.rego:5: + x == y + z + | | | + | | 3 + | 2 + 1 + +data.foo.bar.test_contains_print_fail: FAIL (0s) + + +data.foo.baz.p.q.r.test_quz: FAIL (0s) + + +SUMMARY +-------------------------------------------------------------------------------- +policy1.rego: +data.foo.bar.test_qux: ERROR (0s) + some err +data.foo.bar.test_corge: FAIL (0s) +data.foo.bar.todo_test_qux: SKIPPED + +policy2.rego: +data.foo.bar.test_contains_print_fail: FAIL (0s) + + fake print output2 + + +policy3.rego: +data.foo.baz.p.q.r.test_quz: FAIL (0s) +-------------------------------------------------------------------------------- +PASS: 2/7 +FAIL: 3/7 +SKIPPED: 1/7 +ERROR: 1/7 +` + + if exp != buf.String() { + t.Fatalf("Expected:\n\n%v\n\nGot:\n\n%v", exp, buf.String()) + } +} + func TestPrettyReporter(t *testing.T) { var buf bytes.Buffer diff --git a/topdown/eval.go b/topdown/eval.go index 51147dc138..6263efba64 100644 --- a/topdown/eval.go +++ b/topdown/eval.go @@ -237,6 +237,10 @@ func (e *eval) traceWasm(x ast.Node, target *ast.Ref) { e.traceEvent(WasmOp, x, "", target) } +func (e *eval) traceUnify(a, b *ast.Term) { + e.traceEvent(UnifyOp, ast.Equality.Expr(a, b), "", nil) +} + func (e *eval) traceEvent(op Op, x ast.Node, msg string, target *ast.Ref) { if !e.traceEnabled { @@ -275,6 +279,7 @@ func (e *eval) traceEvent(op Op, x ast.Node, msg string, target *ast.Ref) { evt.Locals = ast.NewValueMap() evt.LocalMetadata = map[ast.Var]VarMetadata{} + evt.localVirtualCacheSnapshot = ast.NewValueMap() _ = e.bindings.Iter(nil, func(k, v *ast.Term) error { original := k.Value.(ast.Var) @@ -290,15 +295,21 @@ func (e *eval) traceEvent(op Op, x ast.Node, msg string, target *ast.Ref) { }) // cannot return error ast.WalkTerms(x, func(term *ast.Term) bool { - if v, ok := term.Value.(ast.Var); ok { - if _, ok := evt.LocalMetadata[v]; !ok { - if rewritten, ok := e.rewrittenVar(v); ok { - evt.LocalMetadata[v] = VarMetadata{ + switch x := term.Value.(type) { + case ast.Var: + if _, ok := evt.LocalMetadata[x]; !ok { + if rewritten, ok := e.rewrittenVar(x); ok { + evt.LocalMetadata[x] = VarMetadata{ Name: rewritten, Location: term.Loc(), } } } + case ast.Ref: + groundRef := x.GroundPrefix() + if v, _ := e.virtualCache.Get(groundRef); v != nil { + evt.localVirtualCacheSnapshot.Put(groundRef, v.Value) + } } return false }) @@ -858,7 +869,7 @@ func (e *eval) biunify(a, b *ast.Term, b1, b2 *bindings, iter unifyIterator) err a, b1 = b1.apply(a) b, b2 = b2.apply(b) if e.traceEnabled { - e.traceEvent(UnifyOp, ast.Equality.Expr(a, b), "", nil) + e.traceUnify(a, b) } switch vA := a.Value.(type) { case ast.Var, ast.Ref, *ast.ArrayComprehension, *ast.SetComprehension, *ast.ObjectComprehension: diff --git a/topdown/lineage/lineage.go b/topdown/lineage/lineage.go index c4631d9e0e..2c0ec7287b 100644 --- a/topdown/lineage/lineage.go +++ b/topdown/lineage/lineage.go @@ -13,7 +13,7 @@ func Debug(trace []*topdown.Event) []*topdown.Event { return trace } -// Full returns a filtered trace that contains everything except Uninfy ops +// Full returns a filtered trace that contains everything except Unify ops func Full(trace []*topdown.Event) (result []*topdown.Event) { // Do not use Filter since this event will only occur at the leaf positions. for _, event := range trace { diff --git a/topdown/trace.go b/topdown/trace.go index 727938d9c6..e77713821b 100644 --- a/topdown/trace.go +++ b/topdown/trace.go @@ -5,8 +5,10 @@ package topdown import ( + "bytes" "fmt" "io" + "slices" "strings" iStrs "github.com/open-policy-agent/opa/internal/strings" @@ -18,7 +20,9 @@ import ( const ( minLocationWidth = 5 // len("query") maxIdealLocationWidth = 64 - locationPadding = 4 + columnPadding = 4 + maxExprVarWidth = 32 + maxPrettyExprVarWidth = 64 ) // Op defines the types of tracing events. @@ -62,7 +66,8 @@ const ( // UnifyOp is emitted when two terms are unified. Node will be set to an // equality expression with the two terms. This Node will not have location // info. - UnifyOp Op = "Unify" + UnifyOp Op = "Unify" + FailedAssertionOp Op = "FailedAssertion" ) // VarMetadata provides some user facing information about @@ -84,8 +89,9 @@ type Event struct { Message string // Contains message for Note events. Ref *ast.Ref // Identifies the subject ref for the event. Only applies to Index and Wasm operations. - input *ast.Term - bindings *bindings + input *ast.Term + bindings *bindings + localVirtualCacheSnapshot *ast.ValueMap } // HasRule returns true if the Event contains an ast.Rule. @@ -236,31 +242,162 @@ func (b *BufferTracer) Config() TraceConfig { // PrettyTrace pretty prints the trace to the writer. func PrettyTrace(w io.Writer, trace []*Event) { - prettyTraceWith(w, trace, false) + PrettyTraceWithOpts(w, trace, PrettyTraceOptions{}) } // PrettyTraceWithLocation prints the trace to the writer and includes location information func PrettyTraceWithLocation(w io.Writer, trace []*Event) { - prettyTraceWith(w, trace, true) + PrettyTraceWithOpts(w, trace, PrettyTraceOptions{Locations: true}) } -func prettyTraceWith(w io.Writer, trace []*Event, locations bool) { +type PrettyTraceOptions struct { + Locations bool // Include location information + ExprVariables bool // Include variables found in the expression + LocalVariables bool // Include all local variables +} + +type traceRow []string + +func (r *traceRow) add(s string) { + *r = append(*r, s) +} + +type traceTable struct { + rows []traceRow + maxWidths []int +} + +func (t *traceTable) add(row traceRow) { + t.rows = append(t.rows, row) + for i := range row { + if i >= len(t.maxWidths) { + t.maxWidths = append(t.maxWidths, len(row[i])) + } else if len(row[i]) > t.maxWidths[i] { + t.maxWidths[i] = len(row[i]) + } + } +} + +func (t *traceTable) write(w io.Writer, padding int) { + for _, row := range t.rows { + for i, cell := range row { + width := t.maxWidths[i] + padding + if i < len(row)-1 { + _, _ = fmt.Fprintf(w, "%-*s ", width, cell) + } else { + _, _ = fmt.Fprintf(w, "%s", cell) + } + } + _, _ = fmt.Fprintln(w) + } +} + +func PrettyTraceWithOpts(w io.Writer, trace []*Event, opts PrettyTraceOptions) { depths := depths{} - filePathAliases, longest := getShortenedFileNames(trace) + // FIXME: Can we shorten each location as we process each trace event instead of beforehand? + filePathAliases, _ := getShortenedFileNames(trace) - // Always include some padding between the trace and location - locationWidth := longest + locationPadding + table := traceTable{} for _, event := range trace { depth := depths.GetOrSet(event.QueryID, event.ParentID) - if locations { + row := traceRow{} + + if opts.Locations { location := formatLocation(event, filePathAliases) - fmt.Fprintf(w, "%-*s %s\n", locationWidth, location, formatEvent(event, depth)) - } else { - fmt.Fprintln(w, formatEvent(event, depth)) + row.add(location) + } + + row.add(formatEvent(event, depth)) + + if opts.ExprVariables { + vars := exprLocalVars(event) + keys := sortedKeys(vars) + + buf := new(bytes.Buffer) + buf.WriteString("{") + for i, k := range keys { + if i > 0 { + buf.WriteString(", ") + } + _, _ = fmt.Fprintf(buf, "%v: %s", k, iStrs.Truncate(vars.Get(k).String(), maxExprVarWidth)) + } + buf.WriteString("}") + row.add(buf.String()) + } + + if opts.LocalVariables { + if locals := event.Locals; locals != nil { + keys := sortedKeys(locals) + + buf := new(bytes.Buffer) + buf.WriteString("{") + for i, k := range keys { + if i > 0 { + buf.WriteString(", ") + } + _, _ = fmt.Fprintf(buf, "%v: %s", k, iStrs.Truncate(locals.Get(k).String(), maxExprVarWidth)) + } + buf.WriteString("}") + row.add(buf.String()) + } else { + row.add("{}") + } + } + + table.add(row) + } + + table.write(w, columnPadding) +} + +func sortedKeys(vm *ast.ValueMap) []ast.Value { + keys := make([]ast.Value, 0, vm.Len()) + vm.Iter(func(k, _ ast.Value) bool { + keys = append(keys, k) + return false + }) + slices.SortFunc(keys, func(a, b ast.Value) int { + return strings.Compare(a.String(), b.String()) + }) + return keys +} + +func exprLocalVars(e *Event) *ast.ValueMap { + vars := ast.NewValueMap() + + findVars := func(term *ast.Term) bool { + //if r, ok := term.Value.(ast.Ref); ok { + // fmt.Printf("ref: %v\n", r) + // //return true + //} + if name, ok := term.Value.(ast.Var); ok { + if meta, ok := e.LocalMetadata[name]; ok { + if val := e.Locals.Get(name); val != nil { + vars.Put(meta.Name, val) + } + } } + return false } + + if r, ok := e.Node.(*ast.Rule); ok { + // We're only interested in vars in the head, not the body + ast.WalkTerms(r.Head, findVars) + return vars + } + + // The local cache snapshot only contains a snapshot for those refs present in the event node, + // so they can all be added to the vars map. + e.localVirtualCacheSnapshot.Iter(func(k, v ast.Value) bool { + vars.Put(k, v) + return false + }) + + ast.WalkTerms(e.Node, findVars) + + return vars } func formatEvent(event *Event, depth int) string { @@ -451,6 +588,310 @@ func rewrite(event *Event) *Event { return &cpy } +type varInfo struct { + VarMetadata + val ast.Value + exprLoc *ast.Location + col int // 0-indexed column +} + +func (v varInfo) Value() string { + if v.val != nil { + return v.val.String() + } + return "undefined" +} + +func (v varInfo) Title() string { + if v.exprLoc != nil && v.exprLoc.Text != nil { + return string(v.exprLoc.Text) + } + return string(v.Name) +} + +func padLocationText(loc *ast.Location) string { + if loc == nil { + return "" + } + + text := string(loc.Text) + + if loc.Col == 0 { + return text + } + + buf := new(bytes.Buffer) + j := 0 + for i := 1; i < loc.Col; i++ { + if len(loc.Tabs) > 0 && j < len(loc.Tabs) && loc.Tabs[j] == i { + buf.WriteString("\t") + j++ + } else { + buf.WriteString(" ") + } + } + + buf.WriteString(text) + return buf.String() +} + +type PrettyEventOpts struct { + PrettyVars bool +} + +func walkTestTerms(x interface{}, f func(*ast.Term) bool) { + var vis *ast.GenericVisitor + vis = ast.NewGenericVisitor(func(x interface{}) bool { + switch x := x.(type) { + case ast.Call: + for _, t := range x[1:] { + vis.Walk(t) + } + return true + case *ast.Expr: + if x.IsCall() { + for _, o := range x.Operands() { + vis.Walk(o) + } + for i := range x.With { + vis.Walk(x.With[i]) + } + return true + } + case *ast.Term: + return f(x) + case *ast.With: + vis.Walk(x.Value) + return true + } + return false + }) + vis.Walk(x) +} + +func PrettyEvent(w io.Writer, e *Event, opts PrettyEventOpts) error { + if !opts.PrettyVars { + _, _ = fmt.Fprintln(w, padLocationText(e.Location)) + return nil + } + + buf := new(bytes.Buffer) + exprVars := map[string]varInfo{} + + findVars := func(unknownAreUndefined bool) func(term *ast.Term) bool { + return func(term *ast.Term) bool { + if term.Location == nil { + return false + } + + switch v := term.Value.(type) { + case *ast.ArrayComprehension, *ast.SetComprehension, *ast.ObjectComprehension: + // we don't report on the internals of a comprehension, as it's already evaluated, and we won't have the local vars. + return true + case ast.Var: + var info *varInfo + if meta, ok := e.LocalMetadata[v]; ok { + info = &varInfo{ + VarMetadata: meta, + val: e.Locals.Get(v), + exprLoc: term.Location, + } + } else if unknownAreUndefined { + info = &varInfo{ + VarMetadata: VarMetadata{Name: v}, + exprLoc: term.Location, + col: term.Location.Col, + } + } + + if info != nil { + if v, exists := exprVars[info.Title()]; !exists || v.val == nil { + if term.Location != nil { + info.col = term.Location.Col + } + exprVars[info.Title()] = *info + } + } + } + return false + } + } + + expr, ok := e.Node.(*ast.Expr) + if !ok || expr == nil { + return nil + } + + base := expr.BaseCogeneratedExpr() + exprText := padLocationText(base.Location) + buf.WriteString(exprText) + + e.localVirtualCacheSnapshot.Iter(func(k, v ast.Value) bool { + var info *varInfo + switch k := k.(type) { + case ast.Ref: + info = &varInfo{ + VarMetadata: VarMetadata{Name: ast.Var(k.String())}, + val: v, + exprLoc: k[0].Location, + col: k[0].Location.Col, + } + case *ast.ArrayComprehension: + info = &varInfo{ + VarMetadata: VarMetadata{Name: ast.Var(k.String())}, + val: v, + exprLoc: k.Term.Location, + col: k.Term.Location.Col, + } + case *ast.SetComprehension: + info = &varInfo{ + VarMetadata: VarMetadata{Name: ast.Var(k.String())}, + val: v, + exprLoc: k.Term.Location, + col: k.Term.Location.Col, + } + case *ast.ObjectComprehension: + info = &varInfo{ + VarMetadata: VarMetadata{Name: ast.Var(k.String())}, + val: v, + exprLoc: k.Key.Location, + col: k.Key.Location.Col, + } + } + + if info != nil { + exprVars[info.Title()] = *info + } + + return false + }) + + // If the expression is negated, we can't confidently assert that vars with unknown values are 'undefined', + // since the compiler might have opted out of the necessary rewrite. + walkTestTerms(expr, findVars(!expr.Negated)) + coExprs := expr.CogeneratedExprs() + for _, coExpr := range coExprs { + // Only the current "co-expr" can have undefined vars, if we don't know the value for a var in any other co-expr, + // it's unknown, not undefined. A var can be unknown if it hasn't been assigned a value yet, because the co-expr + // hasn't been evaluated yet (the fail happened before it). + walkTestTerms(coExpr, findVars(false)) + } + + printPrettyVars(buf, exprVars) + _, _ = fmt.Fprint(w, buf.String()) + return nil +} + +func printPrettyVars(w *bytes.Buffer, exprVars map[string]varInfo) { + containsTabs := false + varRows := make(map[int]interface{}) + for _, info := range exprVars { + if len(info.exprLoc.Tabs) > 0 { + containsTabs = true + } + varRows[info.exprLoc.Row] = nil + } + + if containsTabs && len(varRows) > 1 { + // We can't (currently) reliably point to var locations when they are on different rows that contain tabs. + // So we'll just print them in alphabetical order instead. + byName := make([]varInfo, 0, len(exprVars)) + for _, info := range exprVars { + byName = append(byName, info) + } + slices.SortStableFunc(byName, func(a, b varInfo) int { + return strings.Compare(a.Title(), b.Title()) + }) + + w.WriteString("\n\nWhere:\n") + for _, info := range byName { + w.WriteString(fmt.Sprintf("\n%s: %s", info.Title(), iStrs.Truncate(info.Value(), maxPrettyExprVarWidth))) + } + + return + } + + byCol := make([]varInfo, 0, len(exprVars)) + for _, info := range exprVars { + byCol = append(byCol, info) + } + slices.SortFunc(byCol, func(a, b varInfo) int { + // sort first by column, then by reverse row (to present vars in the same order they appear in the expr) + if a.col == b.col { + if a.exprLoc.Row == b.exprLoc.Row { + return strings.Compare(a.Title(), b.Title()) + } + return b.exprLoc.Row - a.exprLoc.Row + } + return a.col - b.col + }) + + if len(byCol) == 0 { + return + } + + w.WriteString("\n") + printArrows(w, byCol, -1) + for i := len(byCol) - 1; i >= 0; i-- { + w.WriteString("\n") + printArrows(w, byCol, i) + } +} + +func printArrows(w *bytes.Buffer, l []varInfo, printValueAt int) { + prevCol := 0 + var slice []varInfo + if printValueAt >= 0 { + slice = l[:printValueAt+1] + } else { + slice = l + } + isFirst := true + for i, info := range slice { + + isLast := i >= len(slice)-1 + col := info.col + + if !isLast && col == l[i+1].col { + // We're sharing the same column with another, subsequent var + continue + } + + spaces := col - 1 + if i > 0 && !isFirst { + spaces = (col - prevCol) - 1 + } + + for j := 0; j < spaces; j++ { + tab := false + for _, t := range info.exprLoc.Tabs { + if t == j+prevCol+1 { + w.WriteString("\t") + tab = true + break + } + } + if !tab { + w.WriteString(" ") + } + } + + if isLast && printValueAt >= 0 { + valueStr := iStrs.Truncate(info.Value(), maxPrettyExprVarWidth) + if (i > 0 && col == l[i-1].col) || (i < len(l)-1 && col == l[i+1].col) { + // There is another var on this column, so we need to include the name to differentiate them. + w.WriteString(fmt.Sprintf("%s: %s", info.Title(), valueStr)) + } else { + w.WriteString(valueStr) + } + } else { + w.WriteString("|") + } + prevCol = col + isFirst = false + } +} + func init() { RegisterBuiltinFunc(ast.Trace.Name, builtinTrace) } diff --git a/topdown/trace_test.go b/topdown/trace_test.go index ac4d9db159..dec5496cb0 100644 --- a/topdown/trace_test.go +++ b/topdown/trace_test.go @@ -1299,3 +1299,173 @@ func removeUnifyOps(trace []*Event) (result []*Event) { } return } + +func TestPrettyTraceWithLocalVars(t *testing.T) { + { + module := `package test +import rego.v1 + +p if { + x := 1 + y := 2 + z := do_math(x, y) + z == 3 +} + +do_math(a, b) := c if { + c := a + b +} +` + + ctx := context.Background() + compiler := compileModules([]string{module}) + data := loadSmallTestData() + store := inmem.NewFromObject(data) + txn := storage.NewTransactionOrDie(ctx, store) + defer store.Abort(ctx, txn) + + tracer := NewBufferTracer() + query := NewQuery(ast.MustParseBody("data.test = _")). + WithCompiler(compiler). + WithStore(store). + WithTransaction(txn). + WithTracer(tracer) + + _, err := query.Run(ctx) + if err != nil { + panic(err) + } + + expected := `Enter data.test = _ {} +| Eval data.test = _ {} +| Unify data.test = _ {} +| Unify data.test.p = _ {} +| Index data.test.do_math (matched 1 rule) {} +| Unify data.test.p = _ {} +| Index data.test.p (matched 1 rule, early exit) {} +| Enter data.test.p {} +| | Eval x = 1 {} +| | Unify x = 1 {} +| | Eval y = 2 {__local0__: 1} +| | Unify y = 2 {__local0__: 1} +| | Eval data.test.do_math(x, y, __local6__) {__local0__: 1, __local1__: 2} +| | Index data.test.do_math (matched 1 rule) {__local0__: 1, __local1__: 2} +| | Enter data.test.do_math {} +| | | Unify 1 = a {} +| | | Unify 2 = b {__local3__: 1} +| | | Unify __local6__ = c {__local3__: 1, __local4__: 2} +| | | Eval plus(a, b, __local7__) {__local3__: 1, __local4__: 2} +| | | Unify __local7__ = 3 {__local3__: 1, __local4__: 2} +| | | Eval c = __local7__ {__local3__: 1, __local4__: 2, __local7__: 3} +| | | Unify c = 3 {__local3__: 1, __local4__: 2, __local7__: 3} +| | | Exit data.test.do_math {__local3__: 1, __local4__: 2, __local5__: 3, __local7__: 3} +| | Eval z = __local6__ {__local0__: 1, __local1__: 2, __local6__: 3} +| | Unify z = 3 {__local0__: 1, __local1__: 2, __local6__: 3} +| | Eval z = 3 {__local0__: 1, __local1__: 2, __local2__: 3, __local6__: 3} +| | Unify 3 = 3 {__local0__: 1, __local1__: 2, __local2__: 3, __local6__: 3} +| | Exit data.test.p early {__local0__: 1, __local1__: 2, __local2__: 3, __local6__: 3} +| Unify true = _ {} +| Redo data.test.p {__local0__: 1, __local1__: 2, __local2__: 3, __local6__: 3} +| | Redo z = 3 {__local0__: 1, __local1__: 2, __local2__: 3, __local6__: 3} +| | Redo z = __local6__ {__local0__: 1, __local1__: 2, __local2__: 3, __local6__: 3} +| | Redo data.test.do_math(x, y, __local6__) {__local0__: 1, __local1__: 2, __local6__: 3} +| | | Redo c = __local7__ {__local3__: 1, __local4__: 2, __local5__: 3, __local7__: 3} +| | | Redo plus(a, b, __local7__) {__local3__: 1, __local4__: 2, __local7__: 3} +| | Redo y = 2 {__local0__: 1, __local1__: 2} +| | Redo x = 1 {__local0__: 1} +| Unify _ = {"p": true} {} +| Exit data.test = _ {_: {"p": true}} +Redo data.test = _ {_: {"p": true}} +| Redo data.test = _ {_: {"p": true}} +` + + var buf bytes.Buffer + PrettyTraceWithOpts(&buf, *tracer, PrettyTraceOptions{LocalVariables: true}) + compareBuffers(t, expected, buf.String()) + } +} + +func TestPrettyTraceExprVars(t *testing.T) { + { + module := `package test +import rego.v1 + +p if { + x := 1 + y := 2 + z := do_math(x, y) + z == 3 +} + +do_math(a, b) := c if { + c := a + b +} +` + + ctx := context.Background() + compiler := compileModules([]string{module}) + data := loadSmallTestData() + store := inmem.NewFromObject(data) + txn := storage.NewTransactionOrDie(ctx, store) + defer store.Abort(ctx, txn) + + tracer := NewBufferTracer() + query := NewQuery(ast.MustParseBody("data.test = _")). + WithCompiler(compiler). + WithStore(store). + WithTransaction(txn). + WithTracer(tracer) + + _, err := query.Run(ctx) + if err != nil { + panic(err) + } + + expected := `Enter data.test = _ {} +| Eval data.test = _ {} +| Unify data.test = _ {} +| Unify data.test.p = _ {} +| Index data.test.do_math (matched 1 rule) {} +| Unify data.test.p = _ {} +| Index data.test.p (matched 1 rule, early exit) {} +| Enter data.test.p {} +| | Eval x = 1 {} +| | Unify x = 1 {} +| | Eval y = 2 {} +| | Unify y = 2 {} +| | Eval data.test.do_math(x, y, __local6__) {x: 1, y: 2} +| | Index data.test.do_math (matched 1 rule) {x: 1, y: 2} +| | Enter data.test.do_math {} +| | | Unify 1 = a {} +| | | Unify 2 = b {} +| | | Unify __local6__ = c {} +| | | Eval plus(a, b, __local7__) {a: 1, b: 2} +| | | Unify __local7__ = 3 {} +| | | Eval c = __local7__ {__local7__: 3} +| | | Unify c = 3 {} +| | | Exit data.test.do_math {a: 1, b: 2, c: 3} +| | Eval z = __local6__ {__local6__: 3} +| | Unify z = 3 {} +| | Eval z = 3 {z: 3} +| | Unify 3 = 3 {} +| | Exit data.test.p early {} +| Unify true = _ {} +| Redo data.test.p {} +| | Redo z = 3 {z: 3} +| | Redo z = __local6__ {__local6__: 3, z: 3} +| | Redo data.test.do_math(x, y, __local6__) {__local6__: 3, x: 1, y: 2} +| | | Redo c = __local7__ {__local7__: 3, c: 3} +| | | Redo plus(a, b, __local7__) {__local7__: 3, a: 1, b: 2} +| | Redo y = 2 {y: 2} +| | Redo x = 1 {x: 1} +| Unify _ = {"p": true} {} +| Exit data.test = _ {_: {"p": true}} +Redo data.test = _ {_: {"p": true}} +| Redo data.test = _ {_: {"p": true}} +` + + var buf bytes.Buffer + PrettyTraceWithOpts(&buf, *tracer, PrettyTraceOptions{ExprVariables: true}) + compareBuffers(t, expected, buf.String()) + } +}