Add basic REST API support to server mode

- REST APIs

    * CRUDL on policy modules
    * Ad-hoc queries
    * Query and patch base documents
    * Query virtual documents

- Add PolicyStore to manage policy definition/module CRUDL operations.

    * Supports persistence of policy definitons.
    * Serve REST API CRUDL operations.
    * Manage install/uninstall of rules into data store.
    * Manage persistence of policy definitions.

- Misc. refactoring

    * Move storage creation into runtime Init.
    * Make AST types JSON serializable. Tweaked ast.Import to use Term instead
    of Value for the path.

.gitignore

@@ -1,5 +1,11 @@
+# development environment
 .DS_Store
 .vscode
+
+# build artifacts
+coverage
 opa
 ast/parser.go
-coverage
+
+# runtime artifacts
+policies

Makefile

@@ -37,14 +37,17 @@ generate:
 build: generate
 	$(GO) build -o opa $(LDFLAGS)
 
+install: generate
+	$(GO) install $(LDFLAGS)
+
 test: generate
 	$(GO) test -v $(PACKAGES)
 
 COVER_PACKAGES=$(PACKAGES)
 $(COVER_PACKAGES):
 	@mkdir -p coverage/$(shell dirname $@)
-	go test -covermode=count -coverprofile=coverage/$(shell dirname $@)/coverage.out $@
-	go tool cover -html=coverage/$(shell dirname $@)/coverage.out || true
+	$(GO) test -covermode=count -coverprofile=coverage/$(shell dirname $@)/coverage.out $@
+	$(GO) tool cover -html=coverage/$(shell dirname $@)/coverage.out || true
 
 cover: $(COVER_PACKAGES)
 

ast/compile.go

@@ -196,7 +196,7 @@ func (c *Compiler) setGlobals() {
 		// Populate globals with imports within this module.
 		for _, i := range m.Imports {
 			if len(i.Alias) > 0 {
-				switch p := i.Path.(type) {
+				switch p := i.Path.Value.(type) {
 				case Ref:
 					globals[i.Alias] = p
 				case Var:
@@ -205,7 +205,7 @@ func (c *Compiler) setGlobals() {
 					c.err("unexpected %T: %v", p, i)
 				}
 			} else {
-				switch p := i.Path.(type) {
+				switch p := i.Path.Value.(type) {
 				case Ref:
 					switch v := p[len(p)-1].Value.(type) {
 					case String:

ast/parser_test.go

@@ -200,12 +200,12 @@ func TestPackage(t *testing.T) {
 }
 
 func TestImport(t *testing.T) {
-	assertParseImport(t, "single", "import foo", &Import{Path: VarTerm("foo").Value})
-	ref := RefTerm(VarTerm("foo"), StringTerm("bar"), StringTerm("baz")).Value
+	assertParseImport(t, "single", "import foo", &Import{Path: VarTerm("foo")})
+	ref := RefTerm(VarTerm("foo"), StringTerm("bar"), StringTerm("baz"))
 	assertParseImport(t, "multiple", "import foo.bar.baz", &Import{Path: ref})
-	assertParseImport(t, "single alias", "import foo as bar", &Import{Path: VarTerm("foo").Value, Alias: Var("bar")})
+	assertParseImport(t, "single alias", "import foo as bar", &Import{Path: VarTerm("foo"), Alias: Var("bar")})
 	assertParseImport(t, "multiple alias", "import foo.bar.baz as qux", &Import{Path: ref, Alias: Var("qux")})
-	ref2 := RefTerm(VarTerm("foo"), StringTerm("bar"), StringTerm("white space")).Value
+	ref2 := RefTerm(VarTerm("foo"), StringTerm("bar"), StringTerm("white space"))
 	assertParseImport(t, "white space", "import foo.bar[\"white space\"]", &Import{Path: ref2})
 	assertParseError(t, "non-ground ref", "import foo[x]")
 }

ast/policy.go

@@ -4,8 +4,11 @@
 
 package ast
 
-import "fmt"
-import "strings"
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
 
 // DefaultRootDocument is the default root document.
 // All package directives inside source files are implicitly
@@ -31,25 +34,25 @@ type (
 	// Package represents the namespace of the documents produced
 	// by rules inside the module.
 	Package struct {
-		Location *Location
+		Location *Location `json:"-"`
 		Path     Ref
 	}
 
 	// Import represents a dependency on a document outside of the policy
 	// namespace. Imports are optional.
 	Import struct {
-		Location *Location
-		Path     Value
-		Alias    Var
+		Location *Location `json:"-"`
+		Path     *Term
+		Alias    Var `json:",omitempty"`
 	}
 
 	// Rule represents a rule as defined in the language. Rules define the
 	// content of documents that represent policy decisions.
 	Rule struct {
-		Location *Location
+		Location *Location `json:"-"`
 		Name     Var
-		Key      *Term
-		Value    *Term
+		Key      *Term `json:",omitempty"`
+		Value    *Term `json:",omitempty"`
 		Body     Body
 	}
 
@@ -58,8 +61,8 @@ type (
 
 	// Expr represents a single expression contained inside the body of a rule.
 	Expr struct {
-		Location *Location
-		Negated  bool
+		Location *Location `json:"-"`
+		Negated  bool      `json:",omitempty"`
 		Terms    interface{}
 	}
 )
@@ -264,6 +267,51 @@ func (expr *Expr) String() string {
 	return strings.Join(buf, " ")
 }
 
+// UnmarshalJSON parses the byte array and stores the result in expr.
+func (expr *Expr) UnmarshalJSON(bs []byte) error {
+	v := map[string]interface{}{}
+	if err := json.Unmarshal(bs, &v); err != nil {
+		return err
+	}
+
+	n, ok := v["Negated"]
+	if !ok {
+		expr.Negated = false
+	} else {
+		b, ok := n.(bool)
+		if !ok {
+			return unmarshalError(n, "bool")
+		}
+		expr.Negated = b
+	}
+
+	switch ts := v["Terms"].(type) {
+	case map[string]interface{}:
+		v, err := unmarshalValue(ts)
+		if err != nil {
+			return err
+		}
+		expr.Terms = &Term{Value: v}
+	case []interface{}:
+		buf := []*Term{}
+		for _, v := range ts {
+			e, ok := v.(map[string]interface{})
+			if !ok {
+				return unmarshalError(v, "map[string]interface{}")
+			}
+			v, err := unmarshalValue(e)
+			if err != nil {
+				return err
+			}
+			buf = append(buf, &Term{Value: v})
+		}
+		expr.Terms = buf
+	default:
+		return unmarshalError(v["Terms"], "Term or []Term")
+	}
+	return nil
+}
+
 // NewBuiltinExpr creates a new Expr object with the supplied terms.
 // The builtin operator must be the first term.
 func NewBuiltinExpr(terms ...*Term) *Expr {

ast/policy_test.go

@@ -4,7 +4,38 @@
 
 package ast
 
-import "testing"
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+)
+
+func TestModuleJSONRoundTrip(t *testing.T) {
+	mod := MustParseModule(`
+	package a.b.c
+	import data.x.y as z
+	import data.u.i
+	p = [1,2,{"foo":3}] :- r[x] = 1, not q[x]
+	r[y] = v :- i[1] = y, v = i[2]
+	q[x] :- a=[true,false,null,{"x":[1,2,3]}], a[i] = x
+	`)
+
+	bs, err := json.Marshal(mod)
+	if err != nil {
+		panic(err)
+	}
+
+	roundtrip := &Module{}
+
+	err = json.Unmarshal(bs, roundtrip)
+	if err != nil {
+		panic(err)
+	}
+
+	if !roundtrip.Equal(mod) {
+		t.Errorf("Expected roundtripped module to be equal to original:\nExpected:\n\n%v\n\nGot:\n\n%v\n", mod, roundtrip)
+	}
+}
 
 func TestPackageEquals(t *testing.T) {
 	pkg1 := &Package{Path: RefTerm(VarTerm("foo"), StringTerm("bar"), StringTerm("baz")).Value.(Ref)}
@@ -26,12 +57,12 @@ func TestPackageString(t *testing.T) {
 }
 
 func TestImportEquals(t *testing.T) {
-	imp1 := &Import{Path: Var("foo"), Alias: Var("bar")}
-	imp11 := &Import{Path: Var("foo"), Alias: Var("bar")}
-	imp2 := &Import{Path: Var("foo")}
-	imp3 := &Import{Path: RefTerm(VarTerm("bar"), VarTerm("baz"), VarTerm("qux")).Value, Alias: Var("corge")}
-	imp33 := &Import{Path: RefTerm(VarTerm("bar"), VarTerm("baz"), VarTerm("qux")).Value, Alias: Var("corge")}
-	imp4 := &Import{Path: RefTerm(VarTerm("bar"), VarTerm("baz"), VarTerm("qux")).Value}
+	imp1 := &Import{Path: VarTerm("foo"), Alias: Var("bar")}
+	imp11 := &Import{Path: VarTerm("foo"), Alias: Var("bar")}
+	imp2 := &Import{Path: VarTerm("foo")}
+	imp3 := &Import{Path: RefTerm(VarTerm("bar"), VarTerm("baz"), VarTerm("qux")), Alias: Var("corge")}
+	imp33 := &Import{Path: RefTerm(VarTerm("bar"), VarTerm("baz"), VarTerm("qux")), Alias: Var("corge")}
+	imp4 := &Import{Path: RefTerm(VarTerm("bar"), VarTerm("baz"), VarTerm("qux"))}
 	assertImportsEqual(t, imp1, imp1)
 	assertImportsEqual(t, imp1, imp11)
 	assertImportsEqual(t, imp3, imp3)
@@ -47,10 +78,10 @@ func TestImportEquals(t *testing.T) {
 }
 
 func TestImportString(t *testing.T) {
-	imp1 := &Import{Path: Var("foo"), Alias: Var("bar")}
-	imp2 := &Import{Path: Var("foo")}
-	imp3 := &Import{Path: RefTerm(VarTerm("bar"), StringTerm("baz"), StringTerm("qux")).Value, Alias: Var("corge")}
-	imp4 := &Import{Path: RefTerm(VarTerm("bar"), StringTerm("baz"), StringTerm("qux")).Value}
+	imp1 := &Import{Path: VarTerm("foo"), Alias: Var("bar")}
+	imp2 := &Import{Path: VarTerm("foo")}
+	imp3 := &Import{Path: RefTerm(VarTerm("bar"), StringTerm("baz"), StringTerm("qux")), Alias: Var("corge")}
+	imp4 := &Import{Path: RefTerm(VarTerm("bar"), StringTerm("baz"), StringTerm("qux"))}
 	assertImportToString(t, imp1, "import foo as bar")
 	assertImportToString(t, imp2, "import foo")
 	assertImportToString(t, imp3, "import bar.baz.qux as corge")
@@ -126,6 +157,48 @@ func TextExprString(t *testing.T) {
 	assertExprString(t, expr4, "ne({foo: [1, a.b]}, false)")
 }
 
+func TestExprBadJSON(t *testing.T) {
+
+	assert := func(js string, exp error) {
+		expr := Expr{}
+		err := json.Unmarshal([]byte(js), &expr)
+		if !reflect.DeepEqual(exp, err) {
+			t.Errorf("Expected %v but got: %v", exp, err)
+		}
+	}
+
+	js := `
+	{
+		"Negated": 100,
+		"Terms": {
+			"Value": "foo",
+			"Type": "string"
+		}
+	}
+	`
+
+	exp := unmarshalError(100.0, "bool")
+	assert(js, exp)
+
+	js = `
+	{
+		"Terms": [
+			"foo"
+		]
+	}
+	`
+	exp = unmarshalError("foo", "map[string]interface{}")
+	assert(js, exp)
+
+	js = `
+	{
+		"Terms": "bad value" 
+	}
+	`
+	exp = unmarshalError("bad value", "Term or []Term")
+	assert(js, exp)
+}
+
 func TestRuleHeadEquals(t *testing.T) {
 	assertRulesEqual(t, &Rule{}, &Rule{})
 

ast/rego.peg

@@ -74,8 +74,8 @@ Package <- "package" ws val:(Ref / Var) {
 Import <- "import" ws path:(Ref / Var) alias:(ws "as" ws Var)? {
     imp := &Import{}
     imp.Location = currentLocation(c)
-    imp.Path = path.(*Term).Value
-    switch p := imp.Path.(type) {
+    imp.Path = path.(*Term)
+    switch p := imp.Path.Value.(type) {
     case Ref:
         if !p.IsGround() {
             return nil, fmt.Errorf("import cannot contain variables in tail: %v", p)