Add C library implementing operations for WASM

These changes add a C library that implements low-level data operations
and JSON parsing for WASM policies. The output of the WASM build process
are checked into the repository so that OPA can easily access the
bytecode for test and other purposes.

Signed-off-by: Torin Sandall <[email protected]>

.gitignore

@@ -8,6 +8,7 @@ coverage
 opa_*
 .Dockerfile_*
 _release
+wasm/_obj
 site.tar.gz
 policy.wasm
 

Makefile

@@ -9,6 +9,9 @@ GOVERSION := 1.10
 GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)
 
+DOCKER_INSTALLED := $(shell hash docker 2>/dev/null && echo 1 || echo 0)
+DOCKER := docker
+
 BIN := opa_$(GOOS)_$(GOARCH)
 
 REPOSITORY := openpolicyagent
@@ -45,12 +48,24 @@ version:
 deps:
 	@./build/install-deps.sh
 
+.PHONY: wasm-build
+wasm-build:
+ifeq ($(DOCKER_INSTALLED), 1)
+	@$(MAKE) -C wasm build
+	cp wasm/_obj/opa.wasm internal/compiler/wasm/opa/opa.wasm
+else
+	@echo "Docker not installed. Skipping OPA-WASM library build."
+endif
+
 .PHONY: generate
-generate:
+generate: wasm-build
 	$(GO) generate
 
 .PHONY: build
-build: generate
+build: go-build
+
+.PHONY: go-build
+go-build: generate
 	$(GO) build -o $(BIN) -ldflags $(LDFLAGS)
 
 .PHONY: image
@@ -95,9 +110,27 @@ install: generate
 	$(GO) install -ldflags $(LDFLAGS)
 
 .PHONY: test
-test: generate
+test: opa-wasm-test go-test wasm-test
+
+.PHONY: opa-wasm-test
+opa-wasm-test:
+ifeq ($(DOCKER_INSTALLED), 1)
+	@$(MAKE)  -C wasm test
+else
+	@echo "Docker not installed. Skipping OPA-WASM library test."
+endif
+
+.PHONY: go-test
+go-test: generate
 	$(GO) test ./...
 
+.PHONY: wasm-test
+wasm-test:
+ifeq ($(DOCKER_INSTALLED), 1)
+else
+	@echo "Docker not installed. Skipping WASM-based test execution."
+endif
+
 .PHONY: perf
 perf: generate
 	$(GO) test -run=- -bench=. -benchmem ./...
@@ -121,8 +154,12 @@ check-lint:
 fmt:
 	$(GO) fmt ./...
 
+.PHONY: wasm-clean
+wasm-clean:
+	@$(MAKE) -C wasm clean
+
 .PHONY: clean
-clean:
+clean: wasm-clean
 	rm -f .Dockerfile_*
 	rm -f opa_*_*
 	rm -fr site.tar.gz docs/_site docs/node_modules docs/book/_book docs/book/node_modules

internal/cmd/genopawasm/main.go

@@ -0,0 +1,135 @@
+// Copyright 2018 The OPA Authors.  All rights reserved.
+// Use of this source code is governed by an Apache2
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path"
+
+	"github.com/spf13/cobra"
+)
+
+type params struct {
+	Output string
+}
+
+func main() {
+
+	var params params
+	executable := path.Base(os.Args[0])
+
+	command := &cobra.Command{
+		Use:   executable,
+		Short: executable + " <opa.wasm path>",
+		RunE: func(_ *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return fmt.Errorf("provide path of opa.wasm file")
+			}
+			return run(params, args)
+		},
+	}
+
+	command.Flags().StringVarP(&params.Output, "output", "o", "", "set path of output file (default: stdout)")
+
+	if err := command.Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func run(params params, args []string) error {
+
+	var out io.Writer
+
+	if params.Output != "" {
+		f, err := os.Create(params.Output)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		out = f
+	} else {
+		out = os.Stdout
+	}
+
+	_, err := out.Write([]byte(`// Copyright 2018 The OPA Authors.  All rights reserved.
+// Use of this source code is governed by an Apache2
+// license that can be found in the LICENSE file.
+
+// THIS FILE IS GENERATED. DO NOT EDIT.
+
+// Package opa contains bytecode for the OPA-WASM library.
+package opa
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+)
+
+// Bytes returns the OPA-WASM bytecode.
+func Bytes() ([]byte, error) {
+	gr, err := gzip.NewReader(bytes.NewBuffer(gzipped))
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.ReadAll(gr)
+}
+
+`))
+
+	if err != nil {
+		return err
+	}
+
+	_, err = out.Write([]byte(`var gzipped = []byte("`))
+	if err != nil {
+		return err
+	}
+
+	in, err := os.Open(args[0])
+	if err != nil {
+		return err
+	}
+
+	defer in.Close()
+
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	_, err = io.Copy(gw, in)
+	if err != nil {
+		return err
+	}
+
+	gw.Close()
+
+	for _, b := range buf.Bytes() {
+		if _, err := out.Write([]byte(`\x`)); err != nil {
+			return err
+		}
+		if _, err := out.Write(asciihex(b)); err != nil {
+			return err
+		}
+	}
+
+	_, err = out.Write([]byte(`")`))
+	if err != nil {
+		return err
+	}
+
+	_, err = out.Write([]byte("\n"))
+	return err
+}
+
+var digits = "0123456789ABCDEF"
+
+func asciihex(b byte) []byte {
+	lo := digits[(b & 0x0F)]
+	hi := digits[(b >> 4)]
+	return []byte{hi, lo}
+}

main.go

@@ -19,3 +19,4 @@ func main() {
 //
 //go:generate pigeon -o ast/parser.go ast/rego.peg
 //go:generate goimports -w ast/parser.go
+//go:generate go run internal/cmd/genopawasm/main.go -o internal/compiler/wasm/opa/opa.go internal/compiler/wasm/opa/opa.wasm

wasm/Dockerfile

@@ -0,0 +1,33 @@
+FROM ubuntu:18.04
+
+RUN apt-get update && apt-get install -y curl git build-essential
+
+RUN bash -c 'echo -ne "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main\ndeb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" > /etc/apt/sources.list.d/llvm.list'
+
+RUN curl -L https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+
+RUN apt-get update && \
+    apt-get install -y \
+    cmake \
+    clang-8 \
+    lld-8
+
+ENV CC=clang-8
+ENV CXX=clang-8
+ENV LLD=wasm-ld-8
+ENV AR=llvm-ar-8
+ENV RANLIB=llvm-ranlib-8
+
+RUN ln -s /usr/bin/clang-8 /usr/bin/clang && \
+    ln -s /usr/bin/clang++-8 /usr/bin/clang++ && \
+    ln -s /usr/bin/clang-cpp-8 /usr/bin/clang-cpp
+
+RUN git clone https://github.com/WebAssembly/wabt && \
+    cd wabt && \
+    git checkout 1.0.5 && \
+    git submodule update --init && \
+    make
+
+ENV PATH="/wabt/out/clang/Debug:${PATH}"
+
+WORKDIR /src

wasm/Makefile

@@ -0,0 +1,79 @@
+DOCKER := docker
+
+WASM_BUILDER_REPOSITORY := openpolicyagent/opa-wasm-builder
+WASM_BUILDER_VERSION := 1.0
+WASM_BUILDER_IMAGE := $(WASM_BUILDER_REPOSITORY):$(WASM_BUILDER_VERSION)
+WASM_OBJ_DIR := _obj
+
+$(shell mkdir -p $(WASM_OBJ_DIR))
+
+CFLAGS := \
+	-O3 \
+	-nostdinc \
+	-nodefaultlibs \
+	--target=wasm32-unknown-unknown-wasm
+
+.PHONY: all
+all: build test
+
+.PHONY: clean
+clean:
+	rm -fr $(WASM_OBJ_DIR)
+
+.PHONY: builder
+builder: Dockerfile
+	@$(DOCKER) build -t $(WASM_BUILDER_IMAGE) -f Dockerfile .
+
+.PHONY: build
+build:
+	@$(DOCKER) run -it --rm -v $(CURDIR):/src $(WASM_BUILDER_IMAGE) make $(WASM_OBJ_DIR)/opa.wasm
+
+.PHONY: test
+test:
+	@$(DOCKER) run -it --rm -v $(CURDIR):/src $(WASM_BUILDER_IMAGE) make $(WASM_OBJ_DIR)/opa-test.wasm
+	@$(DOCKER) run -it --rm -e VERBOSE=$(VERBOSE) -v $(CURDIR):/src -w /src node:8 node test.js $(WASM_OBJ_DIR)/opa-test.wasm
+
+.PHONY: hack
+hack:
+	@$(DOCKER) run -it --rm -v $(CURDIR):/src $(WASM_BUILDER_IMAGE)
+
+$(WASM_OBJ_DIR)/malloc.wasm: src/malloc.c
+	@$(CC) $(CFLAGS) -c $^ -o $@
+
+$(WASM_OBJ_DIR)/string.wasm: src/string.c
+	@$(CC) $(CFLAGS) -c $^ -o $@
+
+$(WASM_OBJ_DIR)/json.wasm: src/json.c
+	@$(CC) $(CFLAGS) -c $^ -o $@
+
+$(WASM_OBJ_DIR)/value.wasm: src/value.c
+	@$(CC) $(CFLAGS) -c $^ -o $@
+
+$(WASM_OBJ_DIR)/opa.wasm: $(WASM_OBJ_DIR)/malloc.wasm \
+		$(WASM_OBJ_DIR)/value.wasm \
+		$(WASM_OBJ_DIR)/string.wasm \
+		$(WASM_OBJ_DIR)/json.wasm
+	@wasm-ld-8 \
+			--allow-undefined-file=src/undefined.symbols \
+			--import-memory \
+			--no-entry \
+			--export-all \
+			$^ \
+			-o $@
+	@wasm2wat $(WASM_OBJ_DIR)/opa.wasm > $(WASM_OBJ_DIR)/opa.wast
+
+$(WASM_OBJ_DIR)/test.wasm: tests/test.c
+	@$(CC) $(CFLAGS) -I src -c $^ -o $@
+
+$(WASM_OBJ_DIR)/opa-test.wasm: $(WASM_OBJ_DIR)/test.wasm \
+		$(WASM_OBJ_DIR)/malloc.wasm \
+		$(WASM_OBJ_DIR)/value.wasm \
+		$(WASM_OBJ_DIR)/string.wasm \
+		$(WASM_OBJ_DIR)/json.wasm
+	@wasm-ld-8 \
+			--allow-undefined-file=tests/undefined.symbols \
+			--import-memory \
+			--no-entry \
+			--export-all \
+			$^ \
+			-o $@

wasm/README.md

@@ -0,0 +1,42 @@
+# OPA-WASM
+
+This directory contains a library that implements various low-level
+operations for policies compiled into WebAssembly (WASM). Specifically, the
+library implements:
+
+* JSON parsing
+* JSON AST (e.g., comparison, iteration, lookup, etc.)
+* String operations
+* Memory allocation
+
+This library does not make any backwards compatibility guarantees.
+
+## Development
+
+You should have Docker installed to build and test changes to the library. We
+commit the output of the build (`opa.wasm`) into the repository so it's
+important for the build output to be reproducible.
+
+You can build the library by running `make build`. This will produce WASM
+executables under the `_obj` directory.
+
+You can test the library by running `make test`. By default the test runner
+does not print messages when tests pass. If you run `make test VERBOSE=1` it
+will log all of the tests that were run.
+
+You can run `make hack` to start a shell inside the builder image. This is
+useful if you need to interact with low-level WASM tooling like
+`wasm-objdump`, `wasm2wat`, etc. or LLVM itself.
+
+You must manually push the builder image if you make changes to it (run `make
+builder` to produce a new Docker image).
+
+## Vendoring
+
+If you make changes to the library, run the `make generate` in the parent
+directory and commit the results back to the repository. The `generate`
+target will:
+
+1. Build the OPA-WASM library
+2. Copy the library into the [internal/compiler/wasm/opa](../internal/compiler/wasm/opa) directory.
+3. Run the tool to generate the [internal/compiler/wasm/opa/opa.go](../internal/compiler/wasm/opa/opa.go) file.