Add default output validator

BUILD.bazel

@@ -0,0 +1,4 @@
+load("@bazel_gazelle//:def.bzl", "gazelle")
+
+# gazelle:prefix github.com/jsannemo/omogenexec
+gazelle(name = "gazelle")

WORKSPACE

@@ -16,3 +16,69 @@ rust_repositories()
 load("//cargo:crates.bzl", "raze_fetch_remote_crates")
 
 raze_fetch_remote_crates()
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rules_pkg",
+    sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
+        "https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
+    ],
+)
+
+load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
+rules_pkg_dependencies()
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "io_bazel_rules_go",
+    sha256 = "69de5c704a05ff37862f7e0f5534d4f479418afc21806c887db544a316f3cb6b",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
+    ],
+)
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.16")
+
+http_archive(
+    name = "bazel_gazelle",
+    sha256 = "62ca106be173579c0a167deb23358fdfe71ffa1e4cfdddf5582af26520f1c66f",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.23.0/bazel-gazelle-v0.23.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.23.0/bazel-gazelle-v0.23.0.tar.gz",
+    ],
+)
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
+
+go_repository(
+    name = "com_github_google_logger",
+    importpath = "github.com/google/logger",
+    sum = "h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ=",
+    version = "v1.1.1",
+)
+
+gazelle_dependencies()
+
+http_archive(
+    name = "com_google_protobuf",
+    sha256 = "d0f5f605d0d656007ce6c8b5a82df3037e1d8fe8b121ed42e536f569dec16113",
+    strip_prefix = "protobuf-3.14.0",
+    urls = [
+        "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v3.14.0.tar.gz",
+        "https://github.com/protocolbuffers/protobuf/archive/v3.14.0.tar.gz",
+    ],
+)
+
+load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+
+protobuf_deps()

go.mod

@@ -0,0 +1,5 @@
+module github.com/jsannemo/omogenexec
+
+go 1.16
+
+require github.com/google/logger v1.1.1

go.sum

@@ -0,0 +1,4 @@
+github.com/google/logger v1.1.1 h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ=
+github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

runner/diff/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "diff",
+    srcs = ["diff.go"],
+    importpath = "github.com/jsannemo/omogenexec/runner/diff",
+    visibility = ["//visibility:public"],
+    deps = ["@com_github_google_logger//:logger"],
+)
+
+go_test(
+    name = "diff_test",
+    srcs = ["diff_test.go"],
+    embed = [":diff"],
+)

runner/diff/diff.go

@@ -0,0 +1,220 @@
+// Package diff implements a structural, non-exact diff of two strings.
+//
+// The default mode is that the strings are compared in a token-by-token manner, where for example
+// tokens that can be parsed as floating-point integers are parsed as such according to some common
+// formats
+package diff
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/google/logger"
+	"io"
+	"math"
+	"strings"
+)
+
+// DiffResult describes a comparison of two strings.
+type DiffResult struct {
+	// Whether the strings matched.
+	Match bool
+	// A textual description of the difference.
+	Description string
+}
+
+// DiffArgs specifies rules
+type DiffArgs struct {
+	// Whether tokens representing floating point integers should be parsed as such and compared
+	// using precisions RelativePrec and AbsolutePrec rather than as strings.
+	ParseFloats bool
+	// Indicates that floating-point tokens should be accepted if they are within relative error ≤ ε
+	RelativePrec float64
+	// Indicates that floating-point tokens should be accepted if they are within absolute error ≤ ε
+	AbsolutePrec float64
+	// Indicates that comparisons should be case-sensitive
+	CaseSensitive bool
+	// Indicates that changes in the amount of whitespace should be rejected (the default is that
+	// any sequence of 1 or more whitespace characters are equivalent)
+	SpaceSensitive bool
+}
+
+// Diff compares the a Reader against a "correct" reference Reader by tokenizing them.
+func Diff(reference, output io.Reader, args DiffArgs) (*DiffResult, error) {
+	ref := newPositionedScanner(bufio.NewReader(reference), args.SpaceSensitive)
+	out := newPositionedScanner(bufio.NewReader(output), args.SpaceSensitive)
+	for {
+		refToken, err := ref.Scan()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		refStr := string(refToken.Token)
+		outToken, err := out.Scan()
+		if err == io.EOF {
+			return &DiffResult{false, fmt.Sprintf("Expected more output (next reference token: %s at %v)", refStr, refToken.Pos)}, nil
+		} else if err != nil {
+			return nil, err
+		}
+		outStr := string(outToken.Token)
+		match, desc := matchToken(refStr, outStr, args)
+		if !match {
+			return &DiffResult{false, fmt.Sprintf("%s (output %v, reference %v)", desc, outToken.Pos, refToken.Pos)}, nil
+		}
+	}
+	outToken, err := out.Scan()
+	if err != io.EOF {
+		if err != nil {
+			return nil, err
+		}
+		return &DiffResult{false, fmt.Sprintf("Too much output (next output token: %s at %v)", string(outToken.Token), outToken.Pos)}, nil
+	}
+	return &DiffResult{Match: true}, nil
+}
+
+func matchToken(ref, out string, args DiffArgs) (bool, string) {
+	logger.Infof("diff %s %s", ref, out)
+	if args.ParseFloats {
+		var refFloat, outFloat float64
+		strVal := ""
+		if scanned, _ := fmt.Sscanf(ref, "%f%s", &refFloat, &strVal); scanned == 1 {
+			if scanned, _ := fmt.Sscanf(out, "%f%s", &outFloat, &strVal); scanned != 1 {
+				return false, fmt.Sprintf("Reference was decimal, output was: %s", out)
+			}
+			logger.Infof("got decimals %f %f", refFloat, outFloat)
+			diff := math.Abs(refFloat - outFloat)
+			logger.Infof("float diff %f", diff)
+			if !(diff <= args.AbsolutePrec) &&
+				!(diff <= args.RelativePrec*math.Abs(refFloat)) {
+				return false, fmt.Sprintf(
+					"Too large decimal difference. Reference: %s, output: %s, difference: %f (absolute difference > %f and relative difference > %f)",
+					ref, out, diff, args.AbsolutePrec, args.RelativePrec)
+			}
+			return true, ""
+		}
+	}
+	diff := !strings.EqualFold(ref, out)
+	logger.Infof("diff w/o case? %v", diff)
+	if diff {
+		return false, fmt.Sprintf("Output was %s, expected %s", out, ref)
+	}
+	if args.CaseSensitive && ref != out {
+		return false, fmt.Sprintf("Output was %s, expected %s (difference in casing)", out, ref)
+	}
+	return true, ""
+}
+
+type position struct {
+	Line int
+	Col  int
+}
+
+func (p *position) String() string {
+	return fmt.Sprintf("%d:%d", p.Line, p.Col)
+}
+
+type positionedScanner struct {
+	reader         *bufio.Reader
+	spaceSensitive bool
+	pos            position
+}
+
+func newPositionedScanner(reader *bufio.Reader, spaceSensitive bool) positionedScanner {
+	return positionedScanner{
+		reader:         reader,
+		spaceSensitive: spaceSensitive,
+		pos: position{
+			Line: 1,
+			Col:  1,
+		},
+	}
+}
+
+type posToken struct {
+	Pos   position
+	Token []byte
+}
+
+func (sc *positionedScanner) Scan() (*posToken, error) {
+	// Fast-forward through spaces
+	if !sc.spaceSensitive {
+		for {
+			nextByte, err := sc.peekByte()
+			if err != nil {
+				return nil, err
+			}
+			if isSpace(nextByte) {
+				if _, err := sc.eatByte(); err != nil {
+					return nil, err
+				}
+			} else {
+				break
+			}
+		}
+	}
+	token := &posToken{
+		Pos:   sc.pos,
+		Token: make([]byte, 0),
+	}
+	for {
+		nextByte, err := sc.peekByte()
+		if err == io.EOF {
+			if len(token.Token) == 0 {
+				return token, io.EOF
+			} else {
+				return token, nil
+			}
+		} else if err != nil {
+			return nil, err
+		}
+		if isSpace(nextByte) {
+			if !sc.spaceSensitive {
+				if len(token.Token) == 0 {
+					logger.Fatal("Peeked space after empty token!?")
+				}
+				return token, nil
+			} else {
+				// If we're space sensitive, single spaces are tokens, but spaces should never be
+				// included with a token if we had non-space chars already
+				if len(token.Token) == 0 {
+					if _, err := sc.eatByte(); err != nil {
+						return nil, err
+					}
+					token.Token = []byte{nextByte}
+				}
+				return token, nil
+			}
+		} else {
+			if _, err := sc.eatByte(); err != nil {
+				return nil, err
+			}
+			token.Token = append(token.Token, nextByte)
+		}
+	}
+}
+
+func (sc *positionedScanner) peekByte() (byte, error) {
+	peek, err := sc.reader.Peek(1)
+	if err != nil {
+		return 0, err
+	}
+	return peek[0], nil
+}
+
+func (sc *positionedScanner) eatByte() (byte, error) {
+	nextByte, err := sc.reader.ReadByte()
+	if err != nil {
+		return 0, err
+	}
+	if nextByte == '\n' {
+		sc.pos.Line++
+		sc.pos.Col = 1
+	} else {
+		sc.pos.Col++
+	}
+	return nextByte, nil
+}
+
+func isSpace(b byte) bool {
+	return (9 <= b && b <= 13) || b == 32
+}