From 7ec53fad6aaeaf8b7c79d786cec6de810ccd9324 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 5 Jun 2020 18:24:41 +0200 Subject: [PATCH 01/77] Remove executor --- cmd/lbadd/main.go | 14 +------------- internal/executor/doc.go | 3 --- internal/executor/executor.go | 20 -------------------- internal/executor/result.go | 11 ----------- internal/executor/simple_executor.go | 26 -------------------------- internal/node/node.go | 11 ++++------- 6 files changed, 5 insertions(+), 80 deletions(-) delete mode 100644 internal/executor/doc.go delete mode 100644 internal/executor/executor.go delete mode 100644 internal/executor/result.go delete mode 100644 internal/executor/simple_executor.go diff --git a/cmd/lbadd/main.go b/cmd/lbadd/main.go index fe33826d..ce3ed6ce 100644 --- a/cmd/lbadd/main.go +++ b/cmd/lbadd/main.go @@ -11,7 +11,6 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/diode" "github.com/spf13/cobra" - "github.com/tomarrell/lbadd/internal/executor" "github.com/tomarrell/lbadd/internal/node" ) @@ -157,9 +156,7 @@ func startNode(cmd *cobra.Command, args []string) { Str("dbfile", databaseFile). Logger() - exec := createExecutor(log, databaseFile) - - node := node.New(nodeLog, exec) + node := node.New(nodeLog) if err := node.ListenAndServe(cmd.Context(), addr); err != nil { log.Error(). Err(err). @@ -206,12 +203,3 @@ func createLogger(stdin io.Reader, stdout, stderr io.Writer) zerolog.Logger { return log } - -func createExecutor(log zerolog.Logger, databaseFile string) executor.Executor { - execLog := log.With(). - Str("component", "executor"). - Logger() - - exec := executor.New(execLog, databaseFile) - return exec -} diff --git a/internal/executor/doc.go b/internal/executor/doc.go deleted file mode 100644 index 0cda51b3..00000000 --- a/internal/executor/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package executor implements executors, that can execute a command. The -// command is converted from an *ast.SQLStmt. -package executor diff --git a/internal/executor/executor.go b/internal/executor/executor.go deleted file mode 100644 index 9e7ac255..00000000 --- a/internal/executor/executor.go +++ /dev/null @@ -1,20 +0,0 @@ -package executor - -import ( - "github.com/rs/zerolog" - "github.com/tomarrell/lbadd/internal/compiler/command" -) - -// Executor describes a component that can execute a command. A command is the -// intermediate representation of an SQL statement, meaning that it has been -// parsed. -type Executor interface { - // Execute executes a command. The result of the computation is returned - // together with an error, if one occurred. - Execute(command.Command) (Result, error) -} - -// New creates a new, ready to use Executor. -func New(log zerolog.Logger, databaseFile string) Executor { - return newSimpleExecutor(log, databaseFile) -} diff --git a/internal/executor/result.go b/internal/executor/result.go deleted file mode 100644 index bdb2c5ef..00000000 --- a/internal/executor/result.go +++ /dev/null @@ -1,11 +0,0 @@ -package executor - -import "fmt" - -// Result describes the result of a command execution. The result is always a -// table that has a header row. The smallest possible result table is a table -// with one column and two rows, and is generated as a result of a single-value -// computation, e.g. a sum(). -type Result interface { - fmt.Stringer -} diff --git a/internal/executor/simple_executor.go b/internal/executor/simple_executor.go deleted file mode 100644 index 7d53cf8e..00000000 --- a/internal/executor/simple_executor.go +++ /dev/null @@ -1,26 +0,0 @@ -package executor - -import ( - "fmt" - - "github.com/rs/zerolog" - "github.com/tomarrell/lbadd/internal/compiler/command" -) - -var _ Executor = (*simpleExecutor)(nil) - -type simpleExecutor struct { - log zerolog.Logger - databaseFile string -} - -func newSimpleExecutor(log zerolog.Logger, databaseFile string) *simpleExecutor { - return &simpleExecutor{ - log: log, - databaseFile: databaseFile, - } -} - -func (e *simpleExecutor) Execute(cmd command.Command) (Result, error) { - return nil, fmt.Errorf("unimplemented") -} diff --git a/internal/node/node.go b/internal/node/node.go index cd977c03..f92a3a86 100644 --- a/internal/node/node.go +++ b/internal/node/node.go @@ -5,23 +5,20 @@ import ( "fmt" "github.com/rs/zerolog" - "github.com/tomarrell/lbadd/internal/executor" ) // Node is a database node. // -// m := node.New(log, executor) +// m := node.New(log) // err := m.ListenAndServe(ctx, ":34213") type Node struct { - log zerolog.Logger - exec executor.Executor + log zerolog.Logger } // New creates a new node that is executing commands on the given executor. -func New(log zerolog.Logger, exec executor.Executor) *Node { +func New(log zerolog.Logger) *Node { return &Node{ - log: log, - exec: exec, + log: log, } } From 6bc92444de406a0207f8202d7e4676688d62bd18 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 5 Jun 2020 18:30:22 +0200 Subject: [PATCH 02/77] Introduce engine Add a converter that allows primitive to byte conversion for most primitive types. --- internal/engine/converter/converter.go | 156 ++++++++++++++++++++ internal/engine/converter/converter_test.go | 155 +++++++++++++++++++ internal/engine/converter/doc.go | 2 + internal/engine/engine.go | 1 + 4 files changed, 314 insertions(+) create mode 100644 internal/engine/converter/converter.go create mode 100644 internal/engine/converter/converter_test.go create mode 100644 internal/engine/converter/doc.go create mode 100644 internal/engine/engine.go diff --git a/internal/engine/converter/converter.go b/internal/engine/converter/converter.go new file mode 100644 index 00000000..6cefca62 --- /dev/null +++ b/internal/engine/converter/converter.go @@ -0,0 +1,156 @@ +package converter + +import ( + "encoding/binary" + "math" +) + +const ( + falseByte byte = byte(0) + trueByte byte = ^falseByte +) + +var ( + byteOrder = binary.BigEndian +) + +// bool + +// BoolToByte converts the given argument to a byte output, which is then +// returned. +func BoolToByte(v bool) byte { + if v { + return trueByte + } + return falseByte +} + +// BoolToByteArray converts the given argument to a byte output, which is then +// returned. +func BoolToByteArray(v bool) [1]byte { + return [1]byte{BoolToByte(v)} +} + +// BoolToByteSlice converts the given argument to a byte output, which is then +// returned. +func BoolToByteSlice(v bool) []byte { + arr := BoolToByteArray(v) + return arr[:] +} + +// integral + +// Uint16ToByteArray converts the given argument to a byte output, which is then +// returned. +func Uint16ToByteArray(v uint16) (result [2]byte) { + byteOrder.PutUint16(result[:], v) + return +} + +// Uint16ToByteSlice converts the given argument to a byte output, which is then +// returned. +func Uint16ToByteSlice(v uint16) (result []byte) { + result = make([]byte, 2) + byteOrder.PutUint16(result, v) + return +} + +// Uint32ToByteArray converts the given argument to a byte output, which is then +// returned. +func Uint32ToByteArray(v uint32) (result [4]byte) { + byteOrder.PutUint32(result[:], v) + return +} + +// Uint32ToByteSlice converts the given argument to a byte output, which is then +// returned. +func Uint32ToByteSlice(v uint32) (result []byte) { + result = make([]byte, 4) + byteOrder.PutUint32(result, v) + return +} + +// Uint64ToByteArray converts the given argument to a byte output, which is then +// returned. +func Uint64ToByteArray(v uint64) (result [8]byte) { + byteOrder.PutUint64(result[:], v) + return +} + +// Uint64ToByteSlice converts the given argument to a byte output, which is then +// returned. +func Uint64ToByteSlice(v uint64) (result []byte) { + result = make([]byte, 8) + byteOrder.PutUint64(result, v) + return +} + +// fractal + +// Float32ToByteArray converts the given argument to a byte output, which is +// then returned. +func Float32ToByteArray(v float32) (result [4]byte) { + return Uint32ToByteArray(math.Float32bits(v)) +} + +// Float32ToByteSlice converts the given argument to a byte output, which is +// then returned. +func Float32ToByteSlice(v float32) (result []byte) { + return Uint32ToByteSlice(math.Float32bits(v)) +} + +// Float64ToByteArray converts the given argument to a byte output, which is +// then returned. +func Float64ToByteArray(v float64) (result [8]byte) { + return Uint64ToByteArray(math.Float64bits(v)) +} + +// Float64ToByteSlice converts the given argument to a byte output, which is +// then returned. +func Float64ToByteSlice(v float64) (result []byte) { + return Uint64ToByteSlice(math.Float64bits(v)) +} + +// complex + +// Complex64ToByteArray converts the given argument to a byte output, which is +// then returned. +func Complex64ToByteArray(v complex64) (result [8]byte) { + copy(result[:4], Float32ToByteSlice(real(v))) + copy(result[4:], Float32ToByteSlice(imag(v))) + return +} + +// Complex64ToByteSlice converts the given argument to a byte output, which is +// then returned. +func Complex64ToByteSlice(v complex64) (result []byte) { + result = make([]byte, 8) + copy(result[:4], Float32ToByteSlice(real(v))) + copy(result[4:], Float32ToByteSlice(imag(v))) + return +} + +// Complex128ToByteArray converts the given argument to a byte output, which is +// then returned. +func Complex128ToByteArray(v complex128) (result [16]byte) { + copy(result[:8], Float64ToByteSlice(real(v))) + copy(result[8:], Float64ToByteSlice(imag(v))) + return +} + +// Complex128ToByteSlice converts the given argument to a byte output, which is +// then returned. +func Complex128ToByteSlice(v complex128) (result []byte) { + result = make([]byte, 16) + copy(result[:8], Float64ToByteSlice(real(v))) + copy(result[8:], Float64ToByteSlice(imag(v))) + return +} + +// variable-size + +// StringToByteSlice converts the given argument to a byte output, which is then +// returned. +func StringToByteSlice(v string) []byte { + return []byte(v) +} diff --git a/internal/engine/converter/converter_test.go b/internal/engine/converter/converter_test.go new file mode 100644 index 00000000..abd31c50 --- /dev/null +++ b/internal/engine/converter/converter_test.go @@ -0,0 +1,155 @@ +package converter_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/engine/converter" +) + +func TestBool(t *testing.T) { + t.Run("target=byte", func(t *testing.T) { + assert := assert.New(t) + assert.Equal(byte(0x00), converter.BoolToByte(false)) + assert.Equal(byte(255), converter.BoolToByte(true)) + }) + t.Run("target=bytearray", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([1]byte{0x00}, converter.BoolToByteArray(false)) + assert.Equal([1]byte{255}, converter.BoolToByteArray(true)) + }) + t.Run("target=byteslice", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([]byte{0x00}, converter.BoolToByteSlice(false)) + assert.Equal([]byte{255}, converter.BoolToByteSlice(true)) + }) +} + +func TestIntegral(t *testing.T) { + t.Run("cardinality=16bit", func(t *testing.T) { + t.Run("target=bytearray", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([2]byte{0x00, 0x00}, converter.Uint16ToByteArray(0)) + assert.Equal([2]byte{0xCA, 0xFE}, converter.Uint16ToByteArray(0xCAFE)) + assert.Equal([2]byte{0x00, 0xAB}, converter.Uint16ToByteArray(0xAB)) + assert.Equal([2]byte{0xFF, 0xFF}, converter.Uint16ToByteArray(0xFFFF)) + }) + t.Run("target=byteslice", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([]byte{0x00, 0x00}, converter.Uint16ToByteSlice(0)) + assert.Equal([]byte{0xCA, 0xFE}, converter.Uint16ToByteSlice(0xCAFE)) + assert.Equal([]byte{0x00, 0xAB}, converter.Uint16ToByteSlice(0xAB)) + assert.Equal([]byte{0xFF, 0xFF}, converter.Uint16ToByteSlice(0xFFFF)) + }) + }) + t.Run("cardinality=32bit", func(t *testing.T) { + t.Run("target=bytearray", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([4]byte{0x00, 0x00, 0x00, 0x00}, converter.Uint32ToByteArray(0)) + assert.Equal([4]byte{0xCA, 0xFE, 0xBA, 0xBE}, converter.Uint32ToByteArray(0xCAFEBABE)) + assert.Equal([4]byte{0x00, 0x00, 0x00, 0xAB}, converter.Uint32ToByteArray(0xAB)) + assert.Equal([4]byte{0xFF, 0xFF, 0xFF, 0xFF}, converter.Uint32ToByteArray(0xFFFFFFFF)) + }) + t.Run("target=byteslice", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([]byte{0x00, 0x00, 0x00, 0x00}, converter.Uint32ToByteSlice(0)) + assert.Equal([]byte{0xCA, 0xFE, 0xBA, 0xBE}, converter.Uint32ToByteSlice(0xCAFEBABE)) + assert.Equal([]byte{0x00, 0x00, 0x00, 0xAB}, converter.Uint32ToByteSlice(0xAB)) + assert.Equal([]byte{0xFF, 0xFF, 0xFF, 0xFF}, converter.Uint32ToByteSlice(0xFFFFFFFF)) + }) + }) + t.Run("cardinality=64bit", func(t *testing.T) { + t.Run("target=bytearray", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Uint64ToByteArray(0)) + assert.Equal([8]byte{0xCA, 0xFE, 0xBA, 0xBE, 0xDA, 0xDE, 0xFA, 0xBE}, converter.Uint64ToByteArray(0xCAFEBABEDADEFABE)) + assert.Equal([8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB}, converter.Uint64ToByteArray(0xAB)) + assert.Equal([8]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, converter.Uint64ToByteArray(0xFFFFFFFFFFFFFFFF)) + }) + t.Run("target=byteslice", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Uint64ToByteSlice(0)) + assert.Equal([]byte{0xCA, 0xFE, 0xBA, 0xBE, 0xDA, 0xDE, 0xFA, 0xBE}, converter.Uint64ToByteSlice(0xCAFEBABEDADEFABE)) + assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB}, converter.Uint64ToByteSlice(0xAB)) + assert.Equal([]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, converter.Uint64ToByteSlice(0xFFFFFFFFFFFFFFFF)) + }) + }) +} + +func TestFractal(t *testing.T) { + t.Run("cardinality=32bit", func(t *testing.T) { + t.Run("target=bytearray", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([4]byte{0x00, 0x00, 0x00, 0x00}, converter.Float32ToByteArray(0)) + assert.Equal([4]byte{0x4f, 0x4a, 0xfe, 0xbb}, converter.Float32ToByteArray(0xCAFEBABE)) + assert.Equal([4]byte{0x43, 0x2b, 0x00, 0x00}, converter.Float32ToByteArray(0xAB)) + assert.Equal([4]byte{0x4f, 0x80, 0x00, 0x00}, converter.Float32ToByteArray(0xFFFFFFFF)) + }) + t.Run("target=byteslice", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([]byte{0x00, 0x00, 0x00, 0x00}, converter.Float32ToByteSlice(0)) + assert.Equal([]byte{0x4f, 0x4a, 0xfe, 0xbb}, converter.Float32ToByteSlice(0xCAFEBABE)) + assert.Equal([]byte{0x43, 0x2b, 0x00, 0x00}, converter.Float32ToByteSlice(0xAB)) + assert.Equal([]byte{0x4f, 0x80, 0x00, 0x00}, converter.Float32ToByteSlice(0xFFFFFFFF)) + }) + }) + t.Run("cardinality=64bit", func(t *testing.T) { + t.Run("target=bytearray", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteArray(0)) + assert.Equal([8]byte{0x43, 0xe9, 0x5f, 0xd7, 0x57, 0xdb, 0x5b, 0xdf}, converter.Float64ToByteArray(0xCAFEBABEDADEFABE)) + assert.Equal([8]byte{0x40, 0x65, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteArray(0xAB)) + assert.Equal([8]byte{0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteArray(0xFFFFFFFFFFFFFFFF)) + }) + t.Run("target=byteslice", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteSlice(0)) + assert.Equal([]byte{0x43, 0xe9, 0x5f, 0xd7, 0x57, 0xdb, 0x5b, 0xdf}, converter.Float64ToByteSlice(0xCAFEBABEDADEFABE)) + assert.Equal([]byte{0x40, 0x65, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteSlice(0xAB)) + assert.Equal([]byte{0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteSlice(0xFFFFFFFFFFFFFFFF)) + }) + }) +} + +func TestComplex(t *testing.T) { + t.Run("cardinality=64bit", func(t *testing.T) { + t.Run("target=bytearray", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex64ToByteArray(0+0i)) + assert.Equal([8]byte{0x41, 0x60, 0x00, 0x00, 0x41, 0x40, 0x00, 0x00}, converter.Complex64ToByteArray(14+12i)) + assert.Equal([8]byte{0x40, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex64ToByteArray(4+0i)) + assert.Equal([8]byte{0x4f, 0x80, 0x00, 0x00, 0x4f, 0x80, 0x00, 0x00}, converter.Complex64ToByteArray(0xFFFFFFFF+0xFFFFFFFFi)) + }) + t.Run("target=byteslice", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex64ToByteSlice(0+0i)) + assert.Equal([]byte{0x41, 0x60, 0x00, 0x00, 0x41, 0x40, 0x00, 0x00}, converter.Complex64ToByteSlice(14+12i)) + assert.Equal([]byte{0x40, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex64ToByteSlice(4+0i)) + assert.Equal([]byte{0x4f, 0x80, 0x00, 0x00, 0x4f, 0x80, 0x00, 0x00}, converter.Complex64ToByteSlice(0xFFFFFFFF+0xFFFFFFFFi)) + }) + }) + t.Run("cardinality=128bit", func(t *testing.T) { + t.Run("target=bytearray", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([16]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteArray(0+0i)) + assert.Equal([16]byte{0x40, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteArray(14+12i)) + assert.Equal([16]byte{0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteArray(4+0i)) + assert.Equal([16]byte{0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteArray(0xFFFFFFFFFFFFFFFF+0xFFFFFFFFFFFFFFFFi)) + }) + t.Run("target=byteslice", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteSlice(0+0i)) + assert.Equal([]byte{0x40, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteSlice(14+12i)) + assert.Equal([]byte{0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteSlice(4+0i)) + assert.Equal([]byte{0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteSlice(0xFFFFFFFFFFFFFFFF+0xFFFFFFFFFFFFFFFFi)) + }) + }) +} + +func TestVariable(t *testing.T) { + t.Run("type=string", func(t *testing.T) { + assert := assert.New(t) + assert.Equal([]byte{}, converter.StringToByteSlice("")) + assert.Equal([]byte{0x61, 0x62, 0x63, 0x64}, converter.StringToByteSlice("abcd")) + }) +} diff --git a/internal/engine/converter/doc.go b/internal/engine/converter/doc.go new file mode 100644 index 00000000..62b0708d --- /dev/null +++ b/internal/engine/converter/doc.go @@ -0,0 +1,2 @@ +// Package converter implements functions to encode and decode values to bytes. +package converter diff --git a/internal/engine/engine.go b/internal/engine/engine.go new file mode 100644 index 00000000..00a22ef6 --- /dev/null +++ b/internal/engine/engine.go @@ -0,0 +1 @@ +package engine From 358fa9f79b7e5b42c7d0febf2a28d8c9c74587bf Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 5 Jun 2020 19:06:47 +0200 Subject: [PATCH 03/77] Improve godoc --- internal/engine/converter/converter.go | 61 +++++++++++++------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/internal/engine/converter/converter.go b/internal/engine/converter/converter.go index 6cefca62..6d6db06e 100644 --- a/internal/engine/converter/converter.go +++ b/internal/engine/converter/converter.go @@ -16,8 +16,7 @@ var ( // bool -// BoolToByte converts the given argument to a byte output, which is then -// returned. +// BoolToByte converts the given argument bool to a byte which is then returned. func BoolToByte(v bool) byte { if v { return trueByte @@ -25,13 +24,13 @@ func BoolToByte(v bool) byte { return falseByte } -// BoolToByteArray converts the given argument to a byte output, which is then +// BoolToByteArray converts the given argument bool to a [1]byte is then // returned. func BoolToByteArray(v bool) [1]byte { return [1]byte{BoolToByte(v)} } -// BoolToByteSlice converts the given argument to a byte output, which is then +// BoolToByteSlice converts the given argument bool to a []byte which is then // returned. func BoolToByteSlice(v bool) []byte { arr := BoolToByteArray(v) @@ -40,45 +39,45 @@ func BoolToByteSlice(v bool) []byte { // integral -// Uint16ToByteArray converts the given argument to a byte output, which is then -// returned. +// Uint16ToByteArray converts the given argument uint16 to a [2]byte which is +// then returned. func Uint16ToByteArray(v uint16) (result [2]byte) { byteOrder.PutUint16(result[:], v) return } -// Uint16ToByteSlice converts the given argument to a byte output, which is then -// returned. +// Uint16ToByteSlice converts the given argument uint16 to a []byte which is +// then returned. func Uint16ToByteSlice(v uint16) (result []byte) { result = make([]byte, 2) byteOrder.PutUint16(result, v) return } -// Uint32ToByteArray converts the given argument to a byte output, which is then -// returned. +// Uint32ToByteArray converts the given argument uint32 to a [4]byte which is +// then returned. func Uint32ToByteArray(v uint32) (result [4]byte) { byteOrder.PutUint32(result[:], v) return } -// Uint32ToByteSlice converts the given argument to a byte output, which is then -// returned. +// Uint32ToByteSlice converts the given argument uint32 to a []byte which is +// then returned. func Uint32ToByteSlice(v uint32) (result []byte) { result = make([]byte, 4) byteOrder.PutUint32(result, v) return } -// Uint64ToByteArray converts the given argument to a byte output, which is then -// returned. +// Uint64ToByteArray converts the given argument uint64 to a [8]byte which is +// then returned. func Uint64ToByteArray(v uint64) (result [8]byte) { byteOrder.PutUint64(result[:], v) return } -// Uint64ToByteSlice converts the given argument to a byte output, which is then -// returned. +// Uint64ToByteSlice converts the given argument uint64 to a []byte which is +// then returned. func Uint64ToByteSlice(v uint64) (result []byte) { result = make([]byte, 8) byteOrder.PutUint64(result, v) @@ -87,42 +86,42 @@ func Uint64ToByteSlice(v uint64) (result []byte) { // fractal -// Float32ToByteArray converts the given argument to a byte output, which is -// then returned. +// Float32ToByteArray converts the given float32 to a [4]byte, which is then +// returned. func Float32ToByteArray(v float32) (result [4]byte) { return Uint32ToByteArray(math.Float32bits(v)) } -// Float32ToByteSlice converts the given argument to a byte output, which is -// then returned. +// Float32ToByteSlice converts the given float32 to a []byte, which is then +// returned. func Float32ToByteSlice(v float32) (result []byte) { return Uint32ToByteSlice(math.Float32bits(v)) } -// Float64ToByteArray converts the given argument to a byte output, which is -// then returned. +// Float64ToByteArray converts the given float64 to a [8]byte, which is then +// returned. func Float64ToByteArray(v float64) (result [8]byte) { return Uint64ToByteArray(math.Float64bits(v)) } -// Float64ToByteSlice converts the given argument to a byte output, which is -// then returned. +// Float64ToByteSlice converts the given float64 to a []byte, which is then +// returned. func Float64ToByteSlice(v float64) (result []byte) { return Uint64ToByteSlice(math.Float64bits(v)) } // complex -// Complex64ToByteArray converts the given argument to a byte output, which is -// then returned. +// Complex64ToByteArray converts the given complex64 to a [8]byte, which is then +// returned. func Complex64ToByteArray(v complex64) (result [8]byte) { copy(result[:4], Float32ToByteSlice(real(v))) copy(result[4:], Float32ToByteSlice(imag(v))) return } -// Complex64ToByteSlice converts the given argument to a byte output, which is -// then returned. +// Complex64ToByteSlice converts the given complex64 to a []byte, which is then +// returned. func Complex64ToByteSlice(v complex64) (result []byte) { result = make([]byte, 8) copy(result[:4], Float32ToByteSlice(real(v))) @@ -130,7 +129,7 @@ func Complex64ToByteSlice(v complex64) (result []byte) { return } -// Complex128ToByteArray converts the given argument to a byte output, which is +// Complex128ToByteArray converts the given complex128 to a [16]byte, which is // then returned. func Complex128ToByteArray(v complex128) (result [16]byte) { copy(result[:8], Float64ToByteSlice(real(v))) @@ -138,7 +137,7 @@ func Complex128ToByteArray(v complex128) (result [16]byte) { return } -// Complex128ToByteSlice converts the given argument to a byte output, which is +// Complex128ToByteSlice converts the given complex128 to a []byte, which is // then returned. func Complex128ToByteSlice(v complex128) (result []byte) { result = make([]byte, 16) @@ -149,7 +148,7 @@ func Complex128ToByteSlice(v complex128) (result []byte) { // variable-size -// StringToByteSlice converts the given argument to a byte output, which is then +// StringToByteSlice converts the given argument string a byte ytwhich is then // returned. func StringToByteSlice(v string) []byte { return []byte(v) From 5cde6713ad1e677042ef598be988a5605718fbd2 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 5 Jun 2020 19:09:18 +0200 Subject: [PATCH 04/77] Fix typo --- internal/engine/converter/converter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/engine/converter/converter.go b/internal/engine/converter/converter.go index 6d6db06e..8b11f5bb 100644 --- a/internal/engine/converter/converter.go +++ b/internal/engine/converter/converter.go @@ -148,7 +148,7 @@ func Complex128ToByteSlice(v complex128) (result []byte) { // variable-size -// StringToByteSlice converts the given argument string a byte ytwhich is then +// StringToByteSlice converts the given argument string a []byte which is then // returned. func StringToByteSlice(v string) []byte { return []byte(v) From 07c9ebc7b98b84c0b9d6e0ec27d73851d10d24f2 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Sat, 6 Jun 2020 18:08:13 +0200 Subject: [PATCH 05/77] Add a few decoding function to the converter --- internal/engine/converter/converter.go | 85 ++++++++++++++++++++------ 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/internal/engine/converter/converter.go b/internal/engine/converter/converter.go index 8b11f5bb..297b358e 100644 --- a/internal/engine/converter/converter.go +++ b/internal/engine/converter/converter.go @@ -16,7 +16,7 @@ var ( // bool -// BoolToByte converts the given argument bool to a byte which is then returned. +// BoolToByte converts the given bool to a byte which is then returned. func BoolToByte(v bool) byte { if v { return trueByte @@ -24,66 +24,109 @@ func BoolToByte(v bool) byte { return falseByte } -// BoolToByteArray converts the given argument bool to a [1]byte is then -// returned. +// ByteToBool converts the given byte back to a bool. +func ByteToBool(v byte) bool { + return v != falseByte +} + +// BoolToByteArray converts the given bool to a [1]byte is then returned. func BoolToByteArray(v bool) [1]byte { return [1]byte{BoolToByte(v)} } -// BoolToByteSlice converts the given argument bool to a []byte which is then -// returned. +// ByteArrayToBool converts the given [1]byte back to a bool. +func ByteArrayToBool(v [1]byte) bool { + return ByteToBool(v[0]) +} + +// BoolToByteSlice converts the given bool to a []byte which is then returned. func BoolToByteSlice(v bool) []byte { arr := BoolToByteArray(v) return arr[:] } +// ByteSliceToBool converts the given []byte back to a bool. +func ByteSliceToBool(v []byte) bool { + return ByteToBool(v[0]) +} + // integral -// Uint16ToByteArray converts the given argument uint16 to a [2]byte which is -// then returned. +// Uint16ToByteArray converts the given uint16 to a [2]byte which is then +// returned. func Uint16ToByteArray(v uint16) (result [2]byte) { byteOrder.PutUint16(result[:], v) return } -// Uint16ToByteSlice converts the given argument uint16 to a []byte which is -// then returned. +// ByteArrayToUint16 converts the given [2]byte back to a uint16. +func ByteArrayToUint16(v [2]byte) uint16 { + return byteOrder.Uint16(v[:]) +} + +// Uint16ToByteSlice converts the given uint16 to a []byte which is then +// returned. func Uint16ToByteSlice(v uint16) (result []byte) { result = make([]byte, 2) byteOrder.PutUint16(result, v) return } -// Uint32ToByteArray converts the given argument uint32 to a [4]byte which is -// then returned. +// ByteSliceToUint16 converts the given []byte back to a uint16. +func ByteSliceToUint16(v []byte) uint16 { + return byteOrder.Uint16(v) +} + +// Uint32ToByteArray converts the given uint32 to a [4]byte which is then +// returned. func Uint32ToByteArray(v uint32) (result [4]byte) { byteOrder.PutUint32(result[:], v) return } -// Uint32ToByteSlice converts the given argument uint32 to a []byte which is -// then returned. +// ByteArrayToUint32 converts the given [4]byte back to a uint32. +func ByteArrayToUint32(v [4]byte) uint32 { + return byteOrder.Uint32(v[:]) +} + +// Uint32ToByteSlice converts the given uint32 to a []byte which is then +// returned. func Uint32ToByteSlice(v uint32) (result []byte) { result = make([]byte, 4) byteOrder.PutUint32(result, v) return } -// Uint64ToByteArray converts the given argument uint64 to a [8]byte which is -// then returned. +// ByteSliceToUint32 converts the given []byte back to a uint32. +func ByteSliceToUint32(v []byte) uint32 { + return byteOrder.Uint32(v) +} + +// Uint64ToByteArray converts the given uint64 to a [8]byte which is then +// returned. func Uint64ToByteArray(v uint64) (result [8]byte) { byteOrder.PutUint64(result[:], v) return } -// Uint64ToByteSlice converts the given argument uint64 to a []byte which is -// then returned. +// ByteArrayToUint64 converts the given [8]byte back to a uint64. +func ByteArrayToUint64(v [8]byte) uint64 { + return byteOrder.Uint64(v[:]) +} + +// Uint64ToByteSlice converts the given uint64 to a []byte which is then +// returned. func Uint64ToByteSlice(v uint64) (result []byte) { result = make([]byte, 8) byteOrder.PutUint64(result, v) return } +// ByteSliceToUint64 converts the given []byte back to a uint64. +func ByteSliceToUint64(v []byte) uint64 { + return byteOrder.Uint64(v) +} + // fractal // Float32ToByteArray converts the given float32 to a [4]byte, which is then @@ -148,8 +191,12 @@ func Complex128ToByteSlice(v complex128) (result []byte) { // variable-size -// StringToByteSlice converts the given argument string a []byte which is then -// returned. +// StringToByteSlice converts the given string a []byte which is then returned. func StringToByteSlice(v string) []byte { return []byte(v) } + +// ByteSliceToString converts the given []byte back to a string. +func ByteSliceToString(v []byte) string { + return string(v) +} From a06d5a9dad77f4dfe8e9f28b60be82f908da54c5 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Sat, 6 Jun 2020 18:08:34 +0200 Subject: [PATCH 06/77] Add a page abstraction --- internal/engine/storage/page/doc.go | 3 ++ internal/engine/storage/page/error.go | 13 ++++++++ internal/engine/storage/page/page.go | 47 +++++++++++++++++++++++++++ internal/engine/storage/storage.go | 1 + 4 files changed, 64 insertions(+) create mode 100644 internal/engine/storage/page/doc.go create mode 100644 internal/engine/storage/page/error.go create mode 100644 internal/engine/storage/page/page.go create mode 100644 internal/engine/storage/storage.go diff --git a/internal/engine/storage/page/doc.go b/internal/engine/storage/page/doc.go new file mode 100644 index 00000000..f430997c --- /dev/null +++ b/internal/engine/storage/page/doc.go @@ -0,0 +1,3 @@ +// Package page describes generic pages. The generic description is independent +// from any implementation version. +package page diff --git a/internal/engine/storage/page/error.go b/internal/engine/storage/page/error.go new file mode 100644 index 00000000..e4ddb068 --- /dev/null +++ b/internal/engine/storage/page/error.go @@ -0,0 +1,13 @@ +package page + +// Error is a sentinel error. +type Error string + +func (e Error) Error() string { return string(e) } + +// Sentinel errors. +const ( + ErrUnknownHeader = Error("unknown header") + ErrInvalidPageSize = Error("invalid page size") + ErrPageTooSmall = Error("page is too small for the requested data") +) diff --git a/internal/engine/storage/page/page.go b/internal/engine/storage/page/page.go new file mode 100644 index 00000000..f75a86e8 --- /dev/null +++ b/internal/engine/storage/page/page.go @@ -0,0 +1,47 @@ +package page + +// Header is an enum type for header fields. +type Header uint8 + +const ( + // HeaderVersion is the version number, that a page is in. This is a uint16. + HeaderVersion Header = iota + // HeaderID is the page ID, which may be used outside of the page for + // housekeeping. This is a uint16. + HeaderID + // HeaderCellCount is the amount of cells that are currently stored in the + // page. This is a uint16. + HeaderCellCount +) + +// Loader is a function that can load a page from a given byte slice, and return +// errors if any occur. +type Loader func([]byte) (Page, error) + +// Page describes a memory page that stores (page.Cell)s. A page consists of +// header fields and cells, and is a plain store. Obtained cells are always +// ordered ascending by the cell key. A page supports variable size cell keys +// and records. +type Page interface { + // Header obtains a header field from the page's header. If the header is + // not supported, a result=nil,error=ErrUnknownHeader is returned. The type + // of the returned header field value is documented in the header key's + // godoc. + Header(Header) (interface{}, error) + + // StoreCell stores a cell on the page. If the page does not fit the cell + // because of size or too much fragmentation, an error will be returned. + StoreCell(Cell) error + // Cells returns all cells in this page as a slice. Cells are ordered + // ascending by key. Calling this method is relatively cheap. + Cells() []Cell +} + +// Cell is a structure that represents a key-value cell. Both the key and the +// record can be of variable size. +type Cell struct { + // Key is the key of this cell, used for ordering. + Key []byte + // Record is the data stored inside the cell. + Record []byte +} diff --git a/internal/engine/storage/storage.go b/internal/engine/storage/storage.go new file mode 100644 index 00000000..7306f617 --- /dev/null +++ b/internal/engine/storage/storage.go @@ -0,0 +1 @@ +package storage \ No newline at end of file From 893a23c17dbd1c2bbf2f0aeaf504699fa75c865b Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Sat, 6 Jun 2020 18:08:53 +0200 Subject: [PATCH 07/77] Add a slotted page implementation --- internal/engine/storage/page/v1/codec.go | 61 +++++ internal/engine/storage/page/v1/codec_test.go | 138 +++++++++++ internal/engine/storage/page/v1/doc.go | 13 + internal/engine/storage/page/v1/v1_offset.go | 17 ++ internal/engine/storage/page/v1/v1_page.go | 227 ++++++++++++++++++ .../storage/page/v1/v1_page_bench_test.go | 68 ++++++ .../engine/storage/page/v1/v1_page_test.go | 216 +++++++++++++++++ 7 files changed, 740 insertions(+) create mode 100644 internal/engine/storage/page/v1/codec.go create mode 100644 internal/engine/storage/page/v1/codec_test.go create mode 100644 internal/engine/storage/page/v1/doc.go create mode 100644 internal/engine/storage/page/v1/v1_offset.go create mode 100644 internal/engine/storage/page/v1/v1_page.go create mode 100644 internal/engine/storage/page/v1/v1_page_bench_test.go create mode 100644 internal/engine/storage/page/v1/v1_page_test.go diff --git a/internal/engine/storage/page/v1/codec.go b/internal/engine/storage/page/v1/codec.go new file mode 100644 index 00000000..ea4820f6 --- /dev/null +++ b/internal/engine/storage/page/v1/codec.go @@ -0,0 +1,61 @@ +// This file contains encoding and decoding functions. + +package v1 + +import ( + "github.com/tomarrell/lbadd/internal/engine/converter" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +// encodeCell frames the cell key and record, and concatenates them. The +// concatenation is NOT framed itself. +// +// encoded = | frame | key | frame | record | +func encodeCell(cell page.Cell) []byte { + keyFrame := frameData(cell.Key) + recordFrame := frameData(cell.Record) + return append(keyFrame, recordFrame...) +} + +// decodeCell takes data of the format that encodeCell produced, and convert it +// back into a (page.Cell). +func decodeCell(data []byte) page.Cell { + keyDataSize := converter.ByteSliceToUint16(data[:FrameSizeSize]) + keyLo := FrameSizeSize + keyHi := FrameSizeSize + keyDataSize + + dataLo := keyHi + FrameSizeSize + dataSize := converter.ByteSliceToUint16(data[keyHi:dataLo]) + dataHi := dataLo + dataSize + + return page.Cell{ + Key: data[keyLo:keyHi], + Record: data[dataLo:dataHi], + } +} + +// frameData frames the given data and returns the framed bytes. The frame +// contains the length of the data, as uint16. +// +// framed = | length | data | +func frameData(data []byte) []byte { + return append(converter.Uint16ToByteSlice(uint16(len(data))), data...) +} + +// encodeOffset converts the offset to a byte slice of the following form. +// +// encoded = | location | size | +// +// Both the location and size will be encoded as 2 byte uint16. +func encodeOffset(offset Offset) []byte { + return append(converter.Uint16ToByteSlice(offset.Location), converter.Uint16ToByteSlice(offset.Size)...) +} + +// decodeOffset takes bytes of the form produced by encodeOffset and converts it +// back into an Offset. +func decodeOffset(data []byte) Offset { + return Offset{ + Location: converter.ByteSliceToUint16(data[:int(OffsetSize)/2]), + Size: converter.ByteSliceToUint16(data[int(OffsetSize)/2 : int(OffsetSize)]), + } +} diff --git a/internal/engine/storage/page/v1/codec_test.go b/internal/engine/storage/page/v1/codec_test.go new file mode 100644 index 00000000..b6819ec1 --- /dev/null +++ b/internal/engine/storage/page/v1/codec_test.go @@ -0,0 +1,138 @@ +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +func Test_encodeCell(t *testing.T) { + tests := []struct { + name string + cell page.Cell + want []byte + }{ + { + "empty cell", + page.Cell{ + Key: []byte{}, + Record: []byte{}, + }, + []byte{0x00, 0x00, 0x00, 0x00}, + }, + { + "cell", + page.Cell{ + Key: []byte{0x01, 0x02}, + Record: []byte{0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + }, + []byte{ + 0x00, 0x02, // key frame + 0x01, 0x02, // key + 0x00, 0x06, // record frame + 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // record + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + assert.Equal(tt.want, encodeCell(tt.cell)) + }) + } +} + +func Test_decodeCell(t *testing.T) { + tests := []struct { + name string + data []byte + want page.Cell + }{ + { + "empty cell", + []byte{0x00, 0x00, 0x00, 0x00}, + page.Cell{ + Key: []byte{}, + Record: []byte{}, + }, + }, + { + "cell", + []byte{ + 0x00, 0x02, // key frame + 0x01, 0x02, // key + 0x00, 0x06, // record frame + 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // record + }, + page.Cell{ + Key: []byte{0x01, 0x02}, + Record: []byte{0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + assert.Equal(tt.want, decodeCell(tt.data)) + }) + } +} + +func Test_encodeOffset(t *testing.T) { + tests := []struct { + name string + offset Offset + want []byte + }{ + { + "empty offset", + Offset{}, + []byte{0x00, 0x00, 0x00, 0x00}, + }, + { + "offset", + Offset{ + Location: 0xCAFE, + Size: 0xBABE, + }, + []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + assert.Equal(tt.want, encodeOffset(tt.offset)) + }) + } +} + +func Test_decodeOffset(t *testing.T) { + tests := []struct { + name string + data []byte + want Offset + }{ + { + "empty offset", + []byte{0x00, 0x00, 0x00, 0x00}, + Offset{}, + }, + { + "offset", + []byte{0xCA, 0xFE, 0xBA, 0xBE}, + Offset{ + Location: 0xCAFE, + Size: 0xBABE, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + assert.Equal(tt.want, decodeOffset(tt.data)) + }) + } +} diff --git a/internal/engine/storage/page/v1/doc.go b/internal/engine/storage/page/v1/doc.go new file mode 100644 index 00000000..638c33c5 --- /dev/null +++ b/internal/engine/storage/page/v1/doc.go @@ -0,0 +1,13 @@ +// Package v1 provides an implementation for the (page.Page) interface, as well +// as a function to load such a page from a given byte slice. v1 pages consist +// of headers and data, and NO trailer. Header fields are fixed, every header +// field has a fixed offset and memory area. Variable header fields are not +// supported. Cell types are stored in the cells. +// +// Assuming that data is given as a byte slice called data. +// +// data = ... +// p, err := v1.Load(data) +// val, err := p.Header(page.HeaderVersion) +// version := val.(uint16) +package v1 diff --git a/internal/engine/storage/page/v1/v1_offset.go b/internal/engine/storage/page/v1/v1_offset.go new file mode 100644 index 00000000..972d4f41 --- /dev/null +++ b/internal/engine/storage/page/v1/v1_offset.go @@ -0,0 +1,17 @@ +package v1 + +import "unsafe" + +const ( + // OffsetSize is the constant size of an encoded Offset using encodeOffset. + OffsetSize = uint16(unsafe.Sizeof(Offset{})) // #nosec +) + +// Offset describes a memory segment relative to the page start (=0, =before the +// header). +type Offset struct { + // Location is the target location of this offset, relative to the page. + Location uint16 + // Size is the size of the memory segment that is located at Location. + Size uint16 +} diff --git a/internal/engine/storage/page/v1/v1_page.go b/internal/engine/storage/page/v1/v1_page.go new file mode 100644 index 00000000..ba5280c2 --- /dev/null +++ b/internal/engine/storage/page/v1/v1_page.go @@ -0,0 +1,227 @@ +package v1 + +import ( + "bytes" + "sort" + "unsafe" + + "github.com/tomarrell/lbadd/internal/engine/converter" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +// Constants. +const ( + // PageSize is the fixed size of one page. + PageSize uint16 = 1 << 14 + // HeaderSize is the fixed size of the header area in a page. + HeaderSize uint16 = 1 << 9 + // HeaderOffset is the offset in the page's data, at which the header + // starts. + HeaderOffset uint16 = 0 + // ContentOffset is the offset in the page's data, at which the header ends + // and the content starts. This is also, where the offsets are stored. + ContentOffset uint16 = HeaderOffset + HeaderSize + // ContentSize is the size of the content area, equivalent to the page size + // minus the header size. + ContentSize uint16 = PageSize - HeaderSize + + // FrameSizeSize is the byte size of a frame. + FrameSizeSize = uint16(unsafe.Sizeof(uint16(0))) // #nosec + + HeaderVersionOffset = HeaderOffset + HeaderVersionSize = uint16(unsafe.Sizeof(uint16(0))) // #nosec + HeaderVersionLo = HeaderVersionOffset + HeaderVersionHi = HeaderVersionOffset + HeaderVersionSize + + HeaderIDOffset = HeaderVersionOffset + HeaderVersionSize + HeaderIDSize = uint16(unsafe.Sizeof(uint32(0))) // #nosec + HeaderIDLo = HeaderIDOffset + HeaderIDHi = HeaderIDOffset + HeaderIDSize + + HeaderCellCountOffset = HeaderIDOffset + HeaderIDSize + HeaderCellCountSize = uint16(unsafe.Sizeof(uint16(0))) // #nosec + HeaderCellCountLo = HeaderCellCountOffset + HeaderCellCountHi = HeaderCellCountOffset + HeaderCellCountSize +) + +var _ page.Loader = Load +var _ page.Page = (*Page)(nil) + +// Page is an implementation of (page.Page). It implements the concept of +// slotted pages, also it holds all data in memory at all times. The byte slice +// that is passed in when loading the page, is the one that the implementation +// will operate on. It will always stick to that slice, and never replace it +// with another. All changes to the page are immediately reflected in that byte +// slice. This implementation is NOT safe for concurrent use. This +// implementation does not support extension pages. +type Page struct { + data []byte + + header map[page.Header]interface{} +} + +// Load loads the given bytes as a page. The page will operate on the given byte +// slice, and never copy or exchange it. Modifying the byte slice after loading +// a page from it will likely corrupt the page. External changes to the byte +// slice are not guaranteed to be immediately reflected in the page object. +func Load(data []byte) (page.Page, error) { + return load(data) +} + +// Header obtains the header field value of the given key. If the key is not +// supported by this implementation, it will return an error indicating an +// unknown header. +func (p *Page) Header(key page.Header) (interface{}, error) { + val, ok := p.header[key] + if !ok { + return nil, page.ErrUnknownHeader + } + return val, nil +} + +// StoreCell will store the given cell in this page. If there is not enough +// space, NO extension page will be allocated, but an error will be returned, +// indicating insufficient space. +func (p *Page) StoreCell(cell page.Cell) error { + cellData := encodeCell(cell) + insertionOffset, ok := p.findInsertionOffset(uint16(len(cellData))) + if !ok { + return page.ErrPageTooSmall + } + offsetInsertionOffset := p.findOffsetInsertionOffset(cell) + p.insertOffset(insertionOffset, offsetInsertionOffset) + copy(p.data[ContentOffset+insertionOffset.Location:], cellData) + + p.incrementCellCount() + + return nil +} + +// Cells returns all cells that are stored in this page in sorted fashion, +// ordered ascending by key. +func (p *Page) Cells() (cells []page.Cell) { + for _, offset := range p.Offsets() { + cells = append(cells, p.getCell(offset)) + } + return +} + +// Offsets returns all offsets of this page sorted by key of the cell that they +// point to. Following the offsets in the order that they are returned, will +// result in a list of cells, that are sorted ascending by key. +func (p *Page) Offsets() (result []Offset) { + cellCount := p.cellCount() + offsetData := p.data[ContentOffset : ContentOffset+OffsetSize*cellCount] + for i := 0; i < len(offsetData); i += int(OffsetSize) { + result = append(result, decodeOffset(offsetData[i:i+int(OffsetSize)])) + } + return +} + +func load(data []byte) (*Page, error) { + if len(data) != int(PageSize) { + return nil, page.ErrInvalidPageSize + } + p := &Page{ + data: data, + header: make(map[page.Header]interface{}), + } + p.loadHeader() + return p, nil +} + +// insertOffset inserts the given offset at the other given offset. A bit +// confusing, because we need to store an offset, and the location is given by +// another offset. +// +// This method takes care of inserting the offset, and moving other offsets to +// the right, instead of overwriting them. +func (p *Page) insertOffset(offset, at Offset) { + cellCount := p.cellCount() + offsetData := p.data[ContentOffset : ContentOffset+OffsetSize*(cellCount+1)] + encOffset := encodeOffset(offset) + buf := make([]byte, uint16(len(offsetData))-OffsetSize-at.Location) + copy(buf, offsetData[at.Location:]) + copy(offsetData[at.Location+OffsetSize:], buf) + copy(offsetData[at.Location:], encOffset) +} + +// getCell returns the cell that an offset points to. +func (p *Page) getCell(offset Offset) page.Cell { + return decodeCell(p.data[ContentOffset+offset.Location : ContentOffset+offset.Location+offset.Size]) +} + +// findInsertionOffset finds an offset for a data segment of the given size, or +// returns false if no space is available. The space is found by using a +// first-fit approach. +func (p *Page) findInsertionOffset(size uint16) (Offset, bool) { + offsets := p.Offsets() + sort.Slice(offsets, func(i int, j int) bool { + return offsets[i].Location < offsets[j].Location + }) + + // TODO: best fit, this is currently first fit + rightBound := ContentSize + for i := len(offsets) - 1; i >= 0; i-- { + current := offsets[i] + if current.Location+current.Size >= rightBound-size { + // doesn't fit + rightBound = current.Location + } else { + break + } + } + + return Offset{ + Location: rightBound - size, + Size: size, + }, true +} + +// findOffsetInsertionOffset creates an offset to a location, where a new offset +// can be inserted. The new offset that should be inserted, is not part of this +// method. However, we need the cell that the new offset points to, in order to +// compare its key with other cells. This is, because we insert offsets in a +// way, so that all offsets from left to right point to cells, that are ordered +// ascending by key when following all offsets. +func (p *Page) findOffsetInsertionOffset(cell page.Cell) Offset { + // TODO: binary search + offsets := p.Offsets() + for i, offset := range offsets { + if bytes.Compare(p.getCell(offset).Key, cell.Key) > 0 { + return Offset{ + Location: uint16(i) * OffsetSize, + Size: OffsetSize, + } + } + } + return Offset{ + Location: uint16(len(offsets)) * OffsetSize, + Size: OffsetSize, + } +} + +// loadHeader (re-)loads all header values from the page's data. +func (p *Page) loadHeader() { + p.header[page.HeaderVersion] = converter.ByteSliceToUint16(p.data[HeaderVersionLo:HeaderVersionHi]) + p.header[page.HeaderID] = converter.ByteSliceToUint16(p.data[HeaderIDLo:HeaderIDHi]) + p.header[page.HeaderCellCount] = converter.ByteSliceToUint16(p.data[HeaderCellCountLo:HeaderCellCountHi]) +} + +func (p *Page) setHeaderCellCount(count uint16) { + // set in memory cache + p.header[page.HeaderCellCount] = count + // set in data + copy(p.data[HeaderCellCountLo:HeaderCellCountHi], converter.Uint16ToByteSlice(count)) +} + +// incrementCellCount increments the header field HeaderCellCount by one. +func (p *Page) incrementCellCount() { + p.setHeaderCellCount(p.header[page.HeaderCellCount].(uint16) + 1) +} + +// cellCount returns the amount of currently stored cells. This value is +// retrieved from the header field HeaderCellCount. +func (p *Page) cellCount() uint16 { + return p.header[page.HeaderCellCount].(uint16) +} diff --git a/internal/engine/storage/page/v1/v1_page_bench_test.go b/internal/engine/storage/page/v1/v1_page_bench_test.go new file mode 100644 index 00000000..1c418e9e --- /dev/null +++ b/internal/engine/storage/page/v1/v1_page_bench_test.go @@ -0,0 +1,68 @@ +package v1 + +import ( + "testing" + + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +var result interface{} + +func Benchmark_Page_StoreCell(b *testing.B) { + _p, _ := load(make([]byte, PageSize)) + _ = _p.StoreCell(page.Cell{ + Key: []byte{0xAA}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }) + _ = _p.StoreCell(page.Cell{ + Key: []byte{0xFF}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }) + _ = _p.StoreCell(page.Cell{ + Key: []byte{0x11}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + b.StopTimer() + data := make([]byte, PageSize) + copy(data, _p.data) + p, _ := load(data) + b.StartTimer() + + _ = p.StoreCell(page.Cell{ + Key: []byte{0xDD}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }) + } +} + +func Benchmark_Page_Load(b *testing.B) { + _p, _ := load(make([]byte, PageSize)) + _ = _p.StoreCell(page.Cell{ + Key: []byte{0xAA}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }) + _ = _p.StoreCell(page.Cell{ + Key: []byte{0xFF}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }) + _ = _p.StoreCell(page.Cell{ + Key: []byte{0x11}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }) + + var r page.Page + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + p, _ := Load(_p.data) + + r = p + } + + result = r +} diff --git a/internal/engine/storage/page/v1/v1_page_test.go b/internal/engine/storage/page/v1/v1_page_test.go new file mode 100644 index 00000000..48612dd2 --- /dev/null +++ b/internal/engine/storage/page/v1/v1_page_test.go @@ -0,0 +1,216 @@ +package v1 + +import ( + "bytes" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/engine/converter" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +func Test_Page_StoreCell(t *testing.T) { + t.Run("single cell", func(t *testing.T) { + assert := assert.New(t) + + p, err := load(make([]byte, PageSize)) // empty page: version=0,id=0,cellcount=0 + assert.NoError(err) + assert.NotNil(p) + + assert.NoError( + p.StoreCell(page.Cell{ + Key: []byte{0xCA}, + Record: []byte{0xFE, 0xBA, 0xBE}, + }), + ) + + assert.Equal([]byte{ + 0x00, 0x01, // key frame + 0xCA, // key + 0x00, 0x03, // record frame + 0xFE, 0xBA, 0xBE, // record + }, p.data[PageSize-8:]) + + assert.EqualValues(1, p.header[page.HeaderCellCount]) + assert.Equal([]byte{ + 0x3D, 0xF8, // location of our cell + 0x00, 0x08, // size of our cell + }, p.data[ContentOffset:ContentOffset+4]) + + allCells := p.Cells() + assert.Len(allCells, 1) + }) + t.Run("multiple cells", func(t *testing.T) { + assert := assert.New(t) + + p, err := load(make([]byte, PageSize)) // empty page: version=0,id=0,cellcount=0 + assert.NoError(err) + assert.NotNil(p) + + // first cell + + assert.NoError( + p.StoreCell(page.Cell{ + Key: []byte{0xAA}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }), + ) + + assert.Equal([]byte{ + 0x00, 0x01, // key frame + 0xAA, // key + 0x00, 0x04, // record frame + 0xCA, 0xFE, 0xBA, 0xBE, // record + }, p.data[PageSize-9:]) + + // second cell + + assert.NoError( + p.StoreCell(page.Cell{ + Key: []byte{0xFF}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }), + ) + + assert.Equal([]byte{ + 0x00, 0x01, // key frame + 0xFF, // key + 0x00, 0x04, // record frame + 0xCA, 0xFE, 0xBA, 0xBE, // record + }, p.data[PageSize-18:PageSize-9]) + + // third cell + + assert.NoError( + p.StoreCell(page.Cell{ + Key: []byte{0x11}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }), + ) + + assert.Equal([]byte{ + 0x00, 0x01, // key frame + 0x11, // key + 0x00, 0x04, // record frame + 0xCA, 0xFE, 0xBA, 0xBE, // record + }, p.data[PageSize-27:PageSize-18]) + + // check that all the offsets at the beginning of the content area are + // sorted by the key of the cell they point to + + x11Offset := converter.Uint16ToByteArray(ContentSize - 27) + xAAOffset := converter.Uint16ToByteArray(ContentSize - 9) + xFFOffset := converter.Uint16ToByteArray(ContentSize - 18) + + assert.Equal([]byte{ + // first offset + x11Offset[0], x11Offset[1], // location + 0x00, 0x09, + // second offset + xAAOffset[0], xAAOffset[1], // location + 0x00, 0x09, + // third offset + xFFOffset[0], xFFOffset[1], // location + 0x00, 0x09, + }, p.data[ContentOffset:ContentOffset+OffsetSize*3]) + + allCells := p.Cells() + assert.Len(allCells, 3) + assert.True(sort.SliceIsSorted(allCells, func(i, j int) bool { + return bytes.Compare(allCells[i].Key, allCells[j].Key) < 0 + }), "p.Cells() must return all cells ordered by key") + }) +} + +func TestInvalidPageSize(t *testing.T) { + tf := func(t *testing.T, data []byte) { + assert := assert.New(t) + p, err := Load(data) + assert.Equal(page.ErrInvalidPageSize, err) + assert.Nil(p) + } + t.Run("invalid=nil", func(t *testing.T) { + tf(t, nil) + }) + t.Run("invalid=smaller", func(t *testing.T) { + data := make([]byte, PageSize/2) + tf(t, data) + }) + t.Run("invalid=larger", func(t *testing.T) { + data := make([]byte, int(PageSize)*2) + tf(t, data) + }) +} + +func TestHeaderVersion(t *testing.T) { + assert := assert.New(t) + + versionBytes := []byte{0xCA, 0xFE} + + data := make([]byte, PageSize) + copy(data[:2], versionBytes) + + p, err := load(data) + assert.NoError(err) + + version, err := p.Header(page.HeaderVersion) + assert.NoError(err) + assert.IsType(uint16(0), version) + assert.EqualValues(0xCAFE, version) + + assert.Equal(versionBytes, p.data[:2]) + for _, b := range p.data[2:] { + assert.EqualValues(0, b) + } +} + +func TestHeaderID(t *testing.T) { + assert := assert.New(t) + + idBytes := []byte{0xCA, 0xFE, 0xBA, 0xBE} + + data := make([]byte, PageSize) + copy(data[2:6], idBytes) + + p, err := load(data) + assert.NoError(err) + + id, err := p.Header(page.HeaderID) + assert.NoError(err) + assert.IsType(uint16(0), id) + assert.EqualValues(0xCAFE, id) + + for _, b := range p.data[:2] { + assert.EqualValues(0, b) + } + assert.Equal(idBytes, p.data[2:6]) + for _, b := range p.data[6:] { + assert.EqualValues(0, b) + } +} + +func TestHeaderCellCount(t *testing.T) { + assert := assert.New(t) + + cellCountBytes := []byte{0xCA, 0xFE} + + data := make([]byte, PageSize) + copy(data[6:8], cellCountBytes) + + p, err := load(data) + assert.NoError(err) + + cellCount, err := p.Header(page.HeaderCellCount) + assert.NoError(err) + assert.IsType(uint16(0), cellCount) + assert.EqualValues(0xCAFE, cellCount) + + for _, b := range p.data[:6] { + assert.EqualValues(0, b) + } + assert.Equal(cellCountBytes, p.data[6:8]) + for _, b := range p.data[8:] { + assert.EqualValues(0, b) + } +} From 0151cfd0e8268892038db4d1beaefe1b482a8fc6 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Sat, 6 Jun 2020 19:11:51 +0200 Subject: [PATCH 08/77] Add delete and get method to page --- internal/engine/storage/page/page.go | 7 +++++++ internal/engine/storage/page/v1/v1_page.go | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/internal/engine/storage/page/page.go b/internal/engine/storage/page/page.go index f75a86e8..1ecd44e3 100644 --- a/internal/engine/storage/page/page.go +++ b/internal/engine/storage/page/page.go @@ -32,6 +32,13 @@ type Page interface { // StoreCell stores a cell on the page. If the page does not fit the cell // because of size or too much fragmentation, an error will be returned. StoreCell(Cell) error + // Delete deletes the cell with the given bytes as key. If the key couldn't + // be found, nil is returned. If an error occured during deletion, the error + // is returned. + Delete([]byte) error + // Cell returns a cell with the given key, together with a bool indicating + // whether any cell in the page has that key. + Cell([]byte) (Cell, bool) // Cells returns all cells in this page as a slice. Cells are ordered // ascending by key. Calling this method is relatively cheap. Cells() []Cell diff --git a/internal/engine/storage/page/v1/v1_page.go b/internal/engine/storage/page/v1/v1_page.go index ba5280c2..e6e9b149 100644 --- a/internal/engine/storage/page/v1/v1_page.go +++ b/internal/engine/storage/page/v1/v1_page.go @@ -97,6 +97,21 @@ func (p *Page) StoreCell(cell page.Cell) error { return nil } +func (p *Page) Delete(key []byte) error { + panic("TODO") + return nil +} + +func (p *Page) Cell(key []byte) (page.Cell, bool) { + // TODO: binary search + for _, cell := range p.Cells() { + if bytes.Equal(key, cell.Key) { + return cell, true + } + } + return page.Cell{}, false +} + // Cells returns all cells that are stored in this page in sorted fashion, // ordered ascending by key. func (p *Page) Cells() (cells []page.Cell) { From 9d5644d0b1f1e43d738787b811e0f353dfedeec5 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Sat, 6 Jun 2020 19:12:38 +0200 Subject: [PATCH 09/77] Create blackbox tests Create a blackbox test suite that can check any page implementation for validity. --- internal/engine/storage/page/page_test.go | 72 +++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 internal/engine/storage/page/page_test.go diff --git a/internal/engine/storage/page/page_test.go b/internal/engine/storage/page/page_test.go new file mode 100644 index 00000000..37c5273a --- /dev/null +++ b/internal/engine/storage/page/page_test.go @@ -0,0 +1,72 @@ +package page_test + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/engine/storage/page" + v1 "github.com/tomarrell/lbadd/internal/engine/storage/page/v1" +) + +var ( + Loaders = []struct { + Name string + Loader page.Loader + PageSize int + }{ + {"v1", v1.Load, 1 << 14}, + } + + Cells = []page.Cell{ + { + Key: []byte("key[0]"), + Record: []byte("data[0]"), + }, + { + Key: []byte("key[1]"), + Record: []byte("data[1]"), + }, + { + Key: []byte("key[2]"), + Record: []byte("data[2]"), + }, + { + Key: []byte("key[3]"), + Record: []byte("data[3]"), + }, + } +) + +func TestStoreCell(t *testing.T) { + for _, loader := range Loaders { + t.Run("loader="+loader.Name, func(t *testing.T) { _TestStoreAndGetCell(t, loader.Loader, loader.PageSize) }) + } +} + +func _TestStoreAndGetCell(t *testing.T, loader page.Loader, pageSize int) { + assert := assert.New(t) + rand := rand.New(rand.NewSource(87234562678)) // reproducible random + cells := make([]page.Cell, len(Cells)) + copy(cells, Cells) + rand.Shuffle(len(cells), func(i, j int) { cells[i], cells[j] = cells[j], cells[i] }) + + data := make([]byte, pageSize) + p, err := loader(data) + assert.NoError(err) + assert.NotNil(p) + + for _, cell := range cells { + err = p.StoreCell(cell) + assert.NoError(err) + } + + // after insertion, all cells must be returned sorted, as the original Cells + // slice + assert.Equal(Cells, p.Cells()) + for _, cell := range cells { + obtained, ok := p.Cell(cell.Key) + assert.True(ok) + assert.Equal(cell, obtained) + } +} From dffe4b6433f5b74d85c99238db98b31271970eac Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Sat, 6 Jun 2020 20:07:10 +0200 Subject: [PATCH 10/77] Add test and support for deletion of cells --- internal/engine/storage/page/page_test.go | 24 +++++++++++++++++++--- internal/engine/storage/page/v1/codec.go | 6 ++++++ internal/engine/storage/page/v1/v1_page.go | 24 +++++++++++++++++++++- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/internal/engine/storage/page/page_test.go b/internal/engine/storage/page/page_test.go index 37c5273a..c319ad72 100644 --- a/internal/engine/storage/page/page_test.go +++ b/internal/engine/storage/page/page_test.go @@ -38,13 +38,13 @@ var ( } ) -func TestStoreCell(t *testing.T) { +func TestImplementations(t *testing.T) { for _, loader := range Loaders { - t.Run("loader="+loader.Name, func(t *testing.T) { _TestStoreAndGetCell(t, loader.Loader, loader.PageSize) }) + t.Run("loader="+loader.Name, func(t *testing.T) { _TestPageOperations(t, loader.Loader, loader.PageSize) }) } } -func _TestStoreAndGetCell(t *testing.T, loader page.Loader, pageSize int) { +func _TestPageOperations(t *testing.T, loader page.Loader, pageSize int) { assert := assert.New(t) rand := rand.New(rand.NewSource(87234562678)) // reproducible random cells := make([]page.Cell, len(Cells)) @@ -69,4 +69,22 @@ func _TestStoreAndGetCell(t *testing.T, loader page.Loader, pageSize int) { assert.True(ok) assert.Equal(cell, obtained) } + + // page header cell count must be up-to-date + val, err := p.Header(page.HeaderCellCount) + assert.NoError(err) + assert.Equal(uint16(len(Cells)), val) + + // delete one cell + deleteIndex := 2 + err = p.Delete(Cells[deleteIndex].Key) + assert.NoError(err) + + afterDeletionCells := make([]page.Cell, len(Cells)) + copy(afterDeletionCells, Cells) + afterDeletionCells = append(afterDeletionCells[:deleteIndex], afterDeletionCells[deleteIndex+1:]...) + assert.Equal(afterDeletionCells, p.Cells()) + obtained, ok := p.Cell(Cells[deleteIndex].Key) + assert.False(ok, "delete cell must not be obtainable anymore") + assert.Zero(obtained) } diff --git a/internal/engine/storage/page/v1/codec.go b/internal/engine/storage/page/v1/codec.go index ea4820f6..6dbe6cf4 100644 --- a/internal/engine/storage/page/v1/codec.go +++ b/internal/engine/storage/page/v1/codec.go @@ -59,3 +59,9 @@ func decodeOffset(data []byte) Offset { Size: converter.ByteSliceToUint16(data[int(OffsetSize)/2 : int(OffsetSize)]), } } + +func zero(b []byte) { + for i := range b { + b[i] = 0x00 + } +} diff --git a/internal/engine/storage/page/v1/v1_page.go b/internal/engine/storage/page/v1/v1_page.go index e6e9b149..b21d13e3 100644 --- a/internal/engine/storage/page/v1/v1_page.go +++ b/internal/engine/storage/page/v1/v1_page.go @@ -98,7 +98,24 @@ func (p *Page) StoreCell(cell page.Cell) error { } func (p *Page) Delete(key []byte) error { - panic("TODO") + offsets := p.Offsets() + + for i, offset := range offsets { + if bytes.Equal(key, p.getCell(offset).Key) { + offsetData := p.data[ContentOffset : ContentOffset+(p.cellCount()*OffsetSize)] + zero(offsetData) + offsets = append(offsets[:i], offsets[i+1:]...) + for j, off := range offsets { + lo := uint16(j) * OffsetSize + hi := lo + OffsetSize + copy(offsetData[lo:hi], encodeOffset(off)) + } + break + } + } + + p.decrementCellCount() + return nil } @@ -235,6 +252,11 @@ func (p *Page) incrementCellCount() { p.setHeaderCellCount(p.header[page.HeaderCellCount].(uint16) + 1) } +// decrementCellCount decrements the header field HeaderCellCount by one. +func (p *Page) decrementCellCount() { + p.setHeaderCellCount(p.header[page.HeaderCellCount].(uint16) - 1) +} + // cellCount returns the amount of currently stored cells. This value is // retrieved from the header field HeaderCellCount. func (p *Page) cellCount() uint16 { From bb06a4379a9b7beb5f22fe205c44ac1f85090b0b Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Sat, 6 Jun 2020 22:18:57 +0200 Subject: [PATCH 11/77] Add missing godoc --- internal/engine/storage/page/v1/v1_page.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/engine/storage/page/v1/v1_page.go b/internal/engine/storage/page/v1/v1_page.go index b21d13e3..549c0f43 100644 --- a/internal/engine/storage/page/v1/v1_page.go +++ b/internal/engine/storage/page/v1/v1_page.go @@ -97,6 +97,8 @@ func (p *Page) StoreCell(cell page.Cell) error { return nil } +// Delete deletes the cell with the given key. If there is no such cell, this is +// a no-op. This never returns an error. func (p *Page) Delete(key []byte) error { offsets := p.Offsets() @@ -119,6 +121,9 @@ func (p *Page) Delete(key []byte) error { return nil } +// Cell searches for a cell with the given key, and returns a cell object +// representing all the found cell data. If no cell was found, false is +// returned. func (p *Page) Cell(key []byte) (page.Cell, bool) { // TODO: binary search for _, cell := range p.Cells() { From 089e19e8030a99f3206219a790b42bcf7da24c59 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Sun, 7 Jun 2020 11:21:38 +0200 Subject: [PATCH 12/77] Make cell count internal and add a dirty flag --- internal/engine/storage/page/page.go | 20 ++++-- internal/engine/storage/page/page_test.go | 7 +- internal/engine/storage/page/v1/v1_page.go | 69 ++++++++++++++----- .../engine/storage/page/v1/v1_page_test.go | 11 ++- 4 files changed, 68 insertions(+), 39 deletions(-) diff --git a/internal/engine/storage/page/page.go b/internal/engine/storage/page/page.go index 1ecd44e3..a1bc8587 100644 --- a/internal/engine/storage/page/page.go +++ b/internal/engine/storage/page/page.go @@ -9,25 +9,31 @@ const ( // HeaderID is the page ID, which may be used outside of the page for // housekeeping. This is a uint16. HeaderID - // HeaderCellCount is the amount of cells that are currently stored in the - // page. This is a uint16. - HeaderCellCount ) // Loader is a function that can load a page from a given byte slice, and return // errors if any occur. type Loader func([]byte) (Page, error) +//go:generate mockery -case=snake -name=Page + // Page describes a memory page that stores (page.Cell)s. A page consists of // header fields and cells, and is a plain store. Obtained cells are always // ordered ascending by the cell key. A page supports variable size cell keys // and records. type Page interface { // Header obtains a header field from the page's header. If the header is - // not supported, a result=nil,error=ErrUnknownHeader is returned. The type - // of the returned header field value is documented in the header key's - // godoc. - Header(Header) (interface{}, error) + // not supported, result=nil is returned. The type of the returned header + // field value is documented in the header key's godoc. + Header(Header) interface{} + + // Dirty determines whether this page has been modified since the last time + // `Page.ClearDirty` was called. + Dirty() bool + // MarkDirty marks this page as dirty. + MarkDirty() + // ClearDirty unsets the dirty flag from this page. + ClearDirty() // StoreCell stores a cell on the page. If the page does not fit the cell // because of size or too much fragmentation, an error will be returned. diff --git a/internal/engine/storage/page/page_test.go b/internal/engine/storage/page/page_test.go index c319ad72..652dd6a7 100644 --- a/internal/engine/storage/page/page_test.go +++ b/internal/engine/storage/page/page_test.go @@ -70,11 +70,6 @@ func _TestPageOperations(t *testing.T, loader page.Loader, pageSize int) { assert.Equal(cell, obtained) } - // page header cell count must be up-to-date - val, err := p.Header(page.HeaderCellCount) - assert.NoError(err) - assert.Equal(uint16(len(Cells)), val) - // delete one cell deleteIndex := 2 err = p.Delete(Cells[deleteIndex].Key) @@ -85,6 +80,6 @@ func _TestPageOperations(t *testing.T, loader page.Loader, pageSize int) { afterDeletionCells = append(afterDeletionCells[:deleteIndex], afterDeletionCells[deleteIndex+1:]...) assert.Equal(afterDeletionCells, p.Cells()) obtained, ok := p.Cell(Cells[deleteIndex].Key) - assert.False(ok, "delete cell must not be obtainable anymore") + assert.False(ok, "deleted cell must not be obtainable anymore") assert.Zero(obtained) } diff --git a/internal/engine/storage/page/v1/v1_page.go b/internal/engine/storage/page/v1/v1_page.go index 549c0f43..4d935e78 100644 --- a/internal/engine/storage/page/v1/v1_page.go +++ b/internal/engine/storage/page/v1/v1_page.go @@ -9,6 +9,13 @@ import ( "github.com/tomarrell/lbadd/internal/engine/storage/page" ) +// Internal headers, defined from 255 downwards, as opposed to other headers, +// which are defined from 0 upwards. +const ( + InternalHeaderCellCount page.Header = page.Header(^uint8(0)) - iota + InternalHeaderDirty +) + // Constants. const ( // PageSize is the fixed size of one page. @@ -38,10 +45,14 @@ const ( HeaderIDLo = HeaderIDOffset HeaderIDHi = HeaderIDOffset + HeaderIDSize - HeaderCellCountOffset = HeaderIDOffset + HeaderIDSize - HeaderCellCountSize = uint16(unsafe.Sizeof(uint16(0))) // #nosec - HeaderCellCountLo = HeaderCellCountOffset - HeaderCellCountHi = HeaderCellCountOffset + HeaderCellCountSize + InternalHeaderCellCountOffset = HeaderIDOffset + HeaderIDSize + InternalHeaderCellCountSize = uint16(unsafe.Sizeof(uint16(0))) // #nosec + InternalHeaderCellCountLo = InternalHeaderCellCountOffset + InternalHeaderCellCountHi = InternalHeaderCellCountOffset + InternalHeaderCellCountSize + + InternalHeaderDirtyOffset = InternalHeaderCellCountOffset + InternalHeaderCellCountSize + InternalHeaderDirtySize = uint16(unsafe.Sizeof(false)) // #nosec + InternalHeaderDirtyIndex = InternalHeaderDirtyOffset ) var _ page.Loader = Load @@ -69,14 +80,34 @@ func Load(data []byte) (page.Page, error) { } // Header obtains the header field value of the given key. If the key is not -// supported by this implementation, it will return an error indicating an -// unknown header. -func (p *Page) Header(key page.Header) (interface{}, error) { +// supported by this implementation, it will return nil indicating an unknown +// header. +func (p *Page) Header(key page.Header) interface{} { val, ok := p.header[key] if !ok { - return nil, page.ErrUnknownHeader + return nil } - return val, nil + return val +} + +// Dirty returns the value of the header flag dirty, meaning that the page has +// been modified since ClearDirty() has been called the last time. +func (p *Page) Dirty() bool { + return p.header[InternalHeaderDirty].(bool) +} + +// MarkDirty marks this page as dirty. Call this when you have modified the +// page. This will +func (p *Page) MarkDirty() { + p.data[InternalHeaderDirtyIndex] = converter.BoolToByte(true) + p.header[InternalHeaderDirty] = true +} + +// ClearDirty marks this page as NOT dirty (anymore). Call this when the page +// has been written to persistent storage. +func (p *Page) ClearDirty() { + p.data[InternalHeaderDirtyIndex] = converter.BoolToByte(false) + p.header[InternalHeaderDirty] = false } // StoreCell will store the given cell in this page. If there is not enough @@ -242,28 +273,28 @@ func (p *Page) findOffsetInsertionOffset(cell page.Cell) Offset { func (p *Page) loadHeader() { p.header[page.HeaderVersion] = converter.ByteSliceToUint16(p.data[HeaderVersionLo:HeaderVersionHi]) p.header[page.HeaderID] = converter.ByteSliceToUint16(p.data[HeaderIDLo:HeaderIDHi]) - p.header[page.HeaderCellCount] = converter.ByteSliceToUint16(p.data[HeaderCellCountLo:HeaderCellCountHi]) + p.header[InternalHeaderCellCount] = converter.ByteSliceToUint16(p.data[InternalHeaderCellCountLo:InternalHeaderCellCountHi]) } -func (p *Page) setHeaderCellCount(count uint16) { +func (p *Page) setInternalHeaderCellCount(count uint16) { // set in memory cache - p.header[page.HeaderCellCount] = count + p.header[InternalHeaderCellCount] = count // set in data - copy(p.data[HeaderCellCountLo:HeaderCellCountHi], converter.Uint16ToByteSlice(count)) + copy(p.data[InternalHeaderCellCountLo:InternalHeaderCellCountHi], converter.Uint16ToByteSlice(count)) } -// incrementCellCount increments the header field HeaderCellCount by one. +// incrementCellCount increments the header field InternalHeaderCellCount by one. func (p *Page) incrementCellCount() { - p.setHeaderCellCount(p.header[page.HeaderCellCount].(uint16) + 1) + p.setInternalHeaderCellCount(p.header[InternalHeaderCellCount].(uint16) + 1) } -// decrementCellCount decrements the header field HeaderCellCount by one. +// decrementCellCount decrements the header field InternalHeaderCellCount by one. func (p *Page) decrementCellCount() { - p.setHeaderCellCount(p.header[page.HeaderCellCount].(uint16) - 1) + p.setInternalHeaderCellCount(p.header[InternalHeaderCellCount].(uint16) - 1) } // cellCount returns the amount of currently stored cells. This value is -// retrieved from the header field HeaderCellCount. +// retrieved from the header field InternalHeaderCellCount. func (p *Page) cellCount() uint16 { - return p.header[page.HeaderCellCount].(uint16) + return p.header[InternalHeaderCellCount].(uint16) } diff --git a/internal/engine/storage/page/v1/v1_page_test.go b/internal/engine/storage/page/v1/v1_page_test.go index 48612dd2..247d3d33 100644 --- a/internal/engine/storage/page/v1/v1_page_test.go +++ b/internal/engine/storage/page/v1/v1_page_test.go @@ -32,7 +32,7 @@ func Test_Page_StoreCell(t *testing.T) { 0xFE, 0xBA, 0xBE, // record }, p.data[PageSize-8:]) - assert.EqualValues(1, p.header[page.HeaderCellCount]) + assert.EqualValues(1, p.header[InternalHeaderCellCount]) assert.Equal([]byte{ 0x3D, 0xF8, // location of our cell 0x00, 0x08, // size of our cell @@ -154,8 +154,7 @@ func TestHeaderVersion(t *testing.T) { p, err := load(data) assert.NoError(err) - version, err := p.Header(page.HeaderVersion) - assert.NoError(err) + version := p.Header(page.HeaderVersion) assert.IsType(uint16(0), version) assert.EqualValues(0xCAFE, version) @@ -176,8 +175,7 @@ func TestHeaderID(t *testing.T) { p, err := load(data) assert.NoError(err) - id, err := p.Header(page.HeaderID) - assert.NoError(err) + id := p.Header(page.HeaderID) assert.IsType(uint16(0), id) assert.EqualValues(0xCAFE, id) @@ -201,8 +199,7 @@ func TestHeaderCellCount(t *testing.T) { p, err := load(data) assert.NoError(err) - cellCount, err := p.Header(page.HeaderCellCount) - assert.NoError(err) + cellCount := p.Header(InternalHeaderCellCount) assert.IsType(uint16(0), cellCount) assert.EqualValues(0xCAFE, cellCount) From 7ae4eed407baeaf1745da87178a630875901b177 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Sun, 7 Jun 2020 11:21:43 +0200 Subject: [PATCH 13/77] Add a page mock --- internal/engine/storage/page/mocks/doc.go | 2 + internal/engine/storage/page/mocks/page.go | 118 +++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 internal/engine/storage/page/mocks/doc.go create mode 100644 internal/engine/storage/page/mocks/page.go diff --git a/internal/engine/storage/page/mocks/doc.go b/internal/engine/storage/page/mocks/doc.go new file mode 100644 index 00000000..35abed3d --- /dev/null +++ b/internal/engine/storage/page/mocks/doc.go @@ -0,0 +1,2 @@ +// Package mocks provides generated mock implementations for easy testing. +package mocks diff --git a/internal/engine/storage/page/mocks/page.go b/internal/engine/storage/page/mocks/page.go new file mode 100644 index 00000000..3e26117b --- /dev/null +++ b/internal/engine/storage/page/mocks/page.go @@ -0,0 +1,118 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + page "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +// Page is an autogenerated mock type for the Page type +type Page struct { + mock.Mock +} + +// Cell provides a mock function with given fields: _a0 +func (_m *Page) Cell(_a0 []byte) (page.Cell, bool) { + ret := _m.Called(_a0) + + var r0 page.Cell + if rf, ok := ret.Get(0).(func([]byte) page.Cell); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(page.Cell) + } + + var r1 bool + if rf, ok := ret.Get(1).(func([]byte) bool); ok { + r1 = rf(_a0) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// Cells provides a mock function with given fields: +func (_m *Page) Cells() []page.Cell { + ret := _m.Called() + + var r0 []page.Cell + if rf, ok := ret.Get(0).(func() []page.Cell); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]page.Cell) + } + } + + return r0 +} + +// ClearDirty provides a mock function with given fields: +func (_m *Page) ClearDirty() { + _m.Called() +} + +// Delete provides a mock function with given fields: _a0 +func (_m *Page) Delete(_a0 []byte) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func([]byte) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Dirty provides a mock function with given fields: +func (_m *Page) Dirty() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// Header provides a mock function with given fields: _a0 +func (_m *Page) Header(_a0 page.Header) interface{} { + ret := _m.Called(_a0) + + var r0 interface{} + if rf, ok := ret.Get(0).(func(page.Header) interface{}); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + return r0 +} + +// MarkDirty provides a mock function with given fields: +func (_m *Page) MarkDirty() { + _m.Called() +} + +// StoreCell provides a mock function with given fields: _a0 +func (_m *Page) StoreCell(_a0 page.Cell) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(page.Cell) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} From 457d4f3dfdb20c2a4247355c58622d055a68297e Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Wed, 10 Jun 2020 00:08:06 +0200 Subject: [PATCH 14/77] Implement a simple LRU page cache --- internal/engine/storage/page/cache.go | 25 +++ .../page/v1/mock_secondary_storage_test.go | 47 +++++ internal/engine/storage/page/v1/v1_cache.go | 162 ++++++++++++++++++ .../engine/storage/page/v1/v1_cache_test.go | 71 ++++++++ .../engine/storage/page/v1/v1_pageloader.go | 11 ++ 5 files changed, 316 insertions(+) create mode 100644 internal/engine/storage/page/cache.go create mode 100644 internal/engine/storage/page/v1/mock_secondary_storage_test.go create mode 100644 internal/engine/storage/page/v1/v1_cache.go create mode 100644 internal/engine/storage/page/v1/v1_cache_test.go create mode 100644 internal/engine/storage/page/v1/v1_pageloader.go diff --git a/internal/engine/storage/page/cache.go b/internal/engine/storage/page/cache.go new file mode 100644 index 00000000..4ed02f24 --- /dev/null +++ b/internal/engine/storage/page/cache.go @@ -0,0 +1,25 @@ +package page + +import "io" + +// Cache describes a page cache that caches pages from a secondary storage. +type Cache interface { + // FetchAndPin fetches a page from this cache. If it does not exist in the + // cache, it must be loaded from a configured source. After the page was + // fetched, it is pinned, meaning that it's guaranteed that the page is not + // evicted. After working with the page, it must be released again, in order + // for the cache to be able to free memory. If a page with the given id does + // not exist, an error will be returned. + FetchAndPin(id uint32) (Page, error) + // Unpin tells the cache that the page with the given id is no longer + // required directly, and that it can be evicted. Unpin is not a guarantee + // that the page will be evicted. The cache determines, when to evict a + // page. If a page with that id does not exist, this call is a no-op. + Unpin(id uint32) + // Flush writes the contents of the page with the given id to the configured + // source. Before a page is evicted, it is always flushed. Use this method + // to tell the cache that the page must be flushed immediately. If a page + // with the given id does not exist, an error will be returned. + Flush(id uint32) error + io.Closer +} diff --git a/internal/engine/storage/page/v1/mock_secondary_storage_test.go b/internal/engine/storage/page/v1/mock_secondary_storage_test.go new file mode 100644 index 00000000..f109a40b --- /dev/null +++ b/internal/engine/storage/page/v1/mock_secondary_storage_test.go @@ -0,0 +1,47 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package v1 + +import mock "github.com/stretchr/testify/mock" + +// MockSecondaryStorage is an autogenerated mock type for the SecondaryStorage type +type MockSecondaryStorage struct { + mock.Mock +} + +// LoadPage provides a mock function with given fields: _a0 +func (_m *MockSecondaryStorage) LoadPage(_a0 uint32) (*Page, error) { + ret := _m.Called(_a0) + + var r0 *Page + if rf, ok := ret.Get(0).(func(uint32) *Page); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*Page) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint32) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// WritePage provides a mock function with given fields: _a0, _a1 +func (_m *MockSecondaryStorage) WritePage(_a0 uint32, _a1 *Page) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(uint32, *Page) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/internal/engine/storage/page/v1/v1_cache.go b/internal/engine/storage/page/v1/v1_cache.go new file mode 100644 index 00000000..4a429cf4 --- /dev/null +++ b/internal/engine/storage/page/v1/v1_cache.go @@ -0,0 +1,162 @@ +package v1 + +import ( + "fmt" + + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +const ( + // CacheSize is the default amount of pages that a cache can hold. + CacheSize int = 1 << 8 +) + +// Cache is a simple LRU implementation of a page cache. Lookups in the cache +// are performed with amortized O(1) complexity, however, moving elements in the +// LRU list is more expensive. +type Cache struct { + secondaryStorage SecondaryStorage + + pages map[uint32]*Page + pinned map[uint32]struct{} + + size int + lru []uint32 +} + +// NewCache creates a new cache with the given secondary storage. +func NewCache(secondaryStorage SecondaryStorage) page.Cache { + return newCache(secondaryStorage) +} + +// FetchAndPin fetches the page with the given ID from the cache or the +// secondary storage and pins it in the cache. Pinning prevents the page from +// being evicted. +func (c *Cache) FetchAndPin(id uint32) (page.Page, error) { + return c.fetchAndPin(id) +} + +// Unpin marks the given ID as ok to be evicted. This is a no-op if the ID +// doesn't exist or is not pinned. +func (c *Cache) Unpin(id uint32) { + c.unpin(id) +} + +// Flush writes the contents of the page with the given ID to the secondary +// storage. +func (c *Cache) Flush(id uint32) error { + return c.flush(id) +} + +// Close does nothing. +func (c *Cache) Close() error { + return nil +} + +func newCache(secondaryStorage SecondaryStorage) *Cache { + return &Cache{ + secondaryStorage: secondaryStorage, + + pages: make(map[uint32]*Page), + pinned: make(map[uint32]struct{}), + + size: CacheSize, + lru: make([]uint32, 0, CacheSize), + } +} + +func (c *Cache) fetchAndPin(id uint32) (*Page, error) { + // pin id first in order to avoid potential concurrent eviction at this + // point + c.pin(id) + p, err := c.fetch(id) + if err != nil { + // unpin if a page with the given id cannot be loaded + c.unpin(id) + return nil, err + } + return p, nil +} + +func (c *Cache) fetch(id uint32) (*Page, error) { + // check if page is already cached + if p, ok := c.pages[id]; ok { + moveToFront(id, c.lru) + return p, nil + } + + // check if we have to evict pages + if err := c.freeMemoryIfNeeded(); err != nil { + return nil, fmt.Errorf("free mem: %w", err) + } + + // fetch page from secondary storage + p, err := c.secondaryStorage.LoadPage(id) + if err != nil { + return nil, fmt.Errorf("load page: %w", err) + } + // store in cache + c.pages[id] = p + // append in front + c.lru = append([]uint32{id}, c.lru...) + return p, nil +} + +func (c *Cache) pin(id uint32) { + c.pinned[id] = struct{}{} +} + +func (c *Cache) unpin(id uint32) { + delete(c.pinned, id) +} + +func (c *Cache) evict(id uint32) error { + if err := c.flush(id); err != nil { + return fmt.Errorf("flush: %w", err) + } + delete(c.pages, id) + return nil +} + +func (c *Cache) flush(id uint32) error { + if err := c.secondaryStorage.WritePage(id, c.pages[id]); err != nil { + return fmt.Errorf("write page: %w", err) + } + return nil +} + +func (c *Cache) freeMemoryIfNeeded() error { + if len(c.lru) < c.size { + return nil + } + for i := len(c.lru) - 1; i >= 0; i-- { + id := c.lru[i] + if _, ok := c.pinned[id]; ok { + // can't evict current page, pinned + continue + } + c.lru = c.lru[:len(c.lru)-1] + return c.evict(id) + } + return fmt.Errorf("all pages pinned, cache is full") +} + +func moveToFront(needle uint32, haystack []uint32) { + if len(haystack) == 0 || haystack[0] == needle { + return + } + var prev uint32 + for i, elem := range haystack { + switch { + case i == 0: + haystack[0] = needle + prev = elem + case elem == needle: + haystack[i] = prev + return + default: + haystack[i] = prev + prev = elem + } + } +} diff --git a/internal/engine/storage/page/v1/v1_cache_test.go b/internal/engine/storage/page/v1/v1_cache_test.go new file mode 100644 index 00000000..72168de3 --- /dev/null +++ b/internal/engine/storage/page/v1/v1_cache_test.go @@ -0,0 +1,71 @@ +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestCacheQueue(t *testing.T) { + assert := assert.New(t) + must := func(p *Page, err error) *Page { assert.NoError(err); return p } + pages := map[uint32]*Page{ + 0: must(load(make([]byte, PageSize))), + 1: must(load(make([]byte, PageSize))), + 2: must(load(make([]byte, PageSize))), + 3: must(load(make([]byte, PageSize))), + 4: must(load(make([]byte, PageSize))), + } + + secondaryStorage := new(MockSecondaryStorage) + defer secondaryStorage.AssertExpectations(t) + + c := newCache(secondaryStorage) + c.size = 2 + + // first page - unpin after use + secondaryStorage. + On("LoadPage", mock.IsType(uint32(0))). + Return(pages[0], nil). + Once() + p, err := c.FetchAndPin(0) + secondaryStorage.AssertCalled(t, "LoadPage", uint32(0)) + assert.NoError(err) + assert.Same(pages[0], p) + assert.Equal([]uint32{0}, c.lru) + c.Unpin(0) + + // second page - keep pinned + secondaryStorage. + On("LoadPage", mock.IsType(uint32(0))). + Return(pages[1], nil). + Once() + p, err = c.FetchAndPin(1) + secondaryStorage.AssertCalled(t, "LoadPage", uint32(1)) + assert.NoError(err) + assert.Same(pages[1], p) + assert.Equal([]uint32{1, 0}, c.lru) + + // third page - pages[0] must be evicted + secondaryStorage. + On("LoadPage", mock.IsType(uint32(0))). + Return(pages[2], nil). + Once() + secondaryStorage. + On("WritePage", mock.IsType(uint32(0)), mock.IsType(&Page{})). + Return(nil). + Once() + p, err = c.FetchAndPin(2) + secondaryStorage.AssertCalled(t, "LoadPage", uint32(2)) + secondaryStorage.AssertCalled(t, "WritePage", uint32(0), pages[0]) + assert.NoError(err) + assert.Same(pages[2], p) + assert.Equal([]uint32{2, 1}, c.lru) + + // fourth page - can't fetch because cache is full + p, err = c.FetchAndPin(3) + assert.Error(err) + assert.Nil(p) + assert.Equal([]uint32{2, 1}, c.lru) +} diff --git a/internal/engine/storage/page/v1/v1_pageloader.go b/internal/engine/storage/page/v1/v1_pageloader.go new file mode 100644 index 00000000..ba834dee --- /dev/null +++ b/internal/engine/storage/page/v1/v1_pageloader.go @@ -0,0 +1,11 @@ +package v1 + +//go:generate mockery -inpkg -testonly -case=snake -name=SecondaryStorage + +// SecondaryStorage descries a secondary storage component, such as a disk. It +// has to manage pages by ID and must ensure that pages are read and written +// from the underlying storage without any caching. +type SecondaryStorage interface { + LoadPage(uint32) (*Page, error) + WritePage(uint32, *Page) error +} From 3dbee4e9d68cd2bdc387d0fd1c29e17258871eba Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 16 Jun 2020 16:30:25 +0200 Subject: [PATCH 15/77] Remove current page implementation --- internal/engine/storage/page/mocks/doc.go | 2 - internal/engine/storage/page/mocks/page.go | 118 ------- internal/engine/storage/page/page_test.go | 85 ----- internal/engine/storage/page/v1/codec.go | 67 ---- internal/engine/storage/page/v1/codec_test.go | 138 -------- .../page/v1/mock_secondary_storage_test.go | 47 --- internal/engine/storage/page/v1/v1_cache.go | 162 ---------- .../engine/storage/page/v1/v1_cache_test.go | 71 ----- internal/engine/storage/page/v1/v1_offset.go | 17 - internal/engine/storage/page/v1/v1_page.go | 300 ------------------ .../storage/page/v1/v1_page_bench_test.go | 68 ---- .../engine/storage/page/v1/v1_page_test.go | 213 ------------- .../engine/storage/page/v1/v1_pageloader.go | 11 - 13 files changed, 1299 deletions(-) delete mode 100644 internal/engine/storage/page/mocks/doc.go delete mode 100644 internal/engine/storage/page/mocks/page.go delete mode 100644 internal/engine/storage/page/page_test.go delete mode 100644 internal/engine/storage/page/v1/codec.go delete mode 100644 internal/engine/storage/page/v1/codec_test.go delete mode 100644 internal/engine/storage/page/v1/mock_secondary_storage_test.go delete mode 100644 internal/engine/storage/page/v1/v1_cache.go delete mode 100644 internal/engine/storage/page/v1/v1_cache_test.go delete mode 100644 internal/engine/storage/page/v1/v1_offset.go delete mode 100644 internal/engine/storage/page/v1/v1_page.go delete mode 100644 internal/engine/storage/page/v1/v1_page_bench_test.go delete mode 100644 internal/engine/storage/page/v1/v1_page_test.go delete mode 100644 internal/engine/storage/page/v1/v1_pageloader.go diff --git a/internal/engine/storage/page/mocks/doc.go b/internal/engine/storage/page/mocks/doc.go deleted file mode 100644 index 35abed3d..00000000 --- a/internal/engine/storage/page/mocks/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package mocks provides generated mock implementations for easy testing. -package mocks diff --git a/internal/engine/storage/page/mocks/page.go b/internal/engine/storage/page/mocks/page.go deleted file mode 100644 index 3e26117b..00000000 --- a/internal/engine/storage/page/mocks/page.go +++ /dev/null @@ -1,118 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package mocks - -import ( - mock "github.com/stretchr/testify/mock" - page "github.com/tomarrell/lbadd/internal/engine/storage/page" -) - -// Page is an autogenerated mock type for the Page type -type Page struct { - mock.Mock -} - -// Cell provides a mock function with given fields: _a0 -func (_m *Page) Cell(_a0 []byte) (page.Cell, bool) { - ret := _m.Called(_a0) - - var r0 page.Cell - if rf, ok := ret.Get(0).(func([]byte) page.Cell); ok { - r0 = rf(_a0) - } else { - r0 = ret.Get(0).(page.Cell) - } - - var r1 bool - if rf, ok := ret.Get(1).(func([]byte) bool); ok { - r1 = rf(_a0) - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 -} - -// Cells provides a mock function with given fields: -func (_m *Page) Cells() []page.Cell { - ret := _m.Called() - - var r0 []page.Cell - if rf, ok := ret.Get(0).(func() []page.Cell); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]page.Cell) - } - } - - return r0 -} - -// ClearDirty provides a mock function with given fields: -func (_m *Page) ClearDirty() { - _m.Called() -} - -// Delete provides a mock function with given fields: _a0 -func (_m *Page) Delete(_a0 []byte) error { - ret := _m.Called(_a0) - - var r0 error - if rf, ok := ret.Get(0).(func([]byte) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Dirty provides a mock function with given fields: -func (_m *Page) Dirty() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Header provides a mock function with given fields: _a0 -func (_m *Page) Header(_a0 page.Header) interface{} { - ret := _m.Called(_a0) - - var r0 interface{} - if rf, ok := ret.Get(0).(func(page.Header) interface{}); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(interface{}) - } - } - - return r0 -} - -// MarkDirty provides a mock function with given fields: -func (_m *Page) MarkDirty() { - _m.Called() -} - -// StoreCell provides a mock function with given fields: _a0 -func (_m *Page) StoreCell(_a0 page.Cell) error { - ret := _m.Called(_a0) - - var r0 error - if rf, ok := ret.Get(0).(func(page.Cell) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/internal/engine/storage/page/page_test.go b/internal/engine/storage/page/page_test.go deleted file mode 100644 index 652dd6a7..00000000 --- a/internal/engine/storage/page/page_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package page_test - -import ( - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tomarrell/lbadd/internal/engine/storage/page" - v1 "github.com/tomarrell/lbadd/internal/engine/storage/page/v1" -) - -var ( - Loaders = []struct { - Name string - Loader page.Loader - PageSize int - }{ - {"v1", v1.Load, 1 << 14}, - } - - Cells = []page.Cell{ - { - Key: []byte("key[0]"), - Record: []byte("data[0]"), - }, - { - Key: []byte("key[1]"), - Record: []byte("data[1]"), - }, - { - Key: []byte("key[2]"), - Record: []byte("data[2]"), - }, - { - Key: []byte("key[3]"), - Record: []byte("data[3]"), - }, - } -) - -func TestImplementations(t *testing.T) { - for _, loader := range Loaders { - t.Run("loader="+loader.Name, func(t *testing.T) { _TestPageOperations(t, loader.Loader, loader.PageSize) }) - } -} - -func _TestPageOperations(t *testing.T, loader page.Loader, pageSize int) { - assert := assert.New(t) - rand := rand.New(rand.NewSource(87234562678)) // reproducible random - cells := make([]page.Cell, len(Cells)) - copy(cells, Cells) - rand.Shuffle(len(cells), func(i, j int) { cells[i], cells[j] = cells[j], cells[i] }) - - data := make([]byte, pageSize) - p, err := loader(data) - assert.NoError(err) - assert.NotNil(p) - - for _, cell := range cells { - err = p.StoreCell(cell) - assert.NoError(err) - } - - // after insertion, all cells must be returned sorted, as the original Cells - // slice - assert.Equal(Cells, p.Cells()) - for _, cell := range cells { - obtained, ok := p.Cell(cell.Key) - assert.True(ok) - assert.Equal(cell, obtained) - } - - // delete one cell - deleteIndex := 2 - err = p.Delete(Cells[deleteIndex].Key) - assert.NoError(err) - - afterDeletionCells := make([]page.Cell, len(Cells)) - copy(afterDeletionCells, Cells) - afterDeletionCells = append(afterDeletionCells[:deleteIndex], afterDeletionCells[deleteIndex+1:]...) - assert.Equal(afterDeletionCells, p.Cells()) - obtained, ok := p.Cell(Cells[deleteIndex].Key) - assert.False(ok, "deleted cell must not be obtainable anymore") - assert.Zero(obtained) -} diff --git a/internal/engine/storage/page/v1/codec.go b/internal/engine/storage/page/v1/codec.go deleted file mode 100644 index 6dbe6cf4..00000000 --- a/internal/engine/storage/page/v1/codec.go +++ /dev/null @@ -1,67 +0,0 @@ -// This file contains encoding and decoding functions. - -package v1 - -import ( - "github.com/tomarrell/lbadd/internal/engine/converter" - "github.com/tomarrell/lbadd/internal/engine/storage/page" -) - -// encodeCell frames the cell key and record, and concatenates them. The -// concatenation is NOT framed itself. -// -// encoded = | frame | key | frame | record | -func encodeCell(cell page.Cell) []byte { - keyFrame := frameData(cell.Key) - recordFrame := frameData(cell.Record) - return append(keyFrame, recordFrame...) -} - -// decodeCell takes data of the format that encodeCell produced, and convert it -// back into a (page.Cell). -func decodeCell(data []byte) page.Cell { - keyDataSize := converter.ByteSliceToUint16(data[:FrameSizeSize]) - keyLo := FrameSizeSize - keyHi := FrameSizeSize + keyDataSize - - dataLo := keyHi + FrameSizeSize - dataSize := converter.ByteSliceToUint16(data[keyHi:dataLo]) - dataHi := dataLo + dataSize - - return page.Cell{ - Key: data[keyLo:keyHi], - Record: data[dataLo:dataHi], - } -} - -// frameData frames the given data and returns the framed bytes. The frame -// contains the length of the data, as uint16. -// -// framed = | length | data | -func frameData(data []byte) []byte { - return append(converter.Uint16ToByteSlice(uint16(len(data))), data...) -} - -// encodeOffset converts the offset to a byte slice of the following form. -// -// encoded = | location | size | -// -// Both the location and size will be encoded as 2 byte uint16. -func encodeOffset(offset Offset) []byte { - return append(converter.Uint16ToByteSlice(offset.Location), converter.Uint16ToByteSlice(offset.Size)...) -} - -// decodeOffset takes bytes of the form produced by encodeOffset and converts it -// back into an Offset. -func decodeOffset(data []byte) Offset { - return Offset{ - Location: converter.ByteSliceToUint16(data[:int(OffsetSize)/2]), - Size: converter.ByteSliceToUint16(data[int(OffsetSize)/2 : int(OffsetSize)]), - } -} - -func zero(b []byte) { - for i := range b { - b[i] = 0x00 - } -} diff --git a/internal/engine/storage/page/v1/codec_test.go b/internal/engine/storage/page/v1/codec_test.go deleted file mode 100644 index b6819ec1..00000000 --- a/internal/engine/storage/page/v1/codec_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package v1 - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tomarrell/lbadd/internal/engine/storage/page" -) - -func Test_encodeCell(t *testing.T) { - tests := []struct { - name string - cell page.Cell - want []byte - }{ - { - "empty cell", - page.Cell{ - Key: []byte{}, - Record: []byte{}, - }, - []byte{0x00, 0x00, 0x00, 0x00}, - }, - { - "cell", - page.Cell{ - Key: []byte{0x01, 0x02}, - Record: []byte{0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, - }, - []byte{ - 0x00, 0x02, // key frame - 0x01, 0x02, // key - 0x00, 0x06, // record frame - 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // record - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert := assert.New(t) - assert.Equal(tt.want, encodeCell(tt.cell)) - }) - } -} - -func Test_decodeCell(t *testing.T) { - tests := []struct { - name string - data []byte - want page.Cell - }{ - { - "empty cell", - []byte{0x00, 0x00, 0x00, 0x00}, - page.Cell{ - Key: []byte{}, - Record: []byte{}, - }, - }, - { - "cell", - []byte{ - 0x00, 0x02, // key frame - 0x01, 0x02, // key - 0x00, 0x06, // record frame - 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, // record - }, - page.Cell{ - Key: []byte{0x01, 0x02}, - Record: []byte{0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert := assert.New(t) - assert.Equal(tt.want, decodeCell(tt.data)) - }) - } -} - -func Test_encodeOffset(t *testing.T) { - tests := []struct { - name string - offset Offset - want []byte - }{ - { - "empty offset", - Offset{}, - []byte{0x00, 0x00, 0x00, 0x00}, - }, - { - "offset", - Offset{ - Location: 0xCAFE, - Size: 0xBABE, - }, - []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert := assert.New(t) - - assert.Equal(tt.want, encodeOffset(tt.offset)) - }) - } -} - -func Test_decodeOffset(t *testing.T) { - tests := []struct { - name string - data []byte - want Offset - }{ - { - "empty offset", - []byte{0x00, 0x00, 0x00, 0x00}, - Offset{}, - }, - { - "offset", - []byte{0xCA, 0xFE, 0xBA, 0xBE}, - Offset{ - Location: 0xCAFE, - Size: 0xBABE, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert := assert.New(t) - - assert.Equal(tt.want, decodeOffset(tt.data)) - }) - } -} diff --git a/internal/engine/storage/page/v1/mock_secondary_storage_test.go b/internal/engine/storage/page/v1/mock_secondary_storage_test.go deleted file mode 100644 index f109a40b..00000000 --- a/internal/engine/storage/page/v1/mock_secondary_storage_test.go +++ /dev/null @@ -1,47 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package v1 - -import mock "github.com/stretchr/testify/mock" - -// MockSecondaryStorage is an autogenerated mock type for the SecondaryStorage type -type MockSecondaryStorage struct { - mock.Mock -} - -// LoadPage provides a mock function with given fields: _a0 -func (_m *MockSecondaryStorage) LoadPage(_a0 uint32) (*Page, error) { - ret := _m.Called(_a0) - - var r0 *Page - if rf, ok := ret.Get(0).(func(uint32) *Page); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*Page) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(uint32) error); ok { - r1 = rf(_a0) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// WritePage provides a mock function with given fields: _a0, _a1 -func (_m *MockSecondaryStorage) WritePage(_a0 uint32, _a1 *Page) error { - ret := _m.Called(_a0, _a1) - - var r0 error - if rf, ok := ret.Get(0).(func(uint32, *Page) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/internal/engine/storage/page/v1/v1_cache.go b/internal/engine/storage/page/v1/v1_cache.go deleted file mode 100644 index 4a429cf4..00000000 --- a/internal/engine/storage/page/v1/v1_cache.go +++ /dev/null @@ -1,162 +0,0 @@ -package v1 - -import ( - "fmt" - - "github.com/tomarrell/lbadd/internal/engine/storage/page" -) - -const ( - // CacheSize is the default amount of pages that a cache can hold. - CacheSize int = 1 << 8 -) - -// Cache is a simple LRU implementation of a page cache. Lookups in the cache -// are performed with amortized O(1) complexity, however, moving elements in the -// LRU list is more expensive. -type Cache struct { - secondaryStorage SecondaryStorage - - pages map[uint32]*Page - pinned map[uint32]struct{} - - size int - lru []uint32 -} - -// NewCache creates a new cache with the given secondary storage. -func NewCache(secondaryStorage SecondaryStorage) page.Cache { - return newCache(secondaryStorage) -} - -// FetchAndPin fetches the page with the given ID from the cache or the -// secondary storage and pins it in the cache. Pinning prevents the page from -// being evicted. -func (c *Cache) FetchAndPin(id uint32) (page.Page, error) { - return c.fetchAndPin(id) -} - -// Unpin marks the given ID as ok to be evicted. This is a no-op if the ID -// doesn't exist or is not pinned. -func (c *Cache) Unpin(id uint32) { - c.unpin(id) -} - -// Flush writes the contents of the page with the given ID to the secondary -// storage. -func (c *Cache) Flush(id uint32) error { - return c.flush(id) -} - -// Close does nothing. -func (c *Cache) Close() error { - return nil -} - -func newCache(secondaryStorage SecondaryStorage) *Cache { - return &Cache{ - secondaryStorage: secondaryStorage, - - pages: make(map[uint32]*Page), - pinned: make(map[uint32]struct{}), - - size: CacheSize, - lru: make([]uint32, 0, CacheSize), - } -} - -func (c *Cache) fetchAndPin(id uint32) (*Page, error) { - // pin id first in order to avoid potential concurrent eviction at this - // point - c.pin(id) - p, err := c.fetch(id) - if err != nil { - // unpin if a page with the given id cannot be loaded - c.unpin(id) - return nil, err - } - return p, nil -} - -func (c *Cache) fetch(id uint32) (*Page, error) { - // check if page is already cached - if p, ok := c.pages[id]; ok { - moveToFront(id, c.lru) - return p, nil - } - - // check if we have to evict pages - if err := c.freeMemoryIfNeeded(); err != nil { - return nil, fmt.Errorf("free mem: %w", err) - } - - // fetch page from secondary storage - p, err := c.secondaryStorage.LoadPage(id) - if err != nil { - return nil, fmt.Errorf("load page: %w", err) - } - // store in cache - c.pages[id] = p - // append in front - c.lru = append([]uint32{id}, c.lru...) - return p, nil -} - -func (c *Cache) pin(id uint32) { - c.pinned[id] = struct{}{} -} - -func (c *Cache) unpin(id uint32) { - delete(c.pinned, id) -} - -func (c *Cache) evict(id uint32) error { - if err := c.flush(id); err != nil { - return fmt.Errorf("flush: %w", err) - } - delete(c.pages, id) - return nil -} - -func (c *Cache) flush(id uint32) error { - if err := c.secondaryStorage.WritePage(id, c.pages[id]); err != nil { - return fmt.Errorf("write page: %w", err) - } - return nil -} - -func (c *Cache) freeMemoryIfNeeded() error { - if len(c.lru) < c.size { - return nil - } - for i := len(c.lru) - 1; i >= 0; i-- { - id := c.lru[i] - if _, ok := c.pinned[id]; ok { - // can't evict current page, pinned - continue - } - c.lru = c.lru[:len(c.lru)-1] - return c.evict(id) - } - return fmt.Errorf("all pages pinned, cache is full") -} - -func moveToFront(needle uint32, haystack []uint32) { - if len(haystack) == 0 || haystack[0] == needle { - return - } - var prev uint32 - for i, elem := range haystack { - switch { - case i == 0: - haystack[0] = needle - prev = elem - case elem == needle: - haystack[i] = prev - return - default: - haystack[i] = prev - prev = elem - } - } -} diff --git a/internal/engine/storage/page/v1/v1_cache_test.go b/internal/engine/storage/page/v1/v1_cache_test.go deleted file mode 100644 index 72168de3..00000000 --- a/internal/engine/storage/page/v1/v1_cache_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package v1 - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func TestCacheQueue(t *testing.T) { - assert := assert.New(t) - must := func(p *Page, err error) *Page { assert.NoError(err); return p } - pages := map[uint32]*Page{ - 0: must(load(make([]byte, PageSize))), - 1: must(load(make([]byte, PageSize))), - 2: must(load(make([]byte, PageSize))), - 3: must(load(make([]byte, PageSize))), - 4: must(load(make([]byte, PageSize))), - } - - secondaryStorage := new(MockSecondaryStorage) - defer secondaryStorage.AssertExpectations(t) - - c := newCache(secondaryStorage) - c.size = 2 - - // first page - unpin after use - secondaryStorage. - On("LoadPage", mock.IsType(uint32(0))). - Return(pages[0], nil). - Once() - p, err := c.FetchAndPin(0) - secondaryStorage.AssertCalled(t, "LoadPage", uint32(0)) - assert.NoError(err) - assert.Same(pages[0], p) - assert.Equal([]uint32{0}, c.lru) - c.Unpin(0) - - // second page - keep pinned - secondaryStorage. - On("LoadPage", mock.IsType(uint32(0))). - Return(pages[1], nil). - Once() - p, err = c.FetchAndPin(1) - secondaryStorage.AssertCalled(t, "LoadPage", uint32(1)) - assert.NoError(err) - assert.Same(pages[1], p) - assert.Equal([]uint32{1, 0}, c.lru) - - // third page - pages[0] must be evicted - secondaryStorage. - On("LoadPage", mock.IsType(uint32(0))). - Return(pages[2], nil). - Once() - secondaryStorage. - On("WritePage", mock.IsType(uint32(0)), mock.IsType(&Page{})). - Return(nil). - Once() - p, err = c.FetchAndPin(2) - secondaryStorage.AssertCalled(t, "LoadPage", uint32(2)) - secondaryStorage.AssertCalled(t, "WritePage", uint32(0), pages[0]) - assert.NoError(err) - assert.Same(pages[2], p) - assert.Equal([]uint32{2, 1}, c.lru) - - // fourth page - can't fetch because cache is full - p, err = c.FetchAndPin(3) - assert.Error(err) - assert.Nil(p) - assert.Equal([]uint32{2, 1}, c.lru) -} diff --git a/internal/engine/storage/page/v1/v1_offset.go b/internal/engine/storage/page/v1/v1_offset.go deleted file mode 100644 index 972d4f41..00000000 --- a/internal/engine/storage/page/v1/v1_offset.go +++ /dev/null @@ -1,17 +0,0 @@ -package v1 - -import "unsafe" - -const ( - // OffsetSize is the constant size of an encoded Offset using encodeOffset. - OffsetSize = uint16(unsafe.Sizeof(Offset{})) // #nosec -) - -// Offset describes a memory segment relative to the page start (=0, =before the -// header). -type Offset struct { - // Location is the target location of this offset, relative to the page. - Location uint16 - // Size is the size of the memory segment that is located at Location. - Size uint16 -} diff --git a/internal/engine/storage/page/v1/v1_page.go b/internal/engine/storage/page/v1/v1_page.go deleted file mode 100644 index 4d935e78..00000000 --- a/internal/engine/storage/page/v1/v1_page.go +++ /dev/null @@ -1,300 +0,0 @@ -package v1 - -import ( - "bytes" - "sort" - "unsafe" - - "github.com/tomarrell/lbadd/internal/engine/converter" - "github.com/tomarrell/lbadd/internal/engine/storage/page" -) - -// Internal headers, defined from 255 downwards, as opposed to other headers, -// which are defined from 0 upwards. -const ( - InternalHeaderCellCount page.Header = page.Header(^uint8(0)) - iota - InternalHeaderDirty -) - -// Constants. -const ( - // PageSize is the fixed size of one page. - PageSize uint16 = 1 << 14 - // HeaderSize is the fixed size of the header area in a page. - HeaderSize uint16 = 1 << 9 - // HeaderOffset is the offset in the page's data, at which the header - // starts. - HeaderOffset uint16 = 0 - // ContentOffset is the offset in the page's data, at which the header ends - // and the content starts. This is also, where the offsets are stored. - ContentOffset uint16 = HeaderOffset + HeaderSize - // ContentSize is the size of the content area, equivalent to the page size - // minus the header size. - ContentSize uint16 = PageSize - HeaderSize - - // FrameSizeSize is the byte size of a frame. - FrameSizeSize = uint16(unsafe.Sizeof(uint16(0))) // #nosec - - HeaderVersionOffset = HeaderOffset - HeaderVersionSize = uint16(unsafe.Sizeof(uint16(0))) // #nosec - HeaderVersionLo = HeaderVersionOffset - HeaderVersionHi = HeaderVersionOffset + HeaderVersionSize - - HeaderIDOffset = HeaderVersionOffset + HeaderVersionSize - HeaderIDSize = uint16(unsafe.Sizeof(uint32(0))) // #nosec - HeaderIDLo = HeaderIDOffset - HeaderIDHi = HeaderIDOffset + HeaderIDSize - - InternalHeaderCellCountOffset = HeaderIDOffset + HeaderIDSize - InternalHeaderCellCountSize = uint16(unsafe.Sizeof(uint16(0))) // #nosec - InternalHeaderCellCountLo = InternalHeaderCellCountOffset - InternalHeaderCellCountHi = InternalHeaderCellCountOffset + InternalHeaderCellCountSize - - InternalHeaderDirtyOffset = InternalHeaderCellCountOffset + InternalHeaderCellCountSize - InternalHeaderDirtySize = uint16(unsafe.Sizeof(false)) // #nosec - InternalHeaderDirtyIndex = InternalHeaderDirtyOffset -) - -var _ page.Loader = Load -var _ page.Page = (*Page)(nil) - -// Page is an implementation of (page.Page). It implements the concept of -// slotted pages, also it holds all data in memory at all times. The byte slice -// that is passed in when loading the page, is the one that the implementation -// will operate on. It will always stick to that slice, and never replace it -// with another. All changes to the page are immediately reflected in that byte -// slice. This implementation is NOT safe for concurrent use. This -// implementation does not support extension pages. -type Page struct { - data []byte - - header map[page.Header]interface{} -} - -// Load loads the given bytes as a page. The page will operate on the given byte -// slice, and never copy or exchange it. Modifying the byte slice after loading -// a page from it will likely corrupt the page. External changes to the byte -// slice are not guaranteed to be immediately reflected in the page object. -func Load(data []byte) (page.Page, error) { - return load(data) -} - -// Header obtains the header field value of the given key. If the key is not -// supported by this implementation, it will return nil indicating an unknown -// header. -func (p *Page) Header(key page.Header) interface{} { - val, ok := p.header[key] - if !ok { - return nil - } - return val -} - -// Dirty returns the value of the header flag dirty, meaning that the page has -// been modified since ClearDirty() has been called the last time. -func (p *Page) Dirty() bool { - return p.header[InternalHeaderDirty].(bool) -} - -// MarkDirty marks this page as dirty. Call this when you have modified the -// page. This will -func (p *Page) MarkDirty() { - p.data[InternalHeaderDirtyIndex] = converter.BoolToByte(true) - p.header[InternalHeaderDirty] = true -} - -// ClearDirty marks this page as NOT dirty (anymore). Call this when the page -// has been written to persistent storage. -func (p *Page) ClearDirty() { - p.data[InternalHeaderDirtyIndex] = converter.BoolToByte(false) - p.header[InternalHeaderDirty] = false -} - -// StoreCell will store the given cell in this page. If there is not enough -// space, NO extension page will be allocated, but an error will be returned, -// indicating insufficient space. -func (p *Page) StoreCell(cell page.Cell) error { - cellData := encodeCell(cell) - insertionOffset, ok := p.findInsertionOffset(uint16(len(cellData))) - if !ok { - return page.ErrPageTooSmall - } - offsetInsertionOffset := p.findOffsetInsertionOffset(cell) - p.insertOffset(insertionOffset, offsetInsertionOffset) - copy(p.data[ContentOffset+insertionOffset.Location:], cellData) - - p.incrementCellCount() - - return nil -} - -// Delete deletes the cell with the given key. If there is no such cell, this is -// a no-op. This never returns an error. -func (p *Page) Delete(key []byte) error { - offsets := p.Offsets() - - for i, offset := range offsets { - if bytes.Equal(key, p.getCell(offset).Key) { - offsetData := p.data[ContentOffset : ContentOffset+(p.cellCount()*OffsetSize)] - zero(offsetData) - offsets = append(offsets[:i], offsets[i+1:]...) - for j, off := range offsets { - lo := uint16(j) * OffsetSize - hi := lo + OffsetSize - copy(offsetData[lo:hi], encodeOffset(off)) - } - break - } - } - - p.decrementCellCount() - - return nil -} - -// Cell searches for a cell with the given key, and returns a cell object -// representing all the found cell data. If no cell was found, false is -// returned. -func (p *Page) Cell(key []byte) (page.Cell, bool) { - // TODO: binary search - for _, cell := range p.Cells() { - if bytes.Equal(key, cell.Key) { - return cell, true - } - } - return page.Cell{}, false -} - -// Cells returns all cells that are stored in this page in sorted fashion, -// ordered ascending by key. -func (p *Page) Cells() (cells []page.Cell) { - for _, offset := range p.Offsets() { - cells = append(cells, p.getCell(offset)) - } - return -} - -// Offsets returns all offsets of this page sorted by key of the cell that they -// point to. Following the offsets in the order that they are returned, will -// result in a list of cells, that are sorted ascending by key. -func (p *Page) Offsets() (result []Offset) { - cellCount := p.cellCount() - offsetData := p.data[ContentOffset : ContentOffset+OffsetSize*cellCount] - for i := 0; i < len(offsetData); i += int(OffsetSize) { - result = append(result, decodeOffset(offsetData[i:i+int(OffsetSize)])) - } - return -} - -func load(data []byte) (*Page, error) { - if len(data) != int(PageSize) { - return nil, page.ErrInvalidPageSize - } - p := &Page{ - data: data, - header: make(map[page.Header]interface{}), - } - p.loadHeader() - return p, nil -} - -// insertOffset inserts the given offset at the other given offset. A bit -// confusing, because we need to store an offset, and the location is given by -// another offset. -// -// This method takes care of inserting the offset, and moving other offsets to -// the right, instead of overwriting them. -func (p *Page) insertOffset(offset, at Offset) { - cellCount := p.cellCount() - offsetData := p.data[ContentOffset : ContentOffset+OffsetSize*(cellCount+1)] - encOffset := encodeOffset(offset) - buf := make([]byte, uint16(len(offsetData))-OffsetSize-at.Location) - copy(buf, offsetData[at.Location:]) - copy(offsetData[at.Location+OffsetSize:], buf) - copy(offsetData[at.Location:], encOffset) -} - -// getCell returns the cell that an offset points to. -func (p *Page) getCell(offset Offset) page.Cell { - return decodeCell(p.data[ContentOffset+offset.Location : ContentOffset+offset.Location+offset.Size]) -} - -// findInsertionOffset finds an offset for a data segment of the given size, or -// returns false if no space is available. The space is found by using a -// first-fit approach. -func (p *Page) findInsertionOffset(size uint16) (Offset, bool) { - offsets := p.Offsets() - sort.Slice(offsets, func(i int, j int) bool { - return offsets[i].Location < offsets[j].Location - }) - - // TODO: best fit, this is currently first fit - rightBound := ContentSize - for i := len(offsets) - 1; i >= 0; i-- { - current := offsets[i] - if current.Location+current.Size >= rightBound-size { - // doesn't fit - rightBound = current.Location - } else { - break - } - } - - return Offset{ - Location: rightBound - size, - Size: size, - }, true -} - -// findOffsetInsertionOffset creates an offset to a location, where a new offset -// can be inserted. The new offset that should be inserted, is not part of this -// method. However, we need the cell that the new offset points to, in order to -// compare its key with other cells. This is, because we insert offsets in a -// way, so that all offsets from left to right point to cells, that are ordered -// ascending by key when following all offsets. -func (p *Page) findOffsetInsertionOffset(cell page.Cell) Offset { - // TODO: binary search - offsets := p.Offsets() - for i, offset := range offsets { - if bytes.Compare(p.getCell(offset).Key, cell.Key) > 0 { - return Offset{ - Location: uint16(i) * OffsetSize, - Size: OffsetSize, - } - } - } - return Offset{ - Location: uint16(len(offsets)) * OffsetSize, - Size: OffsetSize, - } -} - -// loadHeader (re-)loads all header values from the page's data. -func (p *Page) loadHeader() { - p.header[page.HeaderVersion] = converter.ByteSliceToUint16(p.data[HeaderVersionLo:HeaderVersionHi]) - p.header[page.HeaderID] = converter.ByteSliceToUint16(p.data[HeaderIDLo:HeaderIDHi]) - p.header[InternalHeaderCellCount] = converter.ByteSliceToUint16(p.data[InternalHeaderCellCountLo:InternalHeaderCellCountHi]) -} - -func (p *Page) setInternalHeaderCellCount(count uint16) { - // set in memory cache - p.header[InternalHeaderCellCount] = count - // set in data - copy(p.data[InternalHeaderCellCountLo:InternalHeaderCellCountHi], converter.Uint16ToByteSlice(count)) -} - -// incrementCellCount increments the header field InternalHeaderCellCount by one. -func (p *Page) incrementCellCount() { - p.setInternalHeaderCellCount(p.header[InternalHeaderCellCount].(uint16) + 1) -} - -// decrementCellCount decrements the header field InternalHeaderCellCount by one. -func (p *Page) decrementCellCount() { - p.setInternalHeaderCellCount(p.header[InternalHeaderCellCount].(uint16) - 1) -} - -// cellCount returns the amount of currently stored cells. This value is -// retrieved from the header field InternalHeaderCellCount. -func (p *Page) cellCount() uint16 { - return p.header[InternalHeaderCellCount].(uint16) -} diff --git a/internal/engine/storage/page/v1/v1_page_bench_test.go b/internal/engine/storage/page/v1/v1_page_bench_test.go deleted file mode 100644 index 1c418e9e..00000000 --- a/internal/engine/storage/page/v1/v1_page_bench_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package v1 - -import ( - "testing" - - "github.com/tomarrell/lbadd/internal/engine/storage/page" -) - -var result interface{} - -func Benchmark_Page_StoreCell(b *testing.B) { - _p, _ := load(make([]byte, PageSize)) - _ = _p.StoreCell(page.Cell{ - Key: []byte{0xAA}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }) - _ = _p.StoreCell(page.Cell{ - Key: []byte{0xFF}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }) - _ = _p.StoreCell(page.Cell{ - Key: []byte{0x11}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }) - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - b.StopTimer() - data := make([]byte, PageSize) - copy(data, _p.data) - p, _ := load(data) - b.StartTimer() - - _ = p.StoreCell(page.Cell{ - Key: []byte{0xDD}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }) - } -} - -func Benchmark_Page_Load(b *testing.B) { - _p, _ := load(make([]byte, PageSize)) - _ = _p.StoreCell(page.Cell{ - Key: []byte{0xAA}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }) - _ = _p.StoreCell(page.Cell{ - Key: []byte{0xFF}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }) - _ = _p.StoreCell(page.Cell{ - Key: []byte{0x11}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }) - - var r page.Page - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - p, _ := Load(_p.data) - - r = p - } - - result = r -} diff --git a/internal/engine/storage/page/v1/v1_page_test.go b/internal/engine/storage/page/v1/v1_page_test.go deleted file mode 100644 index 247d3d33..00000000 --- a/internal/engine/storage/page/v1/v1_page_test.go +++ /dev/null @@ -1,213 +0,0 @@ -package v1 - -import ( - "bytes" - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tomarrell/lbadd/internal/engine/converter" - "github.com/tomarrell/lbadd/internal/engine/storage/page" -) - -func Test_Page_StoreCell(t *testing.T) { - t.Run("single cell", func(t *testing.T) { - assert := assert.New(t) - - p, err := load(make([]byte, PageSize)) // empty page: version=0,id=0,cellcount=0 - assert.NoError(err) - assert.NotNil(p) - - assert.NoError( - p.StoreCell(page.Cell{ - Key: []byte{0xCA}, - Record: []byte{0xFE, 0xBA, 0xBE}, - }), - ) - - assert.Equal([]byte{ - 0x00, 0x01, // key frame - 0xCA, // key - 0x00, 0x03, // record frame - 0xFE, 0xBA, 0xBE, // record - }, p.data[PageSize-8:]) - - assert.EqualValues(1, p.header[InternalHeaderCellCount]) - assert.Equal([]byte{ - 0x3D, 0xF8, // location of our cell - 0x00, 0x08, // size of our cell - }, p.data[ContentOffset:ContentOffset+4]) - - allCells := p.Cells() - assert.Len(allCells, 1) - }) - t.Run("multiple cells", func(t *testing.T) { - assert := assert.New(t) - - p, err := load(make([]byte, PageSize)) // empty page: version=0,id=0,cellcount=0 - assert.NoError(err) - assert.NotNil(p) - - // first cell - - assert.NoError( - p.StoreCell(page.Cell{ - Key: []byte{0xAA}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }), - ) - - assert.Equal([]byte{ - 0x00, 0x01, // key frame - 0xAA, // key - 0x00, 0x04, // record frame - 0xCA, 0xFE, 0xBA, 0xBE, // record - }, p.data[PageSize-9:]) - - // second cell - - assert.NoError( - p.StoreCell(page.Cell{ - Key: []byte{0xFF}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }), - ) - - assert.Equal([]byte{ - 0x00, 0x01, // key frame - 0xFF, // key - 0x00, 0x04, // record frame - 0xCA, 0xFE, 0xBA, 0xBE, // record - }, p.data[PageSize-18:PageSize-9]) - - // third cell - - assert.NoError( - p.StoreCell(page.Cell{ - Key: []byte{0x11}, - Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, - }), - ) - - assert.Equal([]byte{ - 0x00, 0x01, // key frame - 0x11, // key - 0x00, 0x04, // record frame - 0xCA, 0xFE, 0xBA, 0xBE, // record - }, p.data[PageSize-27:PageSize-18]) - - // check that all the offsets at the beginning of the content area are - // sorted by the key of the cell they point to - - x11Offset := converter.Uint16ToByteArray(ContentSize - 27) - xAAOffset := converter.Uint16ToByteArray(ContentSize - 9) - xFFOffset := converter.Uint16ToByteArray(ContentSize - 18) - - assert.Equal([]byte{ - // first offset - x11Offset[0], x11Offset[1], // location - 0x00, 0x09, - // second offset - xAAOffset[0], xAAOffset[1], // location - 0x00, 0x09, - // third offset - xFFOffset[0], xFFOffset[1], // location - 0x00, 0x09, - }, p.data[ContentOffset:ContentOffset+OffsetSize*3]) - - allCells := p.Cells() - assert.Len(allCells, 3) - assert.True(sort.SliceIsSorted(allCells, func(i, j int) bool { - return bytes.Compare(allCells[i].Key, allCells[j].Key) < 0 - }), "p.Cells() must return all cells ordered by key") - }) -} - -func TestInvalidPageSize(t *testing.T) { - tf := func(t *testing.T, data []byte) { - assert := assert.New(t) - p, err := Load(data) - assert.Equal(page.ErrInvalidPageSize, err) - assert.Nil(p) - } - t.Run("invalid=nil", func(t *testing.T) { - tf(t, nil) - }) - t.Run("invalid=smaller", func(t *testing.T) { - data := make([]byte, PageSize/2) - tf(t, data) - }) - t.Run("invalid=larger", func(t *testing.T) { - data := make([]byte, int(PageSize)*2) - tf(t, data) - }) -} - -func TestHeaderVersion(t *testing.T) { - assert := assert.New(t) - - versionBytes := []byte{0xCA, 0xFE} - - data := make([]byte, PageSize) - copy(data[:2], versionBytes) - - p, err := load(data) - assert.NoError(err) - - version := p.Header(page.HeaderVersion) - assert.IsType(uint16(0), version) - assert.EqualValues(0xCAFE, version) - - assert.Equal(versionBytes, p.data[:2]) - for _, b := range p.data[2:] { - assert.EqualValues(0, b) - } -} - -func TestHeaderID(t *testing.T) { - assert := assert.New(t) - - idBytes := []byte{0xCA, 0xFE, 0xBA, 0xBE} - - data := make([]byte, PageSize) - copy(data[2:6], idBytes) - - p, err := load(data) - assert.NoError(err) - - id := p.Header(page.HeaderID) - assert.IsType(uint16(0), id) - assert.EqualValues(0xCAFE, id) - - for _, b := range p.data[:2] { - assert.EqualValues(0, b) - } - assert.Equal(idBytes, p.data[2:6]) - for _, b := range p.data[6:] { - assert.EqualValues(0, b) - } -} - -func TestHeaderCellCount(t *testing.T) { - assert := assert.New(t) - - cellCountBytes := []byte{0xCA, 0xFE} - - data := make([]byte, PageSize) - copy(data[6:8], cellCountBytes) - - p, err := load(data) - assert.NoError(err) - - cellCount := p.Header(InternalHeaderCellCount) - assert.IsType(uint16(0), cellCount) - assert.EqualValues(0xCAFE, cellCount) - - for _, b := range p.data[:6] { - assert.EqualValues(0, b) - } - assert.Equal(cellCountBytes, p.data[6:8]) - for _, b := range p.data[8:] { - assert.EqualValues(0, b) - } -} diff --git a/internal/engine/storage/page/v1/v1_pageloader.go b/internal/engine/storage/page/v1/v1_pageloader.go deleted file mode 100644 index ba834dee..00000000 --- a/internal/engine/storage/page/v1/v1_pageloader.go +++ /dev/null @@ -1,11 +0,0 @@ -package v1 - -//go:generate mockery -inpkg -testonly -case=snake -name=SecondaryStorage - -// SecondaryStorage descries a secondary storage component, such as a disk. It -// has to manage pages by ID and must ensure that pages are read and written -// from the underlying storage without any caching. -type SecondaryStorage interface { - LoadPage(uint32) (*Page, error) - WritePage(uint32, *Page) error -} From 1cee316261aa3e0b136ef2dd3c21677bd68f9ef6 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 16 Jun 2020 16:30:48 +0200 Subject: [PATCH 16/77] Add v1 page implementation without store functionality --- internal/engine/storage/page/cache.go | 6 +- internal/engine/storage/page/page.go | 82 +++-- internal/engine/storage/page/v1/cell.go | 148 +++++++++ internal/engine/storage/page/v1/cell_test.go | 298 ++++++++++++++++++ .../engine/storage/page/v1/celltype_string.go | 25 ++ internal/engine/storage/page/v1/offset.go | 34 ++ internal/engine/storage/page/v1/page.go | 252 +++++++++++++++ internal/engine/storage/page/v1/page_test.go | 156 +++++++++ 8 files changed, 976 insertions(+), 25 deletions(-) create mode 100644 internal/engine/storage/page/v1/cell.go create mode 100644 internal/engine/storage/page/v1/cell_test.go create mode 100644 internal/engine/storage/page/v1/celltype_string.go create mode 100644 internal/engine/storage/page/v1/offset.go create mode 100644 internal/engine/storage/page/v1/page.go create mode 100644 internal/engine/storage/page/v1/page_test.go diff --git a/internal/engine/storage/page/cache.go b/internal/engine/storage/page/cache.go index 4ed02f24..9d748295 100644 --- a/internal/engine/storage/page/cache.go +++ b/internal/engine/storage/page/cache.go @@ -10,16 +10,16 @@ type Cache interface { // evicted. After working with the page, it must be released again, in order // for the cache to be able to free memory. If a page with the given id does // not exist, an error will be returned. - FetchAndPin(id uint32) (Page, error) + FetchAndPin(id ID) (Page, error) // Unpin tells the cache that the page with the given id is no longer // required directly, and that it can be evicted. Unpin is not a guarantee // that the page will be evicted. The cache determines, when to evict a // page. If a page with that id does not exist, this call is a no-op. - Unpin(id uint32) + Unpin(id ID) // Flush writes the contents of the page with the given id to the configured // source. Before a page is evicted, it is always flushed. Use this method // to tell the cache that the page must be flushed immediately. If a page // with the given id does not exist, an error will be returned. - Flush(id uint32) error + Flush(id ID) error io.Closer } diff --git a/internal/engine/storage/page/page.go b/internal/engine/storage/page/page.go index a1bc8587..fc22ff38 100644 --- a/internal/engine/storage/page/page.go +++ b/internal/engine/storage/page/page.go @@ -15,46 +15,84 @@ const ( // errors if any occur. type Loader func([]byte) (Page, error) -//go:generate mockery -case=snake -name=Page +// ID is the type of a page ID. This is mainly to avoid any confusion. +// Changing this will break existing database files, so only change during major +// version upgrades. +type ID = uint32 // Page describes a memory page that stores (page.Cell)s. A page consists of // header fields and cells, and is a plain store. Obtained cells are always // ordered ascending by the cell key. A page supports variable size cell keys -// and records. +// and records. A page is generally NOT safe for concurrent writes. type Page interface { - // Header obtains a header field from the page's header. If the header is - // not supported, result=nil is returned. The type of the returned header - // field value is documented in the header key's godoc. - Header(Header) interface{} + // Version returns the version of the page layout. Use this for choosing the + // page implementation to use to decode the page. + Version() uint32 + + // ID returns the page ID, as it is used by any page loader. It is unique in + // the scope of one database. + ID() ID // Dirty determines whether this page has been modified since the last time - // `Page.ClearDirty` was called. + // Page.ClearDirty was called. Dirty() bool // MarkDirty marks this page as dirty. MarkDirty() // ClearDirty unsets the dirty flag from this page. ClearDirty() - // StoreCell stores a cell on the page. If the page does not fit the cell - // because of size or too much fragmentation, an error will be returned. - StoreCell(Cell) error + // StorePointerCell stores the given pointer cell in the page. + // + // If a cell with the same key as the given pointer already exists in the + // page, it will be overwritten. + // + // If a cell with the same key as the given cell does NOT already exist, it + // will be created. + // + // To change the type of a cell, delete it and store a new cell. + StorePointerCell(PointerCell) error + // StoreRecordCell stores the given record cell in the page. + // + // If a cell with the same key as the given cell already exists in the page, + // it will be overwritten. + // + // If a cell with the same key as the given pointer does NOT already exist, + // it will be created. + // + // To change the type of a cell, delete it and store a new cell. + StoreRecordCell(RecordCell) error // Delete deletes the cell with the given bytes as key. If the key couldn't - // be found, nil is returned. If an error occured during deletion, the error - // is returned. - Delete([]byte) error + // be found, false is returned. If an error occured during deletion, the + // error is returned. + DeleteCell([]byte) (bool, error) // Cell returns a cell with the given key, together with a bool indicating - // whether any cell in the page has that key. + // whether any cell in the page has that key. Use a switch statement to + // determine which type of cell you just obtained (pointer, record). Cell([]byte) (Cell, bool) // Cells returns all cells in this page as a slice. Cells are ordered - // ascending by key. Calling this method is relatively cheap. + // ascending by key. Calling this method can be expensive since all cells + // have to be decoded. Cells() []Cell } -// Cell is a structure that represents a key-value cell. Both the key and the -// record can be of variable size. -type Cell struct { - // Key is the key of this cell, used for ordering. - Key []byte - // Record is the data stored inside the cell. - Record []byte +// Cell describes a generic page cell that holds a key. Use a switch statement +// to determine the type of the cell. +type Cell interface { + // Key returns the key of the cell. + Key() []byte +} + +// PointerCell describes a cell that points to another page in memory. +type PointerCell interface { + Cell + // Pointer returns the page ID of the child page that this cell points to. + Pointer() ID +} + +// RecordCell describes a cell that holds some kind of value. What value format +// was used is none of the cells concern, just use it as what you put in. +type RecordCell interface { + Cell + // Record is the data record in this cell, returned as a byte slice. + Record() []byte } diff --git a/internal/engine/storage/page/v1/cell.go b/internal/engine/storage/page/v1/cell.go new file mode 100644 index 00000000..26ced5f6 --- /dev/null +++ b/internal/engine/storage/page/v1/cell.go @@ -0,0 +1,148 @@ +package v1 + +import ( + "bytes" + + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +var _ page.Cell = (*RecordCell)(nil) +var _ page.Cell = (*PointerCell)(nil) + +//go:generate stringer -type=CellType + +// CellType is the type of a page. +type CellType uint8 + +const ( + // CellTypeUnknown indicates a corrupted page or an incorrectly decoded + // cell. + CellTypeUnknown CellType = iota + // CellTypeRecord indicates a RecordCell, which stores a key and a variable + // size record. + CellTypeRecord + // CellTypePointer indicates a PointerCell, which stores a key and an + // uint32, which points to another page. + CellTypePointer +) + +type ( + // Cell is a cell that has a type and a key. + Cell interface { + page.Cell + Type() CellType + } + + cell struct { + key []byte + } + + // RecordCell is a cell with CellTypeRecord. It holds a key and a variable + // size record. + RecordCell struct { + cell + record []byte + } + + // PointerCell is a cell with CellTypePointer. It holds a key and an uint32, + // pointing to another page. + PointerCell struct { + cell + pointer page.ID + } +) + +// Key returns the key of this cell. +func (c cell) Key() []byte { return c.key } + +// Record returns the record data of this cell. +func (c RecordCell) Record() []byte { return c.record } + +// Pointer returns the pointer of this page, that points to another page. +func (c PointerCell) Pointer() page.ID { return c.pointer } + +// Type returns CellTypeRecord. +func (c RecordCell) Type() CellType { return CellTypeRecord } + +// Type returns CellTypePointer. +func (c PointerCell) Type() CellType { return CellTypePointer } + +func decodeCell(data []byte) Cell { + switch t := CellType(data[0]); t { + case CellTypePointer: + return decodePointerCell(data) + case CellTypeRecord: + return decodeRecordCell(data) + default: + return nil + } +} + +func encodeRecordCell(cell RecordCell) []byte { + key := frame(cell.key) + record := frame(cell.record) + + var buf bytes.Buffer + buf.WriteByte(byte(CellTypeRecord)) + buf.Write(key) + buf.Write(record) + + return buf.Bytes() +} + +func decodeRecordCell(data []byte) RecordCell { + cp := copySlice(data) + + keySize := byteOrder.Uint32(cp[1:5]) + key := cp[5 : 5+keySize] + recordSize := byteOrder.Uint32(cp[5+keySize : 5+keySize+4]) + record := cp[5+keySize+4 : 5+keySize+4+recordSize] + return RecordCell{ + cell: cell{ + key: key, + }, + record: record, + } +} + +func encodePointerCell(cell PointerCell) []byte { + key := frame(cell.key) + pointer := make([]byte, 4) + byteOrder.PutUint32(pointer, cell.pointer) + + var buf bytes.Buffer + buf.WriteByte(byte(CellTypePointer)) + buf.Write(key) + buf.Write(pointer) + + return buf.Bytes() +} + +func decodePointerCell(data []byte) PointerCell { + cp := copySlice(data) + + keySize := byteOrder.Uint32(cp[1:5]) + key := cp[5 : 5+keySize] + pointer := byteOrder.Uint32(cp[5+keySize : 5+keySize+4]) + return PointerCell{ + cell: cell{ + key: key, + }, + pointer: pointer, + } +} + +func frame(data []byte) []byte { + // this allocation can be optimized, however, it would mess up the API, but + // it should be considered in the future + result := make([]byte, 4+len(data)) + copy(result[4:], data) + byteOrder.PutUint32(result, uint32(len(data))) + return result +} + +func copySlice(original []byte) []byte { + copied := make([]byte, len(original)) + copy(copied, original) + return copied +} diff --git a/internal/engine/storage/page/v1/cell_test.go b/internal/engine/storage/page/v1/cell_test.go new file mode 100644 index 00000000..7b3279bc --- /dev/null +++ b/internal/engine/storage/page/v1/cell_test.go @@ -0,0 +1,298 @@ +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_frame(t *testing.T) { + tests := []struct { + name string + data []byte + want []byte + }{ + { + "empty", + []byte{}, + []byte{ + 0x00, 0x00, 0x00, 0x00, // frame + // no data + }, + }, + { + "single", + []byte{0xD1}, + []byte{ + 0x00, 0x00, 0x00, 0x01, // frame + 0xD1, // data + }, + }, + { + "double", + []byte{0xD1, 0xCE}, + []byte{ + 0x00, 0x00, 0x00, 0x02, // frame + 0xD1, 0xCE, // data + }, + }, + { + "large", + make([]byte, 1<<16), // 64KB + append( + []byte{0x00, 0x01, 0x00, 0x00}, // frame + make([]byte, 1<<16)..., // data + ), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got := frame(tt.data) + assert.Equal(tt.want, got) + }) + } +} + +func Test_encodeRecordCell(t *testing.T) { + tests := []struct { + name string + cell RecordCell + want []byte + }{ + { + "empty", + RecordCell{}, + []byte{ + byte(CellTypeRecord), // cell type + 0x00, 0x00, 0x00, 0x00, // key frame + // no key + 0x00, 0x00, 0x00, 0x00, // record frame + // no record + }, + }, + { + "small", + RecordCell{ + cell: cell{ + key: []byte{0xD1, 0xCE}, + }, + record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }, + []byte{ + byte(CellTypeRecord), // cell type + 0x00, 0x00, 0x00, 0x02, // key frame + 0xD1, 0xCE, // key + 0x00, 0x00, 0x00, 0x04, // record frame + 0xCA, 0xFE, 0xBA, 0xBE, // record + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got := encodeRecordCell(tt.cell) + assert.Equal(tt.want, got) + }) + } +} + +func Test_encodePointerCell(t *testing.T) { + tests := []struct { + name string + cell PointerCell + want []byte + }{ + { + "empty", + PointerCell{}, + []byte{ + byte(CellTypePointer), // cell type + 0x00, 0x00, 0x00, 0x00, // key frame + // no key + 0x00, 0x00, 0x00, 0x00, // pointer + }, + }, + { + "simple", + PointerCell{ + cell: cell{ + key: []byte{0xD1, 0xCE}, + }, + pointer: 0xCAFEBABE, + }, + []byte{ + byte(CellTypePointer), // cell type + 0x00, 0x00, 0x00, 0x02, // key frame + 0xD1, 0xCE, // key + 0xCA, 0xFE, 0xBA, 0xBE, // pointer + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got := encodePointerCell(tt.cell) + assert.Equal(tt.want, got) + }) + } +} + +func TestAnyCell_Type(t *testing.T) { + assert := assert.New(t) + assert.Equal(CellTypeRecord, RecordCell{}.Type()) + assert.Equal(CellTypePointer, PointerCell{}.Type()) +} + +func Test_decodeRecordCell(t *testing.T) { + tests := []struct { + name string + data []byte + want RecordCell + }{ + { + "zero value", + []byte{ + byte(CellTypeRecord), // cell type + 0x00, 0x00, 0x00, 0x00, // key frame + // no key + 0x00, 0x00, 0x00, 0x00, // record frame + // no record + }, + RecordCell{ + cell: cell{ + key: []byte{}, + }, + record: []byte{}, + }, + }, + { + "simple", + []byte{ + byte(CellTypeRecord), // cell type + 0x00, 0x00, 0x00, 0x02, // key frame + 0xD1, 0xCE, // key + 0x00, 0x00, 0x00, 0x04, // record frame + 0xCA, 0xFE, 0xBA, 0xBE, // record + }, + RecordCell{ + cell: cell{ + key: []byte{0xD1, 0xCE}, + }, + record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got := decodeRecordCell(tt.data) + assert.Equal(tt.want, got) + }) + } +} + +func Test_decodePointerCell(t *testing.T) { + tests := []struct { + name string + data []byte + want PointerCell + }{ + { + "zero value", + []byte{ + byte(CellTypePointer), // cell type + 0x00, 0x00, 0x00, 0x00, // key frame + // no key + 0x00, 0x00, 0x00, 0x00, // pointer + }, + PointerCell{ + cell: cell{ + key: []byte{}, + }, + pointer: 0, + }, + }, + { + "simple", + []byte{ + byte(CellTypePointer), // cell type + 0x00, 0x00, 0x00, 0x02, // key frame + 0xD1, 0xCE, // key + 0xCA, 0xFE, 0xBA, 0xBE, // pointer + }, + PointerCell{ + cell: cell{ + key: []byte{0xD1, 0xCE}, + }, + pointer: 0xCAFEBABE, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got := decodePointerCell(tt.data) + assert.Equal(tt.want, got) + }) + } +} + +func Test_decodeCell(t *testing.T) { + tests := []struct { + name string + data []byte + want Cell + }{ + { + "zero record cell", + []byte{ + byte(CellTypeRecord), // cell type + 0x00, 0x00, 0x00, 0x00, // key frame + // no key + 0x00, 0x00, 0x00, 0x00, // record frame + // no record + }, + RecordCell{ + cell: cell{key: []byte{}}, + record: []byte{}, + }, + }, + { + "zero pointer cell", + []byte{ + byte(CellTypePointer), // cell type + 0x00, 0x00, 0x00, 0x00, // key frame + // no key + 0x00, 0x00, 0x00, 0x00, // pointer + }, + PointerCell{ + cell: cell{key: []byte{}}, + pointer: 0, + }, + }, + { + "invalid cell type", + []byte{ + byte(CellType(123)), // cell type + 0x00, 0x00, 0x00, 0x00, // key frame + // no key + 0x00, 0x00, 0x00, 0x00, // pointer + }, + nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got := decodeCell(tt.data) + assert.Equal(tt.want, got) + }) + } +} diff --git a/internal/engine/storage/page/v1/celltype_string.go b/internal/engine/storage/page/v1/celltype_string.go new file mode 100644 index 00000000..b40a9886 --- /dev/null +++ b/internal/engine/storage/page/v1/celltype_string.go @@ -0,0 +1,25 @@ +// Code generated by "stringer -type=CellType"; DO NOT EDIT. + +package v1 + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[CellTypeUnknown-0] + _ = x[CellTypeRecord-1] + _ = x[CellTypePointer-2] +} + +const _CellType_name = "CellTypeUnknownCellTypeRecordCellTypePointer" + +var _CellType_index = [...]uint8{0, 15, 29, 44} + +func (i CellType) String() string { + if i >= CellType(len(_CellType_index)-1) { + return "CellType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _CellType_name[_CellType_index[i]:_CellType_index[i+1]] +} diff --git a/internal/engine/storage/page/v1/offset.go b/internal/engine/storage/page/v1/offset.go new file mode 100644 index 00000000..5c38b1cb --- /dev/null +++ b/internal/engine/storage/page/v1/offset.go @@ -0,0 +1,34 @@ +package v1 + +import "unsafe" + +const ( + // OffsetSize is the size of an Offset, in the Go memory layout as well as + // in the serialized form. + OffsetSize = uint16(unsafe.Sizeof(Offset{})) // #nosec +) + +// Offset represents a cell Offset in the page data. +type Offset struct { + // Offset is the Offset of the data in the page data slice. If overflow page + // support is added, this might need to be changed to an uint32. + Offset uint16 + // Size is the length of the data segment in the page data slice. If + // overflow page support is added, this might need to be changed to an + // uint32. + Size uint16 +} + +func decodeOffset(data []byte) Offset { + _ = data[3] + return Offset{ + Offset: byteOrder.Uint16(data[0:]), + Size: byteOrder.Uint16(data[2:]), + } +} + +func (o Offset) encodeInto(target []byte) { + _ = target[3] + byteOrder.PutUint16(target[0:], o.Offset) + byteOrder.PutUint16(target[2:], o.Size) +} diff --git a/internal/engine/storage/page/v1/page.go b/internal/engine/storage/page/v1/page.go new file mode 100644 index 00000000..90e36056 --- /dev/null +++ b/internal/engine/storage/page/v1/page.go @@ -0,0 +1,252 @@ +package v1 + +import ( + "bytes" + "encoding/binary" + "fmt" + "sort" + + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +var _ page.Page = (*Page)(nil) +var _ page.Loader = Load + +const ( + // PageSize is the fix size of a page, which is 16KB or 16384 bytes. + PageSize = 1 << 14 + // HeaderSize is the fix size of a page header, which is 10 bytes. + HeaderSize = 10 +) + +// Header field offset in page data. +const ( + versionOffset = 0 // byte 1,2,3,4: version + idOffset = 4 // byte 5,6,7,8: byte page ID + cellCountOffset = 8 // byte 9,10: cell count +) + +var ( + byteOrder = binary.BigEndian +) + +// Page is a page implementation that does not support overflow pages. It is not +// meant for that. Since we want to separate index and data into separate files, +// records should not contain datasets, but rather enough information, to find +// the corresponding dataset in a data file. +type Page struct { + // data is the underlying data byte slice, which holds the header, offsets + // and cells. + data []byte + + dirty bool +} + +// Load loads the given data into the page. The length of the given data byte +// slice may differ from v1.PageSize, however, it cannot exceed ^uint16(0)-1 +// (65535 or 64KB), and must be larger than 22 (HeaderSize(=10) + 1 Offset(=4) + +// 1 empty cell(=8)). +func Load(data []byte) (page.Page, error) { + return load(data) +} + +// Version returns the version of this page. This should always be 1. This value +// must be constant. +func (p *Page) Version() uint32 { return byteOrder.Uint32(p.data[versionOffset:]) } + +// ID returns the ID of this page. This value must be constant. +func (p *Page) ID() page.ID { return byteOrder.Uint32(p.data[idOffset:]) } + +// CellCount returns the amount of stored cells in this page. This value is NOT +// constant. +func (p *Page) CellCount() uint16 { return byteOrder.Uint16(p.data[cellCountOffset:]) } + +// Dirty returns whether the page is dirty (needs syncing with secondary +// storage). +func (p *Page) Dirty() bool { return p.dirty } + +// MarkDirty marks this page as dirty. +func (p *Page) MarkDirty() { p.dirty = true } + +// ClearDirty marks this page as NOT dirty. +func (p *Page) ClearDirty() { p.dirty = false } + +// StorePointerCell stores a pointer cell in this page. A pointer cell points to +// other page IDs. +func (p *Page) StorePointerCell(cell page.PointerCell) error { + if v1cell, ok := cell.(PointerCell); ok { + return p.storePointerCell(v1cell) + } + return fmt.Errorf("can only store v1 pointer cells, but got %T", cell) +} + +// StoreRecordCell stores a record cell in this page. A record cell holds +// arbitrary, variable size data. +func (p *Page) StoreRecordCell(cell page.RecordCell) error { + if v1cell, ok := cell.(RecordCell); ok { + return p.storeRecordCell(v1cell) + } + return fmt.Errorf("can only store v1 record cells, but got %T", cell) +} + +// DeleteCell deletes a cell with the given key. If no such cell could be found, +// false is returned. In this implementation, an error can not occur while +// deleting a cell. +func (p *Page) DeleteCell(key []byte) (bool, error) { + offsetIndex, cellOffset, _, found := p.findCell(key) + if !found { + return false, nil + } + + // delete offset + p.zero(offsetIndex*OffsetSize, OffsetSize) + // delete cell data + p.zero(cellOffset.Offset, cellOffset.Size) + // close gap in offsets due to offset deletion + from := offsetIndex*OffsetSize + OffsetSize // lower bound, right next to gap + to := p.CellCount() * OffsetSize // upper bound of the offset data + p.moveAndZero(from, to-from, from-OffsetSize) // actually move the data + // update cell count + p.decrementCellCount(1) + return true, nil +} + +// Cell returns a cell from this page with the given key, or false if no such +// cell exists in this page. In that case, the returned page is also nil. +func (p *Page) Cell(key []byte) (page.Cell, bool) { + _, _, cell, found := p.findCell(key) + return cell, found +} + +// Cells decodes all cells in this page, which can be expensive, and returns all +// of them. The returned cells do not point back to the original page data, so +// don't modify them. Instead, delete the old cell and store a new one. +func (p *Page) Cells() (result []page.Cell) { + for _, offset := range p.Offsets() { + result = append(result, decodeCell(p.data[offset.Offset:offset.Offset+offset.Size])) + } + return +} + +// RawData returns a copy of the page's internal data, so you can modify it at +// will, and it won't change the original page data. +func (p *Page) RawData() []byte { + cp := make([]byte, len(p.data)) + copy(cp, p.data) + return cp +} + +// Offsets returns all offsets in the page. The offsets can be used to find all +// cells in the page. The amount of offsets will always be equal to the amount +// of cells stored in a page. The amount of offsets in the page depends on the +// cell count of this page, not the other way around. +func (p *Page) Offsets() (result []Offset) { + cellCount := p.CellCount() + offsetsWidth := cellCount * OffsetSize + offsetData := p.data[HeaderSize : HeaderSize+offsetsWidth] + for i := uint16(0); i < cellCount; i++ { + result = append(result, decodeOffset(offsetData[i*OffsetSize:i*OffsetSize+OffsetSize])) + } + return +} + +func load(data []byte) (*Page, error) { + if len(data) > int(^uint16(0))-1 { + return nil, fmt.Errorf("page size too large: %v (max %v)", len(data), int(^uint16(0))-1) + } + + return &Page{ + data: data, + }, nil +} + +// findCell searches for a cell with the given key, as well as the corresponding +// offset and the corresponding offset index. The index is the index of the cell +// offset in all offsets, meaning that the byte location of the offset in the +// page can be obtained with offsetIndex*OffsetSize. The cellOffset is the +// offset that points to the cell. cell is the cell that was found, or nil if no +// cell with the given key could be found. If no cell could be found, +// found=false will be returned, as well as zero values for all other return +// arguments. +func (p *Page) findCell(key []byte) (offsetIndex uint16, cellOffset Offset, cell Cell, found bool) { + offsets := p.Offsets() + result := sort.Search(len(offsets), func(i int) bool { + cell := p.cellAt(offsets[i]) + return bytes.Compare(cell.Key(), key) >= 0 + }) + if result == len(offsets) { + return 0, Offset{}, nil, false + } + return uint16(result), offsets[result], p.cellAt(offsets[result]), true +} + +func (p *Page) storePointerCell(cell PointerCell) error { + return p.storeRawCell(encodePointerCell(cell)) +} + +func (p *Page) storeRecordCell(cell RecordCell) error { + return p.storeRawCell(encodeRecordCell(cell)) +} + +func (p *Page) storeRawCell(rawCell []byte) error { + p.incrementCellCount(1) + _ = Offset{}.encodeInto // to remove linter error + return fmt.Errorf("unimplemented") +} + +func (p *Page) cellAt(offset Offset) Cell { + return decodeCell(p.data[offset.Offset : offset.Offset+offset.Size]) +} + +// moveAndZero moves target bytes in the page's raw data from offset to target, +// and zeros all bytes from offset to offset+size, that do not overlap with the +// target area. Source and target area may overlap. +// +// [1,1,2,2,2,1,1,1,1,1] +// moveAndZero(2, 3, 6) +// [1,1,0,0,0,1,2,2,2,1] +// +// or, with overlap +// +// [1,1,2,2,2,1,1,1,1,1] +// moveAndZero(2, 3, 4) +// [1,1,0,0,2,2,2,1,1,1] +func (p *Page) moveAndZero(offset, size, target uint16) { + _ = p.data[offset+size-1] // bounds check + _ = p.data[target+size-1] // bounds check + + copy(p.data[target:target+size], p.data[offset:offset+size]) + + // area needs zeroing + if target != offset { + if target > offset+size || target+size < offset { + // no overlap + p.zero(offset, size) + } else { + // overlap + if target > offset && target <= offset+size { + // move to right, zero non-overlapping area + p.zero(offset, target-offset) + } else if target < offset && target+size >= offset { + // move to left, zero non-overlapping area + p.zero(target+size, offset-target) + } + } + } +} + +// zero zeroes size bytes, starting at offset in the page's raw data. +func (p *Page) zero(offset, size uint16) { + for i := uint16(0); i < size; i++ { + p.data[offset+i] = 0 + } +} + +func (p *Page) incrementCellCount(delta uint16) { p.incrementUint16(cellCountOffset, delta) } +func (p *Page) decrementCellCount(delta uint16) { p.decrementUint16(cellCountOffset, delta) } + +func (p *Page) storeUint16(at, val uint16) { byteOrder.PutUint16(p.data[at:], val) } +func (p *Page) loadUint16(at uint16) uint16 { return byteOrder.Uint16(p.data[at:]) } + +func (p *Page) incrementUint16(at, delta uint16) { p.storeUint16(at, p.loadUint16(at)+delta) } +func (p *Page) decrementUint16(at, delta uint16) { p.storeUint16(at, p.loadUint16(at)-delta) } diff --git a/internal/engine/storage/page/v1/page_test.go b/internal/engine/storage/page/v1/page_test.go new file mode 100644 index 00000000..233d5bcf --- /dev/null +++ b/internal/engine/storage/page/v1/page_test.go @@ -0,0 +1,156 @@ +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPage_offsets(t *testing.T) { + assert := assert.New(t) + + // create a completely empty page + p, err := load(make([]byte, PageSize)) + assert.NoError(err) + + // create the offset source data + offsetCount := 3 + offsetData := []byte{ + // offset[0] + 0x01, 0x12, // offset + 0x23, 0x34, // size + // offset[1] + 0x45, 0x56, // offset + 0x67, 0x78, // size + // offset[2] + 0x89, 0x9A, // offset + 0xAB, 0xBC, // size + } + // quick check if we made a mistake in the test + assert.EqualValues(OffsetSize, len(offsetData)/offsetCount) + + // inject the offset data + p.incrementCellCount(3) // set the cell count + copy(p.data[HeaderSize:], offsetData) // copy the offset data + + // actual test can start + + offsets := p.Offsets() + assert.Len(offsets, 3) + assert.Equal(Offset{ + Offset: 0x0112, + Size: 0x2334, + }, offsets[0]) + assert.Equal(Offset{ + Offset: 0x4556, + Size: 0x6778, + }, offsets[1]) + assert.Equal(Offset{ + Offset: 0x899A, + Size: 0xABBC, + }, offsets[2]) +} + +func TestPage_moveAndZero(t *testing.T) { + type args struct { + offset uint16 + size uint16 + target uint16 + } + tests := []struct { + name string + data []byte + args args + want []byte + }{ + { + "same position", + []byte{1, 1, 2, 2, 2, 2, 1, 1, 1, 1}, + args{2, 4, 2}, + []byte{1, 1, 2, 2, 2, 2, 1, 1, 1, 1}, + }, + { + "single no overlap to right", + []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + args{0, 1, 2}, + []byte{0, 2, 1, 4, 5, 6, 7, 8, 9}, + }, + { + "double no overlap to right", + []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + args{0, 2, 3}, + []byte{0, 0, 3, 1, 2, 6, 7, 8, 9}, + }, + { + "many no overlap to right", + []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + args{0, 4, 5}, + []byte{0, 0, 0, 0, 5, 1, 2, 3, 4}, + }, + { + "single no overlap to left", + []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + args{8, 1, 2}, + []byte{1, 2, 9, 4, 5, 6, 7, 8, 0}, + }, + { + "double no overlap to left", + []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + args{7, 2, 3}, + []byte{1, 2, 3, 8, 9, 6, 7, 0, 0}, + }, + { + "many no overlap to left", + []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, + args{5, 4, 0}, + []byte{6, 7, 8, 9, 5, 0, 0, 0, 0}, + }, + { + "double 1 overlap to right", + []byte{1, 1, 2, 2, 1, 1, 1, 1, 1, 1}, + args{2, 2, 3}, + []byte{1, 1, 0, 2, 2, 1, 1, 1, 1, 1}, + }, + { + "double 1 overlap to left", + []byte{1, 1, 1, 2, 2, 1, 1, 1, 1, 1}, + args{3, 2, 2}, + []byte{1, 1, 2, 2, 0, 1, 1, 1, 1, 1}, + }, + { + "triple 1 overlap to right", + []byte{1, 1, 2, 2, 2, 1, 1, 1, 1, 1}, + args{2, 3, 4}, + []byte{1, 1, 0, 0, 2, 2, 2, 1, 1, 1}, + }, + { + "triple 2 overlap to right", + []byte{1, 1, 2, 2, 2, 1, 1, 1, 1, 1}, + args{2, 3, 3}, + []byte{1, 1, 0, 2, 2, 2, 1, 1, 1, 1}, + }, + { + "triple 1 overlap to left", + []byte{1, 1, 1, 1, 2, 2, 2, 1, 1, 1}, + args{4, 3, 2}, + []byte{1, 1, 2, 2, 2, 0, 0, 1, 1, 1}, + }, + { + "triple 2 overlap to left", + []byte{1, 1, 1, 2, 2, 2, 1, 1, 1, 1}, + args{3, 3, 2}, + []byte{1, 1, 2, 2, 2, 0, 1, 1, 1, 1}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + p := &Page{ + data: tt.data, + } + p.moveAndZero(tt.args.offset, tt.args.size, tt.args.target) + assert.Equal(tt.want, p.data) + }) + } +} From ca08f13adb82784820fb6f5c23688ae441f629b1 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 18 Jun 2020 17:01:41 +0200 Subject: [PATCH 17/77] Remove converter as it's obsolete --- internal/engine/converter/converter.go | 202 -------------------- internal/engine/converter/converter_test.go | 155 --------------- internal/engine/converter/doc.go | 2 - 3 files changed, 359 deletions(-) delete mode 100644 internal/engine/converter/converter.go delete mode 100644 internal/engine/converter/converter_test.go delete mode 100644 internal/engine/converter/doc.go diff --git a/internal/engine/converter/converter.go b/internal/engine/converter/converter.go deleted file mode 100644 index 297b358e..00000000 --- a/internal/engine/converter/converter.go +++ /dev/null @@ -1,202 +0,0 @@ -package converter - -import ( - "encoding/binary" - "math" -) - -const ( - falseByte byte = byte(0) - trueByte byte = ^falseByte -) - -var ( - byteOrder = binary.BigEndian -) - -// bool - -// BoolToByte converts the given bool to a byte which is then returned. -func BoolToByte(v bool) byte { - if v { - return trueByte - } - return falseByte -} - -// ByteToBool converts the given byte back to a bool. -func ByteToBool(v byte) bool { - return v != falseByte -} - -// BoolToByteArray converts the given bool to a [1]byte is then returned. -func BoolToByteArray(v bool) [1]byte { - return [1]byte{BoolToByte(v)} -} - -// ByteArrayToBool converts the given [1]byte back to a bool. -func ByteArrayToBool(v [1]byte) bool { - return ByteToBool(v[0]) -} - -// BoolToByteSlice converts the given bool to a []byte which is then returned. -func BoolToByteSlice(v bool) []byte { - arr := BoolToByteArray(v) - return arr[:] -} - -// ByteSliceToBool converts the given []byte back to a bool. -func ByteSliceToBool(v []byte) bool { - return ByteToBool(v[0]) -} - -// integral - -// Uint16ToByteArray converts the given uint16 to a [2]byte which is then -// returned. -func Uint16ToByteArray(v uint16) (result [2]byte) { - byteOrder.PutUint16(result[:], v) - return -} - -// ByteArrayToUint16 converts the given [2]byte back to a uint16. -func ByteArrayToUint16(v [2]byte) uint16 { - return byteOrder.Uint16(v[:]) -} - -// Uint16ToByteSlice converts the given uint16 to a []byte which is then -// returned. -func Uint16ToByteSlice(v uint16) (result []byte) { - result = make([]byte, 2) - byteOrder.PutUint16(result, v) - return -} - -// ByteSliceToUint16 converts the given []byte back to a uint16. -func ByteSliceToUint16(v []byte) uint16 { - return byteOrder.Uint16(v) -} - -// Uint32ToByteArray converts the given uint32 to a [4]byte which is then -// returned. -func Uint32ToByteArray(v uint32) (result [4]byte) { - byteOrder.PutUint32(result[:], v) - return -} - -// ByteArrayToUint32 converts the given [4]byte back to a uint32. -func ByteArrayToUint32(v [4]byte) uint32 { - return byteOrder.Uint32(v[:]) -} - -// Uint32ToByteSlice converts the given uint32 to a []byte which is then -// returned. -func Uint32ToByteSlice(v uint32) (result []byte) { - result = make([]byte, 4) - byteOrder.PutUint32(result, v) - return -} - -// ByteSliceToUint32 converts the given []byte back to a uint32. -func ByteSliceToUint32(v []byte) uint32 { - return byteOrder.Uint32(v) -} - -// Uint64ToByteArray converts the given uint64 to a [8]byte which is then -// returned. -func Uint64ToByteArray(v uint64) (result [8]byte) { - byteOrder.PutUint64(result[:], v) - return -} - -// ByteArrayToUint64 converts the given [8]byte back to a uint64. -func ByteArrayToUint64(v [8]byte) uint64 { - return byteOrder.Uint64(v[:]) -} - -// Uint64ToByteSlice converts the given uint64 to a []byte which is then -// returned. -func Uint64ToByteSlice(v uint64) (result []byte) { - result = make([]byte, 8) - byteOrder.PutUint64(result, v) - return -} - -// ByteSliceToUint64 converts the given []byte back to a uint64. -func ByteSliceToUint64(v []byte) uint64 { - return byteOrder.Uint64(v) -} - -// fractal - -// Float32ToByteArray converts the given float32 to a [4]byte, which is then -// returned. -func Float32ToByteArray(v float32) (result [4]byte) { - return Uint32ToByteArray(math.Float32bits(v)) -} - -// Float32ToByteSlice converts the given float32 to a []byte, which is then -// returned. -func Float32ToByteSlice(v float32) (result []byte) { - return Uint32ToByteSlice(math.Float32bits(v)) -} - -// Float64ToByteArray converts the given float64 to a [8]byte, which is then -// returned. -func Float64ToByteArray(v float64) (result [8]byte) { - return Uint64ToByteArray(math.Float64bits(v)) -} - -// Float64ToByteSlice converts the given float64 to a []byte, which is then -// returned. -func Float64ToByteSlice(v float64) (result []byte) { - return Uint64ToByteSlice(math.Float64bits(v)) -} - -// complex - -// Complex64ToByteArray converts the given complex64 to a [8]byte, which is then -// returned. -func Complex64ToByteArray(v complex64) (result [8]byte) { - copy(result[:4], Float32ToByteSlice(real(v))) - copy(result[4:], Float32ToByteSlice(imag(v))) - return -} - -// Complex64ToByteSlice converts the given complex64 to a []byte, which is then -// returned. -func Complex64ToByteSlice(v complex64) (result []byte) { - result = make([]byte, 8) - copy(result[:4], Float32ToByteSlice(real(v))) - copy(result[4:], Float32ToByteSlice(imag(v))) - return -} - -// Complex128ToByteArray converts the given complex128 to a [16]byte, which is -// then returned. -func Complex128ToByteArray(v complex128) (result [16]byte) { - copy(result[:8], Float64ToByteSlice(real(v))) - copy(result[8:], Float64ToByteSlice(imag(v))) - return -} - -// Complex128ToByteSlice converts the given complex128 to a []byte, which is -// then returned. -func Complex128ToByteSlice(v complex128) (result []byte) { - result = make([]byte, 16) - copy(result[:8], Float64ToByteSlice(real(v))) - copy(result[8:], Float64ToByteSlice(imag(v))) - return -} - -// variable-size - -// StringToByteSlice converts the given string a []byte which is then returned. -func StringToByteSlice(v string) []byte { - return []byte(v) -} - -// ByteSliceToString converts the given []byte back to a string. -func ByteSliceToString(v []byte) string { - return string(v) -} diff --git a/internal/engine/converter/converter_test.go b/internal/engine/converter/converter_test.go deleted file mode 100644 index abd31c50..00000000 --- a/internal/engine/converter/converter_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package converter_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tomarrell/lbadd/internal/engine/converter" -) - -func TestBool(t *testing.T) { - t.Run("target=byte", func(t *testing.T) { - assert := assert.New(t) - assert.Equal(byte(0x00), converter.BoolToByte(false)) - assert.Equal(byte(255), converter.BoolToByte(true)) - }) - t.Run("target=bytearray", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([1]byte{0x00}, converter.BoolToByteArray(false)) - assert.Equal([1]byte{255}, converter.BoolToByteArray(true)) - }) - t.Run("target=byteslice", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([]byte{0x00}, converter.BoolToByteSlice(false)) - assert.Equal([]byte{255}, converter.BoolToByteSlice(true)) - }) -} - -func TestIntegral(t *testing.T) { - t.Run("cardinality=16bit", func(t *testing.T) { - t.Run("target=bytearray", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([2]byte{0x00, 0x00}, converter.Uint16ToByteArray(0)) - assert.Equal([2]byte{0xCA, 0xFE}, converter.Uint16ToByteArray(0xCAFE)) - assert.Equal([2]byte{0x00, 0xAB}, converter.Uint16ToByteArray(0xAB)) - assert.Equal([2]byte{0xFF, 0xFF}, converter.Uint16ToByteArray(0xFFFF)) - }) - t.Run("target=byteslice", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([]byte{0x00, 0x00}, converter.Uint16ToByteSlice(0)) - assert.Equal([]byte{0xCA, 0xFE}, converter.Uint16ToByteSlice(0xCAFE)) - assert.Equal([]byte{0x00, 0xAB}, converter.Uint16ToByteSlice(0xAB)) - assert.Equal([]byte{0xFF, 0xFF}, converter.Uint16ToByteSlice(0xFFFF)) - }) - }) - t.Run("cardinality=32bit", func(t *testing.T) { - t.Run("target=bytearray", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([4]byte{0x00, 0x00, 0x00, 0x00}, converter.Uint32ToByteArray(0)) - assert.Equal([4]byte{0xCA, 0xFE, 0xBA, 0xBE}, converter.Uint32ToByteArray(0xCAFEBABE)) - assert.Equal([4]byte{0x00, 0x00, 0x00, 0xAB}, converter.Uint32ToByteArray(0xAB)) - assert.Equal([4]byte{0xFF, 0xFF, 0xFF, 0xFF}, converter.Uint32ToByteArray(0xFFFFFFFF)) - }) - t.Run("target=byteslice", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([]byte{0x00, 0x00, 0x00, 0x00}, converter.Uint32ToByteSlice(0)) - assert.Equal([]byte{0xCA, 0xFE, 0xBA, 0xBE}, converter.Uint32ToByteSlice(0xCAFEBABE)) - assert.Equal([]byte{0x00, 0x00, 0x00, 0xAB}, converter.Uint32ToByteSlice(0xAB)) - assert.Equal([]byte{0xFF, 0xFF, 0xFF, 0xFF}, converter.Uint32ToByteSlice(0xFFFFFFFF)) - }) - }) - t.Run("cardinality=64bit", func(t *testing.T) { - t.Run("target=bytearray", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Uint64ToByteArray(0)) - assert.Equal([8]byte{0xCA, 0xFE, 0xBA, 0xBE, 0xDA, 0xDE, 0xFA, 0xBE}, converter.Uint64ToByteArray(0xCAFEBABEDADEFABE)) - assert.Equal([8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB}, converter.Uint64ToByteArray(0xAB)) - assert.Equal([8]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, converter.Uint64ToByteArray(0xFFFFFFFFFFFFFFFF)) - }) - t.Run("target=byteslice", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Uint64ToByteSlice(0)) - assert.Equal([]byte{0xCA, 0xFE, 0xBA, 0xBE, 0xDA, 0xDE, 0xFA, 0xBE}, converter.Uint64ToByteSlice(0xCAFEBABEDADEFABE)) - assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB}, converter.Uint64ToByteSlice(0xAB)) - assert.Equal([]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, converter.Uint64ToByteSlice(0xFFFFFFFFFFFFFFFF)) - }) - }) -} - -func TestFractal(t *testing.T) { - t.Run("cardinality=32bit", func(t *testing.T) { - t.Run("target=bytearray", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([4]byte{0x00, 0x00, 0x00, 0x00}, converter.Float32ToByteArray(0)) - assert.Equal([4]byte{0x4f, 0x4a, 0xfe, 0xbb}, converter.Float32ToByteArray(0xCAFEBABE)) - assert.Equal([4]byte{0x43, 0x2b, 0x00, 0x00}, converter.Float32ToByteArray(0xAB)) - assert.Equal([4]byte{0x4f, 0x80, 0x00, 0x00}, converter.Float32ToByteArray(0xFFFFFFFF)) - }) - t.Run("target=byteslice", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([]byte{0x00, 0x00, 0x00, 0x00}, converter.Float32ToByteSlice(0)) - assert.Equal([]byte{0x4f, 0x4a, 0xfe, 0xbb}, converter.Float32ToByteSlice(0xCAFEBABE)) - assert.Equal([]byte{0x43, 0x2b, 0x00, 0x00}, converter.Float32ToByteSlice(0xAB)) - assert.Equal([]byte{0x4f, 0x80, 0x00, 0x00}, converter.Float32ToByteSlice(0xFFFFFFFF)) - }) - }) - t.Run("cardinality=64bit", func(t *testing.T) { - t.Run("target=bytearray", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteArray(0)) - assert.Equal([8]byte{0x43, 0xe9, 0x5f, 0xd7, 0x57, 0xdb, 0x5b, 0xdf}, converter.Float64ToByteArray(0xCAFEBABEDADEFABE)) - assert.Equal([8]byte{0x40, 0x65, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteArray(0xAB)) - assert.Equal([8]byte{0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteArray(0xFFFFFFFFFFFFFFFF)) - }) - t.Run("target=byteslice", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteSlice(0)) - assert.Equal([]byte{0x43, 0xe9, 0x5f, 0xd7, 0x57, 0xdb, 0x5b, 0xdf}, converter.Float64ToByteSlice(0xCAFEBABEDADEFABE)) - assert.Equal([]byte{0x40, 0x65, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteSlice(0xAB)) - assert.Equal([]byte{0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Float64ToByteSlice(0xFFFFFFFFFFFFFFFF)) - }) - }) -} - -func TestComplex(t *testing.T) { - t.Run("cardinality=64bit", func(t *testing.T) { - t.Run("target=bytearray", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex64ToByteArray(0+0i)) - assert.Equal([8]byte{0x41, 0x60, 0x00, 0x00, 0x41, 0x40, 0x00, 0x00}, converter.Complex64ToByteArray(14+12i)) - assert.Equal([8]byte{0x40, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex64ToByteArray(4+0i)) - assert.Equal([8]byte{0x4f, 0x80, 0x00, 0x00, 0x4f, 0x80, 0x00, 0x00}, converter.Complex64ToByteArray(0xFFFFFFFF+0xFFFFFFFFi)) - }) - t.Run("target=byteslice", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex64ToByteSlice(0+0i)) - assert.Equal([]byte{0x41, 0x60, 0x00, 0x00, 0x41, 0x40, 0x00, 0x00}, converter.Complex64ToByteSlice(14+12i)) - assert.Equal([]byte{0x40, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex64ToByteSlice(4+0i)) - assert.Equal([]byte{0x4f, 0x80, 0x00, 0x00, 0x4f, 0x80, 0x00, 0x00}, converter.Complex64ToByteSlice(0xFFFFFFFF+0xFFFFFFFFi)) - }) - }) - t.Run("cardinality=128bit", func(t *testing.T) { - t.Run("target=bytearray", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([16]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteArray(0+0i)) - assert.Equal([16]byte{0x40, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteArray(14+12i)) - assert.Equal([16]byte{0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteArray(4+0i)) - assert.Equal([16]byte{0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteArray(0xFFFFFFFFFFFFFFFF+0xFFFFFFFFFFFFFFFFi)) - }) - t.Run("target=byteslice", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteSlice(0+0i)) - assert.Equal([]byte{0x40, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteSlice(14+12i)) - assert.Equal([]byte{0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteSlice(4+0i)) - assert.Equal([]byte{0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, converter.Complex128ToByteSlice(0xFFFFFFFFFFFFFFFF+0xFFFFFFFFFFFFFFFFi)) - }) - }) -} - -func TestVariable(t *testing.T) { - t.Run("type=string", func(t *testing.T) { - assert := assert.New(t) - assert.Equal([]byte{}, converter.StringToByteSlice("")) - assert.Equal([]byte{0x61, 0x62, 0x63, 0x64}, converter.StringToByteSlice("abcd")) - }) -} diff --git a/internal/engine/converter/doc.go b/internal/engine/converter/doc.go deleted file mode 100644 index 62b0708d..00000000 --- a/internal/engine/converter/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package converter implements functions to encode and decode values to bytes. -package converter From ff1faf04d623cda21eb80ef997bfdc0269968047 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 18 Jun 2020 21:59:19 +0200 Subject: [PATCH 18/77] Add storage functionality --- internal/engine/storage/page/error.go | 2 +- internal/engine/storage/page/v1/offset.go | 14 ++- internal/engine/storage/page/v1/page.go | 126 ++++++++++++++++--- internal/engine/storage/page/v1/page_test.go | 51 ++++++++ 4 files changed, 168 insertions(+), 25 deletions(-) diff --git a/internal/engine/storage/page/error.go b/internal/engine/storage/page/error.go index e4ddb068..eb2923a1 100644 --- a/internal/engine/storage/page/error.go +++ b/internal/engine/storage/page/error.go @@ -9,5 +9,5 @@ func (e Error) Error() string { return string(e) } const ( ErrUnknownHeader = Error("unknown header") ErrInvalidPageSize = Error("invalid page size") - ErrPageTooSmall = Error("page is too small for the requested data") + ErrPageFull = Error("page is full") ) diff --git a/internal/engine/storage/page/v1/offset.go b/internal/engine/storage/page/v1/offset.go index 5c38b1cb..44532daa 100644 --- a/internal/engine/storage/page/v1/offset.go +++ b/internal/engine/storage/page/v1/offset.go @@ -19,6 +19,14 @@ type Offset struct { Size uint16 } +func (o Offset) encodeInto(target []byte) { + /* very simple way to avoid a new 4 byte allocation, should probably also be + applied to cells */ + _ = target[3] + byteOrder.PutUint16(target[0:], o.Offset) + byteOrder.PutUint16(target[2:], o.Size) +} + func decodeOffset(data []byte) Offset { _ = data[3] return Offset{ @@ -26,9 +34,3 @@ func decodeOffset(data []byte) Offset { Size: byteOrder.Uint16(data[2:]), } } - -func (o Offset) encodeInto(target []byte) { - _ = target[3] - byteOrder.PutUint16(target[0:], o.Offset) - byteOrder.PutUint16(target[2:], o.Size) -} diff --git a/internal/engine/storage/page/v1/page.go b/internal/engine/storage/page/v1/page.go index 90e36056..2a379443 100644 --- a/internal/engine/storage/page/v1/page.go +++ b/internal/engine/storage/page/v1/page.go @@ -150,6 +150,46 @@ func (p *Page) Offsets() (result []Offset) { return } +// FreeSlots computes all free addressable cell slots in this page. +func (p *Page) FreeSlots() (result []Offset) { + offsets := p.Offsets() + if len(offsets) == 0 { + // if there are no offsets at all, that means that the page is empty, + // and one slot is returned, which reaches from 0+OffsetSize until the + // end of the page + off := HeaderSize + OffsetSize + return []Offset{{ + Offset: HeaderSize + OffsetSize, + Size: uint16(len(p.data)) - off, + }} + } + + sort.Slice(offsets, func(i, j int) bool { + return offsets[i].Offset < offsets[j].Offset + }) + // first slot, from end of offset data until first cell + firstOff := HeaderSize + uint16(len(offsets)+1)*OffsetSize // +1 because we always need space to store one more offset, so if that space is blocked, there is no free slot that is addressable + firstSize := offsets[0].Offset - firstOff + if firstSize > 0 { + result = append(result, Offset{ + Offset: firstOff, + Size: firstSize, + }) + } + // rest of the spaces between cells + for i := 0; i < len(offsets)-1; i++ { + off := offsets[i].Offset + offsets[i].Size + size := offsets[i+1].Offset - off + if size > 0 { + result = append(result, Offset{ + Offset: off, + Size: size, + }) + } + } + return +} + func load(data []byte) (*Page, error) { if len(data) > int(^uint16(0))-1 { return nil, fmt.Errorf("page size too large: %v (max %v)", len(data), int(^uint16(0))-1) @@ -181,17 +221,64 @@ func (p *Page) findCell(key []byte) (offsetIndex uint16, cellOffset Offset, cell } func (p *Page) storePointerCell(cell PointerCell) error { - return p.storeRawCell(encodePointerCell(cell)) + return p.storeRawCell(cell.key, encodePointerCell(cell)) } func (p *Page) storeRecordCell(cell RecordCell) error { - return p.storeRawCell(encodeRecordCell(cell)) + return p.storeRawCell(cell.key, encodeRecordCell(cell)) } -func (p *Page) storeRawCell(rawCell []byte) error { +func (p *Page) storeRawCell(key, rawCell []byte) error { + size := uint16(len(rawCell)) + slot, ok := p.findFreeSlotForSize(size) + if !ok { + return page.ErrPageFull + } + copy(p.data[slot.Offset+slot.Size-size:], rawCell) + p.storeCellOffset(Offset{ + Offset: slot.Offset + slot.Size - size, + Size: size, + }, key) p.incrementCellCount(1) - _ = Offset{}.encodeInto // to remove linter error - return fmt.Errorf("unimplemented") + return nil +} + +func (p *Page) storeCellOffset(offset Offset, cellKey []byte) { + offsets := p.Offsets() + if len(offsets) == 0 { + // directly into the start of the page content, after the header + offset.encodeInto(p.data[HeaderSize:]) + return + } + + index := sort.Search(len(offsets), func(i int) bool { + return bytes.Compare(cellKey, p.cellAt(offsets[i]).Key()) > 0 + }) + + // make room if neccessary + offsetOffset := uint16(index) * OffsetSize + allOffsetsEnd := uint16(len(offsets)) * OffsetSize + p.moveAndZero(offsetOffset, allOffsetsEnd-offsetOffset, offsetOffset+OffsetSize) + offset.encodeInto(p.data[offsetOffset:]) +} + +// findFreeSlotForSize searches for a free slot in this page, matching or +// exceeding the given data size. This is done by using a best-fit algorithm. +func (p *Page) findFreeSlotForSize(dataSize uint16) (Offset, bool) { + // sort all free slots by size + slots := p.FreeSlots() + sort.Slice(slots, func(i, j int) bool { + return slots[i].Size < slots[j].Size + }) + // search for the best fitting slot, i.e. the first slot, whose size is greater + // than or equal to the given data size + index := sort.Search(len(slots), func(i int) bool { + return slots[i].Size >= dataSize + }) + if index == len(slots) { + return Offset{}, false + } + return slots[index], true } func (p *Page) cellAt(offset Offset) Cell { @@ -212,25 +299,28 @@ func (p *Page) cellAt(offset Offset) Cell { // moveAndZero(2, 3, 4) // [1,1,0,0,2,2,2,1,1,1] func (p *Page) moveAndZero(offset, size, target uint16) { + if target == offset { + // no-op when offset and target are the same + return + } + _ = p.data[offset+size-1] // bounds check _ = p.data[target+size-1] // bounds check copy(p.data[target:target+size], p.data[offset:offset+size]) // area needs zeroing - if target != offset { - if target > offset+size || target+size < offset { - // no overlap - p.zero(offset, size) - } else { - // overlap - if target > offset && target <= offset+size { - // move to right, zero non-overlapping area - p.zero(offset, target-offset) - } else if target < offset && target+size >= offset { - // move to left, zero non-overlapping area - p.zero(target+size, offset-target) - } + if target > offset+size || target+size < offset { + // no overlap + p.zero(offset, size) + } else { + // overlap + if target > offset && target <= offset+size { + // move to right, zero non-overlapping area + p.zero(offset, target-offset) + } else if target < offset && target+size >= offset { + // move to left, zero non-overlapping area + p.zero(target+size, offset-target) } } } diff --git a/internal/engine/storage/page/v1/page_test.go b/internal/engine/storage/page/v1/page_test.go index 233d5bcf..2a7168b6 100644 --- a/internal/engine/storage/page/v1/page_test.go +++ b/internal/engine/storage/page/v1/page_test.go @@ -4,8 +4,59 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/engine/storage/page" ) +func TestPage_StoreRecordCell(t *testing.T) { + assert := assert.New(t) + + p, err := load(make([]byte, 36)) + assert.NoError(err) + + c := RecordCell{ + cell: cell{ + key: []byte{0xAB}, + }, + record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + } + + err = p.StoreRecordCell(c) + assert.NoError(err) + assert.Equal([]byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, // header + 0x00, 0x16, 0x00, 0x0E, // offset + 0x00, 0x00, 0x00, 0x00, // reserved for next offset + 0x00, 0x00, 0x00, 0x00, // free slot #0 + 0x01, // cell type + 0x00, 0x00, 0x00, 0x01, // key frame + 0xAB, // key + 0x00, 0x00, 0x00, 0x04, // record frame + 0xCA, 0xFE, 0xBA, 0xBE, // record + }, p.data) + + freeSlots := p.FreeSlots() + assert.Len(freeSlots, 1) + // offset must skipt reserved space for offset, as the offset is not free + // space + assert.Equal(Offset{ + Offset: 18, + Size: 4, + }, freeSlots[0]) + + pageData := make([]byte, len(p.data)) + copy(pageData, p.data) + + anotherCell := RecordCell{ + cell: cell{ + key: []byte("large key"), + }, + record: []byte("way too large record"), + } + err = p.StoreRecordCell(anotherCell) + assert.Equal(page.ErrPageFull, err) + assert.Equal(pageData, p.data) // page must not have been modified +} + func TestPage_offsets(t *testing.T) { assert := assert.New(t) From 80895afc5e41379b4246eee2c01b8964dff2dbb2 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 19 Jun 2020 08:59:20 +0200 Subject: [PATCH 19/77] Rename offset to slot --- internal/engine/storage/page/v1/page.go | 102 +++++++++--------- internal/engine/storage/page/v1/page_test.go | 64 +++++++++-- .../storage/page/v1/{offset.go => slot.go} | 19 ++-- 3 files changed, 119 insertions(+), 66 deletions(-) rename internal/engine/storage/page/v1/{offset.go => slot.go} (59%) diff --git a/internal/engine/storage/page/v1/page.go b/internal/engine/storage/page/v1/page.go index 2a379443..71c42a85 100644 --- a/internal/engine/storage/page/v1/page.go +++ b/internal/engine/storage/page/v1/page.go @@ -99,13 +99,13 @@ func (p *Page) DeleteCell(key []byte) (bool, error) { } // delete offset - p.zero(offsetIndex*OffsetSize, OffsetSize) + p.zero(offsetIndex*SlotByteSize, SlotByteSize) // delete cell data p.zero(cellOffset.Offset, cellOffset.Size) // close gap in offsets due to offset deletion - from := offsetIndex*OffsetSize + OffsetSize // lower bound, right next to gap - to := p.CellCount() * OffsetSize // upper bound of the offset data - p.moveAndZero(from, to-from, from-OffsetSize) // actually move the data + from := offsetIndex*SlotByteSize + SlotByteSize // lower bound, right next to gap + to := p.CellCount() * SlotByteSize // upper bound of the offset data + p.moveAndZero(from, to-from, from-SlotByteSize) // actually move the data // update cell count p.decrementCellCount(1) return true, nil @@ -122,7 +122,7 @@ func (p *Page) Cell(key []byte) (page.Cell, bool) { // of them. The returned cells do not point back to the original page data, so // don't modify them. Instead, delete the old cell and store a new one. func (p *Page) Cells() (result []page.Cell) { - for _, offset := range p.Offsets() { + for _, offset := range p.OccupiedSlots() { result = append(result, decodeCell(p.data[offset.Offset:offset.Offset+offset.Size])) } return @@ -136,30 +136,30 @@ func (p *Page) RawData() []byte { return cp } -// Offsets returns all offsets in the page. The offsets can be used to find all -// cells in the page. The amount of offsets will always be equal to the amount -// of cells stored in a page. The amount of offsets in the page depends on the -// cell count of this page, not the other way around. -func (p *Page) Offsets() (result []Offset) { +// OccupiedSlots returns all occupied slots in the page. The slots all point to +// cells in the page. The amount of slots will always be equal to the amount of +// cells stored in a page. The amount of slots in the page depends on the cell +// count of this page, not the other way around. +func (p *Page) OccupiedSlots() (result []Slot) { cellCount := p.CellCount() - offsetsWidth := cellCount * OffsetSize + offsetsWidth := cellCount * SlotByteSize offsetData := p.data[HeaderSize : HeaderSize+offsetsWidth] for i := uint16(0); i < cellCount; i++ { - result = append(result, decodeOffset(offsetData[i*OffsetSize:i*OffsetSize+OffsetSize])) + result = append(result, decodeOffset(offsetData[i*SlotByteSize:i*SlotByteSize+SlotByteSize])) } return } // FreeSlots computes all free addressable cell slots in this page. -func (p *Page) FreeSlots() (result []Offset) { - offsets := p.Offsets() +func (p *Page) FreeSlots() (result []Slot) { + offsets := p.OccupiedSlots() if len(offsets) == 0 { // if there are no offsets at all, that means that the page is empty, // and one slot is returned, which reaches from 0+OffsetSize until the // end of the page - off := HeaderSize + OffsetSize - return []Offset{{ - Offset: HeaderSize + OffsetSize, + off := HeaderSize + SlotByteSize + return []Slot{{ + Offset: HeaderSize + SlotByteSize, Size: uint16(len(p.data)) - off, }} } @@ -168,10 +168,10 @@ func (p *Page) FreeSlots() (result []Offset) { return offsets[i].Offset < offsets[j].Offset }) // first slot, from end of offset data until first cell - firstOff := HeaderSize + uint16(len(offsets)+1)*OffsetSize // +1 because we always need space to store one more offset, so if that space is blocked, there is no free slot that is addressable + firstOff := HeaderSize + uint16(len(offsets)+1)*SlotByteSize // +1 because we always need space to store one more offset, so if that space is blocked, there is no free slot that is addressable firstSize := offsets[0].Offset - firstOff if firstSize > 0 { - result = append(result, Offset{ + result = append(result, Slot{ Offset: firstOff, Size: firstSize, }) @@ -181,7 +181,7 @@ func (p *Page) FreeSlots() (result []Offset) { off := offsets[i].Offset + offsets[i].Size size := offsets[i+1].Offset - off if size > 0 { - result = append(result, Offset{ + result = append(result, Slot{ Offset: off, Size: size, }) @@ -190,6 +190,25 @@ func (p *Page) FreeSlots() (result []Offset) { return } +// findFreeSlotForSize searches for a free slot in this page, matching or +// exceeding the given data size. This is done by using a best-fit algorithm. +func (p *Page) FindFreeSlotForSize(dataSize uint16) (Slot, bool) { + // sort all free slots by size + slots := p.FreeSlots() + sort.Slice(slots, func(i, j int) bool { + return slots[i].Size < slots[j].Size + }) + // search for the best fitting slot, i.e. the first slot, whose size is greater + // than or equal to the given data size + index := sort.Search(len(slots), func(i int) bool { + return slots[i].Size >= dataSize + }) + if index == len(slots) { + return Slot{}, false + } + return slots[index], true +} + func load(data []byte) (*Page, error) { if len(data) > int(^uint16(0))-1 { return nil, fmt.Errorf("page size too large: %v (max %v)", len(data), int(^uint16(0))-1) @@ -208,14 +227,14 @@ func load(data []byte) (*Page, error) { // cell with the given key could be found. If no cell could be found, // found=false will be returned, as well as zero values for all other return // arguments. -func (p *Page) findCell(key []byte) (offsetIndex uint16, cellOffset Offset, cell Cell, found bool) { - offsets := p.Offsets() +func (p *Page) findCell(key []byte) (offsetIndex uint16, cellSlot Slot, cell Cell, found bool) { + offsets := p.OccupiedSlots() result := sort.Search(len(offsets), func(i int) bool { cell := p.cellAt(offsets[i]) return bytes.Compare(cell.Key(), key) >= 0 }) if result == len(offsets) { - return 0, Offset{}, nil, false + return 0, Slot{}, nil, false } return uint16(result), offsets[result], p.cellAt(offsets[result]), true } @@ -230,12 +249,12 @@ func (p *Page) storeRecordCell(cell RecordCell) error { func (p *Page) storeRawCell(key, rawCell []byte) error { size := uint16(len(rawCell)) - slot, ok := p.findFreeSlotForSize(size) + slot, ok := p.FindFreeSlotForSize(size) if !ok { return page.ErrPageFull } copy(p.data[slot.Offset+slot.Size-size:], rawCell) - p.storeCellOffset(Offset{ + p.storeCellOffset(Slot{ Offset: slot.Offset + slot.Size - size, Size: size, }, key) @@ -243,8 +262,8 @@ func (p *Page) storeRawCell(key, rawCell []byte) error { return nil } -func (p *Page) storeCellOffset(offset Offset, cellKey []byte) { - offsets := p.Offsets() +func (p *Page) storeCellOffset(offset Slot, cellKey []byte) { + offsets := p.OccupiedSlots() if len(offsets) == 0 { // directly into the start of the page content, after the header offset.encodeInto(p.data[HeaderSize:]) @@ -256,33 +275,14 @@ func (p *Page) storeCellOffset(offset Offset, cellKey []byte) { }) // make room if neccessary - offsetOffset := uint16(index) * OffsetSize - allOffsetsEnd := uint16(len(offsets)) * OffsetSize - p.moveAndZero(offsetOffset, allOffsetsEnd-offsetOffset, offsetOffset+OffsetSize) + offsetOffset := uint16(index) * SlotByteSize + allOffsetsEnd := uint16(len(offsets)) * SlotByteSize + p.moveAndZero(offsetOffset, allOffsetsEnd-offsetOffset, offsetOffset+SlotByteSize) offset.encodeInto(p.data[offsetOffset:]) } -// findFreeSlotForSize searches for a free slot in this page, matching or -// exceeding the given data size. This is done by using a best-fit algorithm. -func (p *Page) findFreeSlotForSize(dataSize uint16) (Offset, bool) { - // sort all free slots by size - slots := p.FreeSlots() - sort.Slice(slots, func(i, j int) bool { - return slots[i].Size < slots[j].Size - }) - // search for the best fitting slot, i.e. the first slot, whose size is greater - // than or equal to the given data size - index := sort.Search(len(slots), func(i int) bool { - return slots[i].Size >= dataSize - }) - if index == len(slots) { - return Offset{}, false - } - return slots[index], true -} - -func (p *Page) cellAt(offset Offset) Cell { - return decodeCell(p.data[offset.Offset : offset.Offset+offset.Size]) +func (p *Page) cellAt(slot Slot) Cell { + return decodeCell(p.data[slot.Offset : slot.Offset+slot.Size]) } // moveAndZero moves target bytes in the page's raw data from offset to target, diff --git a/internal/engine/storage/page/v1/page_test.go b/internal/engine/storage/page/v1/page_test.go index 2a7168b6..d7866f8f 100644 --- a/internal/engine/storage/page/v1/page_test.go +++ b/internal/engine/storage/page/v1/page_test.go @@ -38,7 +38,7 @@ func TestPage_StoreRecordCell(t *testing.T) { assert.Len(freeSlots, 1) // offset must skipt reserved space for offset, as the offset is not free // space - assert.Equal(Offset{ + assert.Equal(Slot{ Offset: 18, Size: 4, }, freeSlots[0]) @@ -57,6 +57,43 @@ func TestPage_StoreRecordCell(t *testing.T) { assert.Equal(pageData, p.data) // page must not have been modified } +func TestPage_StoreRecordCell_Multiple(t *testing.T) { + assert := assert.New(t) + + p, err := load(make([]byte, 64)) + assert.NoError(err) + + cells := []RecordCell{ + { + cell: cell{ + key: []byte{0x11}, + }, + record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + }, + { + cell: cell{ + key: []byte{0x33}, + }, + record: []byte{0xD1, 0xCE}, + }, + { + cell: cell{ + key: []byte{0x22}, + }, + record: []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + }, + } + assert.NoError(p.storeRecordCell(cells[0])) + assert.NoError(p.storeRecordCell(cells[1])) + assert.NoError(p.storeRecordCell(cells[2])) + assert.Equal([]byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, // header + 0x00, 0x3C, 0x00, 0x04, // offset #0 + 0x00, 0x3A, 0x00, 0x10, // offset #2 + 0x00, 0x3A, 0x00, 0x02, // offset #1 + }, p.data) +} + func TestPage_offsets(t *testing.T) { assert := assert.New(t) @@ -78,7 +115,7 @@ func TestPage_offsets(t *testing.T) { 0xAB, 0xBC, // size } // quick check if we made a mistake in the test - assert.EqualValues(OffsetSize, len(offsetData)/offsetCount) + assert.EqualValues(SlotByteSize, len(offsetData)/offsetCount) // inject the offset data p.incrementCellCount(3) // set the cell count @@ -86,17 +123,17 @@ func TestPage_offsets(t *testing.T) { // actual test can start - offsets := p.Offsets() + offsets := p.OccupiedSlots() assert.Len(offsets, 3) - assert.Equal(Offset{ + assert.Equal(Slot{ Offset: 0x0112, Size: 0x2334, }, offsets[0]) - assert.Equal(Offset{ + assert.Equal(Slot{ Offset: 0x4556, Size: 0x6778, }, offsets[1]) - assert.Equal(Offset{ + assert.Equal(Slot{ Offset: 0x899A, Size: 0xABBC, }, offsets[2]) @@ -205,3 +242,18 @@ func TestPage_moveAndZero(t *testing.T) { }) } } + +func TestPage_FindFreeSlotForSize(t *testing.T) { + tests := []struct { + name string + offsets []Slot + want Slot + want1 bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + }) + } +} diff --git a/internal/engine/storage/page/v1/offset.go b/internal/engine/storage/page/v1/slot.go similarity index 59% rename from internal/engine/storage/page/v1/offset.go rename to internal/engine/storage/page/v1/slot.go index 44532daa..194175f4 100644 --- a/internal/engine/storage/page/v1/offset.go +++ b/internal/engine/storage/page/v1/slot.go @@ -3,13 +3,14 @@ package v1 import "unsafe" const ( - // OffsetSize is the size of an Offset, in the Go memory layout as well as + // SlotByteSize is the size of an Offset, in the Go memory layout as well as // in the serialized form. - OffsetSize = uint16(unsafe.Sizeof(Offset{})) // #nosec + SlotByteSize = uint16(unsafe.Sizeof(Slot{})) // #nosec ) -// Offset represents a cell Offset in the page data. -type Offset struct { +// Slot represents a data slot in the page. A slot consists of an offset and a +// size. +type Slot struct { // Offset is the Offset of the data in the page data slice. If overflow page // support is added, this might need to be changed to an uint32. Offset uint16 @@ -19,17 +20,17 @@ type Offset struct { Size uint16 } -func (o Offset) encodeInto(target []byte) { +func (s Slot) encodeInto(target []byte) { /* very simple way to avoid a new 4 byte allocation, should probably also be applied to cells */ _ = target[3] - byteOrder.PutUint16(target[0:], o.Offset) - byteOrder.PutUint16(target[2:], o.Size) + byteOrder.PutUint16(target[0:], s.Offset) + byteOrder.PutUint16(target[2:], s.Size) } -func decodeOffset(data []byte) Offset { +func decodeOffset(data []byte) Slot { _ = data[3] - return Offset{ + return Slot{ Offset: byteOrder.Uint16(data[0:]), Size: byteOrder.Uint16(data[2:]), } From b9c1e9941a0915bf35e10494388624436ac0ab2d Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 19 Jun 2020 17:34:28 +0200 Subject: [PATCH 20/77] Make store functionality work (with tests) --- internal/engine/storage/page/v1/page.go | 20 ++-- internal/engine/storage/page/v1/page_test.go | 108 ++++++++++++++++--- 2 files changed, 103 insertions(+), 25 deletions(-) diff --git a/internal/engine/storage/page/v1/page.go b/internal/engine/storage/page/v1/page.go index 71c42a85..955ca7a4 100644 --- a/internal/engine/storage/page/v1/page.go +++ b/internal/engine/storage/page/v1/page.go @@ -190,7 +190,7 @@ func (p *Page) FreeSlots() (result []Slot) { return } -// findFreeSlotForSize searches for a free slot in this page, matching or +// FindFreeSlotForSize searches for a free slot in this page, matching or // exceeding the given data size. This is done by using a best-fit algorithm. func (p *Page) FindFreeSlotForSize(dataSize uint16) (Slot, bool) { // sort all free slots by size @@ -253,16 +253,16 @@ func (p *Page) storeRawCell(key, rawCell []byte) error { if !ok { return page.ErrPageFull } - copy(p.data[slot.Offset+slot.Size-size:], rawCell) - p.storeCellOffset(Slot{ + p.storeCellSlot(Slot{ Offset: slot.Offset + slot.Size - size, Size: size, }, key) + copy(p.data[slot.Offset+slot.Size-size:], rawCell) p.incrementCellCount(1) return nil } -func (p *Page) storeCellOffset(offset Slot, cellKey []byte) { +func (p *Page) storeCellSlot(offset Slot, cellKey []byte) { offsets := p.OccupiedSlots() if len(offsets) == 0 { // directly into the start of the page content, after the header @@ -271,13 +271,15 @@ func (p *Page) storeCellOffset(offset Slot, cellKey []byte) { } index := sort.Search(len(offsets), func(i int) bool { - return bytes.Compare(cellKey, p.cellAt(offsets[i]).Key()) > 0 + return bytes.Compare(cellKey, p.cellAt(offsets[i]).Key()) < 0 }) - // make room if neccessary - offsetOffset := uint16(index) * SlotByteSize - allOffsetsEnd := uint16(len(offsets)) * SlotByteSize - p.moveAndZero(offsetOffset, allOffsetsEnd-offsetOffset, offsetOffset+SlotByteSize) + offsetOffset := HeaderSize + uint16(index)*SlotByteSize + if index != len(offsets) { + // make room if neccessary + allOffsetsEnd := HeaderSize + uint16(len(offsets))*SlotByteSize + p.moveAndZero(offsetOffset, allOffsetsEnd-offsetOffset, offsetOffset+SlotByteSize) + } offset.encodeInto(p.data[offsetOffset:]) } diff --git a/internal/engine/storage/page/v1/page_test.go b/internal/engine/storage/page/v1/page_test.go index d7866f8f..8654bdc5 100644 --- a/internal/engine/storage/page/v1/page_test.go +++ b/internal/engine/storage/page/v1/page_test.go @@ -80,21 +80,40 @@ func TestPage_StoreRecordCell_Multiple(t *testing.T) { cell: cell{ key: []byte{0x22}, }, - record: []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + record: []byte{0xFF}, }, } assert.NoError(p.storeRecordCell(cells[0])) assert.NoError(p.storeRecordCell(cells[1])) assert.NoError(p.storeRecordCell(cells[2])) assert.Equal([]byte{ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, // header - 0x00, 0x3C, 0x00, 0x04, // offset #0 - 0x00, 0x3A, 0x00, 0x10, // offset #2 - 0x00, 0x3A, 0x00, 0x02, // offset #1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // header + 0x00, 0x32, 0x00, 0x0E, // offset #0 + 0x00, 0x1B, 0x00, 0x0B, // offset #2 + 0x00, 0x26, 0x00, 0x0C, // offset #1 + 0x00, 0x00, 0x00, 0x00, 0x00, // free space + // cell #3 + 0x01, // cell #3 type + 0x00, 0x00, 0x00, 0x01, // cell #3 key frame + 0x22, // cell #3 key + 0x00, 0x00, 0x00, 0x01, // cell #3 record frame + 0xFF, // cell #3 record + // cell #2 + 0x01, // cell #2 type + 0x00, 0x00, 0x00, 0x01, // cell #2 key frame + 0x33, // cell #2 key + 0x00, 0x00, 0x00, 0x02, // cell #2 record frame + 0xD1, 0xCE, // cell #2 record + // cell #1 + 0x01, // cell #1 type + 0x00, 0x00, 0x00, 0x01, // cell #1 key frame + 0x11, // cell #1 key + 0x00, 0x00, 0x00, 0x04, // cell #1 record frame + 0xCA, 0xFE, 0xBA, 0xBE, // cell #1 record }, p.data) } -func TestPage_offsets(t *testing.T) { +func TestPage_OccupiedSlots(t *testing.T) { assert := assert.New(t) // create a completely empty page @@ -244,16 +263,73 @@ func TestPage_moveAndZero(t *testing.T) { } func TestPage_FindFreeSlotForSize(t *testing.T) { - tests := []struct { - name string - offsets []Slot - want Slot - want1 bool - }{ - // TODO: Add test cases. + assert := assert.New(t) + + p, err := load(make([]byte, 100)) + assert.NoError(err) + + occupiedSlots := []Slot{ + {90, 10}, + // 1 byte + {80, 9}, + // 25 bytes + {50, 5}, + // 10 bytes + {30, 10}, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - }) + + for i, slot := range occupiedSlots { + slot.encodeInto(p.data[HeaderSize+i*int(SlotByteSize):]) + } + p.incrementCellCount(uint16(len(occupiedSlots))) + + slot, ok := p.FindFreeSlotForSize(1) + assert.True(ok) + assert.Equal(Slot{89, 1}, slot) + + slot, ok = p.FindFreeSlotForSize(15) + assert.True(ok) + assert.Equal(Slot{55, 25}, slot) + + slot, ok = p.FindFreeSlotForSize(25) + assert.True(ok) + assert.Equal(Slot{55, 25}, slot) + + slot, ok = p.FindFreeSlotForSize(10) + assert.True(ok) + assert.Equal(Slot{40, 10}, slot) + + slot, ok = p.FindFreeSlotForSize(5) + assert.True(ok) + assert.Equal(Slot{40, 10}, slot) +} + +func TestPage_FreeSlots(t *testing.T) { + assert := assert.New(t) + + p, err := load(make([]byte, 100)) + assert.NoError(err) + + occupiedSlots := []Slot{ + // 2 bytes + {32, 8}, + // 10 bytes + {50, 5}, + // 25 bytes + {80, 9}, + // 1 byte + {90, 10}, } + + for i, slot := range occupiedSlots { + slot.encodeInto(p.data[HeaderSize+i*int(SlotByteSize):]) + } + p.incrementCellCount(uint16(len(occupiedSlots))) + + assert.EqualValues([]Slot{ + {30, 2}, + {40, 10}, + {55, 25}, + {89, 1}, + }, p.FreeSlots()) } From 5f5952fabd4679c62f334ec272ccdf09f5f5edc4 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 19 Jun 2020 17:53:14 +0200 Subject: [PATCH 21/77] Add a page loader --- internal/engine/storage/page/loader.go | 10 +++++++ internal/engine/storage/page/page.go | 4 --- internal/engine/storage/page/v1/loader.go | 33 +++++++++++++++++++++++ internal/engine/storage/page/v1/page.go | 1 - 4 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 internal/engine/storage/page/loader.go create mode 100644 internal/engine/storage/page/v1/loader.go diff --git a/internal/engine/storage/page/loader.go b/internal/engine/storage/page/loader.go new file mode 100644 index 00000000..2804abd7 --- /dev/null +++ b/internal/engine/storage/page/loader.go @@ -0,0 +1,10 @@ +package page + +// Loader describes a component that is used to load pages from files. The +// manager can infer the exact location of each page from its page ID. Depending +// on the implementation, the manager might have to be fed with the whole +// database. +type Loader interface { + // Load retrieves the page with the given ID from secondary storage. + Load(id ID) (Page, error) +} diff --git a/internal/engine/storage/page/page.go b/internal/engine/storage/page/page.go index fc22ff38..c9b87459 100644 --- a/internal/engine/storage/page/page.go +++ b/internal/engine/storage/page/page.go @@ -11,10 +11,6 @@ const ( HeaderID ) -// Loader is a function that can load a page from a given byte slice, and return -// errors if any occur. -type Loader func([]byte) (Page, error) - // ID is the type of a page ID. This is mainly to avoid any confusion. // Changing this will break existing database files, so only change during major // version upgrades. diff --git a/internal/engine/storage/page/v1/loader.go b/internal/engine/storage/page/v1/loader.go new file mode 100644 index 00000000..479f0aae --- /dev/null +++ b/internal/engine/storage/page/v1/loader.go @@ -0,0 +1,33 @@ +package v1 + +import ( + "github.com/spf13/afero" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +var _ page.Loader = (*Loader)(nil) + +// Loader is the v1 implementation of a page.Loader, used to retrieve pages from +// secondary storage. +type Loader struct { + fs afero.Fs +} + +// NewLoader creates a new, ready to use loader. If during initialization, an +// error occurs, the error will be returned. It may be wrapped. +func NewLoader(fs afero.Fs) (*Loader, error) { + l := &Loader{ + fs: fs, + } + // TODO: add initialization if needed + return l, nil +} + +// Load loads the page with the given ID from the database files. +func (l *Loader) Load(id page.ID) (page.Page, error) { + return l.load(id) +} + +func (l *Loader) load(id page.ID) (*Page, error) { + return nil, nil +} diff --git a/internal/engine/storage/page/v1/page.go b/internal/engine/storage/page/v1/page.go index 955ca7a4..1daa18dc 100644 --- a/internal/engine/storage/page/v1/page.go +++ b/internal/engine/storage/page/v1/page.go @@ -10,7 +10,6 @@ import ( ) var _ page.Page = (*Page)(nil) -var _ page.Loader = Load const ( // PageSize is the fix size of a page, which is 16KB or 16384 bytes. From b53750a09435220bdeb57fc699896d67dad05e1f Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Mon, 22 Jun 2020 16:57:49 +0200 Subject: [PATCH 22/77] Implement and specify part of the file format --- doc/file-format.md | 47 ++ .../engine/storage/{page => cache}/cache.go | 14 +- internal/engine/storage/cache/lru.go | 144 ++++++ internal/engine/storage/db.go | 171 +++++++ internal/engine/storage/db_test.go | 65 +++ internal/engine/storage/doc.go | 3 + internal/engine/storage/error.go | 11 + internal/engine/storage/option.go | 17 + internal/engine/storage/page/{v1 => }/cell.go | 83 ++-- .../engine/storage/page/{v1 => }/cell_test.go | 48 +- internal/engine/storage/page/loader.go | 10 - internal/engine/storage/page/page.go | 425 ++++++++++++++---- .../engine/storage/page/{v1 => }/page_test.go | 71 ++- internal/engine/storage/page/{v1 => }/slot.go | 2 +- .../engine/storage/page/v1/celltype_string.go | 25 -- internal/engine/storage/page/v1/doc.go | 13 - internal/engine/storage/page/v1/loader.go | 33 -- internal/engine/storage/page/v1/page.go | 343 -------------- internal/engine/storage/page_manager.go | 63 +++ internal/engine/storage/storage.go | 7 +- internal/engine/storage/validator.go | 56 +++ 21 files changed, 1010 insertions(+), 641 deletions(-) create mode 100644 doc/file-format.md rename internal/engine/storage/{page => cache}/cache.go (86%) create mode 100644 internal/engine/storage/cache/lru.go create mode 100644 internal/engine/storage/db.go create mode 100644 internal/engine/storage/db_test.go create mode 100644 internal/engine/storage/doc.go create mode 100644 internal/engine/storage/error.go create mode 100644 internal/engine/storage/option.go rename internal/engine/storage/page/{v1 => }/cell.go (54%) rename internal/engine/storage/page/{v1 => }/cell_test.go (88%) delete mode 100644 internal/engine/storage/page/loader.go rename internal/engine/storage/page/{v1 => }/page_test.go (86%) rename internal/engine/storage/page/{v1 => }/slot.go (98%) delete mode 100644 internal/engine/storage/page/v1/celltype_string.go delete mode 100644 internal/engine/storage/page/v1/doc.go delete mode 100644 internal/engine/storage/page/v1/loader.go delete mode 100644 internal/engine/storage/page/v1/page.go create mode 100644 internal/engine/storage/page_manager.go create mode 100644 internal/engine/storage/validator.go diff --git a/doc/file-format.md b/doc/file-format.md new file mode 100644 index 00000000..c5f677ab --- /dev/null +++ b/doc/file-format.md @@ -0,0 +1,47 @@ +# File Format +This document describes the v1.x file format of a database. + +## Terms +This section will quickly describe the terms, that will be used throughout this +document. + +A **database** is a single file, that holds all information of a single +database. + +## Format +The database is a single file. Its size is a multiple of the page size, which is +16K or 16384 bytes for v1.x. The file consists of pages only, meaning there is +no fixed size header, only a header page (and maybe overflow pages). + +### Header page +The page with the **ID 0** is the header page of the whole database. It holds +values for the following keys. The keys are given as strings, the actual key +bytes are the UTF-8 encoding of that string. + +* `pageCount` is a record cell whose entry is an 8 byte big endian unsigned + integer, representing the amount of pages stored in the file. +* `tables` is a pointer cell which points to a page, that contains pointers to + all tables that are stored in this database. The format of the table pages is explained in the next section. + +### Table pages +Table pages do not directly hold data of a table. Instead, they hold pointers to +pages, that do, i.e. the index and data page. Table pages do however hold +information about the table schema. The schema information is a single record +that is to be interpreted as a schema (**TODO: +schemas**). + +The keys of the three values, index page, data page and schema are as follows. + +* `schema` is a record cell containing the schema information about this table. + That is, columns, column types, references, triggers etc. How the schema + information is to be interpreted, is explained [here](#)(**TODO: schemas**). +* `index` is a pointer cell pointing to the index page of this table. The index + page contains pages that are used as nodes in a btree. See more + [here](#index-pages) +* `data` is a pointer cell pointing to the data page of this table. See more + [here](#data-pages) + +### Index pages + +### Data pages \ No newline at end of file diff --git a/internal/engine/storage/page/cache.go b/internal/engine/storage/cache/cache.go similarity index 86% rename from internal/engine/storage/page/cache.go rename to internal/engine/storage/cache/cache.go index 9d748295..b5af39b7 100644 --- a/internal/engine/storage/page/cache.go +++ b/internal/engine/storage/cache/cache.go @@ -1,6 +1,10 @@ -package page +package cache -import "io" +import ( + "io" + + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) // Cache describes a page cache that caches pages from a secondary storage. type Cache interface { @@ -10,16 +14,16 @@ type Cache interface { // evicted. After working with the page, it must be released again, in order // for the cache to be able to free memory. If a page with the given id does // not exist, an error will be returned. - FetchAndPin(id ID) (Page, error) + FetchAndPin(id page.ID) (*page.Page, error) // Unpin tells the cache that the page with the given id is no longer // required directly, and that it can be evicted. Unpin is not a guarantee // that the page will be evicted. The cache determines, when to evict a // page. If a page with that id does not exist, this call is a no-op. - Unpin(id ID) + Unpin(id page.ID) // Flush writes the contents of the page with the given id to the configured // source. Before a page is evicted, it is always flushed. Use this method // to tell the cache that the page must be flushed immediately. If a page // with the given id does not exist, an error will be returned. - Flush(id ID) error + Flush(id page.ID) error io.Closer } diff --git a/internal/engine/storage/cache/lru.go b/internal/engine/storage/cache/lru.go new file mode 100644 index 00000000..16f2c35f --- /dev/null +++ b/internal/engine/storage/cache/lru.go @@ -0,0 +1,144 @@ +package cache + +import ( + "fmt" + + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +type SecondaryStorage interface { + ReadPage(page.ID) (*page.Page, error) + WritePage(*page.Page) error +} + +var _ Cache = (*LRUCache)(nil) + +type LRUCache struct { + store SecondaryStorage + pages map[uint32]*page.Page + pinned map[uint32]struct{} + size int + lru []uint32 +} + +func NewLRUCache(size int, store SecondaryStorage) *LRUCache { + return &LRUCache{ + store: store, + pages: make(map[uint32]*page.Page), + pinned: make(map[uint32]struct{}), + size: size, + lru: make([]uint32, size), + } +} + +func (c *LRUCache) FetchAndPin(id page.ID) (*page.Page, error) { + return c.fetchAndPin(id) +} + +func (c *LRUCache) Unpin(id page.ID) { + c.unpin(id) +} + +func (c *LRUCache) Flush(id page.ID) error { + return c.flush(id) +} + +func (c *LRUCache) Close() error { + return nil +} + +func (c *LRUCache) fetchAndPin(id page.ID) (*page.Page, error) { + // pin id first in order to avoid potential concurrent eviction at this + // point + c.pin(id) + p, err := c.fetch(id) + if err != nil { + // unpin if a page with the given id cannot be loaded + c.unpin(id) + return nil, err + } + return p, nil +} + +func (c *LRUCache) pin(id uint32) { + c.pinned[id] = struct{}{} +} + +func (c *LRUCache) unpin(id uint32) { + delete(c.pinned, id) +} + +func (c *LRUCache) fetch(id uint32) (*page.Page, error) { + // check if page is already cached + if p, ok := c.pages[id]; ok { + moveToFront(id, c.lru) + return p, nil + } + + // check if we have to evict pages + if err := c.freeMemoryIfNeeded(); err != nil { + return nil, fmt.Errorf("free mem: %w", err) + } + + // fetch page from secondary storage + p, err := c.store.ReadPage(id) + if err != nil { + return nil, fmt.Errorf("load page: %w", err) + } + // store in cache + c.pages[id] = p + // append in front + c.lru = append([]uint32{id}, c.lru...) + return p, nil +} + +func (c *LRUCache) evict(id uint32) error { + if err := c.flush(id); err != nil { + return fmt.Errorf("flush: %w", err) + } + delete(c.pages, id) + return nil +} + +func (c *LRUCache) flush(id page.ID) error { + if err := c.store.WritePage(c.pages[id]); err != nil { + return fmt.Errorf("write page: %w", err) + } + return nil +} + +func (c *LRUCache) freeMemoryIfNeeded() error { + if len(c.lru) < c.size { + return nil + } + for i := len(c.lru) - 1; i >= 0; i-- { + id := c.lru[i] + if _, ok := c.pinned[id]; ok { + // can't evict current page, pinned + continue + } + c.lru = c.lru[:len(c.lru)-1] + return c.evict(id) + } + return fmt.Errorf("all pages pinned, cache is full") +} + +func moveToFront(needle page.ID, haystack []page.ID) { + if len(haystack) == 0 || haystack[0] == needle { + return + } + var prev page.ID + for i, elem := range haystack { + switch { + case i == 0: + haystack[0] = needle + prev = elem + case elem == needle: + haystack[i] = prev + return + default: + haystack[i] = prev + prev = elem + } + } +} diff --git a/internal/engine/storage/db.go b/internal/engine/storage/db.go new file mode 100644 index 00000000..c0d20180 --- /dev/null +++ b/internal/engine/storage/db.go @@ -0,0 +1,171 @@ +package storage + +import ( + "encoding/binary" + "fmt" + "sync/atomic" + "unsafe" + + "github.com/rs/zerolog" + "github.com/spf13/afero" + "github.com/tomarrell/lbadd/internal/engine/storage/cache" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +const ( + // DefaultCacheSize is the default amount of pages, that the cache can hold + // at most. Current limit is 256, which, regarding the current page size of + // 16K, means, that the maximum size that a full cache will occupy, is 4M + // (CacheSize * page.Size). + DefaultCacheSize = 1 << 8 + + HeaderPageID page.ID = 0 + + HeaderTables = "tables" + HeaderPageCount = "pageCount" +) + +var ( + byteOrder = binary.BigEndian +) + +type DBFile struct { + closed uint32 + + log zerolog.Logger + cacheSize int + + file afero.File + pageManager *PageManager + cache cache.Cache + + headerPage *page.Page +} + +func Create(file afero.File, opts ...Option) (*DBFile, error) { + if _, err := file.Stat(); err != nil { + return nil, fmt.Errorf("stat: %w", err) + } + + mgr, err := NewPageManager(file) + if err != nil { + return nil, fmt.Errorf("new page manager: %w", err) + } + + headerPage, err := mgr.AllocateNew() + if err != nil { + return nil, fmt.Errorf("allocate header page: %w", err) + } + tablesPage, err := mgr.AllocateNew() + if err != nil { + return nil, fmt.Errorf("allocate tables page: %w", err) + } + + headerPage.StoreRecordCell(page.RecordCell{ + Key: []byte(HeaderPageCount), + Record: encodeUint64(2), // header and tables page + }) + headerPage.StorePointerCell(page.PointerCell{ + Key: []byte(HeaderTables), + Pointer: tablesPage.ID(), + }) + + err = mgr.WritePage(headerPage) // immediately flush + if err != nil { + return nil, fmt.Errorf("write header page: %w", err) + } + err = mgr.WritePage(tablesPage) // immediately flush + if err != nil { + return nil, fmt.Errorf("write tables page: %w", err) + } + + return newDB(file, mgr, headerPage, opts...), nil +} + +func Open(file afero.File, opts ...Option) (*DBFile, error) { + if err := NewValidator(file).Validate(); err != nil { + return nil, fmt.Errorf("validate: %w", err) + } + + mgr, err := NewPageManager(file) + if err != nil { + return nil, fmt.Errorf("new page manager: %w", err) + } + + headerPage, err := mgr.ReadPage(HeaderPageID) + if err != nil { + return nil, fmt.Errorf("read header page: %w", err) + } + + return newDB(file, mgr, headerPage, opts...), nil +} + +func (db *DBFile) AllocateNewPage() (*page.Page, error) { + if db.Closed() { + return nil, ErrClosed + } + + page, err := db.pageManager.AllocateNew() + if err != nil { + return nil, fmt.Errorf("allocate new: %w", err) + } + if err := db.incrementHeaderPageCount(); err != nil { + return nil, fmt.Errorf("increment header page count: %w", err) + } + if err := db.pageManager.WritePage(db.headerPage); err != nil { + return nil, fmt.Errorf("write header page: %w", err) + } + return page, nil +} + +func (db *DBFile) Cache() cache.Cache { + if db.Closed() { + return nil + } + return db.cache +} + +func (db *DBFile) Close() error { + _ = db.cache.Close() + _ = db.file.Close() + atomic.StoreUint32(&db.closed, 1) + return nil +} + +func (db *DBFile) Closed() bool { + return atomic.LoadUint32(&db.closed) == 1 +} + +func newDB(file afero.File, mgr *PageManager, headerPage *page.Page, opts ...Option) *DBFile { + db := &DBFile{ + log: zerolog.Nop(), + cacheSize: DefaultCacheSize, + + file: file, + pageManager: mgr, + headerPage: headerPage, + } + for _, opt := range opts { + opt(db) + } + + db.cache = cache.NewLRUCache(db.cacheSize, mgr) + + return db +} + +func (db *DBFile) incrementHeaderPageCount() error { + val, ok := db.headerPage.Cell([]byte(HeaderPageCount)) + if !ok { + return fmt.Errorf("no page count header field") + } + cell := val.(page.RecordCell) + byteOrder.PutUint64(cell.Record, byteOrder.Uint64(cell.Record)+1) + return nil +} + +func encodeUint64(v uint64) []byte { + buf := make([]byte, unsafe.Sizeof(v)) + byteOrder.PutUint64(buf, v) + return buf +} diff --git a/internal/engine/storage/db_test.go b/internal/engine/storage/db_test.go new file mode 100644 index 00000000..cdbb0071 --- /dev/null +++ b/internal/engine/storage/db_test.go @@ -0,0 +1,65 @@ +package storage + +import ( + "io" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +func TestCreate(t *testing.T) { + assert := assert.New(t) + fs := afero.NewMemMapFs() + + f, err := fs.Create("mydbfile") + assert.NoError(err) + + db, err := Create(f) + assert.NoError(err) + + mustHaveSize(assert, f, 2*page.Size) + + headerPage := loadPageFromOffset(assert, f, 0) + assert.EqualValues(2, headerPage.CellCount()) + assert.Equal( + encodeUint64(2), + mustCell(assert, headerPage, HeaderPageCount).(page.RecordCell).Record, + ) + + _, err = db.AllocateNewPage() + assert.NoError(err) + + mustHaveSize(assert, f, 3*page.Size) + + headerPage = loadPageFromOffset(assert, f, 0) + assert.EqualValues(2, headerPage.CellCount()) + assert.Equal( + encodeUint64(3), + mustCell(assert, headerPage, HeaderPageCount).(page.RecordCell).Record, + ) + + assert.NoError(db.Close()) +} + +func mustHaveSize(assert *assert.Assertions, file afero.File, expectedSize int64) { + stat, err := file.Stat() + assert.NoError(err) + assert.EqualValues(expectedSize, stat.Size()) +} + +func mustCell(assert *assert.Assertions, p *page.Page, key string) interface{} { + val, ok := p.Cell([]byte(key)) + assert.Truef(ok, "page must have cell with key %v", key) + return val +} + +func loadPageFromOffset(assert *assert.Assertions, rd io.ReaderAt, off int64) *page.Page { + buf := make([]byte, page.Size) + _, err := rd.ReadAt(buf, off) + assert.NoError(err) + p, err := page.Load(buf) + assert.NoError(err) + return p +} diff --git a/internal/engine/storage/doc.go b/internal/engine/storage/doc.go new file mode 100644 index 00000000..0a2ab7db --- /dev/null +++ b/internal/engine/storage/doc.go @@ -0,0 +1,3 @@ +// Package storage implements support for the file format of lbadd. Open or +// create a database file with storage.Open / storage.Create respectively. +package storage diff --git a/internal/engine/storage/error.go b/internal/engine/storage/error.go new file mode 100644 index 00000000..70f69e8b --- /dev/null +++ b/internal/engine/storage/error.go @@ -0,0 +1,11 @@ +package storage + +// Error is a sentinel error. +type Error string + +func (e Error) Error() string { return string(e) } + +// Sentinel errors. +const ( + ErrClosed Error = "already closed" +) diff --git a/internal/engine/storage/option.go b/internal/engine/storage/option.go new file mode 100644 index 00000000..9382f403 --- /dev/null +++ b/internal/engine/storage/option.go @@ -0,0 +1,17 @@ +package storage + +import "github.com/rs/zerolog" + +type Option func(*DBFile) + +func WithCacheSize(size int) Option { + return func(db *DBFile) { + db.cacheSize = size + } +} + +func WithLogger(log zerolog.Logger) Option { + return func(db *DBFile) { + db.log = log + } +} diff --git a/internal/engine/storage/page/v1/cell.go b/internal/engine/storage/page/cell.go similarity index 54% rename from internal/engine/storage/page/v1/cell.go rename to internal/engine/storage/page/cell.go index 26ced5f6..a7994c14 100644 --- a/internal/engine/storage/page/v1/cell.go +++ b/internal/engine/storage/page/cell.go @@ -1,13 +1,11 @@ -package v1 +package page import ( "bytes" - - "github.com/tomarrell/lbadd/internal/engine/storage/page" ) -var _ page.Cell = (*RecordCell)(nil) -var _ page.Cell = (*PointerCell)(nil) +var _ CellTyper = (*RecordCell)(nil) +var _ CellTyper = (*PointerCell)(nil) //go:generate stringer -type=CellType @@ -27,47 +25,32 @@ const ( ) type ( - // Cell is a cell that has a type and a key. - Cell interface { - page.Cell + CellTyper interface { Type() CellType } - cell struct { - key []byte - } - // RecordCell is a cell with CellTypeRecord. It holds a key and a variable // size record. RecordCell struct { - cell - record []byte + Key []byte + Record []byte } // PointerCell is a cell with CellTypePointer. It holds a key and an uint32, // pointing to another page. PointerCell struct { - cell - pointer page.ID + Key []byte + Pointer ID } ) -// Key returns the key of this cell. -func (c cell) Key() []byte { return c.key } - -// Record returns the record data of this cell. -func (c RecordCell) Record() []byte { return c.record } - -// Pointer returns the pointer of this page, that points to another page. -func (c PointerCell) Pointer() page.ID { return c.pointer } - // Type returns CellTypeRecord. -func (c RecordCell) Type() CellType { return CellTypeRecord } +func (RecordCell) Type() CellType { return CellTypeRecord } // Type returns CellTypePointer. -func (c PointerCell) Type() CellType { return CellTypePointer } +func (PointerCell) Type() CellType { return CellTypePointer } -func decodeCell(data []byte) Cell { +func decodeCell(data []byte) CellTyper { switch t := CellType(data[0]); t { case CellTypePointer: return decodePointerCell(data) @@ -79,8 +62,8 @@ func decodeCell(data []byte) Cell { } func encodeRecordCell(cell RecordCell) []byte { - key := frame(cell.key) - record := frame(cell.record) + key := frame(cell.Key) + record := frame(cell.Record) var buf bytes.Buffer buf.WriteByte(byte(CellTypeRecord)) @@ -91,24 +74,20 @@ func encodeRecordCell(cell RecordCell) []byte { } func decodeRecordCell(data []byte) RecordCell { - cp := copySlice(data) - - keySize := byteOrder.Uint32(cp[1:5]) - key := cp[5 : 5+keySize] - recordSize := byteOrder.Uint32(cp[5+keySize : 5+keySize+4]) - record := cp[5+keySize+4 : 5+keySize+4+recordSize] + keySize := byteOrder.Uint32(data[1:5]) + key := data[5 : 5+keySize] + recordSize := byteOrder.Uint32(data[5+keySize : 5+keySize+4]) + record := data[5+keySize+4 : 5+keySize+4+recordSize] return RecordCell{ - cell: cell{ - key: key, - }, - record: record, + Key: key, + Record: record, } } func encodePointerCell(cell PointerCell) []byte { - key := frame(cell.key) + key := frame(cell.Key) pointer := make([]byte, 4) - byteOrder.PutUint32(pointer, cell.pointer) + byteOrder.PutUint32(pointer, cell.Pointer) var buf bytes.Buffer buf.WriteByte(byte(CellTypePointer)) @@ -119,16 +98,12 @@ func encodePointerCell(cell PointerCell) []byte { } func decodePointerCell(data []byte) PointerCell { - cp := copySlice(data) - - keySize := byteOrder.Uint32(cp[1:5]) - key := cp[5 : 5+keySize] - pointer := byteOrder.Uint32(cp[5+keySize : 5+keySize+4]) + keySize := byteOrder.Uint32(data[1:5]) + key := data[5 : 5+keySize] + pointer := byteOrder.Uint32(data[5+keySize : 5+keySize+4]) return PointerCell{ - cell: cell{ - key: key, - }, - pointer: pointer, + Key: key, + Pointer: pointer, } } @@ -140,9 +115,3 @@ func frame(data []byte) []byte { byteOrder.PutUint32(result, uint32(len(data))) return result } - -func copySlice(original []byte) []byte { - copied := make([]byte, len(original)) - copy(copied, original) - return copied -} diff --git a/internal/engine/storage/page/v1/cell_test.go b/internal/engine/storage/page/cell_test.go similarity index 88% rename from internal/engine/storage/page/v1/cell_test.go rename to internal/engine/storage/page/cell_test.go index 7b3279bc..ca976fde 100644 --- a/internal/engine/storage/page/v1/cell_test.go +++ b/internal/engine/storage/page/cell_test.go @@ -1,4 +1,4 @@ -package v1 +package page import ( "testing" @@ -75,10 +75,8 @@ func Test_encodeRecordCell(t *testing.T) { { "small", RecordCell{ - cell: cell{ - key: []byte{0xD1, 0xCE}, - }, - record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + Key: []byte{0xD1, 0xCE}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, }, []byte{ byte(CellTypeRecord), // cell type @@ -118,10 +116,8 @@ func Test_encodePointerCell(t *testing.T) { { "simple", PointerCell{ - cell: cell{ - key: []byte{0xD1, 0xCE}, - }, - pointer: 0xCAFEBABE, + Key: []byte{0xD1, 0xCE}, + Pointer: 0xCAFEBABE, }, []byte{ byte(CellTypePointer), // cell type @@ -163,10 +159,8 @@ func Test_decodeRecordCell(t *testing.T) { // no record }, RecordCell{ - cell: cell{ - key: []byte{}, - }, - record: []byte{}, + Key: []byte{}, + Record: []byte{}, }, }, { @@ -179,10 +173,8 @@ func Test_decodeRecordCell(t *testing.T) { 0xCA, 0xFE, 0xBA, 0xBE, // record }, RecordCell{ - cell: cell{ - key: []byte{0xD1, 0xCE}, - }, - record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + Key: []byte{0xD1, 0xCE}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, }, }, } @@ -211,10 +203,8 @@ func Test_decodePointerCell(t *testing.T) { 0x00, 0x00, 0x00, 0x00, // pointer }, PointerCell{ - cell: cell{ - key: []byte{}, - }, - pointer: 0, + Key: []byte{}, + Pointer: 0, }, }, { @@ -226,10 +216,8 @@ func Test_decodePointerCell(t *testing.T) { 0xCA, 0xFE, 0xBA, 0xBE, // pointer }, PointerCell{ - cell: cell{ - key: []byte{0xD1, 0xCE}, - }, - pointer: 0xCAFEBABE, + Key: []byte{0xD1, 0xCE}, + Pointer: 0xCAFEBABE, }, }, } @@ -247,7 +235,7 @@ func Test_decodeCell(t *testing.T) { tests := []struct { name string data []byte - want Cell + want CellTyper }{ { "zero record cell", @@ -259,8 +247,8 @@ func Test_decodeCell(t *testing.T) { // no record }, RecordCell{ - cell: cell{key: []byte{}}, - record: []byte{}, + Key: []byte{}, + Record: []byte{}, }, }, { @@ -272,8 +260,8 @@ func Test_decodeCell(t *testing.T) { 0x00, 0x00, 0x00, 0x00, // pointer }, PointerCell{ - cell: cell{key: []byte{}}, - pointer: 0, + Key: []byte{}, + Pointer: 0, }, }, { diff --git a/internal/engine/storage/page/loader.go b/internal/engine/storage/page/loader.go deleted file mode 100644 index 2804abd7..00000000 --- a/internal/engine/storage/page/loader.go +++ /dev/null @@ -1,10 +0,0 @@ -package page - -// Loader describes a component that is used to load pages from files. The -// manager can infer the exact location of each page from its page ID. Depending -// on the implementation, the manager might have to be fed with the whole -// database. -type Loader interface { - // Load retrieves the page with the given ID from secondary storage. - Load(id ID) (Page, error) -} diff --git a/internal/engine/storage/page/page.go b/internal/engine/storage/page/page.go index c9b87459..f8bc1786 100644 --- a/internal/engine/storage/page/page.go +++ b/internal/engine/storage/page/page.go @@ -1,14 +1,27 @@ package page -// Header is an enum type for header fields. -type Header uint8 +import ( + "bytes" + "encoding/binary" + "fmt" + "sort" +) + +const ( + // Size is the fix size of a page, which is 16KB or 16384 bytes. + Size = 1 << 14 + // HeaderSize is the fix size of a page header, which is 10 bytes. + HeaderSize = 6 +) +// Header field offset in page data. const ( - // HeaderVersion is the version number, that a page is in. This is a uint16. - HeaderVersion Header = iota - // HeaderID is the page ID, which may be used outside of the page for - // housekeeping. This is a uint16. - HeaderID + idOffset = 0 // byte 1,2,3,4: byte page ID + cellCountOffset = 4 // byte 5,6: cell count +) + +var ( + byteOrder = binary.BigEndian ) // ID is the type of a page ID. This is mainly to avoid any confusion. @@ -16,79 +29,327 @@ const ( // version upgrades. type ID = uint32 -// Page describes a memory page that stores (page.Cell)s. A page consists of -// header fields and cells, and is a plain store. Obtained cells are always -// ordered ascending by the cell key. A page supports variable size cell keys -// and records. A page is generally NOT safe for concurrent writes. -type Page interface { - // Version returns the version of the page layout. Use this for choosing the - // page implementation to use to decode the page. - Version() uint32 - - // ID returns the page ID, as it is used by any page loader. It is unique in - // the scope of one database. - ID() ID - - // Dirty determines whether this page has been modified since the last time - // Page.ClearDirty was called. - Dirty() bool - // MarkDirty marks this page as dirty. - MarkDirty() - // ClearDirty unsets the dirty flag from this page. - ClearDirty() - - // StorePointerCell stores the given pointer cell in the page. - // - // If a cell with the same key as the given pointer already exists in the - // page, it will be overwritten. - // - // If a cell with the same key as the given cell does NOT already exist, it - // will be created. - // - // To change the type of a cell, delete it and store a new cell. - StorePointerCell(PointerCell) error - // StoreRecordCell stores the given record cell in the page. - // - // If a cell with the same key as the given cell already exists in the page, - // it will be overwritten. - // - // If a cell with the same key as the given pointer does NOT already exist, - // it will be created. - // - // To change the type of a cell, delete it and store a new cell. - StoreRecordCell(RecordCell) error - // Delete deletes the cell with the given bytes as key. If the key couldn't - // be found, false is returned. If an error occured during deletion, the - // error is returned. - DeleteCell([]byte) (bool, error) - // Cell returns a cell with the given key, together with a bool indicating - // whether any cell in the page has that key. Use a switch statement to - // determine which type of cell you just obtained (pointer, record). - Cell([]byte) (Cell, bool) - // Cells returns all cells in this page as a slice. Cells are ordered - // ascending by key. Calling this method can be expensive since all cells - // have to be decoded. - Cells() []Cell -} - -// Cell describes a generic page cell that holds a key. Use a switch statement -// to determine the type of the cell. -type Cell interface { - // Key returns the key of the cell. - Key() []byte -} - -// PointerCell describes a cell that points to another page in memory. -type PointerCell interface { - Cell - // Pointer returns the page ID of the child page that this cell points to. - Pointer() ID -} - -// RecordCell describes a cell that holds some kind of value. What value format -// was used is none of the cells concern, just use it as what you put in. -type RecordCell interface { - Cell - // Record is the data record in this cell, returned as a byte slice. - Record() []byte +// Page is a page implementation that does not support overflow pages. It is not +// meant for that. Since we want to separate index and data, records should not +// contain datasets, but rather enough information, to find the corresponding +// dataset in a data file. +type Page struct { + // data is the underlying data byte slice, which holds the header, offsets + // and cells. + data []byte + + dirty bool +} + +// New creates a new page with the given ID. +func New(id ID) *Page { + data := make([]byte, Size) + byteOrder.PutUint32(data[idOffset:], id) + return &Page{ + data: data, + } +} + +// Load loads the given data into the page. The length of the given data byte +// slice may differ from v1.PageSize, however, it cannot exceed ^uint16(0)-1 +// (65535 or 64KB), and must be larger than 22 (HeaderSize(=10) + 1 Offset(=4) + +// 1 empty cell(=8)). +func Load(data []byte) (*Page, error) { + return load(data) +} + +// ID returns the ID of this page. This value must be constant. +func (p *Page) ID() ID { return byteOrder.Uint32(p.data[idOffset:]) } + +// CellCount returns the amount of stored cells in this page. This value is NOT +// constant. +func (p *Page) CellCount() uint16 { return byteOrder.Uint16(p.data[cellCountOffset:]) } + +// Dirty returns whether the page is dirty (needs syncing with secondary +// storage). +func (p *Page) Dirty() bool { return p.dirty } + +// MarkDirty marks this page as dirty. +func (p *Page) MarkDirty() { p.dirty = true } + +// ClearDirty marks this page as NOT dirty. +func (p *Page) ClearDirty() { p.dirty = false } + +// StorePointerCell stores a pointer cell in this page. A pointer cell points to +// other page IDs. +func (p *Page) StorePointerCell(cell PointerCell) error { + return p.storePointerCell(cell) +} + +// StoreRecordCell stores a record cell in this page. A record cell holds +// arbitrary, variable size data. +func (p *Page) StoreRecordCell(cell RecordCell) error { + return p.storeRecordCell(cell) +} + +// DeleteCell deletes a cell with the given key. If no such cell could be found, +// false is returned. In this implementation, an error can not occur while +// deleting a cell. +func (p *Page) DeleteCell(key []byte) (bool, error) { + offsetIndex, cellOffset, _, found := p.findCell(key) + if !found { + return false, nil + } + + // delete offset + p.zero(offsetIndex*SlotByteSize, SlotByteSize) + // delete cell data + p.zero(cellOffset.Offset, cellOffset.Size) + // close gap in offsets due to offset deletion + from := offsetIndex*SlotByteSize + SlotByteSize // lower bound, right next to gap + to := p.CellCount() * SlotByteSize // upper bound of the offset data + p.moveAndZero(from, to-from, from-SlotByteSize) // actually move the data + // update cell count + p.decrementCellCount(1) + return true, nil +} + +// Cell returns a cell from this page with the given key, or false if no such +// cell exists in this page. In that case, the returned page is also nil. +func (p *Page) Cell(key []byte) (CellTyper, bool) { + _, _, cell, found := p.findCell(key) + return cell, found +} + +// Cells decodes all cells in this page, which can be expensive, and returns all +// of them. The returned cells do not point back to the original page data, so +// don't modify them. Instead, delete the old cell and store a new one. +func (p *Page) Cells() (result []CellTyper) { + for _, offset := range p.OccupiedSlots() { + result = append(result, decodeCell(p.data[offset.Offset:offset.Offset+offset.Size])) + } + return +} + +// RawData returns a copy of the page's internal data, so you can modify it at +// will, and it won't change the original page data. +func (p *Page) RawData() []byte { + cp := make([]byte, len(p.data)) + copy(cp, p.data) + return cp +} + +// OccupiedSlots returns all occupied slots in the page. The slots all point to +// cells in the page. The amount of slots will always be equal to the amount of +// cells stored in a page. The amount of slots in the page depends on the cell +// count of this page, not the other way around. +func (p *Page) OccupiedSlots() (result []Slot) { + cellCount := p.CellCount() + offsetsWidth := cellCount * SlotByteSize + offsetData := p.data[HeaderSize : HeaderSize+offsetsWidth] + for i := uint16(0); i < cellCount; i++ { + result = append(result, decodeOffset(offsetData[i*SlotByteSize:i*SlotByteSize+SlotByteSize])) + } + return } + +// FreeSlots computes all free addressable cell slots in this page. +func (p *Page) FreeSlots() (result []Slot) { + offsets := p.OccupiedSlots() + if len(offsets) == 0 { + // if there are no offsets at all, that means that the page is empty, + // and one slot is returned, which reaches from 0+OffsetSize until the + // end of the page + off := HeaderSize + SlotByteSize + return []Slot{{ + Offset: HeaderSize + SlotByteSize, + Size: uint16(len(p.data)) - off, + }} + } + + sort.Slice(offsets, func(i, j int) bool { + return offsets[i].Offset < offsets[j].Offset + }) + // first slot, from end of offset data until first cell + firstOff := HeaderSize + uint16(len(offsets)+1)*SlotByteSize // +1 because we always need space to store one more offset, so if that space is blocked, there is no free slot that is addressable + firstSize := offsets[0].Offset - firstOff + if firstSize > 0 { + result = append(result, Slot{ + Offset: firstOff, + Size: firstSize, + }) + } + // rest of the spaces between cells + for i := 0; i < len(offsets)-1; i++ { + off := offsets[i].Offset + offsets[i].Size + size := offsets[i+1].Offset - off + if size > 0 { + result = append(result, Slot{ + Offset: off, + Size: size, + }) + } + } + return +} + +// FindFreeSlotForSize searches for a free slot in this page, matching or +// exceeding the given data size. This is done by using a best-fit algorithm. +func (p *Page) FindFreeSlotForSize(dataSize uint16) (Slot, bool) { + // sort all free slots by size + slots := p.FreeSlots() + sort.Slice(slots, func(i, j int) bool { + return slots[i].Size < slots[j].Size + }) + // search for the best fitting slot, i.e. the first slot, whose size is greater + // than or equal to the given data size + index := sort.Search(len(slots), func(i int) bool { + return slots[i].Size >= dataSize + }) + if index == len(slots) { + return Slot{}, false + } + return slots[index], true +} + +func load(data []byte) (*Page, error) { + if len(data) > int(^uint16(0))-1 { + return nil, fmt.Errorf("page size too large: %v (max %v)", len(data), int(^uint16(0))-1) + } + + return &Page{ + data: data, + }, nil +} + +// findCell searches for a cell with the given key, as well as the corresponding +// offset and the corresponding offset index. The index is the index of the cell +// offset in all offsets, meaning that the byte location of the offset in the +// page can be obtained with offsetIndex*OffsetSize. The cellOffset is the +// offset that points to the cell. cell is the cell that was found, or nil if no +// cell with the given key could be found. If no cell could be found, +// found=false will be returned, as well as zero values for all other return +// arguments. +func (p *Page) findCell(key []byte) (offsetIndex uint16, cellSlot Slot, cell CellTyper, found bool) { + offsets := p.OccupiedSlots() + result := sort.Search(len(offsets), func(i int) bool { + cell := p.cellAt(offsets[i]) + switch c := cell.(type) { + case RecordCell: + return bytes.Compare(c.Key, key) >= 0 + case PointerCell: + return bytes.Compare(c.Key, key) >= 0 + } + return false + }) + if result == len(offsets) { + return 0, Slot{}, nil, false + } + return uint16(result), offsets[result], p.cellAt(offsets[result]), true +} + +func (p *Page) storePointerCell(cell PointerCell) error { + return p.storeRawCell(cell.Key, encodePointerCell(cell)) +} + +func (p *Page) storeRecordCell(cell RecordCell) error { + return p.storeRawCell(cell.Key, encodeRecordCell(cell)) +} + +func (p *Page) storeRawCell(key, rawCell []byte) error { + size := uint16(len(rawCell)) + slot, ok := p.FindFreeSlotForSize(size) + if !ok { + return ErrPageFull + } + p.storeCellSlot(Slot{ + Offset: slot.Offset + slot.Size - size, + Size: size, + }, key) + copy(p.data[slot.Offset+slot.Size-size:], rawCell) + p.incrementCellCount(1) + return nil +} + +func (p *Page) storeCellSlot(offset Slot, cellKey []byte) { + offsets := p.OccupiedSlots() + if len(offsets) == 0 { + // directly into the start of the page content, after the header + offset.encodeInto(p.data[HeaderSize:]) + return + } + + index := sort.Search(len(offsets), func(i int) bool { + cell := p.cellAt(offsets[i]) + switch c := cell.(type) { + case RecordCell: + return bytes.Compare(cellKey, c.Key) < 0 + case PointerCell: + return bytes.Compare(cellKey, c.Key) < 0 + } + return false + }) + + offsetOffset := HeaderSize + uint16(index)*SlotByteSize + if index != len(offsets) { + // make room if neccessary + allOffsetsEnd := HeaderSize + uint16(len(offsets))*SlotByteSize + p.moveAndZero(offsetOffset, allOffsetsEnd-offsetOffset, offsetOffset+SlotByteSize) + } + offset.encodeInto(p.data[offsetOffset:]) +} + +func (p *Page) cellAt(slot Slot) CellTyper { + return decodeCell(p.data[slot.Offset : slot.Offset+slot.Size]) +} + +// moveAndZero moves target bytes in the page's raw data from offset to target, +// and zeros all bytes from offset to offset+size, that do not overlap with the +// target area. Source and target area may overlap. +// +// [1,1,2,2,2,1,1,1,1,1] +// moveAndZero(2, 3, 6) +// [1,1,0,0,0,1,2,2,2,1] +// +// or, with overlap +// +// [1,1,2,2,2,1,1,1,1,1] +// moveAndZero(2, 3, 4) +// [1,1,0,0,2,2,2,1,1,1] +func (p *Page) moveAndZero(offset, size, target uint16) { + if target == offset { + // no-op when offset and target are the same + return + } + + _ = p.data[offset+size-1] // bounds check + _ = p.data[target+size-1] // bounds check + + copy(p.data[target:target+size], p.data[offset:offset+size]) + + // area needs zeroing + if target > offset+size || target+size < offset { + // no overlap + p.zero(offset, size) + } else { + // overlap + if target > offset && target <= offset+size { + // move to right, zero non-overlapping area + p.zero(offset, target-offset) + } else if target < offset && target+size >= offset { + // move to left, zero non-overlapping area + p.zero(target+size, offset-target) + } + } +} + +// zero zeroes size bytes, starting at offset in the page's raw data. +func (p *Page) zero(offset, size uint16) { + for i := uint16(0); i < size; i++ { + p.data[offset+i] = 0 + } +} + +func (p *Page) incrementCellCount(delta uint16) { p.incrementUint16(cellCountOffset, delta) } +func (p *Page) decrementCellCount(delta uint16) { p.decrementUint16(cellCountOffset, delta) } + +func (p *Page) storeUint16(at, val uint16) { byteOrder.PutUint16(p.data[at:], val) } +func (p *Page) loadUint16(at uint16) uint16 { return byteOrder.Uint16(p.data[at:]) } + +func (p *Page) incrementUint16(at, delta uint16) { p.storeUint16(at, p.loadUint16(at)+delta) } +func (p *Page) decrementUint16(at, delta uint16) { p.storeUint16(at, p.loadUint16(at)-delta) } diff --git a/internal/engine/storage/page/v1/page_test.go b/internal/engine/storage/page/page_test.go similarity index 86% rename from internal/engine/storage/page/v1/page_test.go rename to internal/engine/storage/page/page_test.go index 8654bdc5..44156cce 100644 --- a/internal/engine/storage/page/v1/page_test.go +++ b/internal/engine/storage/page/page_test.go @@ -1,30 +1,27 @@ -package v1 +package page import ( "testing" "github.com/stretchr/testify/assert" - "github.com/tomarrell/lbadd/internal/engine/storage/page" ) func TestPage_StoreRecordCell(t *testing.T) { assert := assert.New(t) - p, err := load(make([]byte, 36)) + p, err := load(make([]byte, 32)) assert.NoError(err) c := RecordCell{ - cell: cell{ - key: []byte{0xAB}, - }, - record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + Key: []byte{0xAB}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, } err = p.StoreRecordCell(c) assert.NoError(err) assert.Equal([]byte{ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, // header - 0x00, 0x16, 0x00, 0x0E, // offset + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, // header + 0x00, 0x12, 0x00, 0x0E, // offset 0x00, 0x00, 0x00, 0x00, // reserved for next offset 0x00, 0x00, 0x00, 0x00, // free slot #0 0x01, // cell type @@ -39,7 +36,7 @@ func TestPage_StoreRecordCell(t *testing.T) { // offset must skipt reserved space for offset, as the offset is not free // space assert.Equal(Slot{ - Offset: 18, + Offset: 14, Size: 4, }, freeSlots[0]) @@ -47,50 +44,42 @@ func TestPage_StoreRecordCell(t *testing.T) { copy(pageData, p.data) anotherCell := RecordCell{ - cell: cell{ - key: []byte("large key"), - }, - record: []byte("way too large record"), + Key: []byte("large key"), + Record: []byte("way too large record"), } err = p.StoreRecordCell(anotherCell) - assert.Equal(page.ErrPageFull, err) + assert.Equal(ErrPageFull, err) assert.Equal(pageData, p.data) // page must not have been modified } func TestPage_StoreRecordCell_Multiple(t *testing.T) { assert := assert.New(t) - p, err := load(make([]byte, 64)) + p, err := load(make([]byte, 60)) assert.NoError(err) cells := []RecordCell{ { - cell: cell{ - key: []byte{0x11}, - }, - record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, + Key: []byte{0x11}, + Record: []byte{0xCA, 0xFE, 0xBA, 0xBE}, }, { - cell: cell{ - key: []byte{0x33}, - }, - record: []byte{0xD1, 0xCE}, + Key: []byte{0x33}, + Record: []byte{0xD1, 0xCE}, }, { - cell: cell{ - key: []byte{0x22}, - }, - record: []byte{0xFF}, + Key: []byte{0x22}, + Record: []byte{0xFF}, }, } assert.NoError(p.storeRecordCell(cells[0])) assert.NoError(p.storeRecordCell(cells[1])) assert.NoError(p.storeRecordCell(cells[2])) assert.Equal([]byte{ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // header - 0x00, 0x32, 0x00, 0x0E, // offset #0 - 0x00, 0x1B, 0x00, 0x0B, // offset #2 - 0x00, 0x26, 0x00, 0x0C, // offset #1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // header + 0x00, 0x2E, 0x00, 0x0E, // offset #0 + 0x00, 0x17, 0x00, 0x0B, // offset #2 + 0x00, 0x22, 0x00, 0x0C, // offset #1 0x00, 0x00, 0x00, 0x00, 0x00, // free space // cell #3 0x01, // cell #3 type @@ -117,7 +106,7 @@ func TestPage_OccupiedSlots(t *testing.T) { assert := assert.New(t) // create a completely empty page - p, err := load(make([]byte, PageSize)) + p, err := load(make([]byte, Size)) assert.NoError(err) // create the offset source data @@ -312,13 +301,13 @@ func TestPage_FreeSlots(t *testing.T) { occupiedSlots := []Slot{ // 2 bytes - {32, 8}, + {28, 8}, // 10 bytes - {50, 5}, + {46, 5}, // 25 bytes - {80, 9}, + {76, 9}, // 1 byte - {90, 10}, + {86, 10}, } for i, slot := range occupiedSlots { @@ -327,9 +316,9 @@ func TestPage_FreeSlots(t *testing.T) { p.incrementCellCount(uint16(len(occupiedSlots))) assert.EqualValues([]Slot{ - {30, 2}, - {40, 10}, - {55, 25}, - {89, 1}, + {26, 2}, + {36, 10}, + {51, 25}, + {85, 1}, }, p.FreeSlots()) } diff --git a/internal/engine/storage/page/v1/slot.go b/internal/engine/storage/page/slot.go similarity index 98% rename from internal/engine/storage/page/v1/slot.go rename to internal/engine/storage/page/slot.go index 194175f4..223407f9 100644 --- a/internal/engine/storage/page/v1/slot.go +++ b/internal/engine/storage/page/slot.go @@ -1,4 +1,4 @@ -package v1 +package page import "unsafe" diff --git a/internal/engine/storage/page/v1/celltype_string.go b/internal/engine/storage/page/v1/celltype_string.go deleted file mode 100644 index b40a9886..00000000 --- a/internal/engine/storage/page/v1/celltype_string.go +++ /dev/null @@ -1,25 +0,0 @@ -// Code generated by "stringer -type=CellType"; DO NOT EDIT. - -package v1 - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[CellTypeUnknown-0] - _ = x[CellTypeRecord-1] - _ = x[CellTypePointer-2] -} - -const _CellType_name = "CellTypeUnknownCellTypeRecordCellTypePointer" - -var _CellType_index = [...]uint8{0, 15, 29, 44} - -func (i CellType) String() string { - if i >= CellType(len(_CellType_index)-1) { - return "CellType(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _CellType_name[_CellType_index[i]:_CellType_index[i+1]] -} diff --git a/internal/engine/storage/page/v1/doc.go b/internal/engine/storage/page/v1/doc.go deleted file mode 100644 index 638c33c5..00000000 --- a/internal/engine/storage/page/v1/doc.go +++ /dev/null @@ -1,13 +0,0 @@ -// Package v1 provides an implementation for the (page.Page) interface, as well -// as a function to load such a page from a given byte slice. v1 pages consist -// of headers and data, and NO trailer. Header fields are fixed, every header -// field has a fixed offset and memory area. Variable header fields are not -// supported. Cell types are stored in the cells. -// -// Assuming that data is given as a byte slice called data. -// -// data = ... -// p, err := v1.Load(data) -// val, err := p.Header(page.HeaderVersion) -// version := val.(uint16) -package v1 diff --git a/internal/engine/storage/page/v1/loader.go b/internal/engine/storage/page/v1/loader.go deleted file mode 100644 index 479f0aae..00000000 --- a/internal/engine/storage/page/v1/loader.go +++ /dev/null @@ -1,33 +0,0 @@ -package v1 - -import ( - "github.com/spf13/afero" - "github.com/tomarrell/lbadd/internal/engine/storage/page" -) - -var _ page.Loader = (*Loader)(nil) - -// Loader is the v1 implementation of a page.Loader, used to retrieve pages from -// secondary storage. -type Loader struct { - fs afero.Fs -} - -// NewLoader creates a new, ready to use loader. If during initialization, an -// error occurs, the error will be returned. It may be wrapped. -func NewLoader(fs afero.Fs) (*Loader, error) { - l := &Loader{ - fs: fs, - } - // TODO: add initialization if needed - return l, nil -} - -// Load loads the page with the given ID from the database files. -func (l *Loader) Load(id page.ID) (page.Page, error) { - return l.load(id) -} - -func (l *Loader) load(id page.ID) (*Page, error) { - return nil, nil -} diff --git a/internal/engine/storage/page/v1/page.go b/internal/engine/storage/page/v1/page.go deleted file mode 100644 index 1daa18dc..00000000 --- a/internal/engine/storage/page/v1/page.go +++ /dev/null @@ -1,343 +0,0 @@ -package v1 - -import ( - "bytes" - "encoding/binary" - "fmt" - "sort" - - "github.com/tomarrell/lbadd/internal/engine/storage/page" -) - -var _ page.Page = (*Page)(nil) - -const ( - // PageSize is the fix size of a page, which is 16KB or 16384 bytes. - PageSize = 1 << 14 - // HeaderSize is the fix size of a page header, which is 10 bytes. - HeaderSize = 10 -) - -// Header field offset in page data. -const ( - versionOffset = 0 // byte 1,2,3,4: version - idOffset = 4 // byte 5,6,7,8: byte page ID - cellCountOffset = 8 // byte 9,10: cell count -) - -var ( - byteOrder = binary.BigEndian -) - -// Page is a page implementation that does not support overflow pages. It is not -// meant for that. Since we want to separate index and data into separate files, -// records should not contain datasets, but rather enough information, to find -// the corresponding dataset in a data file. -type Page struct { - // data is the underlying data byte slice, which holds the header, offsets - // and cells. - data []byte - - dirty bool -} - -// Load loads the given data into the page. The length of the given data byte -// slice may differ from v1.PageSize, however, it cannot exceed ^uint16(0)-1 -// (65535 or 64KB), and must be larger than 22 (HeaderSize(=10) + 1 Offset(=4) + -// 1 empty cell(=8)). -func Load(data []byte) (page.Page, error) { - return load(data) -} - -// Version returns the version of this page. This should always be 1. This value -// must be constant. -func (p *Page) Version() uint32 { return byteOrder.Uint32(p.data[versionOffset:]) } - -// ID returns the ID of this page. This value must be constant. -func (p *Page) ID() page.ID { return byteOrder.Uint32(p.data[idOffset:]) } - -// CellCount returns the amount of stored cells in this page. This value is NOT -// constant. -func (p *Page) CellCount() uint16 { return byteOrder.Uint16(p.data[cellCountOffset:]) } - -// Dirty returns whether the page is dirty (needs syncing with secondary -// storage). -func (p *Page) Dirty() bool { return p.dirty } - -// MarkDirty marks this page as dirty. -func (p *Page) MarkDirty() { p.dirty = true } - -// ClearDirty marks this page as NOT dirty. -func (p *Page) ClearDirty() { p.dirty = false } - -// StorePointerCell stores a pointer cell in this page. A pointer cell points to -// other page IDs. -func (p *Page) StorePointerCell(cell page.PointerCell) error { - if v1cell, ok := cell.(PointerCell); ok { - return p.storePointerCell(v1cell) - } - return fmt.Errorf("can only store v1 pointer cells, but got %T", cell) -} - -// StoreRecordCell stores a record cell in this page. A record cell holds -// arbitrary, variable size data. -func (p *Page) StoreRecordCell(cell page.RecordCell) error { - if v1cell, ok := cell.(RecordCell); ok { - return p.storeRecordCell(v1cell) - } - return fmt.Errorf("can only store v1 record cells, but got %T", cell) -} - -// DeleteCell deletes a cell with the given key. If no such cell could be found, -// false is returned. In this implementation, an error can not occur while -// deleting a cell. -func (p *Page) DeleteCell(key []byte) (bool, error) { - offsetIndex, cellOffset, _, found := p.findCell(key) - if !found { - return false, nil - } - - // delete offset - p.zero(offsetIndex*SlotByteSize, SlotByteSize) - // delete cell data - p.zero(cellOffset.Offset, cellOffset.Size) - // close gap in offsets due to offset deletion - from := offsetIndex*SlotByteSize + SlotByteSize // lower bound, right next to gap - to := p.CellCount() * SlotByteSize // upper bound of the offset data - p.moveAndZero(from, to-from, from-SlotByteSize) // actually move the data - // update cell count - p.decrementCellCount(1) - return true, nil -} - -// Cell returns a cell from this page with the given key, or false if no such -// cell exists in this page. In that case, the returned page is also nil. -func (p *Page) Cell(key []byte) (page.Cell, bool) { - _, _, cell, found := p.findCell(key) - return cell, found -} - -// Cells decodes all cells in this page, which can be expensive, and returns all -// of them. The returned cells do not point back to the original page data, so -// don't modify them. Instead, delete the old cell and store a new one. -func (p *Page) Cells() (result []page.Cell) { - for _, offset := range p.OccupiedSlots() { - result = append(result, decodeCell(p.data[offset.Offset:offset.Offset+offset.Size])) - } - return -} - -// RawData returns a copy of the page's internal data, so you can modify it at -// will, and it won't change the original page data. -func (p *Page) RawData() []byte { - cp := make([]byte, len(p.data)) - copy(cp, p.data) - return cp -} - -// OccupiedSlots returns all occupied slots in the page. The slots all point to -// cells in the page. The amount of slots will always be equal to the amount of -// cells stored in a page. The amount of slots in the page depends on the cell -// count of this page, not the other way around. -func (p *Page) OccupiedSlots() (result []Slot) { - cellCount := p.CellCount() - offsetsWidth := cellCount * SlotByteSize - offsetData := p.data[HeaderSize : HeaderSize+offsetsWidth] - for i := uint16(0); i < cellCount; i++ { - result = append(result, decodeOffset(offsetData[i*SlotByteSize:i*SlotByteSize+SlotByteSize])) - } - return -} - -// FreeSlots computes all free addressable cell slots in this page. -func (p *Page) FreeSlots() (result []Slot) { - offsets := p.OccupiedSlots() - if len(offsets) == 0 { - // if there are no offsets at all, that means that the page is empty, - // and one slot is returned, which reaches from 0+OffsetSize until the - // end of the page - off := HeaderSize + SlotByteSize - return []Slot{{ - Offset: HeaderSize + SlotByteSize, - Size: uint16(len(p.data)) - off, - }} - } - - sort.Slice(offsets, func(i, j int) bool { - return offsets[i].Offset < offsets[j].Offset - }) - // first slot, from end of offset data until first cell - firstOff := HeaderSize + uint16(len(offsets)+1)*SlotByteSize // +1 because we always need space to store one more offset, so if that space is blocked, there is no free slot that is addressable - firstSize := offsets[0].Offset - firstOff - if firstSize > 0 { - result = append(result, Slot{ - Offset: firstOff, - Size: firstSize, - }) - } - // rest of the spaces between cells - for i := 0; i < len(offsets)-1; i++ { - off := offsets[i].Offset + offsets[i].Size - size := offsets[i+1].Offset - off - if size > 0 { - result = append(result, Slot{ - Offset: off, - Size: size, - }) - } - } - return -} - -// FindFreeSlotForSize searches for a free slot in this page, matching or -// exceeding the given data size. This is done by using a best-fit algorithm. -func (p *Page) FindFreeSlotForSize(dataSize uint16) (Slot, bool) { - // sort all free slots by size - slots := p.FreeSlots() - sort.Slice(slots, func(i, j int) bool { - return slots[i].Size < slots[j].Size - }) - // search for the best fitting slot, i.e. the first slot, whose size is greater - // than or equal to the given data size - index := sort.Search(len(slots), func(i int) bool { - return slots[i].Size >= dataSize - }) - if index == len(slots) { - return Slot{}, false - } - return slots[index], true -} - -func load(data []byte) (*Page, error) { - if len(data) > int(^uint16(0))-1 { - return nil, fmt.Errorf("page size too large: %v (max %v)", len(data), int(^uint16(0))-1) - } - - return &Page{ - data: data, - }, nil -} - -// findCell searches for a cell with the given key, as well as the corresponding -// offset and the corresponding offset index. The index is the index of the cell -// offset in all offsets, meaning that the byte location of the offset in the -// page can be obtained with offsetIndex*OffsetSize. The cellOffset is the -// offset that points to the cell. cell is the cell that was found, or nil if no -// cell with the given key could be found. If no cell could be found, -// found=false will be returned, as well as zero values for all other return -// arguments. -func (p *Page) findCell(key []byte) (offsetIndex uint16, cellSlot Slot, cell Cell, found bool) { - offsets := p.OccupiedSlots() - result := sort.Search(len(offsets), func(i int) bool { - cell := p.cellAt(offsets[i]) - return bytes.Compare(cell.Key(), key) >= 0 - }) - if result == len(offsets) { - return 0, Slot{}, nil, false - } - return uint16(result), offsets[result], p.cellAt(offsets[result]), true -} - -func (p *Page) storePointerCell(cell PointerCell) error { - return p.storeRawCell(cell.key, encodePointerCell(cell)) -} - -func (p *Page) storeRecordCell(cell RecordCell) error { - return p.storeRawCell(cell.key, encodeRecordCell(cell)) -} - -func (p *Page) storeRawCell(key, rawCell []byte) error { - size := uint16(len(rawCell)) - slot, ok := p.FindFreeSlotForSize(size) - if !ok { - return page.ErrPageFull - } - p.storeCellSlot(Slot{ - Offset: slot.Offset + slot.Size - size, - Size: size, - }, key) - copy(p.data[slot.Offset+slot.Size-size:], rawCell) - p.incrementCellCount(1) - return nil -} - -func (p *Page) storeCellSlot(offset Slot, cellKey []byte) { - offsets := p.OccupiedSlots() - if len(offsets) == 0 { - // directly into the start of the page content, after the header - offset.encodeInto(p.data[HeaderSize:]) - return - } - - index := sort.Search(len(offsets), func(i int) bool { - return bytes.Compare(cellKey, p.cellAt(offsets[i]).Key()) < 0 - }) - - offsetOffset := HeaderSize + uint16(index)*SlotByteSize - if index != len(offsets) { - // make room if neccessary - allOffsetsEnd := HeaderSize + uint16(len(offsets))*SlotByteSize - p.moveAndZero(offsetOffset, allOffsetsEnd-offsetOffset, offsetOffset+SlotByteSize) - } - offset.encodeInto(p.data[offsetOffset:]) -} - -func (p *Page) cellAt(slot Slot) Cell { - return decodeCell(p.data[slot.Offset : slot.Offset+slot.Size]) -} - -// moveAndZero moves target bytes in the page's raw data from offset to target, -// and zeros all bytes from offset to offset+size, that do not overlap with the -// target area. Source and target area may overlap. -// -// [1,1,2,2,2,1,1,1,1,1] -// moveAndZero(2, 3, 6) -// [1,1,0,0,0,1,2,2,2,1] -// -// or, with overlap -// -// [1,1,2,2,2,1,1,1,1,1] -// moveAndZero(2, 3, 4) -// [1,1,0,0,2,2,2,1,1,1] -func (p *Page) moveAndZero(offset, size, target uint16) { - if target == offset { - // no-op when offset and target are the same - return - } - - _ = p.data[offset+size-1] // bounds check - _ = p.data[target+size-1] // bounds check - - copy(p.data[target:target+size], p.data[offset:offset+size]) - - // area needs zeroing - if target > offset+size || target+size < offset { - // no overlap - p.zero(offset, size) - } else { - // overlap - if target > offset && target <= offset+size { - // move to right, zero non-overlapping area - p.zero(offset, target-offset) - } else if target < offset && target+size >= offset { - // move to left, zero non-overlapping area - p.zero(target+size, offset-target) - } - } -} - -// zero zeroes size bytes, starting at offset in the page's raw data. -func (p *Page) zero(offset, size uint16) { - for i := uint16(0); i < size; i++ { - p.data[offset+i] = 0 - } -} - -func (p *Page) incrementCellCount(delta uint16) { p.incrementUint16(cellCountOffset, delta) } -func (p *Page) decrementCellCount(delta uint16) { p.decrementUint16(cellCountOffset, delta) } - -func (p *Page) storeUint16(at, val uint16) { byteOrder.PutUint16(p.data[at:], val) } -func (p *Page) loadUint16(at uint16) uint16 { return byteOrder.Uint16(p.data[at:]) } - -func (p *Page) incrementUint16(at, delta uint16) { p.storeUint16(at, p.loadUint16(at)+delta) } -func (p *Page) decrementUint16(at, delta uint16) { p.storeUint16(at, p.loadUint16(at)-delta) } diff --git a/internal/engine/storage/page_manager.go b/internal/engine/storage/page_manager.go new file mode 100644 index 00000000..206f51d3 --- /dev/null +++ b/internal/engine/storage/page_manager.go @@ -0,0 +1,63 @@ +package storage + +import ( + "fmt" + "sync/atomic" + + "github.com/spf13/afero" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +type PageManager struct { + file afero.File + largestID page.ID +} + +func NewPageManager(file afero.File) (*PageManager, error) { + stat, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("stat: %w", err) + } + + mgr := &PageManager{ + file: file, + largestID: uint32(stat.Size() / page.Size), + } + + return mgr, nil +} + +func (m *PageManager) ReadPage(id page.ID) (*page.Page, error) { + data := make([]byte, page.Size) + _, err := m.file.ReadAt(data, int64(id)*page.Size) + if err != nil { + return nil, fmt.Errorf("read at: %w", err) + } + p, err := page.Load(data) + if err != nil { + return nil, fmt.Errorf("load: %w", err) + } + return p, nil +} + +func (m *PageManager) WritePage(p *page.Page) error { + data := p.RawData() // TODO: avoid copying in RawData() + _, err := m.file.WriteAt(data, int64(p.ID())*page.Size) + if err != nil { + return fmt.Errorf("write at: %w", err) + } + if err := m.file.Sync(); err != nil { + return fmt.Errorf("sync: %w", err) + } + return nil +} + +func (m *PageManager) AllocateNew() (*page.Page, error) { + id := atomic.AddUint32(&m.largestID, 1) - 1 + + p := page.New(id) + if err := m.WritePage(p); err != nil { + return nil, fmt.Errorf("write new page: %w", err) + } + return p, nil +} diff --git a/internal/engine/storage/storage.go b/internal/engine/storage/storage.go index 7306f617..2c43c8d2 100644 --- a/internal/engine/storage/storage.go +++ b/internal/engine/storage/storage.go @@ -1 +1,6 @@ -package storage \ No newline at end of file +package storage + +const ( + // Version is the storage version that is currently implemented. + Version = 1 +) diff --git a/internal/engine/storage/validator.go b/internal/engine/storage/validator.go new file mode 100644 index 00000000..8da4f6a5 --- /dev/null +++ b/internal/engine/storage/validator.go @@ -0,0 +1,56 @@ +package storage + +import ( + "fmt" + "os" + + "github.com/spf13/afero" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +type Validator struct { + file afero.File + info os.FileInfo +} + +func NewValidator(file afero.File) *Validator { + return &Validator{ + file: file, + // info is set on every run of Validate() + } +} + +func (v *Validator) Validate() error { + stat, err := v.file.Stat() + if err != nil { + return fmt.Errorf("stat: %w", err) + } + v.info = stat + + if err := v.validateIsFile(); err != nil { + return fmt.Errorf("is file: %w", err) + } + if err := v.validateSize(); err != nil { + return fmt.Errorf("size: %w", err) + } + + return nil +} + +func (v Validator) validateIsFile() error { + if v.info.IsDir() { + return fmt.Errorf("file is directory") + } + if !v.info.Mode().Perm().IsRegular() { + return fmt.Errorf("file is not a regular file") + } + return nil +} + +func (v Validator) validateSize() error { + size := v.info.Size() + if size%page.Size != 0 { + return fmt.Errorf("invalid file size, must be multiple of page size (=%v), but was %v", page.Size, size) + } + return nil +} From 0123d4091f6f199e88a0471787d8daf89c466165 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Mon, 22 Jun 2020 17:37:54 +0200 Subject: [PATCH 23/77] Add documentation --- internal/engine/storage/cache/lru.go | 31 +++++++++++-- internal/engine/storage/db.go | 61 +++++++++++++++++++------ internal/engine/storage/option.go | 3 ++ internal/engine/storage/page/cell.go | 1 + internal/engine/storage/page_manager.go | 22 +++++++++ internal/engine/storage/validator.go | 7 +++ 6 files changed, 109 insertions(+), 16 deletions(-) diff --git a/internal/engine/storage/cache/lru.go b/internal/engine/storage/cache/lru.go index 16f2c35f..5ecd79ee 100644 --- a/internal/engine/storage/cache/lru.go +++ b/internal/engine/storage/cache/lru.go @@ -6,6 +6,8 @@ import ( "github.com/tomarrell/lbadd/internal/engine/storage/page" ) +// SecondaryStorage is the abstraction that a cache uses, to synchronize dirty +// pages with secondary storage. type SecondaryStorage interface { ReadPage(page.ID) (*page.Page, error) WritePage(*page.Page) error @@ -13,14 +15,20 @@ type SecondaryStorage interface { var _ Cache = (*LRUCache)(nil) +// LRUCache is a simple implementation of an LRU cache. type LRUCache struct { store SecondaryStorage - pages map[uint32]*page.Page - pinned map[uint32]struct{} + pages map[page.ID]*page.Page + pinned map[page.ID]struct{} size int - lru []uint32 + lru []page.ID } +// NewLRUCache creates a new LRU cache with the given size and secondary storage +// to write dirty pages to. The size is the maximum amount of pages, that can be +// held by this cache. If more pages than the size are requested, old pages will +// be evicted from the cache. If all pages are pinned (in use and not released +// yet), requesting a new page will fail. func NewLRUCache(size int, store SecondaryStorage) *LRUCache { return &LRUCache{ store: store, @@ -31,19 +39,36 @@ func NewLRUCache(size int, store SecondaryStorage) *LRUCache { } } +// FetchAndPin will return the page with the given ID. This will fail, if the +// page with the given ID is not in the cache, but the cache is full and all +// pages are pinned. After obtaining a page with this method, you MUST release +// it, once you are done using it, with Unpin(ID). func (c *LRUCache) FetchAndPin(id page.ID) (*page.Page, error) { return c.fetchAndPin(id) } +// Unpin unpins the page with the given ID. If the page with the given ID is not +// pinned, then this is a no-op. func (c *LRUCache) Unpin(id page.ID) { c.unpin(id) } +// Flush writes the contents of the page with the given ID to secondary storage. +// This fails, if the page with the given ID is not in the cache anymore. Only +// call this if you know what you are doing. Pages will always be flushed before +// being evicted. If you really do need to use this, call it before unpinning +// the page, to guarantee that the page will not be evicted between unpinning +// and flushing. func (c *LRUCache) Flush(id page.ID) error { return c.flush(id) } +// Close will flush all dirty pages and then close this cache. func (c *LRUCache) Close() error { + // TODO: can we really just flush on close? + for id := range c.pages { + _ = c.flush(id) + } return nil } diff --git a/internal/engine/storage/db.go b/internal/engine/storage/db.go index c0d20180..8123e6d7 100644 --- a/internal/engine/storage/db.go +++ b/internal/engine/storage/db.go @@ -19,9 +19,12 @@ const ( // (CacheSize * page.Size). DefaultCacheSize = 1 << 8 + // HeaderPageID is the page ID of the header page of the database file. HeaderPageID page.ID = 0 - HeaderTables = "tables" + // HeaderTables is the string key for the header page's cell "tables" + HeaderTables = "tables" + // HeaderPageCount is the string key for the header page's cell "pageCount" HeaderPageCount = "pageCount" ) @@ -29,6 +32,8 @@ var ( byteOrder = binary.BigEndian ) +// DBFile is a database file that can be opened or created. From this file, you +// can obtain a page cache, which you must use for reading pages. type DBFile struct { closed uint32 @@ -42,9 +47,17 @@ type DBFile struct { headerPage *page.Page } +// Create creates a new database in the given file with the given options. The +// file must exist, but be empty and must be a regular file. func Create(file afero.File, opts ...Option) (*DBFile, error) { - if _, err := file.Stat(); err != nil { + if info, err := file.Stat(); err != nil { return nil, fmt.Errorf("stat: %w", err) + } else if info.IsDir() { + return nil, fmt.Errorf("file is directory") + } else if size := info.Size(); size != 0 { + return nil, fmt.Errorf("file is not empty, has %v bytes", size) + } else if !info.Mode().IsRegular() { + return nil, fmt.Errorf("file is not a regular file") } mgr, err := NewPageManager(file) @@ -61,14 +74,18 @@ func Create(file afero.File, opts ...Option) (*DBFile, error) { return nil, fmt.Errorf("allocate tables page: %w", err) } - headerPage.StoreRecordCell(page.RecordCell{ + if err := headerPage.StoreRecordCell(page.RecordCell{ Key: []byte(HeaderPageCount), Record: encodeUint64(2), // header and tables page - }) - headerPage.StorePointerCell(page.PointerCell{ + }); err != nil { + return nil, fmt.Errorf("store record cell: %w", err) + } + if err := headerPage.StorePointerCell(page.PointerCell{ Key: []byte(HeaderTables), Pointer: tablesPage.ID(), - }) + }); err != nil { + return nil, fmt.Errorf("store pointer cell: %w", err) + } err = mgr.WritePage(headerPage) // immediately flush if err != nil { @@ -82,6 +99,9 @@ func Create(file afero.File, opts ...Option) (*DBFile, error) { return newDB(file, mgr, headerPage, opts...), nil } +// Open opens and validates a given file and creates a (*DBFile) with the given +// options. If the validation fails, an error explaining the failure will be +// returned. func Open(file afero.File, opts ...Option) (*DBFile, error) { if err := NewValidator(file).Validate(); err != nil { return nil, fmt.Errorf("validate: %w", err) @@ -100,24 +120,30 @@ func Open(file afero.File, opts ...Option) (*DBFile, error) { return newDB(file, mgr, headerPage, opts...), nil } -func (db *DBFile) AllocateNewPage() (*page.Page, error) { +// AllocateNewPage allocates and immediately persists a new page in secondary +// storage. This will fail if the DBFile is closed. After this method returns, +// the allocated page can immediately be found by the cache, and you can use the +// returned page ID to load the page through the cache. +func (db *DBFile) AllocateNewPage() (page.ID, error) { if db.Closed() { - return nil, ErrClosed + return 0, ErrClosed } page, err := db.pageManager.AllocateNew() if err != nil { - return nil, fmt.Errorf("allocate new: %w", err) + return 0, fmt.Errorf("allocate new: %w", err) } if err := db.incrementHeaderPageCount(); err != nil { - return nil, fmt.Errorf("increment header page count: %w", err) + return 0, fmt.Errorf("increment header page count: %w", err) } if err := db.pageManager.WritePage(db.headerPage); err != nil { - return nil, fmt.Errorf("write header page: %w", err) + return 0, fmt.Errorf("write header page: %w", err) } - return page, nil + return page.ID(), nil } +// Cache returns the cache implementation, that you must use to obtain pages. +// This will fail if the DBFile is closed. func (db *DBFile) Cache() cache.Cache { if db.Closed() { return nil @@ -125,17 +151,22 @@ func (db *DBFile) Cache() cache.Cache { return db.cache } +// Close will close the underlying cache, as well as page manager, as well as +// file. func (db *DBFile) Close() error { _ = db.cache.Close() + _ = db.pageManager.Close() _ = db.file.Close() atomic.StoreUint32(&db.closed, 1) return nil } +// Closed indicates, whether this file was closed. func (db *DBFile) Closed() bool { return atomic.LoadUint32(&db.closed) == 1 } +// newDB creates a new DBFile from the given objects, and applies all options. func newDB(file afero.File, mgr *PageManager, headerPage *page.Page, opts ...Option) *DBFile { db := &DBFile{ log: zerolog.Nop(), @@ -154,6 +185,8 @@ func newDB(file afero.File, mgr *PageManager, headerPage *page.Page, opts ...Opt return db } +// incrementHeaderPageCount will increment the 8 byte uint64 in the +// HeaderPageCount cell by 1. func (db *DBFile) incrementHeaderPageCount() error { val, ok := db.headerPage.Cell([]byte(HeaderPageCount)) if !ok { @@ -164,8 +197,10 @@ func (db *DBFile) incrementHeaderPageCount() error { return nil } +// encodeUint64 will allocate 8 bytes to encode the given uint64 into. This +// newly allocated byte-slice is then returned. func encodeUint64(v uint64) []byte { - buf := make([]byte, unsafe.Sizeof(v)) + buf := make([]byte, unsafe.Sizeof(v)) // #nosec byteOrder.PutUint64(buf, v) return buf } diff --git a/internal/engine/storage/option.go b/internal/engine/storage/option.go index 9382f403..2c157de9 100644 --- a/internal/engine/storage/option.go +++ b/internal/engine/storage/option.go @@ -2,14 +2,17 @@ package storage import "github.com/rs/zerolog" +// Option is an option that can is applied to a DBFile on creation. type Option func(*DBFile) +// WithCacheSize specifies a cache size for the DBFile. func WithCacheSize(size int) Option { return func(db *DBFile) { db.cacheSize = size } } +// WithLogger specifies a logger for the DBFile. func WithLogger(log zerolog.Logger) Option { return func(db *DBFile) { db.log = log diff --git a/internal/engine/storage/page/cell.go b/internal/engine/storage/page/cell.go index a7994c14..b2aa9838 100644 --- a/internal/engine/storage/page/cell.go +++ b/internal/engine/storage/page/cell.go @@ -25,6 +25,7 @@ const ( ) type ( + // CellTyper describes a component that has a cell type. CellTyper interface { Type() CellType } diff --git a/internal/engine/storage/page_manager.go b/internal/engine/storage/page_manager.go index 206f51d3..c133e99d 100644 --- a/internal/engine/storage/page_manager.go +++ b/internal/engine/storage/page_manager.go @@ -8,11 +8,16 @@ import ( "github.com/tomarrell/lbadd/internal/engine/storage/page" ) +// PageManager is a manager that is responsible for reading pages from and +// writing pages to secondary storage. It also can allocate new pages, which +// will immediately be written to secondary storage. type PageManager struct { file afero.File largestID page.ID } +// NewPageManager creates a new page manager over the given file. It is assumed, +// that the file passed validation by a (*storage.Validator). func NewPageManager(file afero.File) (*PageManager, error) { stat, err := file.Stat() if err != nil { @@ -27,6 +32,8 @@ func NewPageManager(file afero.File) (*PageManager, error) { return mgr, nil } +// ReadPage returns the page with the given ID, or an error if reading is +// impossible. func (m *PageManager) ReadPage(id page.ID) (*page.Page, error) { data := make([]byte, page.Size) _, err := m.file.ReadAt(data, int64(id)*page.Size) @@ -40,6 +47,8 @@ func (m *PageManager) ReadPage(id page.ID) (*page.Page, error) { return p, nil } +// WritePage will write the given page into secondary storage. It is guaranteed, +// that after this call returns, the page is present on disk. func (m *PageManager) WritePage(p *page.Page) error { data := p.RawData() // TODO: avoid copying in RawData() _, err := m.file.WriteAt(data, int64(p.ID())*page.Size) @@ -52,6 +61,9 @@ func (m *PageManager) WritePage(p *page.Page) error { return nil } +// AllocateNew will allocate a new page and immediately persist it in secondary +// storage. It is guaranteed, that after this call returns, the page is present +// on disk. func (m *PageManager) AllocateNew() (*page.Page, error) { id := atomic.AddUint32(&m.largestID, 1) - 1 @@ -61,3 +73,13 @@ func (m *PageManager) AllocateNew() (*page.Page, error) { } return p, nil } + +// Close will sync the file with secondary storage and then close it. If syncing +// fails, the file will not be closed, and an error will be returned. +func (m *PageManager) Close() error { + if err := m.file.Sync(); err != nil { + return fmt.Errorf("sync: %w", err) + } + _ = m.file.Close() + return nil +} diff --git a/internal/engine/storage/validator.go b/internal/engine/storage/validator.go index 8da4f6a5..a4cc46ed 100644 --- a/internal/engine/storage/validator.go +++ b/internal/engine/storage/validator.go @@ -8,11 +8,16 @@ import ( "github.com/tomarrell/lbadd/internal/engine/storage/page" ) +// Validator can be used to validate a database file prior to opening it. If +// validation fails, a speaking error is returned. If validation does not fail, +// the file is a valid database file and can be used. Valid means, that the file +// is not structurally corrupted and usable. type Validator struct { file afero.File info os.FileInfo } +// NewValidator creates a new validator over the given file. func NewValidator(file afero.File) *Validator { return &Validator{ file: file, @@ -20,6 +25,8 @@ func NewValidator(file afero.File) *Validator { } } +// Validate runs the file validation and returns a speaking error on why the +// validation failed, if it failed. func (v *Validator) Validate() error { stat, err := v.file.Stat() if err != nil { From 6d1959abc29d02994391d7581a7a03bd9c6f3385 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 23 Jun 2020 11:43:14 +0200 Subject: [PATCH 24/77] Implement page defragmentation --- internal/engine/storage/cache/lru.go | 2 +- internal/engine/storage/page/page.go | 54 ++++++++++++++- internal/engine/storage/page/page_test.go | 82 +++++++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) diff --git a/internal/engine/storage/cache/lru.go b/internal/engine/storage/cache/lru.go index 5ecd79ee..1534e0eb 100644 --- a/internal/engine/storage/cache/lru.go +++ b/internal/engine/storage/cache/lru.go @@ -35,7 +35,7 @@ func NewLRUCache(size int, store SecondaryStorage) *LRUCache { pages: make(map[uint32]*page.Page), pinned: make(map[uint32]struct{}), size: size, - lru: make([]uint32, size), + lru: make([]uint32, 0), } } diff --git a/internal/engine/storage/page/page.go b/internal/engine/storage/page/page.go index f8bc1786..5223d0a5 100644 --- a/internal/engine/storage/page/page.go +++ b/internal/engine/storage/page/page.go @@ -148,7 +148,8 @@ func (p *Page) OccupiedSlots() (result []Slot) { return } -// FreeSlots computes all free addressable cell slots in this page. +// FreeSlots computes all free addressable cell slots in this page. The free +// slots are sorted in ascending order by the offset in the page. func (p *Page) FreeSlots() (result []Slot) { offsets := p.OccupiedSlots() if len(offsets) == 0 { @@ -207,10 +208,61 @@ func (p *Page) FindFreeSlotForSize(dataSize uint16) (Slot, bool) { return slots[index], true } +// Defragment will move the cells in the page in a way that after defragmenting, +// there is only a single free block, and that is located between the offsets +// and the cell data. After calling this method, (*Page).Fragmentation() will +// return 0. +func (p *Page) Defragment() { + occupied := p.OccupiedSlots() + var newSlots []Slot + nextLeftBound := uint16(len(p.data)) + for i := len(occupied) - 1; i >= 0; i-- { + slot := occupied[i] + newOffset := nextLeftBound - slot.Size + p.moveAndZero(slot.Offset, slot.Size, newOffset) + nextLeftBound = newOffset + newSlots = append(newSlots, Slot{ + Offset: newOffset, + Size: slot.Size, + }) + } + // no need to sort new slots, as the order was not modified during + // defragmentation, but we need to reverse it + for i, j := 0, len(newSlots)-1; i < j; i, j = i+1, j-1 { + newSlots[i], newSlots[j] = newSlots[j], newSlots[i] + } + + for i, slot := range newSlots { + slot.encodeInto(p.data[HeaderSize+uint16(i)*SlotByteSize:]) + } +} + +// Fragmentation computes the page fragmentation, which is defined by 1 - +// (largest free block / total free size). Multiply with 100 to get the +// fragmentation percentage. +func (p *Page) Fragmentation() float32 { + slots := p.FreeSlots() + if len(slots) == 0 { + return 0 + } + + var largestFree, totalFree uint16 + for _, slot := range slots { + totalFree += slot.Size + if slot.Size > largestFree { + largestFree = slot.Size + } + } + return 1 - (float32(largestFree) / float32(totalFree)) +} + func load(data []byte) (*Page, error) { if len(data) > int(^uint16(0))-1 { return nil, fmt.Errorf("page size too large: %v (max %v)", len(data), int(^uint16(0))-1) } + if len(data) < HeaderSize { + return nil, fmt.Errorf("page size too small: %v (min %v)", len(data), HeaderSize) + } return &Page{ data: data, diff --git a/internal/engine/storage/page/page_test.go b/internal/engine/storage/page/page_test.go index 44156cce..2fd7729c 100644 --- a/internal/engine/storage/page/page_test.go +++ b/internal/engine/storage/page/page_test.go @@ -322,3 +322,85 @@ func TestPage_FreeSlots(t *testing.T) { {85, 1}, }, p.FreeSlots()) } + +func TestPage_Defragment(t *testing.T) { + tests := []struct { + name string + before []byte + after []byte + }{ + { + "small 2 cells", + []byte{ + /* 0x00 */ 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, // header + /* 0x06 */ 0x00, 0x12, 0x00, 0x04, // offset #0 + /* 0x0a */ 0x00, 0x1A, 0x00, 0x04, // offset #1 + /* 0x0e */ 0x00, 0x00, 0x00, 0x00, // free space + /* 0x12 */ 0x01, 0x01, 0x01, 0x01, // cell #0 + /* 0x16 */ 0x00, 0x00, 0x00, 0x00, // free space + /* 0x1a */ 0x02, 0x02, 0x02, 0x02, // cell #1 + }, + []byte{ + /* 0x00 */ 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, // header + /* 0x06 */ 0x00, 0x16, 0x00, 0x04, // offset #0 + /* 0x0a */ 0x00, 0x1A, 0x00, 0x04, // offset #1 + /* 0x0e */ 0x00, 0x00, 0x00, 0x00, // free space + /* 0x12 */ 0x00, 0x00, 0x00, 0x00, // free space + /* 0x16 */ 0x01, 0x01, 0x01, 0x01, // cell #0 + /* 0x1a */ 0x02, 0x02, 0x02, 0x02, // cell #1 + }, + }, + { + "small 2 cells free end", + []byte{ + /* 0x00 */ 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, // header + /* 0x06 */ 0x00, 0x12, 0x00, 0x04, // offset #0 + /* 0x0a */ 0x00, 0x1A, 0x00, 0x04, // offset #1 + /* 0x0e */ 0x00, 0x00, 0x00, 0x00, // free space + /* 0x12 */ 0x01, 0x01, 0x01, 0x01, // cell #0 + /* 0x16 */ 0x00, 0x00, 0x00, 0x00, // free space + /* 0x1a */ 0x02, 0x02, 0x02, 0x02, // cell #1 + /* 0x1e */ 0x00, 0x00, 0x00, 0x00, // free space + }, + []byte{ + /* 0x00 */ 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, // header + /* 0x06 */ 0x00, 0x1A, 0x00, 0x04, // offset #0 + /* 0x0a */ 0x00, 0x1E, 0x00, 0x04, // offset #1 + /* 0x0e */ 0x00, 0x00, 0x00, 0x00, // free space + /* 0x12 */ 0x00, 0x00, 0x00, 0x00, // free space + /* 0x16 */ 0x00, 0x00, 0x00, 0x00, // free space + /* 0x1a */ 0x01, 0x01, 0x01, 0x01, // cell #0 + /* 0x1e */ 0x02, 0x02, 0x02, 0x02, // cell #1 + }, + }, + { + "full page", + []byte{ + /* 0x00 */ 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, // header + /* 0x06 */ 0x00, 0x0E, 0x00, 0x04, // offset #0 + /* 0x0a */ 0x00, 0x12, 0x00, 0x04, // offset #1 + /* 0x0e */ 0x01, 0x01, 0x01, 0x01, // cell #0 + /* 0x12 */ 0x02, 0x02, 0x02, 0x02, // cell #1 + }, + []byte{ + /* 0x00 */ 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, // header + /* 0x06 */ 0x00, 0x0E, 0x00, 0x04, // offset #0 + /* 0x0a */ 0x00, 0x12, 0x00, 0x04, // offset #1 + /* 0x0e */ 0x01, 0x01, 0x01, 0x01, // cell #0 + /* 0x12 */ 0x02, 0x02, 0x02, 0x02, // cell #1 + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + p, err := load(tt.before) + assert.NoError(err) + + assert.Equal(tt.before, p.data) + p.Defragment() + assert.Equal(tt.after, p.data) + }) + } +} From 91356c0abdbfd9e6a6a60c5440f0663f3302c0bb Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 23 Jun 2020 12:22:28 +0200 Subject: [PATCH 25/77] Test cell deletion --- internal/engine/storage/page/page.go | 11 ++-- internal/engine/storage/page/page_test.go | 71 +++++++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/internal/engine/storage/page/page.go b/internal/engine/storage/page/page.go index 5223d0a5..7e190d89 100644 --- a/internal/engine/storage/page/page.go +++ b/internal/engine/storage/page/page.go @@ -97,13 +97,14 @@ func (p *Page) DeleteCell(key []byte) (bool, error) { } // delete offset - p.zero(offsetIndex*SlotByteSize, SlotByteSize) + p.zero(offsetIndex, SlotByteSize) // delete cell data p.zero(cellOffset.Offset, cellOffset.Size) // close gap in offsets due to offset deletion - from := offsetIndex*SlotByteSize + SlotByteSize // lower bound, right next to gap - to := p.CellCount() * SlotByteSize // upper bound of the offset data - p.moveAndZero(from, to-from, from-SlotByteSize) // actually move the data + from := offsetIndex + SlotByteSize // lower bound, right next to gap + to := offsetIndex // upper bound of the offset data + cellCount := p.CellCount() + p.moveAndZero(from, HeaderSize+cellCount*SlotByteSize-from, to) // actually move the data // update cell count p.decrementCellCount(1) return true, nil @@ -292,7 +293,7 @@ func (p *Page) findCell(key []byte) (offsetIndex uint16, cellSlot Slot, cell Cel if result == len(offsets) { return 0, Slot{}, nil, false } - return uint16(result), offsets[result], p.cellAt(offsets[result]), true + return HeaderSize + uint16(result)*SlotByteSize, offsets[result], p.cellAt(offsets[result]), true } func (p *Page) storePointerCell(cell PointerCell) error { diff --git a/internal/engine/storage/page/page_test.go b/internal/engine/storage/page/page_test.go index 2fd7729c..6bcc933c 100644 --- a/internal/engine/storage/page/page_test.go +++ b/internal/engine/storage/page/page_test.go @@ -404,3 +404,74 @@ func TestPage_Defragment(t *testing.T) { }) } } + +func TestPage_DeleteCell(t *testing.T) { + tests := []struct { + name string + before []byte + deleteKey []byte + after []byte + ok bool + }{ + { + "small 2 cells delete front", + []byte{ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, // header + 0x00, 0x12, 0x00, 0x0A, // offset #0 + 0x00, 0x20, 0x00, 0x0A, // offset #1 + 0x00, 0x00, 0x00, 0x00, // free space + 0x01, 0x00, 0x00, 0x00, 0x01, 0x0A, 0x00, 0x00, 0x00, 0x00, // cell #0 + 0x00, 0x00, 0x00, 0x00, // free space + 0x01, 0x00, 0x00, 0x00, 0x01, 0x0B, 0x00, 0x00, 0x00, 0x00, // cell #1 + }, + []byte{0x0A}, + []byte{ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, // header + 0x00, 0x20, 0x00, 0x0A, // offset #1 + 0x00, 0x00, 0x00, 0x00, // free space + 0x00, 0x00, 0x00, 0x00, // free space + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // free space + 0x00, 0x00, 0x00, 0x00, // free space + 0x01, 0x00, 0x00, 0x00, 0x01, 0x0B, 0x00, 0x00, 0x00, 0x00, // cell #1 + }, + true, + }, + { + "small 2 cells delete end", + []byte{ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, // header + 0x00, 0x12, 0x00, 0x0A, // offset #0 + 0x00, 0x20, 0x00, 0x0A, // offset #1 + 0x00, 0x00, 0x00, 0x00, // free space + 0x01, 0x00, 0x00, 0x00, 0x01, 0x0A, 0x00, 0x00, 0x00, 0x00, // cell #0 + 0x00, 0x00, 0x00, 0x00, // free space + 0x01, 0x00, 0x00, 0x00, 0x01, 0x0B, 0x00, 0x00, 0x00, 0x00, // cell #1 + }, + []byte{0x0B}, + []byte{ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, // header + 0x00, 0x12, 0x00, 0x0A, // offset #0 + 0x00, 0x00, 0x00, 0x00, // free space + 0x00, 0x00, 0x00, 0x00, // free space + 0x01, 0x00, 0x00, 0x00, 0x01, 0x0A, 0x00, 0x00, 0x00, 0x00, // cell #0 + 0x00, 0x00, 0x00, 0x00, // free space + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // free space + }, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + p, err := load(tt.before) + assert.NoError(err) + + assert.Equal(tt.before, p.data) + ok, err := p.DeleteCell(tt.deleteKey) + assert.Equal(tt.ok, ok) + assert.NoError(err) + assert.Equal(tt.after, p.data) + }) + } +} From c18ffa544c40cb721da54b3cfc5c7d6237612ab7 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 23 Jun 2020 12:27:27 +0200 Subject: [PATCH 26/77] Only flush if page is dirty --- internal/engine/storage/cache/lru.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/engine/storage/cache/lru.go b/internal/engine/storage/cache/lru.go index 1534e0eb..cd303ada 100644 --- a/internal/engine/storage/cache/lru.go +++ b/internal/engine/storage/cache/lru.go @@ -126,6 +126,10 @@ func (c *LRUCache) evict(id uint32) error { } func (c *LRUCache) flush(id page.ID) error { + page, ok := c.pages[id] + if !ok || !page.Dirty() { + return nil + } if err := c.store.WritePage(c.pages[id]); err != nil { return fmt.Errorf("write page: %w", err) } From 154327f7cec4d3757d6583e1d7e2319385974d94 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 23 Jun 2020 12:44:11 +0200 Subject: [PATCH 27/77] Add more validating functions --- internal/engine/storage/validator.go | 42 +++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/internal/engine/storage/validator.go b/internal/engine/storage/validator.go index a4cc46ed..a6a433ba 100644 --- a/internal/engine/storage/validator.go +++ b/internal/engine/storage/validator.go @@ -34,11 +34,19 @@ func (v *Validator) Validate() error { } v.info = stat - if err := v.validateIsFile(); err != nil { - return fmt.Errorf("is file: %w", err) + validations := []struct { + name string + validator func() error + }{ + {"is file", v.validateIsFile}, + {"size", v.validateSize}, + {"page count", v.validatePageCount}, } - if err := v.validateSize(); err != nil { - return fmt.Errorf("size: %w", err) + + for _, validation := range validations { + if err := validation.validator(); err != nil { + return fmt.Errorf("%v: %w", validation.name, err) + } } return nil @@ -61,3 +69,29 @@ func (v Validator) validateSize() error { } return nil } + +func (v Validator) validatePageCount() error { + mgr, err := NewPageManager(v.file) + if err != nil { + return fmt.Errorf("new page manager: %w", err) + } + + headerPage, err := mgr.ReadPage(HeaderPageID) + if err != nil { + return fmt.Errorf("read header page: %w", err) + } + + val, ok := headerPage.Cell([]byte(HeaderPageCount)) + if !ok { + return fmt.Errorf("no page count header field in header page") + } + pageCountField, ok := val.(page.RecordCell) + if !ok { + return fmt.Errorf("page count cell is not a record cell (%v)", val.Type()) + } + pageCount := byteOrder.Uint64(pageCountField.Record) + if int64(pageCount) != v.info.Size()/page.Size { + return fmt.Errorf("page count does not match file size (pageCount=%v,size=%v,expected count=%v)", pageCount, v.info.Size(), v.info.Size()/page.Size) + } + return nil +} From 2a26df8a0195da1dd427e07c331923651e170daa Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 23 Jun 2020 18:52:26 +0200 Subject: [PATCH 28/77] Add type system for runtime --- internal/engine/basetype_string.go | 27 ++++++++++++ internal/engine/cmpresult_string.go | 26 ++++++++++++ internal/engine/compare.go | 65 +++++++++++++++++++++++++++++ internal/engine/compare_test.go | 51 ++++++++++++++++++++++ internal/engine/engine.go | 35 ++++++++++++++++ internal/engine/expression.go | 19 +++++++++ internal/engine/expression_test.go | 53 +++++++++++++++++++++++ internal/engine/option.go | 13 ++++++ internal/engine/result.go | 7 ++++ internal/engine/type.go | 20 +++++++++ internal/engine/type_blob.go | 39 +++++++++++++++++ internal/engine/type_bool.go | 46 ++++++++++++++++++++ internal/engine/type_descriptor.go | 39 +++++++++++++++++ internal/engine/type_string.go | 39 +++++++++++++++++ internal/engine/value.go | 7 ++++ 15 files changed, 486 insertions(+) create mode 100644 internal/engine/basetype_string.go create mode 100644 internal/engine/cmpresult_string.go create mode 100644 internal/engine/compare.go create mode 100644 internal/engine/compare_test.go create mode 100644 internal/engine/expression.go create mode 100644 internal/engine/expression_test.go create mode 100644 internal/engine/option.go create mode 100644 internal/engine/result.go create mode 100644 internal/engine/type.go create mode 100644 internal/engine/type_blob.go create mode 100644 internal/engine/type_bool.go create mode 100644 internal/engine/type_descriptor.go create mode 100644 internal/engine/type_string.go create mode 100644 internal/engine/value.go diff --git a/internal/engine/basetype_string.go b/internal/engine/basetype_string.go new file mode 100644 index 00000000..c21ab891 --- /dev/null +++ b/internal/engine/basetype_string.go @@ -0,0 +1,27 @@ +// Code generated by "stringer -type=BaseType"; DO NOT EDIT. + +package engine + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[BaseTypeUnknown-0] + _ = x[BaseTypeBool-1] + _ = x[BaseTypeBinary-2] + _ = x[BaseTypeString-3] + _ = x[BaseTypeNumber-4] +} + +const _BaseType_name = "BaseTypeUnknownBaseTypeBoolBaseTypeBinaryBaseTypeStringBaseTypeNumber" + +var _BaseType_index = [...]uint8{0, 15, 27, 41, 55, 69} + +func (i BaseType) String() string { + if i >= BaseType(len(_BaseType_index)-1) { + return "BaseType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _BaseType_name[_BaseType_index[i]:_BaseType_index[i+1]] +} diff --git a/internal/engine/cmpresult_string.go b/internal/engine/cmpresult_string.go new file mode 100644 index 00000000..f9e5c6ee --- /dev/null +++ b/internal/engine/cmpresult_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=cmpResult"; DO NOT EDIT. + +package engine + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[cmpUncomparable-0] + _ = x[cmpEqual-1] + _ = x[cmpLessThan-2] + _ = x[cmpGreaterThan-3] +} + +const _cmpResult_name = "cmpUncomparablecmpEqualcmpLessThancmpGreaterThan" + +var _cmpResult_index = [...]uint8{0, 15, 23, 34, 48} + +func (i cmpResult) String() string { + if i >= cmpResult(len(_cmpResult_index)-1) { + return "cmpResult(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _cmpResult_name[_cmpResult_index[i]:_cmpResult_index[i+1]] +} diff --git a/internal/engine/compare.go b/internal/engine/compare.go new file mode 100644 index 00000000..288e0386 --- /dev/null +++ b/internal/engine/compare.go @@ -0,0 +1,65 @@ +package engine + +//go:generate stringer -type=cmpResult + +type cmpResult uint8 + +const ( + cmpUncomparable cmpResult = iota + cmpEqual + cmpLessThan + cmpGreaterThan +) + +// cmp compares two values. The result is to be interpreted as R(left, right) or +// left~right, meaning if e.g. cmpLessThan is returned, it is to be understood +// as left= (greater +// than or equal) relation, see (Engine).gteq. +func (e Engine) gt(left, right Value) bool { + return e.lt(right, left) +} + +// lteq checks if the left value is smaller than or equal to the right value. +func (e Engine) lteq(left, right Value) bool { + return e.eq(left, right) || e.lt(left, right) +} + +// gteq checks if the right value is smaller than or equal to the left value. +func (e Engine) gteq(left, right Value) bool { + return e.eq(left, right) || e.gt(left, right) +} diff --git a/internal/engine/compare_test.go b/internal/engine/compare_test.go new file mode 100644 index 00000000..1300d1f4 --- /dev/null +++ b/internal/engine/compare_test.go @@ -0,0 +1,51 @@ +package engine + +import ( + "testing" + + "github.com/rs/zerolog" +) + +func TestEngine_cmp(t *testing.T) { + tests := []struct { + name string + left Value + right Value + want cmpResult + }{ + { + "true <-> true", + BoolValue{Value: true}, + BoolValue{Value: true}, + cmpEqual, + }, + { + "true <-> false", + BoolValue{Value: true}, + BoolValue{Value: false}, + cmpGreaterThan, + }, + { + "false <-> true", + BoolValue{Value: false}, + BoolValue{Value: true}, + cmpLessThan, + }, + { + "false <-> false", + BoolValue{Value: false}, + BoolValue{Value: false}, + cmpEqual, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := Engine{ + log: zerolog.Nop(), + } + if got := e.cmp(tt.left, tt.right); got != tt.want { + t.Errorf("Engine.cmp() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 00a22ef6..0f98bf8a 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -1 +1,36 @@ package engine + +import ( + "github.com/rs/zerolog" + "github.com/tomarrell/lbadd/internal/compiler/command" + "github.com/tomarrell/lbadd/internal/engine/storage" +) + +// Engine is the component that is used to evaluate commands. +type Engine struct { + log zerolog.Logger + dbFile *storage.DBFile +} + +// New creates a new engine object and applies the given options to it. +func New(dbFile *storage.DBFile, opts ...Option) (*Engine, error) { + e := &Engine{ + log: zerolog.Nop(), + dbFile: dbFile, + } + for _, opt := range opts { + opt(e) + } + return e, nil +} + +// Evaluate evaluates the given command. This may mutate the state of the +// database, and changes may occur to the database file. +func (e Engine) Evaluate(cmd command.Command) (Result, error) { + _ = e.eq + _ = e.lt + _ = e.gt + _ = e.lteq + _ = e.gteq + return nil, nil +} diff --git a/internal/engine/expression.go b/internal/engine/expression.go new file mode 100644 index 00000000..d43022f6 --- /dev/null +++ b/internal/engine/expression.go @@ -0,0 +1,19 @@ +package engine + +import ( + "fmt" + + "github.com/tomarrell/lbadd/internal/compiler/command" +) + +// evaluateExpression evaluates the given expression to a runtime-constant +// value, meaning that it can only be evaluated to a constant value with a given +// execution context. This execution context must be inferred from the engine +// receiver. +func (e Engine) evaluateExpression(expr command.Expr) (Value, error) { + switch ex := expr.(type) { + case command.ConstantBooleanExpr: + return BoolValue{Value: ex.Value}, nil + } + return nil, fmt.Errorf("cannot evaluate expression of type %T", expr) +} diff --git a/internal/engine/expression_test.go b/internal/engine/expression_test.go new file mode 100644 index 00000000..434bed54 --- /dev/null +++ b/internal/engine/expression_test.go @@ -0,0 +1,53 @@ +package engine + +import ( + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/compiler/command" +) + +func TestEngine_evaluateExpression(t *testing.T) { + tests := []struct { + name string + expr command.Expr + want Value + wantErr bool + }{ + { + "nil", + nil, + nil, + true, + }, + { + "true", + command.ConstantBooleanExpr{Value: true}, + BoolValue{Value: true}, + false, + }, + { + "false", + command.ConstantBooleanExpr{Value: false}, + BoolValue{Value: false}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + e := Engine{ + log: zerolog.Nop(), + } + got, err := e.evaluateExpression(tt.expr) + assert.Equal(tt.want, got) + if tt.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + } + }) + } +} diff --git a/internal/engine/option.go b/internal/engine/option.go new file mode 100644 index 00000000..097920d8 --- /dev/null +++ b/internal/engine/option.go @@ -0,0 +1,13 @@ +package engine + +import "github.com/rs/zerolog" + +// Option is an option that can is applied to an Engine on creation. +type Option func(*Engine) + +// WithLogger specifies a logger for the Engine. +func WithLogger(log zerolog.Logger) Option { + return func(e *Engine) { + e.log = log + } +} diff --git a/internal/engine/result.go b/internal/engine/result.go new file mode 100644 index 00000000..aba93c0f --- /dev/null +++ b/internal/engine/result.go @@ -0,0 +1,7 @@ +package engine + +// Result represents an evaluation result of the engine. It is a mxn-matrix, +// where m and n is variable and depends on the passed in command. +type Result interface { + // not defined yet +} diff --git a/internal/engine/type.go b/internal/engine/type.go new file mode 100644 index 00000000..eabfef73 --- /dev/null +++ b/internal/engine/type.go @@ -0,0 +1,20 @@ +package engine + +type ( + // Comparator is the interface that wraps the basic compare method. The + // compare method compares the left and right value as follows. -1 if + // leftright. What exectly is considered + // to be <, ==, > is up to the implementation. + Comparator interface { + Compare(Value, Value) (int, error) + } + + // Type is a data type that consists of a type descriptor and a comparator. + // The comparator forces types to define relations between two values of + // this type. A type is only expected to be able to handle values of its own + // type. + Type interface { + TypeDescriptor + Comparator + } +) diff --git a/internal/engine/type_blob.go b/internal/engine/type_blob.go new file mode 100644 index 00000000..2c9e9419 --- /dev/null +++ b/internal/engine/type_blob.go @@ -0,0 +1,39 @@ +package engine + +import "bytes" + +var ( + blobType = BlobType{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeBinary, + }, + } +) + +var _ Type = (*BlobType)(nil) +var _ Value = (*BlobValue)(nil) + +type ( + // BlobType is the type for Binary Large OBjects. The value is basically a + // byte slice. + BlobType struct { + genericTypeDescriptor + } + + // BlobValue is a value of type BlobType. + BlobValue struct { + // Value is the primitive value of this value object. + Value []byte + } +) + +// Compare for the BlobType is defined the lexicographical comparison between +// the primitive underlying values. +func (BlobType) Compare(left, right Value) (int, error) { + leftBlob := left.(BlobValue).Value + rightBlob := right.(BlobValue).Value + return bytes.Compare(leftBlob, rightBlob), nil +} + +// Type returns a blob type. +func (BlobValue) Type() Type { return blobType } diff --git a/internal/engine/type_bool.go b/internal/engine/type_bool.go new file mode 100644 index 00000000..be82b07f --- /dev/null +++ b/internal/engine/type_bool.go @@ -0,0 +1,46 @@ +package engine + +import "fmt" + +var ( + boolType = BoolType{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeBool, + }, + } +) + +var _ Type = (*BoolType)(nil) +var _ Value = (*BoolValue)(nil) + +type ( + // BoolType is the boolean type of this engine. + BoolType struct { + genericTypeDescriptor + } + + // BoolValue is a value of type BoolType. + BoolValue struct { + // Value is the underlying primitive value. + Value bool + } +) + +// Compare for the type BoolType is defined as false %v", leftBool, rightBool) +} + +// Type returns a bool type. +func (BoolValue) Type() Type { return boolType } diff --git a/internal/engine/type_descriptor.go b/internal/engine/type_descriptor.go new file mode 100644 index 00000000..1012300c --- /dev/null +++ b/internal/engine/type_descriptor.go @@ -0,0 +1,39 @@ +package engine + +//go:generate stringer -type=BaseType + +// BaseType is an underlying type for parameterized types. +type BaseType uint8 + +// Known base types. +const ( + BaseTypeUnknown BaseType = iota + BaseTypeBool + BaseTypeBinary + BaseTypeString + BaseTypeNumber +) + +// TypeDescriptor describes a type in more detail than just the type. Every type +// has a type descriptor, which holds the base type and parameterization. Based +// on the parameterization, the type may be interpreted differently. +// +// Example: The simple type INTEGER would have a type descriptor that describes +// a baseType=number with no further parameterization, whereas the more complex +// type VARCHAR(50) would have a type descriptor, that describes a +// baseType=string and a max length of 50. +type TypeDescriptor interface { + Base() BaseType + // TODO: parameters to be done +} + +// genericTypeDescriptor is a type descriptor that has no parameterization and +// just a base type. +type genericTypeDescriptor struct { + baseType BaseType +} + +// Base returns the base type of this type descriptor. +func (td genericTypeDescriptor) Base() BaseType { + return td.baseType +} diff --git a/internal/engine/type_string.go b/internal/engine/type_string.go new file mode 100644 index 00000000..0279b4b3 --- /dev/null +++ b/internal/engine/type_string.go @@ -0,0 +1,39 @@ +package engine + +import "strings" + +var ( + stringType = StringType{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeString, + }, + } +) + +var _ Type = (*StringType)(nil) +var _ Value = (*StringValue)(nil) + +type ( + // StringType is the type for parameterized and non-parameterized string + // types, such as VARCHAR or VARCHAR(n). + StringType struct { + genericTypeDescriptor + } + + // StringValue is a value of type StringType. + StringValue struct { + // Value is the underlying primitive value. + Value string + } +) + +// Compare for the StringType is defined as the lexicographical comparison of +// the two underlying primitive values. +func (StringType) Compare(left, right Value) (int, error) { + leftString := left.(StringValue).Value + rightString := right.(StringValue).Value + return strings.Compare(leftString, rightString), nil +} + +// Type returns a string type. +func (StringValue) Type() Type { return blobType } diff --git a/internal/engine/value.go b/internal/engine/value.go new file mode 100644 index 00000000..575c3260 --- /dev/null +++ b/internal/engine/value.go @@ -0,0 +1,7 @@ +package engine + +// Value describes an object that can be used as a value in the engine. A value +// has a type, which can be used to compare this value to another one. +type Value interface { + Type() Type +} From cb798c1f557b5b1675baf319f4f4fd0c9865550a Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 23 Jun 2020 18:56:54 +0200 Subject: [PATCH 29/77] Add pageCache to engine --- internal/engine/engine.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 0f98bf8a..9a682b7e 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -4,19 +4,22 @@ import ( "github.com/rs/zerolog" "github.com/tomarrell/lbadd/internal/compiler/command" "github.com/tomarrell/lbadd/internal/engine/storage" + "github.com/tomarrell/lbadd/internal/engine/storage/cache" ) // Engine is the component that is used to evaluate commands. type Engine struct { - log zerolog.Logger - dbFile *storage.DBFile + log zerolog.Logger + dbFile *storage.DBFile + pageCache cache.Cache } // New creates a new engine object and applies the given options to it. func New(dbFile *storage.DBFile, opts ...Option) (*Engine, error) { e := &Engine{ - log: zerolog.Nop(), - dbFile: dbFile, + log: zerolog.Nop(), + dbFile: dbFile, + pageCache: dbFile.Cache(), } for _, opt := range opts { opt(e) From 12a388ba869120b4ba57e9fc1a0321772f97a6b3 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 23 Jun 2020 20:21:37 +0200 Subject: [PATCH 30/77] Add cache tests --- internal/engine/doc.go | 2 + internal/engine/engine.go | 12 +++ internal/engine/error.go | 11 +++ internal/engine/storage/cache/lru.go | 44 +++++---- internal/engine/storage/cache/lru_test.go | 94 +++++++++++++++++++ .../cache/mock_secondary_storage_test.go | 50 ++++++++++ .../engine/storage/cache/secondary_storage.go | 12 +++ 7 files changed, 208 insertions(+), 17 deletions(-) create mode 100644 internal/engine/doc.go create mode 100644 internal/engine/error.go create mode 100644 internal/engine/storage/cache/lru_test.go create mode 100644 internal/engine/storage/cache/mock_secondary_storage_test.go create mode 100644 internal/engine/storage/cache/secondary_storage.go diff --git a/internal/engine/doc.go b/internal/engine/doc.go new file mode 100644 index 00000000..3b7f4e9f --- /dev/null +++ b/internal/engine/doc.go @@ -0,0 +1,2 @@ +// Package engine implement an engine that can execute a command. +package engine diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 9a682b7e..f8b81e84 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -37,3 +37,15 @@ func (e Engine) Evaluate(cmd command.Command) (Result, error) { _ = e.gteq return nil, nil } + +// Closed determines whether the underlying database file was closed. If so, +// this engine is considered closed, as it can no longer operate on the +// underlying file. +func (e Engine) Closed() bool { + return e.dbFile.Closed() +} + +// Close closes the underlying database file. +func (e Engine) Close() error { + return e.dbFile.Close() +} diff --git a/internal/engine/error.go b/internal/engine/error.go new file mode 100644 index 00000000..092f9a8f --- /dev/null +++ b/internal/engine/error.go @@ -0,0 +1,11 @@ +package engine + +// Error is a sentinel error. +type Error string + +func (e Error) Error() string { return string(e) } + +// Sentinel errors. +const ( + ErrClosed Error = "already closed" +) diff --git a/internal/engine/storage/cache/lru.go b/internal/engine/storage/cache/lru.go index cd303ada..9c9dc711 100644 --- a/internal/engine/storage/cache/lru.go +++ b/internal/engine/storage/cache/lru.go @@ -2,26 +2,21 @@ package cache import ( "fmt" + "sync" "github.com/tomarrell/lbadd/internal/engine/storage/page" ) -// SecondaryStorage is the abstraction that a cache uses, to synchronize dirty -// pages with secondary storage. -type SecondaryStorage interface { - ReadPage(page.ID) (*page.Page, error) - WritePage(*page.Page) error -} - var _ Cache = (*LRUCache)(nil) // LRUCache is a simple implementation of an LRU cache. type LRUCache struct { - store SecondaryStorage - pages map[page.ID]*page.Page - pinned map[page.ID]struct{} - size int - lru []page.ID + store SecondaryStorage + pages map[page.ID]*page.Page + pageLocks map[page.ID]*sync.Mutex + pinned map[page.ID]struct{} + size int + lru []page.ID } // NewLRUCache creates a new LRU cache with the given size and secondary storage @@ -31,11 +26,12 @@ type LRUCache struct { // yet), requesting a new page will fail. func NewLRUCache(size int, store SecondaryStorage) *LRUCache { return &LRUCache{ - store: store, - pages: make(map[uint32]*page.Page), - pinned: make(map[uint32]struct{}), - size: size, - lru: make([]uint32, 0), + store: store, + pages: make(map[uint32]*page.Page), + pageLocks: make(map[uint32]*sync.Mutex), + pinned: make(map[uint32]struct{}), + size: size, + lru: make([]uint32, 0), } } @@ -73,6 +69,10 @@ func (c *LRUCache) Close() error { } func (c *LRUCache) fetchAndPin(id page.ID) (*page.Page, error) { + // first lock page for others + lock := c.obtainPageLock(id) + lock.Lock() // unpin unlocks this lock + // pin id first in order to avoid potential concurrent eviction at this // point c.pin(id) @@ -85,12 +85,22 @@ func (c *LRUCache) fetchAndPin(id page.ID) (*page.Page, error) { return p, nil } +func (c *LRUCache) obtainPageLock(id page.ID) *sync.Mutex { + lock, ok := c.pageLocks[id] + if !ok { + lock = new(sync.Mutex) + c.pageLocks[id] = lock + } + return lock +} + func (c *LRUCache) pin(id uint32) { c.pinned[id] = struct{}{} } func (c *LRUCache) unpin(id uint32) { delete(c.pinned, id) + c.pageLocks[id].Unlock() // unlock page lock after page is released by user } func (c *LRUCache) fetch(id uint32) (*page.Page, error) { diff --git a/internal/engine/storage/cache/lru_test.go b/internal/engine/storage/cache/lru_test.go new file mode 100644 index 00000000..0c279b7e --- /dev/null +++ b/internal/engine/storage/cache/lru_test.go @@ -0,0 +1,94 @@ +package cache + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +func TestLRUCache(t *testing.T) { + assert := assert.New(t) + + pages := make([]*page.Page, 6) + for i := range pages { + pages[i] = page.New(uint32(i)) + } + + secondaryStorage := new(MockSecondaryStorage) + secondaryStorage. + On("ReadPage", mock.IsType(page.ID(0))). + Return(func(id uint32) *page.Page { + return pages[id] + }, nil) + secondaryStorage. + On("WritePage", mock.IsType((*page.Page)(nil))). + Return(nil) + + c := NewLRUCache(5, secondaryStorage) + defer func() { _ = c.Close() }() + + // load 5 pages, fill cache up + p, err := c.FetchAndPin(0) + assert.NoError(err) + assert.Same(pages[0], p) + secondaryStorage.AssertCalled(t, "ReadPage", uint32(0)) + secondaryStorage.AssertNotCalled(t, "WritePage", mock.Anything) + + p, err = c.FetchAndPin(1) + assert.NoError(err) + assert.Same(pages[1], p) + secondaryStorage.AssertCalled(t, "ReadPage", uint32(1)) + secondaryStorage.AssertNotCalled(t, "WritePage", mock.Anything) + + p, err = c.FetchAndPin(2) + assert.NoError(err) + assert.Same(pages[2], p) + secondaryStorage.AssertCalled(t, "ReadPage", uint32(2)) + secondaryStorage.AssertNotCalled(t, "WritePage", mock.Anything) + + p, err = c.FetchAndPin(3) + assert.NoError(err) + assert.Same(pages[3], p) + secondaryStorage.AssertCalled(t, "ReadPage", uint32(3)) + secondaryStorage.AssertNotCalled(t, "WritePage", mock.Anything) + + p, err = c.FetchAndPin(4) + assert.NoError(err) + assert.Same(pages[4], p) + secondaryStorage.AssertCalled(t, "ReadPage", uint32(4)) + secondaryStorage.AssertNotCalled(t, "WritePage", mock.Anything) + + // all pages are fetched and locked now, cache can not evict any pages + p, err = c.FetchAndPin(5) + assert.Error(err) + assert.Nil(p) + secondaryStorage.AssertNotCalled(t, "ReadPage", uint32(5)) + secondaryStorage.AssertNotCalled(t, "WritePage", mock.Anything) + + // must release a page + c.Unpin(0) // unpin first page + secondaryStorage.AssertNotCalled(t, "WritePage", mock.Anything) + + // load another page + p, err = c.FetchAndPin(5) // page[5] can now be loaded + assert.NoError(err) + assert.Same(pages[5], p) + secondaryStorage.AssertNotCalled(t, "WritePage", mock.Anything) // page 0 evicted, but no writes, because page 0 was not dirty + secondaryStorage.AssertCalled(t, "ReadPage", uint32(5)) // page 5 loaded + + // mark page 1 as dirty + pages[1].MarkDirty() + // release page 1 + c.Unpin(1) + + // load another page again + p, err = c.FetchAndPin(0) // page[0] can now be loaded + assert.NoError(err) + assert.Same(pages[0], p) + secondaryStorage.AssertCalled(t, "WritePage", pages[1]) // page 1 evicted + secondaryStorage.AssertCalled(t, "ReadPage", uint32(0)) // page 0 loaded + + secondaryStorage.AssertNumberOfCalls(t, "ReadPage", 7) +} diff --git a/internal/engine/storage/cache/mock_secondary_storage_test.go b/internal/engine/storage/cache/mock_secondary_storage_test.go new file mode 100644 index 00000000..d89cfdb4 --- /dev/null +++ b/internal/engine/storage/cache/mock_secondary_storage_test.go @@ -0,0 +1,50 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package cache + +import ( + mock "github.com/stretchr/testify/mock" + page "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +// MockSecondaryStorage is an autogenerated mock type for the SecondaryStorage type +type MockSecondaryStorage struct { + mock.Mock +} + +// ReadPage provides a mock function with given fields: _a0 +func (_m *MockSecondaryStorage) ReadPage(_a0 uint32) (*page.Page, error) { + ret := _m.Called(_a0) + + var r0 *page.Page + if rf, ok := ret.Get(0).(func(uint32) *page.Page); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*page.Page) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint32) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// WritePage provides a mock function with given fields: _a0 +func (_m *MockSecondaryStorage) WritePage(_a0 *page.Page) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(*page.Page) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/internal/engine/storage/cache/secondary_storage.go b/internal/engine/storage/cache/secondary_storage.go new file mode 100644 index 00000000..2a744b0b --- /dev/null +++ b/internal/engine/storage/cache/secondary_storage.go @@ -0,0 +1,12 @@ +package cache + +import "github.com/tomarrell/lbadd/internal/engine/storage/page" + +//go:generate mockery -inpkg -case=snake -testonly -name SecondaryStorage + +// SecondaryStorage is the abstraction that a cache uses, to synchronize dirty +// pages with secondary storage. +type SecondaryStorage interface { + ReadPage(page.ID) (*page.Page, error) + WritePage(*page.Page) error +} From 16e034aaa3ccf9ecef7e85c37b3f13dae2108288 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Tue, 23 Jun 2020 20:44:08 +0200 Subject: [PATCH 31/77] Define result --- internal/engine/error.go | 3 ++- internal/engine/result.go | 25 +++++++++++++++++- internal/engine/type_numeric.go | 47 +++++++++++++++++++++++++++++++++ internal/engine/type_string.go | 2 +- 4 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 internal/engine/type_numeric.go diff --git a/internal/engine/error.go b/internal/engine/error.go index 092f9a8f..7bf7bca1 100644 --- a/internal/engine/error.go +++ b/internal/engine/error.go @@ -7,5 +7,6 @@ func (e Error) Error() string { return string(e) } // Sentinel errors. const ( - ErrClosed Error = "already closed" + ErrClosed Error = "already closed" + ErrUnsupported Error = "unsupported" ) diff --git a/internal/engine/result.go b/internal/engine/result.go index aba93c0f..88f5379a 100644 --- a/internal/engine/result.go +++ b/internal/engine/result.go @@ -1,7 +1,30 @@ package engine +import "fmt" + // Result represents an evaluation result of the engine. It is a mxn-matrix, // where m and n is variable and depends on the passed in command. type Result interface { - // not defined yet + Cols() []Column + Rows() []Row + fmt.Stringer +} + +type IndexedGetter interface { + Get(int) Value +} + +type Sizer interface { + Size() int +} + +type Column interface { + Type() Type + IndexedGetter + Sizer +} + +type Row interface { + IndexedGetter + Sizer } diff --git a/internal/engine/type_numeric.go b/internal/engine/type_numeric.go new file mode 100644 index 00000000..24011924 --- /dev/null +++ b/internal/engine/type_numeric.go @@ -0,0 +1,47 @@ +package engine + +import "fmt" + +var ( + numericType = NumericType{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeString, + }, + } +) + +var _ Type = (*NumericType)(nil) +var _ Value = (*NumericValue)(nil) + +type ( + // NumericType is the type for parameterized and non-parameterized numeric + // types, such as DECIMAL. + NumericType struct { + genericTypeDescriptor + } + + // NumericValue is a value of type NumericType. + NumericValue struct { + // Value is the underlying primitive value. + Value float64 + } +) + +// Compare for the NumericType is defined as the lexicographical comparison of +// the two underlying primitive values. +func (NumericType) Compare(left, right Value) (int, error) { + leftNum := left.(NumericValue).Value + rightNum := right.(NumericValue).Value + switch { + case leftNum < rightNum: + return -1, nil + case leftNum == rightNum: + return 0, nil + case leftNum > rightNum: + return 1, nil + } + return -2, fmt.Errorf("unhandled constellation: %v <-> %v", leftNum, rightNum) +} + +// Type returns a string type. +func (NumericValue) Type() Type { return numericType } diff --git a/internal/engine/type_string.go b/internal/engine/type_string.go index 0279b4b3..efff7093 100644 --- a/internal/engine/type_string.go +++ b/internal/engine/type_string.go @@ -36,4 +36,4 @@ func (StringType) Compare(left, right Value) (int, error) { } // Type returns a string type. -func (StringValue) Type() Type { return blobType } +func (StringValue) Type() Type { return numericType } From d92ef6e06bf9a1593109a21c1af4696039bb7632 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Wed, 24 Jun 2020 11:51:45 +0200 Subject: [PATCH 32/77] Add a simple profiler for the engine --- internal/engine/engine.go | 12 +++-- internal/engine/engine_test.go | 65 ++++++++++++++++++++++++ internal/engine/evaluate.go | 31 ++++++++++++ internal/engine/option.go | 11 ++++- internal/engine/profile/profile.go | 5 ++ internal/engine/profile/profiler.go | 76 +++++++++++++++++++++++++++++ internal/engine/profiling.go | 25 ++++++++++ 7 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 internal/engine/engine_test.go create mode 100644 internal/engine/evaluate.go create mode 100644 internal/engine/profile/profile.go create mode 100644 internal/engine/profile/profiler.go create mode 100644 internal/engine/profiling.go diff --git a/internal/engine/engine.go b/internal/engine/engine.go index f8b81e84..8db77e31 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -3,6 +3,7 @@ package engine import ( "github.com/rs/zerolog" "github.com/tomarrell/lbadd/internal/compiler/command" + "github.com/tomarrell/lbadd/internal/engine/profile" "github.com/tomarrell/lbadd/internal/engine/storage" "github.com/tomarrell/lbadd/internal/engine/storage/cache" ) @@ -12,17 +13,18 @@ type Engine struct { log zerolog.Logger dbFile *storage.DBFile pageCache cache.Cache + profiler *profile.Profiler } // New creates a new engine object and applies the given options to it. -func New(dbFile *storage.DBFile, opts ...Option) (*Engine, error) { - e := &Engine{ +func New(dbFile *storage.DBFile, opts ...Option) (Engine, error) { + e := Engine{ log: zerolog.Nop(), dbFile: dbFile, pageCache: dbFile.Cache(), } for _, opt := range opts { - opt(e) + opt(&e) } return e, nil } @@ -30,12 +32,14 @@ func New(dbFile *storage.DBFile, opts ...Option) (*Engine, error) { // Evaluate evaluates the given command. This may mutate the state of the // database, and changes may occur to the database file. func (e Engine) Evaluate(cmd command.Command) (Result, error) { + defer e.profiler.Enter(EvtEvaluate).Exit() + _ = e.eq _ = e.lt _ = e.gt _ = e.lteq _ = e.gteq - return nil, nil + return e.evaluate(cmd) } // Closed determines whether the underlying database file was closed. If so, diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go new file mode 100644 index 00000000..6ec61309 --- /dev/null +++ b/internal/engine/engine_test.go @@ -0,0 +1,65 @@ +package engine + +import ( + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/compiler/command" + "github.com/tomarrell/lbadd/internal/engine/storage" +) + +func TestEngine(t *testing.T) { + assert := assert.New(t) + + fs := afero.NewMemMapFs() + f, err := fs.Create("mydbfile") + assert.NoError(err) + dbFile, err := storage.Create(f) + assert.NoError(err) + + e, err := New(dbFile) + assert.NoError(err) + result, err := e.Evaluate(command.Values{ + Values: [][]command.Expr{ + {command.LiteralExpr{Value: "hello"}, command.LiteralExpr{Value: "world"}, command.ConstantBooleanExpr{Value: true}}, + {command.LiteralExpr{Value: "foo"}, command.LiteralExpr{Value: "bar"}, command.ConstantBooleanExpr{Value: false}}, + }, + }) + assert.NoError(err) + assert.NotNil(result) + + // check cols + cols := result.Cols() + assert.Len(cols, 3) + // col[0] + assert.Equal(2, cols[0].Size()) + assert.Equal(stringType, cols[0].Type()) + // col[1] + assert.Equal(2, cols[1].Size()) + assert.Equal(stringType, cols[1].Type()) + // col[2] + assert.Equal(2, cols[2].Size()) + assert.Equal(numericType, cols[2].Type()) + // col value types + assert.Equal(cols[0].Type(), cols[0].Get(0).Type()) + assert.Equal(cols[0].Type(), cols[0].Get(1).Type()) + assert.Equal(cols[1].Type(), cols[1].Get(0).Type()) + assert.Equal(cols[1].Type(), cols[1].Get(1).Type()) + assert.Equal(cols[2].Type(), cols[2].Get(0).Type()) + assert.Equal(cols[2].Type(), cols[2].Get(1).Type()) + + // check rows + rows := result.Rows() + assert.Len(rows, 2) + // row[0] + assert.Equal(3, rows[0].Size()) + assert.Equal("hello", rows[0].Get(0).(StringValue).Value) + assert.Equal("world", rows[0].Get(1).(StringValue).Value) + assert.Equal(true, rows[0].Get(2).(BoolValue).Value) + // row[0] + assert.Equal(3, rows[1].Size()) + assert.Equal("foo", rows[1].Get(0).(StringValue).Value) + assert.Equal("bar", rows[1].Get(1).(StringValue).Value) + assert.Equal(false, rows[1].Get(2).(BoolValue).Value) +} diff --git a/internal/engine/evaluate.go b/internal/engine/evaluate.go new file mode 100644 index 00000000..d4ae5faa --- /dev/null +++ b/internal/engine/evaluate.go @@ -0,0 +1,31 @@ +package engine + +import ( + "fmt" + + "github.com/tomarrell/lbadd/internal/compiler/command" +) + +func (e Engine) evaluate(c command.Command) (Result, error) { + switch cmd := c.(type) { + case command.Values: + _ = cmd + } + return nil, nil +} + +func (e Engine) evaluateValues(v command.Values) ([][]Value, error) { + result := make([][]Value, len(v.Values)) + for y, values := range v.Values { + rowValues := make([]Value, len(values)) + for x, value := range values { + internalValue, err := e.evaluateExpression(value) + if err != nil { + return nil, fmt.Errorf("expr: %w", err) + } + rowValues[x] = internalValue + } + result[y] = rowValues + } + return result, nil +} diff --git a/internal/engine/option.go b/internal/engine/option.go index 097920d8..a4b97db7 100644 --- a/internal/engine/option.go +++ b/internal/engine/option.go @@ -1,6 +1,9 @@ package engine -import "github.com/rs/zerolog" +import ( + "github.com/rs/zerolog" + "github.com/tomarrell/lbadd/internal/engine/profile" +) // Option is an option that can is applied to an Engine on creation. type Option func(*Engine) @@ -11,3 +14,9 @@ func WithLogger(log zerolog.Logger) Option { e.log = log } } + +func WithProfiler(profiler *profile.Profiler) Option { + return func(e *Engine) { + e.profiler = profiler + } +} diff --git a/internal/engine/profile/profile.go b/internal/engine/profile/profile.go new file mode 100644 index 00000000..397b0153 --- /dev/null +++ b/internal/engine/profile/profile.go @@ -0,0 +1,5 @@ +package profile + +type Profile struct { + Events []Event +} diff --git a/internal/engine/profile/profiler.go b/internal/engine/profile/profiler.go new file mode 100644 index 00000000..9daa0b13 --- /dev/null +++ b/internal/engine/profile/profiler.go @@ -0,0 +1,76 @@ +package profile + +import ( + "fmt" + "time" +) + +type Clearer interface { + Clear() +} + +func NewProfiler() *Profiler { + return &Profiler{} +} + +type Profiler struct { + events []Event +} + +type Event struct { + origin *Profiler + Object fmt.Stringer + Start time.Time + Duration time.Duration +} + +func (p *Profiler) Enter(object fmt.Stringer) Event { + if p == nil { + return Event{} + } + + return Event{ + origin: p, + Object: object, + Start: time.Now(), + } +} + +func (p *Profiler) Exit(evt Event) { + if p == nil { + return + } + + p.events = append(p.events, evt) +} + +func (p *Profiler) Clear() { + if p == nil { + return + } + + p.events = nil +} + +func (p *Profiler) Profile() Profile { + if p == nil { + return Profile{} + } + + return Profile{ + Events: p.events, + } +} + +func (e Event) Exit() { + if e.origin == nil { + return + } + + e.origin.Exit(Event{ + origin: e.origin, + Object: e.Object, + Start: e.Start, + Duration: time.Since(e.Start), + }) +} diff --git a/internal/engine/profiling.go b/internal/engine/profiling.go new file mode 100644 index 00000000..01640845 --- /dev/null +++ b/internal/engine/profiling.go @@ -0,0 +1,25 @@ +package engine + +type Evt string + +func (e Evt) String() string { return string(e) } + +type ParameterizedEvt struct { + Name string + Param string +} + +func (e ParameterizedEvt) String() string { + return e.Name + "[" + e.Param + "]" +} + +const ( + EvtEvaluate Evt = "evaluate" +) + +func EvtFullTableScan(tableName string) ParameterizedEvt { + return ParameterizedEvt{ + Name: "full table scan", + Param: "table=" + tableName, + } +} From fe281987ffa2a144815f4c51a6bdee45af131507 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Wed, 24 Jun 2020 17:25:20 +0200 Subject: [PATCH 33/77] Implement a type system --- internal/engine/types/basetype_string.go | 28 +++++++++++ internal/engine/types/doc.go | 4 ++ internal/engine/types/error.go | 14 ++++++ internal/engine/types/type.go | 23 +++++++++ internal/engine/types/type_blob.go | 53 ++++++++++++++++++++ internal/engine/types/type_bool.go | 59 ++++++++++++++++++++++ internal/engine/types/type_descriptor.go | 40 +++++++++++++++ internal/engine/types/type_function.go | 63 ++++++++++++++++++++++++ internal/engine/types/type_numeric.go | 60 ++++++++++++++++++++++ internal/engine/types/type_string.go | 52 +++++++++++++++++++ internal/engine/types/value.go | 13 +++++ 11 files changed, 409 insertions(+) create mode 100644 internal/engine/types/basetype_string.go create mode 100644 internal/engine/types/doc.go create mode 100644 internal/engine/types/error.go create mode 100644 internal/engine/types/type.go create mode 100644 internal/engine/types/type_blob.go create mode 100644 internal/engine/types/type_bool.go create mode 100644 internal/engine/types/type_descriptor.go create mode 100644 internal/engine/types/type_function.go create mode 100644 internal/engine/types/type_numeric.go create mode 100644 internal/engine/types/type_string.go create mode 100644 internal/engine/types/value.go diff --git a/internal/engine/types/basetype_string.go b/internal/engine/types/basetype_string.go new file mode 100644 index 00000000..6d6404b1 --- /dev/null +++ b/internal/engine/types/basetype_string.go @@ -0,0 +1,28 @@ +// Code generated by "stringer -type=BaseType"; DO NOT EDIT. + +package types + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[BaseTypeUnknown-0] + _ = x[BaseTypeBool-1] + _ = x[BaseTypeBinary-2] + _ = x[BaseTypeString-3] + _ = x[BaseTypeNumeric-4] + _ = x[BaseTypeFunction-5] +} + +const _BaseType_name = "BaseTypeUnknownBaseTypeBoolBaseTypeBinaryBaseTypeStringBaseTypeNumericBaseTypeFunction" + +var _BaseType_index = [...]uint8{0, 15, 27, 41, 55, 70, 86} + +func (i BaseType) String() string { + if i >= BaseType(len(_BaseType_index)-1) { + return "BaseType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _BaseType_name[_BaseType_index[i]:_BaseType_index[i+1]] +} diff --git a/internal/engine/types/doc.go b/internal/engine/types/doc.go new file mode 100644 index 00000000..232ae1fe --- /dev/null +++ b/internal/engine/types/doc.go @@ -0,0 +1,4 @@ +// Package types provides a type system for the lbadd engine. Values have a +// type, and types are described by type descriptors. Types define operations on +// their values, such as comparison. +package types diff --git a/internal/engine/types/error.go b/internal/engine/types/error.go new file mode 100644 index 00000000..7bcb4a3f --- /dev/null +++ b/internal/engine/types/error.go @@ -0,0 +1,14 @@ +package types + +import "fmt" + +// Error is a sentinel error. +type Error string + +func (e Error) Error() string { return string(e) } + +// ErrTypeMismatch returns an error that indicates a type mismatch, and includes +// the expected and the actual type. +func ErrTypeMismatch(expected, got Type) Error { + return Error(fmt.Sprintf("type mismatch: want %v, got %v", expected, got)) +} diff --git a/internal/engine/types/type.go b/internal/engine/types/type.go new file mode 100644 index 00000000..c2f778d5 --- /dev/null +++ b/internal/engine/types/type.go @@ -0,0 +1,23 @@ +package types + +import "fmt" + +type ( + // Comparator is the interface that wraps the basic compare method. The + // compare method compares the left and right value as follows. -1 if + // leftright. What exectly is considered + // to be <, ==, > is up to the implementation. + Comparator interface { + Compare(Value, Value) (int, error) + } + + // Type is a data type that consists of a type descriptor and a comparator. + // The comparator forces types to define relations between two values of + // this type. A type is only expected to be able to handle values of its own + // type. + Type interface { + TypeDescriptor + Comparator + fmt.Stringer + } +) diff --git a/internal/engine/types/type_blob.go b/internal/engine/types/type_blob.go new file mode 100644 index 00000000..003739f2 --- /dev/null +++ b/internal/engine/types/type_blob.go @@ -0,0 +1,53 @@ +package types + +import "bytes" + +var ( + // Blob is the blob type. A Blob is a Binary Large OBject, and its base type + // is a byte slice. + Blob = BlobTypeDescriptor{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeBinary, + }, + } +) + +var _ Type = (*BlobTypeDescriptor)(nil) +var _ Value = (*BlobValue)(nil) + +type ( + // BlobTypeDescriptor is the type descriptor for Binary Large OBjects. The + // value is basically a byte slice. + BlobTypeDescriptor struct { + genericTypeDescriptor + } + + // BlobValue is a value of type Blob. + BlobValue struct { + // Value is the primitive value of this value object. + Value []byte + } +) + +// Compare for the Blob is defined the lexicographical comparison between +// the primitive underlying values. +func (BlobTypeDescriptor) Compare(left, right Value) (int, error) { + if !left.Is(Blob) { + return 0, ErrTypeMismatch(Blob, left.Type()) + } + if !right.Is(Blob) { + return 0, ErrTypeMismatch(Blob, right.Type()) + } + + leftBlob := left.(BlobValue).Value + rightBlob := right.(BlobValue).Value + return bytes.Compare(leftBlob, rightBlob), nil +} + +func (BlobTypeDescriptor) String() string { return "Blob" } + +// Type returns a blob type. +func (BlobValue) Type() Type { return Blob } + +// Is checks if this value is of type Blob. +func (BlobValue) Is(t Type) bool { return t == Blob } diff --git a/internal/engine/types/type_bool.go b/internal/engine/types/type_bool.go new file mode 100644 index 00000000..86d151bd --- /dev/null +++ b/internal/engine/types/type_bool.go @@ -0,0 +1,59 @@ +package types + +import "fmt" + +var ( + // Bool is the boolean type. Its base type is a bool. + Bool = BoolTypeDescriptor{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeBool, + }, + } +) + +var _ Type = (*BoolTypeDescriptor)(nil) +var _ Value = (*BoolValue)(nil) + +type ( + // BoolTypeDescriptor is the boolean type of this engine. + BoolTypeDescriptor struct { + genericTypeDescriptor + } + + // BoolValue is a value of type Bool. + BoolValue struct { + // Value is the underlying primitive value. + Value bool + } +) + +// Compare for the type BoolType is defined as false %v", leftBool, rightBool) +} + +func (BoolTypeDescriptor) String() string { return "Bool" } + +// Type returns a bool type. +func (BoolValue) Type() Type { return Bool } + +// Is checks if this value is of type Bool. +func (BoolValue) Is(t Type) bool { return t == Bool } diff --git a/internal/engine/types/type_descriptor.go b/internal/engine/types/type_descriptor.go new file mode 100644 index 00000000..ab0ca6d5 --- /dev/null +++ b/internal/engine/types/type_descriptor.go @@ -0,0 +1,40 @@ +package types + +//go:generate stringer -type=BaseType + +// BaseType is an underlying type for parameterized types. +type BaseType uint8 + +// Known base types. +const ( + BaseTypeUnknown BaseType = iota + BaseTypeBool + BaseTypeBinary + BaseTypeString + BaseTypeNumeric + BaseTypeFunction +) + +// TypeDescriptor describes a type in more detail than just the type. Every type +// has a type descriptor, which holds the base type and parameterization. Based +// on the parameterization, the type may be interpreted differently. +// +// Example: The simple type INTEGER would have a type descriptor that describes +// a baseType=number with no further parameterization, whereas the more complex +// type VARCHAR(50) would have a type descriptor, that describes a +// baseType=string and a max length of 50. +type TypeDescriptor interface { + Base() BaseType + // TODO: parameters to be done +} + +// genericTypeDescriptor is a type descriptor that has no parameterization and +// just a base type. +type genericTypeDescriptor struct { + baseType BaseType +} + +// Base returns the base type of this type descriptor. +func (td genericTypeDescriptor) Base() BaseType { + return td.baseType +} diff --git a/internal/engine/types/type_function.go b/internal/engine/types/type_function.go new file mode 100644 index 00000000..b960da9b --- /dev/null +++ b/internal/engine/types/type_function.go @@ -0,0 +1,63 @@ +package types + +import "fmt" + +var ( + // Function is the function type. Functions are not comparable. + Function = FunctionTypeDescriptor{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeFunction, + }, + } +) + +type ( + // FunctionTypeDescriptor is the function type of this engine. + FunctionTypeDescriptor struct { + genericTypeDescriptor + } + + functionValue struct { + Name string + value func(...Value) (Value, error) + } +) + +// Compare will always return an error, indicating that functions are not +// comparable. Both arguments have to have a funtion type, otherwise a type +// mismatch error will be returned. +func (FunctionTypeDescriptor) Compare(left, right Value) (int, error) { + if !left.Is(Function) { + return 0, ErrTypeMismatch(Function, left.Type()) + } + if !right.Is(Function) { + return 0, ErrTypeMismatch(Function, right.Type()) + } + return -2, fmt.Errorf("functions are not comparable") +} + +func (FunctionTypeDescriptor) String() string { return "Function" } + +// NewFunctionValue creates a new function value with the given name and +// underlying function. +func NewFunctionValue(name string, fn func(...Value) (Value, error)) Value { + return functionValue{ + Name: name, + value: fn, + } +} + +// CallWithArgs will call the underlying function with the given arguments. +func (f functionValue) CallWithArgs(args ...Value) (Value, error) { + result, err := f.value(args...) + if err != nil { + return nil, fmt.Errorf("call %v: %w", f.Name, err) + } + return result, nil +} + +// Type returns a function type. +func (functionValue) Type() Type { return Function } + +// Is checks if this value is of type function. +func (functionValue) Is(t Type) bool { return t == Function } diff --git a/internal/engine/types/type_numeric.go b/internal/engine/types/type_numeric.go new file mode 100644 index 00000000..30f1d014 --- /dev/null +++ b/internal/engine/types/type_numeric.go @@ -0,0 +1,60 @@ +package types + +import "fmt" + +var ( + // Numeric is the numeric type. Its base type is a numeric type. + Numeric = NumericTypeDescriptor{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeNumeric, + }, + } +) + +var _ Type = (*NumericTypeDescriptor)(nil) +var _ Value = (*NumericValue)(nil) + +type ( + // NumericTypeDescriptor is the type descriptor for parameterized and non-parameterized + // numeric types, such as DECIMAL. + NumericTypeDescriptor struct { + genericTypeDescriptor + } + + // NumericValue is a value of type Numeric. + NumericValue struct { + // Value is the underlying primitive value. + Value float64 + } +) + +// Compare for the Numeric is defined as the lexicographical comparison of the +// two underlying primitive values. +func (NumericTypeDescriptor) Compare(left, right Value) (int, error) { + if !left.Is(Numeric) { + return 0, ErrTypeMismatch(Numeric, left.Type()) + } + if !right.Is(Numeric) { + return 0, ErrTypeMismatch(Numeric, right.Type()) + } + + leftNum := left.(NumericValue).Value + rightNum := right.(NumericValue).Value + switch { + case leftNum < rightNum: + return -1, nil + case leftNum == rightNum: + return 0, nil + case leftNum > rightNum: + return 1, nil + } + return -2, fmt.Errorf("unhandled constellation: %v <-> %v", leftNum, rightNum) +} + +func (NumericTypeDescriptor) String() string { return "Numeric" } + +// Type returns a string type. +func (NumericValue) Type() Type { return Numeric } + +// Is checks if this value is of type Numeric. +func (NumericValue) Is(t Type) bool { return t == Numeric } diff --git a/internal/engine/types/type_string.go b/internal/engine/types/type_string.go new file mode 100644 index 00000000..73eecfe2 --- /dev/null +++ b/internal/engine/types/type_string.go @@ -0,0 +1,52 @@ +package types + +import "strings" + +var ( + // String is the string type. Its base type is a string. + String = StringTypeDescriptor{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeString, + }, + } +) + +var _ Type = (*StringTypeDescriptor)(nil) +var _ Value = (*StringValue)(nil) + +type ( + // StringTypeDescriptor is the type descriptor for parameterized and + // non-parameterized string types, such as VARCHAR or VARCHAR(n). + StringTypeDescriptor struct { + genericTypeDescriptor + } + + // StringValue is a value of type String. + StringValue struct { + // Value is the underlying primitive value. + Value string + } +) + +// Compare for the String is defined as the lexicographical comparison of +// the two underlying primitive values. +func (StringTypeDescriptor) Compare(left, right Value) (int, error) { + if !left.Is(String) { + return 0, ErrTypeMismatch(String, left.Type()) + } + if !right.Is(String) { + return 0, ErrTypeMismatch(String, right.Type()) + } + + leftString := left.(StringValue).Value + rightString := right.(StringValue).Value + return strings.Compare(leftString, rightString), nil +} + +func (StringTypeDescriptor) String() string { return "String" } + +// Type returns a string type. +func (StringValue) Type() Type { return String } + +// Is checks if this value is of type String. +func (StringValue) Is(t Type) bool { return t == String } diff --git a/internal/engine/types/value.go b/internal/engine/types/value.go new file mode 100644 index 00000000..6c28031a --- /dev/null +++ b/internal/engine/types/value.go @@ -0,0 +1,13 @@ +package types + +// Value describes an object that can be used as a value in the engine. A value +// has a type, which can be used to compare this value to another one. +type Value interface { + Type() Type + IsTyper +} + +// IsTyper wraps the basic method Is, which can check the type of a value. +type IsTyper interface { + Is(Type) bool +} From ca0ab8ca41a5503d967b4d32a64835896856d0a7 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Wed, 24 Jun 2020 17:25:33 +0200 Subject: [PATCH 34/77] Add basic profiling options --- internal/engine/profile/profile.go | 2 ++ internal/engine/profile/profiler.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/internal/engine/profile/profile.go b/internal/engine/profile/profile.go index 397b0153..32f156bd 100644 --- a/internal/engine/profile/profile.go +++ b/internal/engine/profile/profile.go @@ -1,5 +1,7 @@ package profile +// Profile is a collection of profiling events that were collected by a +// profiler. type Profile struct { Events []Event } diff --git a/internal/engine/profile/profiler.go b/internal/engine/profile/profiler.go index 9daa0b13..516cb4dd 100644 --- a/internal/engine/profile/profiler.go +++ b/internal/engine/profile/profiler.go @@ -5,18 +5,24 @@ import ( "time" ) +// Clearer wraps a basic clear method, which will clear the components contents. +// What exactly is cleared, must be documented by the component. type Clearer interface { Clear() } +// NewProfiler returns a new, ready to use profiler. func NewProfiler() *Profiler { return &Profiler{} } +// Profiler is a profiler that can collect events. type Profiler struct { events []Event } +// Event is a simple profiling event. It keeps a back reference to the profiler +// it originated from. type Event struct { origin *Profiler Object fmt.Stringer @@ -24,6 +30,9 @@ type Event struct { Duration time.Duration } +// Enter creates a profiling event. Use like this: +// +// defer profiler.Enter(MyEvent).Exit() func (p *Profiler) Enter(object fmt.Stringer) Event { if p == nil { return Event{} @@ -36,6 +45,8 @@ func (p *Profiler) Enter(object fmt.Stringer) Event { } } +// Exit collects the given event. You can call Exit multiple times with the same +// event, it will them appear multiple times in the profiler's profile. func (p *Profiler) Exit(evt Event) { if p == nil { return @@ -44,6 +55,7 @@ func (p *Profiler) Exit(evt Event) { p.events = append(p.events, evt) } +// Clear removes all collected events. func (p *Profiler) Clear() { if p == nil { return @@ -52,6 +64,9 @@ func (p *Profiler) Clear() { p.events = nil } +// Profile returns a profile with all collected events from the profiler. The +// collected events are NOT cleared after this. To clear all events, use +// (*Profiler).Clear(). func (p *Profiler) Profile() Profile { if p == nil { return Profile{} @@ -62,6 +77,8 @@ func (p *Profiler) Profile() Profile { } } +// Exit passes the event back to the origin profiler. When using this, unlike +// using (*Profiler).Exit(Event), an event duration will be set. func (e Event) Exit() { if e.origin == nil { return From 5f9918491b55a39554cf7c4495f7f23a65f9b243 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Wed, 24 Jun 2020 17:25:57 +0200 Subject: [PATCH 35/77] Add builtin functions and use profiling --- internal/engine/basetype_string.go | 27 --------- internal/engine/builtin.go | 89 ++++++++++++++++++++++++++++++ internal/engine/builtin_test.go | 79 ++++++++++++++++++++++++++ internal/engine/compare.go | 16 +++--- internal/engine/compare_test.go | 21 +++---- internal/engine/engine.go | 12 ++++ internal/engine/engine_test.go | 21 ++++--- internal/engine/error.go | 5 +- internal/engine/evaluate.go | 9 +-- internal/engine/expression.go | 5 +- internal/engine/expression_test.go | 7 ++- internal/engine/option.go | 11 ++++ internal/engine/profiling.go | 9 ++- internal/engine/result.go | 15 ++++- internal/engine/type.go | 20 ------- internal/engine/type_blob.go | 39 ------------- internal/engine/type_bool.go | 46 --------------- internal/engine/type_descriptor.go | 39 ------------- internal/engine/type_numeric.go | 47 ---------------- internal/engine/type_string.go | 39 ------------- internal/engine/value.go | 7 --- 21 files changed, 258 insertions(+), 305 deletions(-) delete mode 100644 internal/engine/basetype_string.go create mode 100644 internal/engine/builtin.go create mode 100644 internal/engine/builtin_test.go delete mode 100644 internal/engine/type.go delete mode 100644 internal/engine/type_blob.go delete mode 100644 internal/engine/type_bool.go delete mode 100644 internal/engine/type_descriptor.go delete mode 100644 internal/engine/type_numeric.go delete mode 100644 internal/engine/type_string.go delete mode 100644 internal/engine/value.go diff --git a/internal/engine/basetype_string.go b/internal/engine/basetype_string.go deleted file mode 100644 index c21ab891..00000000 --- a/internal/engine/basetype_string.go +++ /dev/null @@ -1,27 +0,0 @@ -// Code generated by "stringer -type=BaseType"; DO NOT EDIT. - -package engine - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[BaseTypeUnknown-0] - _ = x[BaseTypeBool-1] - _ = x[BaseTypeBinary-2] - _ = x[BaseTypeString-3] - _ = x[BaseTypeNumber-4] -} - -const _BaseType_name = "BaseTypeUnknownBaseTypeBoolBaseTypeBinaryBaseTypeStringBaseTypeNumber" - -var _BaseType_index = [...]uint8{0, 15, 27, 41, 55, 69} - -func (i BaseType) String() string { - if i >= BaseType(len(_BaseType_index)-1) { - return "BaseType(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _BaseType_name[_BaseType_index[i]:_BaseType_index[i+1]] -} diff --git a/internal/engine/builtin.go b/internal/engine/builtin.go new file mode 100644 index 00000000..ca725b41 --- /dev/null +++ b/internal/engine/builtin.go @@ -0,0 +1,89 @@ +package engine + +import ( + "fmt" + + "github.com/tomarrell/lbadd/internal/engine/types" +) + +type builtinFunction = func(...types.Value) (types.Value, error) + +func builtinRandom(args ...types.Value) (types.Value, error) { + return nil, ErrUnimplemented +} + +func builtinCount(args ...types.Value) (types.Value, error) { + return types.NumericValue{Value: float64(len(args))}, nil +} + +func builtinUCase(args ...types.Value) (types.Value, error) { + return nil, ErrUnimplemented +} + +func builtinLCase(args ...types.Value) (types.Value, error) { + return nil, ErrUnimplemented +} + +func builtinNow(args ...types.Value) (types.Value, error) { + return nil, ErrUnimplemented +} + +func builtinMax(args ...types.Value) (types.Value, error) { + if len(args) == 0 { + return nil, nil + } + + if err := ensureSameType(args...); err != nil { + return nil, err + } + + largest := args[0] + t := largest.Type() + for i := 1; i < len(args); i++ { + res, err := t.Compare(largest, args[i]) + if err != nil { + return nil, fmt.Errorf("compare: %w", err) + } + if res < 0 { + largest = args[i] + } + } + return largest, nil +} + +func builtinMin(args ...types.Value) (types.Value, error) { + if len(args) == 0 { + return nil, nil + } + + if err := ensureSameType(args...); err != nil { + return nil, err + } + + smallest := args[0] + t := smallest.Type() + for i := 1; i < len(args); i++ { + res, err := t.Compare(smallest, args[i]) + if err != nil { + return nil, fmt.Errorf("compare: %w", err) + } + if res > 0 { + smallest = args[i] + } + } + return smallest, nil +} + +func ensureSameType(args ...types.Value) error { + if len(args) == 0 { + return nil + } + + base := args[0] + for i := 1; i < len(args); i++ { + if !base.Is(args[i].Type()) { // Is is transitive + return types.ErrTypeMismatch(base.Type(), args[i].Type()) + } + } + return nil +} diff --git a/internal/engine/builtin_test.go b/internal/engine/builtin_test.go new file mode 100644 index 00000000..52fda364 --- /dev/null +++ b/internal/engine/builtin_test.go @@ -0,0 +1,79 @@ +package engine + +import ( + "reflect" + "testing" + + "github.com/tomarrell/lbadd/internal/engine/types" +) + +func Test_builtinMax(t *testing.T) { + type args struct { + args []types.Value + } + tests := []struct { + name string + args args + want types.Value + wantErr bool + }{ + { + "empty", + args{ + []types.Value{}, + }, + nil, + false, + }, + { + "bools", + args{ + []types.Value{ + types.BoolValue{Value: true}, + types.BoolValue{Value: false}, + types.BoolValue{Value: false}, + types.BoolValue{Value: true}, + types.BoolValue{Value: false}, + types.BoolValue{Value: true}, + types.BoolValue{Value: false}, + types.BoolValue{Value: true}, + types.BoolValue{Value: true}, + }, + }, + types.BoolValue{Value: true}, + false, + }, + { + "numbers", + args{ + []types.Value{ + types.NumericValue{Value: 32698.7236}, + types.NumericValue{Value: 33020.4705}, + types.NumericValue{Value: 28550.057}, + types.NumericValue{Value: 17980.2620}, + types.NumericValue{Value: 37105.784}, + types.NumericValue{Value: 623164325426457348.4231}, + types.NumericValue{Value: 53226.854}, + types.NumericValue{Value: 49344.266}, + types.NumericValue{Value: 10634.3037}, + types.NumericValue{Value: 36735.083}, + types.NumericValue{Value: 14828.1674}, + }, + }, + types.NumericValue{Value: 623164325426457348.4231}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := builtinMax(tt.args.args...) + if (err != nil) != tt.wantErr { + t.Errorf("builtinMax() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("builtinMax() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/engine/compare.go b/internal/engine/compare.go index 288e0386..55ba9168 100644 --- a/internal/engine/compare.go +++ b/internal/engine/compare.go @@ -1,5 +1,7 @@ package engine +import "github.com/tomarrell/lbadd/internal/engine/types" + //go:generate stringer -type=cmpResult type cmpResult uint8 @@ -15,9 +17,9 @@ const ( // left~right, meaning if e.g. cmpLessThan is returned, it is to be understood // as left= (greater // than or equal) relation, see (Engine).gteq. -func (e Engine) gt(left, right Value) bool { +func (e Engine) gt(left, right types.Value) bool { return e.lt(right, left) } // lteq checks if the left value is smaller than or equal to the right value. -func (e Engine) lteq(left, right Value) bool { +func (e Engine) lteq(left, right types.Value) bool { return e.eq(left, right) || e.lt(left, right) } // gteq checks if the right value is smaller than or equal to the left value. -func (e Engine) gteq(left, right Value) bool { +func (e Engine) gteq(left, right types.Value) bool { return e.eq(left, right) || e.gt(left, right) } diff --git a/internal/engine/compare_test.go b/internal/engine/compare_test.go index 1300d1f4..c8518109 100644 --- a/internal/engine/compare_test.go +++ b/internal/engine/compare_test.go @@ -4,37 +4,38 @@ import ( "testing" "github.com/rs/zerolog" + "github.com/tomarrell/lbadd/internal/engine/types" ) func TestEngine_cmp(t *testing.T) { tests := []struct { name string - left Value - right Value + left types.Value + right types.Value want cmpResult }{ { "true <-> true", - BoolValue{Value: true}, - BoolValue{Value: true}, + types.BoolValue{Value: true}, + types.BoolValue{Value: true}, cmpEqual, }, { "true <-> false", - BoolValue{Value: true}, - BoolValue{Value: false}, + types.BoolValue{Value: true}, + types.BoolValue{Value: false}, cmpGreaterThan, }, { "false <-> true", - BoolValue{Value: false}, - BoolValue{Value: true}, + types.BoolValue{Value: false}, + types.BoolValue{Value: true}, cmpLessThan, }, { "false <-> false", - BoolValue{Value: false}, - BoolValue{Value: false}, + types.BoolValue{Value: false}, + types.BoolValue{Value: false}, cmpEqual, }, } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 8db77e31..32bf3d41 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -14,6 +14,8 @@ type Engine struct { dbFile *storage.DBFile pageCache cache.Cache profiler *profile.Profiler + + builtinFunctions map[string]builtinFunction } // New creates a new engine object and applies the given options to it. @@ -22,6 +24,16 @@ func New(dbFile *storage.DBFile, opts ...Option) (Engine, error) { log: zerolog.Nop(), dbFile: dbFile, pageCache: dbFile.Cache(), + + builtinFunctions: map[string]builtinFunction{ + "RAND": builtinRandom, + "COUNT": builtinCount, + "UCASE": builtinUCase, + "LCASE": builtinLCase, + "NOW": builtinNow, + "MAX": builtinMax, + "MIN": builtinMin, + }, } for _, opt := range opts { opt(&e) diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 6ec61309..15d8b098 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -7,9 +7,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/tomarrell/lbadd/internal/compiler/command" "github.com/tomarrell/lbadd/internal/engine/storage" + "github.com/tomarrell/lbadd/internal/engine/types" ) func TestEngine(t *testing.T) { + t.Skip("not working yet") + assert := assert.New(t) fs := afero.NewMemMapFs() @@ -34,13 +37,13 @@ func TestEngine(t *testing.T) { assert.Len(cols, 3) // col[0] assert.Equal(2, cols[0].Size()) - assert.Equal(stringType, cols[0].Type()) + assert.Equal(types.String, cols[0].Type()) // col[1] assert.Equal(2, cols[1].Size()) - assert.Equal(stringType, cols[1].Type()) + assert.Equal(types.String, cols[1].Type()) // col[2] assert.Equal(2, cols[2].Size()) - assert.Equal(numericType, cols[2].Type()) + assert.Equal(types.Numeric, cols[2].Type()) // col value types assert.Equal(cols[0].Type(), cols[0].Get(0).Type()) assert.Equal(cols[0].Type(), cols[0].Get(1).Type()) @@ -54,12 +57,12 @@ func TestEngine(t *testing.T) { assert.Len(rows, 2) // row[0] assert.Equal(3, rows[0].Size()) - assert.Equal("hello", rows[0].Get(0).(StringValue).Value) - assert.Equal("world", rows[0].Get(1).(StringValue).Value) - assert.Equal(true, rows[0].Get(2).(BoolValue).Value) + assert.Equal("hello", rows[0].Get(0).(types.StringValue).Value) + assert.Equal("world", rows[0].Get(1).(types.StringValue).Value) + assert.Equal(true, rows[0].Get(2).(types.BoolValue).Value) // row[0] assert.Equal(3, rows[1].Size()) - assert.Equal("foo", rows[1].Get(0).(StringValue).Value) - assert.Equal("bar", rows[1].Get(1).(StringValue).Value) - assert.Equal(false, rows[1].Get(2).(BoolValue).Value) + assert.Equal("foo", rows[1].Get(0).(types.StringValue).Value) + assert.Equal("bar", rows[1].Get(1).(types.StringValue).Value) + assert.Equal(false, rows[1].Get(2).(types.BoolValue).Value) } diff --git a/internal/engine/error.go b/internal/engine/error.go index 7bf7bca1..c43c9c54 100644 --- a/internal/engine/error.go +++ b/internal/engine/error.go @@ -7,6 +7,7 @@ func (e Error) Error() string { return string(e) } // Sentinel errors. const ( - ErrClosed Error = "already closed" - ErrUnsupported Error = "unsupported" + ErrClosed Error = "already closed" + ErrUnsupported Error = "unsupported" + ErrUnimplemented Error = "unimplemented" ) diff --git a/internal/engine/evaluate.go b/internal/engine/evaluate.go index d4ae5faa..ed5764b7 100644 --- a/internal/engine/evaluate.go +++ b/internal/engine/evaluate.go @@ -4,20 +4,21 @@ import ( "fmt" "github.com/tomarrell/lbadd/internal/compiler/command" + "github.com/tomarrell/lbadd/internal/engine/types" ) func (e Engine) evaluate(c command.Command) (Result, error) { switch cmd := c.(type) { case command.Values: - _ = cmd + _, _ = e.evaluateValues(cmd) } return nil, nil } -func (e Engine) evaluateValues(v command.Values) ([][]Value, error) { - result := make([][]Value, len(v.Values)) +func (e Engine) evaluateValues(v command.Values) ([][]types.Value, error) { + result := make([][]types.Value, len(v.Values)) for y, values := range v.Values { - rowValues := make([]Value, len(values)) + rowValues := make([]types.Value, len(values)) for x, value := range values { internalValue, err := e.evaluateExpression(value) if err != nil { diff --git a/internal/engine/expression.go b/internal/engine/expression.go index d43022f6..becd48c7 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -4,16 +4,17 @@ import ( "fmt" "github.com/tomarrell/lbadd/internal/compiler/command" + "github.com/tomarrell/lbadd/internal/engine/types" ) // evaluateExpression evaluates the given expression to a runtime-constant // value, meaning that it can only be evaluated to a constant value with a given // execution context. This execution context must be inferred from the engine // receiver. -func (e Engine) evaluateExpression(expr command.Expr) (Value, error) { +func (e Engine) evaluateExpression(expr command.Expr) (types.Value, error) { switch ex := expr.(type) { case command.ConstantBooleanExpr: - return BoolValue{Value: ex.Value}, nil + return types.BoolValue{Value: ex.Value}, nil } return nil, fmt.Errorf("cannot evaluate expression of type %T", expr) } diff --git a/internal/engine/expression_test.go b/internal/engine/expression_test.go index 434bed54..8ebab307 100644 --- a/internal/engine/expression_test.go +++ b/internal/engine/expression_test.go @@ -6,13 +6,14 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/tomarrell/lbadd/internal/compiler/command" + "github.com/tomarrell/lbadd/internal/engine/types" ) func TestEngine_evaluateExpression(t *testing.T) { tests := []struct { name string expr command.Expr - want Value + want types.Value wantErr bool }{ { @@ -24,13 +25,13 @@ func TestEngine_evaluateExpression(t *testing.T) { { "true", command.ConstantBooleanExpr{Value: true}, - BoolValue{Value: true}, + types.BoolValue{Value: true}, false, }, { "false", command.ConstantBooleanExpr{Value: false}, - BoolValue{Value: false}, + types.BoolValue{Value: false}, false, }, } diff --git a/internal/engine/option.go b/internal/engine/option.go index a4b97db7..2a2d9093 100644 --- a/internal/engine/option.go +++ b/internal/engine/option.go @@ -15,8 +15,19 @@ func WithLogger(log zerolog.Logger) Option { } } +// WithProfiler passes a profiler into the engine. The default for the engine is +// not using a profiler at all. func WithProfiler(profiler *profile.Profiler) Option { return func(e *Engine) { e.profiler = profiler } } + +// WithBuiltinFunction registeres or overwrites an existing builtin function. +// Use this to (e.g.) overwrite the SQL function NOW() to get constant +// timestamps. +func WithBuiltinFunction(name string, fn builtinFunction) Option { + return func(e *Engine) { + e.builtinFunctions[name] = fn + } +} diff --git a/internal/engine/profiling.go b/internal/engine/profiling.go index 01640845..169dd5f7 100644 --- a/internal/engine/profiling.go +++ b/internal/engine/profiling.go @@ -1,11 +1,16 @@ package engine +// Evt is an event this engine uses. type Evt string func (e Evt) String() string { return string(e) } +// ParameterizedEvt is a parameterized event, which will be printed in the form +// Name[Param]. type ParameterizedEvt struct { - Name string + // Name is the name of the event. + Name string + // Param is the parameterization of the event. Param string } @@ -14,9 +19,11 @@ func (e ParameterizedEvt) String() string { } const ( + // EvtEvaluate is the event 'evaluate'. EvtEvaluate Evt = "evaluate" ) +// EvtFullTableScan creates an event 'full table scan[table=]'. func EvtFullTableScan(tableName string) ParameterizedEvt { return ParameterizedEvt{ Name: "full table scan", diff --git a/internal/engine/result.go b/internal/engine/result.go index 88f5379a..9eba3836 100644 --- a/internal/engine/result.go +++ b/internal/engine/result.go @@ -1,6 +1,10 @@ package engine -import "fmt" +import ( + "fmt" + + "github.com/tomarrell/lbadd/internal/engine/types" +) // Result represents an evaluation result of the engine. It is a mxn-matrix, // where m and n is variable and depends on the passed in command. @@ -10,20 +14,25 @@ type Result interface { fmt.Stringer } +// IndexedGetter wraps a Get(index) method. type IndexedGetter interface { - Get(int) Value + Get(int) types.Value } +// Sizer wraps the basic Size() method. type Sizer interface { Size() int } +// Column is an iterator over cells in a column. All of the cells must have the +// same type. type Column interface { - Type() Type + Type() types.Type IndexedGetter Sizer } +// Row is an iterator over cells in a row. The cells may have different types. type Row interface { IndexedGetter Sizer diff --git a/internal/engine/type.go b/internal/engine/type.go deleted file mode 100644 index eabfef73..00000000 --- a/internal/engine/type.go +++ /dev/null @@ -1,20 +0,0 @@ -package engine - -type ( - // Comparator is the interface that wraps the basic compare method. The - // compare method compares the left and right value as follows. -1 if - // leftright. What exectly is considered - // to be <, ==, > is up to the implementation. - Comparator interface { - Compare(Value, Value) (int, error) - } - - // Type is a data type that consists of a type descriptor and a comparator. - // The comparator forces types to define relations between two values of - // this type. A type is only expected to be able to handle values of its own - // type. - Type interface { - TypeDescriptor - Comparator - } -) diff --git a/internal/engine/type_blob.go b/internal/engine/type_blob.go deleted file mode 100644 index 2c9e9419..00000000 --- a/internal/engine/type_blob.go +++ /dev/null @@ -1,39 +0,0 @@ -package engine - -import "bytes" - -var ( - blobType = BlobType{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeBinary, - }, - } -) - -var _ Type = (*BlobType)(nil) -var _ Value = (*BlobValue)(nil) - -type ( - // BlobType is the type for Binary Large OBjects. The value is basically a - // byte slice. - BlobType struct { - genericTypeDescriptor - } - - // BlobValue is a value of type BlobType. - BlobValue struct { - // Value is the primitive value of this value object. - Value []byte - } -) - -// Compare for the BlobType is defined the lexicographical comparison between -// the primitive underlying values. -func (BlobType) Compare(left, right Value) (int, error) { - leftBlob := left.(BlobValue).Value - rightBlob := right.(BlobValue).Value - return bytes.Compare(leftBlob, rightBlob), nil -} - -// Type returns a blob type. -func (BlobValue) Type() Type { return blobType } diff --git a/internal/engine/type_bool.go b/internal/engine/type_bool.go deleted file mode 100644 index be82b07f..00000000 --- a/internal/engine/type_bool.go +++ /dev/null @@ -1,46 +0,0 @@ -package engine - -import "fmt" - -var ( - boolType = BoolType{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeBool, - }, - } -) - -var _ Type = (*BoolType)(nil) -var _ Value = (*BoolValue)(nil) - -type ( - // BoolType is the boolean type of this engine. - BoolType struct { - genericTypeDescriptor - } - - // BoolValue is a value of type BoolType. - BoolValue struct { - // Value is the underlying primitive value. - Value bool - } -) - -// Compare for the type BoolType is defined as false %v", leftBool, rightBool) -} - -// Type returns a bool type. -func (BoolValue) Type() Type { return boolType } diff --git a/internal/engine/type_descriptor.go b/internal/engine/type_descriptor.go deleted file mode 100644 index 1012300c..00000000 --- a/internal/engine/type_descriptor.go +++ /dev/null @@ -1,39 +0,0 @@ -package engine - -//go:generate stringer -type=BaseType - -// BaseType is an underlying type for parameterized types. -type BaseType uint8 - -// Known base types. -const ( - BaseTypeUnknown BaseType = iota - BaseTypeBool - BaseTypeBinary - BaseTypeString - BaseTypeNumber -) - -// TypeDescriptor describes a type in more detail than just the type. Every type -// has a type descriptor, which holds the base type and parameterization. Based -// on the parameterization, the type may be interpreted differently. -// -// Example: The simple type INTEGER would have a type descriptor that describes -// a baseType=number with no further parameterization, whereas the more complex -// type VARCHAR(50) would have a type descriptor, that describes a -// baseType=string and a max length of 50. -type TypeDescriptor interface { - Base() BaseType - // TODO: parameters to be done -} - -// genericTypeDescriptor is a type descriptor that has no parameterization and -// just a base type. -type genericTypeDescriptor struct { - baseType BaseType -} - -// Base returns the base type of this type descriptor. -func (td genericTypeDescriptor) Base() BaseType { - return td.baseType -} diff --git a/internal/engine/type_numeric.go b/internal/engine/type_numeric.go deleted file mode 100644 index 24011924..00000000 --- a/internal/engine/type_numeric.go +++ /dev/null @@ -1,47 +0,0 @@ -package engine - -import "fmt" - -var ( - numericType = NumericType{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeString, - }, - } -) - -var _ Type = (*NumericType)(nil) -var _ Value = (*NumericValue)(nil) - -type ( - // NumericType is the type for parameterized and non-parameterized numeric - // types, such as DECIMAL. - NumericType struct { - genericTypeDescriptor - } - - // NumericValue is a value of type NumericType. - NumericValue struct { - // Value is the underlying primitive value. - Value float64 - } -) - -// Compare for the NumericType is defined as the lexicographical comparison of -// the two underlying primitive values. -func (NumericType) Compare(left, right Value) (int, error) { - leftNum := left.(NumericValue).Value - rightNum := right.(NumericValue).Value - switch { - case leftNum < rightNum: - return -1, nil - case leftNum == rightNum: - return 0, nil - case leftNum > rightNum: - return 1, nil - } - return -2, fmt.Errorf("unhandled constellation: %v <-> %v", leftNum, rightNum) -} - -// Type returns a string type. -func (NumericValue) Type() Type { return numericType } diff --git a/internal/engine/type_string.go b/internal/engine/type_string.go deleted file mode 100644 index efff7093..00000000 --- a/internal/engine/type_string.go +++ /dev/null @@ -1,39 +0,0 @@ -package engine - -import "strings" - -var ( - stringType = StringType{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeString, - }, - } -) - -var _ Type = (*StringType)(nil) -var _ Value = (*StringValue)(nil) - -type ( - // StringType is the type for parameterized and non-parameterized string - // types, such as VARCHAR or VARCHAR(n). - StringType struct { - genericTypeDescriptor - } - - // StringValue is a value of type StringType. - StringValue struct { - // Value is the underlying primitive value. - Value string - } -) - -// Compare for the StringType is defined as the lexicographical comparison of -// the two underlying primitive values. -func (StringType) Compare(left, right Value) (int, error) { - leftString := left.(StringValue).Value - rightString := right.(StringValue).Value - return strings.Compare(leftString, rightString), nil -} - -// Type returns a string type. -func (StringValue) Type() Type { return numericType } diff --git a/internal/engine/value.go b/internal/engine/value.go deleted file mode 100644 index 575c3260..00000000 --- a/internal/engine/value.go +++ /dev/null @@ -1,7 +0,0 @@ -package engine - -// Value describes an object that can be used as a value in the engine. A value -// has a type, which can be used to compare this value to another one. -type Value interface { - Type() Type -} From 09f8a9e9674a6877f3cea0d085c70b6c146ee9c9 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Wed, 24 Jun 2020 19:29:34 +0200 Subject: [PATCH 36/77] Add testing framework for parser->engine e2e tests --- internal/engine/engine.go | 3 +- internal/engine/engine_test.go | 4 +- internal/engine/evaluate.go | 13 ++- internal/engine/expression.go | 2 + internal/engine/{ => result}/result.go | 2 +- internal/engine/result/value.go | 104 +++++++++++++++++ internal/engine/types/type_blob.go | 6 +- internal/engine/types/type_bool.go | 7 +- internal/engine/types/type_function.go | 2 + internal/engine/types/type_numeric.go | 7 +- internal/engine/types/type_string.go | 2 + internal/engine/types/value.go | 3 + internal/test/base_test.go | 154 +++++++++++++++++++++++++ internal/test/issue187_test.go | 10 ++ internal/test/testdata/issue187/output | 3 + 15 files changed, 312 insertions(+), 10 deletions(-) rename internal/engine/{ => result}/result.go (98%) create mode 100644 internal/engine/result/value.go create mode 100644 internal/test/base_test.go create mode 100644 internal/test/issue187_test.go create mode 100644 internal/test/testdata/issue187/output diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 32bf3d41..cc391c31 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -4,6 +4,7 @@ import ( "github.com/rs/zerolog" "github.com/tomarrell/lbadd/internal/compiler/command" "github.com/tomarrell/lbadd/internal/engine/profile" + "github.com/tomarrell/lbadd/internal/engine/result" "github.com/tomarrell/lbadd/internal/engine/storage" "github.com/tomarrell/lbadd/internal/engine/storage/cache" ) @@ -43,7 +44,7 @@ func New(dbFile *storage.DBFile, opts ...Option) (Engine, error) { // Evaluate evaluates the given command. This may mutate the state of the // database, and changes may occur to the database file. -func (e Engine) Evaluate(cmd command.Command) (Result, error) { +func (e Engine) Evaluate(cmd command.Command) (result.Result, error) { defer e.profiler.Enter(EvtEvaluate).Exit() _ = e.eq diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 15d8b098..e45823c2 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -11,8 +11,6 @@ import ( ) func TestEngine(t *testing.T) { - t.Skip("not working yet") - assert := assert.New(t) fs := afero.NewMemMapFs() @@ -43,7 +41,7 @@ func TestEngine(t *testing.T) { assert.Equal(types.String, cols[1].Type()) // col[2] assert.Equal(2, cols[2].Size()) - assert.Equal(types.Numeric, cols[2].Type()) + assert.Equal(types.Bool, cols[2].Type()) // col value types assert.Equal(cols[0].Type(), cols[0].Get(0).Type()) assert.Equal(cols[0].Type(), cols[0].Get(1).Type()) diff --git a/internal/engine/evaluate.go b/internal/engine/evaluate.go index ed5764b7..2413dbe7 100644 --- a/internal/engine/evaluate.go +++ b/internal/engine/evaluate.go @@ -4,13 +4,22 @@ import ( "fmt" "github.com/tomarrell/lbadd/internal/compiler/command" + "github.com/tomarrell/lbadd/internal/engine/result" "github.com/tomarrell/lbadd/internal/engine/types" ) -func (e Engine) evaluate(c command.Command) (Result, error) { +func (e Engine) evaluate(c command.Command) (result.Result, error) { switch cmd := c.(type) { case command.Values: - _, _ = e.evaluateValues(cmd) + values, err := e.evaluateValues(cmd) + if err != nil { + return nil, fmt.Errorf("values: %w", err) + } + res, err := result.FromValues(values) + if err != nil { + return nil, fmt.Errorf("from values: %w", err) + } + return res, nil } return nil, nil } diff --git a/internal/engine/expression.go b/internal/engine/expression.go index becd48c7..f3dec1e1 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -15,6 +15,8 @@ func (e Engine) evaluateExpression(expr command.Expr) (types.Value, error) { switch ex := expr.(type) { case command.ConstantBooleanExpr: return types.BoolValue{Value: ex.Value}, nil + case command.LiteralExpr: + return types.StringValue{Value: ex.Value}, nil } return nil, fmt.Errorf("cannot evaluate expression of type %T", expr) } diff --git a/internal/engine/result.go b/internal/engine/result/result.go similarity index 98% rename from internal/engine/result.go rename to internal/engine/result/result.go index 9eba3836..64267f9a 100644 --- a/internal/engine/result.go +++ b/internal/engine/result/result.go @@ -1,4 +1,4 @@ -package engine +package result import ( "fmt" diff --git a/internal/engine/result/value.go b/internal/engine/result/value.go new file mode 100644 index 00000000..34a82bdd --- /dev/null +++ b/internal/engine/result/value.go @@ -0,0 +1,104 @@ +package result + +import ( + "bytes" + "fmt" + "strings" + "text/tabwriter" + + "github.com/tomarrell/lbadd/internal/engine/types" +) + +type valueResult struct { + vals [][]types.Value +} + +type valueColumn struct { + valueResult + colIndex int +} + +type valueRow struct { + valueResult + rowIndex int +} + +// FromValues wraps the given values in a result, performing a type check on all +// columns. If any column does not have a consistent type, an error will be +// returned, together with NO result. +func FromValues(vals [][]types.Value) (Result, error) { + for x := 0; x < len(vals[0]); x++ { // cols + t := vals[0][x].Type() + for y := 0; y < len(vals); y++ { // rows + if !vals[y][x].Is(t) { + return nil, types.ErrTypeMismatch(t, vals[y][x].Type()) + } + } + } + return valueResult{ + vals: vals, + }, nil +} + +func (r valueResult) Cols() []Column { + result := make([]Column, 0) + for i := 0; i < len(r.vals[0]); i++ { + result = append(result, valueColumn{ + valueResult: r, + colIndex: i, + }) + } + return result +} + +func (r valueResult) Rows() []Row { + result := make([]Row, 0) + for i := 0; i < len(r.vals); i++ { + result = append(result, valueRow{ + valueResult: r, + rowIndex: i, + }) + } + return result +} + +func (r valueResult) String() string { + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 1, 3, ' ', 0) + + var types []string + for _, col := range r.Cols() { + types = append(types, col.Type().String()) + } + _, _ = fmt.Fprintln(w, strings.Join(types, "\t")) + + for _, row := range r.Rows() { + var strVals []string + for i := 0; i < row.Size(); i++ { + strVals = append(strVals, row.Get(i).String()) + } + _, _ = fmt.Fprintln(w, strings.Join(strVals, "\t")) + } + _ = w.Flush() + return buf.String() +} + +func (c valueColumn) Type() types.Type { + return c.Get(0).Type() +} + +func (c valueColumn) Get(index int) types.Value { + return c.valueResult.vals[index][c.colIndex] +} + +func (c valueColumn) Size() int { + return len(c.valueResult.vals) +} + +func (r valueRow) Get(index int) types.Value { + return r.valueResult.vals[r.rowIndex][index] +} + +func (r valueRow) Size() int { + return len(r.valueResult.vals[0]) +} diff --git a/internal/engine/types/type_blob.go b/internal/engine/types/type_blob.go index 003739f2..36341ca6 100644 --- a/internal/engine/types/type_blob.go +++ b/internal/engine/types/type_blob.go @@ -1,6 +1,8 @@ package types -import "bytes" +import ( + "bytes" +) var ( // Blob is the blob type. A Blob is a Binary Large OBject, and its base type @@ -51,3 +53,5 @@ func (BlobValue) Type() Type { return Blob } // Is checks if this value is of type Blob. func (BlobValue) Is(t Type) bool { return t == Blob } + +func (v BlobValue) String() string { return string(v.Value) } diff --git a/internal/engine/types/type_bool.go b/internal/engine/types/type_bool.go index 86d151bd..19e1afff 100644 --- a/internal/engine/types/type_bool.go +++ b/internal/engine/types/type_bool.go @@ -1,6 +1,9 @@ package types -import "fmt" +import ( + "fmt" + "strconv" +) var ( // Bool is the boolean type. Its base type is a bool. @@ -57,3 +60,5 @@ func (BoolValue) Type() Type { return Bool } // Is checks if this value is of type Bool. func (BoolValue) Is(t Type) bool { return t == Bool } + +func (v BoolValue) String() string { return strconv.FormatBool(v.Value) } diff --git a/internal/engine/types/type_function.go b/internal/engine/types/type_function.go index b960da9b..be227305 100644 --- a/internal/engine/types/type_function.go +++ b/internal/engine/types/type_function.go @@ -61,3 +61,5 @@ func (functionValue) Type() Type { return Function } // Is checks if this value is of type function. func (functionValue) Is(t Type) bool { return t == Function } + +func (f functionValue) String() string { return f.Name + "()" } diff --git a/internal/engine/types/type_numeric.go b/internal/engine/types/type_numeric.go index 30f1d014..bd1650d0 100644 --- a/internal/engine/types/type_numeric.go +++ b/internal/engine/types/type_numeric.go @@ -1,6 +1,9 @@ package types -import "fmt" +import ( + "fmt" + "strconv" +) var ( // Numeric is the numeric type. Its base type is a numeric type. @@ -58,3 +61,5 @@ func (NumericValue) Type() Type { return Numeric } // Is checks if this value is of type Numeric. func (NumericValue) Is(t Type) bool { return t == Numeric } + +func (v NumericValue) String() string { return strconv.FormatFloat(v.Value, 'f', -1, 64) } diff --git a/internal/engine/types/type_string.go b/internal/engine/types/type_string.go index 73eecfe2..514fc73b 100644 --- a/internal/engine/types/type_string.go +++ b/internal/engine/types/type_string.go @@ -50,3 +50,5 @@ func (StringValue) Type() Type { return String } // Is checks if this value is of type String. func (StringValue) Is(t Type) bool { return t == String } + +func (v StringValue) String() string { return v.Value } diff --git a/internal/engine/types/value.go b/internal/engine/types/value.go index 6c28031a..d13634e0 100644 --- a/internal/engine/types/value.go +++ b/internal/engine/types/value.go @@ -1,10 +1,13 @@ package types +import "fmt" + // Value describes an object that can be used as a value in the engine. A value // has a type, which can be used to compare this value to another one. type Value interface { Type() Type IsTyper + fmt.Stringer } // IsTyper wraps the basic method Is, which can check the type of a value. diff --git a/internal/test/base_test.go b/internal/test/base_test.go new file mode 100644 index 00000000..7278d2cd --- /dev/null +++ b/internal/test/base_test.go @@ -0,0 +1,154 @@ +package test + +import ( + "flag" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/compiler" + "github.com/tomarrell/lbadd/internal/engine" + "github.com/tomarrell/lbadd/internal/engine/storage" + "github.com/tomarrell/lbadd/internal/parser" +) + +var ( + overwriteExpected bool +) + +type Test struct { + Name string + + CompileOptions []compiler.Option + EngineOptions []engine.Option + DBFileName string + + Statement string +} + +func TestMain(m *testing.M) { + flag.BoolVar(&overwriteExpected, "update", false, "overwrite / update expected output files") + flag.Parse() + + os.Exit(m.Run()) +} + +func RunAndCompare(t *testing.T, tt Test) { + t.Run(tt.Name, func(t *testing.T) { + runAndCompare(t, tt) + }) +} + +func runAndCompare(t *testing.T, tt Test) { + t.Helper() + + assert := assert.New(t) + + if overwriteExpected { + t.Log("OVERWRITING EXPECTED FILE") + t.Fail() + } + + var dbFile *storage.DBFile + if tt.DBFileName == "" { + // create a new im-memory db file if none is set + fs := afero.NewMemMapFs() + f, err := fs.Create("mydbfile") + assert.NoError(err) + + dbFile, err = storage.Create(f) + assert.NoError(err) + } else { + dbFile = loadDBFile(t, tt.Name, tt.DBFileName) + } + + totalStart := time.Now() + parseStart := time.Now() + + p := parser.New(tt.Statement) + stmt, errs, ok := p.Next() + assert.True(ok) + assert.Len(errs, 0) + for _, err := range errs { + assert.NoError(err) + } + + t.Logf("parse: %v", time.Since(parseStart)) + + compileStart := time.Now() + + c := compiler.New(tt.CompileOptions...) + cmd, err := c.Compile(stmt) + assert.NoError(err) + + t.Logf("compile: %v", time.Since(compileStart)) + + engineStart := time.Now() + + e, err := engine.New(dbFile, tt.EngineOptions...) + assert.NoError(err) + + t.Logf("start engine: %v", time.Since(engineStart)) + + evalStart := time.Now() + + result, err := e.Evaluate(cmd) + assert.NoError(err) + + t.Logf("evaluate: %v", time.Since(evalStart)) + t.Logf("TOTAL: %v", time.Since(totalStart)) + + if overwriteExpected { + writeExpectedFile(t, tt.Name, result.String()) + } else { + expectedData := loadExpectedFile(t, tt.Name) + assert.Equal(string(expectedData), result.String()) + } +} + +func loadDBFile(t *testing.T, testName, fileName string) *storage.DBFile { + assert := assert.New(t) + + fs := afero.NewBasePathFs(afero.NewOsFs(), "testdata") + f, err := fs.Open(filepath.Join(testName, fileName)) + assert.NoError(err) + + dbFile, err := storage.Open(f) + assert.NoError(err) + + return dbFile +} + +func writeExpectedFile(t *testing.T, testName string, output string) { + assert := assert.New(t) + + fs := afero.NewBasePathFs(afero.NewOsFs(), "testdata") + + err := fs.MkdirAll(testName, 0700) + assert.NoError(err) + + f, err := fs.Create(filepath.Join(testName, "output")) + assert.NoError(err) + defer func() { _ = f.Close() }() + + _, err = f.WriteString(output) + assert.NoError(err) +} + +func loadExpectedFile(t *testing.T, testName string) []byte { + assert := assert.New(t) + + fs := afero.NewBasePathFs(afero.NewOsFs(), "testdata") + f, err := fs.Open(filepath.Join(testName, "output")) + assert.NoError(err) + defer func() { _ = f.Close() }() + + data, err := ioutil.ReadAll(f) + assert.NoError(err) + + return data +} diff --git a/internal/test/issue187_test.go b/internal/test/issue187_test.go new file mode 100644 index 00000000..41bb5cf3 --- /dev/null +++ b/internal/test/issue187_test.go @@ -0,0 +1,10 @@ +package test + +import "testing" + +func TestIssue187(t *testing.T) { + RunAndCompare(t, Test{ + Name: "issue187", + Statement: "VALUES (1,2,3), (4,5,6)", + }) +} diff --git a/internal/test/testdata/issue187/output b/internal/test/testdata/issue187/output new file mode 100644 index 00000000..f9037d35 --- /dev/null +++ b/internal/test/testdata/issue187/output @@ -0,0 +1,3 @@ +String String String +1 2 3 +4 5 6 From 350d323de752bae0b2851adb896ce261f96b5014 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 25 Jun 2020 10:21:14 +0200 Subject: [PATCH 37/77] Add a date type --- internal/engine/types/type.go | 8 ++++ internal/engine/types/type_date.go | 62 ++++++++++++++++++++++++++ internal/engine/types/type_function.go | 32 ++++++------- 3 files changed, 83 insertions(+), 19 deletions(-) create mode 100644 internal/engine/types/type_date.go diff --git a/internal/engine/types/type.go b/internal/engine/types/type.go index c2f778d5..8bfa766d 100644 --- a/internal/engine/types/type.go +++ b/internal/engine/types/type.go @@ -11,6 +11,14 @@ type ( Compare(Value, Value) (int, error) } + // Caster wraps the Cast method, which is used to transform the input value + // into an output value. Types can implement this interface. E.g. if the + // type String implements Caster, any value passed into the Cast method + // should be attempted to be cast to String, or an error should be returned. + Caster interface { + Cast(Value) (Value, error) + } + // Type is a data type that consists of a type descriptor and a comparator. // The comparator forces types to define relations between two values of // this type. A type is only expected to be able to handle values of its own diff --git a/internal/engine/types/type_date.go b/internal/engine/types/type_date.go new file mode 100644 index 00000000..dd703782 --- /dev/null +++ b/internal/engine/types/type_date.go @@ -0,0 +1,62 @@ +package types + +import ( + "time" +) + +var ( + // Date is the date type. A date is a timestamp, and its base type is a byte + // slice. Internally, the timestamp is represented by time.Time. + Date = DateTypeDescriptor{ + genericTypeDescriptor: genericTypeDescriptor{ + baseType: BaseTypeBinary, + }, + } +) + +var _ Type = (*DateTypeDescriptor)(nil) +var _ Value = (*DateValue)(nil) + +type ( + // DateTypeDescriptor is the type descriptor for date objects. The value is + // a time.Time. + DateTypeDescriptor struct { + genericTypeDescriptor + } + + // DateValue is a value of type Date. + DateValue struct { + // Value is the primitive value of this value object. + Value time.Time + } +) + +// Compare for the Date is defined the lexicographical comparison between +// the primitive underlying values. +func (DateTypeDescriptor) Compare(left, right Value) (int, error) { + if !left.Is(Date) { + return 0, ErrTypeMismatch(Date, left.Type()) + } + if !right.Is(Date) { + return 0, ErrTypeMismatch(Date, right.Type()) + } + + leftDate := left.(DateValue).Value + rightDate := right.(DateValue).Value + if leftDate.After(rightDate) { + return 1, nil + } else if rightDate.After(leftDate) { + return -1, nil + } + return 0, nil +} + +func (DateTypeDescriptor) String() string { return "Date" } + +// Type returns a blob type. +func (DateValue) Type() Type { return Date } + +// Is checks if this value is of type Date. +func (DateValue) Is(t Type) bool { return t == Date } + +func (v DateValue) String() string { return v.Value.Format(time.RFC3339Nano) } diff --git a/internal/engine/types/type_function.go b/internal/engine/types/type_function.go index be227305..129088e9 100644 --- a/internal/engine/types/type_function.go +++ b/internal/engine/types/type_function.go @@ -17,9 +17,12 @@ type ( genericTypeDescriptor } - functionValue struct { - Name string - value func(...Value) (Value, error) + // FunctionValue represents a callable function. It consists of a nameand + // arguments. How the function is evaluated and what code is actually + // executed, is decided by the engine. + FunctionValue struct { + Name string + Args []Value } ) @@ -40,26 +43,17 @@ func (FunctionTypeDescriptor) String() string { return "Function" } // NewFunctionValue creates a new function value with the given name and // underlying function. -func NewFunctionValue(name string, fn func(...Value) (Value, error)) Value { - return functionValue{ - Name: name, - value: fn, +func NewFunctionValue(name string, args ...Value) FunctionValue { + return FunctionValue{ + Name: name, + Args: args, } } -// CallWithArgs will call the underlying function with the given arguments. -func (f functionValue) CallWithArgs(args ...Value) (Value, error) { - result, err := f.value(args...) - if err != nil { - return nil, fmt.Errorf("call %v: %w", f.Name, err) - } - return result, nil -} - // Type returns a function type. -func (functionValue) Type() Type { return Function } +func (FunctionValue) Type() Type { return Function } // Is checks if this value is of type function. -func (functionValue) Is(t Type) bool { return t == Function } +func (FunctionValue) Is(t Type) bool { return t == Function } -func (f functionValue) String() string { return f.Name + "()" } +func (f FunctionValue) String() string { return f.Name + "()" } From 0e86523ce1b827d739851c8e2b1afa7997307297 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 25 Jun 2020 10:21:31 +0200 Subject: [PATCH 38/77] Fix usage and evaluation of function values --- internal/engine/builtin.go | 33 +++++++++----- internal/engine/context.go | 10 ++++ internal/engine/engine.go | 26 ++++++----- internal/engine/error.go | 8 ++++ internal/engine/evaluate.go | 8 ++-- internal/engine/expression.go | 32 ++++++++++++- internal/engine/expression_test.go | 73 ++++++++++++++++++++++++++---- internal/engine/function.go | 13 ++++++ internal/engine/option.go | 18 ++++++-- 9 files changed, 178 insertions(+), 43 deletions(-) create mode 100644 internal/engine/context.go create mode 100644 internal/engine/function.go diff --git a/internal/engine/builtin.go b/internal/engine/builtin.go index ca725b41..699c40c9 100644 --- a/internal/engine/builtin.go +++ b/internal/engine/builtin.go @@ -2,30 +2,41 @@ package engine import ( "fmt" + "strings" "github.com/tomarrell/lbadd/internal/engine/types" ) -type builtinFunction = func(...types.Value) (types.Value, error) +var ( + // suppress warnings, TODO: remove + _ = builtinCount + _ = builtinUCase + _ = builtinLCase + _ = builtinMin +) -func builtinRandom(args ...types.Value) (types.Value, error) { - return nil, ErrUnimplemented +func builtinNow(tp timeProvider) (types.Value, error) { + return types.DateValue{Value: tp()}, nil } func builtinCount(args ...types.Value) (types.Value, error) { return types.NumericValue{Value: float64(len(args))}, nil } -func builtinUCase(args ...types.Value) (types.Value, error) { - return nil, ErrUnimplemented -} - -func builtinLCase(args ...types.Value) (types.Value, error) { - return nil, ErrUnimplemented +func builtinUCase(args ...types.StringValue) ([]types.StringValue, error) { + var output []types.StringValue + for _, arg := range args { + output = append(output, types.StringValue{Value: strings.ToUpper(arg.Value)}) + } + return output, nil } -func builtinNow(args ...types.Value) (types.Value, error) { - return nil, ErrUnimplemented +func builtinLCase(args ...types.StringValue) ([]types.StringValue, error) { + var output []types.StringValue + for _, arg := range args { + output = append(output, types.StringValue{Value: strings.ToLower(arg.Value)}) + } + return output, nil } func builtinMax(args ...types.Value) (types.Value, error) { diff --git a/internal/engine/context.go b/internal/engine/context.go new file mode 100644 index 00000000..d895a8a4 --- /dev/null +++ b/internal/engine/context.go @@ -0,0 +1,10 @@ +package engine + +// ExecutionContext is a context that is passed down throughout a complete +// evaluation. It may be populated further. +type ExecutionContext struct { + *executionContext +} + +type executionContext struct { +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index cc391c31..3a105de3 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -1,6 +1,9 @@ package engine import ( + "math/rand" + "time" + "github.com/rs/zerolog" "github.com/tomarrell/lbadd/internal/compiler/command" "github.com/tomarrell/lbadd/internal/engine/profile" @@ -9,6 +12,9 @@ import ( "github.com/tomarrell/lbadd/internal/engine/storage/cache" ) +type timeProvider func() time.Time +type randomProvider func() float64 + // Engine is the component that is used to evaluate commands. type Engine struct { log zerolog.Logger @@ -16,7 +22,8 @@ type Engine struct { pageCache cache.Cache profiler *profile.Profiler - builtinFunctions map[string]builtinFunction + timeProvider timeProvider + randomProvider randomProvider } // New creates a new engine object and applies the given options to it. @@ -26,15 +33,8 @@ func New(dbFile *storage.DBFile, opts ...Option) (Engine, error) { dbFile: dbFile, pageCache: dbFile.Cache(), - builtinFunctions: map[string]builtinFunction{ - "RAND": builtinRandom, - "COUNT": builtinCount, - "UCASE": builtinUCase, - "LCASE": builtinLCase, - "NOW": builtinNow, - "MAX": builtinMax, - "MIN": builtinMin, - }, + timeProvider: time.Now, + randomProvider: rand.Float64, } for _, opt := range opts { opt(&e) @@ -52,7 +52,11 @@ func (e Engine) Evaluate(cmd command.Command) (result.Result, error) { _ = e.gt _ = e.lteq _ = e.gteq - return e.evaluate(cmd) + + ctx := ExecutionContext{ + executionContext: &executionContext{}, + } + return e.evaluate(ctx, cmd) } // Closed determines whether the underlying database file was closed. If so, diff --git a/internal/engine/error.go b/internal/engine/error.go index c43c9c54..d444cf59 100644 --- a/internal/engine/error.go +++ b/internal/engine/error.go @@ -1,5 +1,7 @@ package engine +import "fmt" + // Error is a sentinel error. type Error string @@ -11,3 +13,9 @@ const ( ErrUnsupported Error = "unsupported" ErrUnimplemented Error = "unimplemented" ) + +// ErrNoSuchFunction returns an error indicating that an error with the given +// name can not be found. +func ErrNoSuchFunction(name string) error { + return fmt.Errorf("no function for name %v(...)", name) +} diff --git a/internal/engine/evaluate.go b/internal/engine/evaluate.go index 2413dbe7..b119094d 100644 --- a/internal/engine/evaluate.go +++ b/internal/engine/evaluate.go @@ -8,10 +8,10 @@ import ( "github.com/tomarrell/lbadd/internal/engine/types" ) -func (e Engine) evaluate(c command.Command) (result.Result, error) { +func (e Engine) evaluate(ctx ExecutionContext, c command.Command) (result.Result, error) { switch cmd := c.(type) { case command.Values: - values, err := e.evaluateValues(cmd) + values, err := e.evaluateValues(ctx, cmd) if err != nil { return nil, fmt.Errorf("values: %w", err) } @@ -24,12 +24,12 @@ func (e Engine) evaluate(c command.Command) (result.Result, error) { return nil, nil } -func (e Engine) evaluateValues(v command.Values) ([][]types.Value, error) { +func (e Engine) evaluateValues(ctx ExecutionContext, v command.Values) ([][]types.Value, error) { result := make([][]types.Value, len(v.Values)) for y, values := range v.Values { rowValues := make([]types.Value, len(values)) for x, value := range values { - internalValue, err := e.evaluateExpression(value) + internalValue, err := e.evaluateExpression(ctx, value) if err != nil { return nil, fmt.Errorf("expr: %w", err) } diff --git a/internal/engine/expression.go b/internal/engine/expression.go index f3dec1e1..a09746aa 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -11,12 +11,40 @@ import ( // value, meaning that it can only be evaluated to a constant value with a given // execution context. This execution context must be inferred from the engine // receiver. -func (e Engine) evaluateExpression(expr command.Expr) (types.Value, error) { +func (e Engine) evaluateExpression(ctx ExecutionContext, expr command.Expr) (types.Value, error) { switch ex := expr.(type) { case command.ConstantBooleanExpr: return types.BoolValue{Value: ex.Value}, nil case command.LiteralExpr: - return types.StringValue{Value: ex.Value}, nil + return e.evaluateLiteralExpr(ctx, ex) + case command.FunctionExpr: + return e.evaluateFunctionExpr(ctx, ex) } return nil, fmt.Errorf("cannot evaluate expression of type %T", expr) } + +func (e Engine) evaluateMultipleExpressions(ctx ExecutionContext, exprs []command.Expr) ([]types.Value, error) { + var vals []types.Value + for _, expr := range exprs { + evaluated, err := e.evaluateExpression(ctx, expr) + if err != nil { + return nil, err + } + vals = append(vals, evaluated) + } + return vals, nil +} + +func (e Engine) evaluateLiteralExpr(ctx ExecutionContext, expr command.LiteralExpr) (types.Value, error) { + return types.StringValue{Value: expr.Value}, nil +} + +func (e Engine) evaluateFunctionExpr(ctx ExecutionContext, expr command.FunctionExpr) (types.Value, error) { + exprs, err := e.evaluateMultipleExpressions(ctx, expr.Args) + if err != nil { + return nil, fmt.Errorf("arguments: %w", err) + } + + function := types.NewFunctionValue(expr.Name, exprs...) + return e.evaluateFunction(ctx, function) +} diff --git a/internal/engine/expression_test.go b/internal/engine/expression_test.go index 8ebab307..8e631401 100644 --- a/internal/engine/expression_test.go +++ b/internal/engine/expression_test.go @@ -2,6 +2,7 @@ package engine import ( "testing" + "time" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" @@ -10,45 +11,97 @@ import ( ) func TestEngine_evaluateExpression(t *testing.T) { + fixedTimestamp, err := time.Parse("2006-01-02T15:04:05", "2020-06-01T14:05:12") + assert.NoError(t, err) + fixedTimeProvider := func() time.Time { return fixedTimestamp } + tests := []struct { name string + e Engine + ctx ExecutionContext expr command.Expr want types.Value - wantErr bool + wantErr string }{ { "nil", + builder().build(), + ExecutionContext{}, nil, nil, - true, + "cannot evaluate expression of type ", }, { "true", + builder().build(), + ExecutionContext{}, command.ConstantBooleanExpr{Value: true}, types.BoolValue{Value: true}, - false, + "", }, { "false", + builder().build(), + ExecutionContext{}, command.ConstantBooleanExpr{Value: false}, types.BoolValue{Value: false}, - false, + "", + }, + { + "function NOW", + builder(). + timeProvider(fixedTimeProvider). + build(), + ExecutionContext{}, + command.FunctionExpr{ + Name: "NOW", + }, + types.DateValue{Value: fixedTimestamp}, + "", + }, + { + "unknown function", + builder().build(), + ExecutionContext{}, + command.FunctionExpr{ + Name: "NOTEXIST", + }, + nil, + "no function for name NOTEXIST(...)", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assert := assert.New(t) - e := Engine{ - log: zerolog.Nop(), - } - got, err := e.evaluateExpression(tt.expr) + got, err := tt.e.evaluateExpression(tt.ctx, tt.expr) assert.Equal(tt.want, got) - if tt.wantErr { - assert.Error(err) + if tt.wantErr != "" { + assert.EqualError(err, tt.wantErr) } else { assert.NoError(err) } }) } } + +type engineBuilder struct { + e Engine +} + +func builder() engineBuilder { + return engineBuilder{ + Engine{ + log: zerolog.Nop(), + }, + } +} + +func (b engineBuilder) timeProvider(tp timeProvider) engineBuilder { + b.e.timeProvider = tp + return b +} + +func (b engineBuilder) build() Engine { + return b.e +} diff --git a/internal/engine/function.go b/internal/engine/function.go new file mode 100644 index 00000000..22fbacc7 --- /dev/null +++ b/internal/engine/function.go @@ -0,0 +1,13 @@ +package engine + +import ( + "github.com/tomarrell/lbadd/internal/engine/types" +) + +func (e Engine) evaluateFunction(ctx ExecutionContext, fn types.FunctionValue) (types.Value, error) { + switch fn.Name { + case "NOW": + return builtinNow(e.timeProvider) + } + return nil, ErrNoSuchFunction(fn.Name) +} diff --git a/internal/engine/option.go b/internal/engine/option.go index 2a2d9093..d424678c 100644 --- a/internal/engine/option.go +++ b/internal/engine/option.go @@ -23,11 +23,19 @@ func WithProfiler(profiler *profile.Profiler) Option { } } -// WithBuiltinFunction registeres or overwrites an existing builtin function. -// Use this to (e.g.) overwrite the SQL function NOW() to get constant -// timestamps. -func WithBuiltinFunction(name string, fn builtinFunction) Option { +// WithTimeProvider sets a time provider, which will be used by the engine to +// evaluate expressions, that require a timestamp, such as the function NOW(). +func WithTimeProvider(tp timeProvider) Option { return func(e *Engine) { - e.builtinFunctions[name] = fn + e.timeProvider = tp + } +} + +// WithRandomProvider sets a random provider, which will be used by the engine +// to evaluate expressions, that require a random source, such as the function +// RAND(). +func WithRandomProvider(rp randomProvider) Option { + return func(e *Engine) { + e.randomProvider = rp } } From 13cbc3476da802236a1177c0857d2815f3707ee8 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 25 Jun 2020 18:50:26 +0200 Subject: [PATCH 39/77] Refactor the type system --- internal/engine/builtin.go | 18 ++- internal/engine/builtin_test.go | 40 ++---- internal/engine/compare.go | 6 +- internal/engine/compare_test.go | 16 +-- internal/engine/error.go | 27 +++- internal/engine/expression.go | 6 +- internal/engine/expression_test.go | 6 +- internal/engine/result/value.go | 2 +- .../engine/storage/page/celltype_string.go | 25 ++++ internal/engine/types/basetype_string.go | 28 ---- internal/engine/types/bool_type.go | 29 ++++ internal/engine/types/bool_value.go | 29 ++++ internal/engine/types/date_type.go | 29 ++++ internal/engine/types/date_value.go | 27 ++++ internal/engine/types/error.go | 6 + internal/engine/types/function_type.go | 13 ++ internal/engine/types/function_value.go | 35 +++++ internal/engine/types/string_type.go | 41 ++++++ internal/engine/types/string_type_test.go | 127 ++++++++++++++++++ internal/engine/types/string_value.go | 25 ++++ internal/engine/types/string_value_test.go | 29 ++++ internal/engine/types/type.go | 32 ++++- internal/engine/types/type_blob.go | 57 -------- internal/engine/types/type_bool.go | 64 --------- internal/engine/types/type_date.go | 62 --------- internal/engine/types/type_descriptor.go | 40 ------ internal/engine/types/type_function.go | 59 -------- internal/engine/types/type_numeric.go | 65 --------- internal/engine/types/type_string.go | 54 -------- internal/engine/types/value.go | 13 ++ 30 files changed, 524 insertions(+), 486 deletions(-) create mode 100644 internal/engine/storage/page/celltype_string.go delete mode 100644 internal/engine/types/basetype_string.go create mode 100644 internal/engine/types/bool_type.go create mode 100644 internal/engine/types/bool_value.go create mode 100644 internal/engine/types/date_type.go create mode 100644 internal/engine/types/date_value.go create mode 100644 internal/engine/types/function_type.go create mode 100644 internal/engine/types/function_value.go create mode 100644 internal/engine/types/string_type.go create mode 100644 internal/engine/types/string_type_test.go create mode 100644 internal/engine/types/string_value.go create mode 100644 internal/engine/types/string_value_test.go delete mode 100644 internal/engine/types/type_blob.go delete mode 100644 internal/engine/types/type_bool.go delete mode 100644 internal/engine/types/type_date.go delete mode 100644 internal/engine/types/type_descriptor.go delete mode 100644 internal/engine/types/type_function.go delete mode 100644 internal/engine/types/type_numeric.go delete mode 100644 internal/engine/types/type_string.go diff --git a/internal/engine/builtin.go b/internal/engine/builtin.go index 699c40c9..da5b20cc 100644 --- a/internal/engine/builtin.go +++ b/internal/engine/builtin.go @@ -16,11 +16,11 @@ var ( ) func builtinNow(tp timeProvider) (types.Value, error) { - return types.DateValue{Value: tp()}, nil + return types.NewDate(tp()), nil } func builtinCount(args ...types.Value) (types.Value, error) { - return types.NumericValue{Value: float64(len(args))}, nil + return nil, ErrUnimplemented } func builtinUCase(args ...types.StringValue) ([]types.StringValue, error) { @@ -48,10 +48,14 @@ func builtinMax(args ...types.Value) (types.Value, error) { return nil, err } - largest := args[0] + largest := args[0] // start at 0 and compare on t := largest.Type() + comparator, ok := t.(types.Comparator) + if !ok { + return nil, ErrUncomparable(t) + } for i := 1; i < len(args); i++ { - res, err := t.Compare(largest, args[i]) + res, err := comparator.Compare(largest, args[i]) if err != nil { return nil, fmt.Errorf("compare: %w", err) } @@ -73,8 +77,12 @@ func builtinMin(args ...types.Value) (types.Value, error) { smallest := args[0] t := smallest.Type() + comparator, ok := t.(types.Comparator) + if !ok { + return nil, ErrUncomparable(t) + } for i := 1; i < len(args); i++ { - res, err := t.Compare(smallest, args[i]) + res, err := comparator.Compare(smallest, args[i]) if err != nil { return nil, fmt.Errorf("compare: %w", err) } diff --git a/internal/engine/builtin_test.go b/internal/engine/builtin_test.go index 52fda364..2350efb9 100644 --- a/internal/engine/builtin_test.go +++ b/internal/engine/builtin_test.go @@ -29,38 +29,18 @@ func Test_builtinMax(t *testing.T) { "bools", args{ []types.Value{ - types.BoolValue{Value: true}, - types.BoolValue{Value: false}, - types.BoolValue{Value: false}, - types.BoolValue{Value: true}, - types.BoolValue{Value: false}, - types.BoolValue{Value: true}, - types.BoolValue{Value: false}, - types.BoolValue{Value: true}, - types.BoolValue{Value: true}, + types.NewBool(true), + types.NewBool(false), + types.NewBool(false), + types.NewBool(true), + types.NewBool(false), + types.NewBool(true), + types.NewBool(false), + types.NewBool(true), + types.NewBool(true), }, }, - types.BoolValue{Value: true}, - false, - }, - { - "numbers", - args{ - []types.Value{ - types.NumericValue{Value: 32698.7236}, - types.NumericValue{Value: 33020.4705}, - types.NumericValue{Value: 28550.057}, - types.NumericValue{Value: 17980.2620}, - types.NumericValue{Value: 37105.784}, - types.NumericValue{Value: 623164325426457348.4231}, - types.NumericValue{Value: 53226.854}, - types.NumericValue{Value: 49344.266}, - types.NumericValue{Value: 10634.3037}, - types.NumericValue{Value: 36735.083}, - types.NumericValue{Value: 14828.1674}, - }, - }, - types.NumericValue{Value: 623164325426457348.4231}, + types.NewBool(true), false, }, } diff --git a/internal/engine/compare.go b/internal/engine/compare.go index 55ba9168..29eb012c 100644 --- a/internal/engine/compare.go +++ b/internal/engine/compare.go @@ -22,7 +22,11 @@ func (e Engine) cmp(left, right types.Value) cmpResult { if !right.Is(left.Type()) { return cmpUncomparable } - res, err := left.Type().Compare(left, right) + comparator, ok := left.Type().(types.Comparator) + if !ok { + return cmpUncomparable + } + res, err := comparator.Compare(left, right) if err != nil { // TODO: log error? return cmpUncomparable diff --git a/internal/engine/compare_test.go b/internal/engine/compare_test.go index c8518109..26de5617 100644 --- a/internal/engine/compare_test.go +++ b/internal/engine/compare_test.go @@ -16,26 +16,26 @@ func TestEngine_cmp(t *testing.T) { }{ { "true <-> true", - types.BoolValue{Value: true}, - types.BoolValue{Value: true}, + types.NewBool(true), + types.NewBool(true), cmpEqual, }, { "true <-> false", - types.BoolValue{Value: true}, - types.BoolValue{Value: false}, + types.NewBool(true), + types.NewBool(false), cmpGreaterThan, }, { "false <-> true", - types.BoolValue{Value: false}, - types.BoolValue{Value: true}, + types.NewBool(false), + types.NewBool(true), cmpLessThan, }, { "false <-> false", - types.BoolValue{Value: false}, - types.BoolValue{Value: false}, + types.NewBool(false), + types.NewBool(false), cmpEqual, }, } diff --git a/internal/engine/error.go b/internal/engine/error.go index d444cf59..5b5ce80c 100644 --- a/internal/engine/error.go +++ b/internal/engine/error.go @@ -1,16 +1,28 @@ package engine -import "fmt" +import ( + "fmt" + + "github.com/tomarrell/lbadd/internal/engine/types" +) // Error is a sentinel error. type Error string func (e Error) Error() string { return string(e) } -// Sentinel errors. const ( - ErrClosed Error = "already closed" - ErrUnsupported Error = "unsupported" + // ErrClosed indicates that the component can not be used anymore, because + // it already has been closed. + ErrClosed Error = "already closed" + // ErrUnsupported indicates that a requested feature is explicitely not + // supported. This is different from ErrUnimplemented, since + // ErrUnimplemented indicates, that the feature has not been implemented + // yet, while ErrUnsupported indicates, that the feature is intentionally + // unimplemented. + ErrUnsupported Error = "unsupported" + // ErrUnimplemented indicates a missing implementation for the requested + // feature. It may be implemented in the next version. ErrUnimplemented Error = "unimplemented" ) @@ -19,3 +31,10 @@ const ( func ErrNoSuchFunction(name string) error { return fmt.Errorf("no function for name %v(...)", name) } + +// ErrUncomparable returns an error indicating that the given type does not +// implement the types.Comparator interface, and thus, values of that type +// cannot be compared. +func ErrUncomparable(t types.Type) error { + return fmt.Errorf("type %v is not comparable", t) +} diff --git a/internal/engine/expression.go b/internal/engine/expression.go index a09746aa..fafbec3c 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -14,7 +14,7 @@ import ( func (e Engine) evaluateExpression(ctx ExecutionContext, expr command.Expr) (types.Value, error) { switch ex := expr.(type) { case command.ConstantBooleanExpr: - return types.BoolValue{Value: ex.Value}, nil + return types.NewBool(ex.Value), nil case command.LiteralExpr: return e.evaluateLiteralExpr(ctx, ex) case command.FunctionExpr: @@ -36,7 +36,7 @@ func (e Engine) evaluateMultipleExpressions(ctx ExecutionContext, exprs []comman } func (e Engine) evaluateLiteralExpr(ctx ExecutionContext, expr command.LiteralExpr) (types.Value, error) { - return types.StringValue{Value: expr.Value}, nil + return types.NewString(expr.Value), nil } func (e Engine) evaluateFunctionExpr(ctx ExecutionContext, expr command.FunctionExpr) (types.Value, error) { @@ -45,6 +45,6 @@ func (e Engine) evaluateFunctionExpr(ctx ExecutionContext, expr command.Function return nil, fmt.Errorf("arguments: %w", err) } - function := types.NewFunctionValue(expr.Name, exprs...) + function := types.NewFunction(expr.Name, exprs...) return e.evaluateFunction(ctx, function) } diff --git a/internal/engine/expression_test.go b/internal/engine/expression_test.go index 8e631401..9b3553fb 100644 --- a/internal/engine/expression_test.go +++ b/internal/engine/expression_test.go @@ -36,7 +36,7 @@ func TestEngine_evaluateExpression(t *testing.T) { builder().build(), ExecutionContext{}, command.ConstantBooleanExpr{Value: true}, - types.BoolValue{Value: true}, + types.NewBool(true), "", }, { @@ -44,7 +44,7 @@ func TestEngine_evaluateExpression(t *testing.T) { builder().build(), ExecutionContext{}, command.ConstantBooleanExpr{Value: false}, - types.BoolValue{Value: false}, + types.NewBool(false), "", }, { @@ -56,7 +56,7 @@ func TestEngine_evaluateExpression(t *testing.T) { command.FunctionExpr{ Name: "NOW", }, - types.DateValue{Value: fixedTimestamp}, + types.NewDate(fixedTimestamp), "", }, { diff --git a/internal/engine/result/value.go b/internal/engine/result/value.go index 34a82bdd..0cf4fb98 100644 --- a/internal/engine/result/value.go +++ b/internal/engine/result/value.go @@ -68,7 +68,7 @@ func (r valueResult) String() string { var types []string for _, col := range r.Cols() { - types = append(types, col.Type().String()) + types = append(types, col.Type().Name()) } _, _ = fmt.Fprintln(w, strings.Join(types, "\t")) diff --git a/internal/engine/storage/page/celltype_string.go b/internal/engine/storage/page/celltype_string.go new file mode 100644 index 00000000..902b3068 --- /dev/null +++ b/internal/engine/storage/page/celltype_string.go @@ -0,0 +1,25 @@ +// Code generated by "stringer -type=CellType"; DO NOT EDIT. + +package page + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[CellTypeUnknown-0] + _ = x[CellTypeRecord-1] + _ = x[CellTypePointer-2] +} + +const _CellType_name = "CellTypeUnknownCellTypeRecordCellTypePointer" + +var _CellType_index = [...]uint8{0, 15, 29, 44} + +func (i CellType) String() string { + if i >= CellType(len(_CellType_index)-1) { + return "CellType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _CellType_name[_CellType_index[i]:_CellType_index[i+1]] +} diff --git a/internal/engine/types/basetype_string.go b/internal/engine/types/basetype_string.go deleted file mode 100644 index 6d6404b1..00000000 --- a/internal/engine/types/basetype_string.go +++ /dev/null @@ -1,28 +0,0 @@ -// Code generated by "stringer -type=BaseType"; DO NOT EDIT. - -package types - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[BaseTypeUnknown-0] - _ = x[BaseTypeBool-1] - _ = x[BaseTypeBinary-2] - _ = x[BaseTypeString-3] - _ = x[BaseTypeNumeric-4] - _ = x[BaseTypeFunction-5] -} - -const _BaseType_name = "BaseTypeUnknownBaseTypeBoolBaseTypeBinaryBaseTypeStringBaseTypeNumericBaseTypeFunction" - -var _BaseType_index = [...]uint8{0, 15, 27, 41, 55, 70, 86} - -func (i BaseType) String() string { - if i >= BaseType(len(_BaseType_index)-1) { - return "BaseType(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _BaseType_name[_BaseType_index[i]:_BaseType_index[i+1]] -} diff --git a/internal/engine/types/bool_type.go b/internal/engine/types/bool_type.go new file mode 100644 index 00000000..c9d83862 --- /dev/null +++ b/internal/engine/types/bool_type.go @@ -0,0 +1,29 @@ +package types + +var ( + Bool = BoolType{ + typ: typ{ + name: "Bool", + }, + } +) + +type BoolType struct { + typ +} + +func (t BoolType) Compare(left, right Value) (int, error) { + if err := t.ensureCanCompare(left, right); err != nil { + return 0, err + } + + leftBool := left.(BoolValue).Value + rightBool := right.(BoolValue).Value + + if leftBool && !rightBool { + return 1, nil + } else if !leftBool && rightBool { + return -1, nil + } + return 0, nil +} diff --git a/internal/engine/types/bool_value.go b/internal/engine/types/bool_value.go new file mode 100644 index 00000000..d002f433 --- /dev/null +++ b/internal/engine/types/bool_value.go @@ -0,0 +1,29 @@ +package types + +import ( + "strconv" +) + +var _ Value = (*BoolValue)(nil) + +// BoolValue is a value of type Bool. +type BoolValue struct { + value + + // Value is the underlying primitive value. + Value bool +} + +// NewBool creates a new value of type Bool. +func NewBool(v bool) BoolValue { + return BoolValue{ + value: value{ + typ: Bool, + }, + Value: v, + } +} + +func (v BoolValue) String() string { + return strconv.FormatBool(v.Value) +} diff --git a/internal/engine/types/date_type.go b/internal/engine/types/date_type.go new file mode 100644 index 00000000..cdfff019 --- /dev/null +++ b/internal/engine/types/date_type.go @@ -0,0 +1,29 @@ +package types + +var ( + Date = DateType{ + typ: typ{ + name: "Date", + }, + } +) + +type DateType struct { + typ +} + +func (t DateType) Compare(left, right Value) (int, error) { + if err := t.ensureCanCompare(left, right); err != nil { + return 0, err + } + + leftDate := left.(DateValue).Value + rightDate := right.(DateValue).Value + + if leftDate.After(rightDate) { + return 1, nil + } else if rightDate.After(leftDate) { + return -1, nil + } + return 0, nil +} diff --git a/internal/engine/types/date_value.go b/internal/engine/types/date_value.go new file mode 100644 index 00000000..6c4736f2 --- /dev/null +++ b/internal/engine/types/date_value.go @@ -0,0 +1,27 @@ +package types + +import "time" + +var _ Value = (*DateValue)(nil) + +// DateValue is a value of type Date. +type DateValue struct { + value + + // Value is the underlying primitive value. + Value time.Time +} + +// NewDate creates a new value of type Date. +func NewDate(v time.Time) DateValue { + return DateValue{ + value: value{ + typ: Date, + }, + Value: v, + } +} + +func (v DateValue) String() string { + return v.Value.Format(time.RFC3339) +} diff --git a/internal/engine/types/error.go b/internal/engine/types/error.go index 7bcb4a3f..4dc2cbf5 100644 --- a/internal/engine/types/error.go +++ b/internal/engine/types/error.go @@ -12,3 +12,9 @@ func (e Error) Error() string { return string(e) } func ErrTypeMismatch(expected, got Type) Error { return Error(fmt.Sprintf("type mismatch: want %v, got %v", expected, got)) } + +// ErrCannotCast returns an error that indicates, that a case from the given +// frmo type to the given to type cannot be performed. +func ErrCannotCast(from, to Type) Error { + return Error(fmt.Sprintf("cannot cast %v to %v", from, to)) +} diff --git a/internal/engine/types/function_type.go b/internal/engine/types/function_type.go new file mode 100644 index 00000000..486bfefc --- /dev/null +++ b/internal/engine/types/function_type.go @@ -0,0 +1,13 @@ +package types + +var ( + Function = FunctionType{ + typ: typ{ + name: "Function", + }, + } +) + +type FunctionType struct { + typ +} diff --git a/internal/engine/types/function_value.go b/internal/engine/types/function_value.go new file mode 100644 index 00000000..d8edb16f --- /dev/null +++ b/internal/engine/types/function_value.go @@ -0,0 +1,35 @@ +package types + +import ( + "fmt" + "strings" +) + +var _ Value = (*FunctionValue)(nil) + +// FunctionValue is a value of type Function. +type FunctionValue struct { + value + + Name string + Args []Value +} + +// NewFunction creates a new value of type Function. +func NewFunction(name string, args ...Value) FunctionValue { + return FunctionValue{ + value: value{ + typ: Function, + }, + Name: name, + Args: args, + } +} + +func (v FunctionValue) String() string { + var args []string + for _, arg := range v.Args { + args = append(args, arg.String()) + } + return fmt.Sprintf("%v(%v)", v.Name, strings.Join(args, ", ")) +} diff --git a/internal/engine/types/string_type.go b/internal/engine/types/string_type.go new file mode 100644 index 00000000..ba949471 --- /dev/null +++ b/internal/engine/types/string_type.go @@ -0,0 +1,41 @@ +package types + +import "strings" + +var ( + // String is the string type. Its base type is a string. + String = StringType{ + typ: typ{ + name: "String", + }, + } +) + +var _ Type = (*StringType)(nil) +var _ Value = (*StringValue)(nil) +var _ Comparator = (*StringType)(nil) +var _ Caster = (*StringType)(nil) + +// StringType is the type descriptor for a string value. +type StringType struct { + typ +} + +// Compare for the String is defined as the lexicographical comparison of +// the two underlying primitive values. +func (t StringType) Compare(left, right Value) (int, error) { + if err := t.ensureCanCompare(left, right); err != nil { + return 0, err + } + + leftString := left.(StringValue).Value + rightString := right.(StringValue).Value + return strings.Compare(leftString, rightString), nil +} + +func (StringType) Cast(v Value) (Value, error) { + if v.Is(String) { + return v, nil + } + return NewString(v.String()), nil +} diff --git a/internal/engine/types/string_type_test.go b/internal/engine/types/string_type_test.go new file mode 100644 index 00000000..a49b666e --- /dev/null +++ b/internal/engine/types/string_type_test.go @@ -0,0 +1,127 @@ +package types + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestStringType_Compare(t *testing.T) { + tests := []struct { + name string + first Value + second Value + want int + wantErr bool + }{ + { + "nils", + nil, + nil, + 0, + true, + }, + { + "empty strings", + NewString(""), + NewString(""), + 0, + false, + }, + { + "simple", + NewString("a"), + NewString("b"), + -1, + false, + }, + { + "equal", + NewString("f"), + NewString("f"), + 0, + false, + }, + { + "long", + NewString("foh382w9fh3wo4rgefisawel"), + NewString("9548h7gor8shuspdofjwepor"), + 1, + false, + }, + { + "different", + NewString("abc"), + NewString("z"), + -1, + false, + }, + { + "one empty", + NewString("abc"), + NewString(""), + 1, + false, + }, + { + "uncomparable", + NewDate(time.Now()), + NewString(""), + 0, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Run("ltr", func(t *testing.T) { + assert := assert.New(t) + res, err := String.Compare(tt.first, tt.second) + if tt.wantErr { + assert.Error(err) + } else { + assert.Equal(tt.want, res) + } + }) + t.Run("rtl", func(t *testing.T) { + assert := assert.New(t) + res, err := String.Compare(tt.second, tt.first) + if tt.wantErr { + assert.Error(err) + } else { + assert.Equal(tt.want, res*-1) + } + }) + }) + } +} + +func TestStringType_Cast(t *testing.T) { + tests := []struct { + name string + from Value + want Value + wantErr bool + }{ + { + "string to string", + NewString("abc"), + NewString("abc"), + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + typ := tt.want.Type() + got, err := typ.(Caster).Cast(tt.from) + assert.Equal(tt.want, got) + if tt.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + } + }) + } +} diff --git a/internal/engine/types/string_value.go b/internal/engine/types/string_value.go new file mode 100644 index 00000000..09ce8caf --- /dev/null +++ b/internal/engine/types/string_value.go @@ -0,0 +1,25 @@ +package types + +var _ Value = (*StringValue)(nil) + +// StringValue is a value of type String. +type StringValue struct { + value + + // Value is the underlying primitive value. + Value string +} + +// NewString creates a new value of type String. +func NewString(v string) StringValue { + return StringValue{ + value: value{ + typ: String, + }, + Value: v, + } +} + +func (v StringValue) String() string { + return v.Value +} diff --git a/internal/engine/types/string_value_test.go b/internal/engine/types/string_value_test.go new file mode 100644 index 00000000..97783e0d --- /dev/null +++ b/internal/engine/types/string_value_test.go @@ -0,0 +1,29 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringValue_Is(t *testing.T) { + assert := assert.New(t) + + v := NewString("foobar") + assert.True(v.Is(String)) +} + +func TestStringValue_Type(t *testing.T) { + assert := assert.New(t) + + v := NewString("foobar") + assert.Equal(String, v.Type()) +} + +func TestStringValue_String(t *testing.T) { + assert := assert.New(t) + + baseStr := "foobar" + v := NewString(baseStr) + assert.Equal(baseStr, v.String()) +} diff --git a/internal/engine/types/type.go b/internal/engine/types/type.go index 8bfa766d..586dbe17 100644 --- a/internal/engine/types/type.go +++ b/internal/engine/types/type.go @@ -19,13 +19,41 @@ type ( Cast(Value) (Value, error) } + // Codec describes a component that can encode and decode values. Types + // embed codec, but are only expected to be able to encode and decode values + // of their own type. If that is not the case, a type mismatch may be + // returned. + Codec interface { + Encode(Value) ([]byte, error) + Decode([]byte) (Value, error) + } + // Type is a data type that consists of a type descriptor and a comparator. // The comparator forces types to define relations between two values of // this type. A type is only expected to be able to handle values of its own // type. Type interface { - TypeDescriptor - Comparator + Name() string fmt.Stringer } ) + +type typ struct { + name string +} + +func (t typ) Name() string { return t.name } +func (t typ) String() string { return t.name } + +func (t typ) ensureCanCompare(left, right Value) error { + if left == nil || right == nil { + return ErrTypeMismatch(t, nil) + } + if !left.Is(t) { + return ErrTypeMismatch(t, left.Type()) + } + if !right.Is(t) { + return ErrTypeMismatch(t, right.Type()) + } + return nil +} diff --git a/internal/engine/types/type_blob.go b/internal/engine/types/type_blob.go deleted file mode 100644 index 36341ca6..00000000 --- a/internal/engine/types/type_blob.go +++ /dev/null @@ -1,57 +0,0 @@ -package types - -import ( - "bytes" -) - -var ( - // Blob is the blob type. A Blob is a Binary Large OBject, and its base type - // is a byte slice. - Blob = BlobTypeDescriptor{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeBinary, - }, - } -) - -var _ Type = (*BlobTypeDescriptor)(nil) -var _ Value = (*BlobValue)(nil) - -type ( - // BlobTypeDescriptor is the type descriptor for Binary Large OBjects. The - // value is basically a byte slice. - BlobTypeDescriptor struct { - genericTypeDescriptor - } - - // BlobValue is a value of type Blob. - BlobValue struct { - // Value is the primitive value of this value object. - Value []byte - } -) - -// Compare for the Blob is defined the lexicographical comparison between -// the primitive underlying values. -func (BlobTypeDescriptor) Compare(left, right Value) (int, error) { - if !left.Is(Blob) { - return 0, ErrTypeMismatch(Blob, left.Type()) - } - if !right.Is(Blob) { - return 0, ErrTypeMismatch(Blob, right.Type()) - } - - leftBlob := left.(BlobValue).Value - rightBlob := right.(BlobValue).Value - return bytes.Compare(leftBlob, rightBlob), nil -} - -func (BlobTypeDescriptor) String() string { return "Blob" } - -// Type returns a blob type. -func (BlobValue) Type() Type { return Blob } - -// Is checks if this value is of type Blob. -func (BlobValue) Is(t Type) bool { return t == Blob } - -func (v BlobValue) String() string { return string(v.Value) } diff --git a/internal/engine/types/type_bool.go b/internal/engine/types/type_bool.go deleted file mode 100644 index 19e1afff..00000000 --- a/internal/engine/types/type_bool.go +++ /dev/null @@ -1,64 +0,0 @@ -package types - -import ( - "fmt" - "strconv" -) - -var ( - // Bool is the boolean type. Its base type is a bool. - Bool = BoolTypeDescriptor{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeBool, - }, - } -) - -var _ Type = (*BoolTypeDescriptor)(nil) -var _ Value = (*BoolValue)(nil) - -type ( - // BoolTypeDescriptor is the boolean type of this engine. - BoolTypeDescriptor struct { - genericTypeDescriptor - } - - // BoolValue is a value of type Bool. - BoolValue struct { - // Value is the underlying primitive value. - Value bool - } -) - -// Compare for the type BoolType is defined as false %v", leftBool, rightBool) -} - -func (BoolTypeDescriptor) String() string { return "Bool" } - -// Type returns a bool type. -func (BoolValue) Type() Type { return Bool } - -// Is checks if this value is of type Bool. -func (BoolValue) Is(t Type) bool { return t == Bool } - -func (v BoolValue) String() string { return strconv.FormatBool(v.Value) } diff --git a/internal/engine/types/type_date.go b/internal/engine/types/type_date.go deleted file mode 100644 index dd703782..00000000 --- a/internal/engine/types/type_date.go +++ /dev/null @@ -1,62 +0,0 @@ -package types - -import ( - "time" -) - -var ( - // Date is the date type. A date is a timestamp, and its base type is a byte - // slice. Internally, the timestamp is represented by time.Time. - Date = DateTypeDescriptor{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeBinary, - }, - } -) - -var _ Type = (*DateTypeDescriptor)(nil) -var _ Value = (*DateValue)(nil) - -type ( - // DateTypeDescriptor is the type descriptor for date objects. The value is - // a time.Time. - DateTypeDescriptor struct { - genericTypeDescriptor - } - - // DateValue is a value of type Date. - DateValue struct { - // Value is the primitive value of this value object. - Value time.Time - } -) - -// Compare for the Date is defined the lexicographical comparison between -// the primitive underlying values. -func (DateTypeDescriptor) Compare(left, right Value) (int, error) { - if !left.Is(Date) { - return 0, ErrTypeMismatch(Date, left.Type()) - } - if !right.Is(Date) { - return 0, ErrTypeMismatch(Date, right.Type()) - } - - leftDate := left.(DateValue).Value - rightDate := right.(DateValue).Value - if leftDate.After(rightDate) { - return 1, nil - } else if rightDate.After(leftDate) { - return -1, nil - } - return 0, nil -} - -func (DateTypeDescriptor) String() string { return "Date" } - -// Type returns a blob type. -func (DateValue) Type() Type { return Date } - -// Is checks if this value is of type Date. -func (DateValue) Is(t Type) bool { return t == Date } - -func (v DateValue) String() string { return v.Value.Format(time.RFC3339Nano) } diff --git a/internal/engine/types/type_descriptor.go b/internal/engine/types/type_descriptor.go deleted file mode 100644 index ab0ca6d5..00000000 --- a/internal/engine/types/type_descriptor.go +++ /dev/null @@ -1,40 +0,0 @@ -package types - -//go:generate stringer -type=BaseType - -// BaseType is an underlying type for parameterized types. -type BaseType uint8 - -// Known base types. -const ( - BaseTypeUnknown BaseType = iota - BaseTypeBool - BaseTypeBinary - BaseTypeString - BaseTypeNumeric - BaseTypeFunction -) - -// TypeDescriptor describes a type in more detail than just the type. Every type -// has a type descriptor, which holds the base type and parameterization. Based -// on the parameterization, the type may be interpreted differently. -// -// Example: The simple type INTEGER would have a type descriptor that describes -// a baseType=number with no further parameterization, whereas the more complex -// type VARCHAR(50) would have a type descriptor, that describes a -// baseType=string and a max length of 50. -type TypeDescriptor interface { - Base() BaseType - // TODO: parameters to be done -} - -// genericTypeDescriptor is a type descriptor that has no parameterization and -// just a base type. -type genericTypeDescriptor struct { - baseType BaseType -} - -// Base returns the base type of this type descriptor. -func (td genericTypeDescriptor) Base() BaseType { - return td.baseType -} diff --git a/internal/engine/types/type_function.go b/internal/engine/types/type_function.go deleted file mode 100644 index 129088e9..00000000 --- a/internal/engine/types/type_function.go +++ /dev/null @@ -1,59 +0,0 @@ -package types - -import "fmt" - -var ( - // Function is the function type. Functions are not comparable. - Function = FunctionTypeDescriptor{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeFunction, - }, - } -) - -type ( - // FunctionTypeDescriptor is the function type of this engine. - FunctionTypeDescriptor struct { - genericTypeDescriptor - } - - // FunctionValue represents a callable function. It consists of a nameand - // arguments. How the function is evaluated and what code is actually - // executed, is decided by the engine. - FunctionValue struct { - Name string - Args []Value - } -) - -// Compare will always return an error, indicating that functions are not -// comparable. Both arguments have to have a funtion type, otherwise a type -// mismatch error will be returned. -func (FunctionTypeDescriptor) Compare(left, right Value) (int, error) { - if !left.Is(Function) { - return 0, ErrTypeMismatch(Function, left.Type()) - } - if !right.Is(Function) { - return 0, ErrTypeMismatch(Function, right.Type()) - } - return -2, fmt.Errorf("functions are not comparable") -} - -func (FunctionTypeDescriptor) String() string { return "Function" } - -// NewFunctionValue creates a new function value with the given name and -// underlying function. -func NewFunctionValue(name string, args ...Value) FunctionValue { - return FunctionValue{ - Name: name, - Args: args, - } -} - -// Type returns a function type. -func (FunctionValue) Type() Type { return Function } - -// Is checks if this value is of type function. -func (FunctionValue) Is(t Type) bool { return t == Function } - -func (f FunctionValue) String() string { return f.Name + "()" } diff --git a/internal/engine/types/type_numeric.go b/internal/engine/types/type_numeric.go deleted file mode 100644 index bd1650d0..00000000 --- a/internal/engine/types/type_numeric.go +++ /dev/null @@ -1,65 +0,0 @@ -package types - -import ( - "fmt" - "strconv" -) - -var ( - // Numeric is the numeric type. Its base type is a numeric type. - Numeric = NumericTypeDescriptor{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeNumeric, - }, - } -) - -var _ Type = (*NumericTypeDescriptor)(nil) -var _ Value = (*NumericValue)(nil) - -type ( - // NumericTypeDescriptor is the type descriptor for parameterized and non-parameterized - // numeric types, such as DECIMAL. - NumericTypeDescriptor struct { - genericTypeDescriptor - } - - // NumericValue is a value of type Numeric. - NumericValue struct { - // Value is the underlying primitive value. - Value float64 - } -) - -// Compare for the Numeric is defined as the lexicographical comparison of the -// two underlying primitive values. -func (NumericTypeDescriptor) Compare(left, right Value) (int, error) { - if !left.Is(Numeric) { - return 0, ErrTypeMismatch(Numeric, left.Type()) - } - if !right.Is(Numeric) { - return 0, ErrTypeMismatch(Numeric, right.Type()) - } - - leftNum := left.(NumericValue).Value - rightNum := right.(NumericValue).Value - switch { - case leftNum < rightNum: - return -1, nil - case leftNum == rightNum: - return 0, nil - case leftNum > rightNum: - return 1, nil - } - return -2, fmt.Errorf("unhandled constellation: %v <-> %v", leftNum, rightNum) -} - -func (NumericTypeDescriptor) String() string { return "Numeric" } - -// Type returns a string type. -func (NumericValue) Type() Type { return Numeric } - -// Is checks if this value is of type Numeric. -func (NumericValue) Is(t Type) bool { return t == Numeric } - -func (v NumericValue) String() string { return strconv.FormatFloat(v.Value, 'f', -1, 64) } diff --git a/internal/engine/types/type_string.go b/internal/engine/types/type_string.go deleted file mode 100644 index 514fc73b..00000000 --- a/internal/engine/types/type_string.go +++ /dev/null @@ -1,54 +0,0 @@ -package types - -import "strings" - -var ( - // String is the string type. Its base type is a string. - String = StringTypeDescriptor{ - genericTypeDescriptor: genericTypeDescriptor{ - baseType: BaseTypeString, - }, - } -) - -var _ Type = (*StringTypeDescriptor)(nil) -var _ Value = (*StringValue)(nil) - -type ( - // StringTypeDescriptor is the type descriptor for parameterized and - // non-parameterized string types, such as VARCHAR or VARCHAR(n). - StringTypeDescriptor struct { - genericTypeDescriptor - } - - // StringValue is a value of type String. - StringValue struct { - // Value is the underlying primitive value. - Value string - } -) - -// Compare for the String is defined as the lexicographical comparison of -// the two underlying primitive values. -func (StringTypeDescriptor) Compare(left, right Value) (int, error) { - if !left.Is(String) { - return 0, ErrTypeMismatch(String, left.Type()) - } - if !right.Is(String) { - return 0, ErrTypeMismatch(String, right.Type()) - } - - leftString := left.(StringValue).Value - rightString := right.(StringValue).Value - return strings.Compare(leftString, rightString), nil -} - -func (StringTypeDescriptor) String() string { return "String" } - -// Type returns a string type. -func (StringValue) Type() Type { return String } - -// Is checks if this value is of type String. -func (StringValue) Is(t Type) bool { return t == String } - -func (v StringValue) String() string { return v.Value } diff --git a/internal/engine/types/value.go b/internal/engine/types/value.go index d13634e0..434ee606 100644 --- a/internal/engine/types/value.go +++ b/internal/engine/types/value.go @@ -14,3 +14,16 @@ type Value interface { type IsTyper interface { Is(Type) bool } + +type value struct { + typ Type +} + +func (v value) Type() Type { + return v.typ +} + +// Is checks whether this value is of the given type. +func (v value) Is(t Type) bool { + return v.typ.Name() == t.Name() +} From ffff33ac94be105a81fbdef7fac892730f0b7c76 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 25 Jun 2020 18:59:31 +0200 Subject: [PATCH 40/77] Add godoc --- internal/engine/types/bool_type.go | 7 +++++++ internal/engine/types/bool_value.go | 4 +++- internal/engine/types/date_type.go | 7 +++++++ internal/engine/types/function_type.go | 3 +++ internal/engine/types/string_type.go | 12 ++++++++---- 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/internal/engine/types/bool_type.go b/internal/engine/types/bool_type.go index c9d83862..bde91866 100644 --- a/internal/engine/types/bool_type.go +++ b/internal/engine/types/bool_type.go @@ -1,6 +1,8 @@ package types var ( + // Bool is the Bool type. Bools are comparable with true>false. The name of + // this type is "Bool". Bool = BoolType{ typ: typ{ name: "Bool", @@ -8,10 +10,15 @@ var ( } ) +// BoolType is a comparable type. type BoolType struct { typ } +// Compare compares two bool values. For this to succeed, both values must be of +// type BoolValue and be not nil. The bool value true is considered larger than +// false. This method will return 1 if left>right, 0 if left==right, and -1 if +// leftright, 0 if left==right, and -1 if +// leftright, 0 +// if left==right, and -1 if left Date: Thu, 25 Jun 2020 19:02:35 +0200 Subject: [PATCH 41/77] Improve godoc --- internal/engine/types/type.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/engine/types/type.go b/internal/engine/types/type.go index 586dbe17..49128620 100644 --- a/internal/engine/types/type.go +++ b/internal/engine/types/type.go @@ -8,7 +8,9 @@ type ( // leftright. What exectly is considered // to be <, ==, > is up to the implementation. Comparator interface { - Compare(Value, Value) (int, error) + // Compare compares the given to values left and right as follows. -1 if + // leftright. + Compare(left, right Value) (int, error) } // Caster wraps the Cast method, which is used to transform the input value From db8d5565d0b9dc73a77e8e017969721a0ea388ff Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 26 Jun 2020 15:28:58 +0200 Subject: [PATCH 42/77] Add doc on page layout --- doc/file-format.md | 38 ++++++++++++++++++++++++++++++-------- doc/page.md | 30 ++++++++++++++++++++++++++++++ doc/page_v1.png | Bin 0 -> 45352 bytes 3 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 doc/page.md create mode 100644 doc/page_v1.png diff --git a/doc/file-format.md b/doc/file-format.md index c5f677ab..9578cebb 100644 --- a/doc/file-format.md +++ b/doc/file-format.md @@ -26,16 +26,15 @@ bytes are the UTF-8 encoding of that string. ### Table pages Table pages do not directly hold data of a table. Instead, they hold pointers to pages, that do, i.e. the index and data page. Table pages do however hold -information about the table schema. The schema information is a single record -that is to be interpreted as a schema (**TODO: -schemas**). +information about the table data definition. The data definition information is +a single record that is to be interpreted as a data definition (**TODO: data definitions**). The keys of the three values, index page, data page and schema are as follows. -* `schema` is a record cell containing the schema information about this table. - That is, columns, column types, references, triggers etc. How the schema - information is to be interpreted, is explained [here](#)(**TODO: schemas**). +* `datadefinition` is a record cell containing the schema information about this + table. That is, columns, column types, references, triggers etc. How the + schema information is to be interpreted, is explained [here](#data-definition). * `index` is a pointer cell pointing to the index page of this table. The index page contains pages that are used as nodes in a btree. See more [here](#index-pages) @@ -44,4 +43,27 @@ The keys of the three values, index page, data page and schema are as follows. ### Index pages -### Data pages \ No newline at end of file +### Data pages +A data page stores plain record in a cell. Cell values are the full records, +cell keys are the RID of the record. The RID (row-ID) is an 8 byte unsigned +integer, which may not be reused for other records, even if a record was +deleted. The only scenario where an RID may be re-used is, when a record is +deleted from a page, while it is also being written into the same page or +another page (i.e. on move only) (this means, that the RID is not actually +re-used, just kept when moving or re-writing a cell). This can happen, if the +size of the record grows, and the cell has to be re-written. The cell keys aka. +RIDs are referenced by cells from the index pages. A full table scan is +performed by obtaining all cells in the data page and checking their records. + +### Data definition +A data definition follows the following format (everything encoded in big +endian). + +* 2 bytes `uint16` the amount of columns +* for each column + * 2 bytes `uint16` frame for the column name + * name bytes + * 1 byte `bool` that is 0 if the table is **NOT**, and 1 if the column is + nullable + * 2 bytes `uint16` frame for the type name + * type name bytes \ No newline at end of file diff --git a/doc/page.md b/doc/page.md new file mode 100644 index 00000000..880a83d6 --- /dev/null +++ b/doc/page.md @@ -0,0 +1,30 @@ +# Page layout +This document describes the layout and format of a single memory page. All pages are +structured like this. + +## Page format +Pages implement the concept of slotted pages. A helpful +resource for understanding is +[this](https://db.in.tum.de/teaching/ss17/moderndbs/chapter3.pdf#page=8) PDF. +Please note though, that we do not follow the exact data structure that is +proposed in that file. + +![Page Structure](./page_v1.png) + +The above image describes the layout of a page. A page has a **fixed size of +16KiB** (16384 byte). + +The **first 6 bytes** are the page header. It has a fixed size and will always +have that 6 byte layout. The **first 4 bytes** represent the page ID. This is +globally unique and is set upon page allocation. A page cannot infer it's own ID +without that header field. The **next 2 bytes** represent the cell count in this +page. This is the amount of slots that occur after the header, and is updated +with every call to `storeCell`. + +After the header, in **4 byte chunks**, slots are defined. A slot points to an absolute offset within the page, and holds a size attribute. The **first 2 bytes** are the offset, the **second 2 bytes** are the size. + +Between slots and data, there is free space. This is the space, where new cells +(slots on the left, and data on the right) will be inserted. +Slots are always **sorted by the key of the cell that they point to**. + +## Cell format \ No newline at end of file diff --git a/doc/page_v1.png b/doc/page_v1.png new file mode 100644 index 0000000000000000000000000000000000000000..c7ccf45da60c0353826b07ad864c60dcba40b279 GIT binary patch literal 45352 zcmeFZ2UJsA*Df3o1Q7*Anjk6`M4E^cr7EIFP(Toabm0h6LPtUiQB)KJEHnWrDxkDT z3oRs3N|Y`wK!6}Ev=AUbfDpJlK|Swz&-ab{k8%HR{CE804GhcPYp*rinscuC%$=vk zhL?Bo?B@Z2K)bH|anS?>+CB#YaYS=(2d)T&yZV4YaL|>D=Wjs!W(E-6Vg`L&L(*VT zksYr^leWX|%*t@{sPyeT#ckvA+?9KGgWy9hIqqG3JnD8bCk~l?42Jo1R=sIi$N%9H zm6|=3pYrPK%YK)bD!OV%#~EOO|9t)?uQms9UvBi z_*f(0U4d?EHAs}?XqCl8MS(xGRe3wKEJ|n3*4+bj-SGkZLg1FVE-x^l(scusH0zZ? zia0fAKTa=D#8!E5Kv^R}GlypN!0s=4$169_mfiBFoO?GeHRTIH>Y5G!RW!i|G9`_L1~he18v zjrVbv4(+qF$%0Sm%~KSPYk*;;gEdXGL7^dQh%MckO1hb5Hu@{i5T_RFzLMCs=I2=Agqm9Y*-G^u_lad2G^09s z;)U*x9kvmzw~m1hRE@X7-CW@Qn#0vu#)TBkff#7?qDDfc0_$mPW$bOdJVA5(2O*L( z^~fXV$yV;(g0fad&mxheut`(AZ4ayKCohN8PRHKvi_BY?%j=I=BrP-06VTKGyZK6I zOrLvkNECgzT5ICVmN=*P{IHhr)tp@kUNv*YE2X{ToqId0E@AoDKIDF1-9tQTVzo?u zEdI6X0R;6-+aT?TR!j!OXp0*)wmt3;q#!skjusnfH|c^S8MV?r)lg&a_jX`Ms5-KI zMGeoVvq)zb+Gb!wj1uXZFKB$_-8BSMoSv#mXq~HDrv8+Bt?xjG9Q&ta&eY92#Z4i- z4cHm?q4|K7G`d()a973^g3@X64c&E}yIUR}#y zB90iaUNG_uH%xH0_Svy|eT4nd^+w0hw9Y5D~`#9W(Rc+ZKn(4j-;4 zu+U20H#z{(K5}1TOFjB}YVqM$915TYl7H+E!XTyVIUG$T(uipVWS*+^pz@Se7;M%Y z0-<+OSD>wXG;Ah+DM0}2qj{!d_D8%2o9ONK<_8p~Nf71js^_xfbBxf2Z--KF2E3#=e+Z|N}2I;!3BMWo$ zm8+U7RmpR(K8-US$V1j&bd|{ZdRtUSPh`BlCuk|LqFJ&|AE+e}^h6Uic{Q~7gy;{- zH9iIPuA@W7#~#v@7W;Kuh26_2ejN)M{shWupuPGIATl#QCvOgR%w{bmCMMkB7eJz3&Ss2lg+8V2<(g!www>K3EI4CUE%$7@zYE{p_u7`g0aq z+q&pYy*7cv*B^Q+SZ-94ZCvhPX{Rz~9Z{6irwqVcSu)-7$k0LMjtAwQ#Ake5uOj1B zf(!H4c`B3Rk#e*$i}N@y@9nv5|UEkSp4H;C5ePzEo9TA@7j8RLM1b^$w3DqApr7O>B!s zsiPM%gM)C$xm*v+CNyHeAD-~#V{$6(dztlE8_h?1!L{;Bo3LC=waZ#q2RCVU`=E~o z&e(I!oxV03!o`}y^KH9Iv@;!-oVA(i_02K7GHG5*ArP&#K@OyEbRb`vrm15pAyagM z8-bj)8bWVEeuryL~-j(-SVnRrP3BLuFr|vTDxCu{m1j7SLtmu=UD#nEq4GP zIYGItx`Yncx{Zw<*7EXdEe|F%eJJNjo!h;2dAIDBshrrso z{-p<2G$C9M0=A4LT7eT9u!9N&DYc1d9`aU<@N?cd#~du<$*`ieF1g!brLateeP}d> zOEql+sU7yYpw7lLcOC024(JyR5WK-k1K;dZ?KU2Mk6i<}0k3S@HMbB^`c~~b-NJ_* zG-E5#$FM5jnjcqoku^_GaU-b`vMkWIMaxu?FYB`TA3S7?F3{Y~fkuWKgyWeG4xgDcJ`4V8uhVQQ6|7l;8xH0m&zam!{V6WI5bIjYMcx$~4ZOpVEoJEO z%sspZ0y)htRLGxUst5bMZhutUPfG&+|DVtQ%@Q!|iX^T5GRCPHAX>xqx#xZvC(9a6 zN@7ian(#5E&ObZ>8T~SIX6BeKc?V9rz;rk%oJ?EO$z&M}=?c>vjis5FumO9A08es5 zpLxkM6}XQ*Z>5DF2aK`qN(HoFnpD{fdwf?0;%CmZN;vf>u+Xx+&~zFyEQ+hyYpOEC-w@>rwXLAy`3g$@z$cLZ$^eqphGU*!90LX_| zR}StzJ*X@+X{5))n;L<^eAU85Ikw;In@T4iXR@QF04!8CEiok*;UyoLj@sWGqDr;PR8O^K=uAyt#AEkZ4B5 zkc}12gd^K;avf%^V#HjC*M%PLTP1PGb3%W~Hxfu@%lsT=8!%r_T02uq7@K0v zX4Sbf)P4Jzq80y3v_i$sqE^0LVdFU#9C^}z?m@Z$_ci|Iz5rnua_%G~f3=*kDbsAW zL`vKqUY4W_F#L|CNfyJN{ImD{&$%ZNYq|?A0Jj(dzh&Xc&yMPtFA^==brN8KoVLZn zYlUqfF0bD8>bV&-cqvG+h{^w{pK8!%Ezk0hTdD=6oBTx)=)+%P08$Hl zN?F!m__fYwKm&itbf#+z`9`57&(<2?EHgic<*ZoaKUi6Cg!Y>QyjNN4QcsD3l_gd) zrTIzgkQY6`y5NH?1emM+X4jtI!Vv!+tLM41JiPezNEY=T7})@4FAlJ#qru@s2-tzjB#MToVK!`cqisHu(pZSZgCVUht@uc*Rei*-?TkzkG>|xj$eI*sH4B z&$#d{h4$$%ZKpd5WCZ^iI)Sh=2O27=VE1@qB@KupPG*h+%C7Mm%*!S9lnH_?%>T@+_I*Ii{6}9z1cu;<)*482q^u6aH}8HNB#w?Up4+J{$|JK>Ix z&HeLJ-~^^A{TpQ<(5V+zyZ(iJpqTcp>t9I!&wwoz{(m6{^C966eI9Da}9hD_8>q*d%~SMyd!BL$hZ%=_{N( z2Hz1Sf@@}z9oa^hYQ#!teQ30DaC|g)g7vc|*m+t7nF1u6ADTy&L9`FRY`490^M_*B z;y@lWJst)RPNogfhQPL_gZ_YX8K*T4AM*cr<>#my|J^78Fe>+dI|?^^$Vu!P2Q;zY zMMmv4>=>31^5O#K%RJAb%%aSk(g_d{iaN)N#(s~=EnXlK%F+)U}PNYfgvYj z5Rk=u!0BCvCQMxrIJ~t`qvR{ORKqgc-U~iV@iojh%rnViQOB+w#(oGOP1AilJ4)=@ zLr{#*oBoZJ`4KR=0~d2J6dgie#;2`bxV&!NSrB;uw*71ZXO_h_l&Qu0HiRdNodKw0 zotFm=c<3eQS^lBw&Wqf0Y6;&U5B)+JOL)X7a&xlkW=PZuh3}x&`uf6HnSg92`^5uB7EeyL+}7X66b9`v4+kN;88g`8XJ7NlcU+2?!U_kh}D&Or<$2 zy&`C!f*5Oll;H~!d3AVMuTTv(8_h@qXF}((Z5G;L;ie97aL~Wa6Ml2LP=}?KZ#`rs z$h)VY7&9YxYS2K`jz*I|_-*maE9DjnkGww)Z&rQb0Oou3b^T;hIs2=QDKJnAEv3n@ zrH;gn2>#Tu=QFefm1s?>q)Uj!m{TeP3$7Vh^8kSgR8^TOGGy&~ngf~#@fM5Z3HYjV z(=Gp2gwnIdp@KYaMnmp02Xsm9v|K=pu6EcUxcT3VO}rR^6R(ILsnG$I^lmrf1CGC6 z0;e3Pv=&DNq-(A#Hw{>X%TpVuy5j%jo--$ms$lG$@JbJ;q=v9cUt-Ln$3d;&74kF? zG;i5k-T>CXe_Y(6zN={1|6YzMBX|+m7Qm)>pc`{OnM{_I{Hi`a|8^{7z^$gtUjT0$ zKIL51vcEMnLOm4_{cgJ{g*5*pUN#BWp;N0k75pLZ_r8?LWPHJ z-`}w)Gc^JB$~2bva&6b}0zL;~c}g7c5b5=3o10C!Bk%DJ{Zbx3p1bR7YT(H`9s$Qt zxMp%?NEYPGM@g($U+SlC1+WO*CA(p*og-rz*YPAXt*l8kANc_)#MY@zt_xvr-zJnp zZog+u2px@iN*|45Sir+Mj{{pMqK1|$BC|V6(ZNk%qyZ^cQe|TR>|xF|L0;NY1`(s^ zKj@z*>~KFAV&|sUBjWI+Xdx)HO7K&P6L+=wDfPqrTrV)zS@^AcjVICU@kk z7}bKldv18O3w%Gl&UBMX_>ONSre8|3?vT=g>C?f@hd0zWJjXVwcg_2m z26noNUBL|fK?<9zd-d%CvWQWz1JmyJ)3W6gJxLWaC4>UZihO5_xK1h!g!;+ zQSFE!k&tSQ&&S8deEe-K-nPfx3qNO5*Z0T6hev7}E$&?clFga#EMk}nypjSj+g(3l zlVWia+gSQ$Od_pD-f2j_7m+l>i!N;MUA=;SG~Rj081Ov4dgCt%FK*dgV7lSlwKcPf zQ)=V>5Lg!s?0N44NZ}w+UT|m<91Z-e35vf%_%G2`tc9sFPYH%C1-g&#Sr0FD2rg?L z!vYk@&`L%WN~NxSmF9$Ih=Z0OKN1W28_U9;X^)(>pHoxu)eizH<5z7DzUp2t?Wi_r zkSVI%?mb@)P32Cq=TBa?wRO3F*Otx3CB}Pbpp?;>Ayu}g{@M=W!YBEq8kBVw%$6Nh zDO*^V%QAO;Oiq=p@)3$NSbwB)B?)PLpnf}ak|%62*jen8uV5%1pH=7pKGAt_1cowl@r%YH$wiU5S&_laylM>&DZJaPh0voI~V$N|rU*(PNhx&rbzDkxXbfCg zD)%8aQhH3#r0|33;Ei(L+b>UZ%Z8x~j0?zgX<=}KQZ1n+XYkJa7(biMSIlMw4bqFV z(?*vmI@RLn6dq5iTz=olR1EJ1;K*MWnaMNrvh0uI^g7D&F=X46n8!GHnieJ1K+%0YR9hnhR2HF;gWq-xGB!Nak4!zGUIT44&TJY&N0|v9E9%9IjPGIj)D_ z@4IiIG(1&=)`d?3GlCYE5$Z>%t3}8GB^o6*ct*bs!y+TTS>yK@MW;lXo9UjNh zJoe}xaK{>9eB;P%7o@`D6#m572c5IY8VMFBbWk3KU}Dr%j5M#5+`-%_=d9($2Qw=> zOT=Zc$rgoA6z?_0o5x-~Z+FXK8%h!UV38Ag&C9l`;L~X{3GH zGx}WNNj7=NWS3k#&SBVZxVl8njU=W2K9{i^5>tjgBs=J^?mq2Ow(rs?MM*-yR>_z1 zwo4V%EpTw5Vy*98FmY1J*GR+gI?qx=Zf{a_KB%hxsZYJpYI%vcLe!4A3xw!>*N{qj zK-i`Aal)FL!qC$yTOHro#bIidclcj4ugpTgCM(u8T&>E`pnqPA-CH*>!tTb~<`&#*ifT}OIxAye%p zZe(K#5z1IoI#AVt&(5u17;&jAlE+ve$k6!Nd|^aFs_LK-&-w!1KB}(Xf(K7_l)30)QBx(=c3(O-~Jf6R``kl}4Ol(!CB6M`(?A+p(2C==v~37^qO3yT1= zM@n@xDLIj{12}4HhQ{o(9dm0J3JTJqkTR3)l*jAJBo1iRV(tPXGuNw6NQWwLwS9!j zxv}7HxmLDLpLU5^ugKBnIMw8*x6-{JajI#9_3DK*$u!)A2lnH z-JQ%c zuIDm3e{5fWrY@nTyM8L92*Re0(U&>U0f2RfiTmXt6myA#)ZE((^pq_9out)yS`T>Vn7*dy4Z5kb!hZu{#C5g>NCt&r3q+gy7EDS&Cv9Zz5SBu;FMMR|S`i&@nyXp*R^VW^ zN-^DrQme1`%GPM9AL+PoFF_85RF8GOCQ-3uDrrHuG6f1*ikE+AD{;( zF{XF%rN=tjmfx~i$jH*F0NhW5Lom$GH!ynr=+lf`)5q+&A!VkbLs0{;9zkeZ0fg{% zVq`2)?rGZ7m9mUihz20zVQ3tr`#pk=tsvx9PT&w@LDD$a`4nb`aRb??wih)1_*V$z zFH!sIb(78Jgl<{MhhF5@mkJzZoBy^g|v-!bqGL-Vdn|91) z-9Qn~(vtS*JIi$6Nuyz>Uut{sd`Nl<<~Rr(P9oPFgnO9Qibx1Q(EJSO1)2MBOPs5( zKGYf2uH3!pa*DPq!Jfqt?rUK+^Np`iovh zEb?5eb!fE6LD{+pwZxL$+dt*Vu7M1Uaf%g=X=H5 z<`qE~z`LUSs_Mg6r{f6@%TZnJ^M-#<;y~vk!^FF`LkmCg@7#0UKw+a!($MPBo}CGW zY63fD3fn{wX7V#x`LV-=h0yj=>|TK`sFm#Qy0ozeu1Y_ik)Q?8H_&RR$GpS{qb|sN zDs}Jth6(*%h|!qC+Q~sYTF80V^5%x@+mP@F*CEPek&dh72lrBA%2wZcopT(GoLDV% zXb^-3s3pR{qhJkE8U%LXLg2yG_~bkMiEEbwXL=kc{jjcXq1C4c=Y!{AR3zaTHj-$1 zC+NK~f}HcMV3{8%>%19OlxEB%fCn=-V?xCQbVf(g3+N|F2+OL-&b6=%v!WM}`s@Aex8;Qa5Rj#g8nQLa2{6ZL+ zN;GCczx*Sc4a~|pBG=pG!>!c*@7+MnlvzJ)L%<7tZcsa&|8giCP<2C-4$-akMo1g= zih-d$zaz1!NCR>oOs4wgt2f{JX?@2pb(BDocaqu=CWm_N`CaVrqG&Rg0(iFm$71)M z6LmlIraC8aU3NAozESun<=c#VSkK`-Zl#bH!k?Y-f|3DPi<<-nTSu3)wdK^x} z(*8Dz98&LdZ(v()aDM@WGX=43{@RfDiLt@~KAf8<(4}bl&y_lbH)6-MbgKC5>7$l$ zm+FMHRvGVxiyIJDk^K46vzz z4&q}Ql&A&eL-v$>D~OMf5$g%T5^19Eakbd9w!tmiN>|E`^3^xw!-icKJXJ!`$JkL5 z3U{~@Flh_b&vxDCgm!RC$J0ee&(O`#Q63haYMpN*U}I2LA$hAjoC3Br^7)ZS@4lD! zMA7rvBZua6j}RyQx}*y!pVaR?lVesBxaatX$X=hogO~HC(T=j(70s=WU0uD#+W`$r+*-j9Y$PJ>(Q6cnID)ZibGgkIBx2A*{Y5zA9`){wgs^^ zT`tcOs4A}6a6-dx{TcD}W!uNB0lPQ!E3LB6^VQ?n&?`Gk<#Nf z&cdGw13uOYpR|P7h|N9&MmrXnIU9%xq1;rhfD}Pa)bAcoK4NoUbuo}#L-rIs}#0|U0@4(Nw+Vg4ULPj~u52ku?9ZiF@A3wsOMv`^!9t7MpT>^$W%d8(MrLk|zw#TS=@v z(&%X{BVMJik|_R@AO+0cwhLJIw=uO>AA!DvyAWw~!N0am9p~ktC#%(j z^!P=jM#iVNLA1l-@}lY)3lf~a<$AV4j#d@(Nrq^O;T^U={XDHpc=G^|LcDsQan)b} zVW8S&7+SU{2)!frzM3J&F@r9T?K}DA(NoUjENlO5_Csf)&ngn-WyV)cL$}LVibau? z53;n!br!IAckg76%-7(DE|5rW#Rv8mk8)plf(N0OCJx(< zd)j&e^QDOV?CL+Z76V|QAj*bWiGruso!`wSk(=CKZ-2he@d$glqSKIGn@~&?;%USu zk0yMbYiYINIXVJ0tiT8%(kITje$uD`#5Nmwb?jcQx!q3)s2!-3Gnr_ac53VUu%%|c ztL~+OYl6ScLQgxa`x58Y=CuRBxHg0i<^u2oX!+!tXo<>6>^6QT0Qkxh0vID;<*bK| zzqh8BUf=R<5&)u>8&O+*%LW4@OUV7&55F~5|8Fatw6F?Xn(9K_i_n%k{o4ur+CKps zycS|W{haBp?tjZi7GasSaF&bMLeAC%Z!9X^h1}TMp3t)D?x7$Cb2XHWE@M4Cq^tWl}UElrOwx=q(ZVIl> zkKkGX9IEPaV9i!@t2lxXP0Gu zb}lesSQmw#&h>Atf4|+wzl{9vt}zuAqrb-L^6$LFgsfOP1=^nA5!_6KkLkLZjb+Ar znBW}K9k81CPf9XTQKl-Gotrv(96CRFXO0Crw*Ziq=_XlyoPv8mKV3ERX1XL3d}M|` zW=kdL#>M~RMd`k*HXTss5{8Q0KZpl#EZD9l$>8L@Jy9R=2;0r|xgRwMQVgCsVYp7fFt8;*@Dq)U1 z@NybVD6ORPbORV0pp9tpq0GRt1L9U@%jo5sUS>o9Hc^EXfETm=VO!ozH-vL(DufvL04_BQp-GYZ8a5$8F7fE+E5&^F6nxnUL8KAeXVPmV^w%(imw7w0@phf za*npFJpJ4hd%mGPuI0wLPL+5sl^eW8^}HLyAxznS3hhR0I5n)9QEAZto)1)eK;F+S zP2mZVN-@LjjHe0Yw8d)D@DdRdN~CKspXkw^29xXaY?&bq4*GiDy^hUB{!Q;0hl8^& zEjNobQXoEWu8&W{cE4cF)* zH)`=I4UExY`aagy5>H*qohN2*K<(IiTAknXAaQBsHzG zitr$|=V2+JRy*b^HSoGyuU`KwUIFa(#Ae3i~0hvuvrKC4sA;$l~Kln61ha zz^fEi$(;FWhn?4di01fjCGCEUZ4mGahW4jey(MYdp&P3|YQobzh-ik{Ds6tMD|d4v zq?1mggu>PqCrJqkwlElDt&64Np^N62q`y>bc?B-KEnMVA@a9H!4x15gll|P@ZLgs* zau_=)e6p$s47R>9kEce9t^j@+`0Q?7>snu80Srv-<{!m2%51*?^m2d&!%~lJ*U%M2 z2sr8a?E=6d=J0SQhRX)fa>Gn80qZlwGup62OVfBv%i`0<4Za`G!&1?ojZraZHh$)C zy71vQOSu8lTN?`N^IRR0K(I}r)bD;gK3g+zyyZrOQl|YWl{-QAd&D$=EkWBGmlm*Z z^Ua3V$(E}TxO#jo?bNdJf}H5q^0+;&N`*)zNAicPrCd^APeBy~>31vd8>FamKuZNZ z?)>y8D?V+ zUw+=%R=+1?zTwpozbxJjCev!EYTtqqAhHQon7;0=RQz-S^2bo@DI-=v9JXC2veSied)-_|cr?0|_|MF5ePxcBm}hn@D0 z*WIBjT;|<9{_vjSM7cb@tB@=V=8ifXG?9njw10_gEE+X`JR=Mv5}>zMUsnR z?}kBli#2;~e$hRHudkhx?to51J@?Ndpqd-x&u(;D0tzU@7_k{p5#nT`COHcaNSM+s#fT`|8XpBbSBK?|CtxuA;A>1#{V2bUOh z|94g(b8NOz8Vt6&Fovgq4NZ3eVHOH;7bUS_1EL%T(IJ)t=DwFezNOY-GSjTi68@p@ zv{g+l{51d`lQCR^GKG_kb~G8VoT=J|spZdrV~`~9?TM;ISX>eT7x zlLtRzPn8!%RQCQ( zsO;r3^ZKz)-{E(4kz_Lyx_s}*57tQ@$1as4;<0>w@Rbl!Ch7k5M7-fAQAT@@x2ij zMP542?$GVtJdmYx@b;A88d+@Qm4*`NKID(7FIM7V zz8eJOz9O;Jy~5jZyQ%K9j34j69E&LRe)?WF`TF(sx-Jq3TQgOh+HMl=T;CzR|1|0N zfPmWaE%f_4vPxYRJ*Bm3*=>bFRzXJFunPd?`@IS@mjhJR^uDXQWQ3Md-VLxC>ex|@ zyXG2BQD_6Fr#-d5U8uU=xbwYCZq~j(72nD4KQ^!Qy(y{vtA1VDtpL#7q;-eU^Vgd+ zER>Em=%%m399OHGkiLV4ephquG|-#hoO?38b|7U9RzYJ`Q zin)3b|2)FCQm|}WHNXY;ui61Vk@?p3%Ey)c9iN^;o@7Nl<^d%gub~+moHyO6gmnR* zOE_;B5L#TINOPIhc-VNk&!|+#*?+j#K0#yW>FOA2`Z-%~-+Y>{hLJ^*($vjHZ}xPd zH$;rSaC7Gr8ijvF6i~B1<7ddoWpFNFP)Xyx9FMO7D zW8W=@2uzPWh<~Vgr=H27y`&D)mf$|?IYys7(ekaY=pB)&dipsDF;M5*HE@+zqbsYN z!OWNq*lc>;A|KaUfcKYqQYxSu<$V<%>TEh4f8JW{Y8iWW<}k73^<(GF%v+Z6?AmWm zmm3zZ?5++!2>PJ68Q9W=$=S_{M?m5NQN}iA$|O7H)eJPn>6?nlTN|7P!Lz#Z>l(xGVJG)1WI@IP9=Q?lznF|A*Lw$W0bBVd$)uF-)IB9kb7L-GN;(# z`1vyjyWRryEg|YrhFfJcDXgh1JDhrG$b%o0!X-BN`*9dm2I!TZkpUGjYI`egn4|)nAzt1v;@N6J)C!IgZ z0nKK<*it%?N%?o`%5x}0jP`bWAYBz-V$6rkGo178xWN-_T4V7gwE&KFfErv1Qh@lZ z`!{SPQqNIIGX$;puS-2d+)nLDunl6~-!=@XfUWQMDP zEjoB*W!}MQWv^{3M*HBNnWh~tz5to<#sZYthT1r?ixWB@=t2-8^e-h1pB+}lgwT;| zKFp@$#@35-@n|WS)+0*?r|siFMSHo2GUkKBS0PoGn`7~>kqR(*8>7Jd0Fi`c-POeT z>iNVFHX9;c5+)TwKeP-EHtOSYZPLFH32YHRiIidlD;H#)Z&;cgx<1NgF3}zP!|b7S zCO6FGhvBM^sgrSBhUG@$JT%}?H_(HmEqvRAsz(KaP7~Iz~X&ACUJmAUe zO%za#N#U{__%B8GNKV=_0)C9ow1NKwA53!H;Jo#Eu*60!CMvG=XyZYpjbv@d4Xe_0 z*|g8#$9nT2&V)w0(-LK_RU=E;{8KteP5#&yqkjD1x0>R<(~+yl zEYb|moXL}YoB5jl62@caO*Ae&^e7~kz7CsIj=OIzF%-9{vZQGoG`K$BG2KK1<3lFw zwYzLc#@Hk-NGT)1!)T-5bWnH)~d}NUVEauvis(ke1>wxJ?(KBqx1~-j|rRQQ`+wh1pWl5AODt%U#n@WBRPgIPmKLR~j1{X|{8J9wh>4#K z0gFjvbDM1DR(J$eV2loyw4Jdr-PgwQojvtT>Em)>c^GrNuugkz4PXIx4=Cr%AFCP+*Cgu9Ze*+Citg# zhgLn0kDf`orFI;vKC0Qagwmy9>FcXLOC%Qqk9!P5ALrMN1;=1 z61ORr(g(%}gwjRF4IfjW0YkbeX`Pm)eF=TA($PAeUfs_H1anUbjf8(%zP&p{)!a4>I1H`4M}Ct|jcwNfZUaw~ zkpCsO7bIS04jpq0AItdeKCJm;;m#S;uL8D_n^~T6e6UKSHs4DVtf11D;UIhU4u?2{ zo8t&+v;tat(;^@SwU}bC9lE+aPO#Wny@1Cw&Jw^I?vlacRP}2MLH%QdSK97<1!$UH zuu&!7?e$h)oRW78u2V@UJ&zxWdg0iRiF<}a3|#iOOLy3;!;JY| zOS*+W20Pv?WI!V{rm1T>?q6MQ*_GDbnneuAsoS7xZ&H6?Hl{HW8??|(+K|9HcM8|LL9$Bh|kppc<={h5OL5BVkotXWY-Ho#kX(W~tnk>iHw*fAle zmQslYIYFyjJ+|^cb!i1%n5hDwaxmuXT%gy) zHb3_@3+P29^_4dy{MmMf4IZxD+!yiqyzFH0B_GKFx@_8fELRB0`(UiYLm%hmehW;b zHIBD&fo8f9z2G(BN)cOqndUsI#B2H5IdQO~Fk}+;GUP|9#Xh(}!mNx&mVc+o<`ll1 zKpSvh9?6;bHt^?2??%vw)1d5k;PoM<{8GdERWy3jbS?{Xub%5SO>&yA{y1Nj>?!D@ zf#2IGdGWjeLA1X7SiCyC)&w z6nN(-@~BHsJdA5z!tnOpK9b9I&hME@3t~Qz4Iu^GWVvgy065m zv1;=`DY5GLDTORDM&!D9(+=|6iE=q2I9qzYc@WU%kohiI+Me6)vfM~n4|eMZ`=#i z@5B$%L-<8F>-~uI9M0=2abPbt5Tt7lk4$w1!#<1)pPZzTAeQS>UUN!|?4!gGN7|AgHs-sEmX&AUs-_=PobKUjX z*Y6C-Q?y-kJzfNdMtB_j(Yo5G>kJx#`VIC3v8#%M24)nHTN^GlOdJ%y$txP@e@{sM79L+Ss&8i6%nZe|>2_Er4LMcx`_BMGa#zyW=HTja0b!@qiCME|-5+y{!NEI{ph+m3zA zda1?Vkx<>gA)$Bwj)d;|I}&>MZ%F8wzaycK{|3Ld@&65e-T!~QXpBQ8>l+q8ZOjv# zYK8R{Ex={EvJGRuQR`VO9YFMIm@w&S3po}}AR-m0OBt*VnprAAS_=fd!(-~N-LZn8 zHcv2%xQa?x0}Mu$eu#3V5%>}=Zveu8tpLDg4S@Fu79u+O?wJL?Q>CY$4D_Jj0PiH+hw0gzAz%85pKjZRIx|QydT+Epz*aLD zfOCJ0;9_2}S1BHmPfH~KIw#USmAQliS z0Q5mW86HC37nnGCq+mFrH{=Ve_OHTgopL}pB+WiE({u^3%s@Fg?HdQYG6z)rFz{l> z;%W#DXEA{Dtqx87R%aj^b1S-hzpK0AK5b&dOI{`~2`;KXmZiOJFBjIJN=!Q4>-CLG zH@fc27`~Yr-dg$+JIecM^BG9!Q@?T#TE;+fA%VZqroMV6s~@F?R%NdMj=H$vyZT;& zt#@uR(Q)kBIEUq)aXr2hK#eHqT;WwxC!y;7>-(ncbBS0~Y5? zv5eOf%~yAHn4}8}`(7r>$j?Qm8t)H4NNxBB%LN+y$g>+|{CH^sRrAjLNTZ_(X1a{a zELrMqLj_;5+{dNZ0r9U~LLE#C&A%i2PA5_(rGB68b<~aprd*;CGmS^RUidFYWj%B= zh*^J2RjzeIbMLtPIH8q=hcfY$&me^C<8B_R$=$Nit5qNEwRx?-Dc@N&WKJ+X$us9Y z`DNy5qhX7LEAS@VC7WOc=H_JA;Ht<6UY8xVo7&5+3qD!O-`7>U41mReR|B3?Ln{Jc zto1Xk5iRfLrjlfPyDQg!1vqPcI9PxB!R0o=x(ggZ&asbl*Va0dSAb{@yYm^?->k=h z22JU;bKh}KjTP3wUWWq)F(<<+QK5p`KJsPPpW=V^3vfV%vt=h{WPnHDBkE_cRii)C zd<_5*9b}pi@adMqCjZR80CuIY1OQwHE@Rg?pwyqm8X!1wK$ExDHctb2Y@mN-1t}np zpI?qNEieS2`oE%|zz_r|>;lu+g|MSmf1Z^9aETq&3`hsKo@WGC4gGoiYz4sFvY%hZ zw6tSx{AVc#fN^%rsQ|VE4F9CXHn{#D;{P8=p6~~p22|{ePh4U`sdHnE3E@B$U2D6m zq76EDHT#wOTHt=5cO-CSeU<7kw+z%$zO_oTtjgS0w4`ZrI^_$1vKb1JM6R&JV7|0oH{ssZx$hevcBiav%@&RK|V^ohQuKtiz&sAwJbv zRkbP7{sg9o^}S1=PiRG9f1tj$=9SfF!N^tCtwNtdpjKFzAO8@f@fG7Kwb}Ug^Dijw z-gqky4GD#aJn0N;4wreQ`QklUV_=CoWF5Z!iT!a$d|m6s^QxI55qPl})|VN9)S@Qn z7d@P=QcgSyFj#hfi$6nsORWkak{z`Ric&qXvPgcH7Ox{1SwIdCW1fJV7dTo+4N_!CrYw1Co1bb-U zYhT~{4yH5k<<{v&{-u;I&9rue05drUX1!OeuzRpocDIC6UYx^hQ6kQ!QR4xQd*8eC zt=(3lu%;I->mVAEW3lO?8hWd7`Uq{hfc7`sS4^Zf`gsiVYD9O^S*IML?u?6v0AORFtZ4DAGF#5Qq(wYN3N5ARvMu zy(EMj5l~v_p$Eax0)!qQlyB{z9)0c|-#5m+&-ae;zVG#qGY&g@uQumeYpywezq#t` zoo`N9m>-8uJpdtlOAC*mcqXSo^mt)B`y7Od}x`G`E!QfC;_dUqqBJNGlhU%BJZn_1#0iVtKd?Mxh!byRg(< z0dT;M5c_5fzg}+yxY}fIZO9G;;|}9I43KPpA4lY|w$#44uZ{9F+%cLRw0AOlDWO#V znC6b@YJX4HJwD6(o)Ee=Dn3jQIKeR*xG%uJyK<0GCh|4tXs6e$Rp&5}`R>4xJA{d{ z%ooJs8>r&H;QI2@Azh-5fr)p_*nzRWS))iG1dogQE)vJ(VZ1!VcW6LS_(GmiF>R!z zi`l4V=XAn^DIt8?VSHb#xq^`I@U5SqyEMpr9B5(05WUt*YPP`avSDHB!Tqh_Ik%uFDKHo&? z;nbkQas1Q*n4{qQ)w*jYq9q~5_&d8O2_n-a0ZhQbgOU04V9zE7!!7~p!g@*iR(PQ< zd2EWGN)W1<>-(19WOKeJ>04h5L%+QRzk>0o=l8>#h(S~kWU@X`o?2$;&+Ide3eF0l z(mvW{z;f98*NSjMdw7?o`4)V?!I@-F|{s8tvR$u_Lm!>liMXcBS!bDaGCoZa0f zH`bSk_3+gU;E|lM{5GxqW1vt$0-JSJMm7aIE1o+X=RHu;<&K~57QFCTj~R}5?kV&W zoY)GX1qZ@tA3?VzyiFb(Xn+s`Mn7T_;rkhuQ`f$po1lN8%IYT!@@&Yjju%()_qA?O z&67RfGneq(sg}Ve113uStAc+n5lzpG12hCR104r|0EBTusyRutM#WQDzs9w1{tt@) zHD~{$DswP^&*z%9q&?!XjaOm${a}(pdH`c(f-7cwv|2^+0;19plTshj!ZYAu$rZ-4 zuJX!$If4`ab-Drn1L>s-N?IGtMo)|}`n}bgO3_y$W+pT>?E^VgelLBuD%h6x5Bz-scsSE~XvP~^W%}ZL z8_!`myE#D?-rEfQbAuxZe30}Y^9l9_w^jbW?AS{m<<#BW3KYxsgy-#{`~1#?{8HbzPTIy z+v{o%lc!MjiTelCtQX%em0IoRFLq4q3+eTg-nBZ#^PT?jJGOzptJj~hvP@s#8gviY zzPKmmiHGGeA>Tf(HXPpx#O;v;Sd1=F1i;B*a@^m!mA-em)X6DI#418y=J~q?$sBGTjMz8%j zSR1%ynjrHFrg9Xw#d_eMirzff!b^YY7|2Gl?Z5Pa(^&Q-fC(*ujHD`CnB@T8ho9mB zg5(V`6`|xcTfv$LQ!$WT>t`cSD@*)C?Kv~^yNhHWmRi5<1)I62pS`792plQ8sO*j2 z4^XVrz@pr2&Ksq5GCZFq@Ws33iO!&GLvt*`xL#*WQ0T)CnI9EzIy#YF~FtZe6!)C_|;H0U-0uu2ctue5aSiupJ zR`)cAG8}-wQ+u|+a3rt;{rm!Bang?{?8;wy68$X6_^}IwO@OSZO2}TSURs_TfsUkn ze#7hQ>)${YCOy*}5Q>LIokuY@;UJ;3)pUVWMm6R#QqIE$Oh%q)<=8{H6C^Du04lZF zDBM2bOWdd**|K%n4zS7t!O5CjMnE#>8C|NQJ!v5!9DKF$nP~0Ybed`qZ|_1OM{I5nSe0FMw$F-n{@& z$n!F`!jXNN=K4(TV`R2KU@ED2?4bvwVMa{<=mI|oUhmV|4?_hV{v|1MXp+NF7ylU{ z$h3n&n!l-#@51QJ2pCZ2pE|V*a*4oxsp%UfXmf_aF8|YmGtdbP_DkDQE4Lx56?Pcd zr9Whu3mKok&4Ka%vl`M^4Ujtcqnfn!HCiP=Ve>12v%vIJAG+p6MJN=MUL%Wvp!D=K zNO~3ksgMtCQ41Onj(>)k<8S@PX3D;+)~7!P-4**(ynWE@kPu7o*CC+UtW+vB3CJ;;21{VLnC zuhsc$)M`A2i^jA3GFcwdFpXatwp#hBkDn#5nL*!|zjXUTb+3lVXDUFc-XGUQcVB;!fW8S%OrA!v}M&8t^KGW*pn!q~`*t(7RmlsSZ ziB?ymnPIrap9>mn&Fq%R8Rcc0;(o=^qXj`R&c!Fa8l-6>{07$#{9bHz&{@&Q*TFG} zN7TBW*ZS6k1up^2L~O(#h0UeXzNqO9IY`?_nlO2=F{!I~Ah+c#+$Svn|ES{6X^nHf zDCnujrEj%fHYnMyvO+g zSQr3V^`E#t>1Uq*!~&qhHUEPVL3ci-yx1=YZEpqY+kYpL_-}vkqY~)HroBggCz5a< zW%`{+qWT!vOLQMC^se>Oy}uJlSlAeUvNiWS~L5dQ1ZhonhbeG z(e|5FgqfFiZA>LdvwzpxGY!(?-Qoof#v5Z0TsJ;rl%*W#Z7#lbut;k(%!K0YOe*Py zZ-8*?VcZZ3CK|#)%aJ@ZpmrGY7zEH=d>>r$CV6Jo=zH7z*efSGUt;(7^4HM$-e-9> z^{c(m3giqOpCZ7Oll&HJh>1ao<$ae+=&t8n6;xDpANI2KAkzx)fe9u_?&n!CXZ(rD#dc&{?rkOw`2f`U;VZoyzwp1vAy z-$B&R3<0Yav|=S?2$e!3EYhVbdh(W`TorQmsDi>ZRv{6LpNIOrO)w!FAehU*y}qBN zd|j74!6E8Qx>?^ORCDB>NqxQTvieu307L0i=>tMCyy&R?nKEkDoaHGQ+mH^>p4q3+4@q zdF-EhZS~VNYBKuTA?93>XWMbhZY`F{mU`T&2WAR7^Vx$|W<5hiqW|g_efVDRfEm|U zt_LQ*kNSXD^hxEyx-DC$B5|X4^pnu1;^R5Vfu95)mcq-;cYeIpp%2-IB?m10PB4~c zh!3>XYBd`M_G`~qw0zlubz^{o^&(m8PT#2y!v~bnZx8T=VmQ~M=eMH8K>iWX@W2r}vV zGCf-gZJgVoU>WB?d*CC73k@p9ASYhr2b(^C#l$f1Hz$)B68Nw=lpIP=M89rT6^en?=av0m(lV z{O{*ZF54o2p*=)s1<<{rAu34Yv+xbjSn?BoIGwZ4SuX>#{*ImJe4-M7J$Jskx|Xr( zjsfeSPI>02vl?X_M3BOe?~eaKi$D`&-$3esP~ek}z$CDUDzGbBp-mDQcE&Q;gv--G~zVqKJYinymL!o2sn^8Xu3-AYs9oYIQzRO1A zN|LcGdWuuvpr7&97rOuEq;^~TzZyvd$wt+vmD^C6`F;}JHbs7&_>noVYJ5W zX`a0}CbTC~#HVBDr2i+3Pbo0gz{j|ida^pu*T;DxfjVA_`m z4ixnY$77v%h^S7fsGhulEK8@E^u_gIVgPLhaO6N7z9;%5jqzK1{op|E^T6jq{H{?| zM&`?n8T!|tQv{JA)a2m+R32_;l__3iO(**d%f$HmB^ zk+W^U&Ksrfz84AVoV8oO!VCE_z}I)UvhnP zYzi+u-B-A8fK+8#iY=o{|Bbz~vj>w`Ga<&yJ8JvH(e1%qNWlHeQSvoEqC2^{i9+2U z5_OQPewA^uDe_`-)D#7h5)KilI|V2Ueq3xTA??=N-STNQ>AiI~;$(ZXc)$)?w{G+< zm2MA#V1f0-BW^I3QER2T8gyF}cr}|knI*w95t?gKB;96#a#-iG+JI+9iz63=>BL+> zLsl{8Zv)RtCh$)TAR-v#Kt)w6y#p>Br>?|Zg&-dT-9(0kAa#LPV<1nN!)ENKA<9jA z1NL*Ag`Y@U3w@G7zKq55EdT#DP@|N6fZ@y>y&4j7V|~p|iNi*c&RRdSOheNo9U5hE zK^Ha^5S%Y{NgxLkz40=Mo#l<60s?)fj8TGxZgdGPxEYKBqRC{{XPiU*QKYOg^s3G6;->xBoz*Cg&VB|T!wuiXGU{1={u?;C2u$aaSYsRxvH@V36Sw) zUAW#cZaPV`F5a|~C@uiQSx-Y2*URr?%cPVmtUvZ?zuUsJzDXf+~a? zYlD!vabg;|wx(HJ18U}H5E`a%?Jtb%3?r12OHSTB5SSb4L%=>`%wr$?z%@`MmCYNQ)pa*; zP)U){RDY>^dG8LKMIHGS1{)aTG2?j6+e30Fxg$%G+L5K4JbW9yetT|#R2QF-18bz^ z6XMrz#=jae+CP#oig4>wIwtJ*y0Hh5GsLr^Puum~&u}1l6E_j5ePcPGN6<3uz$su# zr?vo7TD|gEV3t^wsU1PKE>0XV({p_NgPlbC;@`0#L$FG=MZ-3|)wTO)UlV~)MD%@U z8lqj{Z9QxglYtR?wD0(R#}`(?M3wa4<-m#-x->Ry+Gbo_ZL;s4vX#?@{7u!?$|v&OC373*pqumb9pw6TXTeZWFM;T zm8nmD50U6_x$QbBVxFXS)3$>jv4*aKQ4TO1hmnzsEfwP+5VXlH@3BGYOK~O3ikb6H zS!P3g@hkqWk<9kZ60z4gtMS!J&$fL5IBA7jJ~{_tXG{WLh*~r+&@NMjPi^^fe9G!& zB#Tzh{tbC-B%kZmN2d-p0|LhFG4=K_F&ejq*sP^VH&LuCR%>Qq3A?(jbB0*(7lEDf z4K4x{rertbwJy3TL&Hcdk)TgH7ppfO_KtaW<-O7AEZex@4t{fTX2I%hgKWi|oY_0h z<|d}^J!e-=n^2A{+$7+uDHo7ioe~nu=joVIUZQPS!xz}Us3mFmEM@*jf zo$-XPsthLf;OoeYHA5~JUIE-8J=4@0J};!+_-pt?-d8zE9N7!x;MGBH9mvHabUG8tokaVft)^()ztoUMO6V=ULA3^diWo)xNQH}aLz3SKC(?I7CwYw057;kH&D$@eP|o%{!*NZcp)FV^1kS=P9Q(*TdNQ1+3oro+13mwyotu2tRAbh-L2Q>j(iwVvO+w? zg6Iwl&2mlcRYuO=d4HjzOz-t^I7Bq07(6;v@u@uL#VclHfJD*2#5v-p|7jBveP}}uM6mD2z zxxt?#7QC+T=boD#$T0se(I>P`!8B3z|9m^gOpRV$sY>GGiMRDG&4Ko z!40WGtlY6_CY)7{`P45Zfw)&zr(;WX zTFCfS$u@3H*TmwE={sdd-v0f)1mI3v^195&GfO^Z}WI+0kRvGHN+gzw@5wQL$(?XhBje}rAVHn|Q*vZT3G_g@G3;@VfD zy`#Q|l!&pU`S=?t)%y2eXGzN*5u${oY}USTGq&O2`J1N5gA3aB&l<2W$Imi^cf5#rGO}E4h3TeSYB?ApQUBK-J8=!xER|# zFlxIF_12yQ%G^pqr1@^(NL>y#k>&$UhJ<%HgR91wIkiD=SH{CKI4(y>-YCsl z+_!K6TW&7${JJRv=&kb~l(edlplv$+$KFex76InQoktb|B931GoQ<?!i1hp83wRY&? zfz5y{Wa@3i+fUd_?pc>*Nz1!(5KpwwRNV#iQu}0uvcsKM@jZ&uwXxP$6>s_aE#F4= z6kJ5!G|6Jm6bk;ZrA;5EVH%5E8rW}lY3E=B-3JlM-UGW?FUmdnjNoN7px8;M%D z1Tr}%6ld)K%V$k?POW_{wN4i)nbyy>LS1FdOR`VgC&CECcIr>DiN9dhVq4uUd2?k^ zl?%2AF0kc3d&Cs`&hD}>(2$s_TP831I863zO(C^tx^inG^BHJZ{YPJ&8BnAMjuAKE zQkgWV(OqM;enP65NoCoT3t3_dHJo~uxe-Le1dTavOTjlr$=F)*_wDTM&@kBF`-42C zu-{_oV8b&ptu*WHB&`rm=Li>l0dP%inuKeu{h8vNRcaC`q*}^*gN|CEgm-X1F=j#Myi{fMyfylKL>odvTbGC(GAFafq~! zryJwTL(@kM3*S%*qefb0Kjn*97)uN@`P^OJ{9SvglNmnn>} zZiQR0T;Da5TwO=b%x>R1>vga3DvPmuZRg5+g{Vk(gAe^z-0kgXx-565DCgb?MO97o zOyyy@4cHjKqN`o_r<%oNr(M|@oxJ0kh+4Qb(nkLnD1JuV;_ShOu%%K?L~AMqaba1; zHODu!Ca1g+zMA)}-P06nJsRGVXkz$*>z35>mLX3`hPsiEns@e|lgr8X_1CejMbL1U zE!@;+X_d5f6vPF^zFCaUz@8>Q0e2BwRHBRgY#N#hwPp%e$+^N7uscfH_B8bP>!g+U zQhGT){n^Mn-dKb3G%VwES7za#b9;By%?9Hm#`@+Nm|0nAkV0NvQ+CTosX_0R?3GX< zbREu4+I2ZO`-GlS@|9PjeRpgdKUWZnGCHD#avqBU3S_y^n?oxJx%d&sen?5xWTbBB zho{7kMF1Aj%j_V(TDfG$vH5JTd)k7MY1qK-`=kBU8XfwDL0^G+0d{-X4fz zV>NTvX5eGv({Rj^fsI?0Un|({DJBZq-BB}TEk*4Q@$>K*$_$2NJ{)#XU{Pn%`{>gK z>QK`haNyXw=CDJX-pTQBxBdpJ{?wJj?OdvLH_~D=a@GYC#rfph#WS$ndUdH@M{;s( z7HZ;GPPeOrU6{D_?lsC=Fas&yY4CU0_llJ8mD9+am7PGh+5edOYB<1ofB}`^?%u?o z(o+cm>N2Po2J>08vHoMGGVySL$3R|5f5?l?$p}0Kh5trp2EM!qX$!}8D8TQ?&MzIP zBmD+%S_96W8|w?J!}HGjX&?y>s0moD;GVVl{HXQzrC+6%hA_de$y~e!LjGZCI}-%! zli|-UoG8k5dH7lZo=(?$Ut^AXZ|ZJbEX2 zfD43Eop0XIpGm*Q3U)UT;aR8G_y-*+Y0sCtzz1=5=*q?OH$FkhZ^HCfJ{N3AX5E>e zKd73^%e{aLXMF*_J|lWTZY>k>!~O(l+t5|cwS@BDn5FdrJZXU)X@}~Y|1a9lhu11PcUIjSYPZn! z%UWaJ)hwevI!1cN#63k7y}BsE^7#La*R#+Bu#m~qLOFwsbTDMWB)A7F4Eeew%J@u} zti@(*xO!{9ak1z9GfZ9vWB8xL5UTu4cPi~%_Hew4AzPhoT456d@D{;fBSIHSM;F2c zE>x5z?m9G9_a|pZ@j;R0)g#PyQIMArH}c7!_B&-~!0mrKf7HL*Vde~g`oD8=?#2HN z$huHQL90K)*Weue!?TC~hv!2_!u{~${SK)6hv~yD`+!kKWMB=zg$hj2w!fTt54OQ8 zfQOHzed?DZP~{CnKLsFmR149+oQQj^QB-*l$=03B{^fvV;-(iTapQlF;?9u1<~Rc& z?vG~k+(O&_a37K^H_+bwx1URwIxmBG&NzTvrQ7WL`ChpP!G<2Qy)B2+7^+&gZP*59 zqo-H=1_84A(4K`CGF(4X{QxZf8*n0iQv5MkAFU^|+vz+Z=PVEt2d=iFi2b7gG|vZO zJ_@oQ4~avoHRm>ULHIK>Vp>-qlw+@#9Q0f&@x?QhN}8*&I~h#49T!iwXYR-5+&V;069EyAKsdWm3Kq@%xytUdv6QNF(6qkA zDY-6izYagg3-fb+6RC4nKV{Z_m-_1AeB+ z{6;2td?k(dZitt#NOI5$I!7ujbvnSt6*&3Rv-=9U>G@T)*#<~M?dyMe=d|8lk~lAukk~>pV~pgM z>~~gr_UOnzdwglf^x&3zWsDx`Z>VN|jMzTqDXrrJ=zc$8M)U|R@euEYs(_l~R-dbw zH>~IP7|+Bxs`_kXObs2tmy>TViZ5#Q)`&iA=&paWgzd%}isOPth#wU8TTP;lm4`{l z6>(U9rWJd{Kdp$Hr*dpe#x5VC&L}L<#Gi!OVwDNIr&~&GcjZ6Z3dbFPha7$?CnZu8 zR3@!Xc;Y#OER6MOKndlf5b}n}LEY7e?RUL?sZ;?>^qG)I$~Oh8qOaEl8VU?;x$2gl zb89{9drFeDZCEl7+@{9GH{WpX8Aa4oV*-z@uFQ?=N5D#f7jWIuBT}NvNItOeCD+u{XOAl% zV>DPrHs=qIBCbvi#m+y_v3<^}8S?4vvBiKV$nXq~?6Xqk*X=zT?rgRmj1 zAVZPv&-${sK}f$IlR<%LRrV>rGBLEd{T4}} zuCySlW=~u)xRp!sHJU+R>ERQIRT=ZkK_sd?sQ#FDPlm*2JG1^Zs_P7~!rW9(ZgH4G zU2D#QiK8$5eB7o}s!GyQ{c}M1fYsVbfSF#ph6fhDR@%UL_f z{ux`2c+v6l#?`?$VmYGiM`l)LyOYQY(!&exYj$>^;LgR> z_&g0_jYG^yBAtXDmCZYn0-8HW`TAfgM%3*VJ~$N&(m=7!phLS}6|&B&Doazxza^2h z%@ad*Tpm>HbSLKZJx$dM@GUAbtvS=qEwHX_Es8U=+0mEFV8Y{9-J9;3K^^JbW&m!Ch#fQNt6L=HQKW)OHUAIHRZo1MQYA!ZRhtl9pvS`$ohY1y4eCe2_|V z$0%CnTpwOenc`lK&897On3y7*iO~d*lD(;Dq+*0!w37+qt*%aiP(>Wev`AI&@iS2` zi4~_ELvz>$+9UUAiQly0@2C^93j=>Le(aWtAWAPRgxs&yWVvk9lkjOWHz}A^jm*rw zOgp_!-S>ARxHpL1A+;;5km^A9WoB-@JEj=#sJCpABPoG4F^zp9?S_fw9%wI8Z1DY{ zNW5NSz`%!7rObF|a2WtcXnauZ5E8rd!X$Ba*GMSaK-Cb{zBRj~U4=*{@1&HCFUmNA zv?AbZMbi)3ci#I0OKUR{4qV|4OmGX-^P+D}F>p{)*F<&$>KW^|W#;*8+M`MKGLBnZ zmL1ulQ#K~ywJ{ORS$KXSVS#W84c;ruk|NfV5UI^Q=fj9`kGOYZHFS@$rL!PT2=;!UFO7e9iAL^ytOZO=#l?3+6V$*xh| z6nSFANFTNvVZFSP%7}r7EX$@llAfG{3z?cCuoJs#XV-3I&eUMBt42iXvfzd>t$v5y zTK@Z6DVt1kM+;#95%Q;In_t1?QaLAVy`u1v2lfv0*V_W;=FYdXCu*7$2Ww;G3;liV zG$sO)QsX>!)jM6K1_%4`8*qygJ?gBB*sOy~(=}K72OW>}>!^zVMT<4J*zYi^c=kXR za3GNyDqXrhTu0pWUG~)LS9Nz#^%%V!NoieOUI`6bE=n}x(qiCqI^{pLD_-PQ(9mM^ zN-o7mjKp)RBVP61!1Y2Tu{1-0Kw!jlX2a{3ih*C(ufH41Nn07DnV|=lSCoXpd-i-J z;otw*-5D_vAl{#v<|!dx-5aljJ&7$QZ}Bm0mER~9(+W*u9R@CIJ-&2IMeTL9<))XK zwr3dJ$M^%akH?ym^4_aH@FLI(a%69<_vmuo-sX#>+9-TqlOQ?Q5}dvek?WYkAxh_Q zi5i2InZ^Z367P-#Om&sjS&l9kyR~~J#Ckd3AZFzPS!66MUKjryP~`XZ_=T_;I84~@ zR{N%vwRhk{Rw}q#Dtl>~56VU+8^Mgwqa2FBt8mSE)zRD|@WmGeDvTH%1=|F3Z20E5 zTa<)r#27KykwXSA=UbEy6dvvmYla1HBkB(2cGykWS~iqkSM|X1O68~Pd&KJ3JdL_Y8@GRytRUVgHg%D^M4m5WpR@oTe(@S{q=$XP z7km=%U+ON&&N*k`gl$sh<>P04V1_GM?>CJ+kWcKqu7?QibdltzVi$j1iSxC%%$)mY z)4vf3oOh1m;<`0CX=@fL(h?wF(9yG;;*YMQJbfRi?_98X;3ZK|ja5!Jd7(VdXLK4D zx8kWVOf#?0T=td$jL1n&dGIc?lYw)t8k25es+27iPL?CGG~D2S&ss0>q=DeKOLR*i}<*4^)PNrRtWYZRw@P!5({ z6+Uc?>M6l9u1we}JQMR)z2I`A zjyPD1kKkwzb+d|#0z(5!j+Bb zyZBssp|qKMpwPE}o_(Oos_lp$YXG=%2bk|IS`pWRDxDpsl$t-}IF4)F*XUq&(jvSs+q5SEBe}yIB^^Ef6uSV+nb3?b zGaWd^h&iVlLd?*z_o~H_3`7IHUg)Ty-_B9k)dwQ%#oa4#dc9k6cHbDv&*eE-`WIr+ z6ICaau4j7A!mTWi4en~XJJ6mlZmLY-1fNTzv33M6IY=rCRZ=if z##jm!&We!$nc6Zc2(uI=)+T{orGiSAseHTV%%iSJ*6T)Kvnl&tgFZJuKFJ5yz!#^7 z1}x_dD&(=hEsPf^#$sI_54IsnVXP&owl|G8t#?MuE?O;2^x?;nFMMf&R^sGg8OlT$Ta8jCy;jekV!c@cTy2;*anz?*o>cM2JR&u4fp@B>Dtw$oO$$0GaH&~upM zR9vGT{@?)89_3l@eGMD6Ol-a}(%Hh6Od7dJR(ud?&~itAy-)K=lwP7J$yWXt&d`OD zA{kr)kR$}V@!t8)q1m<>5oM7=A6a`FsmFR-;Tl+7DNhAEMMuG&X^>Yy8J{3`Qncdf zP{}qU+V3JqS_C*Cnh$Ldxls`*l-m2_!*b>6X@arc)!o*yH+pTSbP*#OcErwk6^oOX zy6)%}HFQ2LIroK!{5F~#e43h++x#e(*WzSN@BG7J__g_)ydfDghO|m=ll4iFF&^7K zLY;lqnlOLs9k~Th$wg1Sp0GLmhf;4&5?<)hl0iE8hOR+RK(!oqN=Tizc?b5nQv*#N zR~k|Ca;f*VT(uTbJ!(WxkD#$2oq+8nv;F*3Pukl`WBQQ_&Kb(29#N)dmJlED;fA<|3)Cq#`g zV76LCPjF_sUyfuT^sbT9>XAurI!p>1%T{x{=ikmy&z`ZGnoHT;ZOdT1F3q!HTBLWz z+zzSt8Uqe7q*{E}l2Isrs+;5Npty3VA&EoI=16<*)4A1|T+yn+_>IV#QS)q;+9CLI zYUFMGE|Ws{$dUEJxzKyEq}gGtkG_0`DBeA-2yK+ubsVy#0nW-PKHXr1x?}uGE0oSN zAR|PuPV{|%_xEmufq`Y&e!{3M^!g+E_m_7{T2*3AdLm z^a!R_7~LwJEPUZb^0Kw(BENS}ZtJJqCZ;}O0vXYDv)Qk*(AK7<#fn%9dO32_*wuB` z%ELm=*mfx!wy(4MO9>Vqcg)XwRZtWU*CM$LA}kD9@Vx7MwQRc!2b|su*M)X>&zt39 z%^!jT6%L=&+le5zm0HGf6-sjqw1-jTFQ<{$Q}dz|=I_#rJJfGXq_k^7c@;h_P` z>t~>LQ*LiZgM`*H ztl?`HYnBJ<6P2zs<4KCygraS^pSIFDyR5WNa~*50&oV?0ytTV=5!+}W-*#bA;@tQ> zhn?BEqB{x{(%q3!++u)^%&dMhY^?O)+i$(6 zE@<)@sY+`2{NVITnw*}05s!Lh)vV3mBWD{d6r}lbgvI?#{TDObm%=Ay3bF7rkz94$ zy8s_HYCq-gX}(ASioahCO^Rw#H6%>4!Zwa-@+^X5d z78gK}E()3PR4A}m=aQ28(A7p%#bzZTZKw-e-Oik+&K6?7m6B`$JTG3`S34?9;ZS4U zU_N|2=yc&wW3ScX$Hr9|sd!>xBYZFUOPKu3MgX7CAUq<`_7Y2uw@kXDtY;W0yr%Tt zIHh}$6(6z@*`uVnE1?4GBD=i2*1O{G8gLH?pli=zpab9nWZQY4K^T6GLQgvOhGEeY zU<+$qJ)unrrO}ljgX^&YF4WD9&~3LjU+;zP5?vw?*B6U7n5<6jjM&J}I?@&gm}3vNbrKn1sEnVz z@|Ha(h-iMi20mZ=#_G33Fx^-oJW|hmUAwnCB8A*|ZN|$yyq~zSPPKu9JQ)AsbD48I zh^EJxh09rez22M*22A(a-n}+_Y!$V0au{TenB2yw0Q`t+fIJ&5w>eZxJNcRYkMPH4 z?{W_Nl=dQ0HQKZG1%hpN^_Sw3#J-o$g%kLD_I)$Bq2wY` z|CE|}EI3JQm%yz94{-2nG}#RJG<}|^rZ-Cu-kF_&KoKy`#OHT zjD|a`HlF%ad0YLl9~+|xbW7LaCKwI)fpvlO3W4;Nb_Qqz>D4RsAv=bw%6XTt8&=;9 z9vdNe<%EZir%{4c#L^nLfY?%Pg=Yd{h2n>AtQEIDotkRrZB_UR|-jwbdufO~_9 z`>r2=ACF8y#3dZ}8M;&nj%e2T1Zq^5%_;V6aL?F@w7#$#b-uaXp>@$Ft|b-x|I+Y*LNSbY`;GW~n{l@YwENXe0nr^~VR&?o-(i zo_D%^I1R|=1jyp|V?sfy2f(_h94YCLv@vnOxnX31<`PZ|h!aBMX8qlB8BR^R4+1(r ze);h0msXIzA%Vl%qooq*@${_r&?KMcWn*aRp>c9TOIn6(d%Kur^ zEM^1c-ErWbak+)x9G8y2&>N^8o1Kkt*v)^gda{{(0%N^<(<2sa#?Wn0;q?E|QxDt@ z3pt^JO5c0}J*KCu_~S`9*uR0#187&4$xALU4^U*VgMdFz`}lOcvIH@#Y;Z{vcs!*z zA<9RT0AXIIt@PNI8|63}B@uE0L_h!w9lY4*jTbqrm9hf{o7YAG?NcEXRc-yAc>;uM z<>*zsvBgIOeUWbfggH0O%{TgmI9`Osf9si63voP8YX%p^cx`@SSU4vBdzr;H ztyZz>AeG(dYx)NwAzF#|xwQZf&=a`oVX!g@No3l639u?908Ij~wHU^-PHWOz1xSRT zRi0A!l`uhG%{RdRgM=2~9k-EFg`vJ13-)C*k0P`!D7t!TDdgAWwx)WN2Z)Sn;FDI_R|Gq06eB>;t2C2X@&^Qb+$ zel!edI(W`8)$;9|aa;Zg(0e(0=9@RW!p2Hq6yyD%*w^%sWYZIZn-PkmP_fdZ>I-f= z2c=vMwEt+MG5_L7Ue+sm)7s@dR_gV1Qe+={06OrSZ7`H9LT0{9x%lAl2kK_)K3=WS zPDBLWmyk|e2(P}x2r$a9(W>GXuHD6JvK00GAEti1Czl?rH23_zL_pKjhkK<^Prbw* zt5)d56RTyXUFGl+^92=`=mT(T>_)3qk#y#A(Ufw8XT^1>bjmSt;U$(R9ae|^ z?LBR+6fWXv-+pC}Q5O*VfX$oTM@11#Tc~I8$T^WYKDy{`r|M}Z#^|IV0-GRxQV9d; z!fONP7PimzN>dalo5Ol@<#Iafw6AjIIeIyR;S44SA|^W2B-*o#N*6e8T;Xjiv;D|v zNJ-0O^ilfS#{A+Yc9PUzp~oN#&v9#TfzB zxu5F+0q+eKaG*^ZAp5ds2Dk?XbW0Q*xDz*-3oVB9tX?3 z_28@W&G9QM07!)@=?6Mo^qgX&Rj0OK2=i+R9v9Gam>;jWS)-!l)u0={U~f2TnBaa_ zi0SXbnx?e%64rjGXR01ZQLr@Zq)D5Lgz7R%Pi5$L9P_pY9Ti#M6u@Xp#uiNP%OB13 zcN(QiAeGV4sb-m_*!iF3xNd`&9;x@%f|OuoQ8*;PL+9(-M90#PB5Hwy$*7ksFtt1r z3+gHN>h<#=+U^|KM*7xd>!OS<{5hy|2N_Hpw$Hi2wh$(qH(7-OV;2@$1O^uZ@onFG(S-d1j$(ck{PxnFsa~B? z%0;%Fi{DBeS$ej^Ox-8I6^meHKMu@?Wnqw}f9#BB1b13h)>gPA$-I(6n7T%dk7R}u zPnbLcBb}qP2rVJ7Q84TSt9w9RIUD_Do}5$sht`p%h$%fF87Jf4#Y9XJn2z)fC4`5u zP}4Kr|H9tr0BK&|jQrkO%5(1lHG6iY3d^sG)BUIM90LqZbGiETwK59m(n#?!99n~T z3D+)nXehTNz?jLtr*;J1aQiSd!8bvDPOM%2Kw{p=hzAfqZYAt@A^I59_g##@^ZSk< zHu9-!c#}&?HMX@iQe&4?WX{;$o-!oee$E^yI(CV96dJhO8^2ep?{PHrril7yN0)dKCsEfQ&p~y8&kl z7z|ekOs35`z{YM5J%YJG=_z1FhyFXC7!`A9!Eh{_TOrheh3*OE9_d&{)S=D&)D{?R zccQ||Wx(kF47TmnP4LI{r@Ffec*Hh1V_>Me@tiv!tN1PNtsisBkieZ&KO z*`q_LuZp}i)ORqRnUQlj({-O4=J1$4Gfxy=^JE*XdRJ<{h*{bQxWP&@yVz$uR5f5u znxVCg+^|VoX0rB-N~_rGlKu=<0+_>K3@jt`VhXi`?!-CFmCl%bELx@Q5oM=<{r~PW z7dcWMpClVh8p>-A!@cDkn||TDhcd;!zMfqIeDxDmf@QRMTN!Hj+G@q(hVY}$v@9tB za%1BDiVdf`^1CZGgnNokRU#>~FP@5a71>Q1z;2qiS`F)++kz{k>(<$^;|*cKaeDr) z=x4iH4c2rjU2AuBb0*D*&8&9%m!1`6#Fd7AH-66?i}ojO86e>MB5Ch|(T1|!wCR=^ z?L9xc)E;&ccew{$Z(oPhvFF8+tZ}^_f4lSx)pbFu&UIJ1{e&{4TS9kl$hu5@dtApY zXR4$4WVrQ9Zfb4ZTY|uEICLY zF!grEd_Cg{WE=uhluh#ws!}-X&4+00tDmc7J1ipi>aR zY_nET?yy3QRw_!qYC9%uey$)=BmI4EnnlWoyBVNKJR@ z9N_JHw^o(Y4AKpR2+gzJ{LluL6;^NtM{D{E*4tFLX)HGfRhyGu=Z{2R5VvcEpLFV_ zPRVM8)p}p8lY3rwn)0X^szH{O*>_mUS`Oy$-GFX%<3`nn)?U=bp<#+@D2}6Vf>2I2 z5|yvC;7(0!oqtyv;D)8GgLTi> zeRMtsSo;I9^hGfGhPJK87l~N0eIoqExwa*o3YSpQf)rP(RBk9*pvMtf^ff}oq2mF4 zw8W*xmaf4`RU<{;%%M?);>4PiG`sD4_gRY|EDEj%aa zeILWAa17_&8D5*WX2(wV0f^3R9N#*p#Ud;V!)( z-)q_rWFMQb-L}7!!2deKF@jhM(({ADgeZ>EQZ0_J($eS85awDsMMPP83UcKcf_>GY z6AN~8T|V~dqcJx{p>NXG?sP%11mC-Mk!?V&q#_n-yb`Guhf=1JUr`ks!g>Ww$uwVC zR$9;q!OS=}!9ZsD_MOJ2TZZU^PDA~z_ZcysT03wN^yT~G<;Qn5DEe()xrjjT0ecPy z;?Tn&;Jy?M2|3fNX@@WG+fj0_$1Qf+!J$nOHos{Wo3UN3vIZ?s|qNdbN2d%id(q!%E)i>CjJ0*&)zpz!CNzA8hXiCU`o2#r2k- z&C+WQtzEbX`s9PbGW9%+rWTd;!1>i73EIq|60$V82B5QUu@PM;uz|WA)6hBh#};vv zQ2{y#|9JNc(q%&c{B84&7QvH9w+TTJAHhpdQQ#Uqw$0B#vGjgH&u(oV%3e)cP3C!Z z90O0RHqT^VkYlXp_S~WH%E(JV$^1>|F>209qYoD@xyU3z6ycMZ*Gbd2%Ti{Oqzy1xr_@96OGXnoJ0{?~)Fv|4XvfG>Yd$D#UJ;(hOHSLSp7w$g%e*jmlzV844 literal 0 HcmV?d00001 From 7b9cfb4297589ce557658c4d7951286d9fb5ab02 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 26 Jun 2020 15:53:44 +0200 Subject: [PATCH 43/77] Add info about cells --- doc/cell_v1.png | Bin 0 -> 4070 bytes doc/page.md | 34 ++++++++++++++++++++++++++-------- doc/page_v1.png | Bin 45352 -> 43520 bytes 3 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 doc/cell_v1.png diff --git a/doc/cell_v1.png b/doc/cell_v1.png new file mode 100644 index 0000000000000000000000000000000000000000..3a66151f59cf26cb8fba8d4ee5e7a1ffc9d8e386 GIT binary patch literal 4070 zcmd5Nmnro1Oz0~LJ5&wmAyhfMT|6a z)<#f@igW@BE~ufnkVp-|1rj}>C?Ny_A@{^B&;9;`d!9V!IWzCPGryU4X5L9I#?uv{ zrl$sj!4Pi8zds3s$uptv=PF9jJL&Z~7Z^;l%I$k+-_+4Lc1?Upq>j?$bz}M6yRAAg z5i!xvgJOMW+g}0+X{a-Pr}Ta6D_a|%2l@K@`eG_TZYnR&t8IL>@YS2LrvBZTzc}E# z4bfdZ`vp2@yd_8uvhct5eAd{o*62L&=TFB$i-!1{&>)Gel)A9r z)1k?cMg{UPEt(<@vF}@qJWPcNuT$EpMU#Um;7CC^YBXhVE9_eg9s_J_tSuN_f!0oO zGQ*st>$L>MuhJ+4v@MlLpuc-TD(2U;6IW*_?L>)isJq64J=Bof zTxDp1k-ws|&LK)A=R+%s2V*cSYV&?1jWLBQ@dihXi0fa%sveDj2ZbwP{6#tnA+sWM zCN&rL6UNC-8!IcBNXlwpc&3mwb=)VoL9#UH6;Jd;mM4w*`j}$mTR`Uy{!lnkvJwxE zXO)eIg%2gGzHnOq+&PuW$q@8TwF@&r?<`MNL+l_ia^CBzXWrwv#qA_LSrkAzJP8S~ zPV6~B!P)TJ>`waj#<0U13P;VuDQZJW(w)+mrCcNPj1$1TJ-9jUv7;Ztoo2qZ9)GT7 z=>BN^(ZutVd*dS?C|eJyGHw3|ZLcL#dAl(3-TQ>5$jw@`v@TTH9;9oEJMyk=Q0Na$ z$MQ#g2LlzG57UHJgzei}6>v7wnFU)dGgXuG?aGyAu3l7}K5zf!VIRVKY)>WmW#+h} z5Dh3P;81`LEky}5`!w$wHGtu&cpI%AIeYYC{>C+`&lX6{uqd;jxPWVsWNeD<()0b$ zY%w+eF!?j9ekA(#lkv`Ej$?c9f9b&;;$K}#y-A3*>(+*21F^`vC3?XxV@62Lm)z)A zK>EPUt7PGXu9tP;z3V=>F|_D{9;un{TIiFr820AI!WX)~Q{<=KB-mR%w!+I2WEe<-AjHo?__uYy$7#Fb0!*Nh@^rW51i5W0?|Jj ze5x43CjxD+EIlkm4!T7sdL^o?e}?R8P#fDo9yK@BPrOud;Wrn@%L%XhcI4Y*0wMwz z@*Y{RPIIUB`lT;s(ZqhD2aYq9lNDHr1BHv%U`nh8g~O#!P5_s&9tc_xGsbV8p+Drx zQN2BZ8#m{b>&Ap~9)jw2A}Mg#C?!u6CeqDU1x$}b(l!#-l8D{?xx#`I4~kwN2j(d#5PjS*{bSz_s3YjD45J*( zr}`4XW_mxisO8xo`vf0Wn)q>p-KB{a?FC`dD0j;4iK#v=+Ifc-t;h|iK3djqg2U); zBygg?>*Ags+ur}Vv|t)*(cXmume;fU4QSn3EivYdb;^^L-iPNcHMx;<1oPd3Js)cn zz}=@7tDLc*wC+|pwm+4+vi4wj`8s{MZit?bBQwUeXbw)39tvP4d1F~Hpx=z<_JooM zgWd_QUvo>Hsej?XTOP1>?oL7D2?-}a#VEjjsh|M8f2XU=w#6n|?R0u8o!e5{SlJ&@ z(q}SI<9;B{%KR{ADa6_!*N}G_1snsouP=EoSBtcBh5Zyi0cLE;$|l9(rV(LJngUL! zmRgNs^BO`lo5TC|@uVLq+j_gFH)m_#tVL(}*S}dmd}k&-!(7<1xH9@>ykE;4lLPhL z4Fyv3FN>*`c|89DXLAnV9GZNQ5hL;aJP_jna9m|`)LCS*VwT0HrKjFl?Uohi7f^Vs~8iVoj zkr2HkRNnHz0|D~hdENEeV|kHl+~@Lj1jQaV1B2?(Cv#@(r0`yB%cYq5F%Yicn4qPjb%W19)qXpJV%g!+mFQ!UoSdo?FvQaq+Si=tGRxx|Ql zI*zIelY^RdtqOB0GGia@nrpNeNs4)c$=6u!em`fMd+a=ivX?cq8d!JH$|If6nKfns z&(L(oWZ(hgTvW>%G zx_3J^&7a8!qc9{dT^dWaPPj$=wQnR=5Xd@o!v}#gEYV)RMs=GdS7C2V1>%vk7guEY z0Yd?I*nw^Qt6qH>CE8z>wq^O`1m#5*58!<9uS~{xA2D%*0NhCXPo3g_D}y+*ZpJr$ z_ks{VnyYI|nyW{uo$BWqq9ldyBFB3(ZPVVR<_&*IgMd{09dR`#)xGaVFymkNdUn?H zk1ji+P%i^&U@!%+;2UQ2$(Yr1$cIhVl}cN02Lt{PUs8y2XhAuMl=QS@W>HWdR%d3T z=2m6r_x}d~6=3;4Q`nb*g;rY_xo zKz$p(-A(8YiF($y|DmAAB>1q~bUD>jMxIpX_Rl3k&U6F6{dWrkL&&2^U-4sNyROAg zF(h$B@c^+^ms3TQ636hr10AhuS<)SnI7!2rafd7wO3xCFL)4k+O$4irk>UZgOOBer zbj4??MaWqygw{WMOZ+s4>hj2is?7M30U}*jI`aEX9OlCcJjqBJD6MjeY=nZEZM+nq zI)q;xHlRLrc1X9Xb^`XYfQ}2wU`0YEy;>FW=EW6ZfI|qeg3P{f4)N&B_HKt;TmG8R zzG3THKxdF%`_txIPnB6W;$hidgW-XT`!Vsc{$GFdy6168O0lK-QFmGIgJ_! z{eYJ#BW6%wQ}1lN#nvi-l_}jlY+Cc4 z>~%_6sCL^2XbX2R2yBaKF!@&WB!g`jEDfvVHLdwG#G63z#t&gNq*%_jn5)2oz z#r)b2j!-6-Mge`>cjPd3VtC5}1Cm{w0uG7^mt;4Ucy^*WUL+a|k5gqHb}7@Q$s9NZ z^j+r(k|Mla@7JU#fRLw)b0w^`bb*Cz-=qp$Xj6_gx94o8t+l4L5r}U@RVW1DjI|{- zPg_OQyhr~wl8?%)n@B#gcVrd(H2otS32sf$Alc!Ax>%RfOaZX;V)Qkid!o^jEu)l^hDH3phNQh zRBP>&FPL2s$h`xeVTK-;?;d3xw@E(?Zhdn9z#We>GvSd7O7DOU=j!*JhAEK4xxb2k z13wJ}{$21;`OE4SD*rzVUf+URcxvb>+@<_$^RKfI7<0^w0*;&6qd{tp`=ThOQ6wj^Uxubg#f==xQ{}G;b9o zLblErMb1gijX)?*o3Bd~gzAv`*^3Upe}C#PvFEcppq66KMjzUAI82tiAMn3oIWlE9 zTyAilvIPQi@N?6G@vY@FC!7iT`%X5#K*26rXI3urlIiTp48l533*-ESfGJKcM lZ08>RU;FDS!;`HobQ+5`>yX~uTr8vAe(?Oh`snFP{{hfHU{nAA literal 0 HcmV?d00001 diff --git a/doc/page.md b/doc/page.md index 880a83d6..293636e6 100644 --- a/doc/page.md +++ b/doc/page.md @@ -1,10 +1,16 @@ # Page layout -This document describes the layout and format of a single memory page. All pages are -structured like this. +This document describes the layout and format of a single memory page. All pages +are structured like this. + +**Please note:** "2 bytes indicating a length" implies, that these two bytes, +interpreted as **big endian** encoded **unsigned two byte integer** indicate +said length. In other words, whenever we talk about bytes forming some kind of +number, it is always the big endian encoding of an integer, either 2, 4 or 8 +bytes. ## Page format -Pages implement the concept of slotted pages. A helpful -resource for understanding is +Pages implement the concept of slotted pages. A helpful resource for +understanding is [this](https://db.in.tum.de/teaching/ss17/moderndbs/chapter3.pdf#page=8) PDF. Please note though, that we do not follow the exact data structure that is proposed in that file. @@ -21,10 +27,22 @@ without that header field. The **next 2 bytes** represent the cell count in this page. This is the amount of slots that occur after the header, and is updated with every call to `storeCell`. -After the header, in **4 byte chunks**, slots are defined. A slot points to an absolute offset within the page, and holds a size attribute. The **first 2 bytes** are the offset, the **second 2 bytes** are the size. +After the header, in **4 byte chunks**, slots are defined. A slot points to an +absolute offset within the page, and holds a size attribute. The **first 2 +bytes** are the offset, the **second 2 bytes** are the size. Between slots and data, there is free space. This is the space, where new cells -(slots on the left, and data on the right) will be inserted. -Slots are always **sorted by the key of the cell that they point to**. +(slots on the left, and data on the right) will be inserted. Slots are always +**sorted by the key of the cell that they point to**. + +A single "slot data" is a full cell, as described [below](#cell-format). + +## Cell format +Cells are simple key-value entities. + +![Cell Structure](./cell_v1.png) -## Cell format \ No newline at end of file +The above image describes the layout of a cell. A cell contains of a single key +and a single value, which is called the record. Both in front of the key and in +front of the record, there are **2 bytes** indicating the length of the key +respectively the record. \ No newline at end of file diff --git a/doc/page_v1.png b/doc/page_v1.png index c7ccf45da60c0353826b07ad864c60dcba40b279..c5aad14d0eb680d4e1a3f842eb6334ff363babf7 100644 GIT binary patch literal 43520 zcmeFZ2UJtr_BR@(h*A}4BF%yY5RfiS#Db!rQl(22qy(h*fFL4*z!B+9l%gU;O6U+! zAxaB|5=ej`F%)Bngc=Cp?Es#0?z#89@x~kPKfdo9<0fOES$plZ)~su;-<*5LUN^eL zz30Fl5D3J5+2H(55QrTO0*E-5(W8bq%eY)xv;&ynzn}kd;QwF_jNkOgBi%LMqTqdc*3=p04R(`6)Z&j7guNDK z^dUy`AVM{~^fgW-LB(z@WV@$A14r+n@zo*vL8Fw#elE^_@mx|E%1t{ruq#q|YcRp( z>KUFU>XY|mk0MOH?zeP|a6!{O){w2M4STImO+GLkxhxnEH!4Odr!CG_)F?HXKlE1P zPYijghb_67#I#=!=soA$y9#eoyQ*>9)YxLCBJ6RPNo> zm>V&B^VY71(l5--;ecMlT%45p1vZx&B&tpu9Ad=1Y@U_wPMkW>AQ1tX@~7(5FuojiE>P2RRbX;+#NdVNcEd{t9|a-w&FFuHYNaD z*S)q;InE;o`BhKz!*$UEDeMPPka<#SCQUF~epfTk*20cW??JcMezBWc{De6kRn;(w z0hJ2M5^r0HJQ?K;#3Ls$FI?S|{B_4IO=>ZIS7Gdp7lU)GM};=;b`ZHc?c1UHFoWzJV1bH<;k&)Hzi_KH> z^A0C@pLe{U$tS2#fBi5+&GkRex5b2vDvt($c(f-!&N;78SLRu_mhrOM8G35t2GolAFl88{N_HIXC#E zZc*_BeD7G!v+S7cQSruc}}0<5DO4!uQAsHBI~ z<^*ctjo(?}nkg%E^+^s8#rO8sjl@r**NnSsokNM%{1;!(D-h+bIZ3Zp(UO=6~7p5K{J26_g7b#g{g?mXgrb7lB2DJ|<8cRkcmU!wmp5qz-Gg%9TAlsT5mi&%ft0w^Ja(dE z{Kp$WTOwdGe4SC1m~IsA%vgeYZZl>!r0Lg0;JcU(l9o7PYV8m55d)UZL4;p~P7a%x z@CFc=?=Wg=7hrH*ZGopwt2YE!waa?BjYNEKOWblv zJG^YfiGPdh4{|f@^i{tvCkYwqOkAlvqJw(sVvSy}tU!&o!tWH$AiZI`uBmhmtu9 zQaV?G%mpW4ra2XK#qs#cuUkVHJ+BtJirc57Jc`&n!>k0N@d<~CxIm9>%2A4J&WfhT3g^{c4@= zCJ6=V zLdheQ?|r)il-)h*1zKXLOLiJ=ZWi~0P8f6=zGer#A0ID~)8a2+YX3V!IRk#6bJsMZ z&Q2SH%zcn1$y96y+to(5bB7Ld7UuC@*)ghlnQW_ZPQaywNFBxzu>-H~|> zAorb?X(jyC@4E}1SHKU0&c@(4AG+@9W^O)AULDqA0SOP|y;3-^DYUHS z=@wVyZf>?(#^R>xzj6Whf`o+Jls_mj|M%a||Gpepq7Z$;EVlL1Z;=fII;)pd^joEw zT5w(JPrW_L)PUD~1;2HEAJaTdpX*@_nYMjjfR*Vav^W1^E;H}#)o1>_pYzWP7L!YL zJSe~76RO0?y4gI^w-u%H)-EOUn*-V$19aNv4@>3naQdr_0u5EEb0|!To;Cd-n*{cX zbxtBvl(zYtd&en2*VU%?1Co!J_BLa~quKYDyW6qMDD;Lu1;6eD5(kM>WtGRzEfY;2pi$OHYEXR>soSSZZZ%* zku7S04LL&Wj49)D%|P;`6`bRY$&l1Szd81Y_g`Yu+>e8cS)GK?R+d&aD9V_(;R!BD zOXqy<0mIz*YZw`VhzoA6mwqs~S^ywL&iGGR-@gm2a;{#aTKm}FE_9{s81EQS-8cmb zIfY(rI%vrpFV}z5$Jf$zi5>)*1m4@~1Rs1Sn#e-2_)M3fb1z1eX7e(3VqeJIN(e zt3zD~4tM4fIXCD!;%i^lJ5~qutX*Uv@H3HBE|A~Oz=ZVvSskFR5w^7&1jnlA{&iLt z2Q*r$!n@_E-joG1O_F#Hj+_dX z2QG@OwGy~EmkpP;sJxO?{gEkkhq(DuKBAzq4qGM4A$WBE-ER2fJi%FtDtb!{GW_*o z3d6vGW=Vl^x@O0?YvF|-z>WqfHM;dF9B1S74VthmhKycv#20B`ON(dj5<4Dx?7FJD0~Dd%)zAe2z4+MeqpgJ@T}<-d4T)QUCF-VS)kFE|cu$#0H z<5og!YkzC-KRZQM8HDxjkE@52xqZ`FEU{+laLVpco^*G>RfnVpWZ(e5;_iE{#z{I0 zaBv0C#zA{ovbc@4>2D}a*taWXM*%H{|Ykq6-XJdtPGFNo%bm|@z+)kbkI8x zsT{Eptd)CDT!qb$E)`WbZGqjh{&s;jKb4mt!_fU0p5V4}>x_N==6A;8eQJ3*QA_W@ z*{m8Qf4B9u2a?>$S0QZHKP)1v`h++aWU^sr5$6y)5mAxCK(7Ye#2ufK%X|g5pS>w{ zkXgTyixYvw;sI0_-&0f$^-bxFVDT2ew?Bw;9&v8J^-`)Ukq z@IXOno&9cfHY^*~3tIqIgV^~44DJj9cz}*emF!+h*jj=`<>&dy?koxN{tft0WfK(M zE(sw6Yo81yhDJjR?Pn}j1+GhoTK53^*z>V!R+s94^GS~`7gbmEew1EKkt8p)?OgAm zbY|ZQitp&a=l6 zaHxpj>D_2?<-L*0S5HqZ5Al)fU1?FA%N_bW1- zytc34Tfna>yo?^U3ZQ1sluR$cv=+(>NdV&eRoM&?gKWy8D_V%|hEJ0upJvde@+f&a zVN|Zfr%NSpb{Dr)j!y;CPhP-nnxg}?CgI~9JZ4=mvSPI*XJ_T?b=ZRRq zSFGkaX8Cj20bvDBs$K##!Kc}~`P?=~dLX4!JN4ovxnDRx8>=LkM4k{Q>rAcV*YRb1_(pj|FMQm7t(ny&Eu)L5i&)91DE*J}w0Q|b(rFaL zhTY;&U`sG&)JN>3pA4|g>{#>CJ#El{xx;}TT05e24#t6SoocmYU71JehSM^287r^m z>9pC=2K#gBdf0lQkmgX@OGQ;IY%tK~pBxx}7N&jbtZgSP|JY-VgDkI#r&iS+wHG1t z6eD~2M7NR0U(#BLhSDkrOo*sQ+1FvJ z?-?fzSjyUmwp0^>oZN$Uh4(|J3>Qikf)#&>;XLM4$^*?y_V*HDG6vSg3-MYwUci=9 zHceVTN68>AE*QOX4&e>x9KbQ&u&}GI1%-+)PW%11(BP$W<=b#?ugvHPHR++aXhs3muJ!9TW+T$Y^lVD-Gvr1 zW6=YAR|S6N!=~1?KP;uICGKvREc2ZjuIsTVf3)xRXbYd0oM!ZHbVP~5pHpf?RoOt< zTy>7;5PGVPu9`vNKZpF?LM~c~>S52c706*-V#x+_3)5RXCs*17Jns}5UI2Uq;Q00| z^9qah3zy(80eojH?)Nn76d;N< z2c)%}-Bki|pK3e%yZfEkq~*R^U&?`i%x8vdF0@1EWZC;8GrjNkt0{D_JTJq$o|5kt{8>HENQ9#BTZqx$&!OX~=N5IA?gCK34=G zBr)$ULz@!~au9R8s&t3*O?7Er46BQxxG+`R#T}=@?iO6#V-)Av9pT)`p$4jbSgvOU zc`C(ND&#bEv^|2jS76h5LOyA@1CLnIVFQBl@zdhYd(h%4N8NU#sS(oLNv`YFa#u^_ zls~y8gynEt?kI^y&wWRxi$B7_$De&jt5zrr&0DE4iF{KSRn@PfFZk+|7!Q$m?P;DF zT^5~7gzF^L7zXos9e9pJqEU)Esqz4R2O6h#OsN${yg7*Qv%S|RRL~aq0unE5wq{wE z+zK#UqVqdA6C2Wo*wIp#bD+T1$9gv!@GU{v^-SM#%Pc8%*JYx>1+QhxzRDa2k=f?~ z0flm2RJK^lh}S1r>b&!hB4lLjKfRW#eGW^Rz0+x5LeExTdegJFAca%2_|%B3%5)jq z%P>wOxX$o`mJrjWf~&pR*7vNt;;3_`WS&y*V-HS$-xH>1#h6=JwGvLtl^XDCy`drK zCW>5O3ny9@YdJAD$?=hId>?UFk^tZ=RSm=3r0$sMS;+=hUF+yjDCJV3HpIWgdQjwH z=BMn78%LWVoS#=3sVBdA&mBt0mpMCeD_lX3fx=5txw%zU*6eC7j^jT(6@)J9eeM9c-&-wPzgb zf9beiPTwO|7h3s3v|t#@u@82?d1sH}B^6^aj5+fJ`M$BP+OMkVaLFF@YFQvkNWo~> z=)Kz?M|6vtfn_9AmX&12xMR}gPZ_T>SY0qmzlyB)v>Aszl9=y6WqBoBo9gzB4ykwZ zof(%-0C$MJGFcnIvb?EY;w#Xwhg_~RX?d>sp`raT&1ditD+kRynON&#kYt_!Zy(!H zz>Jx)LlRtb&yZ zH$+yIgNw_&NY9-MRFIHff?v#%(58&CeN5;3y&y-K{-iUw{5pWw^m%`ggnag?1{ip^ zrO(3l8zzY?5Ob>b>KhZGHzalKjztiN~2_WSzAA5qB{$?if_ z4=_j)?2totJ`v-s&t2OCn&OEO9@~=dyTIJxzMVu37aL^*QSW67OyqglS-4#8rw^AK zp>Oj8{@|`N$(ISl`6Q`gRI@%vLE4tuGzvy;%HWjbTu6ynH20uy<6-D9)rSG0iWiZ;p=QWG7+%>yy(t>6vrVOhv^4B}Kp%jQ{ z_%2tyqc&FAoUZHP1{)C3T}C3BDN>dg%XfyX;lzYJ5Gg5;&6$Iafm?oP)kyvQD!_Q# zYqqV{b;2T?M<5ml)i_80Fc&v5ahL0_frD?E2vq?}(os@jY+I{1J!o#2K@C0Fej-mm zWUBr&{mNAON>D)?xoUb1v9X5GmE$Xh83)`TPG$HO--0e%ksDKUn*=Foc4`U7*gR=# zZ9ySjsxBsLX5`Q;3G0LUp`Nr0duU4OXgq>m+lfNyKcM#@Rx%q$(I{zN865T%*Ut&? z*V&o*7_ANdsS?9jJnbH4x)`3M%_u!6a|#Y8^lPGB!L=)5?hS$OLjALtXSDmX_aA@T z(yvHmR+p%*#rUf7RwK+*LQ~&K`}CPT%RPQ6zceDNs3J#-$Nt!~_9Ra;B$a%y^-7%> z_)Om9RJ$+3MbmzA>Q!~#?fPUHJKbpT3<>O|fiRInkYwx~a#ycbzgd4Pc(SN%k$N!m z+UrwvN9g{0GBY-nb`Xtta;Y*Ed}DY?gmE>pozqCYmX z#TN~^zJn-E+E|d~#R^hz`*VQpj7EvTQq}J?^d#Oc_4MQMy1{y6pWMa=DK9{GA9uPqC0S9%_0xtBbG?-@g z2`&fA>=@LrjXMCp@XR}MNT4GHX@BZjJog?#Q~j9dWbJ>#qku9e!DY~P=x%d2E&b*G2&tA0VuWoEt8Ik3o;6YVM_{~VH; zM{t{(?k@sIR$=k6)p;U47HQ9R7BtV9H!r%-maAldy&)+~$U22ENQDD_Wo}J04-+7z zSErn1eRJxOpX zPu&>PQ!`^@uUS)LW`DoR6ag%M_2jx-`hkLh{^kI^%=v*qVtxpIXL$3+)^tsv1Yo#+ zXW8j@GWz7^a!bY{+_{cBOzs~$G^+~-n*BsnRW-0Pm#$VLW7(cjZ)wjoP54pe?o&1Q zyK>5Q&vlH^7aeaai5Wv};L%yskauB5q2aU-*DPKPIbTT<#M{%Y?dG&E2BykiO^F*a zNxY$6P&#CvXY(VW%I1uGN;oG~6a7f1a_+e#`MLi|V6r$BEF-n{F!zs+@wfiVu6sb4 zu_yx>U2$-W)dnRbk=i)muV}8JQ`Tewl}er40do6S_PX9ykmLKl!-7t+G(9e~3!9r_ z(onC9N0)r5PJG7}NZWal>`}SDK4pg~KGIP3{&QsS9b94xr7vK>EANCJHvI`flxoBG zt(VH;X&H4i?~g(0@KN7p_lsIJuVvJ&Rs+<+!d=|nN#Q;c!{vd%5x!g^*Tab(O0J$W zj~|`b`39?At~&Wc6f@BNcy0sm=yMDR5DCYnpx08rIRwL2pf6qRDMnHtAnN|U4o$dy z%G7r&;dBI){zmu({| ziF}K&{Dw5Uj;(a=Tz-mj5lu`5f36UKF<1-X02RG`5)$%@dXIw3dyHb!>E!ee`K6jUTC%L~7ta)T}yo zY)oE`4;fAnp3t5gI;q%<3QRPGtIFsWm!PW?KjhzSLmk1^!FcbLd_(WNi?8%n5BpYc z^Z99q5Om(>Goaekq*IK5P+4f%jL&=?1p46YnS@Z8&!>|N*AIuy_BN;&EtObVf9IT= z3-)RZ!sYBMNI_XVff;|ZBA@!O9p)^0y6Pl!Au|;xe+2VDuyv9-KJ^9oVc!7D13v8t zZtepU+UQz^urQqeAH*EL8EwM4D`6MYc+LWWEf8%0P>4%XHSs}GiITDBN4J!+7ZmHN zKJ$&Js3>v_urY%}}HQomOsIA@yeMh~>BzPD>MtqOmBL_~-TFivG+9z*xyJ+qz0GEn6TW3vDa>8ZV9Ei4o@TFF@ zlQSK^){{x-iTgO@{4z?2&y{o3MH5vGQSA#MS}gfD)$HzA@rl16URG{j=uqZgbGDD_ zuF6C^dtnhKlk<~r2-wX-#HNcf{xB#EY)`2Lkk*@l`#6(nOSk6`at1w0=|A~*m_SI? zLvP=a6JkRfDk@x2?S=tvRf}I5!>hDWCsy^z>I4;B?jNSmu{RJl8yMXtW9RCMW@=B2 z^{9~uC}Oc^s@nt!Phsje2b!;dyJFJ00M2$0ezFGXxav@07+f0!6>@-DILPu%1$bXb zH)DUfw>Cu=N4<5vnB8T@^5$znSRS4zEYXQ5rfXcBL-Z~A6;}zI#YOKr#Q?G~v9!>B z^I6NM<=!^;1$};{uceV#6`hPW#0A`VUCMM8MOv9b{pQB;B{QmukEK8hbV6q5UMXxN z3#BaY!fLi~%Vb;P8xto7&a{e?o{SbP63_%k6AyOzxupc%Ml>SwZ!SVRfYWJe=e}yP~=}5vM4W z$f~;qzs!ap`Immfy<32#Wiu>IV18oAj((H+%ROQI9c&`Z5t1)=e*d;ID_i2c(B_Xv|G7lS+ToZKU!<*)IOB-a_SA~z6r)4+Zn*O?lByR)C06Ybw=2&1) zYE#E;gpTAXX_58}bj0y6PbEwmxCL|WsEP#y4^)k=4=mLO-BZ1kG+134B}vwLx!yOJ zGlwtz+2{G!inoW2ERBsb1t)0U>HM_+?X|ETOC0UQhjmMN;W`MRPf`8ML6WNVMtwI! z`$yZH55aAWH!Lx<#PE0M64fT1L5oo0eNq2C-y8#K63p!0Td)`gi>#WLUHbH}?NH)x zoCyGXHsvy|HT5{1j-VdyqfgTT{&&m+ zKhyRO$f|yEuORPz#Pc?bbL@}7WoV?bECAL3Sp}x4h6n^a&Z%f}9Ql%x8^dIK$a( z$j7vEDPd~AwO)IUtHf^=;qFoW(k0=om)ir{cfr8O^jZ_h_sTAI-8k7v8j&*2hPmjO8J<_VWcTN~<}N!ymn~Vr_6TSx zRF?NO^}ZZKo44)I`zsGD#hatG96D>FudX#sAvFV%Qt=|r=b)L>hvzEESS)$kp>tasX{fUCX~T{hU%~$L^?j z90l`^+Yk%LN{O1+>c2vT8S?5HZKy7bx!e^kXu@CnY;6DiG9;z!%BORQTltU_fcMPA z$or>0MqM?XPl6aj_|}xtr|s2=#=i}mJ?`4+c{%a9#RBsfB1DkX1VBA*34oX8W(!^rmGW-9g_?wA@$n_1LGGEAo`JYTHzwC4s+qcBK z_aS%PA-SEfm64Uq*BqmdLTwu@$YJP#`$Vz(Gx&ytSLBoR)F+z!)IOn*4#?T^-d9YB zj|uX*^iq=Q{->w3^m5SO^ zwi3{beJ#noX}my6?ZVwZrn*-ImG|BOD`t?L!vQe`i`M~ZKvIs-eyad zG7PaQRRFeUo_szYF|odJ>pP8HgNIi2WvO|}E^Q;lOyO*Z4rvIyMtRme%wppp?P}&S#ukRpiQ;wOR4+$>&m>N|j ze0_a=8IS0%@x;di6$l`Zs_bA4kVpIlE_H?=RlFe!v$gCNgQZ(>z)(AiLiW*qr)?0Y z*EfaA3+H+<))-_WT4hQCWU~gyvRJ}?4Pzg^dzeYL>I)k@jY{nNkx){h8m!KK(QC&V zGM{9DT&%+&VNLkUpP6n~(MYv{Mk@>s=j!(uyl=BXpGPI`xcSfl99b!~2;|Rz+#6#p z5tF#gT%Ln1cF^}!Q?7|dSqzwC$*X6vgkUvVBJd8gDZa?6mpbAb2EZm(D_Bz7Nk-{z z><7v=fx6N27%L+iUooM}_7fVeX;uhVMpGxb?GdwRQu47^#V{nv+>S<_;L69YD>jO% zL_F3rdFXIA$9tDG*9S*V1134AibcxjuiHpA`Mwq;T`2dcdIU^<2HUj78%s=%{oc3Q zs`EelLX)GNqdY>4Ee<>t5pEN6nsbM*FcX3VBoqtYb$j+Evk+Px;WRoRrcuMM?fmZ;-O5E6-Ew{EXtSo1H#*|Mq1W#J4;MyR>pM7 zN{yw`6Zq?uOu15fF?rK)jgM^Lk68~De7K^w9mLNYU}P5fg8qoI)38EfZm|aN~oFI z9Uwuzxw~dd8FUuPgh~JGYs{1hXrIQOWcG<5-a`BstQ;uZ8c*Fh$y_x3Q}b6v^;qp9 z!pZd7933-_af1!<@DYMrHMOyvnSEg1IOV+Vy0W~PG>KF;7c3B|+Faogx6^%FI6Kpr zxv7gs#{MDIM;@d;s>@ac;fpnza$JrOX#xq5weAfe3^ z|9mA>CG59qc5Un;=nAkrLoim|1fmi(*u}#HDEt}uZ|Yi zNtcbp%EUNcSDvh19@0Fs^fUn++X4F&yOG1tK3Ucui+T!+rQDtl`EJbyY*N7I)ZsxV zM~L%h?AOdeST1R%cGh2g5K=B3@OeAdMwJ8Q5WwL(QhCzPhpw->!t2*;?89aBNav@D zjvBizxNPKd)C&y{OyYwJ2Yt9sx3ju!v7j>+UTkS{pfcBo=|$_Y%)r`rZtbJjr8mpn zRhpRX<)52KTqnhbzX-S44fv826(YRH%Np7UZA7K(Z9ayS#xG=@SJ5<)4S4o?>%gYj zSz+|k-9V0ilX{Wan{l`fyy_H=KqDMwDXGQY`wUrSQf6Z}a$7KQ@v$bV2M!&m@bEQ? zA>_D*CAtE&>dZsEGmy^rr(#$b2FANl6NwvP!?07@9BvJUOjC{Ui(Skf`k**#DoT^B zY(*V1Occ9ynJ@WNWH_@35a7n(F>+tFc;uPMGI%2L>?uYs-fyu)kon0E<@$vl$fXjMScpWQ-R9or zZt)OYK4@b8&{O}NI#9GBA1t)BYD_O|@ZNL;^8wa~Fzi`ApN3}yAV%29drY)UZgVpp zuiyQDS|Fi3xDa#ocd7TyD{oYUHg}9j<-bK*tcw4k4bTB5Jo7L8w%SC0Sb#v;Ju7=a z4s~-We}>nf79+u65GXw$^zQE{6?g(9Ia(Y9niR9YwH4zrW3Hcf@^^tEf{+%27l937 zD=O2xHN6AmF!q1Oa1g{S)45>ay)2;jM}dQ#?fOc`R;XW1$;0>gh*cjfXp8v4^ofPd zVQWrRyFp4TblUu~ynQo$WvxTCRLY>_DW)jZOB`f_QJ(~=ok~RGj5r);6Ws3KUDpj+ zhKbfxkM0O`I2(P^?1xRg_TtPsYgzxr&xdQTr<^pid3|-SWJJmSQ#*BAq92$(aIJZ} zGvKXF9iPM1Mqej<14<_=Jm%v!E(EV-?NkRkN&o5R?nQ#M{I+4c&>3SJZ!w0vXvg0D z-HpO$M@L5b+2sc65ePh#L8lR&^Kx^Q?LiKS(ygPTqn+g|-@e^mTUlB04rS2hi@E&{ zAwGr`D#)~(Dpo&vf;tx}eUn?HSnQ?a&or3vYGcvjHShimAqOKJAE*jlQnvamGnOHGsIo1}iu-FYC(^ zK}fs+cK7}XHM-kmM>+T2m_PB_ON(Xy9qVV9q5ZWa^Zw#dy;bkqB){HAmu<{o!@EGE z-Emj=JI>-;E6*sV698!$KWKK z{o>@qtthm-QUfPh+DIjK15xPWsYX#{W)vDk$_>jpr^pdkQobXC{IfCwEw_T^dQqt3 z0KJS>0)w-NHCFLB2{}36%$G(d=p}5Qb^1UG>z2yx2fW^CFL>8jJ|E6?`r{488kZ+isP{*tg} zF*A{Zt(B)Nen-V$6Ozxlx#1d+zv{_vLqEklcHYo*_}-l>Y%k|j zMtORc?}R`c==Ny-6X}l$Gm6eTA?9=~*wB(kj>gl|`*R3BVlZxCf>nzwXieBQq8Gz~ z?kKWQW}mF5`tL?b#}SbO2ncC6C{w7f1ICbZzPB6qsJqb+C*SSp9ruH9g5|q%2?X!l zxFe!`+CH&v)nZ^9=Z@h)d(T|S+q>*y*Oj8FFDh}Z2H0r2;|K;y5!4pBaHpYi&!cr3 zy>g6+ra~N@D3yUT%B!YVW}j09YV3ci2@W8(N4f;0`B;*%0@ZNmkna|kk*(U$Nz((m zb2=*Wpx#FHZh`}&vE+0_XURciziVEG7aY?WRZa~O+gagfNcdS>p_3WW{t_e{_rrcM zC~phN0pHfk)Ds1%^Xz1>?`ZM5pYbnoc#9VwoqJ~lLDKp+EXWM6k$S(>?nQyd)eT^O zC6w7AKtt`lIjw}M(Fug{x)L8sKm+cV;GDf8$_hfZs(AdF72O=8^#V}!Z_IiR*lQF7 zZRh_b(!YPuMp+L#2gFh>mcHbFtmyt%Ru?4mQ3>!@_`ihx5h>!k!s#n>`>w9jX_c5| z4qP7V!vwAD_x~D$x+kvCkfb(1meWwm;PP zhO6-m4{Ac?>24623F1IqxL=%~N3d~ud?0Fhd3hi&Bsh4r6LlVx+I$SE6MZ$zxr3-@ z)wnz~OROM}u8~A>PM_DVl7w`HD94>Kf%g`Jvq_V}ae(lBHyyy@=#$2p>+S`4}Wh z!)Zn&Zl{Cn4|R2?KQkFjL1xpu6nI}!dkeV!IMjdq0?35(R!5%bnOCp55L8S&pbu}V zCFM5iIQ$O%I)Q-+8WHm?W#yq|T+tn8j1SLqOzfU%WExEjoGIkueWQ5fIvsf03g4Ji$Tyd|~flHr>6x z-}5huw13}m_2K;!3UPTT{q&WZW@?5R&8>Z75|$xfB4|le!glmZH_MlXi14+OR7(RI zH}*z3M@EyO7ja>>g#^Xd%zkwU=-uak{ceMpiQGZp^M~NHe44Y;-a8P;tkdsS8 zd+?NQ*3QO-jL-VzrwuxFg@S%rl$nU^nI!QeDlL14tW6JW3i#l^Xw`UEVpI^HS(cxm zRC|%PVYAljllp~6qE+VV!#*TweR?~}khjNFbrng=`=+4X#IK%L4<};@AA@Yp5t7mWp%(e z0DfsL{Tsi6+>n|3uMi(xmK!>;NXspvcm|7G;#Q|RhkUgXWeRSY?af|Hfr0xlxYXe( zrQSuIzEU1Hd(ym5K1sxbh6rf#VAOqfMlB9M3NolM=5vwV14@WO4s90WFo7+G*j`eyQ*mL3r}a?P zEZE7!uUod$xwr7FEWVitQA-ZJcw79rYWE&P*KaOmE~+--Eh>;@y2gF|kH(wi*`WW8 zyuSD{8#x0*`?%``C^xn4^bVEor33X0q7T0QWVroaEuz$IYn9S99X%`D{Pzw>hJ2;J z=4a}ZD*{pU;+@P8DR{Km45$?dHIKi*bXL3nO^9pkF7Tqel zY4tyS08s5jt9+k%4wnDz=0o$W1Z#n%TO0K1DE-`UL9`rqmJ@byH+qT2r7TxN5^+gP zOdQB78QoVg6Od_GWA9$S2<}rtQ-?#tKK6QxGM(H1%^Z72e537xUZbZnoUi2@@2gBh z`&8e&-(SVZ`YGaajXHciELCDv>o5e5I(v;AbdR>GI=b)lMxB(N6*R6rlfImXxY#U~ z=_HkkYZ_tSv`np(e{t0JCy}i{si+P~rrEAX=LzOYz6s8!Ix|&3>1D^_o6*&Bm-TsT z<-swF)jCTg9ZyJo2+wfe%lKaTs{@mB+CV0%V83KhI4bJ(!Xh=S0DptwL!-O~YD=Oj zEb~G$%CnWhOm~HQFPK@FC-+eo*u|#5WMluS4dX+Sa^G*8?K$gMZ7M$I&aR^*i)aGx zv7T_7L!9{D!IECIh+@gvuiVD|7u|_vari%bv`3}n(ECJSmPtG+F(#=ENl+|UPfO?ZP;Qz)AKpi-& zJ6fb+95*sOC>Uvk*ljt74rg->9 zpt=d|ma2;pDDZ3Y=&}@Ds~b`tS-Js0%E70LtGBVL_D{Rwi>CBsuPG$^-7V|AA+a@zopix*PKqbi_K@6UQ71O2e;keX1{hg1-;n%SUFVrl%0Bm{s{84M9pe&6vie0H4OP@I0=qZ>-7v%O z+tL}q^GLBmo%n;++ZE8jl7(T$+_DB=ZO6}|FYFKa4@EWs$eb@8H z-tFOo&|#T<`qyK+tk$YHzfYVEA6}}niWH7Wu~69o>U^%dCd(!OceX3cobMoBOSHaJ zWGy#YW1T&rifLCY?xP##d*SBUcSJxZss~)Is{8&FYQ4qk?PXK;6e%eDsj+Nf>Nh5M zLEYc%OEX%6zr70>N_COhc9^iu#SDiX7wc?#^jOxPL(>M8ij z0&1xu$Ta=V;euK+l$jute`ulFc9IcTOY%it5U4K!w(u9AB@ z|AUKr3YPf%w@js#H^3!k4)pJ@3t>JR`Oh{ccmH#nmm~9WdH|J<{ogS|`Tv+qwT|K3 zTD##Q0hxHAENzPd>lgquVOTWj5GBbpTF~11+A^a5<}NEHl?(h$`H2#`%?rV))pjf) zuhI}d-rf{)9t+>kgoHJ-l6lc+3gzRe>ml&(l5!u)kw_#NNSW2Bsj0#B4B@l0vv%yZ z02Vh2U0b^LnVHMIzmv-7Szli_w0CwPnaw=4Bkz1Q-4yl%hrrtYVt6c9Vx0F?F!Acl zoNnMLy*D~!o_jDsY8_TbTN{YfX{V2?lT7A<_kL~z$vVL|b5yd|gsxuFodV`Hw1Qz8 z_8?lp;r8`RuHArh9a*3b+1QG3gQj*-xb7f*G&AWuj zp$IsZEAZVV8_BTg>vE$SE=m(U25`neZh*s?!3)2bwv{PaTBrkIDCUGdSyQ&Be5m9N z!$^4`g;8fjsY$~tV;t?`;^KDfC-m7h%4IA2nS)NMNjp5bE*E@uZ!j6x7mK^Z6ZX>} z^Mu-wp*Z6CjgwI6LG9V>DiQxRWMR+EO>|x}YkoJ{@j74r*A|)EI%K=4<bMXDS*EHtcUk~xvTBvf@l^$U(kf6-wQs?_%p#>+EP^R&*Qd(O+ayz~Nup8H1 zGeCIUP&RVwT%b8Wsm~XvkzyKW%Qqm{_nvwAYcEQ~x~r>F>Ws(M<6uBehX;LSrupk$ z^)@}toz>w(;v+f<=7pFE!!_j3e=CPy1C%@GLiqA)dFlHa<(ztjNK^+;QOSh zIAt%zWXt!;Di~PDc{|*-W-8&qv1UDN=3mf6_tqQw6m<2lD}RNwm7A}o0CgEBsY3t6 z2C(j1NwInUf~WXDkAte)uNtDWx)A<5?EgFy($x(B2j)K;J$M(nX0vjUmnY3UR2d|z%{sLTTnw{wwq62?vp@KR# z0JtOjRg16l2XFz~7~w7+^vj8H@m;Hhu}SiSrCeR@gMcIKB(;qniL5&XxC+3V&{DK> zrnxT|7aQw38}0pdA@9V_@)K(oyu7^fFx;`T+W@E%0E@+!sT)X`>_HU?2QG-De=^g( zCiRl}Vh)f46tbQ@Eh;w8V;B^udTHZiDu3lj=DG7*@t0zg8sdiuZpRs0q7p_H{>C}? zDffWm^=I}oo9{RQrQ7Gly`unr+sUJQ0FkMr<;DpVJdSUbu(jWA#EP}(_=>$Sc`bMw z^L+VL0w%7tC+U}LWa&!A$}k6#D8>~Zy%#rzMjNl+t3cKP(w0$HYQxxIqD zM=P!wOF{eyj?@`#D+3Y^1_+}q`|TpQ1|%_*sgmobqe|YD*SKjdG?g6&#XHEe16Ifs z1~Cr$Rj0-xUdOC794->x(ALSaOga@HHJbFh#PQ%z$!Ll^MfFnS)O@yMwM`!G>Xuq0 z$UCKTH=PZ}yH1PB%^eEdX1>pB|E6KjGeIU{*l0gkK&J+PdB&aCxE7_b<+v)&zZP82 zPJg4i)MJn(o+!4PbGPHm#HFG_3uf}P^;X=Q`WoQ9MV7AbmvE<8Lv$6?8%F7b+a;yD zD!_5Q@3OJ-n<-Jt>cH$vtZqOmcACtU0T*P3or*ZQSqD=q2fq1wbRIvDxv+eo)}Z?i z@2?qIUgrOlp_Tpw)12-gCST!qu!6r<7CL+FZVrx-1bdRVQ5qP21L3^kf7AecQK$iB zb1Utzjz!#o(6*FITb$|>{ARfj(D8&}EWyd+SmxgZVAqmQ1(d&e#_>|qZ{4y;!JHtG zSv8CD!|$hM>EaPfCPy8ngGceIgd!$&TT^!E}FcEk_iF1r3AJUAgXm>)|cpRe1UC^(4ts&&R6&S9{+b)#TFk8$5dkRyETGaYfC3Vcs+5QjT4*8=r3FF>5FivokrJr^Lb;QG9?$!IYu)>;b^Ch# zC^(@}(8HdI^;$MqZ6wtw@o%Q7xh>M7wDKVY=k)g5B>GpSELo+t zxod^2Un~VqY5Af2tTfaMl8d}1hjnM9@xQX&PKDEKFU&4nfYb7!30ktJ&fH`^=toM# z9Dx_$N`>xKn=}7S8z-GM3B~oVE888A?5Bnu5mi~cTSKzjL>?62Yq4eV18anCVLA`2 zjc7EP4hfkdgmXmwFBe@ZOLP-Rh&5wvwbcJvO7p0%lqprQGjHxXi7GUt2R&J>tYB^A znT>nv=4P(PUd0%(mMyZirnBR6MF|573|{h^jSo!HV(o71=X$73R<5^Av-Hs|jvs_` zxZkbGeV*`|3UWE{!;nPHWiR-`uxV@B=>z-vp$7|2bzgE%0yOAudBRiU5COBCvv}l#VaLs$o%)+G z^hvu8s>Xi)fN@Ibg38M~k|!tT7K6^2zXGqq9s&Uk(Aw8b=xgn zEvidJltKaHd$any8Uv@*fAM^U-t@JH1*iRg_)2tV4N)=5YT2SceNhR?a)$?$u0Z% zG8|diaP`rdqP*zd-3g!N={eeQ&fA;Hvy_*rDw@+Z|3hM8{%RR|*^uw-*CyCdgc=G3TrZcR2Q? zmWu~WFFaD>r%69V3%ofj)IAl1XmEv^CM@jRsNDTBS2K6GrlX#t7YMkCE$StM%Z6k^92316PQ z4H6QH*Jg$>g;5B#K$ptEjrCb+V7w5hRBAFAF9v$`{N03snn++kK)}xTOwHdz-;R6D zU3>w~MfYNi1Ph>(72Kb@$!G*9iW1xQF(V^m>gCYF&{}_A--20`KALiG(1sD>1~7qw zH1E^6-LC(2ZvS;+1>CPhWyrSQ?&aF;k?!_Evlp-WxNktQduDQ?*dbMBzef!EMm{xj zbl0xAU!=hg^$4+pNq49QuJ@vHK^hH!%zJyqnkEkmT(hcJzOdJhhmxcV0dAVc-QFL6 zfpj+@3o^TC((^VzUG>^_Ct_t_;7&Zq3SK!KChEfc+R<>k?f3+zUM=>#f#Sylggjx!O_G?UC!;SXZ&(3G|)5UJmt z(;)b)Nq_QzcE-p~{Ie^tM+e1#W8#*isfhsTfPB0I4st0~G^n{G8teUkP7FwT z^!Y2f6F4*qK?I0q;5(}U5x=@jEiNfprs7k80FtPbO9Zd=H2{K(YMIZ9#@> zj|{O`Y}l^mg9BlE#J1&`VlE%L;nqhax(b0)yk(JoP^J$90m8YtInSp@#z z@G_1BcWFGxtycS`E8ki?G9t?za*g1XE>Q)ni{qOym(rG~&Tn9>BsTQj385hb)0WSs za%{0`1lfuQD~OGh@J}q*o9SH(&-&*&T$Z6lrQ0C`tG@^3e(+itL8MLx)jawhrYEZG z!(2U^13{^HU~dScBG*F&>zpj+@cqO_tFt0nY2BjC8Z9Wzu-Cs4(5{Zq$9b6 zp!G3QE{{oP?Q06CFbfiHY90}LP;iaxWn}@1-#>S?wY4n{4i4VlW27)9YwW)fCuvm( ze3$z^(RTL4pt2&5z(A1tJViK&&l!uNIWB?Y)Zy&LkF{Ji9yLGsQIP~1wm^4xA^^(y z;pP3&vgZCI&7-=sBrgxVmQs)~1Bq${8vkR{Ie%=7?$UVqVlz#XSjgWg&4Ur5U~i2CT6(J*C(>z(%rIFKTv-pyc?MG&Ur}&LO=w z+FCM}t2}GzwJGHu|2NyNDWNz;ZF(B`q;oT<%&0of-sSH1h4Zudx^R0|_AKKPY1#?t zZPHjKbQ%)6$GJW$^k&ds3}$6SRmSAB-awj}W-VU7uCv`n!|qX<(*6ssCw!G!H!g#W zw{i7Jmga+ve0Pdc?s}&zb)QJvgB-muh>*kh=!DVz!bt$~NWNy@DGgf}UFi6a-x9c)*RnR+T%N67SD*o?(YZ(m}cNpp3qf7I2(1eWN z66Eg1Nkj|IB<9sNDj#?^Y#g^0ikcSA@%pI8p&mii!8oO!WELF{XumR^)S3ZmI8048 zy$59>Ob%&dDi_Ent5IXn+8u2^8y|Kz;_!zOLlNH)$%VIO=ulH2hko0`0@M?RI2Zt> zhMYpe9~Q{|ov)R!g*zVbpkU_jEDqZ(_YguDw)#6=prCv6yf$)7>p{;y|(lzX&SU|`^B#)mA0w7N8){^qnZ_e~^K#?h<^;JE}vV+|BF5I_M*cv6l4 zh?6FuJ~8((K%MB{EKk{(2YJM}7nj%GufltKdsp9k?^DSggTkqcQ)wKswZG<@i`n5s zQ0T)&9MchC14%xNy3>-UU+a|SmIIa(Xwo|1w%W06nJgLaObQOSYH!&`CwxuZ2baEl z`O+}4INq^YUmUUjL4m))U(7$G@{7`+SLfd;>=rr6RvpiDbW3zl3m)&{+D5mHYkd2X zuR+OucT=HN)M*#zmLGCnBA^3?$g=8T#~E^*9odB~RY&>rm2XkenlcVlqd@FPnXrHJ z&usB)VGCck`u$R}YUFxGCFAAA7Fkl&;ccbKjL|G;oRK`|oeL!CZ#UO)__n!r;D%!8 ziXLAWH;!}nqydD}#@4D7x~_vr(%aQhr)rmd)K{i|C!^^JxSvGeZR+QXtG&^$?Pkhnqzxb48^4dVAzN$~ zGQ2XGfBEjr-W%PvH6C^Kta*e~)_i8*zl2Ou0K(W6#PkC2U*3>*8KJ{uu;ApQP0)-B z)Mw1&aSXbTbbTVqp~L(|G5o$p&*&r%+WFC)O6x$$QQlDI0mTo2b@)q|FJ#uuf93e37 zR>!qd8f=&yE;T*~d%uA)12Hq2NZo#FAPyB)8Mz_|ug-t9Gfqr6+y^z$0jHr;gCKLm z-4Z3!jf(wC!tH&Ox)^EXnZSUSAGQENQI$oM44@Y1F}cEGEyV|@v5DOv=0!8}YMqOO z)B>(q1>U%!Gxd9C{G7}8qoakf`+3cPsLBRyhJxDm4t7z<#Eq+*6$_}AQzMRZQRqus zl<)J;_jKBLY3zrMHmOgIMxe(TXwgq<`y^RWVD1-!0}v>P7{R*|r; z{Lmh}RooUqBvP#we8&MSX*}IM5L8-H@}AA~gD1_H1e&@53nJS)yMw;zzXC1EPnMX> zdxP?n!mhdM>>^JTYxvN%MSVEc1ijvi?!#IlwyJ!Ag*QjlUbw-NS#DnMTMId%{!!$! z8KM-8CT!McfWxlCty0MHWcBIEaN-r!;h!xdmVz1Z)4+RR{r;v-Or10-Codcck+1bY zwzrw7t7Uw0Au}-nFOsjPD^ClOE+*R@F2H*PQYCo`Et#`kkrtPhj$c!Hw(!JI5`e8V5I$hWp(^AR)?-pjr?5OXt zV*=&DL)_7g+R*^At&ySXFQ7N%i#(9J4R0O~fz9BtW2e3r=Lh@q3H)!S zzwtxFTH_ze42uTMDN}wBuulaI9$Au74uym!wY2Y4OUVg7T6lXn{MmdVH%oC)gg`bnCDa7&`1v{LC}e$)B25 ziJpafDAHNv#E!elh+EntH;;Tr;lg~DKUx#ay)3OXXQ}U7Xg?qRjwgn4LN7TG zg*I3#+9?shLtC$z0;irAZ#&AgPlJio(4_Q^9WC14aZajhT$&|pVdxYMujg#}`0suI zaL#BauN6b$Xi40<5t5*~Ycumr13(r&R7-!!k*F5{K zRx{;jpJFl@{u#jR;_MW1>`EdvVtLfK)DUXN<=4+J6~M4v+SsYDMDNm$^QC>~+sG$! z{3^=k^OMGl5<$PO?)dMs3F9rQF)br6IW_|XSVw(Z9pSF3z62Af#_&EBUv zFp_QVUX;&^G~*ex6ZOr7hpAWyYASR9Ab~FJF3hnhwerQ!-b62y1fTa`cS<(JV`H0g zRVOOGC5>Y~>5V{45%PjXeZD6m7hva1iFulY9L(r0+&i;wI{~VgD`G|uF(rGq&#;!l z;O0AO=`;;Iwlp}Eh(pW{Ot4WACiB~nSf)r4HIkeKT{~cMOIG%&Ba-W3XU_z`cUSUWlNlC2A>s9JNTfcgz^W$QP1<_-^Tpzo}(_a=ktX* z{BToyW{;}NT)6D>FxAWGuFQk_bWQy393Ulmg3$YxhHW4A>LwK8FBdV_TZtAL>Jq5U zR!HZ!qz3K$U30_>Vh5-@AR}>)&$%l%e@T+5_@QVUXOlsj#fP(gj>Athm>o#GY20HA z0#=?89Uw`aT)l&zLMd^W1)@pvlF_oFam?`W(Ozv=(#2 zpiNr8K}ub)Fa|SMF`Q+dd3Q`Who2}5ST*%&+&!!|ea5JyA0_3Cm@!&bvgbP!b0c|X zUyujVu={rghHXn>hQFNol+X1`GUwk(y>Rh(apuwN(Fhnk!5k4XJc*za#qTG$^`<$q|SB) zj)B9c_gxi@>@x91)!{7v5){FV@@7V;AuP`bzjV^d8~bj@f0#CZD;osm)Vyr{=&gC! zJ)PFEacQc=3FMO`N2Mq>T*#}H1-H}8X9Jgmm7t%mzDQ2i4VaXS^BEI{Ne?V%lXg=z zBE7+?PW1b8m~!J(K98{Y*^<9n`Jdt)vQ~FZ0EFpeHLL#SPuivL6asRyfjqz(oLEKk zK>cCG@Rz+jCu{S{ocv?34@AFw0x##zd~YF5N%!xUt_aP8#V6bP?-=Nv;QupCf9gi4 zokK^`5xLZQ+vevF3RHhu>5{@x>3=rxN zuDjy7x`m<|9YV&q_sD(+NOQTK`|pgO_;ZA$+5JW~d@7{E{Zu|U-(GEiu<6r~#?rUg ztqJFoUUsUPI!u+&Pr(4PeRMgCckj2`y}wNd_;iwqEq19SqE&@4zL22f=7jMV<4^8M z4g~u+CEo9!U6|=X2X%bA{1n2R>ou_=U``{FW_OXUD_DqI`ssYhbhQ~7A0O98URh_a zC%zDF0-X(iLm+zgHOl}|4YDKJEQ60`@A2BiRuIzpd18j&6n4YlbxP3TtgaC_9&Z1Y z8LXMKf3Du%lgG<5iUq6_(nY=I2rRKpujB2_Wp?O;>8y^nvj97ftD}}Dtms72vrajs z+-pohd@75qkVetUfunW$VBX#D8$P$1Y+v9Ld61SqFy3;mC)x~3n!7#cex)Kwik;wH z)+SrQcjv}61@6t+`-f&*+{^Y*`IKY)6=ONRSy8Wstj7dfUy{kXCd5e2pIso8q#I}d zT}D=CHYiV2%L#q6wATgBr-C~5skctoL?@1rsjtCGTeES@n!3YTv&C-$tq7GJbtVqV zJi}QEh-cihX0~~P;HRGA<5u`tVuI}eMS%PrC>h>v?*pR*@2|BsU@pw7f1R2i$UH3E z`F-%J%+A^9qkJN**S;p4LusDx&wPTXTw?SL?oq2wJOdFU}F>imP1el=BlUd^@Niz6n}rC3cYOavH%3BwzRe~jU-#n3|J_B;}5lPz`7C~GX{FYwlIkH z(2!ohRDrzi@-fsOa=`Wx#F2sBQ2QGI44}V&g#n;qEa!W+c1fEdMor@5H$oQ5*~)@^ zf*i|w9Bf!3rm3lkSeYbPog|nm$YF8Do-wfzxHl6=Xr}1KFP0uvwXR`MW}l{o)A;Vy zKu$1mO1*vJnaw}Oga>(XG*>*yle)V~rjdcxZO|~nZ`^9(SB9j`3S%8nlvP`qsISt0vEW02=@?vSk&k%bAwtAt zI`v%3-nIoBYt9M&hv6ch<3kjp^J%jJJViMp9i`r^LZcqezxpZ0zK%yRc1tne&;{Ae^zUPjtSGp%VnCkTPcz%{ zFZru^hgex`a2I|GZ5=dbQ>ii;kKeMtMlCw{>vWW**ew-c%2$19 z4nYBr0E&wigXx4@W~PDjG~F3wFMk6#m#q6ngUx-xceI*{ppgG(Ueic*cnyJ^Kwmn4 z?%MzGaNM&QU;lmo)m3ePWkHV={^6F|Y4e{v1Basj@ZRP#z2jPpAFZlJ!JWaTD_zjv z?j{y)`m-+DDVF|ekTrvUKggu8wDA^K*#d_XHiaO(!4%Ll%jr;Z4|e?YVTM3_Bl>hkNvR-)HmT!z<>%&4?^o0&Qx7GGr{QrES3g+g1m_R`!<@2_sGT>9B z0&2IsP97*~UpdtDtz7Z_rC*WFO!n=p19L!Q@%vBab^w0uNqyrA0*-ytPGM}ggS;S( zxkpu5nU>mMdf=nXpMlK#TTHuznKqB0$!O0qv%`Crz>WprO9XnD0O~hB)4WKPm4Q1+ z2NP`)m)27E2hoxw8Y;0W8ZF`HKUc@s*eP!`#tpfiUb9%u5$x%0nDb$_ZniPQ7kf{; z%Rw3rpq*st)j|_HaaZA80=*KEOC|9+E-o&GAStLHl!7#Axdpu%Ec#}zm}+CzTx<34T7RWJsfr*aRP?S*4lD+Q-H_N0T-kFmOMJg7(zH zeGQZe`u1-2&$K!dJ&A^%W7j~YPVQ>&#&)ad5bHs=?GCgSyv4w*wt(4n1W`5?Tqm#r z7XrLn&^^Ic)Z(sSJ95Y)Th2SRjd>MC?Ak>ap1x?kr)OSPXIquu(RlSh;Df;CbLJ~L z%l<6fk+*^SXcBL;f0Vatxr~t?3!rrGUib8~4Vw8y+f?Jj0(ph7bm!iFxAPwtUA%?M z`pc({j@YAn;Xp|hp%D`TNwt^yUP{*tb*pm*)leh$+rv9WDu^CMA#Vl{9?`Mu#@9u} zOyy%`nH^>(8$H&PfmHBg%|W}E?&2)Qs{UVn!FHT8|I0)~^TnO7jXCc{o0Jm(t_Ii; zCG$JvR9vXMgphz^OQSRg6aad*9yA5MC=%5PERhlU*f$;k-&-B1sI}j+5c;6t`gSu; z`zFp0;)C6{9MXH%*5DIUg5%x~0&~xqKWb9t08?ZEC=_}D*50dOQ!l`o3bAv1RZDqj z;}m*G6(^jto8zWj=T>a9<*4VBah&!py@N|(3p&_|%d+1~pm|fnc0=WwLA?7m1gs_$ zpCw6aX_KX5xey6Gii=Q7--@ec#@8Mdk5>|s*q*65xf=HR7EJl*&iI8^z9|S_dfnAL zL)2bdW#wK8lt>w(B&agZ5aF(j31pKtyF+mH;U7N>@vlLffzmVH_<~4yT%3fd{feLc z3ZV%R7F$VPB_qdO&-B%wiREm#Ocp8hD`Q4@2i<-mb%me#W@VqiuU<+N>{$~UJ`1TYnaa}NsOa~|k>Z*o0CmsHY7B!WWh znOpqZg~4t@--YdXQaj?~eHazqXr*x#pFX9%!F?88MNKEcihO3m>Pyt=OPl3Zj=BEs>dqYW09-+~ZJ=kZ05 zeq%=F6Ric9W3>UW&p2R1`!qV6V^)jt1!ebg3TLzeEDg|Zsn{9!vC<6pmS|Qg$S2th zL=QWI(DCtvBJD0?l%_^0UxD(|9NEW*w!>)FuPIe#Zj=RgFg#P1vA(LX_b7_gpCzr{=GSr8WoM*%*3@x(T|#<#Wu*OI3ST-YxblW+OX>JUsaXds5ChldYGz*59h{PW8pcT2Rv-l5d zf{xhXHaAnRk9#T+x(yvOGh~tu_Vjkq1URZnN!5mAx4{dLCoCNUF(~UwFphl73S;Mt zPn;&K2hsiBZN4#;k9(&4(6*dFS*CNe&JJt%%ovyMm`yLXP+#UF1#^*R4lOQ|c#&Q@ zN_WBCI1G-%6XwUk@u&n^zYqcAK_Xle?hx4a=g0BX)aQ873Gj3_&8QII z9ryZatAR*&Eook+)LxoS$HbJQ@FyO=1&+Dfhmud+`XFmiPl`cy$QWk!s-j+>h4Y=b zJ%C$WEtvgb6MI1Hds9Wa?X_hk@k-=w(*#vKsj=1E@Mkm_R#6N0=@OjUfd(dI@7AZHwbD@IiR16r?rjyW5z zmp+_|o+!EQW9VnEaRcDUxO&}%QF1MmVzKiRgzuSfSA^!}lCr|7M5LcZ)xh`f#cZ!n z^TVh*8Cb1SBg3}!R41mJ>)m6e9fK{{nU~arPY%WslByed6*KS|SJdky_(latwI|K? z6na!lYZu?K1ZlmORGXa-Fs8Ad#=*AFt@#EFp2x#Jp(XxOx)t3ydNXHLzSDD^lYGST zP-hlQ=c+`J+|eoNil;Ea4eo{mpQfqG$Mz4j|I3Ib{Ps<}M7DN*RRfvgNGH5FzMjxD z>48(d5G_~Jj`W^Bb_|0X4V`{(l`{2Vn#Q+MuPM>`^lnvY(F_*cVmAh?gvOMzvuWaq zuTK{^Sd7peHf^3#glN|Kkh0Mpxp>7TkoD@MV$!j>{cU&1LbVISoKqAl*>oP=npyAp zkmKM6JcM7dag`alWPNF4TzFntkL_RtEVV$~)uAZXV&&VnJRQz^buqIElZ79kfq}Qr zDj$aQcn3<&k3I&Gg77?%Kn|OS$aLVAFoX9qL`v5Q&Lwuu$_^EGdpb8^s!21zn-BZp ze$ly7Mk9yjP8^?RwTWeR%N_l&X^F%OH(graR@hZ15VD-ogH0Y%Jzq>s?5X6VohpBW0LZ?jF@1+!pPuCMNLh_ z@uw~4UpLAEU>LN0DyI`TWLM2p!Tk^?s*5Ywf}0*BJ&Q%GV!V3o4lFpN zzPx{lNLseI?(#93sOmTg5Svcrj!Ua85LaGewHhc;Xw#Py4+=gln>vXt!mi~m%(0)M zsJb?q<|!YQ*`Z{}PhC_A;V5@YnOeHWx(5^ajJhgV&)OL;5%T&6WtZEBzhwZ1irt$^ z%1w|z!pdysq9foNb&d@)w|_;XP(KWVo9#fH6V0jvn_I>n28CW&-7sx#CX znZYUc(&TPYu(Sj5O7p=3zOpOP@V01MYatEW%dvqrq}f^Qy)qZ#(GdV`LiDZ?Hzs@+ z0k_#(TQ0AciY7iCcH?K9{g^!wxm`m5z^Fu7L5H6dT5ybF#&T!Q6L?F=ugc5X)Cr{= zJC`BNQ~$y_k%-mw{ld#ny|-&ZkIl5cwJ$BDw16w@!t-*cELEejr_8Qu*k*wH_g1bi z6WV)!78Orgi*T3`wsp@^;@3k$DXTc=g;;}ubgQ;G zpXaw3jiZ@PA^ppGyzZNnR5nXqt&eJgOZ*i!?AT@s^fApt3dLmjtDq%#Pq==QKdRLb zsIM_y0J4iay>dTno)UKAjGZolt^Q6=yV!C4in;y5Gp@xrD)kkYE<=vYIl z#D5xmX?3IjjBq5@Y`Mr61Y2?(x(g6vT&c2CMMIwTS!5@2mcGH8N5^c~6`-@fs z{k+2oil5RLUhR2%_Sqpj_BV?B@|0ctp4p!p1W!Gc8n+0t<6zYJQS@BTc8BC7RHkLj zec^V*L~Pi>BQeMKEyJrmm&e*)O}7vIPF7UD3t*lUR6GSnG1Jgs<5_U5q*dt^8(|SB zwRh`s?xl8FG~bt!Rloy236~5W2KBC(jU)B}pArze;~pfri5#lLngrwx;?9}(+veT9 zf28pYN9E-PZ3L^Dc7Y&!rT`ZZSU+=9IY2u@bnX6Ea6ja# zyehwRDh4vRk+w&CL-}~bcKuPVPK#Bs-3X*j%3@u!z^n~X1F0mYO}InPfiZoE!QJ-@ z_APJ(30DP*mydM~-gG-*a1wf$mv5OfpwM91zZ#$BHVTFw9kO$L;+y``oZBfYO9#~( z0xustSXGH?s|!259K4FVGsnuW(ZEW+?QkAis(kEN(LS-4>phCsPt{+%?aACD&bG$> zcq&B(aaE}XTs5*S_fuEb%mmm zZw9RH7u4T~&Xu_%ZZ+LfqT%3^HtaH*K!cwIX|u7me#ycxrw*ErVd15o!5r83ys%N5 zxv@z5e2@3Su;VXJLU&)%|EZl~Ivd8KF}FWn?#*b66>COYcn)#?pq@j(^a*$LXZ7(1 z(D5~uyJ{6D=Em5=qrp`pVGRoNiZ;A4F61;)#F}bqilug2`kYws>okY-vgzbyXx6?M zP<51&%-M`teRk!Zi?g5&W)|FI#~$pkEH%(Iy4p4B;k#R{vtqSnMIY}v9tFpdYSmUx zh-2*lE)MOW5p={C$^Wk0A`M8fIehT2Y-0Q4E0TjXa*_5sqjX^$yrY~FE11fYY#Sjy zkr7{6CXV{MX5tEFN^C5#T@&F*sj`Zno_un__)~>tZVwO}**<<1cLN4XinFgKB>cS2 zAHgNDKHByO&lAV9aaG=5|HR?22Bk%ny{S<0J1Z8SFMiwsl%d7Fg%970*e70GO1hh{ zJEjOG4tHHqGVVzl@)!;IG~-zZ*X`B(LO$ysF}`kX?=ypJpM&c^LV$NN4a86;vY|c1pd^M!g;gzbYkQzd}K{txIE|w>Qt-%8c@EzzG zaN6_N6|_78>C1kPSkM%3I&u3Z%(MjX zZ3KCK=;*&<0j{ZP1J#fg$BtS10WXuwR99aba8P{IL-i;*P8`>A0_wzN;`?@IGRTz5 z5kRBcfw!jfy|Ld#m;gW-tB!bQ!Rb0L;Bs9+J?Cdn8{$a+D`n%d63^n9r7pI*$Fab( z_<36w&C8W+;~fWFLRG$R2%uk#Cb*R6Y;e^$0O2lv&-emdz`ceIT$`{8++iA1-B&rX z?+Xg#uWbzp;LdCg2^lBN7{-(J@r|%}r)1^o>?pEw-4{2OV}A2qY?K`Od@$lnxH%PE z3lU1IB;U?T9~Ap9vM1IXqlfxWmYET-JyGS4MZWI3Z_H9CA^ynJBiPO+*_;J?@Ejj- zr7hAtQ;J^R?}2TRO~5=&u6b;Pf8@r2zI2GgCezp>)tqIlTI-`*Wxh>y?kV^^agp!4 zVj>T-A|+J?$r;?5UBM7Be{wsi#yBo{4a~$A9U+l@(3?OOCcGSJ>?0gVUc95@7T<3t zCh*x)i>0i3evlfu--kP$jc9nNhINibzzQlLMWm_frBWPv^Fg zd_mQTsT#@}^4WC}=gSlAUJ)JzF}puo+K4Rk?i`F>ggbk38)ullWL zQ>HId#1aTEeGC!J?=IMNI1G3#GiW?24X7U$#SQ`7ORW9D5JbS(zFLtv+;f@20 z)-4`I_637E3Tn@zQFFZlDwM548SIft>FlR|Tu9~^V^aH?E8g2C-kTs^khS+0HD||x zjDfRN)yP+2m`zMhPv2SHZV8vgZ8zoQ-CM!RF0Afr_Dtupbf6r9Y z2T=UB)smCaW=qk&{46-Z9KOFlDg5=x6u@yoIRfYX7EwK0M17xKjOjM^~2LmAHW0dv-B5aCBHpQT#N*-P5}0T zD^li?62Cf-Ss;)CP}Bz=??X?XpYAvbA|HH!Hz0{MbhP?gZ=ECU0oeLz4=CWl!wwL_ zkkO%iQq}Dh%%w0T0JKjhur5GqqQ(21lt21N%xUku;&%~z>PZiiG@t&Lzwcyjz288m zDUp-$6S+@u-%I8@ZZW;k9*uWtbDA-RJ7xU=F8-G4I0TZz4vw(s@ z&=wd7-Q04txJakVc$2Pg?S+v&N`Ua3%{HFf^e8&g$F4;mNNXf)BD4Gu1kwUD6lmb0 zQsEY1Ve|(1VZ=Awshiva!VQLgMXqmL#5~h3YY(XZdqhj74oXqJJG#SxY%BtrVcBfP zXA0cds6Hj}v$R^PWY27$PZ)SxOFp1uebtLj*{jLk!blWBsqW|G0+YTofW8XlZTI)@ z)IN3%-9e!hVM3}hJqU`O_GHstSeKa^Zh*CQ?<9ZQt_~N<8m*2Kaw=2P0%O>tE(lywzXkndResTOmN2j}^AI7FynnqX-A1e;GVn;c2&2KCU`+8%u zA%kmKIm5||kvCqNvvl=0ze0jh1;z4n0bh8}&W@3QKsqb-b~sPGD)bH??Dk9Z0mn&* z{Wpbk@dmKrfr#(+}$X09%Q@Rs-MHuO0zY`}Rm23+|C) zoZn`v{#<%xltT+e*p-V{H!vg< za+jh)uLQkuqwnW>D!tXr)hq7Cl~l&Fg4q_k&*=aw2c?OiZ@uTt4%Erz5*qS11S0Mp zRZzG^)v^7oxT_-9XiA&55~@m}2er78pwpXNaeEFxBHDAa4t$^yV}A(*h3?5=e-V?x zFr8z3ZM)NO? z>`xDT^>I?ZFZ~o!cVmOHzMygxOG5e)u%iwGvfHX;H@gF{2X5tSE+-$^3iVM~oxy+n zeoj(HFB7>cnkBhIgeP-Okssf(^bW>8Qe(vij|;K3J;Lm8g!F=r!3-J!QO+$QMihRt zaqTm4L+J|=kF;ShYNf~lu}6-`Z^O_VD+ZEl1>|o9Qeq7klVUVe;fE1C?(^3bAFx4O z?$a0PNiaM-fvh=`bW7ws;Z_@F)bcrI@tgAO`0%j%UP9qdQ&sY6fPESUhhIx<8kO={ zgSfH&$&xdnDP_~FZA6}QBE_nbJ3R_p%NoMb_p`S2Za-$b1A3<#7vws(y01{d&#v@9<#0`Ha;5PuoS>#9<>e@3 z0PMJ661D{6%&x)Ci?d4&b}9j#P8**0T;g;p$?41v(9;Os&ho<>k~bK8;ss2EimwZ9ogNi#Ld-3lK4|H3NN1V{ZecolF$^gdwAmS6m?<3$UtmQLzK2 zh!|1h6C!YovNpb>?qU{8Q1EXetOW#1$vkBBa#LA|SoH6#l1RYhatVzWsZ4Ow(b7f%tyTAlbh^eJz;3h6WpR_S4^^ZJ3XpBBH#vTkHP) zWl|Y^*UP4t_igg`MUiJaXbbgsTRxWPJJGg6998CjlOdLn%m1#n$Mgau8-&7Xb5po> z>2@}A*7mm!H`D@-Yvm)Dt=Z@u>3o_92v;N=je@OUrn3cuYWF)%@J@7`+-ALl{(E?j zH_<|_KM;sl@hLWIg*y69z15lgao7|}Uyq+i#q?7#MLYollsD8|hU+#&B-p*3=xWLu z_jEX5paz!v%wAG&4}-0AC*20=^irjp6sF0iLdE)rou%&#EuD57l9TyMyxIHApy%XK z9qPz-PHS#M0?jn2;-inw_;_WdSx>2y<(~41dLEq(zkwOvI+cARYpQP5%cp)SCcSW& zW*6fKT1hXY^GTL2BiTBFI>FCusneq^qFsx}p3aoOHx?>yAA6w>hrPgP$S?K=uHoRp zx4jvUr6Ru-j8FXdv^>(l__iW$x>#5j7)-Hr`|9Iq>a-}Y_Ls~(cgnqq7kQM~5>LhI zbzLSB7pxwS-jwIo%&m13cisP*2~K!MVlc3BNd1|-^06LbO2>+-3Ux+*$>Fp@BcsLD z+!OywouXo@*BaUTq(@!8KXSD)P3Jlv)PN>OvbNsrv+#t7y`)W@^#vTn$z$NKyVX7C z%h9pTYT;*Drv$WlOoY$~@|#$E1z1|wzvRIL=Df-vn=2z|-_vt=g9EwL>Ze`cw)H*g zpD5FQ1mq~=u@nsy;-fO;hr*^gCH7{shdT3R>#}Se z(@#hYZb@5n@SLvBjBiZ}I^?C{E8K?efjgvoysdKh(bJ=6UH!g4JqU?&-?a5iPt#U@ zRec#pA|%8wy?kF%0$*xys|}b0?#L_rq&*J{WLo@=>umJm3qOKOsP_yU)Y~-rwCQ4F z`ug$os9$s+<=DfSZH`OchVt(Gz)BLHMj>2Ts;8f0+FB+RUSpgn9nCW+s-xuVib%NJ zRGr06gQgOg0Bs??r^~M%(?(_WlRWf&hl~QR-6^#pnlfd#+rZ{lazL=-qiCbjn~`P$ zi$PP%x+$3GM5WCkp4e^}wf59H^Hk_tnR?O4lMUqC+!v!`dKcyb#y9$O{RIruBN}86 zvb{~}TBP*)7M)};-8=?BvQ1ToD7|+iwIQ)|%M^fr7-!|!WZPx!4cb9pwUDaN4oA9q zt4&`40O{jHAAL>4f-@e_=@3yPG=1TVZPxWdj8AU2q&{1r+N~XWU#Y*8#FNDx6ikl5m_6u5@Skqt%O)Eys}v(c%&Ld@3tXN7e^A(^=ceN^(_pcHt`Hpmngz2c0n zM4JH#rbp=q8oGuZ&;dH0G)?R~bP;^Ga>e`1?6Me5_1c4CJ0OjMw6B6_#dQEkk6usr z?;rjzU`g*jy^&vMS7-&8n_~ChpjGrEmVxkZ1YP=$yyKrg)1>a-|NhN^e{4M(*Ty^UQ{|yy)@jU+CB#YaYS=(2d)T&yZV4YaL|>D=Wjs!W(E-6Vg`L&L(*VT zksYr^leWX|%*t@{sPyeT#ckvA+?9KGgWy9hIqqG3JnD8bCk~l?42Jo1R=sIi$N%9H zm6|=3pYrPK%YK)bD!OV%#~EOO|9t)?uQms9UvBi z_*f(0U4d?EHAs}?XqCl8MS(xGRe3wKEJ|n3*4+bj-SGkZLg1FVE-x^l(scusH0zZ? zia0fAKTa=D#8!E5Kv^R}GlypN!0s=4$169_mfiBFoO?GeHRTIH>Y5G!RW!i|G9`_L1~he18v zjrVbv4(+qF$%0Sm%~KSPYk*;;gEdXGL7^dQh%MckO1hb5Hu@{i5T_RFzLMCs=I2=Agqm9Y*-G^u_lad2G^09s z;)U*x9kvmzw~m1hRE@X7-CW@Qn#0vu#)TBkff#7?qDDfc0_$mPW$bOdJVA5(2O*L( z^~fXV$yV;(g0fad&mxheut`(AZ4ayKCohN8PRHKvi_BY?%j=I=BrP-06VTKGyZK6I zOrLvkNECgzT5ICVmN=*P{IHhr)tp@kUNv*YE2X{ToqId0E@AoDKIDF1-9tQTVzo?u zEdI6X0R;6-+aT?TR!j!OXp0*)wmt3;q#!skjusnfH|c^S8MV?r)lg&a_jX`Ms5-KI zMGeoVvq)zb+Gb!wj1uXZFKB$_-8BSMoSv#mXq~HDrv8+Bt?xjG9Q&ta&eY92#Z4i- z4cHm?q4|K7G`d()a973^g3@X64c&E}yIUR}#y zB90iaUNG_uH%xH0_Svy|eT4nd^+w0hw9Y5D~`#9W(Rc+ZKn(4j-;4 zu+U20H#z{(K5}1TOFjB}YVqM$915TYl7H+E!XTyVIUG$T(uipVWS*+^pz@Se7;M%Y z0-<+OSD>wXG;Ah+DM0}2qj{!d_D8%2o9ONK<_8p~Nf71js^_xfbBxf2Z--KF2E3#=e+Z|N}2I;!3BMWo$ zm8+U7RmpR(K8-US$V1j&bd|{ZdRtUSPh`BlCuk|LqFJ&|AE+e}^h6Uic{Q~7gy;{- zH9iIPuA@W7#~#v@7W;Kuh26_2ejN)M{shWupuPGIATl#QCvOgR%w{bmCMMkB7eJz3&Ss2lg+8V2<(g!www>K3EI4CUE%$7@zYE{p_u7`g0aq z+q&pYy*7cv*B^Q+SZ-94ZCvhPX{Rz~9Z{6irwqVcSu)-7$k0LMjtAwQ#Ake5uOj1B zf(!H4c`B3Rk#e*$i}N@y@9nv5|UEkSp4H;C5ePzEo9TA@7j8RLM1b^$w3DqApr7O>B!s zsiPM%gM)C$xm*v+CNyHeAD-~#V{$6(dztlE8_h?1!L{;Bo3LC=waZ#q2RCVU`=E~o z&e(I!oxV03!o`}y^KH9Iv@;!-oVA(i_02K7GHG5*ArP&#K@OyEbRb`vrm15pAyagM z8-bj)8bWVEeuryL~-j(-SVnRrP3BLuFr|vTDxCu{m1j7SLtmu=UD#nEq4GP zIYGItx`Yncx{Zw<*7EXdEe|F%eJJNjo!h;2dAIDBshrrso z{-p<2G$C9M0=A4LT7eT9u!9N&DYc1d9`aU<@N?cd#~du<$*`ieF1g!brLateeP}d> zOEql+sU7yYpw7lLcOC024(JyR5WK-k1K;dZ?KU2Mk6i<}0k3S@HMbB^`c~~b-NJ_* zG-E5#$FM5jnjcqoku^_GaU-b`vMkWIMaxu?FYB`TA3S7?F3{Y~fkuWKgyWeG4xgDcJ`4V8uhVQQ6|7l;8xH0m&zam!{V6WI5bIjYMcx$~4ZOpVEoJEO z%sspZ0y)htRLGxUst5bMZhutUPfG&+|DVtQ%@Q!|iX^T5GRCPHAX>xqx#xZvC(9a6 zN@7ian(#5E&ObZ>8T~SIX6BeKc?V9rz;rk%oJ?EO$z&M}=?c>vjis5FumO9A08es5 zpLxkM6}XQ*Z>5DF2aK`qN(HoFnpD{fdwf?0;%CmZN;vf>u+Xx+&~zFyEQ+hyYpOEC-w@>rwXLAy`3g$@z$cLZ$^eqphGU*!90LX_| zR}StzJ*X@+X{5))n;L<^eAU85Ikw;In@T4iXR@QF04!8CEiok*;UyoLj@sWGqDr;PR8O^K=uAyt#AEkZ4B5 zkc}12gd^K;avf%^V#HjC*M%PLTP1PGb3%W~Hxfu@%lsT=8!%r_T02uq7@K0v zX4Sbf)P4Jzq80y3v_i$sqE^0LVdFU#9C^}z?m@Z$_ci|Iz5rnua_%G~f3=*kDbsAW zL`vKqUY4W_F#L|CNfyJN{ImD{&$%ZNYq|?A0Jj(dzh&Xc&yMPtFA^==brN8KoVLZn zYlUqfF0bD8>bV&-cqvG+h{^w{pK8!%Ezk0hTdD=6oBTx)=)+%P08$Hl zN?F!m__fYwKm&itbf#+z`9`57&(<2?EHgic<*ZoaKUi6Cg!Y>QyjNN4QcsD3l_gd) zrTIzgkQY6`y5NH?1emM+X4jtI!Vv!+tLM41JiPezNEY=T7})@4FAlJ#qru@s2-tzjB#MToVK!`cqisHu(pZSZgCVUht@uc*Rei*-?TkzkG>|xj$eI*sH4B z&$#d{h4$$%ZKpd5WCZ^iI)Sh=2O27=VE1@qB@KupPG*h+%C7Mm%*!S9lnH_?%>T@+_I*Ii{6}9z1cu;<)*482q^u6aH}8HNB#w?Up4+J{$|JK>Ix z&HeLJ-~^^A{TpQ<(5V+zyZ(iJpqTcp>t9I!&wwoz{(m6{^C966eI9Da}9hD_8>q*d%~SMyd!BL$hZ%=_{N( z2Hz1Sf@@}z9oa^hYQ#!teQ30DaC|g)g7vc|*m+t7nF1u6ADTy&L9`FRY`490^M_*B z;y@lWJst)RPNogfhQPL_gZ_YX8K*T4AM*cr<>#my|J^78Fe>+dI|?^^$Vu!P2Q;zY zMMmv4>=>31^5O#K%RJAb%%aSk(g_d{iaN)N#(s~=EnXlK%F+)U}PNYfgvYj z5Rk=u!0BCvCQMxrIJ~t`qvR{ORKqgc-U~iV@iojh%rnViQOB+w#(oGOP1AilJ4)=@ zLr{#*oBoZJ`4KR=0~d2J6dgie#;2`bxV&!NSrB;uw*71ZXO_h_l&Qu0HiRdNodKw0 zotFm=c<3eQS^lBw&Wqf0Y6;&U5B)+JOL)X7a&xlkW=PZuh3}x&`uf6HnSg92`^5uB7EeyL+}7X66b9`v4+kN;88g`8XJ7NlcU+2?!U_kh}D&Or<$2 zy&`C!f*5Oll;H~!d3AVMuTTv(8_h@qXF}((Z5G;L;ie97aL~Wa6Ml2LP=}?KZ#`rs z$h)VY7&9YxYS2K`jz*I|_-*maE9DjnkGww)Z&rQb0Oou3b^T;hIs2=QDKJnAEv3n@ zrH;gn2>#Tu=QFefm1s?>q)Uj!m{TeP3$7Vh^8kSgR8^TOGGy&~ngf~#@fM5Z3HYjV z(=Gp2gwnIdp@KYaMnmp02Xsm9v|K=pu6EcUxcT3VO}rR^6R(ILsnG$I^lmrf1CGC6 z0;e3Pv=&DNq-(A#Hw{>X%TpVuy5j%jo--$ms$lG$@JbJ;q=v9cUt-Ln$3d;&74kF? zG;i5k-T>CXe_Y(6zN={1|6YzMBX|+m7Qm)>pc`{OnM{_I{Hi`a|8^{7z^$gtUjT0$ zKIL51vcEMnLOm4_{cgJ{g*5*pUN#BWp;N0k75pLZ_r8?LWPHJ z-`}w)Gc^JB$~2bva&6b}0zL;~c}g7c5b5=3o10C!Bk%DJ{Zbx3p1bR7YT(H`9s$Qt zxMp%?NEYPGM@g($U+SlC1+WO*CA(p*og-rz*YPAXt*l8kANc_)#MY@zt_xvr-zJnp zZog+u2px@iN*|45Sir+Mj{{pMqK1|$BC|V6(ZNk%qyZ^cQe|TR>|xF|L0;NY1`(s^ zKj@z*>~KFAV&|sUBjWI+Xdx)HO7K&P6L+=wDfPqrTrV)zS@^AcjVICU@kk z7}bKldv18O3w%Gl&UBMX_>ONSre8|3?vT=g>C?f@hd0zWJjXVwcg_2m z26noNUBL|fK?<9zd-d%CvWQWz1JmyJ)3W6gJxLWaC4>UZihO5_xK1h!g!;+ zQSFE!k&tSQ&&S8deEe-K-nPfx3qNO5*Z0T6hev7}E$&?clFga#EMk}nypjSj+g(3l zlVWia+gSQ$Od_pD-f2j_7m+l>i!N;MUA=;SG~Rj081Ov4dgCt%FK*dgV7lSlwKcPf zQ)=V>5Lg!s?0N44NZ}w+UT|m<91Z-e35vf%_%G2`tc9sFPYH%C1-g&#Sr0FD2rg?L z!vYk@&`L%WN~NxSmF9$Ih=Z0OKN1W28_U9;X^)(>pHoxu)eizH<5z7DzUp2t?Wi_r zkSVI%?mb@)P32Cq=TBa?wRO3F*Otx3CB}Pbpp?;>Ayu}g{@M=W!YBEq8kBVw%$6Nh zDO*^V%QAO;Oiq=p@)3$NSbwB)B?)PLpnf}ak|%62*jen8uV5%1pH=7pKGAt_1cowl@r%YH$wiU5S&_laylM>&DZJaPh0voI~V$N|rU*(PNhx&rbzDkxXbfCg zD)%8aQhH3#r0|33;Ei(L+b>UZ%Z8x~j0?zgX<=}KQZ1n+XYkJa7(biMSIlMw4bqFV z(?*vmI@RLn6dq5iTz=olR1EJ1;K*MWnaMNrvh0uI^g7D&F=X46n8!GHnieJ1K+%0YR9hnhR2HF;gWq-xGB!Nak4!zGUIT44&TJY&N0|v9E9%9IjPGIj)D_ z@4IiIG(1&=)`d?3GlCYE5$Z>%t3}8GB^o6*ct*bs!y+TTS>yK@MW;lXo9UjNh zJoe}xaK{>9eB;P%7o@`D6#m572c5IY8VMFBbWk3KU}Dr%j5M#5+`-%_=d9($2Qw=> zOT=Zc$rgoA6z?_0o5x-~Z+FXK8%h!UV38Ag&C9l`;L~X{3GH zGx}WNNj7=NWS3k#&SBVZxVl8njU=W2K9{i^5>tjgBs=J^?mq2Ow(rs?MM*-yR>_z1 zwo4V%EpTw5Vy*98FmY1J*GR+gI?qx=Zf{a_KB%hxsZYJpYI%vcLe!4A3xw!>*N{qj zK-i`Aal)FL!qC$yTOHro#bIidclcj4ugpTgCM(u8T&>E`pnqPA-CH*>!tTb~<`&#*ifT}OIxAye%p zZe(K#5z1IoI#AVt&(5u17;&jAlE+ve$k6!Nd|^aFs_LK-&-w!1KB}(Xf(K7_l)30)QBx(=c3(O-~Jf6R``kl}4Ol(!CB6M`(?A+p(2C==v~37^qO3yT1= zM@n@xDLIj{12}4HhQ{o(9dm0J3JTJqkTR3)l*jAJBo1iRV(tPXGuNw6NQWwLwS9!j zxv}7HxmLDLpLU5^ugKBnIMw8*x6-{JajI#9_3DK*$u!)A2lnH z-JQ%c zuIDm3e{5fWrY@nTyM8L92*Re0(U&>U0f2RfiTmXt6myA#)ZE((^pq_9out)yS`T>Vn7*dy4Z5kb!hZu{#C5g>NCt&r3q+gy7EDS&Cv9Zz5SBu;FMMR|S`i&@nyXp*R^VW^ zN-^DrQme1`%GPM9AL+PoFF_85RF8GOCQ-3uDrrHuG6f1*ikE+AD{;( zF{XF%rN=tjmfx~i$jH*F0NhW5Lom$GH!ynr=+lf`)5q+&A!VkbLs0{;9zkeZ0fg{% zVq`2)?rGZ7m9mUihz20zVQ3tr`#pk=tsvx9PT&w@LDD$a`4nb`aRb??wih)1_*V$z zFH!sIb(78Jgl<{MhhF5@mkJzZoBy^g|v-!bqGL-Vdn|91) z-9Qn~(vtS*JIi$6Nuyz>Uut{sd`Nl<<~Rr(P9oPFgnO9Qibx1Q(EJSO1)2MBOPs5( zKGYf2uH3!pa*DPq!Jfqt?rUK+^Np`iovh zEb?5eb!fE6LD{+pwZxL$+dt*Vu7M1Uaf%g=X=H5 z<`qE~z`LUSs_Mg6r{f6@%TZnJ^M-#<;y~vk!^FF`LkmCg@7#0UKw+a!($MPBo}CGW zY63fD3fn{wX7V#x`LV-=h0yj=>|TK`sFm#Qy0ozeu1Y_ik)Q?8H_&RR$GpS{qb|sN zDs}Jth6(*%h|!qC+Q~sYTF80V^5%x@+mP@F*CEPek&dh72lrBA%2wZcopT(GoLDV% zXb^-3s3pR{qhJkE8U%LXLg2yG_~bkMiEEbwXL=kc{jjcXq1C4c=Y!{AR3zaTHj-$1 zC+NK~f}HcMV3{8%>%19OlxEB%fCn=-V?xCQbVf(g3+N|F2+OL-&b6=%v!WM}`s@Aex8;Qa5Rj#g8nQLa2{6ZL+ zN;GCczx*Sc4a~|pBG=pG!>!c*@7+MnlvzJ)L%<7tZcsa&|8giCP<2C-4$-akMo1g= zih-d$zaz1!NCR>oOs4wgt2f{JX?@2pb(BDocaqu=CWm_N`CaVrqG&Rg0(iFm$71)M z6LmlIraC8aU3NAozESun<=c#VSkK`-Zl#bH!k?Y-f|3DPi<<-nTSu3)wdK^x} z(*8Dz98&LdZ(v()aDM@WGX=43{@RfDiLt@~KAf8<(4}bl&y_lbH)6-MbgKC5>7$l$ zm+FMHRvGVxiyIJDk^K46vzz z4&q}Ql&A&eL-v$>D~OMf5$g%T5^19Eakbd9w!tmiN>|E`^3^xw!-icKJXJ!`$JkL5 z3U{~@Flh_b&vxDCgm!RC$J0ee&(O`#Q63haYMpN*U}I2LA$hAjoC3Br^7)ZS@4lD! zMA7rvBZua6j}RyQx}*y!pVaR?lVesBxaatX$X=hogO~HC(T=j(70s=WU0uD#+W`$r+*-j9Y$PJ>(Q6cnID)ZibGgkIBx2A*{Y5zA9`){wgs^^ zT`tcOs4A}6a6-dx{TcD}W!uNB0lPQ!E3LB6^VQ?n&?`Gk<#Nf z&cdGw13uOYpR|P7h|N9&MmrXnIU9%xq1;rhfD}Pa)bAcoK4NoUbuo}#L-rIs}#0|U0@4(Nw+Vg4ULPj~u52ku?9ZiF@A3wsOMv`^!9t7MpT>^$W%d8(MrLk|zw#TS=@v z(&%X{BVMJik|_R@AO+0cwhLJIw=uO>AA!DvyAWw~!N0am9p~ktC#%(j z^!P=jM#iVNLA1l-@}lY)3lf~a<$AV4j#d@(Nrq^O;T^U={XDHpc=G^|LcDsQan)b} zVW8S&7+SU{2)!frzM3J&F@r9T?K}DA(NoUjENlO5_Csf)&ngn-WyV)cL$}LVibau? z53;n!br!IAckg76%-7(DE|5rW#Rv8mk8)plf(N0OCJx(< zd)j&e^QDOV?CL+Z76V|QAj*bWiGruso!`wSk(=CKZ-2he@d$glqSKIGn@~&?;%USu zk0yMbYiYINIXVJ0tiT8%(kITje$uD`#5Nmwb?jcQx!q3)s2!-3Gnr_ac53VUu%%|c ztL~+OYl6ScLQgxa`x58Y=CuRBxHg0i<^u2oX!+!tXo<>6>^6QT0Qkxh0vID;<*bK| zzqh8BUf=R<5&)u>8&O+*%LW4@OUV7&55F~5|8Fatw6F?Xn(9K_i_n%k{o4ur+CKps zycS|W{haBp?tjZi7GasSaF&bMLeAC%Z!9X^h1}TMp3t)D?x7$Cb2XHWE@M4Cq^tWl}UElrOwx=q(ZVIl> zkKkGX9IEPaV9i!@t2lxXP0Gu zb}lesSQmw#&h>Atf4|+wzl{9vt}zuAqrb-L^6$LFgsfOP1=^nA5!_6KkLkLZjb+Ar znBW}K9k81CPf9XTQKl-Gotrv(96CRFXO0Crw*Ziq=_XlyoPv8mKV3ERX1XL3d}M|` zW=kdL#>M~RMd`k*HXTss5{8Q0KZpl#EZD9l$>8L@Jy9R=2;0r|xgRwMQVgCsVYp7fFt8;*@Dq)U1 z@NybVD6ORPbORV0pp9tpq0GRt1L9U@%jo5sUS>o9Hc^EXfETm=VO!ozH-vL(DufvL04_BQp-GYZ8a5$8F7fE+E5&^F6nxnUL8KAeXVPmV^w%(imw7w0@phf za*npFJpJ4hd%mGPuI0wLPL+5sl^eW8^}HLyAxznS3hhR0I5n)9QEAZto)1)eK;F+S zP2mZVN-@LjjHe0Yw8d)D@DdRdN~CKspXkw^29xXaY?&bq4*GiDy^hUB{!Q;0hl8^& zEjNobQXoEWu8&W{cE4cF)* zH)`=I4UExY`aagy5>H*qohN2*K<(IiTAknXAaQBsHzG zitr$|=V2+JRy*b^HSoGyuU`KwUIFa(#Ae3i~0hvuvrKC4sA;$l~Kln61ha zz^fEi$(;FWhn?4di01fjCGCEUZ4mGahW4jey(MYdp&P3|YQobzh-ik{Ds6tMD|d4v zq?1mggu>PqCrJqkwlElDt&64Np^N62q`y>bc?B-KEnMVA@a9H!4x15gll|P@ZLgs* zau_=)e6p$s47R>9kEce9t^j@+`0Q?7>snu80Srv-<{!m2%51*?^m2d&!%~lJ*U%M2 z2sr8a?E=6d=J0SQhRX)fa>Gn80qZlwGup62OVfBv%i`0<4Za`G!&1?ojZraZHh$)C zy71vQOSu8lTN?`N^IRR0K(I}r)bD;gK3g+zyyZrOQl|YWl{-QAd&D$=EkWBGmlm*Z z^Ua3V$(E}TxO#jo?bNdJf}H5q^0+;&N`*)zNAicPrCd^APeBy~>31vd8>FamKuZNZ z?)>y8D?V+ zUw+=%R=+1?zTwpozbxJjCev!EYTtqqAhHQon7;0=RQz-S^2bo@DI-=v9JXC2veSied)-_|cr?0|_|MF5ePxcBm}hn@D0 z*WIBjT;|<9{_vjSM7cb@tB@=V=8ifXG?9njw10_gEE+X`JR=Mv5}>zMUsnR z?}kBli#2;~e$hRHudkhx?to51J@?Ndpqd-x&u(;D0tzU@7_k{p5#nT`COHcaNSM+s#fT`|8XpBbSBK?|CtxuA;A>1#{V2bUOh z|94g(b8NOz8Vt6&Fovgq4NZ3eVHOH;7bUS_1EL%T(IJ)t=DwFezNOY-GSjTi68@p@ zv{g+l{51d`lQCR^GKG_kb~G8VoT=J|spZdrV~`~9?TM;ISX>eT7x zlLtRzPn8!%RQCQ( zsO;r3^ZKz)-{E(4kz_Lyx_s}*57tQ@$1as4;<0>w@Rbl!Ch7k5M7-fAQAT@@x2ij zMP542?$GVtJdmYx@b;A88d+@Qm4*`NKID(7FIM7V zz8eJOz9O;Jy~5jZyQ%K9j34j69E&LRe)?WF`TF(sx-Jq3TQgOh+HMl=T;CzR|1|0N zfPmWaE%f_4vPxYRJ*Bm3*=>bFRzXJFunPd?`@IS@mjhJR^uDXQWQ3Md-VLxC>ex|@ zyXG2BQD_6Fr#-d5U8uU=xbwYCZq~j(72nD4KQ^!Qy(y{vtA1VDtpL#7q;-eU^Vgd+ zER>Em=%%m399OHGkiLV4ephquG|-#hoO?38b|7U9RzYJ`Q zin)3b|2)FCQm|}WHNXY;ui61Vk@?p3%Ey)c9iN^;o@7Nl<^d%gub~+moHyO6gmnR* zOE_;B5L#TINOPIhc-VNk&!|+#*?+j#K0#yW>FOA2`Z-%~-+Y>{hLJ^*($vjHZ}xPd zH$;rSaC7Gr8ijvF6i~B1<7ddoWpFNFP)Xyx9FMO7D zW8W=@2uzPWh<~Vgr=H27y`&D)mf$|?IYys7(ekaY=pB)&dipsDF;M5*HE@+zqbsYN z!OWNq*lc>;A|KaUfcKYqQYxSu<$V<%>TEh4f8JW{Y8iWW<}k73^<(GF%v+Z6?AmWm zmm3zZ?5++!2>PJ68Q9W=$=S_{M?m5NQN}iA$|O7H)eJPn>6?nlTN|7P!Lz#Z>l(xGVJG)1WI@IP9=Q?lznF|A*Lw$W0bBVd$)uF-)IB9kb7L-GN;(# z`1vyjyWRryEg|YrhFfJcDXgh1JDhrG$b%o0!X-BN`*9dm2I!TZkpUGjYI`egn4|)nAzt1v;@N6J)C!IgZ z0nKK<*it%?N%?o`%5x}0jP`bWAYBz-V$6rkGo178xWN-_T4V7gwE&KFfErv1Qh@lZ z`!{SPQqNIIGX$;puS-2d+)nLDunl6~-!=@XfUWQMDP zEjoB*W!}MQWv^{3M*HBNnWh~tz5to<#sZYthT1r?ixWB@=t2-8^e-h1pB+}lgwT;| zKFp@$#@35-@n|WS)+0*?r|siFMSHo2GUkKBS0PoGn`7~>kqR(*8>7Jd0Fi`c-POeT z>iNVFHX9;c5+)TwKeP-EHtOSYZPLFH32YHRiIidlD;H#)Z&;cgx<1NgF3}zP!|b7S zCO6FGhvBM^sgrSBhUG@$JT%}?H_(HmEqvRAsz(KaP7~Iz~X&ACUJmAUe zO%za#N#U{__%B8GNKV=_0)C9ow1NKwA53!H;Jo#Eu*60!CMvG=XyZYpjbv@d4Xe_0 z*|g8#$9nT2&V)w0(-LK_RU=E;{8KteP5#&yqkjD1x0>R<(~+yl zEYb|moXL}YoB5jl62@caO*Ae&^e7~kz7CsIj=OIzF%-9{vZQGoG`K$BG2KK1<3lFw zwYzLc#@Hk-NGT)1!)T-5bWnH)~d}NUVEauvis(ke1>wxJ?(KBqx1~-j|rRQQ`+wh1pWl5AODt%U#n@WBRPgIPmKLR~j1{X|{8J9wh>4#K z0gFjvbDM1DR(J$eV2loyw4Jdr-PgwQojvtT>Em)>c^GrNuugkz4PXIx4=Cr%AFCP+*Cgu9Ze*+Citg# zhgLn0kDf`orFI;vKC0Qagwmy9>FcXLOC%Qqk9!P5ALrMN1;=1 z61ORr(g(%}gwjRF4IfjW0YkbeX`Pm)eF=TA($PAeUfs_H1anUbjf8(%zP&p{)!a4>I1H`4M}Ct|jcwNfZUaw~ zkpCsO7bIS04jpq0AItdeKCJm;;m#S;uL8D_n^~T6e6UKSHs4DVtf11D;UIhU4u?2{ zo8t&+v;tat(;^@SwU}bC9lE+aPO#Wny@1Cw&Jw^I?vlacRP}2MLH%QdSK97<1!$UH zuu&!7?e$h)oRW78u2V@UJ&zxWdg0iRiF<}a3|#iOOLy3;!;JY| zOS*+W20Pv?WI!V{rm1T>?q6MQ*_GDbnneuAsoS7xZ&H6?Hl{HW8??|(+K|9HcM8|LL9$Bh|kppc<={h5OL5BVkotXWY-Ho#kX(W~tnk>iHw*fAle zmQslYIYFyjJ+|^cb!i1%n5hDwaxmuXT%gy) zHb3_@3+P29^_4dy{MmMf4IZxD+!yiqyzFH0B_GKFx@_8fELRB0`(UiYLm%hmehW;b zHIBD&fo8f9z2G(BN)cOqndUsI#B2H5IdQO~Fk}+;GUP|9#Xh(}!mNx&mVc+o<`ll1 zKpSvh9?6;bHt^?2??%vw)1d5k;PoM<{8GdERWy3jbS?{Xub%5SO>&yA{y1Nj>?!D@ zf#2IGdGWjeLA1X7SiCyC)&w z6nN(-@~BHsJdA5z!tnOpK9b9I&hME@3t~Qz4Iu^GWVvgy065m zv1;=`DY5GLDTORDM&!D9(+=|6iE=q2I9qzYc@WU%kohiI+Me6)vfM~n4|eMZ`=#i z@5B$%L-<8F>-~uI9M0=2abPbt5Tt7lk4$w1!#<1)pPZzTAeQS>UUN!|?4!gGN7|AgHs-sEmX&AUs-_=PobKUjX z*Y6C-Q?y-kJzfNdMtB_j(Yo5G>kJx#`VIC3v8#%M24)nHTN^GlOdJ%y$txP@e@{sM79L+Ss&8i6%nZe|>2_Er4LMcx`_BMGa#zyW=HTja0b!@qiCME|-5+y{!NEI{ph+m3zA zda1?Vkx<>gA)$Bwj)d;|I}&>MZ%F8wzaycK{|3Ld@&65e-T!~QXpBQ8>l+q8ZOjv# zYK8R{Ex={EvJGRuQR`VO9YFMIm@w&S3po}}AR-m0OBt*VnprAAS_=fd!(-~N-LZn8 zHcv2%xQa?x0}Mu$eu#3V5%>}=Zveu8tpLDg4S@Fu79u+O?wJL?Q>CY$4D_Jj0PiH+hw0gzAz%85pKjZRIx|QydT+Epz*aLD zfOCJ0;9_2}S1BHmPfH~KIw#USmAQliS z0Q5mW86HC37nnGCq+mFrH{=Ve_OHTgopL}pB+WiE({u^3%s@Fg?HdQYG6z)rFz{l> z;%W#DXEA{Dtqx87R%aj^b1S-hzpK0AK5b&dOI{`~2`;KXmZiOJFBjIJN=!Q4>-CLG zH@fc27`~Yr-dg$+JIecM^BG9!Q@?T#TE;+fA%VZqroMV6s~@F?R%NdMj=H$vyZT;& zt#@uR(Q)kBIEUq)aXr2hK#eHqT;WwxC!y;7>-(ncbBS0~Y5? zv5eOf%~yAHn4}8}`(7r>$j?Qm8t)H4NNxBB%LN+y$g>+|{CH^sRrAjLNTZ_(X1a{a zELrMqLj_;5+{dNZ0r9U~LLE#C&A%i2PA5_(rGB68b<~aprd*;CGmS^RUidFYWj%B= zh*^J2RjzeIbMLtPIH8q=hcfY$&me^C<8B_R$=$Nit5qNEwRx?-Dc@N&WKJ+X$us9Y z`DNy5qhX7LEAS@VC7WOc=H_JA;Ht<6UY8xVo7&5+3qD!O-`7>U41mReR|B3?Ln{Jc zto1Xk5iRfLrjlfPyDQg!1vqPcI9PxB!R0o=x(ggZ&asbl*Va0dSAb{@yYm^?->k=h z22JU;bKh}KjTP3wUWWq)F(<<+QK5p`KJsPPpW=V^3vfV%vt=h{WPnHDBkE_cRii)C zd<_5*9b}pi@adMqCjZR80CuIY1OQwHE@Rg?pwyqm8X!1wK$ExDHctb2Y@mN-1t}np zpI?qNEieS2`oE%|zz_r|>;lu+g|MSmf1Z^9aETq&3`hsKo@WGC4gGoiYz4sFvY%hZ zw6tSx{AVc#fN^%rsQ|VE4F9CXHn{#D;{P8=p6~~p22|{ePh4U`sdHnE3E@B$U2D6m zq76EDHT#wOTHt=5cO-CSeU<7kw+z%$zO_oTtjgS0w4`ZrI^_$1vKb1JM6R&JV7|0oH{ssZx$hevcBiav%@&RK|V^ohQuKtiz&sAwJbv zRkbP7{sg9o^}S1=PiRG9f1tj$=9SfF!N^tCtwNtdpjKFzAO8@f@fG7Kwb}Ug^Dijw z-gqky4GD#aJn0N;4wreQ`QklUV_=CoWF5Z!iT!a$d|m6s^QxI55qPl})|VN9)S@Qn z7d@P=QcgSyFj#hfi$6nsORWkak{z`Ric&qXvPgcH7Ox{1SwIdCW1fJV7dTo+4N_!CrYw1Co1bb-U zYhT~{4yH5k<<{v&{-u;I&9rue05drUX1!OeuzRpocDIC6UYx^hQ6kQ!QR4xQd*8eC zt=(3lu%;I->mVAEW3lO?8hWd7`Uq{hfc7`sS4^Zf`gsiVYD9O^S*IML?u?6v0AORFtZ4DAGF#5Qq(wYN3N5ARvMu zy(EMj5l~v_p$Eax0)!qQlyB{z9)0c|-#5m+&-ae;zVG#qGY&g@uQumeYpywezq#t` zoo`N9m>-8uJpdtlOAC*mcqXSo^mt)B`y7Od}x`G`E!QfC;_dUqqBJNGlhU%BJZn_1#0iVtKd?Mxh!byRg(< z0dT;M5c_5fzg}+yxY}fIZO9G;;|}9I43KPpA4lY|w$#44uZ{9F+%cLRw0AOlDWO#V znC6b@YJX4HJwD6(o)Ee=Dn3jQIKeR*xG%uJyK<0GCh|4tXs6e$Rp&5}`R>4xJA{d{ z%ooJs8>r&H;QI2@Azh-5fr)p_*nzRWS))iG1dogQE)vJ(VZ1!VcW6LS_(GmiF>R!z zi`l4V=XAn^DIt8?VSHb#xq^`I@U5SqyEMpr9B5(05WUt*YPP`avSDHB!Tqh_Ik%uFDKHo&? z;nbkQas1Q*n4{qQ)w*jYq9q~5_&d8O2_n-a0ZhQbgOU04V9zE7!!7~p!g@*iR(PQ< zd2EWGN)W1<>-(19WOKeJ>04h5L%+QRzk>0o=l8>#h(S~kWU@X`o?2$;&+Ide3eF0l z(mvW{z;f98*NSjMdw7?o`4)V?!I@-F|{s8tvR$u_Lm!>liMXcBS!bDaGCoZa0f zH`bSk_3+gU;E|lM{5GxqW1vt$0-JSJMm7aIE1o+X=RHu;<&K~57QFCTj~R}5?kV&W zoY)GX1qZ@tA3?VzyiFb(Xn+s`Mn7T_;rkhuQ`f$po1lN8%IYT!@@&Yjju%()_qA?O z&67RfGneq(sg}Ve113uStAc+n5lzpG12hCR104r|0EBTusyRutM#WQDzs9w1{tt@) zHD~{$DswP^&*z%9q&?!XjaOm${a}(pdH`c(f-7cwv|2^+0;19plTshj!ZYAu$rZ-4 zuJX!$If4`ab-Drn1L>s-N?IGtMo)|}`n}bgO3_y$W+pT>?E^VgelLBuD%h6x5Bz-scsSE~XvP~^W%}ZL z8_!`myE#D?-rEfQbAuxZe30}Y^9l9_w^jbW?AS{m<<#BW3KYxsgy-#{`~1#?{8HbzPTIy z+v{o%lc!MjiTelCtQX%em0IoRFLq4q3+eTg-nBZ#^PT?jJGOzptJj~hvP@s#8gviY zzPKmmiHGGeA>Tf(HXPpx#O;v;Sd1=F1i;B*a@^m!mA-em)X6DI#418y=J~q?$sBGTjMz8%j zSR1%ynjrHFrg9Xw#d_eMirzff!b^YY7|2Gl?Z5Pa(^&Q-fC(*ujHD`CnB@T8ho9mB zg5(V`6`|xcTfv$LQ!$WT>t`cSD@*)C?Kv~^yNhHWmRi5<1)I62pS`792plQ8sO*j2 z4^XVrz@pr2&Ksq5GCZFq@Ws33iO!&GLvt*`xL#*WQ0T)CnI9EzIy#YF~FtZe6!)C_|;H0U-0uu2ctue5aSiupJ zR`)cAG8}-wQ+u|+a3rt;{rm!Bang?{?8;wy68$X6_^}IwO@OSZO2}TSURs_TfsUkn ze#7hQ>)${YCOy*}5Q>LIokuY@;UJ;3)pUVWMm6R#QqIE$Oh%q)<=8{H6C^Du04lZF zDBM2bOWdd**|K%n4zS7t!O5CjMnE#>8C|NQJ!v5!9DKF$nP~0Ybed`qZ|_1OM{I5nSe0FMw$F-n{@& z$n!F`!jXNN=K4(TV`R2KU@ED2?4bvwVMa{<=mI|oUhmV|4?_hV{v|1MXp+NF7ylU{ z$h3n&n!l-#@51QJ2pCZ2pE|V*a*4oxsp%UfXmf_aF8|YmGtdbP_DkDQE4Lx56?Pcd zr9Whu3mKok&4Ka%vl`M^4Ujtcqnfn!HCiP=Ve>12v%vIJAG+p6MJN=MUL%Wvp!D=K zNO~3ksgMtCQ41Onj(>)k<8S@PX3D;+)~7!P-4**(ynWE@kPu7o*CC+UtW+vB3CJ;;21{VLnC zuhsc$)M`A2i^jA3GFcwdFpXatwp#hBkDn#5nL*!|zjXUTb+3lVXDUFc-XGUQcVB;!fW8S%OrA!v}M&8t^KGW*pn!q~`*t(7RmlsSZ ziB?ymnPIrap9>mn&Fq%R8Rcc0;(o=^qXj`R&c!Fa8l-6>{07$#{9bHz&{@&Q*TFG} zN7TBW*ZS6k1up^2L~O(#h0UeXzNqO9IY`?_nlO2=F{!I~Ah+c#+$Svn|ES{6X^nHf zDCnujrEj%fHYnMyvO+g zSQr3V^`E#t>1Uq*!~&qhHUEPVL3ci-yx1=YZEpqY+kYpL_-}vkqY~)HroBggCz5a< zW%`{+qWT!vOLQMC^se>Oy}uJlSlAeUvNiWS~L5dQ1ZhonhbeG z(e|5FgqfFiZA>LdvwzpxGY!(?-Qoof#v5Z0TsJ;rl%*W#Z7#lbut;k(%!K0YOe*Py zZ-8*?VcZZ3CK|#)%aJ@ZpmrGY7zEH=d>>r$CV6Jo=zH7z*efSGUt;(7^4HM$-e-9> z^{c(m3giqOpCZ7Oll&HJh>1ao<$ae+=&t8n6;xDpANI2KAkzx)fe9u_?&n!CXZ(rD#dc&{?rkOw`2f`U;VZoyzwp1vAy z-$B&R3<0Yav|=S?2$e!3EYhVbdh(W`TorQmsDi>ZRv{6LpNIOrO)w!FAehU*y}qBN zd|j74!6E8Qx>?^ORCDB>NqxQTvieu307L0i=>tMCyy&R?nKEkDoaHGQ+mH^>p4q3+4@q zdF-EhZS~VNYBKuTA?93>XWMbhZY`F{mU`T&2WAR7^Vx$|W<5hiqW|g_efVDRfEm|U zt_LQ*kNSXD^hxEyx-DC$B5|X4^pnu1;^R5Vfu95)mcq-;cYeIpp%2-IB?m10PB4~c zh!3>XYBd`M_G`~qw0zlubz^{o^&(m8PT#2y!v~bnZx8T=VmQ~M=eMH8K>iWX@W2r}vV zGCf-gZJgVoU>WB?d*CC73k@p9ASYhr2b(^C#l$f1Hz$)B68Nw=lpIP=M89rT6^en?=av0m(lV z{O{*ZF54o2p*=)s1<<{rAu34Yv+xbjSn?BoIGwZ4SuX>#{*ImJe4-M7J$Jskx|Xr( zjsfeSPI>02vl?X_M3BOe?~eaKi$D`&-$3esP~ek}z$CDUDzGbBp-mDQcE&Q;gv--G~zVqKJYinymL!o2sn^8Xu3-AYs9oYIQzRO1A zN|LcGdWuuvpr7&97rOuEq;^~TzZyvd$wt+vmD^C6`F;}JHbs7&_>noVYJ5W zX`a0}CbTC~#HVBDr2i+3Pbo0gz{j|ida^pu*T;DxfjVA_`m z4ixnY$77v%h^S7fsGhulEK8@E^u_gIVgPLhaO6N7z9;%5jqzK1{op|E^T6jq{H{?| zM&`?n8T!|tQv{JA)a2m+R32_;l__3iO(**d%f$HmB^ zk+W^U&Ksrfz84AVoV8oO!VCE_z}I)UvhnP zYzi+u-B-A8fK+8#iY=o{|Bbz~vj>w`Ga<&yJ8JvH(e1%qNWlHeQSvoEqC2^{i9+2U z5_OQPewA^uDe_`-)D#7h5)KilI|V2Ueq3xTA??=N-STNQ>AiI~;$(ZXc)$)?w{G+< zm2MA#V1f0-BW^I3QER2T8gyF}cr}|knI*w95t?gKB;96#a#-iG+JI+9iz63=>BL+> zLsl{8Zv)RtCh$)TAR-v#Kt)w6y#p>Br>?|Zg&-dT-9(0kAa#LPV<1nN!)ENKA<9jA z1NL*Ag`Y@U3w@G7zKq55EdT#DP@|N6fZ@y>y&4j7V|~p|iNi*c&RRdSOheNo9U5hE zK^Ha^5S%Y{NgxLkz40=Mo#l<60s?)fj8TGxZgdGPxEYKBqRC{{XPiU*QKYOg^s3G6;->xBoz*Cg&VB|T!wuiXGU{1={u?;C2u$aaSYsRxvH@V36Sw) zUAW#cZaPV`F5a|~C@uiQSx-Y2*URr?%cPVmtUvZ?zuUsJzDXf+~a? zYlD!vabg;|wx(HJ18U}H5E`a%?Jtb%3?r12OHSTB5SSb4L%=>`%wr$?z%@`MmCYNQ)pa*; zP)U){RDY>^dG8LKMIHGS1{)aTG2?j6+e30Fxg$%G+L5K4JbW9yetT|#R2QF-18bz^ z6XMrz#=jae+CP#oig4>wIwtJ*y0Hh5GsLr^Puum~&u}1l6E_j5ePcPGN6<3uz$su# zr?vo7TD|gEV3t^wsU1PKE>0XV({p_NgPlbC;@`0#L$FG=MZ-3|)wTO)UlV~)MD%@U z8lqj{Z9QxglYtR?wD0(R#}`(?M3wa4<-m#-x->Ry+Gbo_ZL;s4vX#?@{7u!?$|v&OC373*pqumb9pw6TXTeZWFM;T zm8nmD50U6_x$QbBVxFXS)3$>jv4*aKQ4TO1hmnzsEfwP+5VXlH@3BGYOK~O3ikb6H zS!P3g@hkqWk<9kZ60z4gtMS!J&$fL5IBA7jJ~{_tXG{WLh*~r+&@NMjPi^^fe9G!& zB#Tzh{tbC-B%kZmN2d-p0|LhFG4=K_F&ejq*sP^VH&LuCR%>Qq3A?(jbB0*(7lEDf z4K4x{rertbwJy3TL&Hcdk)TgH7ppfO_KtaW<-O7AEZex@4t{fTX2I%hgKWi|oY_0h z<|d}^J!e-=n^2A{+$7+uDHo7ioe~nu=joVIUZQPS!xz}Us3mFmEM@*jf zo$-XPsthLf;OoeYHA5~JUIE-8J=4@0J};!+_-pt?-d8zE9N7!x;MGBH9mvHabUG8tokaVft)^()ztoUMO6V=ULA3^diWo)xNQH}aLz3SKC(?I7CwYw057;kH&D$@eP|o%{!*NZcp)FV^1kS=P9Q(*TdNQ1+3oro+13mwyotu2tRAbh-L2Q>j(iwVvO+w? zg6Iwl&2mlcRYuO=d4HjzOz-t^I7Bq07(6;v@u@uL#VclHfJD*2#5v-p|7jBveP}}uM6mD2z zxxt?#7QC+T=boD#$T0se(I>P`!8B3z|9m^gOpRV$sY>GGiMRDG&4Ko z!40WGtlY6_CY)7{`P45Zfw)&zr(;WX zTFCfS$u@3H*TmwE={sdd-v0f)1mI3v^195&GfO^Z}WI+0kRvGHN+gzw@5wQL$(?XhBje}rAVHn|Q*vZT3G_g@G3;@VfD zy`#Q|l!&pU`S=?t)%y2eXGzN*5u${oY}USTGq&O2`J1N5gA3aB&l<2W$Imi^cf5#rGO}E4h3TeSYB?ApQUBK-J8=!xER|# zFlxIF_12yQ%G^pqr1@^(NL>y#k>&$UhJ<%HgR91wIkiD=SH{CKI4(y>-YCsl z+_!K6TW&7${JJRv=&kb~l(edlplv$+$KFex76InQoktb|B931GoQ<?!i1hp83wRY&? zfz5y{Wa@3i+fUd_?pc>*Nz1!(5KpwwRNV#iQu}0uvcsKM@jZ&uwXxP$6>s_aE#F4= z6kJ5!G|6Jm6bk;ZrA;5EVH%5E8rW}lY3E=B-3JlM-UGW?FUmdnjNoN7px8;M%D z1Tr}%6ld)K%V$k?POW_{wN4i)nbyy>LS1FdOR`VgC&CECcIr>DiN9dhVq4uUd2?k^ zl?%2AF0kc3d&Cs`&hD}>(2$s_TP831I863zO(C^tx^inG^BHJZ{YPJ&8BnAMjuAKE zQkgWV(OqM;enP65NoCoT3t3_dHJo~uxe-Le1dTavOTjlr$=F)*_wDTM&@kBF`-42C zu-{_oV8b&ptu*WHB&`rm=Li>l0dP%inuKeu{h8vNRcaC`q*}^*gN|CEgm-X1F=j#Myi{fMyfylKL>odvTbGC(GAFafq~! zryJwTL(@kM3*S%*qefb0Kjn*97)uN@`P^OJ{9SvglNmnn>} zZiQR0T;Da5TwO=b%x>R1>vga3DvPmuZRg5+g{Vk(gAe^z-0kgXx-565DCgb?MO97o zOyyy@4cHjKqN`o_r<%oNr(M|@oxJ0kh+4Qb(nkLnD1JuV;_ShOu%%K?L~AMqaba1; zHODu!Ca1g+zMA)}-P06nJsRGVXkz$*>z35>mLX3`hPsiEns@e|lgr8X_1CejMbL1U zE!@;+X_d5f6vPF^zFCaUz@8>Q0e2BwRHBRgY#N#hwPp%e$+^N7uscfH_B8bP>!g+U zQhGT){n^Mn-dKb3G%VwES7za#b9;By%?9Hm#`@+Nm|0nAkV0NvQ+CTosX_0R?3GX< zbREu4+I2ZO`-GlS@|9PjeRpgdKUWZnGCHD#avqBU3S_y^n?oxJx%d&sen?5xWTbBB zho{7kMF1Aj%j_V(TDfG$vH5JTd)k7MY1qK-`=kBU8XfwDL0^G+0d{-X4fz zV>NTvX5eGv({Rj^fsI?0Un|({DJBZq-BB}TEk*4Q@$>K*$_$2NJ{)#XU{Pn%`{>gK z>QK`haNyXw=CDJX-pTQBxBdpJ{?wJj?OdvLH_~D=a@GYC#rfph#WS$ndUdH@M{;s( z7HZ;GPPeOrU6{D_?lsC=Fas&yY4CU0_llJ8mD9+am7PGh+5edOYB<1ofB}`^?%u?o z(o+cm>N2Po2J>08vHoMGGVySL$3R|5f5?l?$p}0Kh5trp2EM!qX$!}8D8TQ?&MzIP zBmD+%S_96W8|w?J!}HGjX&?y>s0moD;GVVl{HXQzrC+6%hA_de$y~e!LjGZCI}-%! zli|-UoG8k5dH7lZo=(?$Ut^AXZ|ZJbEX2 zfD43Eop0XIpGm*Q3U)UT;aR8G_y-*+Y0sCtzz1=5=*q?OH$FkhZ^HCfJ{N3AX5E>e zKd73^%e{aLXMF*_J|lWTZY>k>!~O(l+t5|cwS@BDn5FdrJZXU)X@}~Y|1a9lhu11PcUIjSYPZn! z%UWaJ)hwevI!1cN#63k7y}BsE^7#La*R#+Bu#m~qLOFwsbTDMWB)A7F4Eeew%J@u} zti@(*xO!{9ak1z9GfZ9vWB8xL5UTu4cPi~%_Hew4AzPhoT456d@D{;fBSIHSM;F2c zE>x5z?m9G9_a|pZ@j;R0)g#PyQIMArH}c7!_B&-~!0mrKf7HL*Vde~g`oD8=?#2HN z$huHQL90K)*Weue!?TC~hv!2_!u{~${SK)6hv~yD`+!kKWMB=zg$hj2w!fTt54OQ8 zfQOHzed?DZP~{CnKLsFmR149+oQQj^QB-*l$=03B{^fvV;-(iTapQlF;?9u1<~Rc& z?vG~k+(O&_a37K^H_+bwx1URwIxmBG&NzTvrQ7WL`ChpP!G<2Qy)B2+7^+&gZP*59 zqo-H=1_84A(4K`CGF(4X{QxZf8*n0iQv5MkAFU^|+vz+Z=PVEt2d=iFi2b7gG|vZO zJ_@oQ4~avoHRm>ULHIK>Vp>-qlw+@#9Q0f&@x?QhN}8*&I~h#49T!iwXYR-5+&V;069EyAKsdWm3Kq@%xytUdv6QNF(6qkA zDY-6izYagg3-fb+6RC4nKV{Z_m-_1AeB+ z{6;2td?k(dZitt#NOI5$I!7ujbvnSt6*&3Rv-=9U>G@T)*#<~M?dyMe=d|8lk~lAukk~>pV~pgM z>~~gr_UOnzdwglf^x&3zWsDx`Z>VN|jMzTqDXrrJ=zc$8M)U|R@euEYs(_l~R-dbw zH>~IP7|+Bxs`_kXObs2tmy>TViZ5#Q)`&iA=&paWgzd%}isOPth#wU8TTP;lm4`{l z6>(U9rWJd{Kdp$Hr*dpe#x5VC&L}L<#Gi!OVwDNIr&~&GcjZ6Z3dbFPha7$?CnZu8 zR3@!Xc;Y#OER6MOKndlf5b}n}LEY7e?RUL?sZ;?>^qG)I$~Oh8qOaEl8VU?;x$2gl zb89{9drFeDZCEl7+@{9GH{WpX8Aa4oV*-z@uFQ?=N5D#f7jWIuBT}NvNItOeCD+u{XOAl% zV>DPrHs=qIBCbvi#m+y_v3<^}8S?4vvBiKV$nXq~?6Xqk*X=zT?rgRmj1 zAVZPv&-${sK}f$IlR<%LRrV>rGBLEd{T4}} zuCySlW=~u)xRp!sHJU+R>ERQIRT=ZkK_sd?sQ#FDPlm*2JG1^Zs_P7~!rW9(ZgH4G zU2D#QiK8$5eB7o}s!GyQ{c}M1fYsVbfSF#ph6fhDR@%UL_f z{ux`2c+v6l#?`?$VmYGiM`l)LyOYQY(!&exYj$>^;LgR> z_&g0_jYG^yBAtXDmCZYn0-8HW`TAfgM%3*VJ~$N&(m=7!phLS}6|&B&Doazxza^2h z%@ad*Tpm>HbSLKZJx$dM@GUAbtvS=qEwHX_Es8U=+0mEFV8Y{9-J9;3K^^JbW&m!Ch#fQNt6L=HQKW)OHUAIHRZo1MQYA!ZRhtl9pvS`$ohY1y4eCe2_|V z$0%CnTpwOenc`lK&897On3y7*iO~d*lD(;Dq+*0!w37+qt*%aiP(>Wev`AI&@iS2` zi4~_ELvz>$+9UUAiQly0@2C^93j=>Le(aWtAWAPRgxs&yWVvk9lkjOWHz}A^jm*rw zOgp_!-S>ARxHpL1A+;;5km^A9WoB-@JEj=#sJCpABPoG4F^zp9?S_fw9%wI8Z1DY{ zNW5NSz`%!7rObF|a2WtcXnauZ5E8rd!X$Ba*GMSaK-Cb{zBRj~U4=*{@1&HCFUmNA zv?AbZMbi)3ci#I0OKUR{4qV|4OmGX-^P+D}F>p{)*F<&$>KW^|W#;*8+M`MKGLBnZ zmL1ulQ#K~ywJ{ORS$KXSVS#W84c;ruk|NfV5UI^Q=fj9`kGOYZHFS@$rL!PT2=;!UFO7e9iAL^ytOZO=#l?3+6V$*xh| z6nSFANFTNvVZFSP%7}r7EX$@llAfG{3z?cCuoJs#XV-3I&eUMBt42iXvfzd>t$v5y zTK@Z6DVt1kM+;#95%Q;In_t1?QaLAVy`u1v2lfv0*V_W;=FYdXCu*7$2Ww;G3;liV zG$sO)QsX>!)jM6K1_%4`8*qygJ?gBB*sOy~(=}K72OW>}>!^zVMT<4J*zYi^c=kXR za3GNyDqXrhTu0pWUG~)LS9Nz#^%%V!NoieOUI`6bE=n}x(qiCqI^{pLD_-PQ(9mM^ zN-o7mjKp)RBVP61!1Y2Tu{1-0Kw!jlX2a{3ih*C(ufH41Nn07DnV|=lSCoXpd-i-J z;otw*-5D_vAl{#v<|!dx-5aljJ&7$QZ}Bm0mER~9(+W*u9R@CIJ-&2IMeTL9<))XK zwr3dJ$M^%akH?ym^4_aH@FLI(a%69<_vmuo-sX#>+9-TqlOQ?Q5}dvek?WYkAxh_Q zi5i2InZ^Z367P-#Om&sjS&l9kyR~~J#Ckd3AZFzPS!66MUKjryP~`XZ_=T_;I84~@ zR{N%vwRhk{Rw}q#Dtl>~56VU+8^Mgwqa2FBt8mSE)zRD|@WmGeDvTH%1=|F3Z20E5 zTa<)r#27KykwXSA=UbEy6dvvmYla1HBkB(2cGykWS~iqkSM|X1O68~Pd&KJ3JdL_Y8@GRytRUVgHg%D^M4m5WpR@oTe(@S{q=$XP z7km=%U+ON&&N*k`gl$sh<>P04V1_GM?>CJ+kWcKqu7?QibdltzVi$j1iSxC%%$)mY z)4vf3oOh1m;<`0CX=@fL(h?wF(9yG;;*YMQJbfRi?_98X;3ZK|ja5!Jd7(VdXLK4D zx8kWVOf#?0T=td$jL1n&dGIc?lYw)t8k25es+27iPL?CGG~D2S&ss0>q=DeKOLR*i}<*4^)PNrRtWYZRw@P!5({ z6+Uc?>M6l9u1we}JQMR)z2I`A zjyPD1kKkwzb+d|#0z(5!j+Bb zyZBssp|qKMpwPE}o_(Oos_lp$YXG=%2bk|IS`pWRDxDpsl$t-}IF4)F*XUq&(jvSs+q5SEBe}yIB^^Ef6uSV+nb3?b zGaWd^h&iVlLd?*z_o~H_3`7IHUg)Ty-_B9k)dwQ%#oa4#dc9k6cHbDv&*eE-`WIr+ z6ICaau4j7A!mTWi4en~XJJ6mlZmLY-1fNTzv33M6IY=rCRZ=if z##jm!&We!$nc6Zc2(uI=)+T{orGiSAseHTV%%iSJ*6T)Kvnl&tgFZJuKFJ5yz!#^7 z1}x_dD&(=hEsPf^#$sI_54IsnVXP&owl|G8t#?MuE?O;2^x?;nFMMf&R^sGg8OlT$Ta8jCy;jekV!c@cTy2;*anz?*o>cM2JR&u4fp@B>Dtw$oO$$0GaH&~upM zR9vGT{@?)89_3l@eGMD6Ol-a}(%Hh6Od7dJR(ud?&~itAy-)K=lwP7J$yWXt&d`OD zA{kr)kR$}V@!t8)q1m<>5oM7=A6a`FsmFR-;Tl+7DNhAEMMuG&X^>Yy8J{3`Qncdf zP{}qU+V3JqS_C*Cnh$Ldxls`*l-m2_!*b>6X@arc)!o*yH+pTSbP*#OcErwk6^oOX zy6)%}HFQ2LIroK!{5F~#e43h++x#e(*WzSN@BG7J__g_)ydfDghO|m=ll4iFF&^7K zLY;lqnlOLs9k~Th$wg1Sp0GLmhf;4&5?<)hl0iE8hOR+RK(!oqN=Tizc?b5nQv*#N zR~k|Ca;f*VT(uTbJ!(WxkD#$2oq+8nv;F*3Pukl`WBQQ_&Kb(29#N)dmJlED;fA<|3)Cq#`g zV76LCPjF_sUyfuT^sbT9>XAurI!p>1%T{x{=ikmy&z`ZGnoHT;ZOdT1F3q!HTBLWz z+zzSt8Uqe7q*{E}l2Isrs+;5Npty3VA&EoI=16<*)4A1|T+yn+_>IV#QS)q;+9CLI zYUFMGE|Ws{$dUEJxzKyEq}gGtkG_0`DBeA-2yK+ubsVy#0nW-PKHXr1x?}uGE0oSN zAR|PuPV{|%_xEmufq`Y&e!{3M^!g+E_m_7{T2*3AdLm z^a!R_7~LwJEPUZb^0Kw(BENS}ZtJJqCZ;}O0vXYDv)Qk*(AK7<#fn%9dO32_*wuB` z%ELm=*mfx!wy(4MO9>Vqcg)XwRZtWU*CM$LA}kD9@Vx7MwQRc!2b|su*M)X>&zt39 z%^!jT6%L=&+le5zm0HGf6-sjqw1-jTFQ<{$Q}dz|=I_#rJJfGXq_k^7c@;h_P` z>t~>LQ*LiZgM`*H ztl?`HYnBJ<6P2zs<4KCygraS^pSIFDyR5WNa~*50&oV?0ytTV=5!+}W-*#bA;@tQ> zhn?BEqB{x{(%q3!++u)^%&dMhY^?O)+i$(6 zE@<)@sY+`2{NVITnw*}05s!Lh)vV3mBWD{d6r}lbgvI?#{TDObm%=Ay3bF7rkz94$ zy8s_HYCq-gX}(ASioahCO^Rw#H6%>4!Zwa-@+^X5d z78gK}E()3PR4A}m=aQ28(A7p%#bzZTZKw-e-Oik+&K6?7m6B`$JTG3`S34?9;ZS4U zU_N|2=yc&wW3ScX$Hr9|sd!>xBYZFUOPKu3MgX7CAUq<`_7Y2uw@kXDtY;W0yr%Tt zIHh}$6(6z@*`uVnE1?4GBD=i2*1O{G8gLH?pli=zpab9nWZQY4K^T6GLQgvOhGEeY zU<+$qJ)unrrO}ljgX^&YF4WD9&~3LjU+;zP5?vw?*B6U7n5<6jjM&J}I?@&gm}3vNbrKn1sEnVz z@|Ha(h-iMi20mZ=#_G33Fx^-oJW|hmUAwnCB8A*|ZN|$yyq~zSPPKu9JQ)AsbD48I zh^EJxh09rez22M*22A(a-n}+_Y!$V0au{TenB2yw0Q`t+fIJ&5w>eZxJNcRYkMPH4 z?{W_Nl=dQ0HQKZG1%hpN^_Sw3#J-o$g%kLD_I)$Bq2wY` z|CE|}EI3JQm%yz94{-2nG}#RJG<}|^rZ-Cu-kF_&KoKy`#OHT zjD|a`HlF%ad0YLl9~+|xbW7LaCKwI)fpvlO3W4;Nb_Qqz>D4RsAv=bw%6XTt8&=;9 z9vdNe<%EZir%{4c#L^nLfY?%Pg=Yd{h2n>AtQEIDotkRrZB_UR|-jwbdufO~_9 z`>r2=ACF8y#3dZ}8M;&nj%e2T1Zq^5%_;V6aL?F@w7#$#b-uaXp>@$Ft|b-x|I+Y*LNSbY`;GW~n{l@YwENXe0nr^~VR&?o-(i zo_D%^I1R|=1jyp|V?sfy2f(_h94YCLv@vnOxnX31<`PZ|h!aBMX8qlB8BR^R4+1(r ze);h0msXIzA%Vl%qooq*@${_r&?KMcWn*aRp>c9TOIn6(d%Kur^ zEM^1c-ErWbak+)x9G8y2&>N^8o1Kkt*v)^gda{{(0%N^<(<2sa#?Wn0;q?E|QxDt@ z3pt^JO5c0}J*KCu_~S`9*uR0#187&4$xALU4^U*VgMdFz`}lOcvIH@#Y;Z{vcs!*z zA<9RT0AXIIt@PNI8|63}B@uE0L_h!w9lY4*jTbqrm9hf{o7YAG?NcEXRc-yAc>;uM z<>*zsvBgIOeUWbfggH0O%{TgmI9`Osf9si63voP8YX%p^cx`@SSU4vBdzr;H ztyZz>AeG(dYx)NwAzF#|xwQZf&=a`oVX!g@No3l639u?908Ij~wHU^-PHWOz1xSRT zRi0A!l`uhG%{RdRgM=2~9k-EFg`vJ13-)C*k0P`!D7t!TDdgAWwx)WN2Z)Sn;FDI_R|Gq06eB>;t2C2X@&^Qb+$ zel!edI(W`8)$;9|aa;Zg(0e(0=9@RW!p2Hq6yyD%*w^%sWYZIZn-PkmP_fdZ>I-f= z2c=vMwEt+MG5_L7Ue+sm)7s@dR_gV1Qe+={06OrSZ7`H9LT0{9x%lAl2kK_)K3=WS zPDBLWmyk|e2(P}x2r$a9(W>GXuHD6JvK00GAEti1Czl?rH23_zL_pKjhkK<^Prbw* zt5)d56RTyXUFGl+^92=`=mT(T>_)3qk#y#A(Ufw8XT^1>bjmSt;U$(R9ae|^ z?LBR+6fWXv-+pC}Q5O*VfX$oTM@11#Tc~I8$T^WYKDy{`r|M}Z#^|IV0-GRxQV9d; z!fONP7PimzN>dalo5Ol@<#Iafw6AjIIeIyR;S44SA|^W2B-*o#N*6e8T;Xjiv;D|v zNJ-0O^ilfS#{A+Yc9PUzp~oN#&v9#TfzB zxu5F+0q+eKaG*^ZAp5ds2Dk?XbW0Q*xDz*-3oVB9tX?3 z_28@W&G9QM07!)@=?6Mo^qgX&Rj0OK2=i+R9v9Gam>;jWS)-!l)u0={U~f2TnBaa_ zi0SXbnx?e%64rjGXR01ZQLr@Zq)D5Lgz7R%Pi5$L9P_pY9Ti#M6u@Xp#uiNP%OB13 zcN(QiAeGV4sb-m_*!iF3xNd`&9;x@%f|OuoQ8*;PL+9(-M90#PB5Hwy$*7ksFtt1r z3+gHN>h<#=+U^|KM*7xd>!OS<{5hy|2N_Hpw$Hi2wh$(qH(7-OV;2@$1O^uZ@onFG(S-d1j$(ck{PxnFsa~B? z%0;%Fi{DBeS$ej^Ox-8I6^meHKMu@?Wnqw}f9#BB1b13h)>gPA$-I(6n7T%dk7R}u zPnbLcBb}qP2rVJ7Q84TSt9w9RIUD_Do}5$sht`p%h$%fF87Jf4#Y9XJn2z)fC4`5u zP}4Kr|H9tr0BK&|jQrkO%5(1lHG6iY3d^sG)BUIM90LqZbGiETwK59m(n#?!99n~T z3D+)nXehTNz?jLtr*;J1aQiSd!8bvDPOM%2Kw{p=hzAfqZYAt@A^I59_g##@^ZSk< zHu9-!c#}&?HMX@iQe&4?WX{;$o-!oee$E^yI(CV96dJhO8^2ep?{PHrril7yN0)dKCsEfQ&p~y8&kl z7z|ekOs35`z{YM5J%YJG=_z1FhyFXC7!`A9!Eh{_TOrheh3*OE9_d&{)S=D&)D{?R zccQ||Wx(kF47TmnP4LI{r@Ffec*Hh1V_>Me@tiv!tN1PNtsisBkieZ&KO z*`q_LuZp}i)ORqRnUQlj({-O4=J1$4Gfxy=^JE*XdRJ<{h*{bQxWP&@yVz$uR5f5u znxVCg+^|VoX0rB-N~_rGlKu=<0+_>K3@jt`VhXi`?!-CFmCl%bELx@Q5oM=<{r~PW z7dcWMpClVh8p>-A!@cDkn||TDhcd;!zMfqIeDxDmf@QRMTN!Hj+G@q(hVY}$v@9tB za%1BDiVdf`^1CZGgnNokRU#>~FP@5a71>Q1z;2qiS`F)++kz{k>(<$^;|*cKaeDr) z=x4iH4c2rjU2AuBb0*D*&8&9%m!1`6#Fd7AH-66?i}ojO86e>MB5Ch|(T1|!wCR=^ z?L9xc)E;&ccew{$Z(oPhvFF8+tZ}^_f4lSx)pbFu&UIJ1{e&{4TS9kl$hu5@dtApY zXR4$4WVrQ9Zfb4ZTY|uEICLY zF!grEd_Cg{WE=uhluh#ws!}-X&4+00tDmc7J1ipi>aR zY_nET?yy3QRw_!qYC9%uey$)=BmI4EnnlWoyBVNKJR@ z9N_JHw^o(Y4AKpR2+gzJ{LluL6;^NtM{D{E*4tFLX)HGfRhyGu=Z{2R5VvcEpLFV_ zPRVM8)p}p8lY3rwn)0X^szH{O*>_mUS`Oy$-GFX%<3`nn)?U=bp<#+@D2}6Vf>2I2 z5|yvC;7(0!oqtyv;D)8GgLTi> zeRMtsSo;I9^hGfGhPJK87l~N0eIoqExwa*o3YSpQf)rP(RBk9*pvMtf^ff}oq2mF4 zw8W*xmaf4`RU<{;%%M?);>4PiG`sD4_gRY|EDEj%aa zeILWAa17_&8D5*WX2(wV0f^3R9N#*p#Ud;V!)( z-)q_rWFMQb-L}7!!2deKF@jhM(({ADgeZ>EQZ0_J($eS85awDsMMPP83UcKcf_>GY z6AN~8T|V~dqcJx{p>NXG?sP%11mC-Mk!?V&q#_n-yb`Guhf=1JUr`ks!g>Ww$uwVC zR$9;q!OS=}!9ZsD_MOJ2TZZU^PDA~z_ZcysT03wN^yT~G<;Qn5DEe()xrjjT0ecPy z;?Tn&;Jy?M2|3fNX@@WG+fj0_$1Qf+!J$nOHos{Wo3UN3vIZ?s|qNdbN2d%id(q!%E)i>CjJ0*&)zpz!CNzA8hXiCU`o2#r2k- z&C+WQtzEbX`s9PbGW9%+rWTd;!1>i73EIq|60$V82B5QUu@PM;uz|WA)6hBh#};vv zQ2{y#|9JNc(q%&c{B84&7QvH9w+TTJAHhpdQQ#Uqw$0B#vGjgH&u(oV%3e)cP3C!Z z90O0RHqT^VkYlXp_S~WH%E(JV$^1>|F>209qYoD@xyU3z6ycMZ*Gbd2%Ti{Oqzy1xr_@96OGXnoJ0{?~)Fv|4XvfG>Yd$D#UJ;(hOHSLSp7w$g%e*jmlzV844 From 8b66d7f0ca9c2ca1aa35af5c22a212e936c86cb4 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 26 Jun 2020 15:58:54 +0200 Subject: [PATCH 44/77] Test --- doc/{page_v1.png => _page_v1.png} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/{page_v1.png => _page_v1.png} (100%) diff --git a/doc/page_v1.png b/doc/_page_v1.png similarity index 100% rename from doc/page_v1.png rename to doc/_page_v1.png From c2eff0174ef7b7076613c61862dae767d9c7569a Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 26 Jun 2020 15:59:05 +0200 Subject: [PATCH 45/77] Test --- doc/{_page_v1.png => page_v1.png} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/{_page_v1.png => page_v1.png} (100%) diff --git a/doc/_page_v1.png b/doc/page_v1.png similarity index 100% rename from doc/_page_v1.png rename to doc/page_v1.png From a7d49e26b78b4abff982ec36090da8fa1f72a214 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 26 Jun 2020 16:01:50 +0200 Subject: [PATCH 46/77] Test --- doc/{page_v1.png => _page_v1.png} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/{page_v1.png => _page_v1.png} (100%) diff --git a/doc/page_v1.png b/doc/_page_v1.png similarity index 100% rename from doc/page_v1.png rename to doc/_page_v1.png From c2b83370ec560903cad2fc6aa5e577704e6b3ac6 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 26 Jun 2020 16:02:29 +0200 Subject: [PATCH 47/77] Test --- doc/{_page_v1.png => page_v1.png} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/{_page_v1.png => page_v1.png} (100%) diff --git a/doc/_page_v1.png b/doc/page_v1.png similarity index 100% rename from doc/_page_v1.png rename to doc/page_v1.png From 30124c31b295c62a97cf07deb5256b5be1c0f31a Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Mon, 29 Jun 2020 16:15:47 +0200 Subject: [PATCH 48/77] Clean up type system --- internal/engine/types/bool_type.go | 38 ++++- internal/engine/types/bool_type_test.go | 188 ++++++++++++++++++++++++ internal/engine/types/caster.go | 9 ++ internal/engine/types/comparator.go | 11 ++ internal/engine/types/date_type.go | 2 +- internal/engine/types/error.go | 6 +- internal/engine/types/serializer.go | 9 ++ internal/engine/types/string_type.go | 2 +- internal/engine/types/type.go | 48 ++---- 9 files changed, 274 insertions(+), 39 deletions(-) create mode 100644 internal/engine/types/bool_type_test.go create mode 100644 internal/engine/types/caster.go create mode 100644 internal/engine/types/comparator.go create mode 100644 internal/engine/types/serializer.go diff --git a/internal/engine/types/bool_type.go b/internal/engine/types/bool_type.go index bde91866..e7073eab 100644 --- a/internal/engine/types/bool_type.go +++ b/internal/engine/types/bool_type.go @@ -10,7 +10,12 @@ var ( } ) -// BoolType is a comparable type. +var _ Type = (*BoolType)(nil) // BoolType is a type +var _ Comparator = (*BoolType)(nil) // BoolType is comparable +var _ Serializer = (*BoolType)(nil) // BoolType is serializable + +// BoolType is a basic type. Values of this type describe a boolean value, +// either true or false. type BoolType struct { typ } @@ -20,7 +25,7 @@ type BoolType struct { // false. This method will return 1 if left>right, 0 if left==right, and -1 if // left", + }, + { + "left nil", + args{nil, NewBool(false)}, + 0, + "type mismatch: want Bool, got ", + }, + { + "right nil", + args{NewBool(false), nil}, + 0, + "type mismatch: want Bool, got ", + }, + { + "equal true", + args{NewBool(true), NewBool(true)}, + 0, + "", + }, + { + "equal false", + args{NewBool(false), NewBool(false)}, + 0, + "", + }, + { + "less", + args{NewBool(false), NewBool(true)}, + -1, + "", + }, + { + "greater", + args{NewBool(true), NewBool(false)}, + 1, + "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got, err := Bool.Compare(tt.args.left, tt.args.right) + if tt.wantErr != "" { + assert.EqualError(err, tt.wantErr) + } else { + assert.NoError(err) + } + assert.Equal(tt.want, got) + }) + } +} + +func TestBoolType_Deserialize(t *testing.T) { + tests := []struct { + name string + data []byte + want Value + wantErr string + }{ + { + "nil", + nil, + nil, + "unexpected data size 0, need 1", + }, + { + "too large", + []byte{1, 2}, + nil, + "unexpected data size 2, need 1", + }, + { + "too large", + []byte{1, 2, 3}, + nil, + "unexpected data size 3, need 1", + }, + { + "true", + []byte{1}, + NewBool(true), + "", + }, + { + "true", + []byte{2}, + NewBool(true), + "", + }, + { + "true", + []byte{^uint8(0)}, + NewBool(true), + "", + }, + { + "false", + []byte{0}, + NewBool(false), + "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got, err := Bool.Deserialize(tt.data) + if tt.wantErr != "" { + assert.EqualError(err, tt.wantErr) + } else { + assert.NoError(err) + } + assert.Equal(tt.want, got) + }) + } +} + +func TestBoolType_Serialize(t *testing.T) { + tests := []struct { + name string + v Value + want []byte + wantErr string + }{ + { + "nil", + nil, + nil, + "type mismatch: want Bool, got ", + }, + { + "string", + NewString("foobar"), + nil, + "type mismatch: want Bool, got String", + }, + { + "true", + NewBool(true), + []byte{0x01}, + "", + }, + { + "false", + NewBool(false), + []byte{0x00}, + "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got, err := Bool.Serialize(tt.v) + if tt.wantErr != "" { + assert.EqualError(err, tt.wantErr) + } else { + assert.NoError(err) + } + assert.Equal(tt.want, got) + }) + } +} diff --git a/internal/engine/types/caster.go b/internal/engine/types/caster.go new file mode 100644 index 00000000..c8637319 --- /dev/null +++ b/internal/engine/types/caster.go @@ -0,0 +1,9 @@ +package types + +// Caster wraps the Cast method, which is used to transform the input value +// into an output value. Types can implement this interface. E.g. if the +// type String implements Caster, any value passed into the Cast method +// should be attempted to be cast to String, or an error should be returned. +type Caster interface { + Cast(Value) (Value, error) +} diff --git a/internal/engine/types/comparator.go b/internal/engine/types/comparator.go new file mode 100644 index 00000000..b8b81b8a --- /dev/null +++ b/internal/engine/types/comparator.go @@ -0,0 +1,11 @@ +package types + +// Comparator is the interface that wraps the basic compare method. The +// compare method compares the left and right value as follows. -1 if +// leftright. What exectly is considered +// to be <, ==, > is up to the implementation. +type Comparator interface { + // Compare compares the given to values left and right as follows. -1 if + // leftright. + Compare(left, right Value) (int, error) +} diff --git a/internal/engine/types/date_type.go b/internal/engine/types/date_type.go index 2324c228..ba83ff05 100644 --- a/internal/engine/types/date_type.go +++ b/internal/engine/types/date_type.go @@ -20,7 +20,7 @@ type DateType struct { // larger. This method will return 1 if left>right, 0 if left==right, and -1 if // leftright, 0 // if left==right, and -1 if leftright. What exectly is considered - // to be <, ==, > is up to the implementation. - Comparator interface { - // Compare compares the given to values left and right as follows. -1 if - // leftright. - Compare(left, right Value) (int, error) - } - - // Caster wraps the Cast method, which is used to transform the input value - // into an output value. Types can implement this interface. E.g. if the - // type String implements Caster, any value passed into the Cast method - // should be attempted to be cast to String, or an error should be returned. - Caster interface { - Cast(Value) (Value, error) - } - - // Codec describes a component that can encode and decode values. Types - // embed codec, but are only expected to be able to encode and decode values - // of their own type. If that is not the case, a type mismatch may be - // returned. - Codec interface { - Encode(Value) ([]byte, error) - Decode([]byte) (Value, error) - } - // Type is a data type that consists of a type descriptor and a comparator. // The comparator forces types to define relations between two values of // this type. A type is only expected to be able to handle values of its own @@ -47,15 +20,22 @@ type typ struct { func (t typ) Name() string { return t.name } func (t typ) String() string { return t.name } -func (t typ) ensureCanCompare(left, right Value) error { - if left == nil || right == nil { - return ErrTypeMismatch(t, nil) +func (t typ) ensureHaveThisType(left, right Value) error { + if err := t.ensureHasThisType(left); err != nil { + return err + } + if err := t.ensureHasThisType(right); err != nil { + return err } - if !left.Is(t) { - return ErrTypeMismatch(t, left.Type()) + return nil +} + +func (t typ) ensureHasThisType(v Value) error { + if v == nil { + return ErrTypeMismatch(t, nil) } - if !right.Is(t) { - return ErrTypeMismatch(t, right.Type()) + if !v.Is(t) { + return ErrTypeMismatch(t, v.Type()) } return nil } From 6328d57dfc0934903a075c58d374f29c6ee378e3 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Mon, 29 Jun 2020 16:19:20 +0200 Subject: [PATCH 49/77] Move string tests to value tests --- internal/engine/types/string_value_test.go | 29 ---------------------- internal/engine/types/value_test.go | 15 +++++++++++ 2 files changed, 15 insertions(+), 29 deletions(-) delete mode 100644 internal/engine/types/string_value_test.go create mode 100644 internal/engine/types/value_test.go diff --git a/internal/engine/types/string_value_test.go b/internal/engine/types/string_value_test.go deleted file mode 100644 index 97783e0d..00000000 --- a/internal/engine/types/string_value_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package types - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestStringValue_Is(t *testing.T) { - assert := assert.New(t) - - v := NewString("foobar") - assert.True(v.Is(String)) -} - -func TestStringValue_Type(t *testing.T) { - assert := assert.New(t) - - v := NewString("foobar") - assert.Equal(String, v.Type()) -} - -func TestStringValue_String(t *testing.T) { - assert := assert.New(t) - - baseStr := "foobar" - v := NewString(baseStr) - assert.Equal(baseStr, v.String()) -} diff --git a/internal/engine/types/value_test.go b/internal/engine/types/value_test.go new file mode 100644 index 00000000..d6ba8aa7 --- /dev/null +++ b/internal/engine/types/value_test.go @@ -0,0 +1,15 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValue_Is(t *testing.T) { + assert := assert.New(t) + + v := NewString("foobar") + assert.True(v.Is(String)) // Is must yield the same result as .Type() == + assert.Equal(String, v.Type()) +} From daad9b5e13f4119006ef753a1a93e7d1d20c4d97 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Mon, 29 Jun 2020 16:21:27 +0200 Subject: [PATCH 50/77] Add godoc --- internal/engine/types/error.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/engine/types/error.go b/internal/engine/types/error.go index a1c63510..a7dc7438 100644 --- a/internal/engine/types/error.go +++ b/internal/engine/types/error.go @@ -19,6 +19,9 @@ func ErrCannotCast(from, to Type) Error { return Error(fmt.Sprintf("cannot cast %v to %v", from, to)) } +// ErrDataSizeMismatch returns an error that indicates, that data which had an +// unexpected size was passed in. This will be useful for functions that expect +// fixed-size data. func ErrDataSizeMismatch(expectedSize, gotSize int) Error { return Error(fmt.Sprintf("unexpected data size %v, need %v", gotSize, expectedSize)) } From bd883a8a79c6a91b6e4dd0ad3dccf6a2996c6d70 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Wed, 1 Jul 2020 22:50:43 +0200 Subject: [PATCH 51/77] Improve on the type system and run go generate --- internal/compiler/command/insertor_string.go | 28 +++++++++++++++ internal/engine/builtin.go | 28 +++++++++++++-- internal/engine/types/function_value.go | 5 ++- internal/engine/types/integer_type.go | 36 +++++++++++++++++++ internal/engine/types/integer_value.go.go | 29 +++++++++++++++ internal/engine/types/null_value.go | 24 +++++++++++++ internal/engine/types/real_type.go | 34 ++++++++++++++++++ internal/engine/types/real_value.go | 29 +++++++++++++++ internal/engine/types/serializer.go | 16 +++++++++ internal/engine/types/string_type.go | 28 +++++++++++++++ internal/engine/types/typeindicator_string.go | 28 +++++++++++++++ internal/engine/types/types.go | 33 +++++++++++++++++ internal/engine/types/value_test.go | 1 + 13 files changed, 315 insertions(+), 4 deletions(-) create mode 100644 internal/compiler/command/insertor_string.go create mode 100644 internal/engine/types/integer_type.go create mode 100644 internal/engine/types/integer_value.go.go create mode 100644 internal/engine/types/null_value.go create mode 100644 internal/engine/types/real_type.go create mode 100644 internal/engine/types/real_value.go create mode 100644 internal/engine/types/typeindicator_string.go create mode 100644 internal/engine/types/types.go diff --git a/internal/compiler/command/insertor_string.go b/internal/compiler/command/insertor_string.go new file mode 100644 index 00000000..7066d447 --- /dev/null +++ b/internal/compiler/command/insertor_string.go @@ -0,0 +1,28 @@ +// Code generated by "stringer -type=InsertOr"; DO NOT EDIT. + +package command + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[InsertOrUnknown-0] + _ = x[InsertOrReplace-1] + _ = x[InsertOrRollback-2] + _ = x[InsertOrAbort-3] + _ = x[InsertOrFail-4] + _ = x[InsertOrIgnore-5] +} + +const _InsertOr_name = "InsertOrUnknownInsertOrReplaceInsertOrRollbackInsertOrAbortInsertOrFailInsertOrIgnore" + +var _InsertOr_index = [...]uint8{0, 15, 30, 46, 59, 71, 85} + +func (i InsertOr) String() string { + if i >= InsertOr(len(_InsertOr_index)-1) { + return "InsertOr(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _InsertOr_name[_InsertOr_index[i]:_InsertOr_index[i+1]] +} diff --git a/internal/engine/builtin.go b/internal/engine/builtin.go index da5b20cc..0992230e 100644 --- a/internal/engine/builtin.go +++ b/internal/engine/builtin.go @@ -1,3 +1,12 @@ +// This file contains implementations for builtin functions, such as RAND() or +// NOW(). The arguments for the herein implemented functions differ from those +// that are required in the SQL statement. For example, COUNT(x) takes only one +// argument, but builtinCount requires many values. The engine is responsible to +// interpret COUNT(x), and instead of the single value 'x', pass in all values +// in the column 'x'. How SQL arguments are to be interpreted, depends on the +// SQL function. The builtin functions in this file don't access the result +// table, but instead rely on the engine to pass in the correct values. + package engine import ( @@ -15,14 +24,20 @@ var ( _ = builtinMin ) -func builtinNow(tp timeProvider) (types.Value, error) { +// builtinNow returns a new date value, containing the timestamp provided by the +// given timeProvider. +func builtinNow(tp timeProvider) (types.DateValue, error) { return types.NewDate(tp()), nil } -func builtinCount(args ...types.Value) (types.Value, error) { - return nil, ErrUnimplemented +// builtinCount returns a new integral value, representing the count of the +// passed in values. +func builtinCount(args ...types.Value) (types.IntegerValue, error) { + return types.NewInteger(int64(len(args))), nil } +// builtinUCase maps all passed in string values to new string values with the +// internal string value folded to upper case. func builtinUCase(args ...types.StringValue) ([]types.StringValue, error) { var output []types.StringValue for _, arg := range args { @@ -31,6 +46,8 @@ func builtinUCase(args ...types.StringValue) ([]types.StringValue, error) { return output, nil } +// builtinLCase maps all passed in string values to new string values with the +// internal string value folded to lower case. func builtinLCase(args ...types.StringValue) ([]types.StringValue, error) { var output []types.StringValue for _, arg := range args { @@ -39,6 +56,8 @@ func builtinLCase(args ...types.StringValue) ([]types.StringValue, error) { return output, nil } +// builtinMax returns the largest value out of all passed in values. The largest +// value is determined by comparing one element to all others. func builtinMax(args ...types.Value) (types.Value, error) { if len(args) == 0 { return nil, nil @@ -66,6 +85,8 @@ func builtinMax(args ...types.Value) (types.Value, error) { return largest, nil } +// builtinMin returns the smallest value out of all passed in values. The +// smallest value is determined by comparing one element to all others. func builtinMin(args ...types.Value) (types.Value, error) { if len(args) == 0 { return nil, nil @@ -93,6 +114,7 @@ func builtinMin(args ...types.Value) (types.Value, error) { return smallest, nil } +// ensureSameType returns an error if not all given values have the same type. func ensureSameType(args ...types.Value) error { if len(args) == 0 { return nil diff --git a/internal/engine/types/function_value.go b/internal/engine/types/function_value.go index d8edb16f..a7b555e5 100644 --- a/internal/engine/types/function_value.go +++ b/internal/engine/types/function_value.go @@ -7,7 +7,10 @@ import ( var _ Value = (*FunctionValue)(nil) -// FunctionValue is a value of type Function. +// FunctionValue is a value of type Function. This can not be called, it is +// simply a shell that holds a function name and the arguments, that were used +// in the SQL statement. It is the engine's responsibility, to execute the +// appropriate code to make this function call happen. type FunctionValue struct { value diff --git a/internal/engine/types/integer_type.go b/internal/engine/types/integer_type.go new file mode 100644 index 00000000..ac9f03dc --- /dev/null +++ b/internal/engine/types/integer_type.go @@ -0,0 +1,36 @@ +package types + +var ( + // Integer is the date type. Integers are comparable. The name of this type + // is "Integer". + Integer = IntegerType{ + typ: typ{ + name: "Integer", + }, + } +) + +// IntegerType is a comparable type. +type IntegerType struct { + typ +} + +// Compare compares two date values. For this to succeed, both values must be of +// type IntegerValue and be not nil. A date later than another date is considered +// larger. This method will return 1 if left>right, 0 if left==right, and -1 if +// left leftInteger { + return 1, nil + } + return 0, nil +} diff --git a/internal/engine/types/integer_value.go.go b/internal/engine/types/integer_value.go.go new file mode 100644 index 00000000..59708a1e --- /dev/null +++ b/internal/engine/types/integer_value.go.go @@ -0,0 +1,29 @@ +package types + +import ( + "strconv" +) + +var _ Value = (*IntegerValue)(nil) + +// IntegerValue is a value of type Integer. +type IntegerValue struct { + value + + // Value is the underlying primitive value. + Value int64 +} + +// NewInteger creates a new value of type Integer. +func NewInteger(v int64) IntegerValue { + return IntegerValue{ + value: value{ + typ: Integer, + }, + Value: v, + } +} + +func (v IntegerValue) String() string { + return strconv.FormatInt(v.Value, 10) +} diff --git a/internal/engine/types/null_value.go b/internal/engine/types/null_value.go new file mode 100644 index 00000000..b547d964 --- /dev/null +++ b/internal/engine/types/null_value.go @@ -0,0 +1,24 @@ +package types + +import "fmt" + +var _ Value = (*NullValue)(nil) + +// NullValue is a value of type null. It has no actual value. +type NullValue struct { + value +} + +// NewNull creates a new value of type null, with the given type. +func NewNull(t Type) NullValue { + return NullValue{ + value: value{ + typ: t, + }, + } +} + +// String "NULL", appended to the type of this value in parenthesis. +func (v NullValue) String() string { + return fmt.Sprintf("(%v)NULL", v.typ.Name()) +} diff --git a/internal/engine/types/real_type.go b/internal/engine/types/real_type.go new file mode 100644 index 00000000..3d90cc04 --- /dev/null +++ b/internal/engine/types/real_type.go @@ -0,0 +1,34 @@ +package types + +var ( + // Real is the date type. Reals are comparable. The name of this type + // is "Real". + Real = RealType{ + typ: typ{ + name: "Real", + }, + } +) + +// RealType is a comparable type. +type RealType struct { + typ +} + +// Compare compares two real values. For this to succeed, both values must be of +// type RealValue and be not nil. +func (t RealType) Compare(left, right Value) (int, error) { + if err := t.ensureHaveThisType(left, right); err != nil { + return 0, err + } + + leftReal := left.(RealValue).Value + rightReal := right.(RealValue).Value + + if leftReal < rightReal { + return -1, nil + } else if rightReal > leftReal { + return 1, nil + } + return 0, nil +} diff --git a/internal/engine/types/real_value.go b/internal/engine/types/real_value.go new file mode 100644 index 00000000..2a8fb62e --- /dev/null +++ b/internal/engine/types/real_value.go @@ -0,0 +1,29 @@ +package types + +import ( + "strconv" +) + +var _ Value = (*RealValue)(nil) + +// RealValue is a value of type Real. +type RealValue struct { + value + + // Value is the underlying primitive value. + Value float64 +} + +// NewReal creates a new value of type Real. +func NewReal(v float64) RealValue { + return RealValue{ + value: value{ + typ: Real, + }, + Value: v, + } +} + +func (v RealValue) String() string { + return strconv.FormatFloat(v.Value, 'e', -1, 8) +} diff --git a/internal/engine/types/serializer.go b/internal/engine/types/serializer.go index 3e93f796..8c1f4f4b 100644 --- a/internal/engine/types/serializer.go +++ b/internal/engine/types/serializer.go @@ -1,5 +1,11 @@ package types +import "encoding/binary" + +var ( + byteOrder = binary.BigEndian +) + // Serializer wraps two basic methods for two-way serializing values. Which // values can be serialized and deserialized, is up to the implementing object. // It must be documented, what can and can not be serialized and deserialized. @@ -7,3 +13,13 @@ type Serializer interface { Serialize(Value) ([]byte, error) Deserialize([]byte) (Value, error) } + +// frame prepends an 4 byte big endian encoded unsigned integer to the given +// data, which represents the length of the data. +func frame(data []byte) []byte { + size := len(data) + result := make([]byte, 4+len(data)) + byteOrder.PutUint32(result, uint32(size)) + copy(result[4:], data) + return result +} diff --git a/internal/engine/types/string_type.go b/internal/engine/types/string_type.go index 4c854fea..7bcf7276 100644 --- a/internal/engine/types/string_type.go +++ b/internal/engine/types/string_type.go @@ -16,6 +16,7 @@ var _ Type = (*StringType)(nil) var _ Value = (*StringValue)(nil) var _ Comparator = (*StringType)(nil) var _ Caster = (*StringType)(nil) +var _ Serializer = (*StringType)(nil) // StringType is a comparable type. type StringType struct { @@ -43,3 +44,30 @@ func (StringType) Cast(v Value) (Value, error) { } return NewString(v.String()), nil } + +// Serialize serializes the internal string value as 4-byte-framed byte +// sequence. +func (t StringType) Serialize(v Value) ([]byte, error) { + if err := t.ensureHasThisType(v); err != nil { + return nil, err + } + + str := v.(StringValue).Value + payload := frame([]byte(str)) + payloadLength := len(payload) + data := make([]byte, 1+payloadLength) // string byte length + 1 for the indicator + data[0] = byte(TypeIndicatorString) + copy(data[1:], str) + + return data, nil +} + +// Deserialize reads the data size from the first 4 passed-in bytes, and then +// converts the rest of the bytes to a string leveraging the Go runtime. +func (t StringType) Deserialize(data []byte) (Value, error) { + payloadSize := int(byteOrder.Uint32(data[0:])) + if payloadSize+4 != len(data) { + return nil, ErrDataSizeMismatch(payloadSize+4, len(data)) + } + return NewString(string(data[4:])), nil +} diff --git a/internal/engine/types/typeindicator_string.go b/internal/engine/types/typeindicator_string.go new file mode 100644 index 00000000..5c6f121d --- /dev/null +++ b/internal/engine/types/typeindicator_string.go @@ -0,0 +1,28 @@ +// Code generated by "stringer -type=TypeIndicator"; DO NOT EDIT. + +package types + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[TypeIndicatorUnknown-0] + _ = x[TypeIndicatorBool-1] + _ = x[TypeIndicatorDate-2] + _ = x[TypeIndicatorInteger-3] + _ = x[TypeIndicatorReal-4] + _ = x[TypeIndicatorString-5] +} + +const _TypeIndicator_name = "TypeIndicatorUnknownTypeIndicatorBoolTypeIndicatorDateTypeIndicatorIntegerTypeIndicatorRealTypeIndicatorString" + +var _TypeIndicator_index = [...]uint8{0, 20, 37, 54, 74, 91, 110} + +func (i TypeIndicator) String() string { + if i >= TypeIndicator(len(_TypeIndicator_index)-1) { + return "TypeIndicator(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _TypeIndicator_name[_TypeIndicator_index[i]:_TypeIndicator_index[i+1]] +} diff --git a/internal/engine/types/types.go b/internal/engine/types/types.go new file mode 100644 index 00000000..561a31c4 --- /dev/null +++ b/internal/engine/types/types.go @@ -0,0 +1,33 @@ +package types + +//go:generate stringer -type=TypeIndicator + +// TypeIndicator is a type that is used to serialize types. Each indicator +// corresponds to a type. +type TypeIndicator uint8 + +// Known type indicators corresponding to known types. +const ( + TypeIndicatorUnknown TypeIndicator = iota + TypeIndicatorBool + TypeIndicatorDate + TypeIndicatorInteger + TypeIndicatorReal + TypeIndicatorString +) + +var ( + byIndicator = map[TypeIndicator]Type{ + TypeIndicatorBool: Bool, + TypeIndicatorDate: Date, + TypeIndicatorInteger: Integer, + TypeIndicatorReal: Real, + TypeIndicatorString: String, + } +) + +// ByIndicator accepts a type indicator and returns the corresponding type. If +// the returned type is nil, the type indicator is unknown. +func ByIndicator(indicator TypeIndicator) Type { + return byIndicator[indicator] +} diff --git a/internal/engine/types/value_test.go b/internal/engine/types/value_test.go index d6ba8aa7..819cd80d 100644 --- a/internal/engine/types/value_test.go +++ b/internal/engine/types/value_test.go @@ -11,5 +11,6 @@ func TestValue_Is(t *testing.T) { v := NewString("foobar") assert.True(v.Is(String)) // Is must yield the same result as .Type() == + assert.True(v.Is(v.Type())) assert.Equal(String, v.Type()) } From 0c339535aca8a0b9fbc350bd849557141bba5853 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 2 Jul 2020 00:16:46 +0200 Subject: [PATCH 52/77] Add literal parser and support for numeric literal --- internal/engine/expression.go | 3 + internal/engine/numeric_parser.go | 177 +++++++++++++++++++ internal/engine/numeric_parser_bench_test.go | 24 +++ internal/engine/numeric_parser_test.go | 123 +++++++++++++ internal/test/issue187_test.go | 2 +- internal/test/testdata/issue187/output | 6 +- 6 files changed, 331 insertions(+), 4 deletions(-) create mode 100644 internal/engine/numeric_parser.go create mode 100644 internal/engine/numeric_parser_bench_test.go create mode 100644 internal/engine/numeric_parser_test.go diff --git a/internal/engine/expression.go b/internal/engine/expression.go index fafbec3c..c504e169 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -36,6 +36,9 @@ func (e Engine) evaluateMultipleExpressions(ctx ExecutionContext, exprs []comman } func (e Engine) evaluateLiteralExpr(ctx ExecutionContext, expr command.LiteralExpr) (types.Value, error) { + if numVal, ok := ToNumericValue(expr.Value); ok { + return numVal, nil + } return types.NewString(expr.Value), nil } diff --git a/internal/engine/numeric_parser.go b/internal/engine/numeric_parser.go new file mode 100644 index 00000000..1742787e --- /dev/null +++ b/internal/engine/numeric_parser.go @@ -0,0 +1,177 @@ +package engine + +import ( + "bytes" + "strconv" + "strings" + + "github.com/tomarrell/lbadd/internal/engine/types" +) + +type state func(*numericParser) state + +type numericParser struct { + candidate string + index int + + isReal bool + isHexadecimal bool + isErronous bool + hasDigitsBeforeExponent bool + + value *bytes.Buffer + + current state +} + +// ToNumericValue checks whether the given string is of this form +// https://www.sqlite.org/lang_expr.html#literal_values_constants_ . If it is, a +// appropriate value is returned (either types.IntegerValue or types.RealValue). +// If it is not, false will be returned. This will never return the NULL value, +// even if the given string is empty. In that case, nil and false is returned. +func ToNumericValue(s string) (types.Value, bool) { + p := numericParser{ + candidate: s, + index: 0, + + value: &bytes.Buffer{}, + + current: stateInitial, + } + p.parse() + if p.isErronous { + return nil, false + } + switch { + case p.isReal: + val, err := strconv.ParseFloat(p.value.String(), 64) + if err != nil { + return nil, false + } + return types.NewReal(val), true + case p.isHexadecimal: + val, err := strconv.ParseInt(p.value.String(), 16, 64) + if err != nil { + return nil, false + } + return types.NewInteger(val), true + default: // is integral + val, err := strconv.ParseInt(p.value.String(), 10, 64) + if err != nil { + return nil, false + } + return types.NewInteger(val), true + } +} + +func (p numericParser) done() bool { + return p.index >= len(p.candidate) +} + +func (p *numericParser) parse() { + for p.current != nil && !p.done() { + p.current = p.current(p) + } +} + +func (p *numericParser) get() byte { + return p.candidate[p.index] +} + +func (p *numericParser) step() { + _ = p.value.WriteByte(p.get()) + p.index++ +} + +func stateInitial(p *numericParser) state { + switch { + case strings.HasPrefix(p.candidate, "0x"): + p.index += 2 + p.isHexadecimal = true + return stateHex + case isDigit(p.get()): + return stateFirstDigits + case p.get() == '.': + return stateDecimalPoint + } + return nil +} + +func stateHex(p *numericParser) state { + if isDigit(p.get()) || (p.get()-'A' <= 15) { + p.step() + return stateHex + } + p.isErronous = true + return nil +} + +func stateFirstDigits(p *numericParser) state { + if isDigit(p.get()) { + p.hasDigitsBeforeExponent = true + p.step() + return stateFirstDigits + } else if p.get() == '.' { + return stateDecimalPoint + } + p.isErronous = true + return nil +} + +func stateDecimalPoint(p *numericParser) state { + if p.get() == '.' { + p.step() + p.isReal = true + return stateSecondDigits + } + p.isErronous = true + return nil +} + +func stateSecondDigits(p *numericParser) state { + if isDigit(p.get()) { + p.hasDigitsBeforeExponent = true + p.step() + return stateSecondDigits + } else if p.get() == 'E' { + if p.hasDigitsBeforeExponent { + return stateExponent + } + p.isErronous = true // if there were no first digits, + } + p.isErronous = true + return nil +} + +func stateExponent(p *numericParser) state { + if p.get() == 'E' { + p.step() + return stateOptionalSign + } + p.isErronous = true + return nil +} + +func stateOptionalSign(p *numericParser) state { + if p.get() == '+' || p.get() == '-' { + p.step() + return stateThirdDigits + } else if isDigit(p.get()) { + return stateThirdDigits + } + p.isErronous = true + return nil +} + +func stateThirdDigits(p *numericParser) state { + if isDigit(p.get()) { + p.step() + return stateThirdDigits + } + p.isErronous = true + return nil +} + +func isDigit(b byte) bool { + return b-'0' <= 9 +} diff --git a/internal/engine/numeric_parser_bench_test.go b/internal/engine/numeric_parser_bench_test.go new file mode 100644 index 00000000..2bf44dec --- /dev/null +++ b/internal/engine/numeric_parser_bench_test.go @@ -0,0 +1,24 @@ +package engine + +import ( + "testing" + + "github.com/tomarrell/lbadd/internal/engine/types" +) + +func BenchmarkToNumericValue(b *testing.B) { + str := "75610342.92389E-21423" + expVal := 75610342.92389E-21423 + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + val, ok := ToNumericValue(str) + if !ok { + b.FailNow() + } + if expVal != val.(types.RealValue).Value { + b.FailNow() + } + } +} diff --git a/internal/engine/numeric_parser_test.go b/internal/engine/numeric_parser_test.go new file mode 100644 index 00000000..a17917d8 --- /dev/null +++ b/internal/engine/numeric_parser_test.go @@ -0,0 +1,123 @@ +package engine + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/engine/types" +) + +func TestToNumericValue(t *testing.T) { + tests := []struct { + name string + s string + want types.Value + want1 bool + }{ + { + "empty", + "", + nil, + false, + }, + { + "half hex", + "0x", + nil, + false, + }, + { + "hex", + "0x123ABC", + types.NewInteger(0x123ABC), + true, + }, + { + "hex", + "0xFF", + types.NewInteger(0xFF), + true, + }, + { + "full hex spectrum", + "0x0123456789ABCDEF", + types.NewInteger(0x0123456789ABCDEF), + true, + }, + { + "full hex spectrum", + "0xFEDCBA987654321", + types.NewInteger(0xFEDCBA987654321), + true, + }, + { + "small integral", + "0", + types.NewInteger(0), + true, + }, + { + "small integral", + "1", + types.NewInteger(1), + true, + }, + { + "integral", + "1234567", + types.NewInteger(1234567), + true, + }, + { + "integral", + "42", + types.NewInteger(42), + true, + }, + { + "real", + "0.0", + types.NewReal(0), + true, + }, + { + "real", + ".0", + types.NewReal(0), + true, + }, + { + "only decimal point", + ".", + nil, + false, + }, + { + "real with exponent", + ".0E2", + types.NewReal(0), + true, + }, + { + "real with exponent", + "5.7E-242", + types.NewReal(5.7E-242), + true, + }, + { + "invalid exponent", + ".0e2", // lowercase 'e' is not allowed + nil, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + got, ok := ToNumericValue(tt.s) + assert.Equal(tt.want, got) + assert.Equal(tt.want1, ok) + }) + } +} diff --git a/internal/test/issue187_test.go b/internal/test/issue187_test.go index 41bb5cf3..7aabbc1c 100644 --- a/internal/test/issue187_test.go +++ b/internal/test/issue187_test.go @@ -5,6 +5,6 @@ import "testing" func TestIssue187(t *testing.T) { RunAndCompare(t, Test{ Name: "issue187", - Statement: "VALUES (1,2,3), (4,5,6)", + Statement: `VALUES (1,"2",3), (4,"5",6)`, }) } diff --git a/internal/test/testdata/issue187/output b/internal/test/testdata/issue187/output index f9037d35..04cd20f6 100644 --- a/internal/test/testdata/issue187/output +++ b/internal/test/testdata/issue187/output @@ -1,3 +1,3 @@ -String String String -1 2 3 -4 5 6 +Integer String Integer +1 "2" 3 +4 "5" 6 From 161fb049ac757dcfa96e7834dff56b0fcdb4b427 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 2 Jul 2020 00:34:52 +0200 Subject: [PATCH 53/77] Create tests to reflect requirements of #188 --- internal/test/issue188_test.go | 10 ++++++++++ internal/test/testdata/issue187/output | 4 ++-- internal/test/testdata/issue188/output | 2 ++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 internal/test/issue188_test.go create mode 100644 internal/test/testdata/issue188/output diff --git a/internal/test/issue188_test.go b/internal/test/issue188_test.go new file mode 100644 index 00000000..cd07a6d2 --- /dev/null +++ b/internal/test/issue188_test.go @@ -0,0 +1,10 @@ +package test + +import "testing" + +func TestIssue188(t *testing.T) { + RunAndCompare(t, Test{ + Name: "issue188", + Statement: `VALUES ("abc", "a\"bc", "a\u0042c")`, + }) +} diff --git a/internal/test/testdata/issue187/output b/internal/test/testdata/issue187/output index 04cd20f6..429fb9ae 100644 --- a/internal/test/testdata/issue187/output +++ b/internal/test/testdata/issue187/output @@ -1,3 +1,3 @@ Integer String Integer -1 "2" 3 -4 "5" 6 +1 2 3 +4 5 6 diff --git a/internal/test/testdata/issue188/output b/internal/test/testdata/issue188/output new file mode 100644 index 00000000..fc5f7099 --- /dev/null +++ b/internal/test/testdata/issue188/output @@ -0,0 +1,2 @@ +String String String +abc a"bc abc From efb21d682a9d905ba2a236f3c118a3d3678b2dc6 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 2 Jul 2020 01:04:00 +0200 Subject: [PATCH 54/77] Fix #188 --- internal/engine/expression.go | 8 +++++++- internal/engine/numeric_parser.go | 26 +++++++++++++++----------- internal/test/issue188_test.go | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/internal/engine/expression.go b/internal/engine/expression.go index c504e169..47999814 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -2,6 +2,7 @@ package engine import ( "fmt" + "strconv" "github.com/tomarrell/lbadd/internal/compiler/command" "github.com/tomarrell/lbadd/internal/engine/types" @@ -39,7 +40,12 @@ func (e Engine) evaluateLiteralExpr(ctx ExecutionContext, expr command.LiteralEx if numVal, ok := ToNumericValue(expr.Value); ok { return numVal, nil } - return types.NewString(expr.Value), nil + resolved, err := strconv.Unquote(expr.Value) + if err != nil { + // use the original string + return types.NewString(expr.Value), nil + } + return types.NewString(resolved), nil } func (e Engine) evaluateFunctionExpr(ctx ExecutionContext, expr command.FunctionExpr) (types.Value, error) { diff --git a/internal/engine/numeric_parser.go b/internal/engine/numeric_parser.go index 1742787e..925c4ffd 100644 --- a/internal/engine/numeric_parser.go +++ b/internal/engine/numeric_parser.go @@ -8,7 +8,7 @@ import ( "github.com/tomarrell/lbadd/internal/engine/types" ) -type state func(*numericParser) state +type numericParserState func(*numericParser) numericParserState type numericParser struct { candidate string @@ -21,7 +21,7 @@ type numericParser struct { value *bytes.Buffer - current state + current numericParserState } // ToNumericValue checks whether the given string is of this form @@ -83,7 +83,7 @@ func (p *numericParser) step() { p.index++ } -func stateInitial(p *numericParser) state { +func stateInitial(p *numericParser) numericParserState { switch { case strings.HasPrefix(p.candidate, "0x"): p.index += 2 @@ -97,8 +97,8 @@ func stateInitial(p *numericParser) state { return nil } -func stateHex(p *numericParser) state { - if isDigit(p.get()) || (p.get()-'A' <= 15) { +func stateHex(p *numericParser) numericParserState { + if isHexDigit(p.get()) { p.step() return stateHex } @@ -106,7 +106,7 @@ func stateHex(p *numericParser) state { return nil } -func stateFirstDigits(p *numericParser) state { +func stateFirstDigits(p *numericParser) numericParserState { if isDigit(p.get()) { p.hasDigitsBeforeExponent = true p.step() @@ -118,7 +118,7 @@ func stateFirstDigits(p *numericParser) state { return nil } -func stateDecimalPoint(p *numericParser) state { +func stateDecimalPoint(p *numericParser) numericParserState { if p.get() == '.' { p.step() p.isReal = true @@ -128,7 +128,7 @@ func stateDecimalPoint(p *numericParser) state { return nil } -func stateSecondDigits(p *numericParser) state { +func stateSecondDigits(p *numericParser) numericParserState { if isDigit(p.get()) { p.hasDigitsBeforeExponent = true p.step() @@ -143,7 +143,7 @@ func stateSecondDigits(p *numericParser) state { return nil } -func stateExponent(p *numericParser) state { +func stateExponent(p *numericParser) numericParserState { if p.get() == 'E' { p.step() return stateOptionalSign @@ -152,7 +152,7 @@ func stateExponent(p *numericParser) state { return nil } -func stateOptionalSign(p *numericParser) state { +func stateOptionalSign(p *numericParser) numericParserState { if p.get() == '+' || p.get() == '-' { p.step() return stateThirdDigits @@ -163,7 +163,7 @@ func stateOptionalSign(p *numericParser) state { return nil } -func stateThirdDigits(p *numericParser) state { +func stateThirdDigits(p *numericParser) numericParserState { if isDigit(p.get()) { p.step() return stateThirdDigits @@ -175,3 +175,7 @@ func stateThirdDigits(p *numericParser) state { func isDigit(b byte) bool { return b-'0' <= 9 } + +func isHexDigit(b byte) bool { + return isDigit(b) || (b-'A' <= 15) || (b-'a' <= 15) +} diff --git a/internal/test/issue188_test.go b/internal/test/issue188_test.go index cd07a6d2..d8d5b8d0 100644 --- a/internal/test/issue188_test.go +++ b/internal/test/issue188_test.go @@ -5,6 +5,6 @@ import "testing" func TestIssue188(t *testing.T) { RunAndCompare(t, Test{ Name: "issue188", - Statement: `VALUES ("abc", "a\"bc", "a\u0042c")`, + Statement: `VALUES ("abc", "a\"bc", "a\u0062c")`, }) } From ecf1e054d6406ed77b082ab662193da060c338ea Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 2 Jul 2020 12:11:40 +0200 Subject: [PATCH 55/77] Add more godoc and another test for builtin max --- internal/engine/builtin_test.go | 18 ++++++++++++++++++ internal/engine/expression.go | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/internal/engine/builtin_test.go b/internal/engine/builtin_test.go index 2350efb9..c4485167 100644 --- a/internal/engine/builtin_test.go +++ b/internal/engine/builtin_test.go @@ -43,6 +43,24 @@ func Test_builtinMax(t *testing.T) { types.NewBool(true), false, }, + { + "integers", + args{ + []types.Value{ + types.NewInteger(3456), + types.NewInteger(0), + types.NewInteger(76526), + types.NewInteger(1), + types.NewInteger(23685245), + types.NewInteger(45634), + types.NewInteger(1345), + types.NewInteger(346), + types.NewInteger(5697), + }, + }, + types.NewInteger(23685245), + false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/engine/expression.go b/internal/engine/expression.go index 47999814..0063783e 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -36,10 +36,16 @@ func (e Engine) evaluateMultipleExpressions(ctx ExecutionContext, exprs []comman return vals, nil } +// evaluateLiteralExpr evaluates the given literal expression based on the +// current execution context. The returned value will either be a numeric value +// (integer or real) or a string value. func (e Engine) evaluateLiteralExpr(ctx ExecutionContext, expr command.LiteralExpr) (types.Value, error) { + // Check whether the expression value is a numeric literal. In the future, + // this evaluation might depend on the execution context. if numVal, ok := ToNumericValue(expr.Value); ok { return numVal, nil } + // if not a numeric literal, remove quotes and resolve escapes resolved, err := strconv.Unquote(expr.Value) if err != nil { // use the original string From 78382e728b0fa40b761b8a629a5460acd459720b Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 2 Jul 2020 12:11:54 +0200 Subject: [PATCH 56/77] Introduce a string representation for profiles --- internal/engine/profile/profile.go | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/internal/engine/profile/profile.go b/internal/engine/profile/profile.go index 32f156bd..5f757f52 100644 --- a/internal/engine/profile/profile.go +++ b/internal/engine/profile/profile.go @@ -1,7 +1,60 @@ package profile +import ( + "bytes" + "fmt" + "sort" + "strings" + "time" +) + // Profile is a collection of profiling events that were collected by a // profiler. type Profile struct { Events []Event } + +func (p Profile) String() string { + var buf bytes.Buffer + + evts := p.Events + sort.Slice(evts, func(i, j int) bool { return strings.Compare(evts[i].Object.String(), evts[j].Object.String()) < 0 }) + + firstEvt, lastEvt := evts[0], evts[0] + for _, evt := range evts { + if evt.Start.Before(firstEvt.Start) { + firstEvt = evt + } + if evt.Start.After(lastEvt.Start) { + lastEvt = evt + } + } + + startTime := firstEvt.Start + endTime := lastEvt.Start.Add(lastEvt.Duration) + + _, _ = fmt.Fprintf(&buf, "Profile\n\tfrom %v\n\tto %v\n\ttook %v\n", fmtTime(startTime), fmtTime(endTime), endTime.Sub(startTime)) + _, _ = fmt.Fprintf(&buf, "Events (%v):\n", len(evts)) + + buckets := make(map[string][]Event) + for _, evt := range evts { + str := evt.Object.String() + buckets[str] = append(buckets[str], evt) + } + + for bucket, bucketEvts := range buckets { + _, _ = fmt.Fprintf(&buf, "\t%v (%v events)\n", bucket, len(bucketEvts)) + totalDuration := 0 * time.Second + for _, bucketEvt := range bucketEvts { + totalDuration += bucketEvt.Duration + _, _ = fmt.Fprintf(&buf, "\t\t- %v took %v\n", fmtTime(bucketEvt.Start), bucketEvt.Duration) + } + _, _ = fmt.Fprintf(&buf, "\t\taverage %v, total %v\n", totalDuration/time.Duration(len(bucketEvts)), totalDuration) + } + + return buf.String() +} + +func fmtTime(t time.Time) string { + return t.Format(time.RFC3339Nano) +} From 8c3ea22fc371533562a4937cdcb3fd3c2c211808 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 2 Jul 2020 12:12:02 +0200 Subject: [PATCH 57/77] Create an example test case with profiling --- internal/test/issue187_test.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/internal/test/issue187_test.go b/internal/test/issue187_test.go index 7aabbc1c..4728bbbf 100644 --- a/internal/test/issue187_test.go +++ b/internal/test/issue187_test.go @@ -1,6 +1,11 @@ package test -import "testing" +import ( + "testing" + + "github.com/tomarrell/lbadd/internal/engine" + "github.com/tomarrell/lbadd/internal/engine/profile" +) func TestIssue187(t *testing.T) { RunAndCompare(t, Test{ @@ -8,3 +13,15 @@ func TestIssue187(t *testing.T) { Statement: `VALUES (1,"2",3), (4,"5",6)`, }) } + +func TestIssue187WithProfile(t *testing.T) { + prof := profile.NewProfiler() + RunAndCompare(t, Test{ + Name: "issue187", + Statement: `VALUES (1,"2",3), (4,"5",6)`, + EngineOptions: []engine.Option{ + engine.WithProfiler(prof), + }, + }) + t.Logf("engine profile:\n%v", prof.Profile().String()) +} From 11fa3acf98c880ad30b78f868a988aa5878517c0 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 2 Jul 2020 12:56:56 +0200 Subject: [PATCH 58/77] Add documentation on profile package --- internal/engine/profile/doc.go | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 internal/engine/profile/doc.go diff --git a/internal/engine/profile/doc.go b/internal/engine/profile/doc.go new file mode 100644 index 00000000..0d385bd9 --- /dev/null +++ b/internal/engine/profile/doc.go @@ -0,0 +1,35 @@ +// Package profile implements profiling with generic events. An event can be +// defined by the user of this package and just needs to implement fmt.Stringer. +// This profiler works with string-based events. Use the profiler like this in +// your code. +// +// ... +// type Evt string +// func (e Evt) String() string { return string(e) } +// ... +// const MyEvt = Evt("my expensive func") +// ... +// func main() { +// prof = profile.NewProfiler() +// MyExpensiveFunc() +// fmt.Println(prof.Profile().String()) // will print the profile +// } +// ... +// func MyExpensiveFunc() { +// defer prof.Enter(MyEvt).Exit() +// ... +// } +// +// The above example will print a profile with one event, the respective +// timestamps and durations etc. +// +// NOTE: You don't need an actual profiler. All methods will also work on nil +// profilers, such as the following. +// +// var prof *profile.Profiler +// defer prof.Enter(MyEvt).Exit() +// +// Enter(...) will just create an empty Event, and Exit() will do nothing. This +// was implemented this way, so that no no-op implementation of a profiler is +// neccessary. +package profile \ No newline at end of file From 4f2711102702f391d14b0ae3b0d8b1069cb0ae81 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 2 Jul 2020 12:57:10 +0200 Subject: [PATCH 59/77] Implement random function --- internal/engine/builtin.go | 4 ++++ internal/engine/engine.go | 4 ++-- internal/engine/function.go | 2 ++ internal/engine/option.go | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/engine/builtin.go b/internal/engine/builtin.go index 0992230e..77d01d54 100644 --- a/internal/engine/builtin.go +++ b/internal/engine/builtin.go @@ -30,6 +30,10 @@ func builtinNow(tp timeProvider) (types.DateValue, error) { return types.NewDate(tp()), nil } +func builtinRand(rp randomProvider) (types.IntegerValue, error) { + return types.NewInteger(rp()), nil +} + // builtinCount returns a new integral value, representing the count of the // passed in values. func builtinCount(args ...types.Value) (types.IntegerValue, error) { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 3a105de3..3668cf4b 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -13,7 +13,7 @@ import ( ) type timeProvider func() time.Time -type randomProvider func() float64 +type randomProvider func() int64 // Engine is the component that is used to evaluate commands. type Engine struct { @@ -34,7 +34,7 @@ func New(dbFile *storage.DBFile, opts ...Option) (Engine, error) { pageCache: dbFile.Cache(), timeProvider: time.Now, - randomProvider: rand.Float64, + randomProvider: func() int64 { return int64(rand.Uint64()) }, } for _, opt := range opts { opt(&e) diff --git a/internal/engine/function.go b/internal/engine/function.go index 22fbacc7..58e4384d 100644 --- a/internal/engine/function.go +++ b/internal/engine/function.go @@ -8,6 +8,8 @@ func (e Engine) evaluateFunction(ctx ExecutionContext, fn types.FunctionValue) ( switch fn.Name { case "NOW": return builtinNow(e.timeProvider) + case "RANDOM": + return builtinRand(e.randomProvider) } return nil, ErrNoSuchFunction(fn.Name) } diff --git a/internal/engine/option.go b/internal/engine/option.go index d424678c..5d53a162 100644 --- a/internal/engine/option.go +++ b/internal/engine/option.go @@ -33,7 +33,7 @@ func WithTimeProvider(tp timeProvider) Option { // WithRandomProvider sets a random provider, which will be used by the engine // to evaluate expressions, that require a random source, such as the function -// RAND(). +// RANDOM(). func WithRandomProvider(rp randomProvider) Option { return func(e *Engine) { e.randomProvider = rp From c86da7f1f3814b6b5ac8e2d56e37723a822afcc6 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 2 Jul 2020 12:57:18 +0200 Subject: [PATCH 60/77] Add example tests with engine options --- internal/test/examples_test.go | 33 +++++++++++++++++++++++++ internal/test/testdata/example01/output | 2 ++ internal/test/testdata/example02/output | 2 ++ 3 files changed, 37 insertions(+) create mode 100644 internal/test/examples_test.go create mode 100644 internal/test/testdata/example01/output create mode 100644 internal/test/testdata/example02/output diff --git a/internal/test/examples_test.go b/internal/test/examples_test.go new file mode 100644 index 00000000..955d256e --- /dev/null +++ b/internal/test/examples_test.go @@ -0,0 +1,33 @@ +package test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/engine" +) + +func TestExample01(t *testing.T) { + RunAndCompare(t, Test{ + Name: "example01", + Statement: `VALUES (RANDOM())`, + EngineOptions: []engine.Option{ + engine.WithRandomProvider(func() int64 { return 85734726843 }), + }, + }) +} + +func TestExample02(t *testing.T) { + timestamp, err := time.Parse(time.RFC3339, "2020-07-02T14:03:27Z") + assert.NoError(t, err) + + RunAndCompare(t, Test{ + Name: "example02", + Statement: `VALUES (NOW(), RANDOM())`, + EngineOptions: []engine.Option{ + engine.WithTimeProvider(func() time.Time { return timestamp }), + engine.WithRandomProvider(func() int64 { return 85734726843 }), + }, + }) +} diff --git a/internal/test/testdata/example01/output b/internal/test/testdata/example01/output new file mode 100644 index 00000000..f0b88611 --- /dev/null +++ b/internal/test/testdata/example01/output @@ -0,0 +1,2 @@ +Integer +85734726843 diff --git a/internal/test/testdata/example02/output b/internal/test/testdata/example02/output new file mode 100644 index 00000000..985c6de7 --- /dev/null +++ b/internal/test/testdata/example02/output @@ -0,0 +1,2 @@ +Date Integer +2020-07-02T14:03:27Z 85734726843 From a026b24e23e145acf56cec0b14e6ad10f1affb6b Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Wed, 8 Jul 2020 22:00:22 +0200 Subject: [PATCH 61/77] Introduce a config page --- doc/file-format.md | 12 ++- internal/engine/storage/config.go | 48 ++++++++++ internal/engine/storage/db.go | 79 +++++++++++++--- internal/engine/storage/db_test.go | 28 ++++-- internal/engine/storage/error.go | 11 ++- internal/engine/storage/page/page_test.go | 105 ++++++++++++++++++++++ internal/test/base_test.go | 2 + 7 files changed, 262 insertions(+), 23 deletions(-) create mode 100644 internal/engine/storage/config.go diff --git a/doc/file-format.md b/doc/file-format.md index 9578cebb..75eaea95 100644 --- a/doc/file-format.md +++ b/doc/file-format.md @@ -20,8 +20,11 @@ bytes are the UTF-8 encoding of that string. * `pageCount` is a record cell whose entry is an 8 byte big endian unsigned integer, representing the amount of pages stored in the file. +* `config` is a pointer cell which points to a page, that contains configuration + parameters for this database. * `tables` is a pointer cell which points to a page, that contains pointers to - all tables that are stored in this database. The format of the table pages is explained in the next section. + all tables that are stored in this database. The format of the table pages is + explained in the next section. ### Table pages Table pages do not directly hold data of a table. Instead, they hold pointers to @@ -34,14 +37,15 @@ The keys of the three values, index page, data page and schema are as follows. * `datadefinition` is a record cell containing the schema information about this table. That is, columns, column types, references, triggers etc. How the - schema information is to be interpreted, is explained [here](#data-definition). + schema information is to be interpreted, is explained + [here](#data-definition). * `index` is a pointer cell pointing to the index page of this table. The index - page contains pages that are used as nodes in a btree. See more + page points to pages that are an actual index in the table. See more [here](#index-pages) * `data` is a pointer cell pointing to the data page of this table. See more [here](#data-pages) -### Index pages +### Index page ### Data pages A data page stores plain record in a cell. Cell values are the full records, diff --git a/internal/engine/storage/config.go b/internal/engine/storage/config.go new file mode 100644 index 00000000..db5859e1 --- /dev/null +++ b/internal/engine/storage/config.go @@ -0,0 +1,48 @@ +package storage + +import ( + "fmt" + + "github.com/tomarrell/lbadd/internal/engine/storage/page" +) + +// Config is an intermediate layer to interact with configuration keys and +// values of a database file. It holds a pointer to the database file, so this +// struct may be copied and re-used. +type Config struct { + db *DBFile +} + +// Config returns an intermediate layer to interact with the keys and values +// stored in the config page of the database file. The returned struct may be +// copied and re-used. +func (db *DBFile) Config() Config { + return Config{db} +} + +// GetString returns the value associated with the given key, or an error, if +// there is no such value. +func (c Config) GetString(key string) (string, error) { + cell, ok := c.db.configPage.Cell([]byte(key)) + if !ok { + return "", ErrNoSuchConfigKey + } + if cell.Type() != page.CellTypeRecord { + return "", fmt.Errorf("expected cell '%v' to be a record cell, but was %v", key, cell.Type()) + } + return string(cell.(page.RecordCell).Record), nil +} + +// SetString associates the given value with the given key. If there already is +// such a key present in the config, the value will be overwritten. +func (c Config) SetString(key, value string) error { + err := c.db.configPage.StoreRecordCell(page.RecordCell{ + Key: []byte(key), + Record: []byte(value), + }) + if err != nil { + return fmt.Errorf("store record cell: %w", err) + } + c.db.configPage.MarkDirty() + return nil +} diff --git a/internal/engine/storage/db.go b/internal/engine/storage/db.go index 8123e6d7..da37024a 100644 --- a/internal/engine/storage/db.go +++ b/internal/engine/storage/db.go @@ -26,6 +26,8 @@ const ( HeaderTables = "tables" // HeaderPageCount is the string key for the header page's cell "pageCount" HeaderPageCount = "pageCount" + // HeaderConfig is the string key for the header page's cell "config" + HeaderConfig = "config" ) var ( @@ -45,6 +47,7 @@ type DBFile struct { cache cache.Cache headerPage *page.Page + configPage *page.Page } // Create creates a new database in the given file with the given options. The @@ -69,34 +72,51 @@ func Create(file afero.File, opts ...Option) (*DBFile, error) { if err != nil { return nil, fmt.Errorf("allocate header page: %w", err) } + configPage, err := mgr.AllocateNew() + if err != nil { + return nil, fmt.Errorf("allocate config page: %w", err) + } tablesPage, err := mgr.AllocateNew() if err != nil { return nil, fmt.Errorf("allocate tables page: %w", err) } + // store page count if err := headerPage.StoreRecordCell(page.RecordCell{ Key: []byte(HeaderPageCount), - Record: encodeUint64(2), // header and tables page + Record: encodeUint64(3), // header, config and tables page }); err != nil { - return nil, fmt.Errorf("store record cell: %w", err) + return nil, fmt.Errorf("store page count: %w", err) } + // store pointer to config page + if err := headerPage.StorePointerCell(page.PointerCell{ + Key: []byte(HeaderConfig), + Pointer: configPage.ID(), + }); err != nil { + return nil, fmt.Errorf("store config pointer: %w", err) + } + // store pointer to tables page if err := headerPage.StorePointerCell(page.PointerCell{ Key: []byte(HeaderTables), Pointer: tablesPage.ID(), }); err != nil { - return nil, fmt.Errorf("store pointer cell: %w", err) + return nil, fmt.Errorf("store tables pointer: %w", err) } err = mgr.WritePage(headerPage) // immediately flush if err != nil { return nil, fmt.Errorf("write header page: %w", err) } + err = mgr.WritePage(configPage) // immediately flush + if err != nil { + return nil, fmt.Errorf("write config page: %w", err) + } err = mgr.WritePage(tablesPage) // immediately flush if err != nil { return nil, fmt.Errorf("write tables page: %w", err) } - return newDB(file, mgr, headerPage, opts...), nil + return newDB(file, mgr, headerPage, opts...) } // Open opens and validates a given file and creates a (*DBFile) with the given @@ -117,13 +137,14 @@ func Open(file afero.File, opts ...Option) (*DBFile, error) { return nil, fmt.Errorf("read header page: %w", err) } - return newDB(file, mgr, headerPage, opts...), nil + return newDB(file, mgr, headerPage, opts...) } // AllocateNewPage allocates and immediately persists a new page in secondary // storage. This will fail if the DBFile is closed. After this method returns, -// the allocated page can immediately be found by the cache, and you can use the -// returned page ID to load the page through the cache. +// the allocated page can immediately be found by the cache (it is not loaded +// yet), and you can use the returned page ID to load the page through the +// cache. func (db *DBFile) AllocateNewPage() (page.ID, error) { if db.Closed() { return 0, ErrClosed @@ -152,8 +173,14 @@ func (db *DBFile) Cache() cache.Cache { } // Close will close the underlying cache, as well as page manager, as well as -// file. +// the file. Everything will be closed after writing the config and header page. func (db *DBFile) Close() error { + if err := db.pageManager.WritePage(db.headerPage); err != nil { + return fmt.Errorf("write header page: %w", err) + } + if err := db.pageManager.WritePage(db.configPage); err != nil { + return fmt.Errorf("write config page: %w", err) + } _ = db.cache.Close() _ = db.pageManager.Close() _ = db.file.Close() @@ -167,7 +194,7 @@ func (db *DBFile) Closed() bool { } // newDB creates a new DBFile from the given objects, and applies all options. -func newDB(file afero.File, mgr *PageManager, headerPage *page.Page, opts ...Option) *DBFile { +func newDB(file afero.File, mgr *PageManager, headerPage *page.Page, opts ...Option) (*DBFile, error) { db := &DBFile{ log: zerolog.Nop(), cacheSize: DefaultCacheSize, @@ -180,9 +207,30 @@ func newDB(file afero.File, mgr *PageManager, headerPage *page.Page, opts ...Opt opt(db) } + if err := db.initialize(); err != nil { + return nil, fmt.Errorf("initialize: %w", err) + } + db.cache = cache.NewLRUCache(db.cacheSize, mgr) - return db + return db, nil +} + +func (db *DBFile) initialize() error { + // get config page id + cfgPageID, err := pointerCellValue(db.headerPage, HeaderConfig) + if err != nil { + return err + } + + // read config page + cfgPage, err := db.pageManager.ReadPage(cfgPageID) + if err != nil { + return fmt.Errorf("can't read config page: %w", err) + } + db.configPage = cfgPage + + return nil } // incrementHeaderPageCount will increment the 8 byte uint64 in the @@ -204,3 +252,14 @@ func encodeUint64(v uint64) []byte { byteOrder.PutUint64(buf, v) return buf } + +func pointerCellValue(p *page.Page, cellKey string) (page.ID, error) { + cell, ok := p.Cell([]byte(cellKey)) + if !ok { + return 0, ErrNoSuchCell(cellKey) + } + if cell.Type() != page.CellTypePointer { + return 0, fmt.Errorf("cell '%v' is %v, which is not a pointer cell", cellKey, cell.Type()) + } + return cell.(page.PointerCell).Pointer, nil +} diff --git a/internal/engine/storage/db_test.go b/internal/engine/storage/db_test.go index cdbb0071..398e8040 100644 --- a/internal/engine/storage/db_test.go +++ b/internal/engine/storage/db_test.go @@ -16,27 +16,38 @@ func TestCreate(t *testing.T) { f, err := fs.Create("mydbfile") assert.NoError(err) + // actual tests + + // create the database file contents in file f db, err := Create(f) assert.NoError(err) - mustHaveSize(assert, f, 2*page.Size) + // f must have the size of 3 pages, header, tables and config page + mustHaveSize(assert, f, 3*page.Size) + // load the header page, which is the first page (offset 0) in the file headerPage := loadPageFromOffset(assert, f, 0) - assert.EqualValues(2, headerPage.CellCount()) + assert.EqualValues(3, headerPage.CellCount()) assert.Equal( - encodeUint64(2), + encodeUint64(3), mustCell(assert, headerPage, HeaderPageCount).(page.RecordCell).Record, ) + // Allocating a new page must persist it in the created database file. This + // check ensures, that the file is writable. _, err = db.AllocateNewPage() assert.NoError(err) - mustHaveSize(assert, f, 3*page.Size) + // after allocating a new page, the file must have grown to 4 times the size + // of a single page + mustHaveSize(assert, f, 4*page.Size) + // check the header page again, which must have the same amount of cells, + // but the page count cell value must have been incremented by 1 headerPage = loadPageFromOffset(assert, f, 0) - assert.EqualValues(2, headerPage.CellCount()) + assert.EqualValues(3, headerPage.CellCount()) assert.Equal( - encodeUint64(3), + encodeUint64(4), mustCell(assert, headerPage, HeaderPageCount).(page.RecordCell).Record, ) @@ -49,7 +60,7 @@ func mustHaveSize(assert *assert.Assertions, file afero.File, expectedSize int64 assert.EqualValues(expectedSize, stat.Size()) } -func mustCell(assert *assert.Assertions, p *page.Page, key string) interface{} { +func mustCell(assert *assert.Assertions, p *page.Page, key string) page.CellTyper { val, ok := p.Cell([]byte(key)) assert.Truef(ok, "page must have cell with key %v", key) return val @@ -57,7 +68,8 @@ func mustCell(assert *assert.Assertions, p *page.Page, key string) interface{} { func loadPageFromOffset(assert *assert.Assertions, rd io.ReaderAt, off int64) *page.Page { buf := make([]byte, page.Size) - _, err := rd.ReadAt(buf, off) + n, err := rd.ReadAt(buf, off) + assert.Equal(len(buf), n) assert.NoError(err) p, err := page.Load(buf) assert.NoError(err) diff --git a/internal/engine/storage/error.go b/internal/engine/storage/error.go index 70f69e8b..5419fb94 100644 --- a/internal/engine/storage/error.go +++ b/internal/engine/storage/error.go @@ -1,5 +1,7 @@ package storage +import "fmt" + // Error is a sentinel error. type Error string @@ -7,5 +9,12 @@ func (e Error) Error() string { return string(e) } // Sentinel errors. const ( - ErrClosed Error = "already closed" + ErrClosed Error = "already closed" + ErrNoSuchConfigKey Error = "no such configuration key" ) + +// ErrNoSuchCell returns an error that indicates, that a cell with the given +// name could not be found. +func ErrNoSuchCell(cellKey string) Error { + return Error(fmt.Sprintf("no such cell '%v'", cellKey)) +} diff --git a/internal/engine/storage/page/page_test.go b/internal/engine/storage/page/page_test.go index 6bcc933c..8674f974 100644 --- a/internal/engine/storage/page/page_test.go +++ b/internal/engine/storage/page/page_test.go @@ -475,3 +475,108 @@ func TestPage_DeleteCell(t *testing.T) { }) } } + +func TestPage_findCell(t *testing.T) { + pageID := ID(0) + p := New(pageID) + cells := []CellTyper{ + // these cells should remain sorted, as sorted insertion is tested + // somewhere else, and by being sorted, the tests are more readable + // regarding the offset indexes + PointerCell{ + Key: []byte("001 first"), + Pointer: ID(1), + }, + PointerCell{ + Key: []byte("002 second"), + Pointer: ID(2), + }, + PointerCell{ + Key: []byte("003 third"), + Pointer: ID(3), + }, + PointerCell{ + Key: []byte("004 fourth"), + Pointer: ID(4), + }, + } + for _, cell := range cells { + switch c := cell.(type) { + case RecordCell: + assert.NoError(t, p.StoreRecordCell(c)) + case PointerCell: + assert.NoError(t, p.StorePointerCell(c)) + default: + assert.FailNow(t, "unknown cell type") + } + } + + // actual tests + + tests := []struct { + name string + p *Page + key string + wantOffsetIndex uint16 + wantCellSlot Slot + wantCell CellTyper + wantFound bool + }{ + { + name: "first", + p: p, + key: "001 first", + wantOffsetIndex: 6, + wantCellSlot: Slot{Offset: 16366, Size: 18}, + wantCell: cells[0], + wantFound: true, + }, + { + name: "second", + p: p, + key: "002 second", + wantOffsetIndex: 10, + wantCellSlot: Slot{Offset: 16347, Size: 19}, + wantCell: cells[1], + wantFound: true, + }, + { + name: "third", + p: p, + key: "003 third", + wantOffsetIndex: 14, + wantCellSlot: Slot{Offset: 16329, Size: 18}, + wantCell: cells[2], + wantFound: true, + }, + { + name: "fourth", + p: p, + key: "004 fourth", + wantOffsetIndex: 18, + wantCellSlot: Slot{Offset: 16310, Size: 19}, + wantCell: cells[3], + wantFound: true, + }, + { + name: "missing cell", + p: p, + key: "some key that doesn't exist", + wantOffsetIndex: 0, + wantCellSlot: Slot{Offset: 0, Size: 0}, + wantCell: nil, + wantFound: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + + offsetIndex, cellSlot, cell, found := p.findCell([]byte(tt.key)) + assert.Equal(tt.wantOffsetIndex, offsetIndex, "offset indexes don't match") + assert.Equal(tt.wantCellSlot, cellSlot, "cell slot don't match") + assert.Equal(tt.wantCell, cell, "cell don't match") + assert.Equal(tt.wantFound, found, "found don't match") + }) + } +} diff --git a/internal/test/base_test.go b/internal/test/base_test.go index 7278d2cd..32852d25 100644 --- a/internal/test/base_test.go +++ b/internal/test/base_test.go @@ -38,7 +38,9 @@ func TestMain(m *testing.M) { } func RunAndCompare(t *testing.T, tt Test) { + t.Helper() t.Run(tt.Name, func(t *testing.T) { + t.Helper() runAndCompare(t, tt) }) } From 86d68b5adcd605facfbe1cb7413816a77283f7ca Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Wed, 8 Jul 2020 22:35:23 +0200 Subject: [PATCH 62/77] Introduce NULLs into Comparator implementations --- internal/engine/types/bool_type.go | 6 ++++++ internal/engine/types/bool_type_test.go | 24 ++++++++++++++++++++++++ internal/engine/types/comparator.go | 14 +++++++++----- internal/engine/types/date_type.go | 6 ++++++ internal/engine/types/integer_type.go | 6 ++++++ internal/engine/types/null_value.go | 3 ++- internal/engine/types/real_type.go | 6 ++++++ internal/engine/types/string_type.go | 6 ++++++ internal/engine/types/value.go | 14 +++++++++++++- 9 files changed, 78 insertions(+), 7 deletions(-) diff --git a/internal/engine/types/bool_type.go b/internal/engine/types/bool_type.go index e7073eab..488d8d57 100644 --- a/internal/engine/types/bool_type.go +++ b/internal/engine/types/bool_type.go @@ -29,6 +29,12 @@ func (t BoolType) Compare(left, right Value) (int, error) { return 0, err } + if left.IsNull() { + return -1, nil + } else if right.IsNull() { + return 1, nil + } + leftBool := left.(BoolValue).Value rightBool := right.(BoolValue).Value diff --git a/internal/engine/types/bool_type_test.go b/internal/engine/types/bool_type_test.go index c22fadec..5dd549d8 100644 --- a/internal/engine/types/bool_type_test.go +++ b/internal/engine/types/bool_type_test.go @@ -23,6 +23,30 @@ func TestBoolType_Compare(t *testing.T) { 0, "type mismatch: want Bool, got ", }, + { + "null <-> nil", + args{NewNull(Bool), nil}, + 0, + "type mismatch: want Bool, got ", + }, + { + "null <-> null", + args{NewNull(Bool), NewNull(Bool)}, + -1, + "", + }, + { + "null <-> true", + args{NewNull(Bool), NewBool(true)}, + -1, + "", + }, + { + "null <-> false", + args{NewNull(Bool), NewBool(false)}, + -1, + "", + }, { "left nil", args{nil, NewBool(false)}, diff --git a/internal/engine/types/comparator.go b/internal/engine/types/comparator.go index b8b81b8a..4f011463 100644 --- a/internal/engine/types/comparator.go +++ b/internal/engine/types/comparator.go @@ -1,11 +1,15 @@ package types -// Comparator is the interface that wraps the basic compare method. The -// compare method compares the left and right value as follows. -1 if -// leftright. What exectly is considered -// to be <, ==, > is up to the implementation. +// Comparator is the interface that wraps the basic compare method. The compare +// method compares the left and right value as follows. -1 if leftright. What exectly is considered to be <, ==, > is up +// to the implementation. By definition, the NULL value is smaller than any +// other value. When comparing NULL to another NULL value, and both NULLs have +// the same type, the result is undefined, however, no error must be returned. type Comparator interface { // Compare compares the given to values left and right as follows. -1 if - // leftright. + // leftright. However, NULL Date: Thu, 9 Jul 2020 15:53:54 +0200 Subject: [PATCH 63/77] Make tests also compare a Go string Tests now not only check the string representation of a command, but also a Go string representation. This is, to enable developers, to easily check whether the output is actually correct when recording new tests. --- internal/compiler/command/command.go | 11 +++++++++++ internal/compiler/golden_test.go | 8 ++++++-- internal/compiler/simple_compiler_fixture_test.go | 11 +++++++++++ .../testdata/TestCompileGolden/delete/#00.golden | 3 +++ .../testdata/TestCompileGolden/delete/#01.golden | 3 +++ .../testdata/TestCompileGolden/delete/#02.golden | 3 +++ .../testdata/TestCompileGolden/drop/#00.golden | 3 +++ .../testdata/TestCompileGolden/drop/#01.golden | 3 +++ .../testdata/TestCompileGolden/drop/#02.golden | 3 +++ .../testdata/TestCompileGolden/drop/#03.golden | 3 +++ .../testdata/TestCompileGolden/drop/#04.golden | 3 +++ .../testdata/TestCompileGolden/drop/#05.golden | 3 +++ .../testdata/TestCompileGolden/drop/#06.golden | 3 +++ .../testdata/TestCompileGolden/drop/#07.golden | 3 +++ .../testdata/TestCompileGolden/drop/#08.golden | 3 +++ .../testdata/TestCompileGolden/drop/#09.golden | 3 +++ .../testdata/TestCompileGolden/drop/#10.golden | 3 +++ .../testdata/TestCompileGolden/drop/#11.golden | 3 +++ .../testdata/TestCompileGolden/drop/#12.golden | 3 +++ .../testdata/TestCompileGolden/drop/#13.golden | 3 +++ .../testdata/TestCompileGolden/drop/#14.golden | 3 +++ .../testdata/TestCompileGolden/drop/#15.golden | 3 +++ .../testdata/TestCompileGolden/expressions/#00.golden | 4 ++++ .../testdata/TestCompileGolden/expressions/#01.golden | 4 ++++ .../testdata/TestCompileGolden/select/#00.golden | 3 +++ .../testdata/TestCompileGolden/select/#01.golden | 3 +++ .../testdata/TestCompileGolden/select/#02.golden | 3 +++ .../testdata/TestCompileGolden/select/#03.golden | 3 +++ .../testdata/TestCompileGolden/select/#04.golden | 3 +++ .../testdata/TestCompileGolden/select/#05.golden | 3 +++ .../testdata/TestCompileGolden/select/#06.golden | 3 +++ .../testdata/TestCompileGolden/select/#07.golden | 3 +++ .../testdata/TestCompileGolden/select/#08.golden | 3 +++ .../testdata/TestCompileGolden/select/#09.golden | 3 +++ .../testdata/TestCompileGolden/select/#10.golden | 3 +++ .../testdata/TestCompileGolden/select/#11.golden | 3 +++ .../testdata/TestCompileGolden/select/#12.golden | 3 +++ .../testdata/TestCompileGolden/update/#00.golden | 3 +++ .../testdata/TestCompileGolden/update/#01.golden | 3 +++ .../testdata/TestCompileGolden/update/#02.golden | 3 +++ .../testdata/TestCompileGolden/update/#03.golden | 3 +++ 41 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 internal/compiler/testdata/TestCompileGolden/expressions/#00.golden create mode 100644 internal/compiler/testdata/TestCompileGolden/expressions/#01.golden diff --git a/internal/compiler/command/command.go b/internal/compiler/command/command.go index 5b2eae22..9787e089 100644 --- a/internal/compiler/command/command.go +++ b/internal/compiler/command/command.go @@ -104,6 +104,7 @@ type ( // select statements. Table interface { _table() + QualifiedName() string } // SimpleTable is a table that is only specified by schema and table name, @@ -313,6 +314,16 @@ func (Values) _list() {} func (SimpleTable) _table() {} +// QualifiedName returns '.', or only '' if no +// schema is specified. +func (t SimpleTable) QualifiedName() string { + qualifiedName := t.Table + if t.Schema != "" { + qualifiedName = t.Schema + "." + qualifiedName + } + return qualifiedName +} + func (e Explain) String() string { return fmt.Sprintf("explanation: %v", e.Command) } diff --git a/internal/compiler/golden_test.go b/internal/compiler/golden_test.go index 22e19447..51eb2726 100644 --- a/internal/compiler/golden_test.go +++ b/internal/compiler/golden_test.go @@ -2,6 +2,7 @@ package compiler import ( "flag" + "fmt" "io/ioutil" "os" "path/filepath" @@ -43,19 +44,22 @@ func runGolden(t *testing.T, input string) { got, err := c.Compile(stmt) require.NoError(err) + gotGoString := fmt.Sprintf("%#v", got) gotString := got.String() + gotFull := gotGoString + "\n\nString:\n" + gotString + testFilePath := "testdata/" + t.Name() + ".golden" if *record { t.Logf("overwriting golden file %v", testFilePath) err := os.MkdirAll(filepath.Dir(testFilePath), 0777) require.NoError(err) - err = ioutil.WriteFile(testFilePath, []byte(gotString), 0666) + err = ioutil.WriteFile(testFilePath, []byte(gotFull), 0666) require.NoError(err) t.Fail() } else { data, err := ioutil.ReadFile(testFilePath) require.NoError(err) - require.Equal(string(data), gotString) + require.Equal(string(data), gotFull) } } diff --git a/internal/compiler/simple_compiler_fixture_test.go b/internal/compiler/simple_compiler_fixture_test.go index 80fc07dc..6e64fcbd 100644 --- a/internal/compiler/simple_compiler_fixture_test.go +++ b/internal/compiler/simple_compiler_fixture_test.go @@ -7,6 +7,17 @@ func TestCompileGolden(t *testing.T) { t.Run("delete", _TestCompileDelete) t.Run("drop", _TestCompileDrop) t.Run("update", _TestCompileUpdate) + t.Run("expressions", _TestCompileExpressions) +} + +func _TestCompileExpressions(t *testing.T) { + tests := []string{ + "VALUES (7)", + "VALUES (-7)", + } + for _, test := range tests { + RunGolden(t, test) + } } func _TestCompileUpdate(t *testing.T) { diff --git a/internal/compiler/testdata/TestCompileGolden/delete/#00.golden b/internal/compiler/testdata/TestCompileGolden/delete/#00.golden index 262579a9..230a5bce 100644 --- a/internal/compiler/testdata/TestCompileGolden/delete/#00.golden +++ b/internal/compiler/testdata/TestCompileGolden/delete/#00.golden @@ -1 +1,4 @@ +command.Delete{Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}, Filter:command.ConstantBooleanExpr{Value:true}} + +String: Delete[filter=true](myTable) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/delete/#01.golden b/internal/compiler/testdata/TestCompileGolden/delete/#01.golden index 40def1e3..5cf3a28e 100644 --- a/internal/compiler/testdata/TestCompileGolden/delete/#01.golden +++ b/internal/compiler/testdata/TestCompileGolden/delete/#01.golden @@ -1 +1,4 @@ +command.Delete{Table:command.SimpleTable{Schema:"mySchema", Table:"myTable", Alias:"", Indexed:false, Index:""}, Filter:command.ConstantBooleanExpr{Value:true}} + +String: Delete[filter=true](mySchema.myTable) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/delete/#02.golden b/internal/compiler/testdata/TestCompileGolden/delete/#02.golden index 067c2191..08c35459 100644 --- a/internal/compiler/testdata/TestCompileGolden/delete/#02.golden +++ b/internal/compiler/testdata/TestCompileGolden/delete/#02.golden @@ -1 +1,4 @@ +command.Delete{Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}, Filter:command.BinaryExpr{Operator:"==", Left:command.LiteralExpr{Value:"col1"}, Right:command.LiteralExpr{Value:"col2"}}} + +String: Delete[filter=col1 == col2](myTable) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#00.golden b/internal/compiler/testdata/TestCompileGolden/drop/#00.golden index e13ff203..a50b4346 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#00.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#00.golden @@ -1 +1,4 @@ +command.DropTable{IfExists:false, Schema:"", Name:"myTable"} + +String: DropTable[table=myTable,ifexists=false]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#01.golden b/internal/compiler/testdata/TestCompileGolden/drop/#01.golden index aba0c190..9445ff27 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#01.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#01.golden @@ -1 +1,4 @@ +command.DropTable{IfExists:true, Schema:"", Name:"myTable"} + +String: DropTable[table=myTable,ifexists=true]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#02.golden b/internal/compiler/testdata/TestCompileGolden/drop/#02.golden index e642a229..89df6920 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#02.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#02.golden @@ -1 +1,4 @@ +command.DropTable{IfExists:false, Schema:"mySchema", Name:"myTable"} + +String: DropTable[table=mySchema.myTable,ifexists=false]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#03.golden b/internal/compiler/testdata/TestCompileGolden/drop/#03.golden index 53ee2a06..a7ccb29d 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#03.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#03.golden @@ -1 +1,4 @@ +command.DropTable{IfExists:true, Schema:"mySchema", Name:"myTable"} + +String: DropTable[table=mySchema.myTable,ifexists=true]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#04.golden b/internal/compiler/testdata/TestCompileGolden/drop/#04.golden index 1cfee780..6d4e6bf4 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#04.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#04.golden @@ -1 +1,4 @@ +command.DropView{IfExists:false, Schema:"", Name:"myView"} + +String: DropView[view=myView,ifexists=false]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#05.golden b/internal/compiler/testdata/TestCompileGolden/drop/#05.golden index bd885ecf..5b8d8e5d 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#05.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#05.golden @@ -1 +1,4 @@ +command.DropView{IfExists:true, Schema:"", Name:"myView"} + +String: DropView[view=myView,ifexists=true]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#06.golden b/internal/compiler/testdata/TestCompileGolden/drop/#06.golden index 39317bbe..04ea4c11 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#06.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#06.golden @@ -1 +1,4 @@ +command.DropView{IfExists:false, Schema:"mySchema", Name:"myView"} + +String: DropView[view=mySchema.myView,ifexists=false]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#07.golden b/internal/compiler/testdata/TestCompileGolden/drop/#07.golden index 93ff65dc..8cdc6904 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#07.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#07.golden @@ -1 +1,4 @@ +command.DropView{IfExists:true, Schema:"mySchema", Name:"myView"} + +String: DropView[view=mySchema.myView,ifexists=true]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#08.golden b/internal/compiler/testdata/TestCompileGolden/drop/#08.golden index 408abfdf..1afd82b1 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#08.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#08.golden @@ -1 +1,4 @@ +command.DropIndex{IfExists:false, Schema:"", Name:"myIndex"} + +String: DropIndex[index=myIndex,ifexists=false]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#09.golden b/internal/compiler/testdata/TestCompileGolden/drop/#09.golden index ec6eaefa..e3504d80 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#09.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#09.golden @@ -1 +1,4 @@ +command.DropIndex{IfExists:true, Schema:"", Name:"myIndex"} + +String: DropIndex[index=myIndex,ifexists=true]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#10.golden b/internal/compiler/testdata/TestCompileGolden/drop/#10.golden index ea55a689..d1eff223 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#10.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#10.golden @@ -1 +1,4 @@ +command.DropIndex{IfExists:false, Schema:"mySchema", Name:"myIndex"} + +String: DropIndex[index=mySchema.myIndex,ifexists=false]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#11.golden b/internal/compiler/testdata/TestCompileGolden/drop/#11.golden index ccbc9394..096d9cc5 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#11.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#11.golden @@ -1 +1,4 @@ +command.DropIndex{IfExists:true, Schema:"mySchema", Name:"myIndex"} + +String: DropIndex[index=mySchema.myIndex,ifexists=true]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#12.golden b/internal/compiler/testdata/TestCompileGolden/drop/#12.golden index b9a97746..acacd916 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#12.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#12.golden @@ -1 +1,4 @@ +command.DropTrigger{IfExists:false, Schema:"", Name:"myTrigger"} + +String: DropTrigger[trigger=myTrigger,ifexists=false]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#13.golden b/internal/compiler/testdata/TestCompileGolden/drop/#13.golden index ec80c2d6..03140074 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#13.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#13.golden @@ -1 +1,4 @@ +command.DropTrigger{IfExists:true, Schema:"", Name:"myTrigger"} + +String: DropTrigger[trigger=myTrigger,ifexists=true]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#14.golden b/internal/compiler/testdata/TestCompileGolden/drop/#14.golden index e42e58d5..8cb9639f 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#14.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#14.golden @@ -1 +1,4 @@ +command.DropTrigger{IfExists:false, Schema:"mySchema", Name:"myTrigger"} + +String: DropTrigger[trigger=mySchema.myTrigger,ifexists=false]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/drop/#15.golden b/internal/compiler/testdata/TestCompileGolden/drop/#15.golden index bea9b287..d1d9a6ad 100644 --- a/internal/compiler/testdata/TestCompileGolden/drop/#15.golden +++ b/internal/compiler/testdata/TestCompileGolden/drop/#15.golden @@ -1 +1,4 @@ +command.DropTrigger{IfExists:true, Schema:"mySchema", Name:"myTrigger"} + +String: DropTrigger[trigger=mySchema.myTrigger,ifexists=true]() \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/expressions/#00.golden b/internal/compiler/testdata/TestCompileGolden/expressions/#00.golden new file mode 100644 index 00000000..65e5bd02 --- /dev/null +++ b/internal/compiler/testdata/TestCompileGolden/expressions/#00.golden @@ -0,0 +1,4 @@ +command.Values{Values:[][]command.Expr{[]command.Expr{command.LiteralExpr{Value:"7"}}}} + +String: +Values[]((7)) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/expressions/#01.golden b/internal/compiler/testdata/TestCompileGolden/expressions/#01.golden new file mode 100644 index 00000000..702c9ddb --- /dev/null +++ b/internal/compiler/testdata/TestCompileGolden/expressions/#01.golden @@ -0,0 +1,4 @@ +command.Values{Values:[][]command.Expr{[]command.Expr{command.UnaryExpr{Operator:"-", Value:command.LiteralExpr{Value:"7"}}}}} + +String: +Values[]((- 7)) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#00.golden b/internal/compiler/testdata/TestCompileGolden/select/#00.golden index 1471d004..a5650b1b 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#00.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#00.golden @@ -1 +1,4 @@ +command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"*"}, Alias:""}}, Input:command.Scan{Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}}} + +String: Project[cols=*](Scan[table=myTable]()) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#01.golden b/internal/compiler/testdata/TestCompileGolden/select/#01.golden index c9103c39..fab9b558 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#01.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#01.golden @@ -1 +1,4 @@ +command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"*"}, Alias:""}}, Input:command.Select{Filter:command.ConstantBooleanExpr{Value:true}, Input:command.Scan{Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}}}} + +String: Project[cols=*](Select[filter=true](Scan[table=myTable]())) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#02.golden b/internal/compiler/testdata/TestCompileGolden/select/#02.golden index 035f3cdc..2c173dc3 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#02.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#02.golden @@ -1 +1,4 @@ +command.Limit{Limit:command.LiteralExpr{Value:"5"}, Input:command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"*"}, Alias:""}}, Input:command.Scan{Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}}}} + +String: Limit[limit=5](Project[cols=*](Scan[table=myTable]())) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#03.golden b/internal/compiler/testdata/TestCompileGolden/select/#03.golden index d0fd93c8..d4757eb2 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#03.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#03.golden @@ -1 +1,4 @@ +command.Limit{Limit:command.LiteralExpr{Value:"5"}, Input:command.Offset{Offset:command.LiteralExpr{Value:"10"}, Input:command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"*"}, Alias:""}}, Input:command.Scan{Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}}}}} + +String: Limit[limit=5](Offset[offset=10](Project[cols=*](Scan[table=myTable]()))) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#04.golden b/internal/compiler/testdata/TestCompileGolden/select/#04.golden index d0fd93c8..d4757eb2 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#04.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#04.golden @@ -1 +1,4 @@ +command.Limit{Limit:command.LiteralExpr{Value:"5"}, Input:command.Offset{Offset:command.LiteralExpr{Value:"10"}, Input:command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"*"}, Alias:""}}, Input:command.Scan{Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}}}}} + +String: Limit[limit=5](Offset[offset=10](Project[cols=*](Scan[table=myTable]()))) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#05.golden b/internal/compiler/testdata/TestCompileGolden/select/#05.golden index 14df34c1..69244a73 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#05.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#05.golden @@ -1 +1,4 @@ +command.Distinct{Input:command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"*"}, Alias:""}}, Input:command.Select{Filter:command.ConstantBooleanExpr{Value:true}, Input:command.Scan{Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}}}}} + +String: Distinct[](Project[cols=*](Select[filter=true](Scan[table=myTable]()))) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#06.golden b/internal/compiler/testdata/TestCompileGolden/select/#06.golden index 32d0c1c4..1c56843d 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#06.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#06.golden @@ -1 +1,4 @@ +command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"*"}, Alias:""}}, Input:command.Select{Filter:command.ConstantBooleanExpr{Value:true}, Input:command.Join{Natural:false, Type:0x0, Filter:command.Expr(nil), Left:command.Scan{Table:command.SimpleTable{Schema:"", Table:"a", Alias:"", Indexed:false, Index:""}}, Right:command.Scan{Table:command.SimpleTable{Schema:"", Table:"b", Alias:"", Indexed:false, Index:""}}}}} + +String: Project[cols=*](Select[filter=true](Join[](Scan[table=a](),Scan[table=b]()))) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#07.golden b/internal/compiler/testdata/TestCompileGolden/select/#07.golden index 32d0c1c4..1c56843d 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#07.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#07.golden @@ -1 +1,4 @@ +command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"*"}, Alias:""}}, Input:command.Select{Filter:command.ConstantBooleanExpr{Value:true}, Input:command.Join{Natural:false, Type:0x0, Filter:command.Expr(nil), Left:command.Scan{Table:command.SimpleTable{Schema:"", Table:"a", Alias:"", Indexed:false, Index:""}}, Right:command.Scan{Table:command.SimpleTable{Schema:"", Table:"b", Alias:"", Indexed:false, Index:""}}}}} + +String: Project[cols=*](Select[filter=true](Join[](Scan[table=a](),Scan[table=b]()))) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#08.golden b/internal/compiler/testdata/TestCompileGolden/select/#08.golden index 9a1188f8..6f66baf9 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#08.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#08.golden @@ -1 +1,4 @@ +command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"*"}, Alias:""}}, Input:command.Select{Filter:command.ConstantBooleanExpr{Value:true}, Input:command.Join{Natural:false, Type:0x0, Filter:command.Expr(nil), Left:command.Join{Natural:false, Type:0x0, Filter:command.Expr(nil), Left:command.Scan{Table:command.SimpleTable{Schema:"", Table:"a", Alias:"", Indexed:false, Index:""}}, Right:command.Scan{Table:command.SimpleTable{Schema:"", Table:"b", Alias:"", Indexed:false, Index:""}}}, Right:command.Scan{Table:command.SimpleTable{Schema:"", Table:"c", Alias:"", Indexed:false, Index:""}}}}} + +String: Project[cols=*](Select[filter=true](Join[](Join[](Scan[table=a](),Scan[table=b]()),Scan[table=c]()))) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#09.golden b/internal/compiler/testdata/TestCompileGolden/select/#09.golden index 9133264f..eb08cffe 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#09.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#09.golden @@ -1 +1,4 @@ +command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.LiteralExpr{Value:"name"}, Alias:""}, command.Column{Table:"", Column:command.BinaryExpr{Operator:"*", Left:command.LiteralExpr{Value:"amount"}, Right:command.LiteralExpr{Value:"price"}}, Alias:"total_price"}}, Input:command.Join{Natural:false, Type:0x0, Filter:command.Expr(nil), Left:command.Scan{Table:command.SimpleTable{Schema:"", Table:"items", Alias:"", Indexed:false, Index:""}}, Right:command.Scan{Table:command.SimpleTable{Schema:"", Table:"prices", Alias:"", Indexed:false, Index:""}}}} + +String: Project[cols=name,amount * price AS total_price](Join[](Scan[table=items](),Scan[table=prices]())) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#10.golden b/internal/compiler/testdata/TestCompileGolden/select/#10.golden index 53319ca2..7e3d3245 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#10.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#10.golden @@ -1 +1,4 @@ +command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.FunctionExpr{Name:"AVG", Distinct:false, Args:[]command.Expr{command.LiteralExpr{Value:"price"}}}, Alias:"avg_price"}}, Input:command.Join{Natural:false, Type:0x1, Filter:command.Expr(nil), Left:command.Scan{Table:command.SimpleTable{Schema:"", Table:"items", Alias:"", Indexed:false, Index:""}}, Right:command.Scan{Table:command.SimpleTable{Schema:"", Table:"prices", Alias:"", Indexed:false, Index:""}}}} + +String: Project[cols=AVG(price) AS avg_price](Join[type=JoinLeft](Scan[table=items](),Scan[table=prices]())) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#11.golden b/internal/compiler/testdata/TestCompileGolden/select/#11.golden index 79f349bb..88291aa2 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#11.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#11.golden @@ -1 +1,4 @@ +command.Project{Cols:[]command.Column{command.Column{Table:"", Column:command.FunctionExpr{Name:"AVG", Distinct:true, Args:[]command.Expr{command.LiteralExpr{Value:"price"}}}, Alias:"avg_price"}}, Input:command.Join{Natural:false, Type:0x1, Filter:command.Expr(nil), Left:command.Scan{Table:command.SimpleTable{Schema:"", Table:"items", Alias:"", Indexed:false, Index:""}}, Right:command.Scan{Table:command.SimpleTable{Schema:"", Table:"prices", Alias:"", Indexed:false, Index:""}}}} + +String: Project[cols=AVG(DISTINCT price) AS avg_price](Join[type=JoinLeft](Scan[table=items](),Scan[table=prices]())) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/select/#12.golden b/internal/compiler/testdata/TestCompileGolden/select/#12.golden index facaa6dc..c869a32e 100644 --- a/internal/compiler/testdata/TestCompileGolden/select/#12.golden +++ b/internal/compiler/testdata/TestCompileGolden/select/#12.golden @@ -1 +1,4 @@ +command.Values{Values:[][]command.Expr{[]command.Expr{command.LiteralExpr{Value:"1"}, command.LiteralExpr{Value:"2"}, command.LiteralExpr{Value:"3"}}, []command.Expr{command.LiteralExpr{Value:"4"}, command.LiteralExpr{Value:"5"}, command.LiteralExpr{Value:"6"}}, []command.Expr{command.LiteralExpr{Value:"7"}, command.LiteralExpr{Value:"8"}, command.LiteralExpr{Value:"9"}}}} + +String: Values[]((1,2,3),(4,5,6),(7,8,9)) \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/update/#00.golden b/internal/compiler/testdata/TestCompileGolden/update/#00.golden index fdbb2c6c..334d5c13 100644 --- a/internal/compiler/testdata/TestCompileGolden/update/#00.golden +++ b/internal/compiler/testdata/TestCompileGolden/update/#00.golden @@ -1 +1,4 @@ +command.Update{UpdateOr:0x5, Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}, Updates:[]command.UpdateSetter{command.UpdateSetter{Cols:[]string{"myCol"}, Value:command.LiteralExpr{Value:"7"}}}, Filter:command.ConstantBooleanExpr{Value:true}} + +String: Update[or=UpdateOrIgnore,table=myTable,sets=((myCol)=7),filter=true] \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/update/#01.golden b/internal/compiler/testdata/TestCompileGolden/update/#01.golden index 3ee04b65..cdf59922 100644 --- a/internal/compiler/testdata/TestCompileGolden/update/#01.golden +++ b/internal/compiler/testdata/TestCompileGolden/update/#01.golden @@ -1 +1,4 @@ +command.Update{UpdateOr:0x5, Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}, Updates:[]command.UpdateSetter{command.UpdateSetter{Cols:[]string{"myCol"}, Value:command.LiteralExpr{Value:"7"}}}, Filter:command.BinaryExpr{Operator:"==", Left:command.LiteralExpr{Value:"myOtherCol"}, Right:command.LiteralExpr{Value:"9"}}} + +String: Update[or=UpdateOrIgnore,table=myTable,sets=((myCol)=7),filter=myOtherCol == 9] \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/update/#02.golden b/internal/compiler/testdata/TestCompileGolden/update/#02.golden index 5b33c733..b1b795bb 100644 --- a/internal/compiler/testdata/TestCompileGolden/update/#02.golden +++ b/internal/compiler/testdata/TestCompileGolden/update/#02.golden @@ -1 +1,4 @@ +command.Update{UpdateOr:0x4, Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}, Updates:[]command.UpdateSetter{command.UpdateSetter{Cols:[]string{"myCol"}, Value:command.LiteralExpr{Value:"7"}}}, Filter:command.BinaryExpr{Operator:"==", Left:command.LiteralExpr{Value:"myOtherCol"}, Right:command.LiteralExpr{Value:"9"}}} + +String: Update[or=UpdateOrFail,table=myTable,sets=((myCol)=7),filter=myOtherCol == 9] \ No newline at end of file diff --git a/internal/compiler/testdata/TestCompileGolden/update/#03.golden b/internal/compiler/testdata/TestCompileGolden/update/#03.golden index de51b43b..a2c7d9ce 100644 --- a/internal/compiler/testdata/TestCompileGolden/update/#03.golden +++ b/internal/compiler/testdata/TestCompileGolden/update/#03.golden @@ -1 +1,4 @@ +command.Update{UpdateOr:0x5, Table:command.SimpleTable{Schema:"", Table:"myTable", Alias:"", Indexed:false, Index:""}, Updates:[]command.UpdateSetter{command.UpdateSetter{Cols:[]string{"myCol1", "myCol2"}, Value:command.LiteralExpr{Value:"7"}}, command.UpdateSetter{Cols:[]string{"myOtherCol1", "myOtherCol2"}, Value:command.LiteralExpr{Value:"8"}}}, Filter:command.BinaryExpr{Operator:"==", Left:command.LiteralExpr{Value:"myOtherCol"}, Right:command.LiteralExpr{Value:"9"}}} + +String: Update[or=UpdateOrIgnore,table=myTable,sets=((myCol1,myCol2)=7,(myOtherCol1,myOtherCol2)=8),filter=myOtherCol == 9] \ No newline at end of file From 0e9a36efe1847bdbc23b53219eea6811437d8105 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Thu, 9 Jul 2020 16:26:42 +0200 Subject: [PATCH 64/77] Remove obsolete length check of parser errors --- internal/test/base_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/test/base_test.go b/internal/test/base_test.go index 32852d25..090ebef0 100644 --- a/internal/test/base_test.go +++ b/internal/test/base_test.go @@ -74,7 +74,6 @@ func runAndCompare(t *testing.T, tt Test) { p := parser.New(tt.Statement) stmt, errs, ok := p.Next() assert.True(ok) - assert.Len(errs, 0) for _, err := range errs { assert.NoError(err) } From c7a01700b5eef069da2596296f2c1ebe9add23fb Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 10 Jul 2020 09:19:02 +0200 Subject: [PATCH 65/77] Merge package ID from branch raft --- internal/id/doc.go | 3 ++ internal/id/id.go | 64 ++++++++++++++++++++++++++++++++++++++++++ internal/id/id_test.go | 29 +++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 internal/id/doc.go create mode 100644 internal/id/id.go create mode 100644 internal/id/id_test.go diff --git a/internal/id/doc.go b/internal/id/doc.go new file mode 100644 index 00000000..d18da919 --- /dev/null +++ b/internal/id/doc.go @@ -0,0 +1,3 @@ +// Package id provides functions for creating globally unique IDs that can be +// used by the application. +package id diff --git a/internal/id/id.go b/internal/id/id.go new file mode 100644 index 00000000..5f2e9521 --- /dev/null +++ b/internal/id/id.go @@ -0,0 +1,64 @@ +package id + +import ( + "fmt" + "log" + "math/rand" + "sync" + "time" + + "github.com/oklog/ulid" +) + +// ID describes a general identifier. An ID has to be unique application-wide. +// IDs must not be re-used. +type ID interface { + fmt.Stringer + Bytes() []byte +} + +var _ ID = (*id)(nil) + +type id ulid.ULID + +var ( + lock sync.Mutex + randSource = rand.New(rand.NewSource(time.Now().UnixNano())) + entropy = ulid.Monotonic(randSource, 0) +) + +// Create creates a globally unique ID. This function is safe for concurrent +// use. +func Create() ID { + lock.Lock() + defer lock.Unlock() + + genID, err := ulid.New(ulid.Now(), entropy) + if err != nil { + // For this to happen, the random module would have to fail. Since we + // use Go's pseudo RNG, which just jumps around a few numbers, instead + // of using crypto/rand, and we also made this function safe for + // concurrent use, this is nearly impossible to happen. However, with + // the current version of oklog/ulid v1.3.1, this will also break after + // 2121-04-11 11:53:25.01172576 UTC. + log.Fatal(fmt.Errorf("new ulid: %w", err)) + } + return id(genID) +} + +// Parse parses an ID from a byte slice. +func Parse(idBytes []byte) (ID, error) { + parsed, err := ulid.Parse(string(idBytes)) + if err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + return id(parsed), nil +} + +func (id id) String() string { + return ulid.ULID(id).String() +} + +func (id id) Bytes() []byte { + return []byte(id.String()) +} diff --git a/internal/id/id_test.go b/internal/id/id_test.go new file mode 100644 index 00000000..d6ba1fe3 --- /dev/null +++ b/internal/id/id_test.go @@ -0,0 +1,29 @@ +package id_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tomarrell/lbadd/internal/id" +) + +func TestIDThreadSafe(t *testing.T) { + // This is sufficient for the race detector to detect a race if Create() is + // not safe for concurrent use. + for i := 0; i < 5; i++ { + go func() { + _ = id.Create() + }() + } +} + +func TestIDEquality(t *testing.T) { + assert := assert.New(t) + + id1 := id.Create() + id2, err := id.Parse(id1.Bytes()) + assert.NoError(err) + + assert.Equal(id1, id2) + assert.True(id1 == id2) +} From 164f492633a66e2ae28bf840a74e3968dac1fe81 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 10 Jul 2020 11:49:06 +0200 Subject: [PATCH 66/77] Implement projection This unfortunately is kind of a god commit. * remove result package and replace with engine.Table * extend profiling and tests * add column names to table * fixture tests will write column names * implement project evaluation --- internal/engine/compare.go | 5 ++ internal/engine/context.go | 39 ++++++++- internal/engine/engine.go | 18 ++-- internal/engine/engine_test.go | 62 ++++++------- internal/engine/error.go | 17 ++-- internal/engine/evaluate.go | 112 +++++++++++++++++++++--- internal/engine/evaluate_test.go | 60 +++++++++++++ internal/engine/profiling.go | 15 ++-- internal/engine/result/result.go | 39 --------- internal/engine/result/value.go | 104 ---------------------- internal/engine/scan.go | 17 ++++ internal/engine/table.go | 85 ++++++++++++++++++ internal/test/testdata/example01/output | 2 +- internal/test/testdata/example02/output | 2 +- internal/test/testdata/issue187/output | 6 +- internal/test/testdata/issue188/output | 4 +- 16 files changed, 368 insertions(+), 219 deletions(-) create mode 100644 internal/engine/evaluate_test.go delete mode 100644 internal/engine/result/result.go delete mode 100644 internal/engine/result/value.go create mode 100644 internal/engine/scan.go create mode 100644 internal/engine/table.go diff --git a/internal/engine/compare.go b/internal/engine/compare.go index 29eb012c..9e65f402 100644 --- a/internal/engine/compare.go +++ b/internal/engine/compare.go @@ -8,8 +8,11 @@ type cmpResult uint8 const ( cmpUncomparable cmpResult = iota + // cmpEqual is returned if left==right. cmpEqual + // cmpLessThan is returned if leftright. cmpGreaterThan ) @@ -18,6 +21,8 @@ const ( // as left]'. func EvtFullTableScan(tableName string) ParameterizedEvt { return ParameterizedEvt{ diff --git a/internal/engine/result/result.go b/internal/engine/result/result.go deleted file mode 100644 index 64267f9a..00000000 --- a/internal/engine/result/result.go +++ /dev/null @@ -1,39 +0,0 @@ -package result - -import ( - "fmt" - - "github.com/tomarrell/lbadd/internal/engine/types" -) - -// Result represents an evaluation result of the engine. It is a mxn-matrix, -// where m and n is variable and depends on the passed in command. -type Result interface { - Cols() []Column - Rows() []Row - fmt.Stringer -} - -// IndexedGetter wraps a Get(index) method. -type IndexedGetter interface { - Get(int) types.Value -} - -// Sizer wraps the basic Size() method. -type Sizer interface { - Size() int -} - -// Column is an iterator over cells in a column. All of the cells must have the -// same type. -type Column interface { - Type() types.Type - IndexedGetter - Sizer -} - -// Row is an iterator over cells in a row. The cells may have different types. -type Row interface { - IndexedGetter - Sizer -} diff --git a/internal/engine/result/value.go b/internal/engine/result/value.go deleted file mode 100644 index 0cf4fb98..00000000 --- a/internal/engine/result/value.go +++ /dev/null @@ -1,104 +0,0 @@ -package result - -import ( - "bytes" - "fmt" - "strings" - "text/tabwriter" - - "github.com/tomarrell/lbadd/internal/engine/types" -) - -type valueResult struct { - vals [][]types.Value -} - -type valueColumn struct { - valueResult - colIndex int -} - -type valueRow struct { - valueResult - rowIndex int -} - -// FromValues wraps the given values in a result, performing a type check on all -// columns. If any column does not have a consistent type, an error will be -// returned, together with NO result. -func FromValues(vals [][]types.Value) (Result, error) { - for x := 0; x < len(vals[0]); x++ { // cols - t := vals[0][x].Type() - for y := 0; y < len(vals); y++ { // rows - if !vals[y][x].Is(t) { - return nil, types.ErrTypeMismatch(t, vals[y][x].Type()) - } - } - } - return valueResult{ - vals: vals, - }, nil -} - -func (r valueResult) Cols() []Column { - result := make([]Column, 0) - for i := 0; i < len(r.vals[0]); i++ { - result = append(result, valueColumn{ - valueResult: r, - colIndex: i, - }) - } - return result -} - -func (r valueResult) Rows() []Row { - result := make([]Row, 0) - for i := 0; i < len(r.vals); i++ { - result = append(result, valueRow{ - valueResult: r, - rowIndex: i, - }) - } - return result -} - -func (r valueResult) String() string { - var buf bytes.Buffer - w := tabwriter.NewWriter(&buf, 0, 1, 3, ' ', 0) - - var types []string - for _, col := range r.Cols() { - types = append(types, col.Type().Name()) - } - _, _ = fmt.Fprintln(w, strings.Join(types, "\t")) - - for _, row := range r.Rows() { - var strVals []string - for i := 0; i < row.Size(); i++ { - strVals = append(strVals, row.Get(i).String()) - } - _, _ = fmt.Fprintln(w, strings.Join(strVals, "\t")) - } - _ = w.Flush() - return buf.String() -} - -func (c valueColumn) Type() types.Type { - return c.Get(0).Type() -} - -func (c valueColumn) Get(index int) types.Value { - return c.valueResult.vals[index][c.colIndex] -} - -func (c valueColumn) Size() int { - return len(c.valueResult.vals) -} - -func (r valueRow) Get(index int) types.Value { - return r.valueResult.vals[r.rowIndex][index] -} - -func (r valueRow) Size() int { - return len(r.valueResult.vals[0]) -} diff --git a/internal/engine/scan.go b/internal/engine/scan.go new file mode 100644 index 00000000..5915ed8d --- /dev/null +++ b/internal/engine/scan.go @@ -0,0 +1,17 @@ +package engine + +import "github.com/tomarrell/lbadd/internal/compiler/command" + +func (e Engine) scanSimpleTable(ctx ExecutionContext, table command.SimpleTable) (Table, error) { + tableName := table.QualifiedName() + + // only perform scan if not already scanned + if table, alreadyScanned := ctx.getScannedTable(tableName); alreadyScanned { + return table, nil + } + + // TODO: load table from the database file + + ctx.putScannedTable(table.QualifiedName(), Table{}) + return Table{}, ErrUnimplemented("scan simple table") +} diff --git a/internal/engine/table.go b/internal/engine/table.go new file mode 100644 index 00000000..07bfcd63 --- /dev/null +++ b/internal/engine/table.go @@ -0,0 +1,85 @@ +package engine + +import ( + "bytes" + "fmt" + "strings" + "text/tabwriter" + + "github.com/tomarrell/lbadd/internal/engine/types" +) + +var ( + EmptyTable = Table{ + Cols: make([]Col, 0), + Rows: make([]Row, 0), + } +) + +// Table is a one-dimensional collection of Rows. +type Table struct { + Cols []Col + Rows []Row +} + +// Col is a header for a single column in a table, containing the qualified name +// of the col, a possible alias and the col data type. +type Col struct { + QualifiedName string + Alias string + Type types.Type +} + +// Row is a one-dimensional collection of values. +type Row struct { + Values []types.Value +} + +func (t Table) RemoveColumnByQualifiedName(qualifiedName string) Table { + index := -1 + for i, col := range t.Cols { + if col.QualifiedName == qualifiedName { + index = i + break + } + } + if index != -1 { + return t.RemoveColumn(index) + } + return t +} + +// RemoveColumn works on a copy of the table, and removes the column with the +// given index from the copy. After removal, the copy is returned. +func (t Table) RemoveColumn(index int) Table { + t.Cols = append(t.Cols[:index], t.Cols[index+1:]...) + for i := range t.Rows { + t.Rows[i].Values = append(t.Rows[i].Values[:index], t.Rows[i].Values[index+1:]...) + } + return t +} + +func (t Table) String() string { + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 1, 3, ' ', 0) + + var colNames []string + for _, col := range t.Cols { + colName := col.QualifiedName + if col.Alias != "" { + colName = col.Alias + } + colNames = append(colNames, colName+" ("+col.Type.String()+")") + } + _, _ = fmt.Fprintln(w, strings.Join(colNames, "\t")) + + for _, row := range t.Rows { + var strVals []string + for i := 0; i < len(row.Values); i++ { + strVals = append(strVals, row.Values[i].String()) + } + _, _ = fmt.Fprintln(w, strings.Join(strVals, "\t")) + } + _ = w.Flush() + return buf.String() +} diff --git a/internal/test/testdata/example01/output b/internal/test/testdata/example01/output index f0b88611..5ce07659 100644 --- a/internal/test/testdata/example01/output +++ b/internal/test/testdata/example01/output @@ -1,2 +1,2 @@ -Integer +column1 (Integer) 85734726843 diff --git a/internal/test/testdata/example02/output b/internal/test/testdata/example02/output index 985c6de7..4f90ea6a 100644 --- a/internal/test/testdata/example02/output +++ b/internal/test/testdata/example02/output @@ -1,2 +1,2 @@ -Date Integer +column1 (Date) column2 (Integer) 2020-07-02T14:03:27Z 85734726843 diff --git a/internal/test/testdata/issue187/output b/internal/test/testdata/issue187/output index 429fb9ae..6e3d2f21 100644 --- a/internal/test/testdata/issue187/output +++ b/internal/test/testdata/issue187/output @@ -1,3 +1,3 @@ -Integer String Integer -1 2 3 -4 5 6 +column1 (Integer) column2 (String) column3 (Integer) +1 2 3 +4 5 6 diff --git a/internal/test/testdata/issue188/output b/internal/test/testdata/issue188/output index fc5f7099..1ce6afe5 100644 --- a/internal/test/testdata/issue188/output +++ b/internal/test/testdata/issue188/output @@ -1,2 +1,2 @@ -String String String -abc a"bc abc +column1 (String) column2 (String) column3 (String) +abc a"bc abc From e14c42b7518f2651a95107a00d7b0dad165776cc Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 10 Jul 2020 11:55:17 +0200 Subject: [PATCH 67/77] Add godoc --- internal/engine/table.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/engine/table.go b/internal/engine/table.go index 07bfcd63..7d88bed1 100644 --- a/internal/engine/table.go +++ b/internal/engine/table.go @@ -10,6 +10,7 @@ import ( ) var ( + // EmptyTable is the empty table, with 0 cols and 0 rows. EmptyTable = Table{ Cols: make([]Col, 0), Rows: make([]Row, 0), @@ -35,6 +36,10 @@ type Row struct { Values []types.Value } +// RemoveColumnByQualifiedName will remove the first column with the given +// qualified name from the table, and return the new table. The original table +// will not be modified. If no such column exists, the original table is +// returned. func (t Table) RemoveColumnByQualifiedName(qualifiedName string) Table { index := -1 for i, col := range t.Cols { From eb29cdc6754b6baa5ba30a44ed1796ae848f74e9 Mon Sep 17 00:00:00 2001 From: Tim Satke <48135919+TimSatke@users.noreply.github.com> Date: Fri, 10 Jul 2020 12:22:34 +0200 Subject: [PATCH 68/77] Update internal/engine/error.go --- internal/engine/error.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/engine/error.go b/internal/engine/error.go index 52efb678..ab4e515f 100644 --- a/internal/engine/error.go +++ b/internal/engine/error.go @@ -23,7 +23,7 @@ const ( ErrUnsupported Error = "unsupported" ) -// ErrNoSuchFunction returns an error indicating that an error with the given +// ErrNoSuchFunction returns an error indicating that a function with the given // name can not be found. func ErrNoSuchFunction(name string) Error { return Error(fmt.Sprintf("no function for name %v(...)", name)) From 4f64a98c9c405e58eecf8c6c742aef1641d43e6c Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 10 Jul 2020 12:47:59 +0200 Subject: [PATCH 69/77] Remove todos --- internal/engine/compare.go | 1 - internal/engine/evaluate.go | 2 +- internal/engine/scan.go | 2 -- internal/engine/storage/cache/lru.go | 1 - internal/engine/storage/page_manager.go | 2 +- 5 files changed, 2 insertions(+), 6 deletions(-) diff --git a/internal/engine/compare.go b/internal/engine/compare.go index 9e65f402..d145309f 100644 --- a/internal/engine/compare.go +++ b/internal/engine/compare.go @@ -33,7 +33,6 @@ func (e Engine) cmp(left, right types.Value) cmpResult { } res, err := comparator.Compare(left, right) if err != nil { - // TODO: log error? return cmpUncomparable } switch res { diff --git a/internal/engine/evaluate.go b/internal/engine/evaluate.go index 4da8afb2..9378b471 100644 --- a/internal/engine/evaluate.go +++ b/internal/engine/evaluate.go @@ -64,7 +64,7 @@ func (e Engine) evaluateProjection(ctx ExecutionContext, proj command.Project) ( colName = col.Table + "." + colName } - // TODO: support alias + // #193: support alias expectedColumnNames = append(expectedColumnNames, colName) } diff --git a/internal/engine/scan.go b/internal/engine/scan.go index 5915ed8d..9100ab62 100644 --- a/internal/engine/scan.go +++ b/internal/engine/scan.go @@ -10,8 +10,6 @@ func (e Engine) scanSimpleTable(ctx ExecutionContext, table command.SimpleTable) return table, nil } - // TODO: load table from the database file - ctx.putScannedTable(table.QualifiedName(), Table{}) return Table{}, ErrUnimplemented("scan simple table") } diff --git a/internal/engine/storage/cache/lru.go b/internal/engine/storage/cache/lru.go index 9c9dc711..dd95ed4a 100644 --- a/internal/engine/storage/cache/lru.go +++ b/internal/engine/storage/cache/lru.go @@ -61,7 +61,6 @@ func (c *LRUCache) Flush(id page.ID) error { // Close will flush all dirty pages and then close this cache. func (c *LRUCache) Close() error { - // TODO: can we really just flush on close? for id := range c.pages { _ = c.flush(id) } diff --git a/internal/engine/storage/page_manager.go b/internal/engine/storage/page_manager.go index c133e99d..d36478a7 100644 --- a/internal/engine/storage/page_manager.go +++ b/internal/engine/storage/page_manager.go @@ -50,7 +50,7 @@ func (m *PageManager) ReadPage(id page.ID) (*page.Page, error) { // WritePage will write the given page into secondary storage. It is guaranteed, // that after this call returns, the page is present on disk. func (m *PageManager) WritePage(p *page.Page) error { - data := p.RawData() // TODO: avoid copying in RawData() + data := p.RawData() _, err := m.file.WriteAt(data, int64(p.ID())*page.Size) if err != nil { return fmt.Errorf("write at: %w", err) From cc49763aa773d60b7bb97cae35de452703d537ce Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 10 Jul 2020 12:50:46 +0200 Subject: [PATCH 70/77] Remove database package --- internal/database/column/basetype_string.go | 25 - internal/database/column/column.go | 12 - internal/database/column/doc.go | 4 - internal/database/column/type.go | 40 - internal/database/database.go | 8 - internal/database/doc.go | 3 - internal/database/schema/doc.go | 4 - internal/database/schema/schema.go | 8 - internal/database/storage/btree/btree.go | 291 ------- internal/database/storage/btree/btree_test.go | 744 ------------------ internal/database/storage/btree/doc.go | 9 - internal/database/storage/doc.go | 2 - internal/database/storage/securefs/doc.go | 5 - .../database/storage/securefs/secure_file.go | 237 ------ .../database/storage/securefs/secure_fs.go | 85 -- .../storage/securefs/secure_fs_test.go | 52 -- internal/database/storage/storage.go | 4 - internal/database/table/doc.go | 2 - internal/database/table/table.go | 15 - 19 files changed, 1550 deletions(-) delete mode 100644 internal/database/column/basetype_string.go delete mode 100644 internal/database/column/column.go delete mode 100644 internal/database/column/doc.go delete mode 100644 internal/database/column/type.go delete mode 100644 internal/database/database.go delete mode 100644 internal/database/doc.go delete mode 100644 internal/database/schema/doc.go delete mode 100644 internal/database/schema/schema.go delete mode 100644 internal/database/storage/btree/btree.go delete mode 100644 internal/database/storage/btree/btree_test.go delete mode 100644 internal/database/storage/btree/doc.go delete mode 100644 internal/database/storage/doc.go delete mode 100644 internal/database/storage/securefs/doc.go delete mode 100644 internal/database/storage/securefs/secure_file.go delete mode 100644 internal/database/storage/securefs/secure_fs.go delete mode 100644 internal/database/storage/securefs/secure_fs_test.go delete mode 100644 internal/database/storage/storage.go delete mode 100644 internal/database/table/doc.go delete mode 100644 internal/database/table/table.go diff --git a/internal/database/column/basetype_string.go b/internal/database/column/basetype_string.go deleted file mode 100644 index 5f685666..00000000 --- a/internal/database/column/basetype_string.go +++ /dev/null @@ -1,25 +0,0 @@ -// Code generated by "stringer -type=BaseType"; DO NOT EDIT. - -package column - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[Unknown-0] - _ = x[Decimal-1] - _ = x[Varchar-2] -} - -const _BaseType_name = "UnknownDecimalVarchar" - -var _BaseType_index = [...]uint8{0, 7, 14, 21} - -func (i BaseType) String() string { - if i >= BaseType(len(_BaseType_index)-1) { - return "BaseType(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _BaseType_name[_BaseType_index[i]:_BaseType_index[i+1]] -} diff --git a/internal/database/column/column.go b/internal/database/column/column.go deleted file mode 100644 index 297ffd96..00000000 --- a/internal/database/column/column.go +++ /dev/null @@ -1,12 +0,0 @@ -package column - -// Column describes a database column, that consists of a type and multiple -// attributes, such as nullability, if it is a primary key etc. -type Column interface { - Type() Type - IsNullable() bool - IsPrimaryKey() bool - ShouldAutoincrement() bool - // extend this as we add support for more things, such as default values, - // uniqueness etc. -} diff --git a/internal/database/column/doc.go b/internal/database/column/doc.go deleted file mode 100644 index 203adbbb..00000000 --- a/internal/database/column/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package column describes columns inside the database. A column consists of a -// type, type parameters and a few additional attributes, such as if the column -// is nullable, if it is a primary key etc. -package column diff --git a/internal/database/column/type.go b/internal/database/column/type.go deleted file mode 100644 index a09dc1e7..00000000 --- a/internal/database/column/type.go +++ /dev/null @@ -1,40 +0,0 @@ -package column - -//go:generate stringer -type=BaseType - -// BaseType is the base type of a column, in unparameterized form. To -// parameterize a base type, use a (column.Type). -type BaseType uint16 - -var ( - parameterCount = map[BaseType]uint8{ - Decimal: 2, - Varchar: 1, - } -) - -// NumParameters returns the amount of parameters, that the base type supports. -// For example, this is 1 for the VARCHAR type, and 2 for the DECIMAL type. -func (t BaseType) NumParameters() uint8 { - return parameterCount[t] // zero is default value -} - -// Supported base types. -const ( - Unknown BaseType = iota - Decimal - Varchar -) - -// Type describes a type that consists of a base type and zero, one or two -// number parameters. -type Type interface { - // BaseType returns the base type of this column type. Depending on the base - // type, IsParameterized implies different constellations. Some base types - // support only one parameter, some support two. If it supports one or two - // parameters can be determined by calling NumParameters() of the BaseType. - BaseType() BaseType - IsParameterized() bool - FirstParameter() float64 - SecondParameter() float64 -} diff --git a/internal/database/database.go b/internal/database/database.go deleted file mode 100644 index ec4191c2..00000000 --- a/internal/database/database.go +++ /dev/null @@ -1,8 +0,0 @@ -package database - -import "github.com/tomarrell/lbadd/internal/database/schema" - -// DB describes a database. -type DB interface { - Schema(name string) (schema.Schema, bool) -} diff --git a/internal/database/doc.go b/internal/database/doc.go deleted file mode 100644 index 2d17d0e4..00000000 --- a/internal/database/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package database implements the database structure. Every method and function -// defined in here works with arguments instead of an SQL statement. -package database diff --git a/internal/database/schema/doc.go b/internal/database/schema/doc.go deleted file mode 100644 index e67a384c..00000000 --- a/internal/database/schema/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package schema implements a schema structure. Every method and function -// defined in here works with arguments instead of an SQL statement. A schema -// consists of zero or more tables that can be retrieved. -package schema diff --git a/internal/database/schema/schema.go b/internal/database/schema/schema.go deleted file mode 100644 index 96f78647..00000000 --- a/internal/database/schema/schema.go +++ /dev/null @@ -1,8 +0,0 @@ -package schema - -import "github.com/tomarrell/lbadd/internal/database/table" - -// Schema describes a schema, which consists of zero or more tables. -type Schema interface { - Table(name string) (table.Table, bool) -} diff --git a/internal/database/storage/btree/btree.go b/internal/database/storage/btree/btree.go deleted file mode 100644 index e8289bb1..00000000 --- a/internal/database/storage/btree/btree.go +++ /dev/null @@ -1,291 +0,0 @@ -package btree - -const defaultOrder = 3 - -// Btree describes a btree. -type Btree interface { - get(k key) (v *entry, exists bool) - insert(k key, v value) - remove(k key) (removed bool) - getAll(limit int) []*entry - getAbove(k key, limit int) []*entry - getBelow(k key, limit int) []*entry - getBetween(low, high key, limit int) []*entry -} - -type ( - key int - value interface{} -) - -// node defines the stuct which contains keys (entries) and -// the child nodes of a particular node in the b-tree -type node struct { - parent *node - entries []*entry - children []*node -} - -// entry is a key/value pair that is stored in the b-tree -type entry struct { - key key - value value -} - -// btree is the main structure. -// -// "order" invariants: -// - every node except root must contain at least order-1 keys -// - every node may contain at most (2*order)-1 keys -type btree struct { - root *node - size int - order int -} - -// newBtree creates a new instance of Btree -func newBtree() *btree { - return &btree{ - root: nil, - size: 0, - order: defaultOrder, - } -} - -func newBtreeOrder(order int) *btree { - return &btree{ - root: nil, - size: 0, - order: order, - } -} - -// get searches for a specific key in the btree, -// returning a pointer to the resulting entry -// and a boolean as to whether it exists in the tree -func (b *btree) get(k key) (result *entry, exists bool) { - if b.root == nil || len(b.root.entries) == 0 { - return nil, false - } - - return b.getNode(b.root, k) -} - -func (b *btree) getNode(node *node, k key) (result *entry, exists bool) { - i, exists := b.search(node.entries, k) - if exists { - return node.entries[i], true - } - - if i > len(node.children) { - return nil, false - } - - return b.getNode(node.children[i], k) -} - -// insert takes a key and value, creats a new -// entry and inserts it in the tree according to the key -func (b *btree) insert(k key, v value) { - if b.root == nil { - b.size++ - b.root = &node{ - parent: nil, - entries: []*entry{{k, v}}, - children: []*node{}, - } - return - } - - b.insertNode(b.root, &entry{k, v}) -} - -// insertNode takes a node and the entry to insert -func (b *btree) insertNode(node *node, entry *entry) (inserted bool) { - // If the root node is already full, we need to split it - if node == b.root && node.isFull(b.order) { - b.root = node.split() - } - - // Search for the key in the node's entries - idx, exists := b.search(node.entries, entry.key) - - // The entry already exists, so it should be updated - if exists { - node.entries[idx] = entry - return false - } - - // If the node is a leaf node, add entry to the entries list - // We can guarantee that we have room as it would otherwise have - // been split. - if node.isLeaf() { - node.entries = append(node.entries, nil) - copy(node.entries[idx+1:], node.entries[idx:]) - node.entries[idx] = entry - b.size++ - return true - } - - // The node is not a leaf, so we we need to check - // if the appropriate child is already full, - // and conditionally split it. Otherwise traverse - // to that child. - if node.children[idx].isFull(b.order) { - node.children[idx] = node.children[idx].split() - } - - return b.insertNode(node.children[idx], entry) -} - -// remove tries to delete an entry from the tree, and -// returns true if the entry was removed, and false if -// the key was not found in the tree -func (b *btree) remove(k key) (removed bool) { - if b.root == nil { - return false - } - - return b.removeNode(b.root, k) -} - -// removeNode takes a node and key and bool, and recursively deletes -// k from the node, while maintaining the order invariants -func (b *btree) removeNode(node *node, k key) (removed bool) { - idx, exists := b.search(node.entries, k) - - // If the key exists in a leaf node, we can simply remove - // it outright - if node.isLeaf() { - if exists { - b.size-- - node.entries = append(node.entries[:idx], node.entries[idx+1:]...) - return true - } - // We've reached the bottom and couldn't find the key - return false - } - - // If the key exists in the node, but it is not a leaf - if exists { - child := node.children[idx] - // There are enough entries in left child to take one - if child.canSteal(b.order) { - stolen := child.entries[len(child.entries)-1] - node.entries[idx] = stolen - return b.removeNode(child, stolen.key) - } - - // child = node.children[idx] - // There are enough entries in the right child to take one - // if child.canSteal(b.order) { - // TODO implement this - // } - - // Both children don't have enough entries, so we need - // to merge the left and right children and take a key - // TODO - } - - return b.removeNode(node.children[idx], k) -} - -// -func (b *btree) getAll(limit int) []*entry { - if b.size == 0 || limit == 0 { - return []*entry{} - } - - // TODO unimplemented - - return nil -} - -// -func (b *btree) getAbove(k key, limit int) []*entry { - // TODO unimplemented - return []*entry{} -} - -// -func (b *btree) getBelow(k key, limit int) []*entry { - // TODO unimplemented - return []*entry{} -} - -// -func (b *btree) getBetween(low, high key, limit int) []*entry { - // TODO unimplemented - return []*entry{} -} - -// search takes a slice of entries and a key, and returns -// the position that the key would fit relative to all -// other entries' keys. -// e.g. -// b.search([1, 2, 4], 3) => (2, false) -func (b *btree) search(entries []*entry, k key) (index int, exists bool) { - var ( - low = 0 - mid = 0 - high = len(entries) - 1 - ) - - for low <= high { - mid = (high + low) / 2 - - entryKey := entries[mid].key - switch { - case k > entryKey: - low = mid + 1 - case k < entryKey: - high = mid - 1 - case k == entryKey: - return mid, true - } - } - - return low, false -} - -func (n *node) isLeaf() bool { - return len(n.children) == 0 -} - -// isFull returns a bool indication whether the node -// already contains the maximum number of entries -// allowed for a given order -func (n *node) isFull(order int) bool { - return len(n.entries) >= ((order * 2) - 1) -} - -// canSteal returns a bool indicating whether or not -// the node contains enough entries to be able to take one -func (n *node) canSteal(order int) bool { - return len(n.entries)-1 > order-1 -} - -// Splits a full node to have a single, median, -// entry, and two child nodes containing the left -// and right halves of the entries -func (n *node) split() *node { - if len(n.entries) == 0 { - return n - } - - mid := len(n.entries) / 2 - - left := &node{ - parent: n, - entries: append([]*entry{}, n.entries[:mid]...), - } - right := &node{ - parent: n, - entries: append([]*entry{}, n.entries[mid:]...), - } - - n.entries = []*entry{{n.entries[mid].key, nil}} - n.children = append(n.children, left, right) - - return n -} diff --git a/internal/database/storage/btree/btree_test.go b/internal/database/storage/btree/btree_test.go deleted file mode 100644 index 7629da99..00000000 --- a/internal/database/storage/btree/btree_test.go +++ /dev/null @@ -1,744 +0,0 @@ -package btree - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestBTree(t *testing.T) { - t.Skip() - cases := []struct { - name string - insert []entry - get []entry - }{ - { - name: "set and get", - insert: []entry{{1, 1}}, - get: []entry{{1, 1}}, - }, - } - - order := 3 - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - bt := newBtreeOrder(order) - - for _, e := range tc.insert { - bt.insert(e.key, e.value) - } - - for _, g := range tc.get { - assert.Equal(t, - g.value, - func() *entry { - e, _ := bt.get(g.key) - return e - }(), - ) - } - }) - } -} - -func TestGet(t *testing.T) { - cases := []struct { - name string - root *node - key key - expectedExists bool - }{ - { - name: "no root", - root: nil, - expectedExists: false, - }, - { - name: "empty root", - root: &node{}, - expectedExists: false, - }, - { - name: "entries only in root", - root: &node{entries: []*entry{{1, 1}, {2, 2}, {3, 3}}}, - key: 2, - expectedExists: true, - }, - { - name: "entry one level deep left of root", - root: &node{ - entries: []*entry{{2, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}}}, - {entries: []*entry{{2, 2}, {3, 3}}}, - }, - }, - key: 1, - expectedExists: true, - }, - { - name: "entry one level deep right of root", - root: &node{ - entries: []*entry{{2, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}}}, - {entries: []*entry{{2, 2}, {3, 3}}}, - }, - }, - key: 3, - expectedExists: true, - }, - { - name: "depth > 1 and key not exist", - root: &node{ - entries: []*entry{{2, 2}}, - children: []*node{ - {entries: []*entry{{1, 1}}}, - {entries: []*entry{{2, 2}, {3, 3}}}, - }, - }, - key: 4, - expectedExists: false, - }, - { - name: "depth = 3 found", - root: &node{ - entries: []*entry{{2, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}}}, - { - entries: []*entry{{3, nil}}, - children: []*node{ - {entries: []*entry{{2, 2}}}, - {entries: []*entry{{3, 3}, {4, 4}}}, - }, - }, - }, - }, - key: 4, - expectedExists: true, - }, - { - name: "depth = 3 not found", - root: &node{ - entries: []*entry{{2, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}}}, - { - entries: []*entry{{3, nil}}, - children: []*node{ - {entries: []*entry{{2, 2}}}, - {entries: []*entry{{3, 3}, {4, 4}}}, - }, - }, - }, - }, - key: 5, - expectedExists: false, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - btree := newBtree() - btree.root = tc.root - - _, exists := btree.get(tc.key) - assert.Equal(t, tc.expectedExists, exists) - }) - } -} - -func TestKeySearch(t *testing.T) { - cases := []struct { - name string - entries []*entry - key key - exists bool - index int - }{ - { - name: "single value", - entries: []*entry{{key: 1}}, - key: 2, - exists: false, - index: 1, - }, - { - name: "single value, already exists", - entries: []*entry{{key: 1}}, - key: 1, - exists: true, - index: 0, - }, - { - name: "already exists", - entries: []*entry{{key: 1}, {key: 2}, {key: 4}, {key: 5}}, - key: 4, - exists: true, - index: 2, - }, - { - name: "doc example", - entries: []*entry{{key: 1}, {key: 2}, {key: 4}}, - key: 3, - exists: false, - index: 2, - }, - { - name: "no entries", - entries: []*entry{}, - key: 2, - exists: false, - index: 0, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - bt := newBtree() - - idx, exists := bt.search(tc.entries, tc.key) - - assert.Equal(t, tc.exists, exists) - assert.Equal(t, tc.index, idx) - }) - } -} - -func TestNodeSplit(t *testing.T) { - parent := &node{} - - cases := []struct { - name string - root bool - input *node - expected *node - }{ - { - name: "simple node", - input: &node{parent: parent, entries: []*entry{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}}, - expected: &node{ - parent: parent, - entries: []*entry{{3, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}, {2, 2}}}, - {entries: []*entry{{3, 3}, {4, 4}, {5, 5}}}, - }, - }, - }, - { - name: "even entries node", - input: &node{parent: parent, entries: []*entry{{1, 1}, {2, 2}, {3, 3}, {4, 4}}}, - expected: &node{ - parent: parent, - entries: []*entry{{3, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}, {2, 2}}}, - {entries: []*entry{{3, 3}, {4, 4}}}, - }, - }, - }, - { - name: "no parent", - input: &node{entries: []*entry{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}}, - root: true, - expected: &node{ - entries: []*entry{{3, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}, {2, 2}}}, - {entries: []*entry{{3, 3}, {4, 4}, {5, 5}}}, - }, - }, - }, - { - name: "empty node", - input: &node{parent: parent, entries: []*entry{}}, - expected: &node{ - parent: parent, - entries: []*entry{}, - children: []*node{}, - }, - }, - { - name: "single entry", - input: &node{parent: parent, entries: []*entry{{1, 1}}}, - expected: &node{ - parent: parent, - entries: []*entry{{1, nil}}, - children: []*node{}, - }, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - newNode := tc.input.split() - assert.Equal(t, tc.expected.parent, newNode.parent) - assert.Equal(t, tc.expected.entries, newNode.entries) - - for i := range tc.expected.children { - expectedChild, newChild := tc.expected.children[i], newNode.children[i] - - assert.Equal(t, &tc.input, &newChild.parent) - assert.Equal(t, expectedChild.entries, newChild.entries) - assert.Equal(t, expectedChild.children, newChild.children) - } - }) - } -} - -func TestInsertNode(t *testing.T) { - type fields struct { - root *node - size int - } - type args struct { - node *node - entry *entry - } - - tests := []struct { - name string - fields fields - args args - wantSize int - wantInserted bool - }{ - { - name: "insert single entry", - fields: fields{}, - args: args{&node{}, &entry{1, 1}}, - wantSize: 1, - wantInserted: true, - }, - { - name: "entry already exists", - fields: fields{size: 1}, - args: args{&node{entries: []*entry{{1, 1}}}, &entry{1, 1}}, - wantSize: 1, - wantInserted: false, - }, - { - name: "entry exists one level down right", - fields: fields{size: 2}, - args: args{ - &node{ - entries: []*entry{{2, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}}}, - {entries: []*entry{{2, 2}}}, - }, - }, - &entry{2, 2}, - }, - wantSize: 2, - wantInserted: false, - }, - { - name: "entry exists one level down right unbalanced", - fields: fields{size: 2}, - args: args{ - &node{ - entries: []*entry{{1, nil}}, - children: []*node{ - {}, - {entries: []*entry{{1, 1}, {2, 2}}}, - }, - }, - &entry{2, 2}, - }, - wantSize: 2, - wantInserted: false, - }, - { - name: "entry exists one level down left", - fields: fields{size: 2}, - args: args{ - &node{ - entries: []*entry{{2, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}}}, - {entries: []*entry{{2, 2}}}, - }, - }, - &entry{1, 1}, - }, - wantSize: 2, - wantInserted: false, - }, - { - name: "entry inserted one level down right", - fields: fields{size: 2}, - args: args{ - &node{ - entries: []*entry{{3, nil}}, - children: []*node{ - {entries: []*entry{{1, 1}}}, - {entries: []*entry{{3, 3}}}, - }, - }, - &entry{4, 4}, - }, - wantSize: 3, - wantInserted: true, - }, - { - name: "entry inserted one level down left, would overflow", - fields: fields{size: 6}, - args: args{ - &node{ - entries: []*entry{{10, nil}}, - children: []*node{ - {entries: []*entry{{3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}}}, - {entries: []*entry{{10, 10}}}, - }, - }, - &entry{1, 1}, - }, - wantSize: 7, - wantInserted: true, - }, - { - name: "entry inserted one level down right, would more than overflow", - fields: fields{size: 4}, - args: args{ - &node{ - entries: []*entry{{10, nil}}, - children: []*node{ - {entries: []*entry{{3, 3}, {4, 4}, {5, 5}}}, - {entries: []*entry{{10, 10}, {11, 11}, {12, 12}, {13, 13}, {14, 14}, {15, 15}, {16, 16}, {17, 17}, {18, 18}, {19, 19}, {29, 29}}}, - }, - }, - &entry{30, 30}, - }, - wantSize: 5, - wantInserted: true, - }, - { - name: "entry inserted one level down right, would more than overflow", - fields: fields{size: 4}, - args: args{ - &node{ - entries: []*entry{{10, nil}}, - children: []*node{ - {entries: []*entry{{3, 3}, {4, 4}, {5, 5}}}, - {entries: []*entry{{10, 10}, {11, 11}, {12, 12}, {13, 13}, {14, 14}, {15, 15}, {16, 16}, {17, 17}, {18, 18}, {19, 19}, {29, 29}}}, - }, - }, - &entry{30, 30}, - }, - wantSize: 5, - wantInserted: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &btree{ - root: tt.fields.root, - size: tt.fields.size, - order: 3, - } - - got := b.insertNode(tt.args.node, tt.args.entry) - assert.Equal(t, tt.wantInserted, got) - assert.Equal(t, tt.wantSize, b.size) - }) - } -} - -func TestRemove(t *testing.T) { - type fields struct { - root *node - size int - order int - } - type args struct { - k key - } - - tests := []struct { - name string - fields fields - args args - wantRemoved bool - wantSize int - }{ - { - name: "no root", - fields: fields{root: nil, size: 0, order: 3}, - args: args{k: 1}, - wantRemoved: false, - wantSize: 0, - }, - { - name: "remove entry from root", - fields: fields{root: &node{entries: []*entry{{1, 1}}}, size: 1, order: 3}, - args: args{k: 1}, - wantRemoved: true, - wantSize: 0, - }, - { - name: "remove entry from non-leaf node with children", - fields: fields{ - size: 8, - order: 3, - root: &node{ - entries: []*entry{{5, 5}}, - children: []*node{ - {entries: []*entry{{1, 1}, {2, 2}, {3, 3}, {4, 4}}}, - {entries: []*entry{{6, 6}, {7, 7}, {8, 8}}}, - }, - }, - }, - args: args{k: 5}, - wantRemoved: true, - wantSize: 7, - }, - { - name: "remove entry depth 1 right is leaf", - fields: fields{ - size: 3, - order: 2, - root: &node{ - entries: []*entry{{1, 1}}, - children: []*node{ - {}, - {entries: []*entry{{2, 2}, {3, 3}}}, - }, - }, - }, - args: args{k: 2}, - wantRemoved: true, - wantSize: 2, - }, - { - name: "remove entry depth 1 left is leaf", - fields: fields{ - size: 3, - order: 2, - root: &node{ - entries: []*entry{{3, 3}}, - children: []*node{ - {entries: []*entry{{1, 1}, {2, 2}}}, - {}, - }, - }, - }, - args: args{k: 1}, - wantRemoved: true, - wantSize: 2, - }, - { - name: "key doesn't exist", - fields: fields{ - size: 2, - order: 2, - root: &node{ - entries: []*entry{{2, 2}}, - children: []*node{ - {entries: []*entry{{1, 1}}}, - {}, - }, - }, - }, - args: args{k: 3}, - wantRemoved: false, - wantSize: 2, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &btree{ - root: tt.fields.root, - size: tt.fields.size, - order: tt.fields.order, - } - - gotRemoved := b.remove(tt.args.k) - assert.Equal(t, tt.wantRemoved, gotRemoved) - assert.Equal(t, tt.wantSize, b.size) - }) - } -} - -func TestNode_isFull(t *testing.T) { - type fields struct { - parent *node - entries []*entry - children []*node - } - type args struct { - order int - } - - tests := []struct { - name string - fields fields - args args - want bool - }{ - { - name: "order 3, node not full", - fields: fields{entries: []*entry{}}, - args: args{order: 3}, - want: false, - }, - { - name: "order 3, node full", - fields: fields{entries: []*entry{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}}, - args: args{order: 3}, - want: true, - }, - { - name: "order 3, node nearly full", - fields: fields{entries: []*entry{{1, 1}, {2, 2}, {3, 3}, {4, 4}}}, - args: args{order: 3}, - want: false, - }, - { - name: "order 3, node over filled (bug case)", - fields: fields{entries: []*entry{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}}}, - args: args{order: 3}, - want: true, - }, - { - name: "order 5, node full", - fields: fields{entries: []*entry{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}, {8, 8}, {9, 9}}}, - args: args{order: 5}, - want: true, - }, - { - name: "order 5, node almost full", - fields: fields{entries: []*entry{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}, {7, 7}, {8, 8}}}, - args: args{order: 5}, - want: false, - }, - { - name: "order 5, node empty", - fields: fields{entries: []*entry{}}, - args: args{order: 5}, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - n := &node{ - parent: tt.fields.parent, - entries: tt.fields.entries, - children: tt.fields.children, - } - - got := n.isFull(tt.args.order) - assert.Equal(t, tt.want, got) - }) - } -} - -func TestNode_canSteal(t *testing.T) { - type fields struct { - parent *node - entries []*entry - children []*node - } - type args struct { - order int - } - - tests := []struct { - name string - fields fields - args args - want bool - }{ - // TODO add test cases - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - n := &node{ - parent: tt.fields.parent, - entries: tt.fields.entries, - children: tt.fields.children, - } - - got := n.canSteal(tt.args.order) - assert.Equal(t, tt.want, got) - }) - } -} - -func Test_btree_getAll(t *testing.T) { - type fields struct { - root *node - size int - order int - } - type args struct { - limit int - } - - root := &node{} - root.entries = []*entry{{4, 4}, {8, 8}} - root.children = []*node{ - { - parent: root, - entries: []*entry{{0, 0}, {1, 1}, {2, 2}}, - }, - { - parent: root, - entries: []*entry{{5, 5}, {7, 7}}, - }, - { - parent: root, - entries: []*entry{{9, 9}, {11, 11}, {12, 12}}, - }, - } - - f := fields{root: root, size: 10} - - tests := []struct { - name string - fields fields - args args - want []*entry - }{ - { - name: "empty tree returns empty entries", - fields: fields{size: 0}, - args: args{limit: 10}, - want: []*entry{}, - }, - { - name: "limit of 0 returns empty entries", - fields: f, - args: args{limit: 0}, - want: []*entry{}, - }, - { - name: "returns all entries up to limit", - fields: f, - args: args{limit: 0}, - want: []*entry{}, - }, - // TODO add more cases - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &btree{ - root: tt.fields.root, - size: tt.fields.size, - order: tt.fields.order, - } - - got := b.getAll(tt.args.limit) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/internal/database/storage/btree/doc.go b/internal/database/storage/btree/doc.go deleted file mode 100644 index f3401185..00000000 --- a/internal/database/storage/btree/doc.go +++ /dev/null @@ -1,9 +0,0 @@ -// Package btree contains the btree struct, which is used as the primary data store of -// the database. -// -// The btree supports 3 primary operations: -// - get: given a key, retrieve the corresponding entry -// - put: given a key and a value, create an entry in the btree -// - remove: given a key, remove the corresponding entry in the tree if it -// exists -package btree diff --git a/internal/database/storage/doc.go b/internal/database/storage/doc.go deleted file mode 100644 index 597379ec..00000000 --- a/internal/database/storage/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package storage does something. TODO(tomarrell) create doc. -package storage \ No newline at end of file diff --git a/internal/database/storage/securefs/doc.go b/internal/database/storage/securefs/doc.go deleted file mode 100644 index c0d883e0..00000000 --- a/internal/database/storage/securefs/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -// Package securefs implements an afero.Fs that reads a file's content into -// memory when opening or creating it. All data is held encrypted by using -// github.com/awnumar/memguard. File writes need to be synced manually with the -// disk contents by calling file.Sync(). -package securefs diff --git a/internal/database/storage/securefs/secure_file.go b/internal/database/storage/securefs/secure_file.go deleted file mode 100644 index 7528f19e..00000000 --- a/internal/database/storage/securefs/secure_file.go +++ /dev/null @@ -1,237 +0,0 @@ -package securefs - -import ( - "fmt" - "io" - "os" - - "github.com/awnumar/memguard" - "github.com/spf13/afero" -) - -var _ afero.File = (*secureFile)(nil) - -type secureFile struct { - file afero.File - - enclave *memguard.Enclave - pointer int64 - closed bool -} - -func newSecureFile(file afero.File) (*secureFile, error) { - secureFile := &secureFile{ - file: file, - } - if err := secureFile.load(); err != nil { - return nil, fmt.Errorf("load: %w", err) - } - return secureFile, nil -} - -// ReadAt reads len(p) bytes into p, starting from off. If EOF is reached before -// reading was finished, all read bytes are returned, together with an io.EOF -// error. BE AWARE THAT BYTES ARE COPIED FROM A SECURE AREA TO A POTENTIALLY -// INSECURE (your byte slice), AND THAT ALL READ BYTES ARE NO LONGER SECURE. -func (f *secureFile) ReadAt(p []byte, off int64) (int, error) { - if err := f.ensureOpen(); err != nil { - return 0, err - } - - buffer, err := f.enclave.Open() - if err != nil { - return 0, fmt.Errorf("open enclave: %w", err) - } - defer func() { - f.enclave = buffer.Seal() - }() - data := buffer.Bytes() - - n := copy(p, data[off:]) - if n < len(p) { - return n, io.EOF - } - return n, nil -} - -func (f *secureFile) WriteAt(p []byte, off int64) (n int, err error) { - if err = f.ensureOpen(); err != nil { - return 0, err - } - - if f.enclave == nil || int(off)+len(p) > f.enclave.Size() { - if err := f.grow(int(off) + len(p)); err != nil { - return 0, fmt.Errorf("grow: %w", err) - } - } - - buffer, err := f.enclave.Open() - if err != nil { - return 0, fmt.Errorf("open enclave: %w", err) - } - defer func() { - f.enclave = buffer.Seal() - - if syncErr := f.Sync(); syncErr != nil { - err = fmt.Errorf("sync: %w", syncErr) - } - }() - buffer.Melt() - data := buffer.Bytes() - - n = copy(data[off:off+int64(len(p))], p) - if n != len(p) { - return n, fmt.Errorf("unable to write all bytes") - } - return n, nil -} - -func (f *secureFile) Close() error { - if f.closed { - return nil - } - - if err := f.Sync(); err != nil { - return fmt.Errorf("sync: %w", err) - } - if err := f.file.Close(); err != nil { - return fmt.Errorf("close underlying: %w", err) - } - f.enclave = nil - f.closed = true - return nil -} - -// Reads len(p) bytes into p and returns the number of bytes read. BE AWARE THAT -// BYTES ARE COPIED FROM A SECURE AREA TO A POTENTIALLY INSECURE (your byte -// slice), AND THAT ALL READ BYTES ARE NO LONGER SECURE. -func (f *secureFile) Read(p []byte) (n int, err error) { - if err := f.ensureOpen(); err != nil { - return 0, err - } - return f.ReadAt(p, f.pointer) -} - -func (f *secureFile) Seek(offset int64, whence int) (int64, error) { - if err := f.ensureOpen(); err != nil { - return 0, err - } - - switch whence { - case io.SeekCurrent: - f.pointer += offset - case io.SeekStart: - f.pointer = offset - case io.SeekEnd: - f.pointer = int64(f.enclave.Size()) - offset - default: - return f.pointer, fmt.Errorf("unsupported whence: %v", whence) - } - return f.pointer, nil -} - -func (f *secureFile) Write(p []byte) (n int, err error) { - if err := f.ensureOpen(); err != nil { - return 0, err - } - - return f.WriteAt(p, f.pointer) -} - -func (f *secureFile) Name() string { - return f.file.Name() -} - -func (f *secureFile) Readdir(count int) ([]os.FileInfo, error) { - return f.file.Readdir(count) -} - -func (f *secureFile) Readdirnames(n int) ([]string, error) { - return f.file.Readdirnames(n) -} - -func (f *secureFile) Stat() (os.FileInfo, error) { - return f.file.Stat() -} - -func (f *secureFile) Sync() error { - if err := f.ensureOpen(); err != nil { - return err - } - - buffer, err := f.enclave.Open() - if err != nil { - return fmt.Errorf("open enclave: %w", err) - } - defer func() { - f.enclave = buffer.Seal() - }() - - if err = f.file.Truncate(0); err != nil { - return fmt.Errorf("truncate: %w", err) - } - // Truncate alone doesn't work for, golang doesn't touch the file position - // indicator on truncate at all - _, err = f.file.Seek(0, io.SeekStart) - if err != nil { - return fmt.Errorf("seek: %w", err) - } - - _, err = buffer.Reader().WriteTo(f.file) - if err != nil { - return fmt.Errorf("write to: %w", err) - } - return nil -} - -func (f *secureFile) Truncate(size int64) error { - if err := f.grow(int(size)); err != nil { - return fmt.Errorf("grow: %w", err) - } - return nil -} - -func (f *secureFile) WriteString(s string) (ret int, err error) { - if err := f.ensureOpen(); err != nil { - return 0, err - } - - return f.Write([]byte(s)) -} - -func (f *secureFile) load() error { - buffer, err := memguard.NewBufferFromEntireReader(f.file) - if err != nil { - return fmt.Errorf("read all: %w", err) - } - f.enclave = buffer.Seal() - return nil -} - -func (f *secureFile) grow(newSize int) error { - if f.enclave == nil { - f.enclave = memguard.NewBuffer(newSize).Seal() - return nil - } - - oldBuffer, err := f.enclave.Open() - if err != nil { - return fmt.Errorf("open enclave: %w", err) - } - defer oldBuffer.Destroy() - - // allocate new memory and copy old data - newBuffer := memguard.NewBuffer(newSize) - newBuffer.Melt() - newBuffer.Copy(oldBuffer.Bytes()) - - f.enclave = newBuffer.Seal() - return nil -} - -func (f *secureFile) ensureOpen() error { - if f.closed { - return afero.ErrFileClosed - } - return nil -} diff --git a/internal/database/storage/securefs/secure_fs.go b/internal/database/storage/securefs/secure_fs.go deleted file mode 100644 index 5dbd6757..00000000 --- a/internal/database/storage/securefs/secure_fs.go +++ /dev/null @@ -1,85 +0,0 @@ -package securefs - -import ( - "os" - "time" - - "github.com/spf13/afero" -) - -var _ afero.Fs = (*secureFs)(nil) - -type secureFs struct { - fs afero.Fs -} - -// New creates a new secure fs, that only holds data in memory encrypted. Read -// operations read bytes in passed in byte slices, which makes the bytes leave a -// protected area. The given byte slice should also be in a protected area, but -// this is the caller's responsibility. Files that are opened and/or created -// with the returned Fs are 100% in memory. -func New(fs afero.Fs) afero.Fs { - return &secureFs{ - fs: fs, - } -} - -func (fs *secureFs) Create(name string) (afero.File, error) { - file, err := fs.fs.Create(name) - if err != nil { - return nil, err - } - return newSecureFile(file) -} - -func (fs *secureFs) Mkdir(name string, perm os.FileMode) error { - return fs.fs.Mkdir(name, perm) -} - -func (fs *secureFs) MkdirAll(path string, perm os.FileMode) error { - return fs.fs.MkdirAll(path, perm) -} - -func (fs *secureFs) Open(name string) (afero.File, error) { - file, err := fs.fs.Open(name) - if err != nil { - return nil, err - } - return newSecureFile(file) -} - -func (fs *secureFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { - file, err := fs.fs.OpenFile(name, flag, perm) - if err != nil { - return nil, err - } - return newSecureFile(file) -} - -func (fs *secureFs) Remove(name string) error { - return fs.fs.Remove(name) -} - -func (fs *secureFs) RemoveAll(path string) error { - return fs.fs.RemoveAll(path) -} - -func (fs *secureFs) Rename(oldname, newname string) error { - return fs.fs.Rename(oldname, newname) -} - -func (fs *secureFs) Stat(name string) (os.FileInfo, error) { - return fs.fs.Stat(name) -} - -func (fs *secureFs) Name() string { - return "secure/" + fs.fs.Name() -} - -func (fs *secureFs) Chmod(name string, mode os.FileMode) error { - return fs.fs.Chmod(name, mode) -} - -func (fs *secureFs) Chtimes(name string, atime time.Time, mtime time.Time) error { - return fs.fs.Chtimes(name, atime, mtime) -} diff --git a/internal/database/storage/securefs/secure_fs_test.go b/internal/database/storage/securefs/secure_fs_test.go deleted file mode 100644 index f1b314ff..00000000 --- a/internal/database/storage/securefs/secure_fs_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package securefs_test - -import ( - "io" - "io/ioutil" - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/tomarrell/lbadd/internal/database/storage/securefs" -) - -func mustRead(t *testing.T, r io.Reader) []byte { - data, err := ioutil.ReadAll(r) - if err != nil { - assert.NoError(t, err) - } - return data -} - -func TestSecureFs_FileOperations(t *testing.T) { - assert := assert.New(t) - - underlying := afero.NewMemMapFs() - fs := securefs.New(underlying) - - filename := "myfile.dat" - content := "hello, world!" - modContent := "hello, World!" - - file, err := fs.Create(filename) - assert.NoError(err) - assert.Equal(filename, file.Name()) - - n, err := file.WriteString(content) - assert.NoError(err) - assert.Equal(len(content), n) - - underlyingFile, err := underlying.Open(filename) - assert.NoError(err) - assert.Equal(content, string(mustRead(t, underlyingFile))) - assert.NoError(underlyingFile.Close()) - - n, err = file.WriteAt([]byte("W"), 7) - assert.Equal(1, n) - assert.NoError(err) - - underlyingFile, err = underlying.Open(filename) - assert.NoError(err) - assert.Equal(modContent, string(mustRead(t, underlyingFile))) - assert.NoError(underlyingFile.Close()) -} diff --git a/internal/database/storage/storage.go b/internal/database/storage/storage.go deleted file mode 100644 index 4ec35fe7..00000000 --- a/internal/database/storage/storage.go +++ /dev/null @@ -1,4 +0,0 @@ -package storage - -// Storage describes a storage component. -type Storage interface{} diff --git a/internal/database/table/doc.go b/internal/database/table/doc.go deleted file mode 100644 index 32b6aa67..00000000 --- a/internal/database/table/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package table implements a table that holds data in a storage and in memory. -package table diff --git a/internal/database/table/table.go b/internal/database/table/table.go deleted file mode 100644 index 36776ce8..00000000 --- a/internal/database/table/table.go +++ /dev/null @@ -1,15 +0,0 @@ -package table - -import ( - "github.com/tomarrell/lbadd/internal/database/column" - "github.com/tomarrell/lbadd/internal/database/storage" -) - -// Table describes a table that consists of a schema, a name, columns and a -// storage component, that is used to store the table's data. -type Table interface { - Schema() string - Name() string - Columns() []column.Column - Storage() storage.Storage -} From f023d210051f0b398549ce783959f97e03bb1680 Mon Sep 17 00:00:00 2001 From: Tim Satke <48135919+TimSatke@users.noreply.github.com> Date: Fri, 10 Jul 2020 12:51:22 +0200 Subject: [PATCH 71/77] Update internal/engine/expression.go --- internal/engine/expression.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/engine/expression.go b/internal/engine/expression.go index 0063783e..28a9c699 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -10,8 +10,7 @@ import ( // evaluateExpression evaluates the given expression to a runtime-constant // value, meaning that it can only be evaluated to a constant value with a given -// execution context. This execution context must be inferred from the engine -// receiver. +// execution context. func (e Engine) evaluateExpression(ctx ExecutionContext, expr command.Expr) (types.Value, error) { switch ex := expr.(type) { case command.ConstantBooleanExpr: From 1427b99d5150430b787f0e55c2c498b54a2cbf9d Mon Sep 17 00:00:00 2001 From: Tim Satke <48135919+TimSatke@users.noreply.github.com> Date: Fri, 10 Jul 2020 12:51:31 +0200 Subject: [PATCH 72/77] Update internal/engine/numeric_parser.go --- internal/engine/numeric_parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/engine/numeric_parser.go b/internal/engine/numeric_parser.go index 925c4ffd..99ca92dc 100644 --- a/internal/engine/numeric_parser.go +++ b/internal/engine/numeric_parser.go @@ -25,7 +25,7 @@ type numericParser struct { } // ToNumericValue checks whether the given string is of this form -// https://www.sqlite.org/lang_expr.html#literal_values_constants_ . If it is, a +// https://www.sqlite.org/lang_expr.html#literal_values_constants_ . If it is, an // appropriate value is returned (either types.IntegerValue or types.RealValue). // If it is not, false will be returned. This will never return the NULL value, // even if the given string is empty. In that case, nil and false is returned. From 6be3efbcf9ef4a72629cc79382c13dee8d8cadde Mon Sep 17 00:00:00 2001 From: Tim Satke <48135919+TimSatke@users.noreply.github.com> Date: Fri, 10 Jul 2020 12:51:38 +0200 Subject: [PATCH 73/77] Update internal/engine/expression.go --- internal/engine/expression.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/engine/expression.go b/internal/engine/expression.go index 28a9c699..c05fd5ef 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -20,7 +20,7 @@ func (e Engine) evaluateExpression(ctx ExecutionContext, expr command.Expr) (typ case command.FunctionExpr: return e.evaluateFunctionExpr(ctx, ex) } - return nil, fmt.Errorf("cannot evaluate expression of type %T", expr) + return nil, ErrUnimplemented(fmt.Sprintf("evaluate %T", expr)) } func (e Engine) evaluateMultipleExpressions(ctx ExecutionContext, exprs []command.Expr) ([]types.Value, error) { From 1cb364bee3f105dd4415e220682c9c29dccbc878 Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 10 Jul 2020 12:55:38 +0200 Subject: [PATCH 74/77] Fix test failure --- internal/engine/expression.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/engine/expression.go b/internal/engine/expression.go index c05fd5ef..209d964b 100644 --- a/internal/engine/expression.go +++ b/internal/engine/expression.go @@ -12,6 +12,10 @@ import ( // value, meaning that it can only be evaluated to a constant value with a given // execution context. func (e Engine) evaluateExpression(ctx ExecutionContext, expr command.Expr) (types.Value, error) { + if expr == nil { + return nil, fmt.Errorf("cannot evaluate expression of type %T", expr) + } + switch ex := expr.(type) { case command.ConstantBooleanExpr: return types.NewBool(ex.Value), nil From 7c2c4b213671ae0d5aaa37ed3569093c75be88ea Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 10 Jul 2020 15:59:35 +0200 Subject: [PATCH 75/77] Embed a pointer --- internal/engine/context.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/engine/context.go b/internal/engine/context.go index 553e349f..0f22699c 100644 --- a/internal/engine/context.go +++ b/internal/engine/context.go @@ -9,7 +9,7 @@ import ( // ExecutionContext is a context that is passed down throughout a complete // evaluation. It may be populated further. type ExecutionContext struct { - ctx *executionContext + *executionContext } type executionContext struct { @@ -20,7 +20,7 @@ type executionContext struct { func newEmptyExecutionContext() ExecutionContext { return ExecutionContext{ - ctx: &executionContext{ + executionContext: &executionContext{ id: id.Create(), scannedTables: make(map[string]Table), }, @@ -28,20 +28,20 @@ func newEmptyExecutionContext() ExecutionContext { } func (c ExecutionContext) putScannedTable(name string, table Table) { - c.ctx.scannedTablesLock.Lock() - defer c.ctx.scannedTablesLock.Unlock() + c.scannedTablesLock.Lock() + defer c.scannedTablesLock.Unlock() - c.ctx.scannedTables[name] = table + c.scannedTables[name] = table } func (c ExecutionContext) getScannedTable(name string) (Table, bool) { - c.ctx.scannedTablesLock.Lock() - defer c.ctx.scannedTablesLock.Unlock() + c.scannedTablesLock.Lock() + defer c.scannedTablesLock.Unlock() - tbl, ok := c.ctx.scannedTables[name] + tbl, ok := c.scannedTables[name] return tbl, ok } func (c ExecutionContext) String() string { - return c.ctx.id.String() + return c.id.String() } From 15a41cf676dba25c2fae28c0bceb921ada90ef8a Mon Sep 17 00:00:00 2001 From: Tim Satke Date: Fri, 10 Jul 2020 15:59:40 +0200 Subject: [PATCH 76/77] go mod tidy --- go.mod | 4 ---- go.sum | 26 -------------------------- 2 files changed, 30 deletions(-) diff --git a/go.mod b/go.mod index 6b37e0dd..d6768e87 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/tomarrell/lbadd go 1.13 require ( - github.com/awnumar/memguard v0.22.2 github.com/google/go-cmp v0.5.0 github.com/kr/text v0.2.0 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect @@ -13,12 +12,9 @@ require ( github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.6.1 - golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 // indirect golang.org/x/net v0.0.0-20200505041828-1ed23360d12c golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a - golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 // indirect golang.org/x/text v0.3.3 golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect - gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum index 9ff814a3..f50f4f3e 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/awnumar/memcall v0.0.0-20191004114545-73db50fd9f80 h1:8kObYoBO4LNmQ+fLiScBfxEdxF1w2MHlvH/lr9MLaTg= -github.com/awnumar/memcall v0.0.0-20191004114545-73db50fd9f80/go.mod h1:S911igBPR9CThzd/hYQQmTc9SWNu3ZHIlCGaWsWsoJo= -github.com/awnumar/memguard v0.22.2 h1:tMxcq1WamhG13gigK8Yaj9i/CHNUO3fFlpS9ABBQAxw= -github.com/awnumar/memguard v0.22.2/go.mod h1:33OwJBHC+T4eEfFcDrQb78TMlBMBvcOPCXWU9xE34gM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -40,8 +36,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -97,8 +91,6 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= -github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= github.com/rs/zerolog v1.19.0 h1:hYz4ZVdUgjXTBUmrkrw55j1nHx68LfOKIQk5IYtyScg= github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -108,8 +100,6 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.1 h1:GPTpEAuNr98px18yNQ66JllNil98wfRZ/5Ukny8FeQA= github.com/spf13/afero v1.3.1/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= @@ -130,8 +120,6 @@ github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -139,7 +127,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -148,10 +135,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA= -golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= -golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -185,15 +168,8 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 h1:5B6i6EAiSYyejWfvc5Rc9BbI3rzIsrrXfAQBWnYfn+w= -golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -223,8 +199,6 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 8a51bd41a8cf27bfc1ef53a4986c626cc6b12e01 Mon Sep 17 00:00:00 2001 From: Tim Satke <48135919+TimSatke@users.noreply.github.com> Date: Fri, 10 Jul 2020 21:52:00 +0200 Subject: [PATCH 77/77] Update internal/engine/storage/db_test.go Co-authored-by: Sumukha Pk --- internal/engine/storage/db_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/engine/storage/db_test.go b/internal/engine/storage/db_test.go index 398e8040..0e7748d8 100644 --- a/internal/engine/storage/db_test.go +++ b/internal/engine/storage/db_test.go @@ -33,7 +33,7 @@ func TestCreate(t *testing.T) { mustCell(assert, headerPage, HeaderPageCount).(page.RecordCell).Record, ) - // Allocating a new page must persist it in the created database file. This + // Allocating a new page must persist it is in the created database file. This // check ensures, that the file is writable. _, err = db.AllocateNewPage() assert.NoError(err)