Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: clickhouse.JSON Serializer interface #1491

Merged
merged 7 commits into from
Feb 7, 2025
Merged

feat: clickhouse.JSON Serializer interface #1491

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
43cc90d
use pointer receiver for json.Marshaler for consistency, add test
SpencerTorres Feb 6, 2025
9a787ff
Add JSON (De)Serializer interfaces for faster encoding/decoding to/fr…
SpencerTorres Feb 6, 2025
dfd932e
extend comments from chcol module
SpencerTorres Feb 6, 2025
8c31f9d
fix json serializer detection
SpencerTorres Feb 6, 2025
a4bca42
add note in comment
SpencerTorres Feb 6, 2025
cd14a4e
Add example/test/benchmark for JSON serializer, reorganize JSON helpers
SpencerTorres Feb 6, 2025
56c284a
update JSON serializer func names
SpencerTorres Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 44 additions & 127 deletions benchmark/json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,10 @@ import (
"crypto/tls"
"encoding/json"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/chcol"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
clickhouse_tests "github.com/ClickHouse/clickhouse-go/v2/tests"
"os"
"testing"
"time"
)

const testSet string = "json_bench"
Expand Down Expand Up @@ -79,7 +77,7 @@ func prepareJSONReadTest(ctx context.Context, b *testing.B) (driver.Conn, driver
b.Fatal(err)
}

jsonRow := buildTestJSONPaths()
jsonRow := clickhouse_tests.BuildFastTestJSONStruct()
for i := 0; i < b.N; i++ {
if err := batch.Append(jsonRow); err != nil {
b.Fatal(err)
Expand All @@ -98,94 +96,14 @@ func prepareJSONReadTest(ctx context.Context, b *testing.B) (driver.Conn, driver
return conn, rows
}

var jsonTestDate, _ = time.Parse(time.RFC3339, "2024-12-13T02:09:30.123Z")

type Address struct {
Street string `chType:"String"`
City string `chType:"String"`
Country string `chType:"String"`
}

type TestStruct struct {
Name string
Age int64
Active bool
Score float64

Tags []string
Numbers []int64

Address Address

KeysNumbers map[string]int64
Metadata map[string]interface{}

Timestamp time.Time `chType:"DateTime64(3)"`

DynamicString chcol.Dynamic
DynamicInt chcol.Dynamic
DynamicMap chcol.Dynamic
}

func buildTestJSONPaths() *chcol.JSON {
jsonRow := chcol.NewJSON()
jsonRow.SetValueAtPath("Name", "JSON")
jsonRow.SetValueAtPath("Age", int64(42))
jsonRow.SetValueAtPath("Active", true)
jsonRow.SetValueAtPath("Score", 3.14)
jsonRow.SetValueAtPath("Tags", []string{"a", "b"})
jsonRow.SetValueAtPath("Numbers", []int64{20, 40})
jsonRow.SetValueAtPath("Address.Street", "Street")
jsonRow.SetValueAtPath("Address.City", "City")
jsonRow.SetValueAtPath("Address.Country", "Country")
jsonRow.SetValueAtPath("KeysNumbers", map[string]int64{"FieldA": 42, "FieldB": 32})
jsonRow.SetValueAtPath("Metadata.FieldA", "a")
jsonRow.SetValueAtPath("Metadata.FieldB", "b")
jsonRow.SetValueAtPath("Metadata.FieldC.FieldD", "d")
jsonRow.SetValueAtPath("Timestamp", jsonTestDate)
jsonRow.SetValueAtPath("DynamicString", clickhouse.NewDynamic("str"))
jsonRow.SetValueAtPath("DynamicInt", clickhouse.NewDynamic(int64(48)))
jsonRow.SetValueAtPath("DynamicMap", clickhouse.NewDynamic(map[string]string{"a": "a", "b": "b"}))

return jsonRow
}

func buildTestJSONStruct() TestStruct {
return TestStruct{
Name: "JSON",
Age: 42,
Active: true,
Score: 3.14,
Tags: []string{"a", "b"},
Numbers: []int64{20, 40},
Address: Address{
Street: "Street",
City: "City",
Country: "Country",
},
KeysNumbers: map[string]int64{"FieldA": 42, "FieldB": 32},
Metadata: map[string]interface{}{
"FieldA": "a",
"FieldB": "b",
"FieldC": map[string]interface{}{
"FieldD": "d",
},
},
Timestamp: jsonTestDate,
DynamicString: chcol.NewDynamic("str").WithType("String"),
DynamicInt: chcol.NewDynamic(int64(48)).WithType("Int64"),
DynamicMap: chcol.NewDynamic(map[string]string{"a": "a", "b": "b"}).WithType("Map(String, String)"),
}
}

// BenchmarkJSONInsert tests the performance for appending to a JSON column batch
func BenchmarkJSONInsert(b *testing.B) {
b.Run("paths", func(b *testing.B) {
ctx := context.Background()
conn, batch := prepareJSONInsertTest(ctx, b)
defer conn.Close()

jsonRow := buildTestJSONPaths()
jsonRow := clickhouse_tests.BuildTestJSONPaths()

b.ReportAllocs()
b.ResetTimer()
Expand All @@ -202,31 +120,7 @@ func BenchmarkJSONInsert(b *testing.B) {
conn, batch := prepareJSONInsertTest(ctx, b)
defer conn.Close()

inputRow := TestStruct{
Name: "JSON",
Age: 42,
Active: true,
Score: 3.14,
Tags: []string{"a", "b"},
Numbers: []int64{20, 40},
Address: Address{
Street: "Street",
City: "City",
Country: "Country",
},
KeysNumbers: map[string]int64{"FieldA": 42, "FieldB": 32},
Metadata: map[string]interface{}{
"FieldA": "a",
"FieldB": "b",
"FieldC": map[string]interface{}{
"FieldD": "d",
},
},
Timestamp: jsonTestDate,
DynamicString: chcol.NewDynamic("str").WithType("String"),
DynamicInt: chcol.NewDynamic(int64(48)).WithType("Int64"),
DynamicMap: chcol.NewDynamic(map[string]string{"a": "a", "b": "b"}).WithType("Map(String, String)"),
}
inputRow := clickhouse_tests.BuildTestJSONStruct()

b.ReportAllocs()
b.ResetTimer()
Expand All @@ -238,12 +132,29 @@ func BenchmarkJSONInsert(b *testing.B) {
b.StopTimer()
})

b.Run("fast_structs", func(b *testing.B) {
ctx := context.Background()
conn, batch := prepareJSONInsertTest(ctx, b)
defer conn.Close()

inputRow := clickhouse_tests.BuildFastTestJSONStruct()

b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := batch.Append(&inputRow); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
})

b.Run("marshal_strings", func(b *testing.B) {
ctx := context.Background()
conn, batch := prepareJSONInsertTest(ctx, b)
defer conn.Close()

inputRow := buildTestJSONStruct()
inputRow := clickhouse_tests.BuildTestJSONStruct()

b.ReportAllocs()
b.ResetTimer()
Expand All @@ -265,7 +176,7 @@ func BenchmarkJSONInsert(b *testing.B) {
conn, batch := prepareJSONInsertTest(ctx, b)
defer conn.Close()

inputRow := buildTestJSONStruct()
inputRow := clickhouse_tests.BuildTestJSONStruct()

inputRowStr, err := json.Marshal(inputRow)
if err != nil {
Expand Down Expand Up @@ -314,7 +225,26 @@ func BenchmarkJSONRead(b *testing.B) {
for i := 0; i < b.N; i++ {
rows.Next()

var row TestStruct
var row clickhouse_tests.TestStruct
err := rows.Scan(&row)
if err != nil {
b.Fatal(err)
}
}
b.StopTimer()
})

b.Run("fast_structs", func(b *testing.B) {
ctx := context.Background()
conn, rows := prepareJSONReadTest(ctx, b)
defer conn.Close()

b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
rows.Next()

var row clickhouse_tests.FastTestStruct
err := rows.Scan(&row)
if err != nil {
b.Fatal(err)
Expand All @@ -334,21 +264,8 @@ func BenchmarkJSONRead(b *testing.B) {

// BenchmarkJSONMarshal compares the different ways to turn JSON data back into a string
func BenchmarkJSONMarshal(b *testing.B) {
b.Run("paths_direct", func(b *testing.B) {
pathsRow := buildTestJSONPaths()

b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := pathsRow.MarshalJSON()
if err != nil {
b.Fatal(err)
}
}
})

b.Run("paths", func(b *testing.B) {
pathsRow := buildTestJSONPaths()
pathsRow := clickhouse_tests.BuildTestJSONPaths()

b.ReportAllocs()
b.ResetTimer()
Expand All @@ -361,7 +278,7 @@ func BenchmarkJSONMarshal(b *testing.B) {
})

b.Run("structs", func(b *testing.B) {
structRow := buildTestJSONStruct()
structRow := clickhouse_tests.BuildTestJSONStruct()

b.ReportAllocs()
b.ResetTimer()
Expand Down
19 changes: 18 additions & 1 deletion chcol.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,20 @@ import "github.com/ClickHouse/clickhouse-go/v2/lib/chcol"
// Re-export chcol types/funcs to top level clickhouse package

type (
// Variant represents a ClickHouse Variant type that can hold multiple possible types
Variant = chcol.Variant
// Dynamic is an alias for the Variant type
Dynamic = chcol.Dynamic
JSON = chcol.JSON
// JSON represents a ClickHouse JSON type that can hold multiple possible types
JSON = chcol.JSON

// JSONSerializer interface allows a struct to be manually converted to an optimized JSON structure instead of relying
// on recursive reflection.
// Note that the struct must be a pointer in order for the interface to be matched, reflection will be used otherwise.
JSONSerializer = chcol.JSONSerializer
// JSONDeserializer interface allows a struct to load its data from an optimized JSON structure instead of relying
// on recursive reflection to set its fields.
JSONDeserializer = chcol.JSONDeserializer
)

// NewVariant creates a new Variant with the given value
Expand All @@ -51,3 +62,9 @@ func NewDynamicWithType(v any, chType string) Dynamic {
func NewJSON() *JSON {
return chcol.NewJSON()
}

// ExtractJSONPathAs is a convenience function for asserting a path to a specific type.
// The underlying value is also extracted from its Dynamic wrapper if present.
func ExtractJSONPathAs[T any](o *JSON, path string) (valueAs T, ok bool) {
return chcol.ExtractJSONPathAs[T](o, path)
}
Loading
Loading