diff --git a/go.mod b/go.mod index de4393d58d..b36bbe995a 100644 --- a/go.mod +++ b/go.mod @@ -79,7 +79,7 @@ require ( github.com/jackc/pgx/v4 v4.18.2 github.com/jhump/protoreflect v1.15.6 github.com/jmespath/go-jmespath v0.4.0 - github.com/klauspost/compress v1.17.7 + github.com/klauspost/compress v1.17.9 github.com/klauspost/pgzip v1.2.6 github.com/lib/pq v1.10.9 github.com/linkedin/goavro/v2 v2.12.0 @@ -97,7 +97,7 @@ require ( github.com/opensearch-project/opensearch-go/v3 v3.0.0 github.com/ory/dockertest/v3 v3.10.0 github.com/oschwald/geoip2-golang v1.9.0 - github.com/parquet-go/parquet-go v0.20.0 + github.com/parquet-go/parquet-go v0.23.0 github.com/pebbe/zmq4 v1.2.10 github.com/pierrec/lz4/v4 v4.1.21 github.com/pkg/sftp v1.13.6 @@ -147,7 +147,7 @@ require ( golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 google.golang.org/api v0.162.0 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.34.2 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.28.0 @@ -305,10 +305,10 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rickb777/plural v1.4.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect - github.com/segmentio/encoding v0.3.6 // indirect + github.com/segmentio/encoding v0.4.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/objx v0.5.2 // indirect @@ -327,7 +327,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.21.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.16.1 // indirect diff --git a/go.sum b/go.sum index f9d71b5f71..f9cdcf55f1 100644 --- a/go.sum +++ b/go.sum @@ -726,8 +726,8 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= @@ -869,8 +869,8 @@ github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrz github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y= github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0= github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg= -github.com/parquet-go/parquet-go v0.20.0 h1:a6tV5XudF893P1FMuyp01zSReXbBelquKQgRxBgJ29w= -github.com/parquet-go/parquet-go v0.20.0/go.mod h1:4YfUo8TkoGoqwzhA/joZKZ8f77wSMShOLHESY4Ys0bY= +github.com/parquet-go/parquet-go v0.23.0 h1:dyEU5oiHCtbASyItMCD2tXtT2nPmoPbKpqf0+nnGrmk= +github.com/parquet-go/parquet-go v0.23.0/go.mod h1:MnwbUcFHU6uBYMymKAlPPAw9yh3kE1wWl6Gl1uLdkNk= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= @@ -944,8 +944,8 @@ github.com/rickb777/date v1.20.5/go.mod h1:6BPrm3/aQI0I8jvlD1fAlm/86k5eSeTQ2mR5F github.com/rickb777/plural v1.4.1 h1:5MMLcbIaapLFmvDGRT5iPk8877hpTPt8Y9cdSKRw9sU= github.com/rickb777/plural v1.4.1/go.mod h1:kdmXUpmKBJTS0FtG/TFumd//VBWsNTD7zOw7x4umxNw= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -961,11 +961,10 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/segmentio/encoding v0.3.6 h1:E6lVLyDPseWEulBmCmAKPanDd3jiyGDo5gMcugCRwZQ= -github.com/segmentio/encoding v0.3.6/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= +github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOVmy8= +github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -1312,7 +1311,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1327,8 +1325,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1497,8 +1495,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/impl/sql/output_sql_deprecated.go b/internal/impl/sql/output_sql_deprecated.go index 9f8156843e..282a22cf05 100644 --- a/internal/impl/sql/output_sql_deprecated.go +++ b/internal/impl/sql/output_sql_deprecated.go @@ -77,5 +77,5 @@ func newSQLDeprecatedOutputFromConfig(conf *service.ParsedConfig, mgr *service.R if err != nil { return nil, err } - return newSQLRawOutput(mgr.Logger(), driverStr, dsnStr, queryStatic, nil, argsMapping, connSettings), nil + return newSQLRawOutput(mgr.Logger(), driverStr, dsnStr, queryStatic, nil, argsMapping, connSettings, []string{}, map[string]any{}), nil } diff --git a/internal/impl/sql/output_sql_insert.go b/internal/impl/sql/output_sql_insert.go index e6223bc628..ce5a328cd3 100644 --- a/internal/impl/sql/output_sql_insert.go +++ b/internal/impl/sql/output_sql_insert.go @@ -3,6 +3,7 @@ package sql import ( "context" "database/sql" + "errors" "fmt" "sync" @@ -27,7 +28,8 @@ func sqlInsertOutputConfig() *service.ConfigSpec { Example("foo")). Field(service.NewStringListField("columns"). Description("A list of columns to insert."). - Example([]string{"foo", "bar", "baz"})). + Example([]string{"foo", "bar", "baz"}). + Default([]string{})). Field(service.NewAnyListField("data_types").Description("The columns data types.").Optional().Example([]any{ map[string]any{ "name": "foo", @@ -47,8 +49,8 @@ func sqlInsertOutputConfig() *service.ConfigSpec { "format": "2006-01-02", }, }, - }), - ). + }). + Default([]any{})). Field(service.NewBloblangField("args_mapping"). Description("A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of columns specified."). Example("root = [ this.cat.meow, this.doc.woofs[0] ]"). @@ -297,13 +299,17 @@ func (s *sqlInsertOutput) WriteBatch(ctx context.Context, batch service.MessageB } if tx == nil { - if applyDataTypeFn, found := applyDataTypeMap[s.driver]; found { - for i, arg := range args { - newArg, err := applyDataTypeFn(arg, s.columns[i], s.dataTypes) - if err != nil { - return err + if applyDataTypeFn, found := applyDataTypeMap[s.driver]; found && len(s.dataTypes) > 0 { + if len(s.dataTypes) == len(args) { + for i, arg := range args { + newArg, err := applyDataTypeFn(arg, s.columns[i], s.dataTypes) + if err != nil { + return err + } + args[i] = newArg } - args[i] = newArg + } else { + return errors.New("number of data types must match number of columns") } } insertBuilder = insertBuilder.Values(args...) diff --git a/internal/impl/sql/output_sql_raw.go b/internal/impl/sql/output_sql_raw.go index c04f0e2e01..b5d469bc1c 100644 --- a/internal/impl/sql/output_sql_raw.go +++ b/internal/impl/sql/output_sql_raw.go @@ -3,6 +3,7 @@ package sql import ( "context" "database/sql" + "errors" "fmt" "sync" @@ -26,6 +27,31 @@ func sqlRawOutputConfig() *service.ConfigSpec { Description("Whether to enable [interpolation functions](/docs/configuration/interpolation/#bloblang-queries) in the query. Great care should be made to ensure your queries are defended against injection attacks."). Advanced(). Default(false)). + Field(service.NewStringListField("columns"). + Description("A list of columns to insert."). + Example([]string{"foo", "bar", "baz"}). + Default([]string{})). + Field(service.NewAnyListField("data_types").Description("The columns data types.").Optional().Example([]any{ + map[string]any{ + "name": "foo", + "type": "VARCHAR", + }, + map[string]any{ + "name": "bar", + "type": "DATETIME", + "datetime": map[string]any{ + "format": "2006-01-02 15:04:05.999", + }, + }, + map[string]any{ + "name": "baz", + "type": "DATE", + "date": map[string]any{ + "format": "2006-01-02", + }, + }, + }). + Default([]any{})). Field(service.NewBloblangField("args_mapping"). Description("An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`."). Example("root = [ this.cat.meow, this.doc.woofs[0] ]"). @@ -96,6 +122,9 @@ type sqlRawOutput struct { logger *service.Logger shutSig *shutdown.Signaller + + columns []string + dataTypes map[string]any } func newSQLRawOutputFromConfig(conf *service.ParsedConfig, mgr *service.Resources) (*sqlRawOutput, error) { @@ -123,6 +152,24 @@ func newSQLRawOutputFromConfig(conf *service.ParsedConfig, mgr *service.Resource } } + columns, err := conf.FieldStringList("columns") + if err != nil { + return nil, err + } + + dataTypesField, err := conf.FieldAnyList("data_types") + if err != nil { + return nil, err + } + dataTypes := map[string]any{} + for _, dataTypeField := range dataTypesField { + field, err := dataTypeField.FieldAny() + if err != nil { + return nil, err + } + dataTypes[field.(map[string]any)["name"].(string)] = field + } + var argsMapping *bloblang.Executor if conf.Contains("args_mapping") { if argsMapping, err = conf.FieldBloblang("args_mapping"); err != nil { @@ -134,7 +181,7 @@ func newSQLRawOutputFromConfig(conf *service.ParsedConfig, mgr *service.Resource if err != nil { return nil, err } - return newSQLRawOutput(mgr.Logger(), driverStr, dsnStr, queryStatic, queryDyn, argsMapping, connSettings), nil + return newSQLRawOutput(mgr.Logger(), driverStr, dsnStr, queryStatic, queryDyn, argsMapping, connSettings, columns, dataTypes), nil } func newSQLRawOutput( @@ -144,6 +191,8 @@ func newSQLRawOutput( queryDyn *service.InterpolatedString, argsMapping *bloblang.Executor, connSettings *connSettings, + columns []string, + dataTypes map[string]any, ) *sqlRawOutput { return &sqlRawOutput{ logger: logger, @@ -154,6 +203,8 @@ func newSQLRawOutput( queryDyn: queryDyn, argsMapping: argsMapping, connSettings: connSettings, + columns: columns, + dataTypes: dataTypes, } } @@ -205,6 +256,20 @@ func (s *sqlRawOutput) WriteBatch(ctx context.Context, batch service.MessageBatc if args, ok = iargs.([]any); !ok { return fmt.Errorf("mapping returned non-array result: %T", iargs) } + + if applyDataTypeFn, found := applyDataTypeMap[s.driver]; found && len(s.dataTypes) > 0 { + if len(s.dataTypes) == len(args) { + for i, arg := range args { + newArg, err := applyDataTypeFn(arg, s.columns[i], s.dataTypes) + if err != nil { + return err + } + args[i] = newArg + } + } else { + return errors.New("number of data types must match number of columns") + } + } } queryStr := s.queryStatic diff --git a/website/docs/components/outputs/sql_insert.md b/website/docs/components/outputs/sql_insert.md index 2407e32b34..9788c92b8b 100644 --- a/website/docs/components/outputs/sql_insert.md +++ b/website/docs/components/outputs/sql_insert.md @@ -35,8 +35,8 @@ output: driver: "" # No default (required) dsn: clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60 # No default (required) table: foo # No default (required) - columns: [] # No default (required) - data_types: [] # No default (optional) + columns: [] + data_types: [] args_mapping: root = [ this.cat.meow, this.doc.woofs[0] ] # No default (required) max_in_flight: 64 batching: @@ -57,8 +57,8 @@ output: driver: "" # No default (required) dsn: clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60 # No default (required) table: foo # No default (required) - columns: [] # No default (required) - data_types: [] # No default (optional) + columns: [] + data_types: [] args_mapping: root = [ this.cat.meow, this.doc.woofs[0] ] # No default (required) prefix: "" # No default (optional) suffix: ON CONFLICT (name) DO NOTHING # No default (optional) @@ -185,6 +185,7 @@ A list of columns to insert. Type: `array` +Default: `[]` ```yml # Examples @@ -201,6 +202,7 @@ The columns data types. Type: `array` +Default: `[]` ```yml # Examples diff --git a/website/docs/components/outputs/sql_raw.md b/website/docs/components/outputs/sql_raw.md index fcaab6fc00..5e6b5b3f16 100644 --- a/website/docs/components/outputs/sql_raw.md +++ b/website/docs/components/outputs/sql_raw.md @@ -35,6 +35,8 @@ output: driver: "" # No default (required) dsn: clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60 # No default (required) query: INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?); # No default (required) + columns: [] + data_types: [] args_mapping: root = [ this.cat.meow, this.doc.woofs[0] ] # No default (optional) max_in_flight: 64 batching: @@ -56,6 +58,8 @@ output: dsn: clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60 # No default (required) query: INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?); # No default (required) unsafe_dynamic_query: false + columns: [] + data_types: [] args_mapping: root = [ this.cat.meow, this.doc.woofs[0] ] # No default (optional) max_in_flight: 64 init_files: [] # No default (optional) @@ -193,6 +197,47 @@ Whether to enable [interpolation functions](/docs/configuration/interpolation/#b Type: `bool` Default: `false` +### `columns` + +A list of columns to insert. + + +Type: `array` +Default: `[]` + +```yml +# Examples + +columns: + - foo + - bar + - baz +``` + +### `data_types` + +The columns data types. + + +Type: `array` +Default: `[]` + +```yml +# Examples + +data_types: + - name: foo + type: VARCHAR + - datetime: + format: "2006-01-02 15:04:05.999" + name: bar + type: DATETIME + - date: + format: "2006-01-02" + name: baz + type: DATE +``` + ### `args_mapping` An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.