Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

create appender ext #164

Merged
merged 1 commit into from
Feb 22, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions api/pkgs/@duckdb/node-api/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -644,7 +644,7 @@ await connection.run(
`create or replace table target_table(i integer, v varchar)`
);

const appender = await connection.createAppender('main', 'target_table');
const appender = await connection.createAppender('target_table');

appender.appendInteger(42);
appender.appendVarchar('duck');
Expand All @@ -670,7 +670,7 @@ await connection.run(
`create or replace table target_table(i integer, v varchar)`
);

const appender = await connection.createAppender('main', 'target_table');
const appender = await connection.createAppender('target_table');

const chunk = DuckDBDataChunk.create([INTEGER, VARCHAR]);
chunk.setColumns([
Expand Down
12 changes: 9 additions & 3 deletions api/src/DuckDBConnection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,17 @@ export class DuckDBConnection {
);
}
public async createAppender(
schema: string,
table: string
table: string,
schema?: string | null,
catalog?: string | null
): Promise<DuckDBAppender> {
return new DuckDBAppender(
duckdb.appender_create(this.connection, schema, table)
duckdb.appender_create_ext(
this.connection,
catalog ?? null,
schema ?? null,
table
)
);
}
}
24 changes: 12 additions & 12 deletions api/test/api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1374,7 +1374,7 @@ describe('api', () => {
chunk.setColumnValues(0, values);

await connection.run('create table target(col0 int)');
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand Down Expand Up @@ -1408,7 +1408,7 @@ describe('api', () => {
}

await connection.run('create table target1(col0 int)');
const appender1 = await connection.createAppender('main', 'target1');
const appender1 = await connection.createAppender('target1');
appender1.appendDataChunk(chunk);
appender1.flush();

Expand All @@ -1425,7 +1425,7 @@ describe('api', () => {
vector.flush();

await connection.run('create table target2(col0 int)');
const appender2 = await connection.createAppender('main', 'target2');
const appender2 = await connection.createAppender('target2');
appender2.appendDataChunk(chunk);
appender2.flush();

Expand All @@ -1447,7 +1447,7 @@ describe('api', () => {
chunk.setColumnValues(0, values);

await connection.run('create table target(col0 varchar)');
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand Down Expand Up @@ -1483,7 +1483,7 @@ describe('api', () => {
chunk.setColumnValues(0, values);

await connection.run('create table target(col0 blob)');
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand All @@ -1510,7 +1510,7 @@ describe('api', () => {
chunk.setColumnValues(0, values);

await connection.run('create table target(col0 integer[])');
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand Down Expand Up @@ -1557,7 +1557,7 @@ describe('api', () => {
]);

await connection.run('create table target(col0 integer[][])');
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand All @@ -1584,7 +1584,7 @@ describe('api', () => {
chunk.setColumnValues(0, values);

await connection.run('create table target(col0 integer[3])');
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand All @@ -1611,7 +1611,7 @@ describe('api', () => {
chunk.setColumnValues(0, values);

await connection.run('create table target(col0 varchar[3])');
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand Down Expand Up @@ -1643,7 +1643,7 @@ describe('api', () => {
await connection.run(
'create table target(col0 struct(num integer, str varchar))'
);
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand Down Expand Up @@ -1683,7 +1683,7 @@ describe('api', () => {
int integer\
)'
);
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand Down Expand Up @@ -1720,7 +1720,7 @@ describe('api', () => {
.map(({ name, type }) => `"${name.replace(`"`, `""`)}" ${type}`)
.join(', ')})`
);
const appender = await connection.createAppender('main', 'target');
const appender = await connection.createAppender('target');
appender.appendDataChunk(chunk);
appender.flush();

Expand Down
16 changes: 8 additions & 8 deletions api/test/bench/read.bench.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ for (const full of [false, true]) {
},
{
setup,
iterations: 200,
iterations: 20,
}
);
bench(
Expand All @@ -126,7 +126,7 @@ for (const full of [false, true]) {
},
{
setup,
iterations: 200,
iterations: 20,
}
);
bench(
Expand Down Expand Up @@ -160,7 +160,7 @@ for (const full of [false, true]) {
},
{
setup,
iterations: 200,
iterations: 20,
}
);

Expand All @@ -179,7 +179,7 @@ for (const full of [false, true]) {
},
{
setup,
iterations: 200,
iterations: 20,
}
);
bench(
Expand All @@ -205,7 +205,7 @@ for (const full of [false, true]) {
},
{
setup,
iterations: 200,
iterations: 20,
}
);
bench(
Expand Down Expand Up @@ -239,7 +239,7 @@ for (const full of [false, true]) {
},
{
setup,
iterations: 200,
iterations: 20,
}
);
bench(
Expand All @@ -265,7 +265,7 @@ for (const full of [false, true]) {
},
{
setup,
iterations: 200,
iterations: 20,
}
);
bench(
Expand All @@ -291,7 +291,7 @@ for (const full of [false, true]) {
},
{
setup,
iterations: 200,
iterations: 20,
}
);
});
Expand Down
8 changes: 4 additions & 4 deletions api/test/bench/write.bench.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { bench, describe } from "vitest";
import { bench, describe } from 'vitest';
import {
DuckDBConnection,
DuckDBInstance,
DuckDBTimestampValue,
} from "../../src";
} from '../../src';

let instance: DuckDBInstance;
let connection: DuckDBConnection;
Expand All @@ -26,7 +26,7 @@ for (const batchSize of [1, 1000]) {
`${batchSize} insert bind`,
async () => {
const query = await connection.prepare(
"INSERT INTO test (timestamp, value) VALUES ($1, $2);"
'INSERT INTO test (timestamp, value) VALUES ($1, $2);'
);

for (let index = 0; index < batchSize; index++) {
Expand All @@ -46,7 +46,7 @@ for (const batchSize of [1, 1000]) {
bench(
`${batchSize} row append`,
async () => {
const appender = await connection.createAppender("main", "test");
const appender = await connection.createAppender('test');

for (let index = 0; index < batchSize; index++) {
appender.appendTimestamp(
Expand Down
3 changes: 2 additions & 1 deletion bindings/pkgs/@duckdb/node-bindings/duckdb.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1054,9 +1054,10 @@ export function validity_set_row_valid(validity: Uint8Array, row_index: number):
// DUCKDB_API duckdb_profiling_info duckdb_profiling_info_get_child(duckdb_profiling_info info, idx_t index);

// DUCKDB_API duckdb_state duckdb_appender_create(duckdb_connection connection, const char *schema, const char *table, duckdb_appender *out_appender);
export function appender_create(connection: Connection, schema: string, table: string): Appender;
export function appender_create(connection: Connection, schema: string | null, table: string): Appender;

// DUCKDB_API duckdb_state duckdb_appender_create_ext(duckdb_connection connection, const char *catalog, const char *schema, const char *table, duckdb_appender *out_appender);
export function appender_create_ext(connection: Connection, catalog: string | null, schema: string | null, table: string): Appender;

// DUCKDB_API idx_t duckdb_appender_column_count(duckdb_appender appender);
export function appender_column_count(appender: Appender): number;
Expand Down
44 changes: 39 additions & 5 deletions bindings/src/duckdb_node_bindings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1273,6 +1273,7 @@ class DuckDBNodeAddon : public Napi::Addon<DuckDBNodeAddon> {
InstanceMethod("validity_set_row_valid", &DuckDBNodeAddon::validity_set_row_valid),

InstanceMethod("appender_create", &DuckDBNodeAddon::appender_create),
InstanceMethod("appender_create_ext", &DuckDBNodeAddon::appender_create_ext),
InstanceMethod("appender_column_count", &DuckDBNodeAddon::appender_column_count),
InstanceMethod("appender_column_type", &DuckDBNodeAddon::appender_column_type),
InstanceMethod("appender_flush", &DuckDBNodeAddon::appender_flush),
Expand Down Expand Up @@ -3707,17 +3708,24 @@ class DuckDBNodeAddon : public Napi::Addon<DuckDBNodeAddon> {
// DUCKDB_API duckdb_profiling_info duckdb_profiling_info_get_child(duckdb_profiling_info info, idx_t index);

// DUCKDB_API duckdb_state duckdb_appender_create(duckdb_connection connection, const char *schema, const char *table, duckdb_appender *out_appender);
// function appender_create(connection: Connection, schema: string, table: string): Appender
// function appender_create(connection: Connection, schema: string | null, table: string): Appender
Napi::Value appender_create(const Napi::CallbackInfo& info) {
auto env = info.Env();
auto connection = GetConnectionFromExternal(env, info[0]);
if (!connection) {
throw Napi::Error::New(env, "Failed to create appender: connection disconnected");
}
std::string schema = info[1].As<Napi::String>();
std::string schema = info[1].IsNull() ? std::string() : info[1].As<Napi::String>();
std::string table = info[2].As<Napi::String>();
duckdb_appender appender;
if (duckdb_appender_create(connection, schema.c_str(), table.c_str(), &appender)) {
if (
duckdb_appender_create(
connection,
info[1].IsNull() ? nullptr : schema.c_str(),
table.c_str(),
&appender
)
) {
std::string error = duckdb_appender_error(appender);
duckdb_appender_destroy(&appender);
throw Napi::Error::New(env, error);
Expand All @@ -3726,6 +3734,32 @@ class DuckDBNodeAddon : public Napi::Addon<DuckDBNodeAddon> {
}

// DUCKDB_API duckdb_state duckdb_appender_create_ext(duckdb_connection connection, const char *catalog, const char *schema, const char *table, duckdb_appender *out_appender);
// function appender_create_ext(connection: Connection, catalog: string | null, schema: string | null, table: string): Appender
Napi::Value appender_create_ext(const Napi::CallbackInfo& info) {
auto env = info.Env();
auto connection = GetConnectionFromExternal(env, info[0]);
if (!connection) {
throw Napi::Error::New(env, "Failed to create appender: connection disconnected");
}
std::string catalog = info[1].IsNull() ? std::string() : info[1].As<Napi::String>();
std::string schema = info[2].IsNull() ? std::string() : info[2].As<Napi::String>();
std::string table = info[3].As<Napi::String>();
duckdb_appender appender;
if (
duckdb_appender_create_ext(
connection,
info[1].IsNull() ? nullptr : catalog.c_str(),
info[2].IsNull() ? nullptr : schema.c_str(),
table.c_str(),
&appender
)
) {
std::string error = duckdb_appender_error(appender);
duckdb_appender_destroy(&appender);
throw Napi::Error::New(env, error);
}
return CreateExternalForAppender(env, appender);
}

// DUCKDB_API idx_t duckdb_appender_column_count(duckdb_appender appender);
// function appender_column_count(appender: Appender): number
Expand Down Expand Up @@ -4183,7 +4217,7 @@ NODE_API_ADDON(DuckDBNodeAddon)
---
411 total functions

237 instance methods
238 instance methods
3 unimplemented instance cache functions
1 unimplemented logical type function
13 unimplemented scalar function functions
Expand All @@ -4196,7 +4230,7 @@ NODE_API_ADDON(DuckDBNodeAddon)
5 unimplemented function info functions
4 unimplemented replacement scan functions
5 unimplemented profiling info functions
5 unimplemented appender functions
4 unimplemented appender functions
6 unimplemented table description functions
8 unimplemented tasks functions
12 unimplemented cast function functions
Expand Down
8 changes: 4 additions & 4 deletions bindings/test/appender.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { data } from './utils/expectedVectors';
suite('appender', () => {
test('error: no table', async () => {
await withConnection(async (connection) => {
expect(() => duckdb.appender_create(connection, 'main', 'bogus_table'))
expect(() => duckdb.appender_create_ext(connection, 'memory', 'main', 'bogus_table'))
.toThrowError(`Table "main.bogus_table" could not be found`);
});
});
Expand All @@ -27,7 +27,7 @@ suite('appender', () => {
chunks: [],
});

const appender = duckdb.appender_create(connection, 'main', 'appender_target');
const appender = duckdb.appender_create_ext(connection, 'memory', 'main', 'appender_target');
expect(duckdb.appender_column_count(appender)).toBe(1);
const column_type = duckdb.appender_column_type(appender, 0);
expectLogicalType(column_type, INTEGER);
Expand Down Expand Up @@ -92,7 +92,7 @@ suite('appender', () => {
});


const appender = duckdb.appender_create(connection, 'main', 'appender_target');
const appender = duckdb.appender_create_ext(connection, 'memory', 'main', 'appender_target');
expect(duckdb.appender_column_count(appender)).toBe(21);

const expectedLogicalTypes = [
Expand Down Expand Up @@ -222,7 +222,7 @@ suite('appender', () => {
chunks: [],
});

const appender = duckdb.appender_create(connection, 'main', 'appender_target');
const appender = duckdb.appender_create_ext(connection, 'memory', 'main', 'appender_target');
expect(duckdb.appender_column_count(appender)).toBe(2);

const source_result = await duckdb.query(connection, 'select int, varchar from test_all_types()');
Expand Down