diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..68570ae5 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,37 @@ + + +### What changes are being made and why? + + +--- + +### How the changes have been QAed? + + + +--- + +### Setup Instructions + + diff --git a/README.md b/README.md index 21dade25..8b248385 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,49 @@ -# Kestra JDBC Plugin +

+ + Kestra workflow orchestrator + +

+ +

+ Event-Driven Declarative Orchestrator +

+ +
+ Last Version + License + Github star
+Kestra infinitely scalable orchestration and scheduling platform +Slack +
+ +
+ +

+ twitter   + linkedin   +youtube   +

+

- Kestra workflow orchestrator + + Get started in 4 minutes with Kestra +

+

Get started with Kestra in 4 minutes.

+ +# Kestra JDBC Plugin > Plugin to interact with SQL Databases via JDBC -![Kestra orchestrator](https://kestra.io/ui.gif) + +![Kestra orchestrator](https://kestra.io/video.gif) ## Documentation -* The official Kestra documentation can be found under: [kestra.io](https://kestra.io) -* This plugin documentation is available here : +* Full documentation can be found under [kestra.io/docs](https://kestra.io/docs) +* Documentation for developing a plugin is included in the [Plugin Developer Guide](https://kestra.io/docs/plugin-developer-guide/) +* The documentation for specific sub-plugins: * [for ClickHouse](https://kestra.io/plugins/plugin-jdbc-clickhouse/) * [for DuckDb](https://kestra.io/plugins/plugin-jdbc-duckdb/) * [for MySQL](https://kestra.io/plugins/plugin-jdbc-mysql/) @@ -25,7 +57,15 @@ * [for Trino](https://kestra.io/plugins/plugin-jdbc-trino/) * [for Vectorwise](https://kestra.io/plugins/plugin-jdbc-vectorwise/) * [for Vertica](https://kestra.io/plugins/plugin-jdbc-vertica/) - + * ...and more! + + ## License Apache 2.0 © [Kestra Technologies](https://kestra.io) + +## Stay up to date + +We release new versions every month. Give the [main repository](https://github.com/kestra-io/kestra) a star to stay up to date with the latest releases and get notified about future updates. + +![Star the repo](https://kestra.io/star.gif) diff --git a/plugin-jdbc-duckdb/build.gradle b/plugin-jdbc-duckdb/build.gradle index 487c2ce9..47d610b9 100644 --- a/plugin-jdbc-duckdb/build.gradle +++ b/plugin-jdbc-duckdb/build.gradle @@ -13,7 +13,7 @@ jar { } dependencies { - implementation("org.duckdb:duckdb_jdbc:0.9.0") + implementation("org.duckdb:duckdb_jdbc:0.8.1") implementation project(':plugin-jdbc') testImplementation project(':plugin-jdbc').sourceSets.test.output diff --git a/plugin-jdbc-duckdb/src/test/java/io/kestra/plugin/jdbc/duckdb/DuckDbTest.java b/plugin-jdbc-duckdb/src/test/java/io/kestra/plugin/jdbc/duckdb/DuckDbTest.java index bf00c065..555511c5 100644 --- a/plugin-jdbc-duckdb/src/test/java/io/kestra/plugin/jdbc/duckdb/DuckDbTest.java +++ b/plugin-jdbc-duckdb/src/test/java/io/kestra/plugin/jdbc/duckdb/DuckDbTest.java @@ -177,6 +177,7 @@ void select() throws Exception { @Test void inputOutputFiles() throws Exception { URI source = storageInterface.put( + null, new URI("/" + IdUtils.create()), new FileInputStream(new File(Objects.requireNonNull(DuckDbTest.class.getClassLoader() .getResource("full.csv")) @@ -199,7 +200,7 @@ void inputOutputFiles() throws Exception { System.out.println(JacksonMapper.ofYaml().writeValueAsString(task)); assertThat( - IOUtils.toString(storageInterface.get(runOutput.getOutputFiles().get("out")), Charsets.UTF_8), + IOUtils.toString(storageInterface.get(null, runOutput.getOutputFiles().get("out")), Charsets.UTF_8), is( "id,name\n" + "4814976,Viva\n" + "1010871,Voomm\n" + diff --git a/plugin-jdbc-mysql/src/test/java/io/kestra/plugin/jdbc/mysql/LoadTest.java b/plugin-jdbc-mysql/src/test/java/io/kestra/plugin/jdbc/mysql/LoadTest.java index f75bc042..36a2ad19 100644 --- a/plugin-jdbc-mysql/src/test/java/io/kestra/plugin/jdbc/mysql/LoadTest.java +++ b/plugin-jdbc-mysql/src/test/java/io/kestra/plugin/jdbc/mysql/LoadTest.java @@ -34,6 +34,7 @@ void load() throws Exception { URL resource = LoadTest.class.getClassLoader().getResource("load.csv"); URI put = storageInterface.put( + null, new URI("/file/storage/get.yml"), new FileInputStream(Objects.requireNonNull(resource).getFile()) ); diff --git a/plugin-jdbc-oracle/src/test/java/io/kestra/plugin/jdbc/oracle/BatchTest.java b/plugin-jdbc-oracle/src/test/java/io/kestra/plugin/jdbc/oracle/BatchTest.java index 97b0f630..a6c9be17 100644 --- a/plugin-jdbc-oracle/src/test/java/io/kestra/plugin/jdbc/oracle/BatchTest.java +++ b/plugin-jdbc-oracle/src/test/java/io/kestra/plugin/jdbc/oracle/BatchTest.java @@ -60,7 +60,7 @@ void insert() throws Exception { )); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) @@ -91,7 +91,7 @@ public void namedInsert() throws Exception { } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) @@ -122,7 +122,7 @@ public void namedColumnsInsert() throws Exception { } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); ArrayList columns = new ArrayList<>(); columns.add("t_id"); diff --git a/plugin-jdbc-postgres/src/main/java/io/kestra/plugin/jdbc/postgresql/CopyOut.java b/plugin-jdbc-postgres/src/main/java/io/kestra/plugin/jdbc/postgresql/CopyOut.java index 88cbe2ce..51a52c03 100644 --- a/plugin-jdbc-postgres/src/main/java/io/kestra/plugin/jdbc/postgresql/CopyOut.java +++ b/plugin-jdbc-postgres/src/main/java/io/kestra/plugin/jdbc/postgresql/CopyOut.java @@ -25,12 +25,12 @@ @Getter @NoArgsConstructor @Schema( - title = "Copy from a file to PostgreSQL table" + title = "Copy tabular data from a Postgres table to a file" ) @Plugin( examples = { @Example( - title = "Export a csv or tsv from a postgres table or query", + title = "Export a Postgres table or query to a CSV or TSV file", code = { "url: jdbc:postgresql://127.0.0.1:56982/", "username: postgres", diff --git a/plugin-jdbc-postgres/src/test/java/io/kestra/plugin/jdbc/postgresql/BatchTest.java b/plugin-jdbc-postgres/src/test/java/io/kestra/plugin/jdbc/postgresql/BatchTest.java index ca984ff5..75a43399 100644 --- a/plugin-jdbc-postgres/src/test/java/io/kestra/plugin/jdbc/postgresql/BatchTest.java +++ b/plugin-jdbc-postgres/src/test/java/io/kestra/plugin/jdbc/postgresql/BatchTest.java @@ -66,7 +66,7 @@ void insert() throws Exception { )); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) @@ -154,7 +154,7 @@ public void namedInsert() throws Exception { ); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(TestUtils.url()) @@ -191,7 +191,7 @@ public void namedColumnsInsert() throws Exception { ); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(TestUtils.url()) diff --git a/plugin-jdbc-postgres/src/test/java/io/kestra/plugin/jdbc/postgresql/PgsqlTest.java b/plugin-jdbc-postgres/src/test/java/io/kestra/plugin/jdbc/postgresql/PgsqlTest.java index e3303bea..be6570ea 100644 --- a/plugin-jdbc-postgres/src/test/java/io/kestra/plugin/jdbc/postgresql/PgsqlTest.java +++ b/plugin-jdbc-postgres/src/test/java/io/kestra/plugin/jdbc/postgresql/PgsqlTest.java @@ -165,7 +165,7 @@ void selectAndFetchToFile() throws Exception { AbstractJdbcQuery.Output runOutput = task.run(runContext); assertThat(runOutput.getUri(), notNullValue()); - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(this.storageInterface.get(runOutput.getUri()))); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(this.storageInterface.get(null, runOutput.getUri()))); int lines = 0; while (bufferedReader.readLine() != null) { lines++; @@ -208,6 +208,7 @@ void selectWithCompositeType() throws Exception { @Test void updateFromFlow() throws Exception { Execution execution = runnerUtils.runOne( + null, "io.kestra.jdbc.postgres", "update_postgres", null, diff --git a/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/UploadDownloadTest.java b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/UploadDownloadTest.java index 40da1ef6..5c9f6a8f 100644 --- a/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/UploadDownloadTest.java +++ b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/UploadDownloadTest.java @@ -45,6 +45,7 @@ void success() throws Exception { URL resource = UploadDownloadTest.class.getClassLoader().getResource("scripts/snowflake.sql"); URI put = storageInterface.put( + null, new URI("/file/storage/snowflake.sql"), new FileInputStream(Objects.requireNonNull(resource).getFile()) ); @@ -82,8 +83,8 @@ void success() throws Exception { assertThat(downloadRun.getUri(), notNullValue()); assertThat( - IOUtils.toString(this.storageInterface.get(downloadRun.getUri()), Charsets.UTF_8), - is(IOUtils.toString(this.storageInterface.get(put), Charsets.UTF_8)) + IOUtils.toString(this.storageInterface.get(null, downloadRun.getUri()), Charsets.UTF_8), + is(IOUtils.toString(this.storageInterface.get(null, put), Charsets.UTF_8)) ); } } diff --git a/plugin-jdbc-sqlserver/src/test/java/io/kestra/plugin/jdbc/sqlserver/BatchTest.java b/plugin-jdbc-sqlserver/src/test/java/io/kestra/plugin/jdbc/sqlserver/BatchTest.java index e3b0cc2b..87383084 100644 --- a/plugin-jdbc-sqlserver/src/test/java/io/kestra/plugin/jdbc/sqlserver/BatchTest.java +++ b/plugin-jdbc-sqlserver/src/test/java/io/kestra/plugin/jdbc/sqlserver/BatchTest.java @@ -64,7 +64,7 @@ void insert() throws Exception { ); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) @@ -94,7 +94,7 @@ public void namedInsert() throws Exception { ); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) @@ -125,7 +125,7 @@ public void namedColumnsInsert() throws Exception { } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); ArrayList columns = new ArrayList<>(); columns.add("t_id"); diff --git a/plugin-jdbc-vectorwise/src/test/java/io/kestra/plugin/jdbc/vectorwise/BatchTest.java b/plugin-jdbc-vectorwise/src/test/java/io/kestra/plugin/jdbc/vectorwise/BatchTest.java index 04dacaad..68340f28 100644 --- a/plugin-jdbc-vectorwise/src/test/java/io/kestra/plugin/jdbc/vectorwise/BatchTest.java +++ b/plugin-jdbc-vectorwise/src/test/java/io/kestra/plugin/jdbc/vectorwise/BatchTest.java @@ -71,7 +71,7 @@ void insert() throws Exception { )); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) @@ -123,7 +123,7 @@ public void namedInsert() throws Exception { } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) @@ -174,7 +174,7 @@ public void namedColumnsInsert() throws Exception { } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) diff --git a/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/BatchTest.java b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/BatchTest.java index 1798282a..aeb0170c 100644 --- a/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/BatchTest.java +++ b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/BatchTest.java @@ -71,7 +71,7 @@ void insert() throws Exception { )); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) @@ -103,7 +103,7 @@ public void namedInsert() throws Exception { ); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl()) @@ -134,7 +134,7 @@ public void namedColumnsInsert() throws Exception { ); } - URI uri = storageInterface.put(URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + URI uri = storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); Batch task = Batch.builder() .url(getUrl())