Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implemented CLI task for ClickHouse #370

Merged
merged 11 commits into from
Sep 2, 2024
Merged
2 changes: 2 additions & 0 deletions plugin-jdbc-clickhouse/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,7 @@ dependencies {
implementation("com.clickhouse:clickhouse-jdbc:0.6.0:all")
implementation project(':plugin-jdbc')

compileOnly group: "io.kestra", name: "script", version: kestraVersion

testImplementation project(':plugin-jdbc').sourceSets.test.output
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package io.kestra.plugin.jdbc.clickhouse;

import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.tasks.*;
import io.kestra.core.models.tasks.runners.ScriptService;
import io.kestra.core.models.tasks.runners.TaskRunner;
import io.kestra.core.runners.RunContext;
import io.kestra.plugin.scripts.exec.scripts.models.ScriptOutput;
import io.kestra.plugin.scripts.exec.scripts.runners.CommandsWrapper;
import io.kestra.plugin.scripts.runner.docker.Docker;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import lombok.*;
import lombok.experimental.SuperBuilder;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;

import static io.kestra.core.utils.Rethrow.throwFunction;

@SuperBuilder
@ToString
@EqualsAndHashCode
@Getter
@NoArgsConstructor
@Schema(
title = "Run clickhouse-local commands."
)
@Plugin(
examples = {
@Example(
title = "Run clickhouse-local commands",
full = true,
code = {
"""
id: clickhouse-local
namespace: company.team
tasks:
- id: query
type: io.kestra.plugin.clickhouse.ClickHouseLocalCLI
commands:
- SELECT count() FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/house_parquet/house_0.parquet')
"""
}
)
}
)
public class ClickHouseLocalCLI extends Task implements RunnableTask<ScriptOutput>, NamespaceFilesInterface, InputFilesInterface, OutputFilesInterface {

public static final String DEFAULT_IMAGE = "clickhouse/clickhouse-server:latest";

@Schema(
title = "The commands to run before main list of commands."
)
@PluginProperty(dynamic = true)
protected List<String> beforeCommands;

@Schema(
title = "The commands to run."
)
@PluginProperty(dynamic = true)
@NotEmpty
protected List<String> commands;

@Schema(
title = "Additional environment variables for the current process."
)
@PluginProperty(dynamic = true)
protected Map<String, String> env;

@Schema(
title = "The task runner to use."
)
@Valid
@PluginProperty
@Builder.Default
private TaskRunner taskRunner = Docker.instance();

@Schema(
title = "The Clickhouse container image."
)
@PluginProperty(dynamic =true)
@Builder.Default
private String containerImage = DEFAULT_IMAGE;

private NamespaceFiles namespaceFiles;

private Object inputFiles;

private List<String> outputFiles;

@Override
public ScriptOutput run(RunContext runContext) throws Exception {
return new CommandsWrapper(runContext)
.withWarningOnStdErr(true)
.withTaskRunner(this.taskRunner)
.withContainerImage(this.containerImage)
.withEnv(Optional.ofNullable(env).orElse(new HashMap<>()))
.withNamespaceFiles(namespaceFiles)
.withInputFiles(inputFiles)
.withOutputFiles(outputFiles)
.withCommands(
ScriptService.scriptCommands(
List.of("clickhouse-local"),
Optional.ofNullable(this.beforeCommands).map(throwFunction(runContext::render)).orElse(null),
runContext.render(this.commands)
)
)
.run();
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package io.kestra.plugin.jdbc.clickhouse;

import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.runners.RunContext;
import io.kestra.core.runners.RunContextFactory;
import io.kestra.core.utils.IdUtils;
import io.kestra.core.utils.TestsUtils;
import io.kestra.plugin.scripts.exec.scripts.models.ScriptOutput;
import jakarta.inject.Inject;
import org.junit.jupiter.api.Test;

import java.util.List;
import java.util.Map;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;

@KestraTest
public class ClickHouseLocalCLITest {

@Inject
private RunContextFactory runContextFactory;

@Test
void run() throws Exception {
ClickHouseLocalCLI clickhouseLocalCLI = ClickHouseLocalCLI.builder()
.id(IdUtils.create())
.type(ClickHouseLocalCLI.class.getName())
.commands(List.of("SELECT * FROM system.tables"))
.build();

RunContext runContext = TestsUtils.mockRunContext(runContextFactory, clickhouseLocalCLI, Map.of());

ScriptOutput output = clickhouseLocalCLI.run(runContext);

assertThat(output.getExitCode(), is(0));
}

}
Loading