From e3ad9f2ad1f5d1b99e7c9fbaf322ba229dbfd494 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 May 2022 08:53:01 -0600 Subject: [PATCH 01/17] Add new ballista-cli crate --- Cargo.toml | 2 +- ballista-cli/Cargo.toml | 40 ++ ballista-cli/Dockerfile | 38 ++ ballista-cli/README.md | 94 ++++ ballista-cli/src/command.rs | 222 ++++++++++ ballista-cli/src/context.rs | 90 ++++ ballista-cli/src/exec.rs | 170 +++++++ ballista-cli/src/functions.rs | 198 +++++++++ ballista-cli/src/helper.rs | 102 +++++ ballista-cli/src/lib.rs | 27 ++ ballista-cli/src/main.rs | 165 +++++++ ballista-cli/src/print_format.rs | 155 +++++++ ballista-cli/src/print_options.rs | 54 +++ datafusion-cli/Cargo.lock | 707 +----------------------------- datafusion-cli/Cargo.toml | 1 - datafusion-cli/src/context.rs | 5 - datafusion-cli/src/main.rs | 11 +- 17 files changed, 1369 insertions(+), 712 deletions(-) create mode 100644 ballista-cli/Cargo.toml create mode 100644 ballista-cli/Dockerfile create mode 100644 ballista-cli/README.md create mode 100644 ballista-cli/src/command.rs create mode 100644 ballista-cli/src/context.rs create mode 100644 ballista-cli/src/exec.rs create mode 100644 ballista-cli/src/functions.rs create mode 100644 ballista-cli/src/helper.rs create mode 100644 ballista-cli/src/lib.rs create mode 100644 ballista-cli/src/main.rs create mode 100644 ballista-cli/src/print_format.rs create mode 100644 ballista-cli/src/print_options.rs diff --git a/Cargo.toml b/Cargo.toml index 297b394b4521..e319c51713b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ members = [ "ballista-examples", "data-access", ] -exclude = ["datafusion-cli"] +exclude = ["ballista-cli", "datafusion-cli"] [profile.release] codegen-units = 1 diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml new file mode 100644 index 000000000000..a06b036ecc0e --- /dev/null +++ b/ballista-cli/Cargo.toml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "ballista-cli" +description = "Command Line Client for Ballista distributed query engine." +version = "0.6.0" +authors = ["Apache Arrow "] +edition = "2021" +keywords = [ "ballista", "cli", ] +license = "Apache-2.0" +homepage = "https://github.com/apache/arrow-datafusion" +repository = "https://github.com/apache/arrow-datafusion" +rust-version = "1.59" +readme = "README.md" + +[dependencies] +arrow = { version = "13" } +ballista = { path = "../ballista/rust/client", version = "0.6.0" } +clap = { version = "3", features = ["derive", "cargo"] } +datafusion = { path = "../datafusion/core", version = "7.0.0" } +dirs = "4.0.0" +env_logger = "0.9" +mimalloc = { version = "*", default-features = false } +rustyline = "9.0" +tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] } diff --git a/ballista-cli/Dockerfile b/ballista-cli/Dockerfile new file mode 100644 index 000000000000..7a35d185b715 --- /dev/null +++ b/ballista-cli/Dockerfile @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM rust:1.59 as builder + +COPY ./datafusion /usr/src/datafusion + +COPY ./ballista-cli /usr/src/ballista-cli + +COPY ./ballista /usr/src/ballista + +WORKDIR /usr/src/ballista-cli + +RUN rustup component add rustfmt + +RUN cargo build --release + +FROM debian:bullseye-slim + +COPY --from=builder /usr/src/ballista-cli/target/release/ballista-cli /usr/local/bin + +ENTRYPOINT ["ballista-cli"] + +CMD ["--data-path", "/data"] diff --git a/ballista-cli/README.md b/ballista-cli/README.md new file mode 100644 index 000000000000..ea47c5ed5f07 --- /dev/null +++ b/ballista-cli/README.md @@ -0,0 +1,94 @@ + + +# Ballista Command-line Interface + +[Ballista](df) is a distributed query execution framework, written in Rust, that uses Apache Arrow as its in-memory format. + +The Ballista CLI allows SQL queries to be executed by an in-process DataFusion context, or by a distributed +Ballista context. + +```ignore +USAGE: + ballista-cli [FLAGS] [OPTIONS] + +FLAGS: + -h, --help Prints help information + -q, --quiet Reduce printing other than the results and work quietly + -V, --version Prints version information + +OPTIONS: + -c, --batch-size The batch size of each query, or use DataFusion default + -p, --data-path Path to your data, default to current directory + -f, --file ... Execute commands from file(s), then exit + --format Output format [default: table] [possible values: csv, tsv, table, json, ndjson] + --host Ballista scheduler host + --port Ballista scheduler port +``` + +## Example + +Create a CSV file to query. + +```bash,ignore +$ echo "1,2" > data.csv +``` + +```sql,ignore +$ ballista-cli + +Ballista CLI v4.0.0-SNAPSHOT + +> CREATE EXTERNAL TABLE foo (a INT, b INT) STORED AS CSV LOCATION 'data.csv'; +0 rows in set. Query took 0.001 seconds. + +> SELECT * FROM foo; ++---+---+ +| a | b | ++---+---+ +| 1 | 2 | ++---+---+ +1 row in set. Query took 0.017 seconds. +``` + +## Ballista-Cli + +Build the `ballista-cli` without the feature of ballista. + +```bash +cd arrow-datafusion/ballista-cli +cargo build +``` + +## Ballista + +If you want to execute the SQL in ballista by `ballista-cli`, you must build/compile the `ballista-cli` with features of "ballista" first. + +```bash +cd arrow-datafusion/ballista-cli +cargo build +``` + +The Ballista CLI can connect to a Ballista scheduler for query execution. + +```bash +ballista-cli --host localhost --port 50050 +``` + +[df]: https://crates.io/crates/datafusion diff --git a/ballista-cli/src/command.rs b/ballista-cli/src/command.rs new file mode 100644 index 000000000000..0fd43a3071e5 --- /dev/null +++ b/ballista-cli/src/command.rs @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Command within CLI + +use crate::context::Context; +use crate::functions::{display_all_functions, Function}; +use crate::print_format::PrintFormat; +use crate::print_options::PrintOptions; +use clap::ArgEnum; +use datafusion::arrow::array::{ArrayRef, StringArray}; +use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::error::{DataFusionError, Result}; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Instant; + +/// Command +#[derive(Debug)] +pub enum Command { + Quit, + Help, + ListTables, + DescribeTable(String), + ListFunctions, + SearchFunctions(String), + QuietMode(Option), + OutputFormat(Option), +} + +pub enum OutputFormat { + ChangeFormat(String), +} + +impl Command { + pub async fn execute( + &self, + ctx: &mut Context, + print_options: &mut PrintOptions, + ) -> Result<()> { + let now = Instant::now(); + match self { + Self::Help => print_options + .print_batches(&[all_commands_info()], now) + .map_err(|e| DataFusionError::Execution(e.to_string())), + Self::ListTables => { + let df = ctx.sql("SHOW TABLES").await?; + let batches = df.collect().await?; + print_options + .print_batches(&batches, now) + .map_err(|e| DataFusionError::Execution(e.to_string())) + } + Self::DescribeTable(name) => { + let df = ctx.sql(&format!("SHOW COLUMNS FROM {}", name)).await?; + let batches = df.collect().await?; + print_options + .print_batches(&batches, now) + .map_err(|e| DataFusionError::Execution(e.to_string())) + } + Self::QuietMode(quiet) => { + if let Some(quiet) = quiet { + print_options.quiet = *quiet; + println!( + "Quiet mode set to {}", + if print_options.quiet { "true" } else { "false" } + ); + } else { + println!( + "Quiet mode is {}", + if print_options.quiet { "true" } else { "false" } + ); + } + Ok(()) + } + Self::Quit => Err(DataFusionError::Execution( + "Unexpected quit, this should be handled outside".into(), + )), + Self::ListFunctions => display_all_functions(), + Self::SearchFunctions(function) => { + if let Ok(func) = function.parse::() { + let details = func.function_details()?; + println!("{}", details); + Ok(()) + } else { + let msg = format!("{} is not a supported function", function); + Err(DataFusionError::Execution(msg)) + } + } + Self::OutputFormat(_) => Err(DataFusionError::Execution( + "Unexpected change output format, this should be handled outside".into(), + )), + } + } + + fn get_name_and_description(&self) -> (&'static str, &'static str) { + match self { + Self::Quit => ("\\q", "quit datafusion-cli"), + Self::ListTables => ("\\d", "list tables"), + Self::DescribeTable(_) => ("\\d name", "describe table"), + Self::Help => ("\\?", "help"), + Self::ListFunctions => ("\\h", "function list"), + Self::SearchFunctions(_) => ("\\h function", "search function"), + Self::QuietMode(_) => ("\\quiet (true|false)?", "print or set quiet mode"), + Self::OutputFormat(_) => { + ("\\pset [NAME [VALUE]]", "set table output option\n(format)") + } + } + } +} + +const ALL_COMMANDS: [Command; 8] = [ + Command::ListTables, + Command::DescribeTable(String::new()), + Command::Quit, + Command::Help, + Command::ListFunctions, + Command::SearchFunctions(String::new()), + Command::QuietMode(None), + Command::OutputFormat(None), +]; + +fn all_commands_info() -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("Command", DataType::Utf8, false), + Field::new("Description", DataType::Utf8, false), + ])); + let (names, description): (Vec<&str>, Vec<&str>) = ALL_COMMANDS + .into_iter() + .map(|c| c.get_name_and_description()) + .unzip(); + RecordBatch::try_new( + schema, + [names, description] + .into_iter() + .map(|i| Arc::new(StringArray::from(i)) as ArrayRef) + .collect::>(), + ) + .expect("This should not fail") +} + +impl FromStr for Command { + type Err = (); + + fn from_str(s: &str) -> std::result::Result { + let (c, arg) = if let Some((a, b)) = s.split_once(' ') { + (a, Some(b)) + } else { + (s, None) + }; + Ok(match (c, arg) { + ("q", None) => Self::Quit, + ("d", None) => Self::ListTables, + ("d", Some(name)) => Self::DescribeTable(name.into()), + ("?", None) => Self::Help, + ("h", None) => Self::ListFunctions, + ("h", Some(function)) => Self::SearchFunctions(function.into()), + ("quiet", Some("true" | "t" | "yes" | "y" | "on")) => { + Self::QuietMode(Some(true)) + } + ("quiet", Some("false" | "f" | "no" | "n" | "off")) => { + Self::QuietMode(Some(false)) + } + ("quiet", None) => Self::QuietMode(None), + ("pset", Some(subcommand)) => { + Self::OutputFormat(Some(subcommand.to_string())) + } + ("pset", None) => Self::OutputFormat(None), + _ => return Err(()), + }) + } +} + +impl FromStr for OutputFormat { + type Err = (); + + fn from_str(s: &str) -> std::result::Result { + let (c, arg) = if let Some((a, b)) = s.split_once(' ') { + (a, Some(b)) + } else { + (s, None) + }; + Ok(match (c, arg) { + ("format", Some(format)) => Self::ChangeFormat(format.to_string()), + _ => return Err(()), + }) + } +} + +impl OutputFormat { + pub async fn execute(&self, print_options: &mut PrintOptions) -> Result<()> { + match self { + Self::ChangeFormat(format) => { + if let Ok(format) = format.parse::() { + print_options.format = format; + println!("Output format is {:?}.", print_options.format); + Ok(()) + } else { + Err(DataFusionError::Execution(format!( + "{:?} is not a valid format type [possible values: {:?}]", + format, + PrintFormat::value_variants() + ))) + } + } + } + } +} diff --git a/ballista-cli/src/context.rs b/ballista-cli/src/context.rs new file mode 100644 index 000000000000..c96b0e76624a --- /dev/null +++ b/ballista-cli/src/context.rs @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Context (remote or local) + +use datafusion::dataframe::DataFrame; +use datafusion::error::{DataFusionError, Result}; +use datafusion::execution::context::{SessionConfig, SessionContext}; +use std::sync::Arc; + +/// The CLI supports using a local DataFusion context or a distributed BallistaContext +pub enum Context { + /// In-process execution with DataFusion + Local(SessionContext), + /// Distributed execution with Ballista (if available) + Remote(BallistaContext), +} + +impl Context { + /// create a new remote context with given host and port + pub async fn new_remote(host: &str, port: u16) -> Result { + Ok(Context::Remote(BallistaContext::try_new(host, port).await?)) + } + + /// create a local context using the given config + pub fn new_local(config: &SessionConfig) -> Context { + Context::Local(SessionContext::with_config(config.clone())) + } + + /// execute an SQL statement against the context + pub async fn sql(&mut self, sql: &str) -> Result> { + match self { + Context::Local(datafusion) => datafusion.sql(sql).await, + Context::Remote(ballista) => ballista.sql(sql).await, + } + } +} + +// implement wrappers around the BallistaContext to support running without ballista + +#[cfg(feature = "ballista")] +pub struct BallistaContext(ballista::context::BallistaContext); +#[cfg(feature = "ballista")] +impl BallistaContext { + pub async fn try_new(host: &str, port: u16) -> Result { + use ballista::context::BallistaContext; + use ballista::prelude::BallistaConfig; + let builder = + BallistaConfig::builder().set("ballista.with_information_schema", "true"); + let config = builder + .build() + .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?; + let remote_ctx = BallistaContext::remote(host, port, &config) + .await + .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?; + Ok(Self(remote_ctx)) + } + pub async fn sql(&mut self, sql: &str) -> Result> { + self.0.sql(sql).await + } +} + +#[cfg(not(feature = "ballista"))] +pub struct BallistaContext(); +#[cfg(not(feature = "ballista"))] +impl BallistaContext { + pub async fn try_new(_host: &str, _port: u16) -> Result { + Err(DataFusionError::NotImplemented( + "Remote execution not supported. Compile with feature 'ballista' to enable" + .to_string(), + )) + } + pub async fn sql(&mut self, _sql: &str) -> Result> { + unreachable!() + } +} diff --git a/ballista-cli/src/exec.rs b/ballista-cli/src/exec.rs new file mode 100644 index 000000000000..dc3c73e5ab43 --- /dev/null +++ b/ballista-cli/src/exec.rs @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Execution functions + +use crate::{ + command::{Command, OutputFormat}, + context::Context, + helper::CliHelper, + print_options::PrintOptions, +}; +use datafusion::error::Result; +use rustyline::error::ReadlineError; +use rustyline::Editor; +use std::fs::File; +use std::io::prelude::*; +use std::io::BufReader; +use std::time::Instant; + +/// run and execute SQL statements and commands from a file, against a context with the given print options +pub async fn exec_from_lines( + ctx: &mut Context, + reader: &mut BufReader, + print_options: &PrintOptions, +) { + let mut query = "".to_owned(); + + for line in reader.lines() { + match line { + Ok(line) if line.starts_with("--") => { + continue; + } + Ok(line) => { + let line = line.trim_end(); + query.push_str(line); + if line.ends_with(';') { + match exec_and_print(ctx, print_options, query).await { + Ok(_) => {} + Err(err) => println!("{:?}", err), + } + query = "".to_owned(); + } else { + query.push('\n'); + } + } + _ => { + break; + } + } + } + + // run the left over query if the last statement doesn't contain ‘;’ + if !query.is_empty() { + match exec_and_print(ctx, print_options, query).await { + Ok(_) => {} + Err(err) => println!("{:?}", err), + } + } +} + +pub async fn exec_from_files( + files: Vec, + ctx: &mut Context, + print_options: &PrintOptions, +) { + let files = files + .into_iter() + .map(|file_path| File::open(file_path).unwrap()) + .collect::>(); + for file in files { + let mut reader = BufReader::new(file); + exec_from_lines(ctx, &mut reader, print_options).await; + } +} + +/// run and execute SQL statements and commands against a context with the given print options +pub async fn exec_from_repl(ctx: &mut Context, print_options: &mut PrintOptions) { + let mut rl = Editor::::new(); + rl.set_helper(Some(CliHelper::default())); + rl.load_history(".history").ok(); + + let mut print_options = print_options.clone(); + + loop { + match rl.readline("❯ ") { + Ok(line) if line.starts_with('\\') => { + rl.add_history_entry(line.trim_end()); + let command = line.split_whitespace().collect::>().join(" "); + if let Ok(cmd) = &command[1..].parse::() { + match cmd { + Command::Quit => break, + Command::OutputFormat(subcommand) => { + if let Some(subcommand) = subcommand { + if let Ok(command) = subcommand.parse::() { + if let Err(e) = + command.execute(&mut print_options).await + { + eprintln!("{}", e) + } + } else { + eprintln!( + "'\\{}' is not a valid command", + &line[1..] + ); + } + } else { + println!("Output format is {:?}.", print_options.format); + } + } + _ => { + if let Err(e) = cmd.execute(ctx, &mut print_options).await { + eprintln!("{}", e) + } + } + } + } else { + eprintln!("'\\{}' is not a valid command", &line[1..]); + } + } + Ok(line) => { + rl.add_history_entry(line.trim_end()); + match exec_and_print(ctx, &print_options, line).await { + Ok(_) => {} + Err(err) => eprintln!("{:?}", err), + } + } + Err(ReadlineError::Interrupted) => { + println!("^C"); + continue; + } + Err(ReadlineError::Eof) => { + println!("\\q"); + break; + } + Err(err) => { + eprintln!("Unknown error happened {:?}", err); + break; + } + } + } + + rl.save_history(".history").ok(); +} + +async fn exec_and_print( + ctx: &mut Context, + print_options: &PrintOptions, + sql: String, +) -> Result<()> { + let now = Instant::now(); + let df = ctx.sql(&sql).await?; + let results = df.collect().await?; + print_options.print_batches(&results, now)?; + + Ok(()) +} diff --git a/ballista-cli/src/functions.rs b/ballista-cli/src/functions.rs new file mode 100644 index 000000000000..98b698ab5fb6 --- /dev/null +++ b/ballista-cli/src/functions.rs @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Functions that are query-able and searchable via the `\h` command +use arrow::array::StringArray; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use arrow::util::pretty::pretty_format_batches; +use datafusion::error::Result; +use std::fmt; +use std::str::FromStr; +use std::sync::Arc; + +#[derive(Debug)] +pub enum Function { + Select, + Explain, + Show, + CreateTable, + CreateTableAs, + Insert, + DropTable, +} + +const ALL_FUNCTIONS: [Function; 7] = [ + Function::CreateTable, + Function::CreateTableAs, + Function::DropTable, + Function::Explain, + Function::Insert, + Function::Select, + Function::Show, +]; + +impl Function { + pub fn function_details(&self) -> Result<&str> { + let details = match self { + Function::Select => { + r#" +Command: SELECT +Description: retrieve rows from a table or view +Syntax: +SELECT [ ALL | DISTINCT [ ON ( expression [, ...] ) ] ] + [ * | expression [ [ AS ] output_name ] [, ...] ] + [ FROM from_item [, ...] ] + [ WHERE condition ] + [ GROUP BY [ ALL | DISTINCT ] grouping_element [, ...] ] + [ HAVING condition ] + [ WINDOW window_name AS ( window_definition ) [, ...] ] + [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ] + [ ORDER BY expression [ ASC | DESC | USING operator ] [ NULLS { FIRST | LAST } ] [, ...] ] + [ LIMIT { count | ALL } ] + [ OFFSET start [ ROW | ROWS ] ] + +where from_item can be one of: + + [ ONLY ] table_name [ * ] [ [ AS ] alias [ ( column_alias [, ...] ) ] ] + [ TABLESAMPLE sampling_method ( argument [, ...] ) [ REPEATABLE ( seed ) ] ] + [ LATERAL ] ( select ) [ AS ] alias [ ( column_alias [, ...] ) ] + with_query_name [ [ AS ] alias [ ( column_alias [, ...] ) ] ] + [ LATERAL ] function_name ( [ argument [, ...] ] ) + [ WITH ORDINALITY ] [ [ AS ] alias [ ( column_alias [, ...] ) ] ] + [ LATERAL ] function_name ( [ argument [, ...] ] ) [ AS ] alias ( column_definition [, ...] ) + [ LATERAL ] function_name ( [ argument [, ...] ] ) AS ( column_definition [, ...] ) + [ LATERAL ] ROWS FROM( function_name ( [ argument [, ...] ] ) [ AS ( column_definition [, ...] ) ] [, ...] ) + [ WITH ORDINALITY ] [ [ AS ] alias [ ( column_alias [, ...] ) ] ] + from_item [ NATURAL ] join_type from_item [ ON join_condition | USING ( join_column [, ...] ) [ AS join_using_alias ] ] + +and grouping_element can be one of: + + ( ) + expression + ( expression [, ...] ) + +and with_query is: + + with_query_name [ ( column_name [, ...] ) ] AS [ [ NOT ] MATERIALIZED ] ( select | values | insert | update | delete ) + +TABLE [ ONLY ] table_name [ * ]"# + } + Function::Explain => { + r#" +Command: EXPLAIN +Description: show the execution plan of a statement +Syntax: +EXPLAIN [ ANALYZE ] statement +"# + } + Function::Show => { + r#" +Command: SHOW +Description: show the value of a run-time parameter +Syntax: +SHOW name +"# + } + Function::CreateTable => { + r#" +Command: CREATE TABLE +Description: define a new table +Syntax: +CREATE [ EXTERNAL ] TABLE table_name ( [ + { column_name data_type } + [, ... ] +] ) +"# + } + Function::CreateTableAs => { + r#" +Command: CREATE TABLE AS +Description: define a new table from the results of a query +Syntax: +CREATE TABLE table_name + [ (column_name [, ...] ) ] + AS query + [ WITH [ NO ] DATA ] +"# + } + Function::Insert => { + r#" +Command: INSERT +Description: create new rows in a table +Syntax: +INSERT INTO table_name [ ( column_name [, ...] ) ] + { VALUES ( { expression } [, ...] ) [, ...] } +"# + } + Function::DropTable => { + r#" +Command: DROP TABLE +Description: remove a table +Syntax: +DROP TABLE [ IF EXISTS ] name [, ...] +"# + } + }; + Ok(details) + } +} + +impl FromStr for Function { + type Err = (); + + fn from_str(s: &str) -> std::result::Result { + Ok(match s.trim().to_uppercase().as_str() { + "SELECT" => Self::Select, + "EXPLAIN" => Self::Explain, + "SHOW" => Self::Show, + "CREATE TABLE" => Self::CreateTable, + "CREATE TABLE AS" => Self::CreateTableAs, + "INSERT" => Self::Insert, + "DROP TABLE" => Self::DropTable, + _ => return Err(()), + }) + } +} + +impl fmt::Display for Function { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + Function::Select => write!(f, "SELECT"), + Function::Explain => write!(f, "EXPLAIN"), + Function::Show => write!(f, "SHOW"), + Function::CreateTable => write!(f, "CREATE TABLE"), + Function::CreateTableAs => write!(f, "CREATE TABLE AS"), + Function::Insert => write!(f, "INSERT"), + Function::DropTable => write!(f, "DROP TABLE"), + } + } +} + +pub fn display_all_functions() -> Result<()> { + println!("Available help:"); + let array = StringArray::from( + ALL_FUNCTIONS + .iter() + .map(|f| format!("{}", f)) + .collect::>(), + ); + let schema = Schema::new(vec![Field::new("Function", DataType::Utf8, false)]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)])?; + println!("{}", pretty_format_batches(&[batch]).unwrap()); + Ok(()) +} diff --git a/ballista-cli/src/helper.rs b/ballista-cli/src/helper.rs new file mode 100644 index 000000000000..e229255f56e3 --- /dev/null +++ b/ballista-cli/src/helper.rs @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Helper that helps with interactive editing, including multi-line parsing and validation, +//! and auto-completion for file name during creating external table. + +use datafusion::sql::parser::{DFParser, Statement}; +use rustyline::completion::Completer; +use rustyline::completion::FilenameCompleter; +use rustyline::completion::Pair; +use rustyline::error::ReadlineError; +use rustyline::highlight::Highlighter; +use rustyline::hint::Hinter; +use rustyline::validate::ValidationContext; +use rustyline::validate::ValidationResult; +use rustyline::validate::Validator; +use rustyline::Context; +use rustyline::Helper; +use rustyline::Result; + +#[derive(Default)] +pub(crate) struct CliHelper { + completer: FilenameCompleter, +} + +impl Highlighter for CliHelper {} + +impl Hinter for CliHelper { + type Hint = String; +} + +/// returns true if the current position is after the open quote for +/// creating an external table. +fn is_open_quote_for_location(line: &str, pos: usize) -> bool { + let mut sql = line[..pos].to_string(); + sql.push('\''); + if let Ok(stmts) = DFParser::parse_sql(&sql) { + if let Some(Statement::CreateExternalTable(_)) = stmts.back() { + return true; + } + } + false +} + +impl Completer for CliHelper { + type Candidate = Pair; + + fn complete( + &self, + line: &str, + pos: usize, + ctx: &Context<'_>, + ) -> std::result::Result<(usize, Vec), ReadlineError> { + if is_open_quote_for_location(line, pos) { + self.completer.complete(line, pos, ctx) + } else { + Ok((0, Vec::with_capacity(0))) + } + } +} + +impl Validator for CliHelper { + fn validate(&self, ctx: &mut ValidationContext<'_>) -> Result { + let input = ctx.input().trim_end(); + if let Some(sql) = input.strip_suffix(';') { + match DFParser::parse_sql(sql) { + Ok(statements) if statements.is_empty() => Ok(ValidationResult::Invalid( + Some(" 🤔 You entered an empty statement".to_string()), + )), + Ok(statements) if statements.len() > 1 => Ok(ValidationResult::Invalid( + Some(" 🤔 You entered more than one statement".to_string()), + )), + Ok(_statements) => Ok(ValidationResult::Valid(None)), + Err(err) => Ok(ValidationResult::Invalid(Some(format!( + " 🤔 Invalid statement: {}", + err + )))), + } + } else if input.starts_with('\\') { + // command + Ok(ValidationResult::Valid(None)) + } else { + Ok(ValidationResult::Incomplete) + } + } +} + +impl Helper for CliHelper {} diff --git a/ballista-cli/src/lib.rs b/ballista-cli/src/lib.rs new file mode 100644 index 000000000000..b75be331259b --- /dev/null +++ b/ballista-cli/src/lib.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![doc = include_str!("../README.md")] +pub const DATAFUSION_CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); + +pub mod command; +pub mod context; +pub mod exec; +pub mod functions; +pub mod helper; +pub mod print_format; +pub mod print_options; diff --git a/ballista-cli/src/main.rs b/ballista-cli/src/main.rs new file mode 100644 index 000000000000..2287fc8a1893 --- /dev/null +++ b/ballista-cli/src/main.rs @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use clap::Parser; +use datafusion::error::Result; +use datafusion::execution::context::SessionConfig; +use ballista_cli::{ + context::Context, exec, print_format::PrintFormat, print_options::PrintOptions, + DATAFUSION_CLI_VERSION, +}; +use mimalloc::MiMalloc; +use std::env; +use std::path::Path; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +#[derive(Debug, Parser, PartialEq)] +#[clap(author, version, about, long_about= None)] +struct Args { + #[clap( + short = 'p', + long, + help = "Path to your data, default to current directory", + validator(is_valid_data_dir) + )] + data_path: Option, + + #[clap( + short = 'c', + long, + help = "The batch size of each query, or use DataFusion default", + validator(is_valid_batch_size) + )] + batch_size: Option, + + #[clap( + short, + long, + multiple_values = true, + help = "Execute commands from file(s), then exit", + validator(is_valid_file) + )] + file: Vec, + + #[clap( + short = 'r', + long, + multiple_values = true, + help = "Run the provided files on startup instead of ~/.datafusionrc", + validator(is_valid_file), + conflicts_with = "file" + )] + rc: Option>, + + #[clap(long, arg_enum, default_value_t = PrintFormat::Table)] + format: PrintFormat, + + #[clap(long, help = "Ballista scheduler host")] + host: Option, + + #[clap(long, help = "Ballista scheduler port")] + port: Option, + + #[clap( + short, + long, + help = "Reduce printing other than the results and work quietly" + )] + quiet: bool, +} + +#[tokio::main] +pub async fn main() -> Result<()> { + env_logger::init(); + let args = Args::parse(); + + if !args.quiet { + println!("DataFusion CLI v{}", DATAFUSION_CLI_VERSION); + } + + if let Some(ref path) = args.data_path { + let p = Path::new(path); + env::set_current_dir(&p).unwrap(); + }; + + let mut session_config = SessionConfig::new().with_information_schema(true); + + if let Some(batch_size) = args.batch_size { + session_config = session_config.with_batch_size(batch_size); + }; + + let mut ctx: Context = match (args.host, args.port) { + (Some(ref h), Some(p)) => Context::new_remote(h, p).await?, + _ => Context::new_local(&session_config), + }; + + let mut print_options = PrintOptions { + format: args.format, + quiet: args.quiet, + }; + + let files = args.file; + let rc = match args.rc { + Some(file) => file, + None => { + let mut files = Vec::new(); + let home = dirs::home_dir(); + if let Some(p) = home { + let home_rc = p.join(".datafusionrc"); + if home_rc.exists() { + files.push(home_rc.into_os_string().into_string().unwrap()); + } + } + files + } + }; + if !files.is_empty() { + exec::exec_from_files(files, &mut ctx, &print_options).await + } else { + if !rc.is_empty() { + exec::exec_from_files(rc, &mut ctx, &print_options).await + } + exec::exec_from_repl(&mut ctx, &mut print_options).await; + } + + Ok(()) +} + +fn is_valid_file(dir: &str) -> std::result::Result<(), String> { + if Path::new(dir).is_file() { + Ok(()) + } else { + Err(format!("Invalid file '{}'", dir)) + } +} + +fn is_valid_data_dir(dir: &str) -> std::result::Result<(), String> { + if Path::new(dir).is_dir() { + Ok(()) + } else { + Err(format!("Invalid data directory '{}'", dir)) + } +} + +fn is_valid_batch_size(size: &str) -> std::result::Result<(), String> { + match size.parse::() { + Ok(size) if size > 0 => Ok(()), + _ => Err(format!("Invalid batch size '{}'", size)), + } +} diff --git a/ballista-cli/src/print_format.rs b/ballista-cli/src/print_format.rs new file mode 100644 index 000000000000..05a1ef7b10d8 --- /dev/null +++ b/ballista-cli/src/print_format.rs @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Print format variants +use arrow::csv::writer::WriterBuilder; +use arrow::json::{ArrayWriter, LineDelimitedWriter}; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::arrow::util::pretty; +use datafusion::error::{DataFusionError, Result}; +use std::str::FromStr; + +/// Allow records to be printed in different formats +#[derive(Debug, PartialEq, Eq, clap::ArgEnum, Clone)] +pub enum PrintFormat { + Csv, + Tsv, + Table, + Json, + NdJson, +} + +impl FromStr for PrintFormat { + type Err = String; + + fn from_str(s: &str) -> std::result::Result { + clap::ArgEnum::from_str(s, true) + } +} + +macro_rules! batches_to_json { + ($WRITER: ident, $batches: expr) => {{ + let mut bytes = vec![]; + { + let mut writer = $WRITER::new(&mut bytes); + writer.write_batches($batches)?; + writer.finish()?; + } + String::from_utf8(bytes).map_err(|e| DataFusionError::Execution(e.to_string()))? + }}; +} + +fn print_batches_with_sep(batches: &[RecordBatch], delimiter: u8) -> Result { + let mut bytes = vec![]; + { + let builder = WriterBuilder::new() + .has_headers(true) + .with_delimiter(delimiter); + let mut writer = builder.build(&mut bytes); + for batch in batches { + writer.write(batch)?; + } + } + let formatted = String::from_utf8(bytes) + .map_err(|e| DataFusionError::Execution(e.to_string()))?; + Ok(formatted) +} + +impl PrintFormat { + /// print the batches to stdout using the specified format + pub fn print_batches(&self, batches: &[RecordBatch]) -> Result<()> { + match self { + Self::Csv => println!("{}", print_batches_with_sep(batches, b',')?), + Self::Tsv => println!("{}", print_batches_with_sep(batches, b'\t')?), + Self::Table => pretty::print_batches(batches)?, + Self::Json => println!("{}", batches_to_json!(ArrayWriter, batches)), + Self::NdJson => { + println!("{}", batches_to_json!(LineDelimitedWriter, batches)) + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::Int32Array; + use arrow::datatypes::{DataType, Field, Schema}; + use datafusion::from_slice::FromSlice; + use std::sync::Arc; + + #[test] + fn test_print_batches_with_sep() { + let batches = vec![]; + assert_eq!("", print_batches_with_sep(&batches, b',').unwrap()); + + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Int32, false), + Field::new("c", DataType::Int32, false), + ])); + + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from_slice(&[1, 2, 3])), + Arc::new(Int32Array::from_slice(&[4, 5, 6])), + Arc::new(Int32Array::from_slice(&[7, 8, 9])), + ], + ) + .unwrap(); + + let batches = vec![batch]; + let r = print_batches_with_sep(&batches, b',').unwrap(); + assert_eq!("a,b,c\n1,4,7\n2,5,8\n3,6,9\n", r); + } + + #[test] + fn test_print_batches_to_json_empty() -> Result<()> { + let batches = vec![]; + let r = batches_to_json!(ArrayWriter, &batches); + assert_eq!("", r); + + let r = batches_to_json!(LineDelimitedWriter, &batches); + assert_eq!("", r); + + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Int32, false), + Field::new("c", DataType::Int32, false), + ])); + + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from_slice(&[1, 2, 3])), + Arc::new(Int32Array::from_slice(&[4, 5, 6])), + Arc::new(Int32Array::from_slice(&[7, 8, 9])), + ], + ) + .unwrap(); + + let batches = vec![batch]; + let r = batches_to_json!(ArrayWriter, &batches); + assert_eq!("[{\"a\":1,\"b\":4,\"c\":7},{\"a\":2,\"b\":5,\"c\":8},{\"a\":3,\"b\":6,\"c\":9}]", r); + + let r = batches_to_json!(LineDelimitedWriter, &batches); + assert_eq!("{\"a\":1,\"b\":4,\"c\":7}\n{\"a\":2,\"b\":5,\"c\":8}\n{\"a\":3,\"b\":6,\"c\":9}\n", r); + Ok(()) + } +} diff --git a/ballista-cli/src/print_options.rs b/ballista-cli/src/print_options.rs new file mode 100644 index 000000000000..5e3792634a4e --- /dev/null +++ b/ballista-cli/src/print_options.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::print_format::PrintFormat; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::error::Result; +use std::time::Instant; + +#[derive(Debug, Clone)] +pub struct PrintOptions { + pub format: PrintFormat, + pub quiet: bool, +} + +fn print_timing_info(row_count: usize, now: Instant) { + println!( + "{} {} in set. Query took {:.3} seconds.", + row_count, + if row_count == 1 { "row" } else { "rows" }, + now.elapsed().as_secs_f64() + ); +} + +impl PrintOptions { + /// print the batches to stdout using the specified format + pub fn print_batches(&self, batches: &[RecordBatch], now: Instant) -> Result<()> { + if batches.is_empty() { + if !self.quiet { + print_timing_info(0, now); + } + } else { + self.format.print_batches(batches)?; + if !self.quiet { + let row_count: usize = batches.iter().map(|b| b.num_rows()).sum(); + print_timing_info(row_count, now); + } + } + Ok(()) + } +} diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 06f978c3ebb4..d67b041f7a3d 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -43,12 +43,6 @@ dependencies = [ "alloc-no-stdlib", ] -[[package]] -name = "anyhow" -version = "1.0.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" - [[package]] name = "arrayref" version = "0.3.6" @@ -86,45 +80,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "arrow-flight" -version = "13.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a3666d2dbc637fa979d1f0bf3031d39a80e709f3b9ec88e3d573c1d666bf553" -dependencies = [ - "arrow", - "base64", - "bytes", - "futures", - "proc-macro2", - "prost", - "prost-derive", - "tokio", - "tonic", - "tonic-build", -] - -[[package]] -name = "async-stream" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" -dependencies = [ - "async-stream-impl", - "futures-core", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-trait" version = "0.1.53" @@ -153,93 +108,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "axum" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4af7447fc1214c1f3a1ace861d0216a6c8bb13965b64bbad9650f375b67689a" -dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa 1.0.1", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde", - "sync_wrapper", - "tokio", - "tower", - "tower-http", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da31c0ed7b4690e2c78fe4b880d21cd7db04a346ebc658b4270251b695437f17" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", -] - -[[package]] -name = "ballista" -version = "0.6.0" -dependencies = [ - "ballista-core", - "datafusion", - "futures", - "log", - "parking_lot", - "sqlparser", - "tempfile", - "tokio", -] - -[[package]] -name = "ballista-core" -version = "0.6.0" -dependencies = [ - "ahash", - "arrow-flight", - "async-trait", - "chrono", - "clap", - "datafusion", - "datafusion-proto", - "futures", - "hashbrown 0.12.1", - "libloading", - "log", - "once_cell", - "parking_lot", - "parse_arg", - "prost", - "prost-types", - "rustc_version", - "serde", - "sqlparser", - "tokio", - "tonic", - "tonic-build", - "uuid", - "walkdir", -] - [[package]] name = "base64" version = "0.13.0" @@ -323,12 +191,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" - [[package]] name = "cc" version = "1.0.73" @@ -406,15 +268,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "cmake" -version = "0.1.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" -dependencies = [ - "cc", -] - [[package]] name = "comfy-table" version = "5.0.1" @@ -494,6 +347,7 @@ dependencies = [ "datafusion-data-access", "datafusion-expr", "datafusion-physical-expr", + "datafusion-row", "futures", "hashbrown 0.12.1", "lazy_static", @@ -518,7 +372,6 @@ name = "datafusion-cli" version = "7.0.0" dependencies = [ "arrow", - "ballista", "clap", "datafusion", "dirs", @@ -545,6 +398,7 @@ dependencies = [ "async-trait", "chrono", "futures", + "glob", "parking_lot", "tempfile", "tokio", @@ -571,6 +425,7 @@ dependencies = [ "chrono", "datafusion-common", "datafusion-expr", + "datafusion-row", "hashbrown 0.12.1", "lazy_static", "md-5", @@ -583,12 +438,13 @@ dependencies = [ ] [[package]] -name = "datafusion-proto" +name = "datafusion-row" version = "7.0.0" dependencies = [ - "datafusion", - "prost", - "tonic-build", + "arrow", + "datafusion-common", + "paste", + "rand", ] [[package]] @@ -643,12 +499,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "either" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" - [[package]] name = "endian-type" version = "0.1.2" @@ -719,12 +569,6 @@ dependencies = [ "windows-sys 0.30.0", ] -[[package]] -name = "fixedbitset" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" - [[package]] name = "flatbuffers" version = "2.1.2" @@ -748,12 +592,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "futures" version = "0.3.21" @@ -861,27 +699,14 @@ checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi", ] [[package]] -name = "h2" -version = "0.3.13" +name = "glob" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "half" @@ -934,88 +759,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "http" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" -dependencies = [ - "bytes", - "fnv", - "itoa 1.0.1", -] - -[[package]] -name = "http-body" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "http-range-header" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" - -[[package]] -name = "httparse" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" - -[[package]] -name = "httpdate" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" - [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" -[[package]] -name = "hyper" -version = "0.14.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa 1.0.1", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - [[package]] name = "indexmap" version = "1.8.1" @@ -1047,15 +796,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9448015e586b611e5d322f6703812bbca2f1e709d5773ecd38ddb4e3bb649504" -[[package]] -name = "itertools" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "0.4.8" @@ -1153,16 +893,6 @@ version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" -[[package]] -name = "libloading" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" -dependencies = [ - "cfg-if", - "winapi", -] - [[package]] name = "libmimalloc-sys" version = "0.1.25" @@ -1217,12 +947,6 @@ dependencies = [ "libc", ] -[[package]] -name = "matchit" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" - [[package]] name = "md-5" version = "0.10.1" @@ -1256,12 +980,6 @@ dependencies = [ "libmimalloc-sys", ] -[[package]] -name = "mime" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" - [[package]] name = "miniz_oxide" version = "0.5.1" @@ -1271,35 +989,6 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" -dependencies = [ - "libc", - "log", - "miow", - "ntapi", - "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - [[package]] name = "multiversion" version = "0.6.1" @@ -1342,15 +1031,6 @@ dependencies = [ "memoffset", ] -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", -] - [[package]] name = "num" version = "0.4.0" @@ -1521,54 +1201,12 @@ dependencies = [ "thrift", ] -[[package]] -name = "parse_arg" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14248cc8eced350e20122a291613de29e4fa129ba2731818c4cdbb44fccd3e55" - [[package]] name = "paste" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" -[[package]] -name = "percent-encoding" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" - -[[package]] -name = "petgraph" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" -dependencies = [ - "fixedbitset", - "indexmap", -] - -[[package]] -name = "pin-project" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1587,16 +1225,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" -[[package]] -name = "prettyplease" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1630,61 +1258,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "prost" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07b0857a71a8cb765763950499cae2413c3f9cede1133478c43600d9e146890" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120fbe7988713f39d780a58cf1a7ef0d7ef66c6d87e5aa3438940c05357929f4" -dependencies = [ - "bytes", - "cfg-if", - "cmake", - "heck 0.4.0", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prost", - "prost-types", - "regex", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" -dependencies = [ - "bytes", - "prost", -] - [[package]] name = "quote" version = "1.0.18" @@ -1786,15 +1359,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.34.6" @@ -1845,35 +1409,17 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "semver" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" - [[package]] name = "serde" version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" -dependencies = [ - "serde_derive", -] [[package]] name = "serde_derive" @@ -1927,16 +1473,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" -[[package]] -name = "socket2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "sqlparser" version = "0.16.0" @@ -2000,12 +1536,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "sync_wrapper" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" - [[package]] name = "tempfile" version = "3.3.0" @@ -2083,27 +1613,11 @@ version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc" dependencies = [ - "bytes", - "libc", - "memchr", - "mio", "num_cpus", "once_cell", "parking_lot", "pin-project-lite", - "socket2", "tokio-macros", - "winapi", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", ] [[package]] @@ -2128,165 +1642,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-util" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "tonic" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30fb54bf1e446f44d870d260d99957e7d11fb9d0a0f5bd1a662ad1411cc103f9" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "prost-derive", - "tokio", - "tokio-stream", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - -[[package]] -name = "tonic-build" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03447cdc9eaf8feffb6412dcb27baf2db11669a6c4789f29da799aabfb99547" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn", -] - -[[package]] -name = "tower" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" -dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e980386f06883cf4d0578d6c9178c81f68b45d77d00f2c2c1bc034b3439c2c56" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" - -[[package]] -name = "tower-service" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" - -[[package]] -name = "tracing" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" -dependencies = [ - "cfg-if", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - -[[package]] -name = "try-lock" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" - [[package]] name = "typenum" version = "1.15.0" @@ -2332,50 +1687,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "walkdir" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" -dependencies = [ - "same-file", - "winapi", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" -dependencies = [ - "log", - "try-lock", -] - [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "which" -version = "4.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" -dependencies = [ - "either", - "lazy_static", - "libc", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 0f0deb3f07ae..b880e8b95b16 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -30,7 +30,6 @@ readme = "README.md" [dependencies] arrow = { version = "13" } -ballista = { path = "../ballista/rust/client", version = "0.6.0", optional = true } clap = { version = "3", features = ["derive", "cargo"] } datafusion = { path = "../datafusion/core", version = "7.0.0" } dirs = "4.0.0" diff --git a/datafusion-cli/src/context.rs b/datafusion-cli/src/context.rs index c96b0e76624a..8645a74d5ac9 100644 --- a/datafusion-cli/src/context.rs +++ b/datafusion-cli/src/context.rs @@ -31,11 +31,6 @@ pub enum Context { } impl Context { - /// create a new remote context with given host and port - pub async fn new_remote(host: &str, port: u16) -> Result { - Ok(Context::Remote(BallistaContext::try_new(host, port).await?)) - } - /// create a local context using the given config pub fn new_local(config: &SessionConfig) -> Context { Context::Local(SessionContext::with_config(config.clone())) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index d76fe38e5fb6..0702ff447936 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -70,12 +70,6 @@ struct Args { #[clap(long, arg_enum, default_value_t = PrintFormat::Table)] format: PrintFormat, - #[clap(long, help = "Ballista scheduler host")] - host: Option, - - #[clap(long, help = "Ballista scheduler port")] - port: Option, - #[clap( short, long, @@ -104,10 +98,7 @@ pub async fn main() -> Result<()> { session_config = session_config.with_batch_size(batch_size); }; - let mut ctx: Context = match (args.host, args.port) { - (Some(ref h), Some(p)) => Context::new_remote(h, p).await?, - _ => Context::new_local(&session_config), - }; + let mut ctx: Context = Context::new_local(&session_config); let mut print_options = PrintOptions { format: args.format, From a583903bcaf1252b3f246c3af3f690604be7d562 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 May 2022 08:53:28 -0600 Subject: [PATCH 02/17] update dependency diagram --- dev/release/crate-deps.dot | 4 +- dev/release/crate-deps.svg | 180 ++++++++++++++++++++----------------- 2 files changed, 99 insertions(+), 85 deletions(-) diff --git a/dev/release/crate-deps.dot b/dev/release/crate-deps.dot index fd292fa2da2a..bedcd0e21bdc 100644 --- a/dev/release/crate-deps.dot +++ b/dev/release/crate-deps.dot @@ -56,6 +56,8 @@ digraph G { ballista -> ballista_executor datafusion_cli -> datafusion - datafusion_cli -> ballista + + ballista_cli -> datafusion + ballista_cli -> ballista } diff --git a/dev/release/crate-deps.svg b/dev/release/crate-deps.svg index c41cd0aa7e78..b409ab8976e8 100644 --- a/dev/release/crate-deps.svg +++ b/dev/release/crate-deps.svg @@ -1,247 +1,259 @@ - - + G - + datafusion_common - -datafusion_common + +datafusion_common datafusion_data_access - -datafusion_data_access + +datafusion_data_access datafusion_expr - -datafusion_expr + +datafusion_expr datafusion_expr->datafusion_common - - + + datafusion_physical_expr - -datafusion_physical_expr + +datafusion_physical_expr datafusion_physical_expr->datafusion_common - - + + datafusion_physical_expr->datafusion_expr - - + + datafusion_jit - -datafusion_jit + +datafusion_jit datafusion_jit->datafusion_common - - + + datafusion_jit->datafusion_expr - - + + datafusion_row - -datafusion_row + +datafusion_row datafusion_row->datafusion_common - - + + datafusion_row->datafusion_jit - - + + datafusion - -datafusion + +datafusion datafusion->datafusion_common - - + + datafusion->datafusion_data_access - - + + datafusion->datafusion_expr - - + + datafusion->datafusion_physical_expr - - + + datafusion->datafusion_jit - - + + datafusion->datafusion_row - - + + datafusion_proto - -datafusion_proto + +datafusion_proto datafusion_proto->datafusion - - + + ballista_core - -ballista_core + +ballista_core ballista_core->datafusion - - + + ballista_core->datafusion_proto - - + + ballista_scheduler - -ballista_scheduler + +ballista_scheduler ballista_scheduler->datafusion - - + + ballista_scheduler->ballista_core - - + + ballista_executor - -ballista_executor + +ballista_executor ballista_executor->datafusion - - + + ballista_executor->ballista_core - - + + ballista - -ballista + +ballista ballista->datafusion - - + + ballista->ballista_core - - + + ballista->ballista_scheduler - - + + ballista->ballista_executor - - + + datafusion_cli - -datafusion_cli + +datafusion_cli datafusion_cli->datafusion - - + + - + + +ballista_cli + +ballista_cli + + -datafusion_cli->ballista - - +ballista_cli->datafusion + + + + + +ballista_cli->ballista + + From c560fb640afb0f9ed34833ed1a020bad7e656ced Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 May 2022 09:00:15 -0600 Subject: [PATCH 03/17] re-use PrintFormat from datafusion-cli --- ballista-cli/Cargo.toml | 1 + ballista-cli/src/command.rs | 2 +- ballista-cli/src/lib.rs | 1 - ballista-cli/src/main.rs | 3 +- ballista-cli/src/print_format.rs | 155 ------------------------------ ballista-cli/src/print_options.rs | 2 +- 6 files changed, 5 insertions(+), 159 deletions(-) delete mode 100644 ballista-cli/src/print_format.rs diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml index a06b036ecc0e..d921f9b0e908 100644 --- a/ballista-cli/Cargo.toml +++ b/ballista-cli/Cargo.toml @@ -33,6 +33,7 @@ arrow = { version = "13" } ballista = { path = "../ballista/rust/client", version = "0.6.0" } clap = { version = "3", features = ["derive", "cargo"] } datafusion = { path = "../datafusion/core", version = "7.0.0" } +datafusion-cli = { path = "../datafusion-cli", version = "7.0.0" } dirs = "4.0.0" env_logger = "0.9" mimalloc = { version = "*", default-features = false } diff --git a/ballista-cli/src/command.rs b/ballista-cli/src/command.rs index 0fd43a3071e5..4598f855e743 100644 --- a/ballista-cli/src/command.rs +++ b/ballista-cli/src/command.rs @@ -19,7 +19,7 @@ use crate::context::Context; use crate::functions::{display_all_functions, Function}; -use crate::print_format::PrintFormat; +use datafusion_cli::print_format::PrintFormat; use crate::print_options::PrintOptions; use clap::ArgEnum; use datafusion::arrow::array::{ArrayRef, StringArray}; diff --git a/ballista-cli/src/lib.rs b/ballista-cli/src/lib.rs index b75be331259b..fb50f414c231 100644 --- a/ballista-cli/src/lib.rs +++ b/ballista-cli/src/lib.rs @@ -23,5 +23,4 @@ pub mod context; pub mod exec; pub mod functions; pub mod helper; -pub mod print_format; pub mod print_options; diff --git a/ballista-cli/src/main.rs b/ballista-cli/src/main.rs index 2287fc8a1893..aa6f611a67c7 100644 --- a/ballista-cli/src/main.rs +++ b/ballista-cli/src/main.rs @@ -18,8 +18,9 @@ use clap::Parser; use datafusion::error::Result; use datafusion::execution::context::SessionConfig; +use datafusion_cli::{print_format::PrintFormat}; use ballista_cli::{ - context::Context, exec, print_format::PrintFormat, print_options::PrintOptions, + context::Context, exec, print_options::PrintOptions, DATAFUSION_CLI_VERSION, }; use mimalloc::MiMalloc; diff --git a/ballista-cli/src/print_format.rs b/ballista-cli/src/print_format.rs deleted file mode 100644 index 05a1ef7b10d8..000000000000 --- a/ballista-cli/src/print_format.rs +++ /dev/null @@ -1,155 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Print format variants -use arrow::csv::writer::WriterBuilder; -use arrow::json::{ArrayWriter, LineDelimitedWriter}; -use datafusion::arrow::record_batch::RecordBatch; -use datafusion::arrow::util::pretty; -use datafusion::error::{DataFusionError, Result}; -use std::str::FromStr; - -/// Allow records to be printed in different formats -#[derive(Debug, PartialEq, Eq, clap::ArgEnum, Clone)] -pub enum PrintFormat { - Csv, - Tsv, - Table, - Json, - NdJson, -} - -impl FromStr for PrintFormat { - type Err = String; - - fn from_str(s: &str) -> std::result::Result { - clap::ArgEnum::from_str(s, true) - } -} - -macro_rules! batches_to_json { - ($WRITER: ident, $batches: expr) => {{ - let mut bytes = vec![]; - { - let mut writer = $WRITER::new(&mut bytes); - writer.write_batches($batches)?; - writer.finish()?; - } - String::from_utf8(bytes).map_err(|e| DataFusionError::Execution(e.to_string()))? - }}; -} - -fn print_batches_with_sep(batches: &[RecordBatch], delimiter: u8) -> Result { - let mut bytes = vec![]; - { - let builder = WriterBuilder::new() - .has_headers(true) - .with_delimiter(delimiter); - let mut writer = builder.build(&mut bytes); - for batch in batches { - writer.write(batch)?; - } - } - let formatted = String::from_utf8(bytes) - .map_err(|e| DataFusionError::Execution(e.to_string()))?; - Ok(formatted) -} - -impl PrintFormat { - /// print the batches to stdout using the specified format - pub fn print_batches(&self, batches: &[RecordBatch]) -> Result<()> { - match self { - Self::Csv => println!("{}", print_batches_with_sep(batches, b',')?), - Self::Tsv => println!("{}", print_batches_with_sep(batches, b'\t')?), - Self::Table => pretty::print_batches(batches)?, - Self::Json => println!("{}", batches_to_json!(ArrayWriter, batches)), - Self::NdJson => { - println!("{}", batches_to_json!(LineDelimitedWriter, batches)) - } - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use arrow::array::Int32Array; - use arrow::datatypes::{DataType, Field, Schema}; - use datafusion::from_slice::FromSlice; - use std::sync::Arc; - - #[test] - fn test_print_batches_with_sep() { - let batches = vec![]; - assert_eq!("", print_batches_with_sep(&batches, b',').unwrap()); - - let schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Int32, false), - Field::new("b", DataType::Int32, false), - Field::new("c", DataType::Int32, false), - ])); - - let batch = RecordBatch::try_new( - schema, - vec![ - Arc::new(Int32Array::from_slice(&[1, 2, 3])), - Arc::new(Int32Array::from_slice(&[4, 5, 6])), - Arc::new(Int32Array::from_slice(&[7, 8, 9])), - ], - ) - .unwrap(); - - let batches = vec![batch]; - let r = print_batches_with_sep(&batches, b',').unwrap(); - assert_eq!("a,b,c\n1,4,7\n2,5,8\n3,6,9\n", r); - } - - #[test] - fn test_print_batches_to_json_empty() -> Result<()> { - let batches = vec![]; - let r = batches_to_json!(ArrayWriter, &batches); - assert_eq!("", r); - - let r = batches_to_json!(LineDelimitedWriter, &batches); - assert_eq!("", r); - - let schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Int32, false), - Field::new("b", DataType::Int32, false), - Field::new("c", DataType::Int32, false), - ])); - - let batch = RecordBatch::try_new( - schema, - vec![ - Arc::new(Int32Array::from_slice(&[1, 2, 3])), - Arc::new(Int32Array::from_slice(&[4, 5, 6])), - Arc::new(Int32Array::from_slice(&[7, 8, 9])), - ], - ) - .unwrap(); - - let batches = vec![batch]; - let r = batches_to_json!(ArrayWriter, &batches); - assert_eq!("[{\"a\":1,\"b\":4,\"c\":7},{\"a\":2,\"b\":5,\"c\":8},{\"a\":3,\"b\":6,\"c\":9}]", r); - - let r = batches_to_json!(LineDelimitedWriter, &batches); - assert_eq!("{\"a\":1,\"b\":4,\"c\":7}\n{\"a\":2,\"b\":5,\"c\":8}\n{\"a\":3,\"b\":6,\"c\":9}\n", r); - Ok(()) - } -} diff --git a/ballista-cli/src/print_options.rs b/ballista-cli/src/print_options.rs index 5e3792634a4e..60d0fe776184 100644 --- a/ballista-cli/src/print_options.rs +++ b/ballista-cli/src/print_options.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::print_format::PrintFormat; +use datafusion_cli::print_format::PrintFormat; use datafusion::arrow::record_batch::RecordBatch; use datafusion::error::Result; use std::time::Instant; From 31d88c0430e2775389725967772f6a3f54c3e1a2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 May 2022 09:03:16 -0600 Subject: [PATCH 04/17] re-use PrintOptions from datafusion-cli --- ballista-cli/src/command.rs | 2 +- ballista-cli/src/exec.rs | 3 ++ ballista-cli/src/lib.rs | 3 +- ballista-cli/src/main.rs | 10 +++--- ballista-cli/src/print_options.rs | 54 ------------------------------- 5 files changed, 11 insertions(+), 61 deletions(-) delete mode 100644 ballista-cli/src/print_options.rs diff --git a/ballista-cli/src/command.rs b/ballista-cli/src/command.rs index 4598f855e743..22a7c798611a 100644 --- a/ballista-cli/src/command.rs +++ b/ballista-cli/src/command.rs @@ -20,7 +20,7 @@ use crate::context::Context; use crate::functions::{display_all_functions, Function}; use datafusion_cli::print_format::PrintFormat; -use crate::print_options::PrintOptions; +use datafusion_cli::print_options::PrintOptions; use clap::ArgEnum; use datafusion::arrow::array::{ArrayRef, StringArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema}; diff --git a/ballista-cli/src/exec.rs b/ballista-cli/src/exec.rs index dc3c73e5ab43..8dadee1c3d42 100644 --- a/ballista-cli/src/exec.rs +++ b/ballista-cli/src/exec.rs @@ -21,6 +21,9 @@ use crate::{ command::{Command, OutputFormat}, context::Context, helper::CliHelper, + +}; +use datafusion_cli::{ print_options::PrintOptions, }; use datafusion::error::Result; diff --git a/ballista-cli/src/lib.rs b/ballista-cli/src/lib.rs index fb50f414c231..25f6dab9374c 100644 --- a/ballista-cli/src/lib.rs +++ b/ballista-cli/src/lib.rs @@ -16,11 +16,10 @@ // under the License. #![doc = include_str!("../README.md")] -pub const DATAFUSION_CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); +pub const BALLISTA_CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); pub mod command; pub mod context; pub mod exec; pub mod functions; pub mod helper; -pub mod print_options; diff --git a/ballista-cli/src/main.rs b/ballista-cli/src/main.rs index aa6f611a67c7..73a5549c4651 100644 --- a/ballista-cli/src/main.rs +++ b/ballista-cli/src/main.rs @@ -18,10 +18,12 @@ use clap::Parser; use datafusion::error::Result; use datafusion::execution::context::SessionConfig; -use datafusion_cli::{print_format::PrintFormat}; +use datafusion_cli::{ + print_format::PrintFormat, + print_options::PrintOptions,}; use ballista_cli::{ - context::Context, exec, print_options::PrintOptions, - DATAFUSION_CLI_VERSION, + context::Context, exec, + BALLISTA_CLI_VERSION, }; use mimalloc::MiMalloc; use std::env; @@ -91,7 +93,7 @@ pub async fn main() -> Result<()> { let args = Args::parse(); if !args.quiet { - println!("DataFusion CLI v{}", DATAFUSION_CLI_VERSION); + println!("Ballista CLI v{}", BALLISTA_CLI_VERSION); } if let Some(ref path) = args.data_path { diff --git a/ballista-cli/src/print_options.rs b/ballista-cli/src/print_options.rs deleted file mode 100644 index 60d0fe776184..000000000000 --- a/ballista-cli/src/print_options.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use datafusion_cli::print_format::PrintFormat; -use datafusion::arrow::record_batch::RecordBatch; -use datafusion::error::Result; -use std::time::Instant; - -#[derive(Debug, Clone)] -pub struct PrintOptions { - pub format: PrintFormat, - pub quiet: bool, -} - -fn print_timing_info(row_count: usize, now: Instant) { - println!( - "{} {} in set. Query took {:.3} seconds.", - row_count, - if row_count == 1 { "row" } else { "rows" }, - now.elapsed().as_secs_f64() - ); -} - -impl PrintOptions { - /// print the batches to stdout using the specified format - pub fn print_batches(&self, batches: &[RecordBatch], now: Instant) -> Result<()> { - if batches.is_empty() { - if !self.quiet { - print_timing_info(0, now); - } - } else { - self.format.print_batches(batches)?; - if !self.quiet { - let row_count: usize = batches.iter().map(|b| b.num_rows()).sum(); - print_timing_info(row_count, now); - } - } - Ok(()) - } -} From 0b549c510c55c005f94254fc2861712b43f262f4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 May 2022 09:06:41 -0600 Subject: [PATCH 05/17] re-use Helper --- ballista-cli/src/command.rs | 4 +- ballista-cli/src/exec.rs | 3 -- ballista-cli/src/helper.rs | 102 ----------------------------------- ballista-cli/src/lib.rs | 3 +- ballista-cli/src/main.rs | 11 ++-- datafusion-cli/src/helper.rs | 2 +- 6 files changed, 9 insertions(+), 116 deletions(-) delete mode 100644 ballista-cli/src/helper.rs diff --git a/ballista-cli/src/command.rs b/ballista-cli/src/command.rs index 22a7c798611a..0fd43a3071e5 100644 --- a/ballista-cli/src/command.rs +++ b/ballista-cli/src/command.rs @@ -19,8 +19,8 @@ use crate::context::Context; use crate::functions::{display_all_functions, Function}; -use datafusion_cli::print_format::PrintFormat; -use datafusion_cli::print_options::PrintOptions; +use crate::print_format::PrintFormat; +use crate::print_options::PrintOptions; use clap::ArgEnum; use datafusion::arrow::array::{ArrayRef, StringArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema}; diff --git a/ballista-cli/src/exec.rs b/ballista-cli/src/exec.rs index 8dadee1c3d42..dc3c73e5ab43 100644 --- a/ballista-cli/src/exec.rs +++ b/ballista-cli/src/exec.rs @@ -21,9 +21,6 @@ use crate::{ command::{Command, OutputFormat}, context::Context, helper::CliHelper, - -}; -use datafusion_cli::{ print_options::PrintOptions, }; use datafusion::error::Result; diff --git a/ballista-cli/src/helper.rs b/ballista-cli/src/helper.rs deleted file mode 100644 index e229255f56e3..000000000000 --- a/ballista-cli/src/helper.rs +++ /dev/null @@ -1,102 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Helper that helps with interactive editing, including multi-line parsing and validation, -//! and auto-completion for file name during creating external table. - -use datafusion::sql::parser::{DFParser, Statement}; -use rustyline::completion::Completer; -use rustyline::completion::FilenameCompleter; -use rustyline::completion::Pair; -use rustyline::error::ReadlineError; -use rustyline::highlight::Highlighter; -use rustyline::hint::Hinter; -use rustyline::validate::ValidationContext; -use rustyline::validate::ValidationResult; -use rustyline::validate::Validator; -use rustyline::Context; -use rustyline::Helper; -use rustyline::Result; - -#[derive(Default)] -pub(crate) struct CliHelper { - completer: FilenameCompleter, -} - -impl Highlighter for CliHelper {} - -impl Hinter for CliHelper { - type Hint = String; -} - -/// returns true if the current position is after the open quote for -/// creating an external table. -fn is_open_quote_for_location(line: &str, pos: usize) -> bool { - let mut sql = line[..pos].to_string(); - sql.push('\''); - if let Ok(stmts) = DFParser::parse_sql(&sql) { - if let Some(Statement::CreateExternalTable(_)) = stmts.back() { - return true; - } - } - false -} - -impl Completer for CliHelper { - type Candidate = Pair; - - fn complete( - &self, - line: &str, - pos: usize, - ctx: &Context<'_>, - ) -> std::result::Result<(usize, Vec), ReadlineError> { - if is_open_quote_for_location(line, pos) { - self.completer.complete(line, pos, ctx) - } else { - Ok((0, Vec::with_capacity(0))) - } - } -} - -impl Validator for CliHelper { - fn validate(&self, ctx: &mut ValidationContext<'_>) -> Result { - let input = ctx.input().trim_end(); - if let Some(sql) = input.strip_suffix(';') { - match DFParser::parse_sql(sql) { - Ok(statements) if statements.is_empty() => Ok(ValidationResult::Invalid( - Some(" 🤔 You entered an empty statement".to_string()), - )), - Ok(statements) if statements.len() > 1 => Ok(ValidationResult::Invalid( - Some(" 🤔 You entered more than one statement".to_string()), - )), - Ok(_statements) => Ok(ValidationResult::Valid(None)), - Err(err) => Ok(ValidationResult::Invalid(Some(format!( - " 🤔 Invalid statement: {}", - err - )))), - } - } else if input.starts_with('\\') { - // command - Ok(ValidationResult::Valid(None)) - } else { - Ok(ValidationResult::Incomplete) - } - } -} - -impl Helper for CliHelper {} diff --git a/ballista-cli/src/lib.rs b/ballista-cli/src/lib.rs index 25f6dab9374c..f3b99dd134c9 100644 --- a/ballista-cli/src/lib.rs +++ b/ballista-cli/src/lib.rs @@ -22,4 +22,5 @@ pub mod command; pub mod context; pub mod exec; pub mod functions; -pub mod helper; + +pub use datafusion_cli::{helper, print_format, print_options}; diff --git a/ballista-cli/src/main.rs b/ballista-cli/src/main.rs index 73a5549c4651..69868091de16 100644 --- a/ballista-cli/src/main.rs +++ b/ballista-cli/src/main.rs @@ -15,16 +15,13 @@ // specific language governing permissions and limitations // under the License. -use clap::Parser; -use datafusion::error::Result; -use datafusion::execution::context::SessionConfig; -use datafusion_cli::{ - print_format::PrintFormat, - print_options::PrintOptions,}; use ballista_cli::{ - context::Context, exec, + context::Context, exec, print_format::PrintFormat, print_options::PrintOptions, BALLISTA_CLI_VERSION, }; +use clap::Parser; +use datafusion::error::Result; +use datafusion::execution::context::SessionConfig; use mimalloc::MiMalloc; use std::env; use std::path::Path; diff --git a/datafusion-cli/src/helper.rs b/datafusion-cli/src/helper.rs index e229255f56e3..42eeb83f86e9 100644 --- a/datafusion-cli/src/helper.rs +++ b/datafusion-cli/src/helper.rs @@ -33,7 +33,7 @@ use rustyline::Helper; use rustyline::Result; #[derive(Default)] -pub(crate) struct CliHelper { +pub struct CliHelper { completer: FilenameCompleter, } From 3ea9404232a2d617c45f1fcb9e67245cea5b8e73 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 May 2022 09:07:31 -0600 Subject: [PATCH 06/17] re-use functions --- ballista-cli/src/functions.rs | 198 ---------------------------------- ballista-cli/src/lib.rs | 3 +- 2 files changed, 1 insertion(+), 200 deletions(-) delete mode 100644 ballista-cli/src/functions.rs diff --git a/ballista-cli/src/functions.rs b/ballista-cli/src/functions.rs deleted file mode 100644 index 98b698ab5fb6..000000000000 --- a/ballista-cli/src/functions.rs +++ /dev/null @@ -1,198 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Functions that are query-able and searchable via the `\h` command -use arrow::array::StringArray; -use arrow::datatypes::{DataType, Field, Schema}; -use arrow::record_batch::RecordBatch; -use arrow::util::pretty::pretty_format_batches; -use datafusion::error::Result; -use std::fmt; -use std::str::FromStr; -use std::sync::Arc; - -#[derive(Debug)] -pub enum Function { - Select, - Explain, - Show, - CreateTable, - CreateTableAs, - Insert, - DropTable, -} - -const ALL_FUNCTIONS: [Function; 7] = [ - Function::CreateTable, - Function::CreateTableAs, - Function::DropTable, - Function::Explain, - Function::Insert, - Function::Select, - Function::Show, -]; - -impl Function { - pub fn function_details(&self) -> Result<&str> { - let details = match self { - Function::Select => { - r#" -Command: SELECT -Description: retrieve rows from a table or view -Syntax: -SELECT [ ALL | DISTINCT [ ON ( expression [, ...] ) ] ] - [ * | expression [ [ AS ] output_name ] [, ...] ] - [ FROM from_item [, ...] ] - [ WHERE condition ] - [ GROUP BY [ ALL | DISTINCT ] grouping_element [, ...] ] - [ HAVING condition ] - [ WINDOW window_name AS ( window_definition ) [, ...] ] - [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ] - [ ORDER BY expression [ ASC | DESC | USING operator ] [ NULLS { FIRST | LAST } ] [, ...] ] - [ LIMIT { count | ALL } ] - [ OFFSET start [ ROW | ROWS ] ] - -where from_item can be one of: - - [ ONLY ] table_name [ * ] [ [ AS ] alias [ ( column_alias [, ...] ) ] ] - [ TABLESAMPLE sampling_method ( argument [, ...] ) [ REPEATABLE ( seed ) ] ] - [ LATERAL ] ( select ) [ AS ] alias [ ( column_alias [, ...] ) ] - with_query_name [ [ AS ] alias [ ( column_alias [, ...] ) ] ] - [ LATERAL ] function_name ( [ argument [, ...] ] ) - [ WITH ORDINALITY ] [ [ AS ] alias [ ( column_alias [, ...] ) ] ] - [ LATERAL ] function_name ( [ argument [, ...] ] ) [ AS ] alias ( column_definition [, ...] ) - [ LATERAL ] function_name ( [ argument [, ...] ] ) AS ( column_definition [, ...] ) - [ LATERAL ] ROWS FROM( function_name ( [ argument [, ...] ] ) [ AS ( column_definition [, ...] ) ] [, ...] ) - [ WITH ORDINALITY ] [ [ AS ] alias [ ( column_alias [, ...] ) ] ] - from_item [ NATURAL ] join_type from_item [ ON join_condition | USING ( join_column [, ...] ) [ AS join_using_alias ] ] - -and grouping_element can be one of: - - ( ) - expression - ( expression [, ...] ) - -and with_query is: - - with_query_name [ ( column_name [, ...] ) ] AS [ [ NOT ] MATERIALIZED ] ( select | values | insert | update | delete ) - -TABLE [ ONLY ] table_name [ * ]"# - } - Function::Explain => { - r#" -Command: EXPLAIN -Description: show the execution plan of a statement -Syntax: -EXPLAIN [ ANALYZE ] statement -"# - } - Function::Show => { - r#" -Command: SHOW -Description: show the value of a run-time parameter -Syntax: -SHOW name -"# - } - Function::CreateTable => { - r#" -Command: CREATE TABLE -Description: define a new table -Syntax: -CREATE [ EXTERNAL ] TABLE table_name ( [ - { column_name data_type } - [, ... ] -] ) -"# - } - Function::CreateTableAs => { - r#" -Command: CREATE TABLE AS -Description: define a new table from the results of a query -Syntax: -CREATE TABLE table_name - [ (column_name [, ...] ) ] - AS query - [ WITH [ NO ] DATA ] -"# - } - Function::Insert => { - r#" -Command: INSERT -Description: create new rows in a table -Syntax: -INSERT INTO table_name [ ( column_name [, ...] ) ] - { VALUES ( { expression } [, ...] ) [, ...] } -"# - } - Function::DropTable => { - r#" -Command: DROP TABLE -Description: remove a table -Syntax: -DROP TABLE [ IF EXISTS ] name [, ...] -"# - } - }; - Ok(details) - } -} - -impl FromStr for Function { - type Err = (); - - fn from_str(s: &str) -> std::result::Result { - Ok(match s.trim().to_uppercase().as_str() { - "SELECT" => Self::Select, - "EXPLAIN" => Self::Explain, - "SHOW" => Self::Show, - "CREATE TABLE" => Self::CreateTable, - "CREATE TABLE AS" => Self::CreateTableAs, - "INSERT" => Self::Insert, - "DROP TABLE" => Self::DropTable, - _ => return Err(()), - }) - } -} - -impl fmt::Display for Function { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - Function::Select => write!(f, "SELECT"), - Function::Explain => write!(f, "EXPLAIN"), - Function::Show => write!(f, "SHOW"), - Function::CreateTable => write!(f, "CREATE TABLE"), - Function::CreateTableAs => write!(f, "CREATE TABLE AS"), - Function::Insert => write!(f, "INSERT"), - Function::DropTable => write!(f, "DROP TABLE"), - } - } -} - -pub fn display_all_functions() -> Result<()> { - println!("Available help:"); - let array = StringArray::from( - ALL_FUNCTIONS - .iter() - .map(|f| format!("{}", f)) - .collect::>(), - ); - let schema = Schema::new(vec![Field::new("Function", DataType::Utf8, false)]); - let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)])?; - println!("{}", pretty_format_batches(&[batch]).unwrap()); - Ok(()) -} diff --git a/ballista-cli/src/lib.rs b/ballista-cli/src/lib.rs index f3b99dd134c9..a51aad013682 100644 --- a/ballista-cli/src/lib.rs +++ b/ballista-cli/src/lib.rs @@ -21,6 +21,5 @@ pub const BALLISTA_CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); pub mod command; pub mod context; pub mod exec; -pub mod functions; -pub use datafusion_cli::{helper, print_format, print_options}; +pub use datafusion_cli::{helper, print_format, print_options, functions}; From cbd96386af7571a2aae13ee1a714fa3355475434 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 May 2022 09:11:32 -0600 Subject: [PATCH 07/17] update dev scripts --- dev/docker/ballista.dockerfile | 3 +++ dev/update_ballista_versions.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dev/docker/ballista.dockerfile b/dev/docker/ballista.dockerfile index c56bfecf0b2e..b461506e7f52 100644 --- a/dev/docker/ballista.dockerfile +++ b/dev/docker/ballista.dockerfile @@ -30,6 +30,7 @@ RUN cargo install cargo-chef --version 0.1.34 FROM base as planner ADD Cargo.toml . COPY ballista ./ballista/ +COPY ballista-cli ./ballista-cli/ COPY ballista-examples ./ballista-examples/ COPY benchmarks ./benchmarks/ COPY data-access ./data-access/ @@ -44,6 +45,7 @@ RUN cargo chef cook $RELEASE_FLAG --recipe-path recipe.json FROM base as builder RUN mkdir /tmp/ballista/ballista +RUN mkdir /tmp/ballista/ballista-cli RUN mkdir /tmp/ballista/ballista-examples RUN mkdir /tmp/ballista/benchmarks RUN mkdir /tmp/ballista/data-access @@ -52,6 +54,7 @@ RUN mkdir /tmp/ballista/datafusion-cli RUN mkdir /tmp/ballista/datafusion-examples ADD Cargo.toml . COPY ballista ./ballista/ +COPY ballista-cli ./ballista-cli/ COPY ballista-examples ./ballista-examples/ COPY benchmarks ./benchmarks/ COPY data-access ./data-access/ diff --git a/dev/update_ballista_versions.py b/dev/update_ballista_versions.py index d30f027d7b97..fa2c8bc18f10 100755 --- a/dev/update_ballista_versions.py +++ b/dev/update_ballista_versions.py @@ -35,7 +35,7 @@ def update_cargo_toml(cargo_toml: str, new_version: str): data = f.read() doc = tomlkit.parse(data) - if "ballista/" in cargo_toml: + if "ballista/" in cargo_toml or "ballista-cli/" in cargo_toml: doc.get('package')['version'] = new_version # ballista crates also depend on each other @@ -77,12 +77,12 @@ def main(): ballista_crates = set([ os.path.join(repo_root, rel_path, "Cargo.toml") for rel_path in [ + 'ballista-cli', 'ballista-examples', 'ballista/rust/core', 'ballista/rust/scheduler', 'ballista/rust/executor', 'ballista/rust/client', - 'datafusion-cli', ] ]) new_version = args.new_version From 34d1d7b9ef71caa7968e16ded6d23751a053a739 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 May 2022 09:15:11 -0600 Subject: [PATCH 08/17] update diagram and docs --- dev/release/README.md | 5 +- dev/release/crate-deps.dot | 1 + dev/release/crate-deps.svg | 178 +++++++++++++++++++------------------ 3 files changed, 95 insertions(+), 89 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index 2c42ad54c762..062fe7813446 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -281,6 +281,7 @@ of the following crates: - [datafusion-proto](https://crates.io/crates/datafusion-proto) - [datafusion-row](https://crates.io/crates/datafusion-row) - [ballista](https://crates.io/crates/ballista) +- [ballista-cli](https://crates.io/crates/ballista-cli) - [ballista-core](https://crates.io/crates/ballista-core) - [ballista-executor](https://crates.io/crates/ballista-executor) - [ballista-scheduler](https://crates.io/crates/ballista-scheduler) @@ -311,9 +312,6 @@ dot -Tsvg dev/release/crate-deps.dot > dev/release/crate-deps.svg (cd datafusion-cli && cargo publish) ``` -Note that `datafusion-cli` will depend on the previously published version of Ballista (which depends on the -previously published version of DataFusion) until https://github.com/apache/arrow-datafusion/issues/2433 is resolved. - If there is a ballista release, run ```shell @@ -321,6 +319,7 @@ If there is a ballista release, run (cd ballista/rust/executor && cargo publish) (cd ballista/rust/scheduler && cargo publish) (cd ballista/rust/client && cargo publish) +(cd ballista-cli && cargo publish) ``` ### Publish datafusion-cli on Homebrew and crates.io diff --git a/dev/release/crate-deps.dot b/dev/release/crate-deps.dot index bedcd0e21bdc..5176814e35f6 100644 --- a/dev/release/crate-deps.dot +++ b/dev/release/crate-deps.dot @@ -58,6 +58,7 @@ digraph G { datafusion_cli -> datafusion ballista_cli -> datafusion + ballista_cli -> datafusion_cli ballista_cli -> ballista } diff --git a/dev/release/crate-deps.svg b/dev/release/crate-deps.svg index b409ab8976e8..eb98a00ce967 100644 --- a/dev/release/crate-deps.svg +++ b/dev/release/crate-deps.svg @@ -4,256 +4,262 @@ - + G - + datafusion_common - -datafusion_common + +datafusion_common datafusion_data_access - -datafusion_data_access + +datafusion_data_access datafusion_expr - -datafusion_expr + +datafusion_expr datafusion_expr->datafusion_common - - + + datafusion_physical_expr - -datafusion_physical_expr + +datafusion_physical_expr datafusion_physical_expr->datafusion_common - - + + datafusion_physical_expr->datafusion_expr - - + + datafusion_jit - -datafusion_jit + +datafusion_jit datafusion_jit->datafusion_common - - + + datafusion_jit->datafusion_expr - - + + datafusion_row - -datafusion_row + +datafusion_row datafusion_row->datafusion_common - - + + datafusion_row->datafusion_jit - - + + datafusion - -datafusion + +datafusion datafusion->datafusion_common - - + + datafusion->datafusion_data_access - - + + datafusion->datafusion_expr - - + + datafusion->datafusion_physical_expr - - + + datafusion->datafusion_jit - - + + datafusion->datafusion_row - - + + datafusion_proto - -datafusion_proto + +datafusion_proto datafusion_proto->datafusion - - + + ballista_core - -ballista_core + +ballista_core ballista_core->datafusion - - + + ballista_core->datafusion_proto - - + + ballista_scheduler - -ballista_scheduler + +ballista_scheduler ballista_scheduler->datafusion - - + + ballista_scheduler->ballista_core - - + + ballista_executor - -ballista_executor + +ballista_executor ballista_executor->datafusion - - + + ballista_executor->ballista_core - - + + ballista - -ballista + +ballista ballista->datafusion - - + + ballista->ballista_core - - + + ballista->ballista_scheduler - - + + ballista->ballista_executor - - + + datafusion_cli - -datafusion_cli + +datafusion_cli datafusion_cli->datafusion - - + + ballista_cli - -ballista_cli + +ballista_cli ballista_cli->datafusion - - + + - + ballista_cli->ballista - - + + + + + +ballista_cli->datafusion_cli + + From 0c99641188633401c01c1cbbd4492870fef0b4d6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 9 May 2022 09:49:05 -0600 Subject: [PATCH 09/17] stop building DataFusion CLI with ballista in CI --- .github/workflows/rust.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 056e83df0c40..40345974d069 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -75,14 +75,6 @@ jobs: env: CARGO_HOME: "/github/home/.cargo" CARGO_TARGET_DIR: "/github/home/target" - - name: Check DataFusion CLI builds with ballista - run: | - (cd datafusion-cli && cargo check) - (cd datafusion-cli && cargo check --no-default-features) - (cd datafusion-cli && cargo check --features=ballista) - env: - CARGO_HOME: "/github/home/.cargo" - CARGO_TARGET_DIR: "/github/home/target" # test the crate linux-test: From 50b764eb136f2b7d4712705fdbafe48ce7e6849c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 May 2022 08:00:55 -0600 Subject: [PATCH 10/17] update user guide --- datafusion-cli/Cargo.toml | 2 +- docs/source/cli/index.rst | 49 +++++----- docs/source/user-guide/cli.md | 2 +- .../user-guide/distributed/clients/cli.rst | 94 +++++++++++++++++++ .../user-guide/distributed/clients/index.rst | 1 + docs/source/user-guide/example-usage.md | 2 +- docs/source/user-guide/library.md | 2 +- 7 files changed, 121 insertions(+), 31 deletions(-) create mode 100644 docs/source/user-guide/distributed/clients/cli.rst diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index b880e8b95b16..08dc44fe7c2f 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-cli" -description = "Command Line Client for DataFusion query engine and Ballista distributed computation engine." +description = "Command Line Client for DataFusion query engine." version = "7.0.0" authors = ["Apache Arrow "] edition = "2021" diff --git a/docs/source/cli/index.rst b/docs/source/cli/index.rst index 89f7f5da7693..c10db36dfd63 100644 --- a/docs/source/cli/index.rst +++ b/docs/source/cli/index.rst @@ -23,10 +23,15 @@ The Arrow DataFusion CLI is a command-line interactive SQL utility that allows queries to be executed against CSV and Parquet files. It is a convenient way to try DataFusion out with your own data sources. +Install and run using Cargo +=========================== + +The easiest way to install DataFusion CLI a spin is via `cargo install datafusion-cli`. + Install and run using Homebrew (on MacOS) ========================================= -The easiest way to give DataFusion CLI a spin is via Homebrew (on MacOS). Install it as any other pre-built software like this: +DataFusion CLI can also be installed via Homebrew (on MacOS). Install it as any other pre-built software like this: .. code-block:: bash @@ -41,26 +46,19 @@ The easiest way to give DataFusion CLI a spin is via Homebrew (on MacOS). Instal datafusion-cli -Run using Cargo -=============== - -Use the following commands to clone this repository and run the CLI. This will require the Rust toolchain to be installed. Rust can be installed from `https://rustup.rs `_. - -.. code-block:: bash - - git clone https://github.com/apache/arrow-datafusion - cd arrow-datafusion/datafusion-cli - cargo run --release - Run using Docker ================ +There is no officially published Docker image for the DataFusion CLI, so it is necessary to build from source +instead. + Use the following commands to clone this repository and build a Docker image containing the CLI tool. Note that there is :code:`.dockerignore` file in the root of the repository that may need to be deleted in order for this to work. .. code-block:: bash git clone https://github.com/apache/arrow-datafusion + git checkout 8.0.0 cd arrow-datafusion docker build -f datafusion-cli/Dockerfile . --tag datafusion-cli docker run -it -v $(your_data_location):/data datafusion-cli @@ -71,25 +69,22 @@ Usage .. code-block:: bash - DataFusion 5.1.0-SNAPSHOT - DataFusion is an in-memory query engine that uses Apache Arrow as the memory model. It supports executing SQL queries - against CSV and Parquet files as well as querying directly against in-memory data. + Apache Arrow + Command Line Client for DataFusion query engine and Ballista distributed computation engine. USAGE: - datafusion-cli [FLAGS] [OPTIONS] - - FLAGS: - -h, --help Prints help information - -q, --quiet Reduce printing other than the results and work quietly - -V, --version Prints version information + datafusion-cli [OPTIONS] OPTIONS: - -c, --batch-size The batch size of each query, or use DataFusion default - -p, --data-path Path to your data, default to current directory - -f, --file ... Execute commands from file(s), then exit - --format Output format [default: table] [possible values: csv, tsv, table, json, ndjson] - --host Ballista scheduler host - --port Ballista scheduler port + -c, --batch-size The batch size of each query, or use DataFusion default + -f, --file ... Execute commands from file(s), then exit + --format [default: table] [possible values: csv, tsv, table, json, + nd-json] + -h, --help Print help information + -p, --data-path Path to your data, default to current directory + -q, --quiet Reduce printing other than the results and work quietly + -r, --rc ... Run the provided files on startup instead of ~/.datafusionrc + -V, --version Print version information Type `exit` or `quit` to exit the CLI. diff --git a/docs/source/user-guide/cli.md b/docs/source/user-guide/cli.md index 9561b5e1d1c6..2ec9c8d49937 100644 --- a/docs/source/user-guide/cli.md +++ b/docs/source/user-guide/cli.md @@ -51,7 +51,7 @@ $ echo "1,2" > data.csv ```bash $ datafusion-cli -DataFusion CLI v5.1.0-SNAPSHOT +DataFusion CLI v8.0.0 > CREATE EXTERNAL TABLE foo (a INT, b INT) STORED AS CSV LOCATION 'data.csv'; 0 rows in set. Query took 0.001 seconds. diff --git a/docs/source/user-guide/distributed/clients/cli.rst b/docs/source/user-guide/distributed/clients/cli.rst new file mode 100644 index 000000000000..6a94bb0c537c --- /dev/null +++ b/docs/source/user-guide/distributed/clients/cli.rst @@ -0,0 +1,94 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +======================= +Ballista Command-line +======================= + +The Arrow Ballista CLI is a command-line interactive SQL utility that allows +queries to be executed against CSV and Parquet files. It is a convenient way to +try Ballista out with your own data sources. + +Install and run using Cargo +=========================== + +The easiest way to install Ballista CLI a spin is via `cargo install ballista-cli`. + +Usage +===== + +.. code-block:: bash + + Apache Arrow + Command Line Client for Ballista distributed query engine. + + USAGE: + ballista-cli [OPTIONS] + + OPTIONS: + -c, --batch-size The batch size of each query, or use DataFusion default + -f, --file ... Execute commands from file(s), then exit + --format [default: table] [possible values: csv, tsv, table, json, + nd-json] + -h, --help Print help information + --host Ballista scheduler host + -p, --data-path Path to your data, default to current directory + --port Ballista scheduler port + -q, --quiet Reduce printing other than the results and work quietly + -r, --rc ... Run the provided files on startup instead of ~/.datafusionrc + -V, --version Print version information + +Type `exit` or `quit` to exit the CLI. + + +Registering Parquet Data Sources +================================ + +Parquet data sources can be registered by executing a :code:`CREATE EXTERNAL TABLE` SQL statement. It is not necessary to provide schema information for Parquet files. + +.. code-block:: sql + + CREATE EXTERNAL TABLE taxi + STORED AS PARQUET + LOCATION '/mnt/nyctaxi/tripdata.parquet'; + + +Registering CSV Data Sources +============================ + +CSV data sources can be registered by executing a :code:`CREATE EXTERNAL TABLE` SQL statement. It is necessary to provide schema information for CSV files since DataFusion does not automatically infer the schema when using SQL to query CSV files. + +.. code-block:: sql + + CREATE EXTERNAL TABLE test ( + c1 VARCHAR NOT NULL, + c2 INT NOT NULL, + c3 SMALLINT NOT NULL, + c4 SMALLINT NOT NULL, + c5 INT NOT NULL, + c6 BIGINT NOT NULL, + c7 SMALLINT NOT NULL, + c8 INT NOT NULL, + c9 BIGINT NOT NULL, + c10 VARCHAR NOT NULL, + c11 FLOAT NOT NULL, + c12 DOUBLE NOT NULL, + c13 VARCHAR NOT NULL + ) + STORED AS CSV + WITH HEADER ROW + LOCATION '/path/to/aggregate_test_100.csv'; diff --git a/docs/source/user-guide/distributed/clients/index.rst b/docs/source/user-guide/distributed/clients/index.rst index c9eb1e1f524b..6199bca5e74b 100644 --- a/docs/source/user-guide/distributed/clients/index.rst +++ b/docs/source/user-guide/distributed/clients/index.rst @@ -21,5 +21,6 @@ Clients .. toctree:: :maxdepth: 2 + cli rust python diff --git a/docs/source/user-guide/example-usage.md b/docs/source/user-guide/example-usage.md index 6be802400a77..de8cae295579 100644 --- a/docs/source/user-guide/example-usage.md +++ b/docs/source/user-guide/example-usage.md @@ -24,7 +24,7 @@ Add the following to your `Cargo.toml` file: ```toml -datafusion = "7.0.0" +datafusion = "8.0.0" tokio = "1.0" ``` diff --git a/docs/source/user-guide/library.md b/docs/source/user-guide/library.md index cb9ca48e78f8..422c9d6d1ec4 100644 --- a/docs/source/user-guide/library.md +++ b/docs/source/user-guide/library.md @@ -44,7 +44,7 @@ To get started, add the following to your `Cargo.toml` file: ```toml [dependencies] -datafusion = "7.0.0" +datafusion = "8.0.0" ``` ## Create a main function From edd2ee00a3cdb47b7008a69c8f9b24a91f67bc2f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 May 2022 08:11:05 -0600 Subject: [PATCH 11/17] docs for building ballista-cli with docker --- .../user-guide/distributed/clients/cli.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/source/user-guide/distributed/clients/cli.rst b/docs/source/user-guide/distributed/clients/cli.rst index 6a94bb0c537c..d5cf30b8a024 100644 --- a/docs/source/user-guide/distributed/clients/cli.rst +++ b/docs/source/user-guide/distributed/clients/cli.rst @@ -28,6 +28,23 @@ Install and run using Cargo The easiest way to install Ballista CLI a spin is via `cargo install ballista-cli`. +Run using Docker +================ + +There is no officially published Docker image for the Ballista CLI, so it is necessary to build from source +instead. + +Use the following commands to clone this repository and build a Docker image containing the CLI tool. Note that there is :code:`.dockerignore` file in the root of the repository that may need to be deleted in order for this to work. + +.. code-block:: bash + + git clone https://github.com/apache/arrow-datafusion + git checkout 8.0.0 + cd arrow-datafusion + docker build -f ballista-cli/Dockerfile . --tag ballista-cli + docker run -it -v $(your_data_location):/data ballista-cli + + Usage ===== From 7660867b1f1709a5e473c2b938dd7ba32fbc5329 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 10 May 2022 08:14:36 -0600 Subject: [PATCH 12/17] make version numbers consistent with repo --- ballista-cli/README.md | 2 +- datafusion-cli/README.md | 2 +- docs/source/cli/index.rst | 2 +- docs/source/user-guide/cli.md | 2 +- docs/source/user-guide/distributed/clients/cli.rst | 2 +- docs/source/user-guide/example-usage.md | 2 +- docs/source/user-guide/library.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ballista-cli/README.md b/ballista-cli/README.md index ea47c5ed5f07..733329e0cc1d 100644 --- a/ballista-cli/README.md +++ b/ballista-cli/README.md @@ -53,7 +53,7 @@ $ echo "1,2" > data.csv ```sql,ignore $ ballista-cli -Ballista CLI v4.0.0-SNAPSHOT +Ballista CLI v0.6.0 > CREATE EXTERNAL TABLE foo (a INT, b INT) STORED AS CSV LOCATION 'data.csv'; 0 rows in set. Query took 0.001 seconds. diff --git a/datafusion-cli/README.md b/datafusion-cli/README.md index 5c72f16a45d4..2b21aa2b80a5 100644 --- a/datafusion-cli/README.md +++ b/datafusion-cli/README.md @@ -53,7 +53,7 @@ $ echo "1,2" > data.csv ```sql,ignore $ datafusion-cli -DataFusion CLI v4.0.0-SNAPSHOT +DataFusion CLI v7.0.0 > CREATE EXTERNAL TABLE foo (a INT, b INT) STORED AS CSV LOCATION 'data.csv'; 0 rows in set. Query took 0.001 seconds. diff --git a/docs/source/cli/index.rst b/docs/source/cli/index.rst index c10db36dfd63..bc22bf6c8348 100644 --- a/docs/source/cli/index.rst +++ b/docs/source/cli/index.rst @@ -58,7 +58,7 @@ Use the following commands to clone this repository and build a Docker image con .. code-block:: bash git clone https://github.com/apache/arrow-datafusion - git checkout 8.0.0 + git checkout 7.0.0 cd arrow-datafusion docker build -f datafusion-cli/Dockerfile . --tag datafusion-cli docker run -it -v $(your_data_location):/data datafusion-cli diff --git a/docs/source/user-guide/cli.md b/docs/source/user-guide/cli.md index 2ec9c8d49937..ec38aad9a92a 100644 --- a/docs/source/user-guide/cli.md +++ b/docs/source/user-guide/cli.md @@ -51,7 +51,7 @@ $ echo "1,2" > data.csv ```bash $ datafusion-cli -DataFusion CLI v8.0.0 +DataFusion CLI v7.0.0 > CREATE EXTERNAL TABLE foo (a INT, b INT) STORED AS CSV LOCATION 'data.csv'; 0 rows in set. Query took 0.001 seconds. diff --git a/docs/source/user-guide/distributed/clients/cli.rst b/docs/source/user-guide/distributed/clients/cli.rst index d5cf30b8a024..71518508222b 100644 --- a/docs/source/user-guide/distributed/clients/cli.rst +++ b/docs/source/user-guide/distributed/clients/cli.rst @@ -39,7 +39,7 @@ Use the following commands to clone this repository and build a Docker image con .. code-block:: bash git clone https://github.com/apache/arrow-datafusion - git checkout 8.0.0 + git checkout 7.0.0 cd arrow-datafusion docker build -f ballista-cli/Dockerfile . --tag ballista-cli docker run -it -v $(your_data_location):/data ballista-cli diff --git a/docs/source/user-guide/example-usage.md b/docs/source/user-guide/example-usage.md index de8cae295579..6be802400a77 100644 --- a/docs/source/user-guide/example-usage.md +++ b/docs/source/user-guide/example-usage.md @@ -24,7 +24,7 @@ Add the following to your `Cargo.toml` file: ```toml -datafusion = "8.0.0" +datafusion = "7.0.0" tokio = "1.0" ``` diff --git a/docs/source/user-guide/library.md b/docs/source/user-guide/library.md index 422c9d6d1ec4..cb9ca48e78f8 100644 --- a/docs/source/user-guide/library.md +++ b/docs/source/user-guide/library.md @@ -44,7 +44,7 @@ To get started, add the following to your `Cargo.toml` file: ```toml [dependencies] -datafusion = "8.0.0" +datafusion = "7.0.0" ``` ## Create a main function From e214e5c43ebdea117c52cac5ac4e2c4dccb7968f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 11 May 2022 10:23:35 -0600 Subject: [PATCH 13/17] update Cargo.lock files for CLIs and add ballista-cli to GitHub workflow --- .github/workflows/rust.yml | 2 + .gitignore | 1 + ballista-cli/Cargo.lock | 2557 ++++++++++++++++++++++++++++++++++++ datafusion-cli/Cargo.lock | 360 +---- 4 files changed, 2568 insertions(+), 352 deletions(-) create mode 100644 ballista-cli/Cargo.lock diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 40345974d069..fe56b621c53b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -201,6 +201,8 @@ jobs: POSTGRES_PASSWORD: postgres - name: Build datafusion-cli run: (cd datafusion-cli && cargo build) + - name: Build ballista-cli + run: (cd ballista-cli && cargo build) - name: Test Psql Parity run: python -m pytest -v integration-tests/test_psql_parity.py env: diff --git a/.gitignore b/.gitignore index 23d609710108..1cfeb1f0b623 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,7 @@ cpp/Brewfile.lock.json target Cargo.lock !datafusion-cli/Cargo.lock +!ballista-cli/Cargo.lock rusty-tags.vi .history diff --git a/ballista-cli/Cargo.lock b/ballista-cli/Cargo.lock new file mode 100644 index 000000000000..528148c9b43f --- /dev/null +++ b/ballista-cli/Cargo.lock @@ -0,0 +1,2557 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "anyhow" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] +name = "arrow" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6bee230122beb516ead31935a61f683715f987c6f003eff44ad6986624105a" +dependencies = [ + "bitflags", + "chrono", + "comfy-table", + "csv", + "flatbuffers", + "half", + "hex", + "indexmap", + "lazy_static", + "lexical-core", + "multiversion", + "num", + "rand", + "regex", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "arrow-flight" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a3666d2dbc637fa979d1f0bf3031d39a80e709f3b9ec88e3d573c1d666bf553" +dependencies = [ + "arrow", + "base64", + "bytes", + "futures", + "proc-macro2", + "prost", + "prost-derive", + "tokio", + "tonic", + "tonic-build", +] + +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "axum" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4af7447fc1214c1f3a1ace861d0216a6c8bb13965b64bbad9650f375b67689a" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa 1.0.1", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da31c0ed7b4690e2c78fe4b880d21cd7db04a346ebc658b4270251b695437f17" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", +] + +[[package]] +name = "ballista" +version = "0.6.0" +dependencies = [ + "ballista-core", + "datafusion", + "futures", + "log", + "parking_lot", + "sqlparser", + "tempfile", + "tokio", +] + +[[package]] +name = "ballista-cli" +version = "0.6.0" +dependencies = [ + "arrow", + "ballista", + "clap", + "datafusion", + "datafusion-cli", + "dirs", + "env_logger", + "mimalloc", + "rustyline", + "tokio", +] + +[[package]] +name = "ballista-core" +version = "0.6.0" +dependencies = [ + "ahash", + "arrow-flight", + "async-trait", + "chrono", + "clap", + "datafusion", + "datafusion-proto", + "futures", + "hashbrown 0.12.1", + "libloading", + "log", + "once_cell", + "parking_lot", + "parse_arg", + "prost", + "prost-types", + "rustc_version", + "serde", + "sqlparser", + "tokio", + "tonic", + "tonic-build", + "uuid", + "walkdir", +] + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "blake2" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cf849ee05b2ee5fba5e36f97ff8ec2533916700fc0758d40d92136a42f3388" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "digest", +] + +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "winapi", +] + +[[package]] +name = "clap" +version = "3.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a35a599b11c089a7f49105658d089b8f2cf0882993c17daf6de15285c2c35d" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap", + "lazy_static", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "clipboard-win" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3e1238132dc01f081e1cbb9dace14e5ef4c3a51ee244bd982275fb514605db" +dependencies = [ + "error-code", + "str-buf", + "winapi", +] + +[[package]] +name = "cmake" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" +dependencies = [ + "cc", +] + +[[package]] +name = "comfy-table" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b103d85ca6e209388771bfb7aa6b68a7aeec4afbf6f0a0264bfbf50360e5212e" +dependencies = [ + "strum", + "strum_macros", + "unicode-width", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "cpufeatures" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "datafusion" +version = "7.0.0" +dependencies = [ + "ahash", + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-data-access", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-row", + "futures", + "hashbrown 0.12.1", + "lazy_static", + "log", + "num_cpus", + "ordered-float 3.0.0", + "parking_lot", + "parquet", + "paste", + "pin-project-lite", + "rand", + "smallvec", + "sqlparser", + "tempfile", + "tokio", + "tokio-stream", + "uuid", +] + +[[package]] +name = "datafusion-cli" +version = "7.0.0" +dependencies = [ + "arrow", + "clap", + "datafusion", + "dirs", + "env_logger", + "mimalloc", + "rustyline", + "tokio", +] + +[[package]] +name = "datafusion-common" +version = "7.0.0" +dependencies = [ + "arrow", + "ordered-float 3.0.0", + "parquet", + "sqlparser", +] + +[[package]] +name = "datafusion-data-access" +version = "7.0.0" +dependencies = [ + "async-trait", + "chrono", + "futures", + "glob", + "parking_lot", + "tempfile", + "tokio", +] + +[[package]] +name = "datafusion-expr" +version = "7.0.0" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "sqlparser", +] + +[[package]] +name = "datafusion-physical-expr" +version = "7.0.0" +dependencies = [ + "ahash", + "arrow", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-row", + "hashbrown 0.12.1", + "lazy_static", + "md-5", + "ordered-float 3.0.0", + "paste", + "rand", + "regex", + "sha2", + "unicode-segmentation", +] + +[[package]] +name = "datafusion-proto" +version = "7.0.0" +dependencies = [ + "datafusion", + "prost", + "tonic-build", +] + +[[package]] +name = "datafusion-row" +version = "7.0.0" +dependencies = [ + "arrow", + "datafusion-common", + "paste", + "rand", +] + +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + +[[package]] +name = "env_logger" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "error-code" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +dependencies = [ + "libc", + "str-buf", +] + +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + +[[package]] +name = "fd-lock" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e245f4c8ec30c6415c56cb132c07e69e74f1942f6b4a4061da748b49f486ca" +dependencies = [ + "cfg-if", + "rustix", + "windows-sys 0.30.0", +] + +[[package]] +name = "fixedbitset" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" + +[[package]] +name = "flatbuffers" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b428b715fdbdd1c364b84573b5fdc0f84f8e423661b9f398735278bc7f2b6a" +dependencies = [ + "bitflags", + "smallvec", + "thiserror", +] + +[[package]] +name = "flate2" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" + +[[package]] +name = "futures-executor" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" + +[[package]] +name = "futures-task" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" + +[[package]] +name = "futures-util" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "hashbrown" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" +dependencies = [ + "ahash", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.1", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "httparse" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa 1.0.1", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "indexmap" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +dependencies = [ + "autocfg", + "hashbrown 0.11.2", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "integer-encoding" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" + +[[package]] +name = "io-lifetimes" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9448015e586b611e5d322f6703812bbca2f1e709d5773ecd38ddb4e3bb649504" + +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "jobserver" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +dependencies = [ + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lexical-core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92912c4af2e7d9075be3e5e3122c4d7263855fa6cce34fbece4dd08e5884624d" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f518eed87c3be6debe6d26b855c97358d8a11bf05acec137e5f53080f5ad2dd8" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc852ec67c6538bbb2b9911116a385b24510e879a69ab516e6a151b15a79168" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c72a9d52c5c4e62fa2cdc2cb6c694a39ae1382d9c2a17a466f18e272a0930eb1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a89ec1d062e481210c309b672f73a0567b7855f21e7d2fae636df44d12e97f9" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "094060bd2a7c2ff3a16d5304a6ae82727cb3cc9d1c70f813cc73f744c319337e" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" + +[[package]] +name = "libloading" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "libmimalloc-sys" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ca136052550448f55df7898c6dbe651c6b574fe38a0d9ea687a9f8088a2e2c" +dependencies = [ + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" + +[[package]] +name = "lock_api" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "lz4" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4edcb94251b1c375c459e5abe9fb0168c1c826c3370172684844f8f3f8d1a885" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7be8908e2ed6f31c02db8a9fa962f03e36c53fbfde437363eae3306b85d7e17" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "matchit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" + +[[package]] +name = "md-5" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" +dependencies = [ + "digest", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mimalloc" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f64ad83c969af2e732e907564deb0d0ed393cec4af80776f77dd77a1a427698" +dependencies = [ + "libmimalloc-sys", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "miniz_oxide" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "multiversion" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025c962a3dd3cc5e0e520aa9c612201d127dcdf28616974961a649dca64f5373" +dependencies = [ + "multiversion-macros", +] + +[[package]] +name = "multiversion-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + +[[package]] +name = "nix" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +dependencies = [ + "bitflags", + "cc", + "cfg-if", + "libc", + "memoffset", +] + +[[package]] +name = "ntapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +dependencies = [ + "winapi", +] + +[[package]] +name = "num" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fbc387afefefd5e9e39493299f3069e14a140dd34dc19b4c1c1a8fddb6a790" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" + +[[package]] +name = "ordered-float" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bcbab4bfea7a59c2c0fe47211a1ac4e3e96bea6eb446d704f310bc5c732ae2" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os_str_bytes" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.36.1", +] + +[[package]] +name = "parquet" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c6d737baed48775e87a69aa262f1fa2f1d6bd074dedbe9cac244b9aabf2a0b4" +dependencies = [ + "arrow", + "base64", + "brotli", + "byteorder", + "chrono", + "flate2", + "lz4", + "num", + "num-bigint", + "parquet-format", + "rand", + "snap", + "thrift", + "zstd", +] + +[[package]] +name = "parquet-format" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f0c06cdcd5460967c485f9c40a821746f5955ad81990533c7fae95dbd9bc0b5" +dependencies = [ + "thrift", +] + +[[package]] +name = "parse_arg" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14248cc8eced350e20122a291613de29e4fa129ba2731818c4cdbb44fccd3e55" + +[[package]] +name = "paste" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "petgraph" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "prettyplease" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "prost" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a07b0857a71a8cb765763950499cae2413c3f9cede1133478c43600d9e146890" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120fbe7988713f39d780a58cf1a7ef0d7ef66c6d87e5aa3438940c05357929f4" +dependencies = [ + "bytes", + "cfg-if", + "cmake", + "heck 0.4.0", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" +dependencies = [ + "bytes", + "prost", +] + +[[package]] +name = "quote" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.34.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3e74b3f02f2b6eb33790923756784614f456de79d821d6b2670dc7d5fbea807" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "winapi", +] + +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + +[[package]] +name = "rustyline" +version = "9.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db7826789c0e25614b03e5a54a0717a86f9ff6e6e5247f92b369472869320039" +dependencies = [ + "bitflags", + "cfg-if", + "clipboard-win", + "dirs-next", + "fd-lock", + "libc", + "log", + "memchr", + "nix", + "radix_trie", + "scopeguard", + "smallvec", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "winapi", +] + +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" + +[[package]] +name = "serde" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f972498cf015f7c0746cac89ebe1d6ef10c293b94175a243a2d9442c163d9944" +dependencies = [ + "indexmap", + "itoa 1.0.1", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "slab" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" + +[[package]] +name = "smallvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" + +[[package]] +name = "snap" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" + +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "sqlparser" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc2739f3a9bfc68e2f7b7695589f6cb0181c88af73ceaee0c84215cd2a2ae28" +dependencies = [ + "log", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "str-buf" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" + +[[package]] +name = "strum_macros" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" + +[[package]] +name = "thiserror" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6d965454947cc7266d22716ebfd07b18d84ebaf35eec558586bbb2a8cb6b5b" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float 1.1.1", + "threadpool", +] + +[[package]] +name = "tokio" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc" +dependencies = [ + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "socket2", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-stream" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30fb54bf1e446f44d870d260d99957e7d11fb9d0a0f5bd1a662ad1411cc103f9" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03447cdc9eaf8feffb6412dcb27baf2db11669a6c4789f29da799aabfb99547" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e980386f06883cf4d0578d6c9178c81f68b45d77d00f2c2c1bc034b3439c2c56" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-segmentation" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" + +[[package]] +name = "utf8parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" + +[[package]] +name = "uuid" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfcd319456c4d6ea10087ed423473267e1a071f3bc0aa89f80d60997843c6f0" +dependencies = [ + "getrandom", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +dependencies = [ + "either", + "lazy_static", + "libc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030b7ff91626e57a05ca64a07c481973cbb2db774e4852c9c7ca342408c6a99a" +dependencies = [ + "windows_aarch64_msvc 0.30.0", + "windows_i686_gnu 0.30.0", + "windows_i686_msvc 0.30.0", + "windows_x86_64_gnu 0.30.0", + "windows_x86_64_msvc 0.30.0", +] + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29277a4435d642f775f63c7d1faeb927adba532886ce0287bd985bffb16b6bca" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145e1989da93956c68d1864f32fb97c8f561a8f89a5125f6a2b7ea75524e4b8" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a09e3a0d4753b73019db171c1339cd4362c8c44baf1bcea336235e955954a6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca64fcb0220d58db4c119e050e7af03c69e6f4f415ef69ec1773d9aab422d5a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08cabc9f0066848fef4bc6a1c1668e6efce38b661d2aeec75d18d8617eebb5f1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "zstd" +version = "0.11.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a16b8414fde0414e90c612eba70985577451c4c504b99885ebed24762cb81a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c12659121420dd6365c5c3de4901f97145b79651fb1d25814020ed2ed0585ae" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +dependencies = [ + "cc", + "libc", +] diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index c1011c789983..4309a4216b1c 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -108,93 +108,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "axum" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f1e8a972137fad81e2a1a60b86ff17ce0338f8017264e45a9723d0083c39a1" -dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa 1.0.1", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde", - "sync_wrapper", - "tokio", - "tower", - "tower-http", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da31c0ed7b4690e2c78fe4b880d21cd7db04a346ebc658b4270251b695437f17" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", -] - -[[package]] -name = "ballista" -version = "0.6.0" -dependencies = [ - "ballista-core", - "datafusion", - "futures", - "log", - "parking_lot", - "sqlparser 0.16.0", - "tempfile", - "tokio", -] - -[[package]] -name = "ballista-core" -version = "0.6.0" -dependencies = [ - "ahash", - "arrow-flight", - "async-trait", - "chrono", - "clap", - "datafusion", - "datafusion-proto", - "futures", - "hashbrown 0.12.1", - "libloading", - "log", - "once_cell", - "parking_lot", - "parse_arg", - "prost", - "prost-types", - "rustc_version", - "serde", - "sqlparser 0.16.0", - "tokio", - "tonic", - "tonic-build", - "uuid", - "walkdir", -] - [[package]] name = "base64" version = "0.13.0" @@ -307,9 +220,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.17" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47582c09be7c8b32c0ab3a6181825ababb713fde6fff20fc573a3870dd45c6a0" +checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" dependencies = [ "atty", "bitflags", @@ -324,9 +237,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.1.7" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" +checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -447,7 +360,7 @@ dependencies = [ "pin-project-lite", "rand", "smallvec", - "sqlparser 0.17.0", + "sqlparser", "tempfile", "tokio", "tokio-stream", @@ -475,12 +388,12 @@ dependencies = [ "arrow", "ordered-float 3.0.0", "parquet", - "sqlparser 0.17.0", + "sqlparser", ] [[package]] name = "datafusion-data-access" -version = "1.0.0" +version = "7.0.0" dependencies = [ "async-trait", "chrono", @@ -498,7 +411,7 @@ dependencies = [ "ahash", "arrow", "datafusion-common", - "sqlparser 0.17.0", + "sqlparser", ] [[package]] @@ -534,16 +447,6 @@ dependencies = [ "rand", ] -[[package]] -name = "datafusion-row" -version = "7.0.0" -dependencies = [ - "arrow", - "datafusion-common", - "paste", - "rand", -] - [[package]] name = "digest" version = "0.10.3" @@ -805,12 +708,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" -[[package]] -name = "h2" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" - [[package]] name = "half" version = "1.8.2" @@ -1092,24 +989,6 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" -dependencies = [ - "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.36.1", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - [[package]] name = "multiversion" version = "0.6.1" @@ -1379,61 +1258,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "prost" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc03e116981ff7d8da8e5c220e374587b98d294af7ba7dd7fda761158f00086f" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65a1118354442de7feb8a2a76f3d80ef01426bd45542c8c1fdffca41a758f846" -dependencies = [ - "bytes", - "cfg-if", - "cmake", - "heck 0.4.0", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prost", - "prost-types", - "regex", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" -dependencies = [ - "bytes", - "prost", -] - [[package]] name = "quote" version = "1.0.18" @@ -1649,15 +1473,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" -[[package]] -name = "sqlparser" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9a527b68048eb95495a1508f6c8395c8defcff5ecdbe8ad4106d08a2ef2a3c" -dependencies = [ - "log", -] - [[package]] name = "sqlparser" version = "0.17.0" @@ -1827,165 +1642,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-util" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "tonic" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be9d60db39854b30b835107500cf0aca0b0d14d6e1c3de124217c23a29c2ddb" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "prost-derive", - "tokio", - "tokio-stream", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - -[[package]] -name = "tonic-build" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9263bf4c9bfaae7317c1c2faf7f18491d2fe476f70c414b73bf5d445b00ffa1" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn", -] - -[[package]] -name = "tower" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" -dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" - -[[package]] -name = "tower-service" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" - -[[package]] -name = "tracing" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" -dependencies = [ - "cfg-if", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - -[[package]] -name = "try-lock" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" - [[package]] name = "typenum" version = "1.15.0" From a7956718b8b6ce29571385ad10dc1029b1c16c4d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 May 2022 06:52:33 -0600 Subject: [PATCH 14/17] disable ballista tests --- .github/workflows/rust.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fe56b621c53b..98c0ff1c3ba8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -127,18 +127,18 @@ jobs: CARGO_HOME: "/github/home/.cargo" CARGO_TARGET_DIR: "/github/home/target" # Ballista is currently not part of the main workspace so requires a separate test step - - name: Run Ballista tests - run: | - export ARROW_TEST_DATA=$(pwd)/testing/data - export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data - cd ballista/rust - # snmalloc requires cmake so build without default features - cargo test --no-default-features --features sled - # Ensure also compiles in standalone mode - cargo test --no-default-features --features standalone - env: - CARGO_HOME: "/github/home/.cargo" - CARGO_TARGET_DIR: "/github/home/target" +# - name: Run Ballista tests +# run: | +# export ARROW_TEST_DATA=$(pwd)/testing/data +# export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data +# cd ballista/rust +# # snmalloc requires cmake so build without default features +# cargo test --no-default-features --features sled +# # Ensure also compiles in standalone mode +# cargo test --no-default-features --features standalone +# env: +# CARGO_HOME: "/github/home/.cargo" +# CARGO_TARGET_DIR: "/github/home/target" integration-test: name: "Integration Test" From f79e6ee6fc49860e084e9b17908531cd1ba05bfe Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 May 2022 07:02:55 -0600 Subject: [PATCH 15/17] fix ci --- .github/workflows/rust.yml | 72 ++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 98c0ff1c3ba8..ab5a24b152a3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -75,6 +75,14 @@ jobs: env: CARGO_HOME: "/github/home/.cargo" CARGO_TARGET_DIR: "/github/home/target" + - name: Check DataFusion CLI builds with ballista + run: | + (cd datafusion-cli && cargo check) + (cd datafusion-cli && cargo check --no-default-features) + (cd datafusion-cli && cargo check --features=ballista) + env: + CARGO_HOME: "/github/home/.cargo" + CARGO_TARGET_DIR: "/github/home/target" # test the crate linux-test: @@ -115,9 +123,9 @@ jobs: run: | export ARROW_TEST_DATA=$(pwd)/testing/data export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data + cargo test --no-default-features # test datafusion examples cd datafusion-examples - cargo test --no-default-features cargo run --example csv_sql cargo run --example parquet_sql cargo run --example avro_sql --features=datafusion/avro @@ -126,19 +134,55 @@ jobs: env: CARGO_HOME: "/github/home/.cargo" CARGO_TARGET_DIR: "/github/home/target" + + # test the crate + ballista-test: + name: Test Ballista on AMD64 Rust ${{ matrix.rust }} + needs: [linux-build-lib] + runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64] + rust: [stable] + container: + image: ${{ matrix.arch }}/rust + env: + # Disable full debug symbol generation to speed up CI build and keep memory down + # "1" means line tables only, which is useful for panic tracebacks. + RUSTFLAGS: "-C debuginfo=1" + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: /github/home/.cargo + # this key equals the ones on `linux-build-lib` for re-use + key: cargo-cache- + - name: Cache Rust dependencies + uses: actions/cache@v2 + with: + path: /github/home/target + # this key equals the ones on `linux-build-lib` for re-use + key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }} + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ matrix.rust }} # Ballista is currently not part of the main workspace so requires a separate test step -# - name: Run Ballista tests -# run: | -# export ARROW_TEST_DATA=$(pwd)/testing/data -# export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data -# cd ballista/rust -# # snmalloc requires cmake so build without default features -# cargo test --no-default-features --features sled -# # Ensure also compiles in standalone mode -# cargo test --no-default-features --features standalone -# env: -# CARGO_HOME: "/github/home/.cargo" -# CARGO_TARGET_DIR: "/github/home/target" + - name: Run Ballista tests + run: | + export ARROW_TEST_DATA=$(pwd)/testing/data + export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data + cd ballista/rust + # snmalloc requires cmake so build without default features + cargo test --no-default-features --features sled + # Ensure also compiles in standalone mode + cargo test --no-default-features --features standalone + env: + CARGO_HOME: "/github/home/.cargo" + CARGO_TARGET_DIR: "/github/home/target" integration-test: name: "Integration Test" @@ -201,8 +245,6 @@ jobs: POSTGRES_PASSWORD: postgres - name: Build datafusion-cli run: (cd datafusion-cli && cargo build) - - name: Build ballista-cli - run: (cd ballista-cli && cargo build) - name: Test Psql Parity run: python -m pytest -v integration-tests/test_psql_parity.py env: From 4a4fa40323c2630d04e4c565e327c19accca80c2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 May 2022 07:15:29 -0600 Subject: [PATCH 16/17] fix merge conflict --- .github/workflows/rust.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ab5a24b152a3..83d1e97bba4f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -75,14 +75,6 @@ jobs: env: CARGO_HOME: "/github/home/.cargo" CARGO_TARGET_DIR: "/github/home/target" - - name: Check DataFusion CLI builds with ballista - run: | - (cd datafusion-cli && cargo check) - (cd datafusion-cli && cargo check --no-default-features) - (cd datafusion-cli && cargo check --features=ballista) - env: - CARGO_HOME: "/github/home/.cargo" - CARGO_TARGET_DIR: "/github/home/target" # test the crate linux-test: @@ -245,6 +237,8 @@ jobs: POSTGRES_PASSWORD: postgres - name: Build datafusion-cli run: (cd datafusion-cli && cargo build) + - name: Build ballista-cli + run: (cd ballista-cli && cargo build) - name: Test Psql Parity run: python -m pytest -v integration-tests/test_psql_parity.py env: From a396d47c49ffa24a5b27ac7d4607a5793bb68e78 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 May 2022 07:33:40 -0600 Subject: [PATCH 17/17] fix --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 83d1e97bba4f..4a4e32f90a16 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -115,7 +115,7 @@ jobs: run: | export ARROW_TEST_DATA=$(pwd)/testing/data export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data - cargo test --no-default-features + cargo test # test datafusion examples cd datafusion-examples cargo run --example csv_sql @@ -127,7 +127,7 @@ jobs: CARGO_HOME: "/github/home/.cargo" CARGO_TARGET_DIR: "/github/home/target" - # test the crate + # run ballista tests ballista-test: name: Test Ballista on AMD64 Rust ${{ matrix.rust }} needs: [linux-build-lib]