Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(risedev): support thread sanitizer #1173

Merged
merged 1 commit into from
Mar 23, 2022
Merged

feat(risedev): support thread sanitizer #1173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 54 additions & 15 deletions Makefile.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,27 @@ env_files = [
"risedev-components.user.env"
]


env_scripts = [
'''
#!@duckscript

is_sanitizer_enabled = get_env ENABLE_SANITIZER

if ${is_sanitizer_enabled}
set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "-Zbuild-std --target ${CARGO_MAKE_RUST_TARGET_TRIPLE}"
set_env RISEDEV_BUILD_TARGET_DIR "${CARGO_MAKE_RUST_TARGET_TRIPLE}/"
Comment on lines +23 to +24
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why specify the target explicitly?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as required by thread sanitizer...

set_env RISEDEV_RUSTFLAGS "-Zsanitizer=thread"
else
set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS ""
set_env RISEDEV_BUILD_TARGET_DIR ""
set_env RISEDEV_RUSTFLAGS ""
end
'''
]

[env]
RISEDEV_CARGO_BUILD_FLAGS = { value = "--features=all-in-one", condition = { env_set = ["ENABLE_ALL_IN_ONE"] } }
RISEDEV_CARGO_BUILD_FEATURES = { value = "all-in-one", condition = { env_set = ["ENABLE_ALL_IN_ONE"] } }

[config]
skip_core_tasks = true
Expand Down Expand Up @@ -86,9 +105,9 @@ set -e
rm -f "${PREFIX_BIN}/compute-node"
rm -f "${PREFIX_BIN}/meta-node"
rm -f "${PREFIX_BIN}/frontend-v2"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/compute-node" "${PREFIX_BIN}/compute-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/meta-node" "${PREFIX_BIN}/meta-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/frontend-v2" "${PREFIX_BIN}/frontend-v2"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/compute-node" "${PREFIX_BIN}/compute-node"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/meta-node" "${PREFIX_BIN}/meta-node"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/frontend-v2" "${PREFIX_BIN}/frontend-v2"
'''

[tasks.link-all-in-one-binaries]
Expand All @@ -101,11 +120,11 @@ set -e

rm -rf "${PREFIX_BIN}/risingwave"
mkdir -p "${PREFIX_BIN}/risingwave"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/meta-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/compute-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/frontend-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/risectl"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/playground"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/meta-node"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/compute-node"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/frontend-node"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/risectl"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/playground"
'''

[tasks.link-user-bin]
Expand All @@ -118,11 +137,11 @@ set -e

rm -rf "${PREFIX_USR_BIN}"
mkdir -p "${PREFIX_USR_BIN}"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/meta-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/compute-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/frontend-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/risectl"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/playground"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/meta-node"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/compute-node"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/frontend-node"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/risectl"
ln -s "$(pwd)/rust/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/playground"
'''

[tasks.post-build-risingwave]
Expand Down Expand Up @@ -169,7 +188,25 @@ script = '''

set -e
echo "**Reminder**: risedev will only build risingwave_cmd and risedev crates"
cd rust && cargo build -p risingwave_cmd -p risedev --profile "${RISINGWAVE_BUILD_PROFILE}" ${RISEDEV_CARGO_BUILD_FLAGS}
cd rust

export RUSTFLAGS="${RISEDEV_RUSTFLAGS}"
echo + RUSTFLAGS="$RUSTFLAGS"
set -xe
cargo build -p risingwave_cmd -p risedev \
--profile "${RISINGWAVE_BUILD_PROFILE}" \
${RISEDEV_CARGO_BUILD_EXTRA_ARGS} \
--features=${RISEDEV_CARGO_BUILD_FEATURES}
'''

[tasks.clean]
category = "RiseDev - Build"
description = "Clean Rust targets"
condition = { env_set = [ "ENABLE_BUILD_RUST" ] }
script = '''
#!@shell
set -e
cd rust && cargo clean
'''

[tasks.build-docs]
Expand Down Expand Up @@ -484,7 +521,9 @@ cd "$DIR"
./risedev "\\$@"
EOF
chmod +x "${INSTALL_PATH}"

echo RiseDev installed to $(tput setaf 4)${INSTALL_PATH}$(tput sgr0)
echo If you still feel this is not enough, you may copy $(tput setaf 4)risedev$(tput sgr0) to something like $(tput setaf 4)r+$(tput sgr0).
"""

[tasks.ci-start]
Expand Down
189 changes: 94 additions & 95 deletions rust/batch/src/executor/order_by.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,14 +245,11 @@ mod tests {
use std::sync::Arc;

use itertools::Itertools;
use rand::distributions::{Alphanumeric, Standard, Uniform};
use rand::Rng;
use risingwave_common::array::column::Column;
use risingwave_common::array::{BoolArray, DataChunk, PrimitiveArray, Utf8Array};
use risingwave_common::catalog::{Field, Schema};
use risingwave_common::types::{DataType, OrderedF32, OrderedF64};
use risingwave_common::util::sort_util::OrderType;
use test::Bencher;

use super::*;
use crate::executor::test_utils::MockExecutor;
Expand Down Expand Up @@ -455,98 +452,100 @@ mod tests {
}
}

fn benchmark_1e4(b: &mut Bencher, enable_encoding: bool) {
// gen random vec for i16 float bool and str
let scale = 10000;
let width = 10;
let int_vec: Vec<Option<i16>> = rand::thread_rng()
.sample_iter(Standard)
.take(scale)
.map(Some)
.collect_vec();
let bool_vec: Vec<Option<bool>> = rand::thread_rng()
.sample_iter(Standard)
.take(scale)
.map(Some)
.collect_vec();
let float_vec: Vec<Option<f32>> = rand::thread_rng()
.sample_iter(Standard)
.take(scale)
.map(Some)
.collect_vec();
let mut str_vec = Vec::<Option<String>>::new();
for _ in 0..scale {
let len = rand::thread_rng().sample(Uniform::<usize>::new(1, width));
let s: String = rand::thread_rng()
.sample_iter(&Alphanumeric)
.take(len)
.map(char::from)
.collect();
str_vec.push(Some(s));
}
b.iter(|| {
let col0 = create_column_i16(int_vec.as_slice()).unwrap();
let col1 = create_column_bool(bool_vec.as_slice()).unwrap();
let col2 = create_column_f32(float_vec.as_slice()).unwrap();
let col3 = create_column_string(str_vec.as_slice()).unwrap();
let data_chunk = DataChunk::builder()
.columns([col0, col1, col2, col3].to_vec())
.build();
let schema = Schema {
fields: vec![
Field::unnamed(DataType::Int16),
Field::unnamed(DataType::Boolean),
Field::unnamed(DataType::Float32),
Field::unnamed(DataType::Varchar),
],
};
let mut mock_executor = MockExecutor::new(schema);
mock_executor.add(data_chunk);
let order_pairs = vec![
OrderPair {
column_idx: 1,
order_type: OrderType::Ascending,
},
OrderPair {
column_idx: 0,
order_type: OrderType::Descending,
},
OrderPair {
column_idx: 3,
order_type: OrderType::Descending,
},
OrderPair {
column_idx: 2,
order_type: OrderType::Ascending,
},
];
let mut order_by_executor = OrderByExecutor {
order_pairs: Arc::new(order_pairs),
child: Box::new(mock_executor),
vis_indices: vec![],
chunks: vec![],
sorted_indices: vec![],
min_heap: BinaryHeap::new(),
encoded_keys: vec![],
encodable: false,
disable_encoding: !enable_encoding,
identity: "OrderByExecutor".to_string(),
};
let future = order_by_executor.open();
tokio_test::block_on(future).unwrap();
let future = order_by_executor.next();
let res = tokio_test::block_on(future).unwrap();
assert!(matches!(res, Some(_)));
});
}
// TODO: enable benches

#[bench]
fn benchmark_bsc_1e4(b: &mut Bencher) {
benchmark_1e4(b, true);
}
// fn benchmark_1e4(b: &mut Bencher, enable_encoding: bool) {
// // gen random vec for i16 float bool and str
// let scale = 10000;
// let width = 10;
// let int_vec: Vec<Option<i16>> = rand::thread_rng()
// .sample_iter(Standard)
// .take(scale)
// .map(Some)
// .collect_vec();
// let bool_vec: Vec<Option<bool>> = rand::thread_rng()
// .sample_iter(Standard)
// .take(scale)
// .map(Some)
// .collect_vec();
// let float_vec: Vec<Option<f32>> = rand::thread_rng()
// .sample_iter(Standard)
// .take(scale)
// .map(Some)
// .collect_vec();
// let mut str_vec = Vec::<Option<String>>::new();
// for _ in 0..scale {
// let len = rand::thread_rng().sample(Uniform::<usize>::new(1, width));
// let s: String = rand::thread_rng()
// .sample_iter(&Alphanumeric)
// .take(len)
// .map(char::from)
// .collect();
// str_vec.push(Some(s));
// }
// b.iter(|| {
// let col0 = create_column_i16(int_vec.as_slice()).unwrap();
// let col1 = create_column_bool(bool_vec.as_slice()).unwrap();
// let col2 = create_column_f32(float_vec.as_slice()).unwrap();
// let col3 = create_column_string(str_vec.as_slice()).unwrap();
// let data_chunk = DataChunk::builder()
// .columns([col0, col1, col2, col3].to_vec())
// .build();
// let schema = Schema {
// fields: vec![
// Field::unnamed(DataType::Int16),
// Field::unnamed(DataType::Boolean),
// Field::unnamed(DataType::Float32),
// Field::unnamed(DataType::Varchar),
// ],
// };
// let mut mock_executor = MockExecutor::new(schema);
// mock_executor.add(data_chunk);
// let order_pairs = vec![
// OrderPair {
// column_idx: 1,
// order_type: OrderType::Ascending,
// },
// OrderPair {
// column_idx: 0,
// order_type: OrderType::Descending,
// },
// OrderPair {
// column_idx: 3,
// order_type: OrderType::Descending,
// },
// OrderPair {
// column_idx: 2,
// order_type: OrderType::Ascending,
// },
// ];
// let mut order_by_executor = OrderByExecutor {
// order_pairs: Arc::new(order_pairs),
// child: Box::new(mock_executor),
// vis_indices: vec![],
// chunks: vec![],
// sorted_indices: vec![],
// min_heap: BinaryHeap::new(),
// encoded_keys: vec![],
// encodable: false,
// disable_encoding: !enable_encoding,
// identity: "OrderByExecutor".to_string(),
// };
// let future = order_by_executor.open();
// tokio_test::block_on(future).unwrap();
// let future = order_by_executor.next();
// let res = tokio_test::block_on(future).unwrap();
// assert!(matches!(res, Some(_)));
// });
// }

#[bench]
fn benchmark_baseline_1e4(b: &mut Bencher) {
benchmark_1e4(b, false);
}
// #[bench]
// fn benchmark_bsc_1e4(b: &mut Bencher) {
// benchmark_1e4(b, true);
// }

// #[bench]
// fn benchmark_baseline_1e4(b: &mut Bencher) {
// benchmark_1e4(b, false);
// }
}
2 changes: 0 additions & 2 deletions rust/batch/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
#![feature(trait_alias)]
#![feature(generic_associated_types)]
#![feature(binary_heap_drain_sorted)]
#![feature(test)]
#![feature(map_first_last)]
#![feature(let_chains)]
#![cfg_attr(coverage, feature(no_coverage))]
Expand All @@ -39,4 +38,3 @@ pub mod task;
extern crate log;
#[macro_use]
extern crate risingwave_common;
extern crate test;
2 changes: 0 additions & 2 deletions rust/compute/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,11 @@
#![feature(trait_alias)]
#![feature(generic_associated_types)]
#![feature(binary_heap_drain_sorted)]
#![feature(test)]
#![feature(map_first_last)]
#![cfg_attr(coverage, feature(no_coverage))]

#[macro_use]
extern crate log;
extern crate test;

pub mod rpc;
pub mod server;
Expand Down
13 changes: 13 additions & 0 deletions rust/risedevtool/src/bin/risedev-config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ pub enum Components {
Dashboard,
Release,
AllInOne,
Sanitizer,
}

impl Components {
Expand All @@ -48,6 +49,7 @@ impl Components {
Self::Tracing => "[Component] Tracing: Jaeger",
Self::Release => "[Build] Enable release mode",
Self::AllInOne => "[Build] Enable all-in-one binary",
Self::Sanitizer => "[Build] Enable sanitizer",
}
.into()
}
Expand Down Expand Up @@ -97,6 +99,15 @@ With this option enabled, RiseDev will help you create
symlinks to `risingwave` all-in-one binary, so as to build
and use `risingwave` in all-in-one mode."
}
Self::Sanitizer => {
"
With this option enabled, RiseDev will build Rust components
with thread sanitizer. The built binaries will be at
`target/<arch-triple>/(debug|release)` instead of simply at
`target/debug`. RiseDev will help link binaries when starting
a dev cluster.
"
}
}
.into()
}
Expand All @@ -111,6 +122,7 @@ and use `risingwave` in all-in-one mode."
"ENABLE_COMPUTE_TRACING" => Some(Self::Tracing),
"ENABLE_RELEASE_PROFILE" => Some(Self::Release),
"ENABLE_ALL_IN_ONE" => Some(Self::AllInOne),
"ENABLE_SANITIZER" => Some(Self::Sanitizer),
_ => None,
}
}
Expand All @@ -126,6 +138,7 @@ and use `risingwave` in all-in-one mode."
Self::Tracing => "ENABLE_COMPUTE_TRACING",
Self::Release => "ENABLE_RELEASE_PROFILE",
Self::AllInOne => "ENABLE_ALL_IN_ONE",
Self::Sanitizer => "ENABLE_SANITIZER",
}
.into()
}
Expand Down
Loading