Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

list terms #2740

Merged
merged 11 commits into from
Feb 8, 2023
Merged

list terms #2740

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
34f1dc8
update tantivy
trinity-1686a Jan 18, 2023
4db2e6d
add mock test on jaeger::get_service
trinity-1686a Jan 24, 2023
cccbaa2
add rpc endpoint to list terms
trinity-1686a Jan 24, 2023
cad7519
add internal support for list terms
trinity-1686a Jan 30, 2023
41e24a5
add some error handling
trinity-1686a Jan 31, 2023
3eca214
fix documentation nits
trinity-1686a Jan 31, 2023
9d39ff5
allow list terms to return every results
trinity-1686a Jan 31, 2023
e4e21b4
dedup terms
trinity-1686a Feb 6, 2023
64a7405
fix typo and remove commented code
trinity-1686a Feb 8, 2023
af092ca
clippy
trinity-1686a Feb 8, 2023
78ea837
Merge branch 'main' into trinity--list-terms
trinity-1686a Feb 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions quickwit/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions quickwit/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,14 +209,14 @@ quickwit-serve = { version = "0.4.0", path = "./quickwit-serve" }
quickwit-storage = { version = "0.4.0", path = "./quickwit-storage" }
quickwit-telemetry = { version = "0.4.0", path = "./quickwit-telemetry" }

fastfield_codecs = { git = "https://github.com/quickwit-oss/tantivy/", rev = "c9cb3d0" }
tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "c9cb3d0", default-features = false, features = [
fastfield_codecs = { git = "https://github.com/quickwit-oss/tantivy/", rev = "d72ea7d" }
tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "d72ea7d", default-features = false, features = [
"mmap",
"lz4-compression",
"zstd-compression",
"quickwit",
] }
tantivy-query-grammar = { git = "https://github.com/quickwit-oss/tantivy/", rev = "c9cb3d0" }
tantivy-query-grammar = { git = "https://github.com/quickwit-oss/tantivy/", rev = "d72ea7d" }

# This is actually not used directly the goal is to fix the version
# used by reqwest. 0.8.30 has an unclear license.
Expand Down
4 changes: 1 addition & 3 deletions quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,7 @@ impl DefaultDocMapper {

fn validate_tag_fields(tag_fields: &[String], schema: &Schema) -> anyhow::Result<()> {
for tag_field in tag_fields {
let field = schema
.get_field(tag_field)
.ok_or_else(|| anyhow::anyhow!("Tag field `{}` does not exist.", tag_field))?;
let field = schema.get_field(tag_field)?;
let field_type = schema.get_field_entry(field).field_type();
match field_type {
FieldType::Str(options) => {
Expand Down
2 changes: 1 addition & 1 deletion quickwit/quickwit-doc-mapper/src/doc_mapper.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ pub trait DocMapper: Send + Sync + Debug + DynClone + 'static {
/// over time. So `split_schema` is the schema of the split being operated on.
fn timestamp_field(&self, split_schema: &Schema) -> Option<Field> {
self.timestamp_field_name()
.and_then(|field_name| split_schema.get_field(&field_name))
.and_then(|field_name| split_schema.get_field(&field_name).ok())
}

/// Returns the timestamp field name.
Expand Down
11 changes: 5 additions & 6 deletions quickwit/quickwit-doc-mapper/src/fast_field_reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,14 @@ pub fn timestamp_field_reader(
let timestamp_field_entry = schema.get_field_entry(timestamp_field);
let fast_field_readers = segment_reader.fast_fields();
let field_schema_type = timestamp_field_entry.field_type().value_type();
let field_name = timestamp_field_entry.name();
let timestamp_field_reader = match field_schema_type {
Type::I64 => GenericFastFieldReader::I64(fast_field_readers.i64(timestamp_field)?),
Type::Date => GenericFastFieldReader::Date(fast_field_readers.date(timestamp_field)?),
Type::I64 => GenericFastFieldReader::I64(fast_field_readers.i64(field_name)?),
Type::Date => GenericFastFieldReader::Date(fast_field_readers.date(field_name)?),
_ => {
return Err(TantivyError::SchemaError(format!(
"Failed to build timestamp filter for field `{:?}`: expected I64 or Date type, \
got `{:?}`.",
timestamp_field_entry.name(),
field_schema_type
"Failed to build timestamp filter for field `{field_name:?}`: expected I64 or \
Date type, got `{field_schema_type:?}`."
)))
}
};
Expand Down
33 changes: 15 additions & 18 deletions quickwit/quickwit-doc-mapper/src/query_builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,7 @@ pub(crate) fn build_query(
fn resolve_fields(schema: &Schema, field_names: &[String]) -> anyhow::Result<Vec<Field>> {
let mut fields = vec![];
for field_name in field_names {
let field = schema
.get_field(field_name)
.ok_or_else(|| TantivyQueryParserError::FieldDoesNotExist(field_name.clone()))?;
let field = schema.get_field(field_name)?;
fields.push(field);
}
Ok(fields)
Expand Down Expand Up @@ -231,8 +229,7 @@ fn validate_requested_snippet_fields(

let field_entry = schema
.get_field(field_name)
.map(|field| schema.get_field_entry(field))
.ok_or_else(|| anyhow::anyhow!("The snippet field `{}` does not exist.", field_name))?;
.map(|field| schema.get_field_entry(field))?;
match field_entry.field_type() {
FieldType::Str(text_options) => {
if !text_options.is_stored() {
Expand Down Expand Up @@ -455,7 +452,7 @@ mod test {
"title:foo desc:bar",
vec!["url".to_string()],
None,
TestExpectation::Err("Field does not exist: 'url'"),
TestExpectation::Err("field does not exist: 'url'"),
)
.unwrap();
check_build_query(
Expand Down Expand Up @@ -522,14 +519,14 @@ mod test {
"dt:[2023-01-10T15:13:35Z TO 2023-01-10T15:13:40Z]",
Vec::new(),
None,
TestExpectation::Ok("RangeQuery { field: Field(10), value_type: Date"),
TestExpectation::Ok("RangeQuery { field: \"dt\", value_type: Date"),
)
.unwrap();
check_build_query(
"dt:<2023-01-10T15:13:35Z",
Vec::new(),
None,
TestExpectation::Ok("RangeQuery { field: Field(10), value_type: Date"),
TestExpectation::Ok("RangeQuery { field: \"dt\", value_type: Date"),
)
.unwrap();
}
Expand All @@ -541,7 +538,7 @@ mod test {
Vec::new(),
None,
TestExpectation::Ok(
"RangeQuery { field: Field(7), value_type: IpAddr, left_bound: Included([0, 0, 0, \
"RangeQuery { field: \"ip\", value_type: IpAddr, left_bound: Included([0, 0, 0, \
0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 0, 0, 1]), right_bound: Included([0, 0, 0, \
0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 1, 1, 1]) }",
),
Expand All @@ -552,7 +549,7 @@ mod test {
Vec::new(),
None,
TestExpectation::Ok(
"RangeQuery { field: Field(7), value_type: IpAddr, left_bound: Excluded([0, 0, 0, \
"RangeQuery { field: \"ip\", value_type: IpAddr, left_bound: Excluded([0, 0, 0, \
0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 0, 0, 1]), right_bound: Unbounded }",
),
)
Expand All @@ -565,14 +562,14 @@ mod test {
"f64_fast:[7.7 TO 77.7]",
Vec::new(),
None,
TestExpectation::Ok("RangeQuery { field: Field(13), value_type: F64"),
TestExpectation::Ok("RangeQuery { field: \"f64_fast\", value_type: F64"),
)
.unwrap();
check_build_query(
"f64_fast:>7",
Vec::new(),
None,
TestExpectation::Ok("RangeQuery { field: Field(13), value_type: F64"),
TestExpectation::Ok("RangeQuery { field: \"f64_fast\", value_type: F64"),
)
.unwrap();
}
Expand All @@ -583,14 +580,14 @@ mod test {
"i64_fast:[-7 TO 77]",
Vec::new(),
None,
TestExpectation::Ok("RangeQuery { field: Field(12), value_type: I64"),
TestExpectation::Ok("RangeQuery { field: \"i64_fast\", value_type: I64"),
)
.unwrap();
check_build_query(
"i64_fast:>7",
Vec::new(),
None,
TestExpectation::Ok("RangeQuery { field: Field(12), value_type: I64"),
TestExpectation::Ok("RangeQuery { field: \"i64_fast\", value_type: I64"),
)
.unwrap();
}
Expand All @@ -601,14 +598,14 @@ mod test {
"u64_fast:[7 TO 77]",
Vec::new(),
None,
TestExpectation::Ok("RangeQuery { field: Field(11), value_type: U64"),
TestExpectation::Ok("RangeQuery { field: \"u64_fast\", value_type: U64"),
)
.unwrap();
check_build_query(
"u64_fast:>7",
Vec::new(),
None,
TestExpectation::Ok("RangeQuery { field: Field(11), value_type: U64"),
TestExpectation::Ok("RangeQuery { field: \"u64_fast\", value_type: U64"),
)
.unwrap();
}
Expand All @@ -620,7 +617,7 @@ mod test {
Vec::new(),
None,
TestExpectation::Ok(
"RangeQuery { field: Field(8), value_type: IpAddr, left_bound: Included([0, 0, 0, \
"RangeQuery { field: \"ips\", value_type: IpAddr, left_bound: Included([0, 0, 0, \
0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 0, 0, 1]), right_bound: Included([0, 0, 0, \
0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 1, 1, 1]) }",
),
Expand Down Expand Up @@ -717,7 +714,7 @@ mod test {
);
assert_eq!(
validation_result.unwrap_err().to_string(),
"The snippet field `summary` does not exist."
"The field does not exist: 'summary'"
);
// Unknown searched field
let validation_result =
Expand Down
9 changes: 2 additions & 7 deletions quickwit/quickwit-indexing/src/actors/merge_executor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ use quickwit_metastore::{Metastore, SplitMetadata};
use quickwit_proto::metastore_api::DeleteTask;
use quickwit_proto::SearchRequest;
use tantivy::directory::{DirectoryClone, MmapDirectory, RamDirectory};
use tantivy::{Directory, Index, IndexMeta, SegmentId, SegmentReader, TantivyError};
use tantivy::{Directory, Index, IndexMeta, SegmentId, SegmentReader};
use tokio::runtime::Handle;
use tracing::{debug, info, instrument, warn};

Expand Down Expand Up @@ -391,12 +391,7 @@ impl MergeExecutor {
if let Some(ref timestamp_field_name) = self.doc_mapper.timestamp_field_name() {
let timestamp_field = merged_segment_reader
.schema()
.get_field(timestamp_field_name)
.ok_or_else(|| {
TantivyError::SchemaError(format!(
"Timestamp field `{timestamp_field_name}` does not exist"
))
})?;
.get_field(timestamp_field_name)?;
let reader = timestamp_field_reader(timestamp_field, &merged_segment_reader)?;
Some(reader.min_value()..=reader.max_value())
} else {
Expand Down
Loading