Skip to content

Commit

Permalink
update tantivy
Browse files Browse the repository at this point in the history
  • Loading branch information
@PSeitz
PSeitz committed Jul 28, 2024
1 parent b6db60d commit 33ddbd5
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 42 deletions.
18 changes: 9 additions & 9 deletions quickwit/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion quickwit/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ quickwit-serve = { path = "quickwit-serve" }
quickwit-storage = { path = "quickwit-storage" }
quickwit-telemetry = { path = "quickwit-telemetry" }

tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "13e9885", default-features = false, features = [
tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "7ebcc15", default-features = false, features = [
"lz4-compression",
"mmap",
"quickwit",
Expand Down
83 changes: 51 additions & 32 deletions quickwit/quickwit-query/src/query_ast/range_query.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,10 @@
use std::ops::Bound;

use serde::{Deserialize, Serialize};
use tantivy::query::{
FastFieldRangeWeight as TantivyFastFieldRangeQuery, RangeQuery as TantivyRangeQuery,
};
use tantivy::fastfield::FastValue;
use tantivy::query::RangeQuery as TantivyRangeQuery;
use tantivy::schema::Schema as TantivySchema;
use tantivy::DateTime;
use tantivy::{DateTime, Term};

use super::QueryAst;
use crate::json_literal::InterpretUserInput;
Expand Down Expand Up @@ -165,7 +164,9 @@ fn compute_numerical_boundaries(

/// Converts a given bound JsonLiteral bound into a bound of type T.
fn convert_bound<'a, T>(bound: &'a Bound<JsonLiteral>) -> Option<Bound<T>>
where T: InterpretUserInput<'a> {
where
T: InterpretUserInput<'a>,
{
match bound {
Bound::Included(val) => {
let val = T::interpret_json(val)?;
Expand Down Expand Up @@ -228,7 +229,7 @@ impl BuildTantivyAst for RangeQuery {
_search_fields: &[String],
_with_validation: bool,
) -> Result<TantivyQueryAst, InvalidQuery> {
let (_field, field_entry, _path) =
let (field, field_entry, json_path) =
super::utils::find_field_or_hit_dynamic(&self.field, schema)?;
if !field_entry.is_fast() {
return Err(InvalidQuery::SchemaError(format!(
Expand All @@ -246,20 +247,29 @@ impl BuildTantivyAst for RangeQuery {
tantivy::schema::FieldType::U64(_) => {
let (lower_bound, upper_bound) =
convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?;
TantivyFastFieldRangeQuery::new::<u64>(self.field.clone(), lower_bound, upper_bound)
.into()
TantivyRangeQuery::new(
lower_bound.map(|val| Term::from_field_u64(field, val)),
upper_bound.map(|val| Term::from_field_u64(field, val)),
)
.into()
}
tantivy::schema::FieldType::I64(_) => {
let (lower_bound, upper_bound) =
convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?;
TantivyFastFieldRangeQuery::new::<i64>(self.field.clone(), lower_bound, upper_bound)
.into()
TantivyRangeQuery::new(
lower_bound.map(|val| Term::from_field_i64(field, val)),
upper_bound.map(|val| Term::from_field_i64(field, val)),
)
.into()
}
tantivy::schema::FieldType::F64(_) => {
let (lower_bound, upper_bound) =
convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?;
TantivyFastFieldRangeQuery::new::<f64>(self.field.clone(), lower_bound, upper_bound)
.into()
TantivyRangeQuery::new(
lower_bound.map(|val| Term::from_field_f64(field, val)),
upper_bound.map(|val| Term::from_field_f64(field, val)),
)
.into()
}
tantivy::schema::FieldType::Bool(_) => {
return Err(InvalidQuery::RangeQueryNotSupportedForField {
Expand All @@ -272,12 +282,11 @@ impl BuildTantivyAst for RangeQuery {
convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?;
let truncate_datetime =
|date: &DateTime| date.truncate(date_options.get_precision());
let truncated_lower_bound = map_bound(&lower_bound, truncate_datetime);
let truncated_upper_bound = map_bound(&upper_bound, truncate_datetime);
TantivyFastFieldRangeQuery::new::<DateTime>(
self.field.clone(),
truncated_lower_bound,
truncated_upper_bound,
let lower_bound = map_bound(&lower_bound, truncate_datetime);
let upper_bound = map_bound(&upper_bound, truncate_datetime);
TantivyRangeQuery::new(
lower_bound.map(|val| Term::from_field_date(field, val)),
upper_bound.map(|val| Term::from_field_date(field, val)),
)
.into()
}
Expand All @@ -288,8 +297,14 @@ impl BuildTantivyAst for RangeQuery {
});
}
tantivy::schema::FieldType::Bytes(_) => todo!(),
tantivy::schema::FieldType::JsonObject(_) => {
let full_path = self.field.clone();
tantivy::schema::FieldType::JsonObject(opt) => {
let empty_term =
Term::from_field_json_path(field, json_path, opt.is_expand_dots_enabled());
fn term_with_val<T: FastValue>(term: &Term, val: T) -> Term {
let mut term = term.clone();
term.append_type_and_fast_value(val);
term
}
let mut sub_queries: Vec<TantivyQueryAst> = Vec::new();
if let Some(NumericalBoundaries {
i64_range,
Expand All @@ -299,29 +314,30 @@ impl BuildTantivyAst for RangeQuery {
{
// Adding the f64 range.
sub_queries.push(
TantivyFastFieldRangeQuery::new(
full_path.clone(),
f64_range.0,
f64_range.1,
TantivyRangeQuery::new(
f64_range.0.map(|val| term_with_val(&empty_term, val)),
f64_range.1.map(|val| term_with_val(&empty_term, val)),
)
.into(),
);
// Adding the i64 range.
if !is_empty(&i64_range).unwrap_or(false) {
sub_queries.push(
TantivyFastFieldRangeQuery::new(
full_path.clone(),
i64_range.0,
i64_range.1,
TantivyRangeQuery::new(
i64_range.0.map(|val| term_with_val(&empty_term, val)),
i64_range.1.map(|val| term_with_val(&empty_term, val)),
)
.into(),
);
}
// Adding the u64 range.
if !is_empty(&u64_range).unwrap_or(false) {
sub_queries.push(
TantivyFastFieldRangeQuery::new(full_path, u64_range.0, u64_range.1)
.into(),
TantivyRangeQuery::new(
u64_range.0.map(|val| term_with_val(&empty_term, val)),
u64_range.1.map(|val| term_with_val(&empty_term, val)),
)
.into(),
);
}
}
Expand All @@ -335,8 +351,11 @@ impl BuildTantivyAst for RangeQuery {
tantivy::schema::FieldType::IpAddr(_) => {
let (lower_bound, upper_bound) =
convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?;
TantivyRangeQuery::new_ip_bounds(self.field.clone(), lower_bound, upper_bound)
.into()
TantivyRangeQuery::new(
lower_bound.map(|val| Term::from_field_ip_addr(field, val)),
upper_bound.map(|val| Term::from_field_ip_addr(field, val)),
)
.into()
}
})
}
Expand Down

0 comments on commit 33ddbd5

Please sign in to comment.