From 196ec06b898c24136be55cb003190434d64df176 Mon Sep 17 00:00:00 2001 From: BubbleCal Date: Tue, 26 Nov 2024 00:09:44 +0800 Subject: [PATCH] fix: panic if all docs are deleted in a posting list (#3163) --- python/python/tests/test_scalar_index.py | 7 +++++++ rust/lance-index/src/scalar/inverted/wand.rs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py index 6793b218db..2dad325f96 100644 --- a/python/python/tests/test_scalar_index.py +++ b/python/python/tests/test_scalar_index.py @@ -300,6 +300,13 @@ def test_fts_with_other_str_scalar_index(dataset): assert dataset.to_table(full_text_query=query).num_rows > 0 +def test_fts_all_deleted(dataset): + dataset.create_scalar_index("doc", index_type="INVERTED", with_position=False) + first_row_doc = dataset.take(indices=[0], columns=["doc"]).column(0)[0].as_py() + dataset.delete(f"doc = '{first_row_doc}'") + dataset.to_table(full_text_query=first_row_doc) + + def test_bitmap_index(tmp_path: Path): """Test create bitmap index""" tbl = pa.Table.from_arrays( diff --git a/rust/lance-index/src/scalar/inverted/wand.rs b/rust/lance-index/src/scalar/inverted/wand.rs index 2901d0e900..d5e1631671 100644 --- a/rust/lance-index/src/scalar/inverted/wand.rs +++ b/rust/lance-index/src/scalar/inverted/wand.rs @@ -127,7 +127,7 @@ impl Wand { threshold: 0.0, cur_doc: None, num_docs, - postings: postings.collect(), + postings: postings.filter(|posting| posting.doc().is_some()).collect(), candidates: BinaryHeap::new(), } }