Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: list indices always shows vector index type is IVF_PQ even it's not #3258

Merged
merged 1 commit into from
Dec 17, 2024
Merged

fix: list indices always shows vector index type is IVF_PQ even it's not #3258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions python/python/tests/test_vector_index.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,37 @@ def test_has_index(dataset, tmp_path):
assert ann_ds.list_indices()[0]["fields"] == ["vector"]


def test_index_type(dataset, tmp_path):
ann_ds = lance.write_dataset(dataset.to_table(), tmp_path / "indexed.lance")

ann_ds = ann_ds.create_index(
"vector",
index_type="IVF_PQ",
num_partitions=4,
num_sub_vectors=16,
replace=True,
)
assert ann_ds.list_indices()[0]["type"] == "IVF_PQ"

ann_ds = ann_ds.create_index(
"vector",
index_type="IVF_HNSW_SQ",
num_partitions=4,
num_sub_vectors=16,
replace=True,
)
assert ann_ds.list_indices()[0]["type"] == "IVF_HNSW_SQ"

ann_ds = ann_ds.create_index(
"vector",
index_type="IVF_HNSW_PQ",
num_partitions=4,
num_sub_vectors=16,
replace=True,
)
assert ann_ds.list_indices()[0]["type"] == "IVF_HNSW_PQ"


def test_create_dot_index(dataset, tmp_path):
assert not dataset.has_index
ann_ds = lance.write_dataset(dataset.to_table(), tmp_path / "indexed.lance")
Expand Down
22 changes: 7 additions & 15 deletions python/src/dataset.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,23 +568,15 @@ impl Dataset {

let idx_schema = schema.project_by_ids(idx.fields.as_slice(), true);

let is_vector = idx_schema
.fields
.iter()
.any(|f| matches!(f.data_type(), DataType::FixedSizeList(_, _)));

let idx_type = if is_vector {
IndexType::Vector
} else {
let ds = self_.ds.clone();
RT.block_on(Some(self_.py()), async {
let scalar_idx = ds
.open_scalar_index(&idx_schema.fields[0].name, &idx.uuid.to_string())
let ds = self_.ds.clone();
let idx_type = RT
.block_on(Some(self_.py()), async {
let idx = ds
.open_generic_index(&idx_schema.fields[0].name, &idx.uuid.to_string())
.await?;
Ok::<_, lance::Error>(scalar_idx.index_type())
Ok::<_, lance::Error>(idx.index_type())
})?
.map_err(|e| PyIOError::new_err(e.to_string()))?
};
.map_err(|e| PyIOError::new_err(e.to_string()))?;

let field_names = idx_schema
.fields
Expand Down
Loading