Skip to content

Commit

Permalink
index: make build_entry raise on FileNotFound
Browse files Browse the repository at this point in the history
Pre-requisite for fixing iterative/dvc#9541
  • Loading branch information
efiop committed Jun 12, 2023
1 parent 96c789c commit f06fd63
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 37 deletions.
22 changes: 11 additions & 11 deletions src/dvc_data/index/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,7 @@ def build_entry(
hash_name: str = DEFAULT_ALGORITHM,
):
if info is None:
try:
info = fs.info(path)
except FileNotFoundError:
return DataIndexEntry()
info = fs.info(path)

if compute_hash and info["type"] != "directory":
meta, hash_info = hash_file(
Expand Down Expand Up @@ -62,13 +59,16 @@ def build_entries(
root_key = fs.path.relparts(root, path)

for name in chain(dirs, files):
entry = build_entry(
fs.path.join(root, name),
fs,
compute_hash=compute_hash,
state=state,
hash_name=hash_name,
)
try:
entry = build_entry(
fs.path.join(root, name),
fs,
compute_hash=compute_hash,
state=state,
hash_name=hash_name,
)
except FileNotFoundError:
entry = DataIndexEntry()
entry.key = (*root_key, name)
yield entry

Expand Down
44 changes: 44 additions & 0 deletions tests/index/test_build.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import pytest

from dvc_data.index.build import DataIndexEntry, build, build_entry


def test_build_entry(tmp_upath, as_filesystem):
(tmp_upath / "foo").write_bytes(b"foo\n")

fs = as_filesystem(tmp_upath.fs)

entry = build_entry(str(tmp_upath / "foo"), fs)
assert isinstance(entry, DataIndexEntry)
assert entry.meta.size == 4
assert entry.key is None
assert entry.hash_info is None

with pytest.raises(FileNotFoundError):
build_entry(str(tmp_upath / "missing"), fs)


def test_build(tmp_upath, as_filesystem):
(tmp_upath / "foo").write_bytes(b"foo\n")
(tmp_upath / "data").mkdir()
(tmp_upath / "data" / "bar").write_bytes(b"bar\n")
(tmp_upath / "data" / "baz").write_bytes(b"baz\n")

fs = as_filesystem(tmp_upath.fs)
index = build(str(tmp_upath), fs)
assert index[("foo",)].meta.size == 4
assert index.storage_map.get_data(index[("foo",)]) == (
fs,
str(tmp_upath / "foo"),
)
assert index[("data",)].meta.isdir
assert index[("data", "bar")].meta.size == 4
assert index.storage_map.get_data(index[("data", "bar")]) == (
fs,
str(tmp_upath / "data" / "bar"),
)
assert index[("data", "baz")].meta.size == 4
assert index.storage_map.get_data(index[("data", "baz")]) == (
fs,
str(tmp_upath / "data" / "baz"),
)
26 changes: 0 additions & 26 deletions tests/index/test_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,32 +146,6 @@ def test_save(tmp_upath, odb, as_filesystem):
assert odb.exists("258622b1688250cb619f3c9ccaefb7eb")


def test_build(tmp_upath, as_filesystem):
(tmp_upath / "foo").write_bytes(b"foo\n")
(tmp_upath / "data").mkdir()
(tmp_upath / "data" / "bar").write_bytes(b"bar\n")
(tmp_upath / "data" / "baz").write_bytes(b"baz\n")

fs = as_filesystem(tmp_upath.fs)
index = build(str(tmp_upath), fs)
assert index[("foo",)].meta.size == 4
assert index.storage_map.get_data(index[("foo",)]) == (
fs,
str(tmp_upath / "foo"),
)
assert index[("data",)].meta.isdir
assert index[("data", "bar")].meta.size == 4
assert index.storage_map.get_data(index[("data", "bar")]) == (
fs,
str(tmp_upath / "data" / "bar"),
)
assert index[("data", "baz")].meta.size == 4
assert index.storage_map.get_data(index[("data", "baz")]) == (
fs,
str(tmp_upath / "data" / "baz"),
)


def test_add(tmp_upath, as_filesystem):
(tmp_upath / "foo").write_bytes(b"foo\n")
(tmp_upath / "data").mkdir()
Expand Down

0 comments on commit f06fd63

Please sign in to comment.