Skip to content

Commit

Permalink
output: save version_id meta
Browse files Browse the repository at this point in the history
  • Loading branch information
pmrowla committed Sep 1, 2022
1 parent aff65d4 commit 2e97e3c
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 4 deletions.
1 change: 1 addition & 0 deletions dvc/config_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ class RelPath(str):
"checksum_jobs": All(Coerce(int), Range(1)),
"jobs": All(Coerce(int), Range(1)),
Optional("no_traverse"): Bool, # obsoleted
Optional("version_aware"): Bool,
}
LOCAL_COMMON = {
"type": supported_cache_type,
Expand Down
41 changes: 37 additions & 4 deletions dvc/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,15 @@ def __init__(
repo=None,
):
self.repo = stage.repo if not repo and stage else repo
fs_cls, fs_config, fs_path = get_cloud_fs(self.repo, url=path)
meta = Meta.from_dict(info)
# NOTE: when version_aware is not passed into get_cloud_fs, it will be
# set based on whether or not path is versioned
fs_kwargs = {"version_aware": True} if meta.version_id else {}
fs_cls, fs_config, fs_path = get_cloud_fs(
self.repo,
url=path,
**fs_kwargs,
)
self.fs = fs_cls(**fs_config)

if (
Expand Down Expand Up @@ -320,7 +328,7 @@ def __init__(
# By resolved path, which contains actual location,
# should be absolute and don't contain remote:// refs.
self.stage = stage
self.meta = Meta.from_dict(info)
self.meta = meta
self.hash_info = HashInfo.from_dict(info)
self.use_cache = False if self.IS_DEPENDENCY else cache
self.metric = False if self.IS_DEPENDENCY else metric
Expand All @@ -335,6 +343,12 @@ def __init__(

self.remote = remote

if self.fs.version_aware:
_, version_id = self.fs.path.coalesce_version(
self.def_path, self.meta.version_id
)
self.meta.version_id = version_id

def _parse_path(self, fs, fs_path):
parsed = urlparse(self.def_path)
if (
Expand All @@ -360,6 +374,10 @@ def __repr__(self):

def __str__(self):
if self.fs.protocol != "local":
if self.meta.version_id:
return self.fs.path.version_path(
self.def_path, self.meta.version_id
)
return self.def_path

if (
Expand Down Expand Up @@ -419,21 +437,29 @@ def cache_path(self):
)

def get_hash(self):
_, hash_info = self._get_hash_meta()
return hash_info

def _get_hash_meta(self):
if self.use_cache:
odb = self.odb
name = self.odb.fs.PARAM_CHECKSUM
else:
odb = self.repo.odb.local
name = self.fs.PARAM_CHECKSUM
_, _, obj = build(
_, meta, obj = build(
odb,
self.fs_path,
self.fs,
name,
ignore=self.dvcignore,
dry_run=not self.use_cache,
)
return obj.hash_info
return meta, obj.hash_info

def get_meta(self) -> Meta:
meta, _ = self._get_hash_meta()
return meta

@property
def is_dir_checksum(self):
Expand Down Expand Up @@ -469,6 +495,11 @@ def changed_cache(self, filter_info=None):
except (FileNotFoundError, ObjectFormatError):
return True

def changed_meta(self) -> bool:
if self.fs.version_aware and self.meta.version_id:
return self.meta.version_id == self.get_meta().version_id
return False

def workspace_status(self):
if not self.exists:
return {str(self): "deleted"}
Expand Down Expand Up @@ -814,6 +845,7 @@ def move(self, out):
self.fs.move(self.fs_path, out.fs_path)
self.def_path = out.def_path
self.fs_path = out.fs_path
self.__dict__.pop("fs_path")
self.save()
self.commit()

Expand Down Expand Up @@ -1095,6 +1127,7 @@ def is_plot(self) -> bool:
Meta.PARAM_SIZE: int,
Meta.PARAM_NFILES: int,
Meta.PARAM_ISEXEC: bool,
Meta.PARAM_VERSION_ID: str,
}

SCHEMA = {
Expand Down
1 change: 1 addition & 0 deletions tests/unit/output/test_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ def test_get_used_objs(exists, expected_message, mocker, caplog):
mocker.patch.object(
stage.repo.dvcignore, "check_ignore", return_value=_no_match("path")
)
stage.repo.fs.version_aware = False

output = Output(stage, "path")

Expand Down

0 comments on commit 2e97e3c

Please sign in to comment.