Skip to content

Commit

Permalink
Add option to specify tars to check or extract
Browse files Browse the repository at this point in the history
  • Loading branch information
@forsyth2
forsyth2 committed Dec 11, 2021
1 parent 5a7a2a1 commit 1f8dbe3
Show file tree
Hide file tree
Showing 7 changed files with 383 additions and 30 deletions.
15 changes: 15 additions & 0 deletions docs/source/usage.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ where
* ``--keep`` to keep a copy of the tar files on the local file system after
they have been extracted from the archive. Normally, they are deleted after
successful transfer.
* ``--tars`` to specify specific tars to check. See below for example usage.
* ``-v`` increases output verbosity.
* ``[files]`` is a list of files to check (standard wildcards supported).

Expand Down Expand Up @@ -156,6 +157,19 @@ You may need to reupload it via ``zstash create``.
Please contact the zstash development team, we're working on
identifying what causes these issues.

Example usage of ``--tars``::

# Starting at 00005a until the end
zstash check --tars=00005a-
# Starting from the beginning to 00005a (included)
zstash check --tars=-00005a
# Specific range
zstash check --tars=00005a-00005c
# Selected tar files
zstash check --tars=00003e,00004e,000059
# Mix and match
zstash check --tars=000030-00003e,00004e,00005a-

Update
======

Expand Down Expand Up @@ -243,6 +257,7 @@ where
* ``--keep`` to keep a copy of the tar files on the local file system after
they have been extracted from the archive. Normally, they are deleted after
successful transfer.
* ``--tars`` to specify specific tars to check. See "Check" above for example usage.
* ``-v`` increases output verbosity.
* ``[files]`` is a list of files to be extracted (standard wildcards supported).

Expand Down
154 changes: 148 additions & 6 deletions tests/test_check.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,13 @@ class TestCheck(TestZstash):

# `zstash check` is tested in TestCheck and TestCheckParallel.
# x = on, no mark = off, b = both on and off tested
# option | Check | CheckMismatch | CheckKeepTars | CheckParallel | CheckParallelVerboseMismatch | CheckParallelKeepTars |
# --hpss |x|x|x|x|x| |
# --workers | | | |x|x|x|
# --cache |b| | | | | |
# --keep | | | | | |b|
# -v |b|x| |b|x| |
# option | Check | CheckMismatch | CheckKeepTars | CheckTars | CheckParallel | CheckParallelVerboseMismatch | CheckParallelKeepTars | CheckParallelTars |
# --hpss |x|x|x|x|x|x| |x|
# --workers | | | | |x|x|x|x|
# --cache |b| | | | | | | |
# --keep | | | | | | |b| |
# --tars | | | |x| | | |x|
# -v |b|x| | |b|x| | |

def helperCheck(self, test_name, hpss_path, cache=None, zstash_path=ZSTASH_PATH):
"""
Expand Down Expand Up @@ -142,6 +143,13 @@ def testCheckVerboseMismatchHPSS(self):
self.conditional_hpss_skip()
self.helperCheckVerboseMismatch("testCheckVerboseMismatchHPSS", HPSS_ARCHIVE)

def testCheckTars(self):
helperCheckTars(self, "testCheckTars", "none")

def testCheckTarsHPSS(self):
self.conditional_hpss_skip()
helperCheckTars(self, "testCheckTarsHPSS", HPSS_ARCHIVE)

def testCheckKeepTars(self):
"""
Test that `zstash check` does not delete tars if `--hpss=none`.
Expand Down Expand Up @@ -211,5 +219,139 @@ def testCheckKeepTars(self):
os.chdir(TOP_LEVEL)


def helperCheckTars(
tester, test_name, hpss_path, worker_str="", zstash_path=ZSTASH_PATH
):
"""
Test `zstash check --tars`
"""
tester.hpss_path = hpss_path
use_hpss = tester.setupDirs(test_name)
tester.create(use_hpss, zstash_path)
tester.add_files(use_hpss, zstash_path)

tester.assertWorkspace()
os.chdir(tester.test_dir)

# Starting at 000001 until the end
zstash_cmd = '{}zstash check --hpss={} --tars="000001-"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Starting from the beginning to 00003 (included)
zstash_cmd = '{}zstash check --hpss={} --tars="-000003"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000004.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Specific range
zstash_cmd = '{}zstash check --hpss={} --tars="000001-000003"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000004.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Selected tar files
zstash_cmd = '{}zstash check --hpss={} --tars="000001,000003"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000004.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Mix and match
zstash_cmd = (
'{}zstash check --hpss={} --tars="000001-00002,000003,000004-"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Ending with nonexistent tar
zstash_cmd = '{}zstash check --hpss={} --tars="000001-00007"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000005.tar",
"INFO: Opening tar archive zstash/000006.tar",
"INFO: Opening tar archive zstash/000007.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Ending with nonexistent tar
zstash_cmd = '{}zstash check --hpss={} --tars="000001-00003"{} file'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = ["ValueError: If --tars is used, <files> should not be listed."]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when checking the files.",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)

os.chdir(TOP_LEVEL)


if __name__ == "__main__":
unittest.main()
21 changes: 15 additions & 6 deletions tests/test_check_parallel.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
print_starred,
run_cmd,
)
from tests.test_check import helperCheckTars

# https://bugs.python.org/issue43743
# error: Module has no attribute "_USE_CP_SENDFILE"
Expand All @@ -24,12 +25,13 @@ class TestCheckParallel(TestZstash):

# `zstash check` is tested in TestCheck and TestCheckParallel.
# x = on, no mark = off, b = both on and off tested
# option | Check | CheckMismatch | CheckKeepTars | CheckParallel | CheckParallelVerboseMismatch | CheckParallelKeepTars |
# --hpss |x|x|x|x|x| |
# --workers | | | |x|x|x|
# --cache |b| | | | | |
# --keep | | | | | |b|
# -v |b|x| |b|x| |
# option | Check | CheckMismatch | CheckKeepTars | CheckTars | CheckParallel | CheckParallelVerboseMismatch | CheckParallelKeepTars | CheckParallelTars |
# --hpss |x|x|x|x|x|x| |x|
# --workers | | | | |x|x|x|x|
# --cache |b| | | | | | | |
# --keep | | | | | | |b| |
# --tars | | | |x| | | |x|
# -v |b|x| | |b|x| | |

def helperCheckParallel(
self, test_name, hpss_path, zstash_path=ZSTASH_PATH, verbose=False
Expand Down Expand Up @@ -226,6 +228,13 @@ def testCheckParallelKeepTarsHPSS(self):
self.conditional_hpss_skip()
self.helperCheckParallelKeepTars("testCheckParallelKeepTarsHPSS", HPSS_ARCHIVE)

def testCheckParallelTars(self):
helperCheckTars(self, "testCheckParallelTars", "none", " --workers=2")

def testCheckParallelTarsHPSS(self):
self.conditional_hpss_skip()
helperCheckTars(self, "testCheckParallelTarsHPSS", HPSS_ARCHIVE, " --workers=2")


if __name__ == "__main__":
unittest.main()
60 changes: 54 additions & 6 deletions tests/test_extract.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,13 @@ class TestExtract(TestZstash):

# `zstash extract` is tested in TestExtract and TestExtractParallel.
# x = on, no mark = off, b = both on and off tested
# option | ExtractVerbose | ExtractKeep | ExtractCache | ExtractParallel |
# --hpss |x|x|x|x|
# --workers | | | |x|
# --cache | | |x| |
# --keep | |x| | |
# -v |x| | |b|
# option | ExtractVerbose | Extract | ExtractCache | ExtractTars | ExtractParallel | ExtractParallelTars |
# --hpss |x|x|x|x|x|x|
# --workers | | | | |x|x|
# --cache | | |x| | | |
# --keep | |x| | | | |
# --tars | | | |x| |x|
# -v |x| | | |b| |

def helperExtractVerbose(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Expand Down Expand Up @@ -228,6 +229,53 @@ def testExtractCacheHPSS(self):
self.conditional_hpss_skip()
self.helperExtractCache("testExtractCacheHPSS", HPSS_ARCHIVE)

def testExtractTars(self):
helperExtractTars(self, "testExtractTars", "none")

def testExtractTarsHPSS(self):
self.conditional_hpss_skip()
helperExtractTars(self, "testExtractTarsHPSS", HPSS_ARCHIVE)


def helperExtractTars(
tester, test_name, hpss_path, worker_str="", zstash_path=ZSTASH_PATH
):
"""
Test `zstash extract --tars`
"""
tester.hpss_path = hpss_path
use_hpss = tester.setupDirs(test_name)
tester.create(use_hpss, zstash_path)
tester.add_files(use_hpss, zstash_path)

os.rename(tester.test_dir, tester.backup_dir)
os.mkdir(tester.test_dir)
os.chdir(tester.test_dir)
if not use_hpss:
shutil.copytree(
"{}/{}/{}".format(TOP_LEVEL, tester.backup_dir, tester.cache),
tester.copy_dir,
)

zstash_cmd = (
'{}zstash extract --hpss={} --tars="000001-00002,000003,000004-"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when extracting the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
os.chdir(TOP_LEVEL)


if __name__ == "__main__":
unittest.main()
23 changes: 17 additions & 6 deletions tests/test_extract_parallel.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
print_starred,
run_cmd,
)
from tests.test_extract import helperExtractTars

# https://bugs.python.org/issue43743
# error: Module has no attribute "_USE_CP_SENDFILE"
Expand All @@ -23,12 +24,13 @@ class TestExtractParallel(TestZstash):

# `zstash extract` is tested in TestExtract and TestExtractParallel.
# x = on, no mark = off, b = both on and off tested
# option | ExtractVerbose | Extract | ExtractCache | ExtractParallel |
# --hpss |x|x|x|x|
# --workers | | | |x|
# --cache | | |x| |
# --keep | |x| | |
# -v |x| | |b|
# option | ExtractVerbose | Extract | ExtractCache | ExtractTars | ExtractParallel | ExtractParallelTars |
# --hpss |x|x|x|x|x|x|
# --workers | | | | |x|x|
# --cache | | |x| | | |
# --keep | |x| | | | |
# --tars | | | |x| |x|
# -v |x| | | |b| |

def helperExtractParallel(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Expand Down Expand Up @@ -116,6 +118,15 @@ def testExtractParallelHPSS(self):
self.conditional_hpss_skip()
self.helperExtractParallel("testExtractParallelHPSS", HPSS_ARCHIVE)

def testExtractParallelTars(self):
helperExtractTars(self, "testExtractParallelTars", "none", " --workers=2")

def testExtractParallelTarsHPSS(self):
self.conditional_hpss_skip()
helperExtractTars(
self, "testExtractParallelTarsHPSS", HPSS_ARCHIVE, " --workers=2"
)


if __name__ == "__main__":
unittest.main()
Loading

0 comments on commit 1f8dbe3

Please sign in to comment.