Add option to specify tars to check or extract

docs/source/usage.rst

@@ -117,6 +117,7 @@ where
 * ``--keep`` to keep a copy of the tar files on the local file system after
   they have been extracted from the archive. Normally, they are deleted after
   successful transfer.
+* ``--tars`` to specify specific tars to check. See below for example usage.
 * ``-v`` increases output verbosity.
 * ``[files]`` is a list of files to check (standard wildcards supported).
 
@@ -156,6 +157,19 @@ You may need to reupload it via ``zstash create``.
 Please contact the zstash development team, we're working on
 identifying what causes these issues.
 
+Example usage of ``--tars``::
+
+  # Starting at 00005a until the end
+  zstash check --tars=00005a-
+  # Starting from the beginning to 00005a (included)
+  zstash check --tars=-00005a
+  # Specific range
+  zstash check --tars=00005a-00005c
+  # Selected tar files
+  zstash check --tars=00003e,00004e,000059
+  # Mix and match
+  zstash check --tars=000030-00003e,00004e,00005a-
+
 Update
 ======
 
@@ -243,6 +257,7 @@ where
 * ``--keep`` to keep a copy of the tar files on the local file system after
   they have been extracted from the archive. Normally, they are deleted after
   successful transfer.
+* ``--tars`` to	specify	specific tars to check. See "Check" above for example usage.
 * ``-v`` increases output verbosity.
 * ``[files]`` is a list of files to be extracted (standard wildcards supported).
 

tests/test_check.py

@@ -26,12 +26,13 @@ class TestCheck(TestZstash):
 
     # `zstash check` is tested in TestCheck and TestCheckParallel.
     # x = on, no mark = off, b = both on and off tested
-    # option | Check | CheckMismatch | CheckKeepTars | CheckParallel | CheckParallelVerboseMismatch | CheckParallelKeepTars |
-    # --hpss    |x|x|x|x|x| |
-    # --workers | | | |x|x|x|
-    # --cache   |b| | | | | |
-    # --keep    | | | | | |b|
-    # -v        |b|x| |b|x| |
+    # option | Check | CheckMismatch | CheckKeepTars | CheckTars | CheckParallel | CheckParallelVerboseMismatch | CheckParallelKeepTars | CheckParallelTars |
+    # --hpss    |x|x|x|x|x|x| |x|
+    # --workers | | | | |x|x|x|x|
+    # --cache   |b| | | | | | | |
+    # --keep    | | | | | | |b| |
+    # --tars    | | | |x| | | |x|
+    # -v        |b|x| | |b|x| | |
 
     def helperCheck(self, test_name, hpss_path, cache=None, zstash_path=ZSTASH_PATH):
         """
@@ -142,6 +143,13 @@ def testCheckVerboseMismatchHPSS(self):
         self.conditional_hpss_skip()
         self.helperCheckVerboseMismatch("testCheckVerboseMismatchHPSS", HPSS_ARCHIVE)
 
+    def testCheckTars(self):
+        helperCheckTars(self, "testCheckTars", "none")
+
+    def testCheckTarsHPSS(self):
+        self.conditional_hpss_skip()
+        helperCheckTars(self, "testCheckTarsHPSS", HPSS_ARCHIVE)
+
     def testCheckKeepTars(self):
         """
         Test that `zstash check` does not delete tars if `--hpss=none`.
@@ -211,5 +219,139 @@ def testCheckKeepTars(self):
         os.chdir(TOP_LEVEL)
 
 
+def helperCheckTars(
+    tester, test_name, hpss_path, worker_str="", zstash_path=ZSTASH_PATH
+):
+    """
+    Test `zstash check --tars`
+    """
+    tester.hpss_path = hpss_path
+    use_hpss = tester.setupDirs(test_name)
+    tester.create(use_hpss, zstash_path)
+    tester.add_files(use_hpss, zstash_path)
+
+    tester.assertWorkspace()
+    os.chdir(tester.test_dir)
+
+    # Starting at 000001 until the end
+    zstash_cmd = '{}zstash check --hpss={} --tars="000001-"{}'.format(
+        zstash_path, tester.hpss_path, worker_str
+    )
+    output, err = run_cmd(zstash_cmd)
+    expected_present = [
+        "INFO: Opening tar archive zstash/000001.tar",
+        "INFO: Opening tar archive zstash/000002.tar",
+        "INFO: Opening tar archive zstash/000003.tar",
+        "INFO: Opening tar archive zstash/000004.tar",
+        "INFO: No failures detected when checking the files.",
+    ]
+    expected_absent = [
+        "INFO: Opening tar archive zstash/000000.tar",
+    ]
+    tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
+    # Starting from the beginning to 00003 (included)
+    zstash_cmd = '{}zstash check --hpss={} --tars="-000003"{}'.format(
+        zstash_path, tester.hpss_path, worker_str
+    )
+    output, err = run_cmd(zstash_cmd)
+    expected_present = [
+        "INFO: Opening tar archive zstash/000000.tar",
+        "INFO: Opening tar archive zstash/000001.tar",
+        "INFO: Opening tar archive zstash/000002.tar",
+        "INFO: Opening tar archive zstash/000003.tar",
+        "INFO: No failures detected when checking the files.",
+    ]
+    expected_absent = [
+        "INFO: Opening tar archive zstash/000004.tar",
+    ]
+    tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
+    # Specific range
+    zstash_cmd = '{}zstash check --hpss={} --tars="000001-000003"{}'.format(
+        zstash_path, tester.hpss_path, worker_str
+    )
+    output, err = run_cmd(zstash_cmd)
+    expected_present = [
+        "INFO: Opening tar archive zstash/000001.tar",
+        "INFO: Opening tar archive zstash/000002.tar",
+        "INFO: Opening tar archive zstash/000003.tar",
+        "INFO: No failures detected when checking the files.",
+    ]
+    expected_absent = [
+        "INFO: Opening tar archive zstash/000000.tar",
+        "INFO: Opening tar archive zstash/000004.tar",
+    ]
+    tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
+    # Selected tar files
+    zstash_cmd = '{}zstash check --hpss={} --tars="000001,000003"{}'.format(
+        zstash_path, tester.hpss_path, worker_str
+    )
+    output, err = run_cmd(zstash_cmd)
+    expected_present = [
+        "INFO: Opening tar archive zstash/000001.tar",
+        "INFO: Opening tar archive zstash/000003.tar",
+        "INFO: No failures detected when checking the files.",
+    ]
+    expected_absent = [
+        "INFO: Opening tar archive zstash/000000.tar",
+        "INFO: Opening tar archive zstash/000002.tar",
+        "INFO: Opening tar archive zstash/000004.tar",
+    ]
+    tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
+    # Mix and match
+    zstash_cmd = (
+        '{}zstash check --hpss={} --tars="000001-00002,000003,000004-"{}'.format(
+            zstash_path, tester.hpss_path, worker_str
+        )
+    )
+    output, err = run_cmd(zstash_cmd)
+    expected_present = [
+        "INFO: Opening tar archive zstash/000001.tar",
+        "INFO: Opening tar archive zstash/000002.tar",
+        "INFO: Opening tar archive zstash/000003.tar",
+        "INFO: Opening tar archive zstash/000004.tar",
+        "INFO: No failures detected when checking the files.",
+    ]
+    expected_absent = [
+        "INFO: Opening tar archive zstash/000000.tar",
+    ]
+    tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
+    # Ending with nonexistent tar
+    zstash_cmd = '{}zstash check --hpss={} --tars="000001-00007"{}'.format(
+        zstash_path, tester.hpss_path, worker_str
+    )
+    output, err = run_cmd(zstash_cmd)
+    expected_present = [
+        "INFO: Opening tar archive zstash/000001.tar",
+        "INFO: Opening tar archive zstash/000002.tar",
+        "INFO: Opening tar archive zstash/000003.tar",
+        "INFO: Opening tar archive zstash/000004.tar",
+        "INFO: No failures detected when checking the files.",
+    ]
+    expected_absent = [
+        "INFO: Opening tar archive zstash/000000.tar",
+        "INFO: Opening tar archive zstash/000005.tar",
+        "INFO: Opening tar archive zstash/000006.tar",
+        "INFO: Opening tar archive zstash/000007.tar",
+    ]
+    tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
+    # Ending with nonexistent tar
+    zstash_cmd = '{}zstash check --hpss={} --tars="000001-00003"{} file'.format(
+        zstash_path, tester.hpss_path, worker_str
+    )
+    output, err = run_cmd(zstash_cmd)
+    expected_present = ["ValueError: If --tars is used, <files> should not be listed."]
+    expected_absent = [
+        "INFO: Opening tar archive zstash/000000.tar",
+        "INFO: Opening tar archive zstash/000001.tar",
+        "INFO: Opening tar archive zstash/000002.tar",
+        "INFO: Opening tar archive zstash/000003.tar",
+        "INFO: Opening tar archive zstash/000004.tar",
+        "INFO: No failures detected when checking the files.",
+    ]
+    tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
+
+    os.chdir(TOP_LEVEL)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_check_parallel.py

@@ -11,6 +11,7 @@
     print_starred,
     run_cmd,
 )
+from tests.test_check import helperCheckTars
 
 # https://bugs.python.org/issue43743
 # error: Module has no attribute "_USE_CP_SENDFILE"
@@ -24,12 +25,13 @@ class TestCheckParallel(TestZstash):
 
     # `zstash check` is tested in TestCheck and TestCheckParallel.
     # x = on, no mark = off, b = both on and off tested
-    # option | Check | CheckMismatch | CheckKeepTars | CheckParallel | CheckParallelVerboseMismatch | CheckParallelKeepTars |
-    # --hpss    |x|x|x|x|x| |
-    # --workers | | | |x|x|x|
-    # --cache   |b| | | | | |
-    # --keep    | | | | | |b|
-    # -v        |b|x| |b|x| |
+    # option | Check | CheckMismatch | CheckKeepTars | CheckTars | CheckParallel | CheckParallelVerboseMismatch | CheckParallelKeepTars | CheckParallelTars |
+    # --hpss    |x|x|x|x|x|x| |x|
+    # --workers | | | | |x|x|x|x|
+    # --cache   |b| | | | | | | |
+    # --keep    | | | | | | |b| |
+    # --tars    | | | |x| | | |x|
+    # -v        |b|x| | |b|x| | |
 
     def helperCheckParallel(
         self, test_name, hpss_path, zstash_path=ZSTASH_PATH, verbose=False
@@ -226,6 +228,13 @@ def testCheckParallelKeepTarsHPSS(self):
         self.conditional_hpss_skip()
         self.helperCheckParallelKeepTars("testCheckParallelKeepTarsHPSS", HPSS_ARCHIVE)
 
+    def testCheckParallelTars(self):
+        helperCheckTars(self, "testCheckParallelTars", "none", " --workers=2")
+
+    def testCheckParallelTarsHPSS(self):
+        self.conditional_hpss_skip()
+        helperCheckTars(self, "testCheckParallelTarsHPSS", HPSS_ARCHIVE, " --workers=2")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_extract.py

@@ -24,12 +24,13 @@ class TestExtract(TestZstash):
 
     # `zstash extract` is tested in TestExtract and TestExtractParallel.
     # x = on, no mark = off, b = both on and off tested
-    # option | ExtractVerbose | ExtractKeep | ExtractCache | ExtractParallel |
-    # --hpss    |x|x|x|x|
-    # --workers | | | |x|
-    # --cache   | | |x| |
-    # --keep    | |x| | |
-    # -v        |x| | |b|
+    # option | ExtractVerbose | Extract | ExtractCache | ExtractTars | ExtractParallel | ExtractParallelTars |
+    # --hpss    |x|x|x|x|x|x|
+    # --workers | | | | |x|x|
+    # --cache   | | |x| | | |
+    # --keep    | |x| | | | |
+    # --tars    | | | |x| |x|
+    # -v        |x| | | |b| |
 
     def helperExtractVerbose(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
         """
@@ -228,6 +229,53 @@ def testExtractCacheHPSS(self):
         self.conditional_hpss_skip()
         self.helperExtractCache("testExtractCacheHPSS", HPSS_ARCHIVE)
 
+    def testExtractTars(self):
+        helperExtractTars(self, "testExtractTars", "none")
+
+    def testExtractTarsHPSS(self):
+        self.conditional_hpss_skip()
+        helperExtractTars(self, "testExtractTarsHPSS", HPSS_ARCHIVE)
+
+
+def helperExtractTars(
+    tester, test_name, hpss_path, worker_str="", zstash_path=ZSTASH_PATH
+):
+    """
+    Test `zstash extract --tars`
+    """
+    tester.hpss_path = hpss_path
+    use_hpss = tester.setupDirs(test_name)
+    tester.create(use_hpss, zstash_path)
+    tester.add_files(use_hpss, zstash_path)
+
+    os.rename(tester.test_dir, tester.backup_dir)
+    os.mkdir(tester.test_dir)
+    os.chdir(tester.test_dir)
+    if not use_hpss:
+        shutil.copytree(
+            "{}/{}/{}".format(TOP_LEVEL, tester.backup_dir, tester.cache),
+            tester.copy_dir,
+        )
+
+    zstash_cmd = (
+        '{}zstash extract --hpss={} --tars="000001-00002,000003,000004-"{}'.format(
+            zstash_path, tester.hpss_path, worker_str
+        )
+    )
+    output, err = run_cmd(zstash_cmd)
+    expected_present = [
+        "INFO: Opening tar archive zstash/000001.tar",
+        "INFO: Opening tar archive zstash/000002.tar",
+        "INFO: Opening tar archive zstash/000003.tar",
+        "INFO: Opening tar archive zstash/000004.tar",
+        "INFO: No failures detected when extracting the files.",
+    ]
+    expected_absent = [
+        "INFO: Opening tar archive zstash/000000.tar",
+    ]
+    tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
+    os.chdir(TOP_LEVEL)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_extract_parallel.py

@@ -10,6 +10,7 @@
     print_starred,
     run_cmd,
 )
+from tests.test_extract import helperExtractTars
 
 # https://bugs.python.org/issue43743
 # error: Module has no attribute "_USE_CP_SENDFILE"
@@ -23,12 +24,13 @@ class TestExtractParallel(TestZstash):
 
     # `zstash extract` is tested in TestExtract and TestExtractParallel.
     # x = on, no mark = off, b = both on and off tested
-    # option | ExtractVerbose | Extract | ExtractCache | ExtractParallel |
-    # --hpss    |x|x|x|x|
-    # --workers | | | |x|
-    # --cache   | | |x| |
-    # --keep    | |x| | |
-    # -v        |x| | |b|
+    # option | ExtractVerbose | Extract | ExtractCache | ExtractTars | ExtractParallel | ExtractParallelTars |
+    # --hpss    |x|x|x|x|x|x|
+    # --workers | | | | |x|x|
+    # --cache   | | |x| | | |
+    # --keep    | |x| | | | |
+    # --tars    | | | |x| |x|
+    # -v        |x| | | |b| |
 
     def helperExtractParallel(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
         """
@@ -116,6 +118,15 @@ def testExtractParallelHPSS(self):
         self.conditional_hpss_skip()
         self.helperExtractParallel("testExtractParallelHPSS", HPSS_ARCHIVE)
 
+    def testExtractParallelTars(self):
+        helperExtractTars(self, "testExtractParallelTars", "none", " --workers=2")
+
+    def testExtractParallelTarsHPSS(self):
+        self.conditional_hpss_skip()
+        helperExtractTars(
+            self, "testExtractParallelTarsHPSS", HPSS_ARCHIVE, " --workers=2"
+        )
+
 
 if __name__ == "__main__":
     unittest.main()