From 927ea5f490d2ecd4efe1cb07c0989b769278c95c Mon Sep 17 00:00:00 2001 From: Rye Mutt Date: Wed, 28 Sep 2022 16:50:52 -0400 Subject: [PATCH 1/4] Add zstandard support for tarfile compression along with xz, and gzip --- autobuild/autobuild_tool_install.py | 14 ++++++++++---- autobuild/autobuild_tool_package.py | 24 +++++++++++++++++++----- autobuild/common.py | 27 +++++++++++++++++++++++++++ setup.py | 2 +- tests/test_graph.py | 2 +- 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/autobuild/autobuild_tool_install.py b/autobuild/autobuild_tool_install.py index f677742..0a9582f 100644 --- a/autobuild/autobuild_tool_install.py +++ b/autobuild/autobuild_tool_install.py @@ -267,7 +267,7 @@ def _install_package(archive_path, install_dir, exclude=[]): logger.error("cannot extract non-existing package: %s" % archive_path) return False logger.info("extracting from %s" % os.path.basename(archive_path)) - if tarfile.is_tarfile(archive_path): + if tarfile.is_tarfile(archive_path) or ".tar.zst" in archive_path: return __extract_tar_file(archive_path, install_dir, exclude=exclude) elif zipfile.is_zipfile(archive_path): return __extract_zip_archive(archive_path, install_dir, exclude=exclude) @@ -285,8 +285,11 @@ def extract_metadata_from_package(archive_path, metadata_file_name): logger.error("no package found at: %s" % archive_path) else: logger.debug("extracting metadata from %s" % os.path.basename(archive_path)) - if tarfile.is_tarfile(archive_path): - tar = tarfile.open(archive_path, 'r') + if tarfile.is_tarfile(archive_path) or ".tar.zst" in archive_path: + if ".tar.zst" in archive_path: + tar = common.ZstdTarFile(archive_path, 'r') + else: + tar = tarfile.open(archive_path, 'r') try: metadata_file = tar.extractfile(metadata_file_name) except KeyError as err: @@ -305,7 +308,10 @@ def extract_metadata_from_package(archive_path, metadata_file_name): def __extract_tar_file(cachename, install_dir, exclude=[]): # Attempt to extract the package from the install cache - tar = tarfile.open(cachename, 'r') + if ".tar.zst" in cachename: + tar = common.ZstdTarFile(cachename, 'r') + else: + tar = tarfile.open(cachename, 'r') extract = [member for member in tar.getmembers() if member.name not in exclude] conflicts = [member.name for member in extract if os.path.exists(os.path.join(install_dir, member.name)) diff --git a/autobuild/autobuild_tool_package.py b/autobuild/autobuild_tool_package.py index 301896d..72e8999 100644 --- a/autobuild/autobuild_tool_package.py +++ b/autobuild/autobuild_tool_package.py @@ -65,7 +65,7 @@ def register(self, parser): parser.add_argument('--archive-format', default=None, dest='archive_format', - help='the format of the archive (tbz2 or zip)') + help='the format of the archive (tbz2, tzst, txz, tgz, or zip)') parser.add_argument('--build-dir', default=None, dest='select_dir', # see common.select_directories() @@ -239,8 +239,8 @@ def package(config, build_directory, platform_name, archive_filename=None, archi else: archive_description = platform_description.archive format = _determine_archive_format(archive_format, archive_description) - if format == 'tbz2': - _create_tarfile(tarfilename + '.tar.bz2', build_directory, files, results) + if format == 'txz' or format == 'tbz2' or format == 'tgz' or format == 'tzst': + _create_tarfile(tarfilename, format, build_directory, files, results) elif format == 'zip': _create_zip_archive(tarfilename + '.zip', build_directory, files, results) else: @@ -288,13 +288,27 @@ def _get_file_list(platform_description, build_directory): os.chdir(current_directory) return [files, missing] -def _create_tarfile(tarfilename, build_directory, filelist, results: dict): +def _create_tarfile(tarfilename, format, build_directory, filelist, results: dict): if not os.path.exists(os.path.dirname(tarfilename)): os.makedirs(os.path.dirname(tarfilename)) current_directory = os.getcwd() os.chdir(build_directory) try: - tfile = tarfile.open(tarfilename, 'w:bz2') + if format == 'txz': + tarfilename = tarfilename + '.tar.xz' + tfile = tarfile.open(tarfilename, 'w:xz') + elif format == 'tbz2': + tarfilename = tarfilename + '.tar.bz2' + tfile = tarfile.open(tarfilename, 'w:bz2') + elif format == 'tgz': + tarfilename = tarfilename + '.tar.gz' + tfile = tarfile.open(tarfilename, 'w:gz') + elif format == 'tzst': + tarfilename = tarfilename + '.tar.zst' + tfile = common.ZstdTarFile(tarfilename, 'w', level=22) + else: + raise PackageError("unknown tar archive format: %s" % format) + for file in filelist: try: # Make sure permissions are set on Windows. diff --git a/autobuild/common.py b/autobuild/common.py index f84229a..f3e9d40 100644 --- a/autobuild/common.py +++ b/autobuild/common.py @@ -9,14 +9,18 @@ import itertools import logging +import multiprocessing import os import platform import pprint import subprocess import sys +import tarfile import tempfile from collections import OrderedDict +from pyzstd import CParameter, ZstdFile + from autobuild.version import AUTOBUILD_VERSION_STRING logger = logging.getLogger(__name__) @@ -535,3 +539,26 @@ def has_cmd(name) -> bool: except OSError: return False return not p.returncode + + +class ZstdTarFile(tarfile.TarFile): + def __init__(self, name, mode='r', *, level=4, zstd_dict=None, **kwargs): + zstdoption = None + if mode != 'r' and mode != 'rb': + zstdoption = {CParameter.compressionLevel : level, + CParameter.nbWorkers : multiprocessing.cpu_count(), + CParameter.checksumFlag : 1} + self.zstd_file = ZstdFile(name, mode, + level_or_option=zstdoption, + zstd_dict=zstd_dict) + try: + super().__init__(fileobj=self.zstd_file, mode=mode, **kwargs) + except: + self.zstd_file.close() + raise + + def close(self): + try: + super().close() + finally: + self.zstd_file.close() diff --git a/setup.py b/setup.py index d2eb7a8..9a936c4 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ 'Programming Language :: Python :: 3', 'Topic :: Software Development :: Libraries :: Python Modules', ], - install_requires=['llsd', 'pydot'], + install_requires=['llsd', 'pydot', 'pyzstd'], extras_require={ 'dev': ['pytest', 'pytest-cov'], 'build': ['build', 'setuptools_scm'], diff --git a/tests/test_graph.py b/tests/test_graph.py index 605a3bb..ceecce4 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -92,7 +92,7 @@ def setUp(self): self.options.graph_type = 'mermaid' def test_output(self): - with CaptureStdout() as out: + with CaptureStdout() as out: self.options.source_file = os.path.join(self.this_dir, "data", "bongo-0.1-common-111.tar.bz2") graph.AutobuildTool().run(self.options) graph_txt = out.getvalue() From 5c25107a6acd7e2b3bc69833e3e8cf7ac5c0481b Mon Sep 17 00:00:00 2001 From: Rye Mutt Date: Wed, 28 Sep 2022 19:55:15 -0400 Subject: [PATCH 2/4] Add tests for packaging of new compression types --- tests/test_package.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/test_package.py b/tests/test_package.py index 8df5306..3bf94d5 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -51,6 +51,9 @@ def setUp(self): self.platform = 'common' self.tar_basename = os.path.join(self.data_dir, "test1-1.0-common-123456") self.tar_name = self.tar_basename + ".tar.bz2" + self.tar_gz_name = self.tar_basename + ".tar.gz" + self.tar_xz_name = self.tar_basename + ".tar.xz" + self.tar_zst_name = self.tar_basename + ".tar.zst" self.zip_name = self.tar_basename + ".zip" self.expected_files=['include/file1','LICENSES/test1.txt','autobuild-package.xml'] self.expected_files.sort() @@ -72,7 +75,10 @@ def tearDown(self): BaseTest.tearDown(self) def tar_has_expected(self,tar): - tarball = tarfile.open(tar, 'r:bz2') + if 'tar.zst' in tar: + tarball = common.ZstdTarFile(tar, 'r') + else: + tarball = tarfile.open(tar, 'r') packaged_files=tarball.getnames() packaged_files.sort() self.assertEqual(packaged_files, self.expected_files) @@ -155,6 +161,27 @@ def test_autobuild_package(self): assert os.path.exists(self.tar_name), "%s does not exist" % self.tar_name self.tar_has_expected(self.tar_name) self.remove(self.tar_name) + self.autobuild("package", + "--config-file=" + self.config_path, + "--archive-format=tgz", + "-p", "common") + assert os.path.exists(self.tar_gz_name), "%s does not exist" % self.tar_gz_name + self.tar_has_expected(self.tar_gz_name) + self.remove(self.tar_gz_name) + self.autobuild("package", + "--config-file=" + self.config_path, + "--archive-format=txz", + "-p", "common") + assert os.path.exists(self.tar_xz_name), "%s does not exist" % self.tar_xz_name + self.tar_has_expected(self.tar_xz_name) + self.remove(self.tar_xz_name) + self.autobuild("package", + "--config-file=" + self.config_path, + "--archive-format=tzst", + "-p", "common") + assert os.path.exists(self.tar_zst_name), "%s does not exist" % self.tar_zst_name + self.tar_has_expected(self.tar_zst_name) + self.remove(self.tar_zst_name) self.autobuild("package", "--config-file=" + self.config_path, "--archive-format=zip", From f44ed197e5505802baaeb24b1ad539a6e0d22c76 Mon Sep 17 00:00:00 2001 From: Rye Mutt Date: Wed, 28 Sep 2022 20:17:13 -0400 Subject: [PATCH 3/4] Add basic functionality test for installation of new package compression types --- tests/data/bogus-0.1-common-111.tar.gz | Bin 0 -> 592 bytes tests/data/bogus-0.1-common-111.tar.xz | Bin 0 -> 592 bytes tests/data/bogus-0.1-common-111.tar.zst | Bin 0 -> 523 bytes tests/data/bogus-0.1-common-111.zip | Bin 0 -> 744 bytes tests/test_install.py | 96 +++++++++++++++++++++++- 5 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 tests/data/bogus-0.1-common-111.tar.gz create mode 100644 tests/data/bogus-0.1-common-111.tar.xz create mode 100644 tests/data/bogus-0.1-common-111.tar.zst create mode 100644 tests/data/bogus-0.1-common-111.zip diff --git a/tests/data/bogus-0.1-common-111.tar.gz b/tests/data/bogus-0.1-common-111.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..845a0183472cd3b0aba6086a4ee700975696d432 GIT binary patch literal 592 zcmV-W0h{_)VR^ccT(E2Gx;C+}<6DMf z`Mw{lG{FClyKkRvzdznSn%jI=SBBinY4q>9?q&TWTf;WNwufxwcuM~mdA6n_(EsH4 ze18u@Uoeb+!vF1||6Abyv-rQ?^M4x!0U?GiJkPGLuz{}hoAoM8(<)X6L5uZDR|uck z%lc3#8y<&d^Y3`Rk|ob}Ym9Mdw(X7(X7*Azq}hsQ6SFERd2#-jr`Okpr#xa= z#d;y#eZM^14N~fHHVIG1iEnrV4J)H!oPVH=QrfZ@Wg+8yz|GgiQlwNSc_~JQJ<5e> zx*d$Vd@0ySykL^XRMHW~FvU|v_M_Y6Li1SHFz>_q59a^D`tP*=6Zn2WKqU)6Yl^LR zJWZzmzwiIJXZt_Mvta*c7W`0aEDTBq{|EmE|NrCv1o?sE!K|uvg+ysD7GA8aFeH-)g z%QNik_&4zC)_ZslSXO!n{gpQj@hS$eNt`VFYDh25xtv&Qh`thHo6(v5>J7qDqp}@I zbY^TN0BfMvI9QIQ#w~95P6?xo(?WAJvQoCx4dbcTrlUDWfSNRIl@1SfxE3*wz2o@u z!&RD~g)y(TX160>G}rau6BJO-2gGG~Qqv1}S7bBSW^8J9`P>BFZ?HXE z@Ev~D{xMR}kXsuNFB*TvZ6?g8Tb|2TXRXI%*7$No>Vk?i%S?}F*IJ^|N-2d!sOUTf z?)E2fV&+NsG{5J|0|Z)i=TfmlF33GK0s3>)KK0vgf6&@|WfPF2M#2-hYsPoL&BYB# z>2VKmlf_q+Qdr)|>u+8#Mw~X zwg(DLJfwpk5fFsg;Ch_flyJjShcBxJBGkhL%;2|*@rg-OK95^2=pu@ScAtK0-*ZfY zl%J9M-O&H{Qx{67cS(D%=F(v0fAsalqieKSwb?ZwBT=@`E@%C`;I_ysnZ#j&>0s+E zvLw;y?lHIsEN$(Y>yE~se0U89+;g#TcE-%c=2>NcwVC?d$TIXsoQeIa<^TY)6hiPU e`Tel~0jvaoPyhe}JMP)B#Ao{g000001X)_zjV5FO literal 0 HcmV?d00001 diff --git a/tests/data/bogus-0.1-common-111.tar.zst b/tests/data/bogus-0.1-common-111.tar.zst new file mode 100644 index 0000000000000000000000000000000000000000..6a6a183e4a97fad79a527d3769b3561631b03ab8 GIT binary patch literal 523 zcmV+m0`&bTwJ-eyi1iNu_7z7XAWh`~z`zs|`N@REFSVPZtg6cky2-9<+A_%iQ3e1W z5hejW06PFa0PA&(F;|&ksv$9w2ZIL&g93$xbu3cK23eEPn|m?3z_c`5L}{yg-Ord> z6|&r0yK}wne10ZW??8~SgawsS3@KAD9fXd(wk-W(H-Al$m=9H*h`L3}r5!Hg2@#{~ z$e{I5;3C_}_SVtRaL~wlO1lF4wes9O4-|?dCt=*ZX+-p9oG=evG!=7#L5^m6``}uc zqZvQ|nmK?#BvE`P1yXG_z;yHW8g_ITomX42+4u?Pr+_CbVLt!!dD=Nu}us!3#?WaOK`B*AEML5#@q(?G`dYqd5|vFK!h zLFRf~@Qk0vyOCl|5})!5@?UKLvg3_JJpd}7KV0oYyAXn91#K&YIDd?#fCmJQn173@8ymjjfJgbqFsnRmm?{`M6}O)T19OIx95=kNyG7jJPT= z;ICt;iLQpSqDz7Kf5zdfIT@L%lxHvip)fww2V0E>(%Fp};OQrPwVDrwf;h43fa4rD N(jO9;CKc@>>g=z7_g4S_ literal 0 HcmV?d00001 diff --git a/tests/data/bogus-0.1-common-111.zip b/tests/data/bogus-0.1-common-111.zip new file mode 100644 index 0000000000000000000000000000000000000000..00347cafac887fa453afa048a1db75c7954c77cb GIT binary patch literal 744 zcmWIWW@Zs#U|`^2*u20dRGn!_IU6Gb!#5@d22r4BVrfZ!QfX#Rif%z-a&}^Rs$NBI zPVc0HUCjmpZSQNjUNX<9I)CeM(6rV~I$xqD7)st<=s872@9pjI>=j{~H^j|3@&E77 z(RcsdnR&CGU@a zFBHu@CL+4h;z$2Awo?s#pZRsVTDEos?Jlt___j-DEmOQ{nENq3skQ94uU`4P_Gm}k zVs=hBr6X34ij|+g66o7pxc+?rJm3`PScmEweXZsI1{fO<^8;~aUUE)pN~(TRetK!K zUIw?PuAbo&CI(bZS5}-X25DdcVj&>*@pN|e3w8~L>ME%yDgONZIl8X(0f#+dR`CLL zpgJ o^MIN$J&vv)-O~vDOMv=Of*`<~6%?qzP-S8G4WzFF6)-RW0Q{WpcmMzZ literal 0 HcmV?d00001 diff --git a/tests/test_install.py b/tests/test_install.py index 41f74ab..f409399 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -510,9 +510,9 @@ def test_versions(self): self.assertEqual(stream.getvalue(), "bogus: 1\n") # ------------------------------------- ------------------------------------- -class TestInstallCachedArchive(BaseTest): +class TestInstallCachedBZ2Archive(BaseTest): def setup_method(self, method): - super(TestInstallCachedArchive, self).setup_method(method) + super(TestInstallCachedBZ2Archive, self).setup_method(method) self.pkg = "bogus" # make sure that the archive is not in the server clean_file(os.path.join(SERVER_DIR, "bogus-0.1-common-111.tar.bz2")) @@ -532,6 +532,98 @@ def test_success(self): autobuild_tool_install.AutobuildTool().run(self.options) self.assertEqual(stream.getvalue(), 'Dirty Packages: \n') +# ------------------------------------- ------------------------------------- +class TestInstallCachedGZArchive(BaseTest): + def setup_method(self, method): + super(TestInstallCachedGZArchive, self).setup_method(method) + self.pkg = "bogus" + # make sure that the archive is not in the server + clean_file(os.path.join(SERVER_DIR, "bogus-0.1-common-111.tar.gz")) + assert not os.path.exists(in_dir(SERVER_DIR, "bogus-0.1-common-111.tar.gz")) + self.cache_name = in_dir(common.get_install_cache_dir(), "bogus-0.1-common-111.tar.gz") + + # Instead copy directly to cache dir. + self.copyto(os.path.join(mydir, "data", "bogus-0.1-common-111.tar.gz"), common.get_install_cache_dir()) + + def test_success(self): + self.options.package = [self.pkg] + autobuild_tool_install.AutobuildTool().run(self.options) + assert os.path.exists(os.path.join(INSTALL_DIR, "lib", "bogus.lib")) + assert os.path.exists(os.path.join(INSTALL_DIR, "include", "bogus.h")) + with CaptureStdout() as stream: + self.options.list_dirty=True + autobuild_tool_install.AutobuildTool().run(self.options) + self.assertEqual(stream.getvalue(), 'Dirty Packages: \n') + +# ------------------------------------- ------------------------------------- +class TestInstallCachedXZArchive(BaseTest): + def setup_method(self, method): + super(TestInstallCachedXZArchive, self).setup_method(method) + self.pkg = "bogus" + # make sure that the archive is not in the server + clean_file(os.path.join(SERVER_DIR, "bogus-0.1-common-111.tar.xz")) + assert not os.path.exists(in_dir(SERVER_DIR, "bogus-0.1-common-111.tar.xz")) + self.cache_name = in_dir(common.get_install_cache_dir(), "bogus-0.1-common-111.tar.xz") + + # Instead copy directly to cache dir. + self.copyto(os.path.join(mydir, "data", "bogus-0.1-common-111.tar.xz"), common.get_install_cache_dir()) + + def test_success(self): + self.options.package = [self.pkg] + autobuild_tool_install.AutobuildTool().run(self.options) + assert os.path.exists(os.path.join(INSTALL_DIR, "lib", "bogus.lib")) + assert os.path.exists(os.path.join(INSTALL_DIR, "include", "bogus.h")) + with CaptureStdout() as stream: + self.options.list_dirty=True + autobuild_tool_install.AutobuildTool().run(self.options) + self.assertEqual(stream.getvalue(), 'Dirty Packages: \n') + +# ------------------------------------- ------------------------------------- +class TestInstallCachedZSTArchive(BaseTest): + def setup_method(self, method): + super(TestInstallCachedZSTArchive, self).setup_method(method) + self.pkg = "bogus" + # make sure that the archive is not in the server + clean_file(os.path.join(SERVER_DIR, "bogus-0.1-common-111.tar.zst")) + assert not os.path.exists(in_dir(SERVER_DIR, "bogus-0.1-common-111.tar.zst")) + self.cache_name = in_dir(common.get_install_cache_dir(), "bogus-0.1-common-111.tar.zst") + + # Instead copy directly to cache dir. + self.copyto(os.path.join(mydir, "data", "bogus-0.1-common-111.tar.zst"), common.get_install_cache_dir()) + + def test_success(self): + self.options.package = [self.pkg] + autobuild_tool_install.AutobuildTool().run(self.options) + assert os.path.exists(os.path.join(INSTALL_DIR, "lib", "bogus.lib")) + assert os.path.exists(os.path.join(INSTALL_DIR, "include", "bogus.h")) + with CaptureStdout() as stream: + self.options.list_dirty=True + autobuild_tool_install.AutobuildTool().run(self.options) + self.assertEqual(stream.getvalue(), 'Dirty Packages: \n') + +# ------------------------------------- ------------------------------------- +class TestInstallCachedZIPArchive(BaseTest): + def setup_method(self, method): + super(TestInstallCachedZIPArchive, self).setup_method(method) + self.pkg = "bogus" + # make sure that the archive is not in the server + clean_file(os.path.join(SERVER_DIR, "bogus-0.1-common-111.zip")) + assert not os.path.exists(in_dir(SERVER_DIR, "bogus-0.1-common-111.zip")) + self.cache_name = in_dir(common.get_install_cache_dir(), "bogus-0.1-common-111.zip") + + # Instead copy directly to cache dir. + self.copyto(os.path.join(mydir, "data", "bogus-0.1-common-111.zip"), common.get_install_cache_dir()) + + def test_success(self): + self.options.package = [self.pkg] + autobuild_tool_install.AutobuildTool().run(self.options) + assert os.path.exists(os.path.join(INSTALL_DIR, "lib", "bogus.lib")) + assert os.path.exists(os.path.join(INSTALL_DIR, "include", "bogus.h")) + with CaptureStdout() as stream: + self.options.list_dirty=True + autobuild_tool_install.AutobuildTool().run(self.options) + self.assertEqual(stream.getvalue(), 'Dirty Packages: \n') + # ------------------------------------- ------------------------------------- class TestInstallLocalArchive(BaseTest): def setup_method(self, method): From e232d9a9c0ce565349afd5c7c8ffb4d892e9a473 Mon Sep 17 00:00:00 2001 From: Rye Mutt Date: Wed, 28 Sep 2022 20:34:14 -0400 Subject: [PATCH 4/4] Fix small codestyle issue --- autobuild/autobuild_tool_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuild/autobuild_tool_package.py b/autobuild/autobuild_tool_package.py index 72e8999..1bbad65 100644 --- a/autobuild/autobuild_tool_package.py +++ b/autobuild/autobuild_tool_package.py @@ -239,7 +239,7 @@ def package(config, build_directory, platform_name, archive_filename=None, archi else: archive_description = platform_description.archive format = _determine_archive_format(archive_format, archive_description) - if format == 'txz' or format == 'tbz2' or format == 'tgz' or format == 'tzst': + if format in ('txz', 'tbz2', 'tgz', 'tzst'): _create_tarfile(tarfilename, format, build_directory, files, results) elif format == 'zip': _create_zip_archive(tarfilename + '.zip', build_directory, files, results)