From 29b0f6310396e2f537d8594c077546c5c7c5188c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ru=C5=BEi=C4=8Dka?= Date: Mon, 2 Dec 2024 14:44:01 +0100 Subject: [PATCH 1/2] docs: Fix too short title underline in jq usage This fixes docs build warning: docs/usage.rst:330: WARNING: Title underline too short. --- docs/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index ab826af..2c4e0d6 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -327,7 +327,7 @@ post_data_type You can use ``pip install 'nvchecker[htmlparser]'``. Search with an JSON Parser (jq) -~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :: source = "jq" From eeddd56156c256efe389d67f8c0850aefcee2b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ru=C5=BEi=C4=8Dka?= Date: Wed, 27 Nov 2024 03:55:10 +0100 Subject: [PATCH 2/2] rpmrepo: new RPM repo source --- docs/usage.rst | 23 ++++++++++ nvchecker_source/rpmrepo.py | 83 +++++++++++++++++++++++++++++++++++++ setup.cfg | 2 + tests/test_rpmrepo.py | 19 +++++++++ 4 files changed, 127 insertions(+) create mode 100644 nvchecker_source/rpmrepo.py create mode 100644 tests/test_rpmrepo.py diff --git a/docs/usage.rst b/docs/usage.rst index 2c4e0d6..374f4d1 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -857,6 +857,29 @@ strip_release Note that either pkg or srcpkg needs to be specified (but not both) or the item name will be used as pkg. +Check RPM repository +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:: + + source = "rpmrepo" + +This enables you to check latest package versions in an arbitrary RPM repository in `repomd` format used by package managers such as ``dnf`` (Fedora, RHEL, AlmaLinux etc.) or ``zypper`` (openSUSE) without the need for native RPM tools. + +pkg + Name of the RPM package (you can also use ``rpmrepo`` as with other sources, but ``pkg`` is preferred for clarity) + +repo + URL of the repository (required, ``repodata/repomd.xml`` should be there) + +arch + Architecture of the RPM package (``binary``, ``src``, ``any``, ``x86_64``, ``aarch64``, etc, defaults to ``binary``) + +This source supports :ref:`list options`. + +.. note:: + An additional dependency "lxml" is required. + You can use ``pip install 'nvchecker[rpmrepo]'``. + Check Git repository ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :: diff --git a/nvchecker_source/rpmrepo.py b/nvchecker_source/rpmrepo.py new file mode 100644 index 0000000..08e6a58 --- /dev/null +++ b/nvchecker_source/rpmrepo.py @@ -0,0 +1,83 @@ +# MIT licensed +# Copyright (c) 2024 Jakub Ružička , et al. + +import asyncio +import gzip +import pathlib +import urllib + +import lxml.etree +from nvchecker.api import session, AsyncCache, Entry, KeyManager, VersionResult + + +# XML namespaces used in repodata (dead links haha) +NS = { + 'common': 'http://linux.duke.edu/metadata/common', + 'repo': 'http://linux.duke.edu/metadata/repo', + 'rpm': 'http://linux.duke.edu/metadata/rpm' +} + + +async def get_version( + name: str, conf: Entry, *, + cache: AsyncCache, keymanager: KeyManager, + **kwargs, +) -> VersionResult: + repo = conf['repo'] + arch = conf.get('arch', 'binary') + pkg = conf.get('pkg') + if not pkg: + pkg = conf.get('rpmrepo', name) + + repo_url = urllib.parse.urlparse(repo) + repo_path = pathlib.PurePosixPath(repo_url.path) + + # get the url of repomd.xml + repomd_path = repo_path / 'repodata' / 'repomd.xml' + repomd_url = repo_url._replace(path=str(repomd_path)).geturl() + # download repomd.xml (use cache) + repomd_body = await cache.get(repomd_url, get_file) + # parse repomd.xml + repomd_xml = lxml.etree.fromstring(repomd_body) + + # get the url of *primary.xml.gz + primary_element = repomd_xml.find('repo:data[@type="primary"]/repo:location', namespaces=NS) + primary_path = repo_path / primary_element.get('href') + primary_url = repo_url._replace(path=str(primary_path)).geturl() + # download and decompress *primary.xml.gz (use cache) + primary_body = await cache.get(primary_url, get_file_gz) + # parse *primary.xml metadata + metadata = lxml.etree.fromstring(primary_body) + + # use set to eliminate duplication + versions_set = set() + # iterate package metadata + for el in metadata.findall(f'common:package[common:name="{pkg}"]', namespaces=NS): + pkg_arch = el.findtext('common:arch', namespaces=NS) + + # filter bych arch + if arch == 'binary': + if pkg_arch == 'src': + continue + elif arch != 'any': + if pkg_arch != arch: + continue + + version_info = el.find('common:version', namespaces=NS) + version = version_info.get('ver') + versions_set.add(version) + + versions = list(versions_set) + return versions + + +async def get_file(url: str) -> str: + res = await session.get(url) + return res.body + + +async def get_file_gz(url: str) -> str: + res = await session.get(url) + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, gzip.decompress, res.body) diff --git a/setup.cfg b/setup.cfg index efa5296..2d757f4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -64,6 +64,8 @@ pypi = packaging htmlparser = lxml +rpmrepo = + lxml jq = jq diff --git a/tests/test_rpmrepo.py b/tests/test_rpmrepo.py new file mode 100644 index 0000000..f8f4638 --- /dev/null +++ b/tests/test_rpmrepo.py @@ -0,0 +1,19 @@ +# MIT licensed +# Copyright (c) 2024 Jakub Ružička , et al. + +import pytest +pytestmark = [pytest.mark.asyncio, pytest.mark.needs_net] + +async def test_rpmrepo_fedora(get_version): + assert await get_version("knot_fedora-39", { + "source": "rpmrepo", + "pkg": "knot", + "repo": "http://ftp.sh.cvut.cz/fedora/linux/updates/39/Everything/x86_64/", + }) == "3.3.9" + +async def test_rpmrepo_alma(get_version): + assert await get_version("knot_fedora-39", { + "source": "rpmrepo", + "pkg": "tmux", + "repo": "http://ftp.sh.cvut.cz/almalinux/9.5/BaseOS/x86_64/os/", + }) == "3.2a"