From 7a9c8b9059179cc1f9c59134bd65d0a221638ff9 Mon Sep 17 00:00:00 2001 From: Maciej Delmanowski Date: Fri, 11 Oct 2024 11:49:57 +0200 Subject: [PATCH 1/7] [ci] Don't pin versions of specific packages in CI Continuous Integration should use newest versions of packages to check if the code is compatible. --- requirements-integration.txt | 45 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/requirements-integration.txt b/requirements-integration.txt index 1936ca9a09..278c94a6ac 100644 --- a/requirements-integration.txt +++ b/requirements-integration.txt @@ -1,28 +1,29 @@ # Copyright (C) 2023 Berkhan Berkdemir +# Copyright (C) 2024 Maciej Delmanowski # Copyright (C) 2024 DebOps # SPDX-License-Identifier: GPL-3.0-or-later # List of Python packages used by GitHub CI pipeline for continuous integration -ansible==8.0.0a1 -ansible-core==2.15.0b2 -cffi==1.16.0 -cryptography==41.0.5 -distro==1.8.0 -dnspython==2.4.2 -future==0.18.3 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -netaddr==0.9.0 -packaging==23.2 -passlib==1.7.4 -pyasn1==0.5.0 -pyasn1-modules==0.3.0 -pycparser==2.21 -pyOpenSSL==23.2.0 -python-dotenv==1.0.0 -python-ldap==3.4.3 -pyxdg==0.28 -PyYAML==6.0.1 -resolvelib==0.8.1 -toml==0.10.2 +ansible +ansible-core +cffi +cryptography +distro +dnspython +future +Jinja2 +MarkupSafe +netaddr +packaging +passlib +pyasn1 +pyasn1-modules +pycparser +pyOpenSSL +python-dotenv +python-ldap +pyxdg +PyYAML +resolvelib +toml From 6d2bef7a3eb3604e527518e56df3c8a0f8daa865 Mon Sep 17 00:00:00 2001 From: Maciej Delmanowski Date: Fri, 11 Oct 2024 12:57:54 +0200 Subject: [PATCH 2/7] Change the way DebOps checks current user The 'os.getlogin()' function relies on the glibc 'getlogin()' function, which requires a control terminal (tty, pty) to work. In an environment without the control terminal (systemd service, remote execution) it results in the 'OSError: [Errno 6] No such device or address' exception. Switching to 'getpass.getuser()' function is a recommended alternative. --- src/debops/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/debops/cli.py b/src/debops/cli.py index f805a126d0..d6aba94760 100644 --- a/src/debops/cli.py +++ b/src/debops/cli.py @@ -10,6 +10,7 @@ from .ansibleplaybookrunner import AnsiblePlaybookRunner from .envrunner import EnvRunner import logging +import getpass import sys import os @@ -23,7 +24,7 @@ def __init__(self, args=None): self.config = Configuration() self.parsed_args = Subcommands(self.args) logger.debug('Command line interpreter initialized') - logger.debug('Control user=' + os.getlogin()) + logger.debug('Control user=' + getpass.getuser()) if self.parsed_args.section == 'project': if self.parsed_args.command == 'init': From cc488e0c5c8842757589af7a3b8b128bedc81e8e Mon Sep 17 00:00:00 2001 From: Maciej Delmanowski Date: Fri, 11 Oct 2024 14:10:19 +0200 Subject: [PATCH 3/7] [ci] Set no_log=false for all tasks in test env This reveals all passwords and sensitive data inside of the DebOps CI Pipeline on GitHub Actions, but since all passwords are randomly generated, this shouldn't be a security risk. --- .github/workflows/prepare-ci-pipeline/action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prepare-ci-pipeline/action.yml b/.github/workflows/prepare-ci-pipeline/action.yml index b581f71d39..3de9b37a2d 100644 --- a/.github/workflows/prepare-ci-pipeline/action.yml +++ b/.github/workflows/prepare-ci-pipeline/action.yml @@ -1,6 +1,6 @@ --- -# Copyright (C) 2021 Maciej Delmanowski -# Copyright (C) 2023 Berkhan Berkdemir +# Copyright (C) 2021-2024 Maciej Delmanowski +# Copyright (C) 2023 Berkhan Berkdemir # Copyright (C) 2021-2024 DebOps # SPDX-License-Identifier: GPL-3.0-or-later @@ -39,5 +39,7 @@ runs: cat <> ansible/inventory/hosts [debops_all_hosts] localhost ansible_connection=local + [all:vars] + debops__no_log=false EOL shell: bash From 58f5bfc2a63b71a78fccc36ba12e2f5c019880f0 Mon Sep 17 00:00:00 2001 From: Maciej Delmanowski Date: Fri, 11 Oct 2024 14:11:43 +0200 Subject: [PATCH 4/7] [ci] Use system-wide Python interpreter for tests This hopefully avoids an issue with Ansible executed against localhost to randomly switch to a different Python environment. We want to be able to use Python modules installed via APT, for example for MariaDB/MySQL access. --- .github/workflows/prepare-ci-pipeline/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prepare-ci-pipeline/action.yml b/.github/workflows/prepare-ci-pipeline/action.yml index 3de9b37a2d..62b2ab553c 100644 --- a/.github/workflows/prepare-ci-pipeline/action.yml +++ b/.github/workflows/prepare-ci-pipeline/action.yml @@ -38,7 +38,7 @@ runs: cd ~/src/controller cat <> ansible/inventory/hosts [debops_all_hosts] - localhost ansible_connection=local + localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 [all:vars] debops__no_log=false EOL From 5c5b6ecc4795c6dfe1bdb62ddfde88bc4eeea5c1 Mon Sep 17 00:00:00 2001 From: Maciej Delmanowski Date: Fri, 11 Oct 2024 17:36:33 +0200 Subject: [PATCH 5/7] [ci] Increase timeout for documentation builds --- .github/workflows/continuous-integration.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 51f58f230b..f94a5312af 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -206,6 +206,7 @@ jobs: debops-wheel: runs-on: 'ubuntu-latest' + timeout-minutes: 120 steps: - name: Add ~/.local/bin to $PATH @@ -233,6 +234,7 @@ jobs: debops-sdist: runs-on: 'ubuntu-latest' + timeout-minutes: 120 steps: - name: Add ~/.local/bin to $PATH @@ -282,6 +284,7 @@ jobs: documentation: runs-on: 'ubuntu-latest' + timeout-minutes: 120 steps: - name: Add ~/.local/bin to $PATH From eb27e50aa644c868eab3e1c1f28a15838b78c501 Mon Sep 17 00:00:00 2001 From: Maciej Delmanowski Date: Mon, 14 Oct 2024 14:32:26 +0200 Subject: [PATCH 6/7] [docs] Remove 'html_theme_path' variable This variable causes a warning on newer Sphinx versions, which results in an error in the CI environment and fails the checks. --- docs/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index c83c05709a..7fbcba64ab 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -206,7 +206,6 @@ def setup(app): try: import sphinx_rtd_theme html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] html_theme_options = { 'collapse_navigation': True, 'logo_only': False From c9d99261ceb4a0ac6735ba069d1f57d05796f75b Mon Sep 17 00:00:00 2001 From: Maciej Delmanowski Date: Mon, 14 Oct 2024 14:34:59 +0200 Subject: [PATCH 7/7] [ci] Use Sphinx v5.3.0 in test environment The newest Sphinx version (v8.1.3) is very slow, no idea why for now. Let's try an older Sphinx in CI to see if this helps make the tests faster. --- .github/workflows/continuous-integration.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index f94a5312af..2b47ace8d5 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -222,7 +222,7 @@ jobs: sudo apt-get -qq update sudo apt-get -yq install pandoc pandoc-data graphviz pip3 install --user --upgrade pip wheel "setuptools==65.7.0" pypandoc - pip3 install --user --upgrade sphinx sphinx-autobuild sphinx_rtd_theme + pip3 install --user --upgrade sphinx==5.3.0 sphinx-autobuild sphinx_rtd_theme - name: Build DebOps Python wheel package run: | @@ -250,7 +250,7 @@ jobs: sudo apt-get -qq update sudo apt-get -yq install pandoc pandoc-data graphviz pip3 install --user --upgrade pip wheel "setuptools==65.7.0" pypandoc - pip3 install --user --upgrade sphinx sphinx-autobuild sphinx_rtd_theme + pip3 install --user --upgrade sphinx==5.3.0 sphinx-autobuild sphinx_rtd_theme - name: Build DebOps Python sdist package run: | @@ -298,7 +298,7 @@ jobs: sudo apt-get -qq update sudo apt-get install -yq graphviz pip3 install --user --upgrade pip wheel "setuptools==65.7.0" - pip3 install --user --upgrade sphinx sphinx-autobuild sphinx_rtd-theme + pip3 install --user --upgrade sphinx==5.3.0 sphinx-autobuild sphinx_rtd-theme - name: Build documentation site run: |