From 7f31521d545e3554f1989e0477985a345428b135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Fri, 29 Nov 2024 11:41:42 +0200 Subject: [PATCH 01/30] Clean-up master.cfg, initial step * Remove star imports * master-private.cfg always loaded from base directory for all masters. --- master-docker-nonstandard-2/master.cfg | 21 +++++++-- master-docker-nonstandard/master.cfg | 21 +++++++-- master-galera/master.cfg | 23 ++++++++-- master-libvirt/master.cfg | 21 +++++++-- master-nonlatent/master.cfg | 22 +++++++-- master-protected-branches/master.cfg | 21 +++++++-- master-web/master.cfg | 19 +++++++- master.cfg | 62 ++++++++++++++++---------- validate_master_cfg.sh | 15 ++----- 9 files changed, 165 insertions(+), 60 deletions(-) diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index 1a018186..6b50952a 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -25,14 +25,27 @@ from utils import * FQDN = os.environ["BUILDMASTER_WG_IP"] -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} +cfg_dir = os.path.abspath(os.path.dirname(__file__)) + +# Non autogen master. For now the directory structure is: +# +# └── +#    ├── buildbot.tac +#     └── master.cfg +# +# Non autogen masters load from for now. +base_dir = os.path.abspath(f'{cfg_dir}/../') # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: + exec(file.read(), config, {}) + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + ####### BUILDBOT SERVICES diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 20515a0f..a4610764 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -25,14 +25,27 @@ from utils import * FQDN = os.environ["BUILDMASTER_WG_IP"] -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} +cfg_dir = os.path.abspath(os.path.dirname(__file__)) + +# Non autogen master. For now the directory structure is: +# +# └── +#    ├── buildbot.tac +#     └── master.cfg +# +# Non autogen masters load from for now. +base_dir = os.path.abspath(f'{cfg_dir}/../') # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: + exec(file.read(), config, {}) + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + ####### BUILDBOT SERVICES diff --git a/master-galera/master.cfg b/master-galera/master.cfg index e5e53b59..fe5a2a5d 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -11,6 +11,8 @@ from twisted.internet import defer import sys import os import docker +import yaml + from datetime import timedelta sys.setrecursionlimit(10000) @@ -19,14 +21,27 @@ sys.path.append(os.getcwd() + "/..") from constants import * from utils import * -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} +cfg_dir = os.path.abspath(os.path.dirname(__file__)) + +# Non autogen master. For now the directory structure is: +# +# └── +#    ├── buildbot.tac +#     └── master.cfg +# +# Non autogen masters load from for now. +base_dir = os.path.abspath(f'{cfg_dir}/../') # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: + exec(file.read(), config, {}) + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + FQDN = os.environ["BUILDMASTER_WG_IP"] diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 6f7ee94d..1614780e 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -21,14 +21,27 @@ sys.path.insert(0, "/srv/buildbot/master") from utils import * from constants import OS_INFO -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} +cfg_dir = os.path.abspath(os.path.dirname(__file__)) + +# Non autogen master. For now the directory structure is: +# +# └── +#    ├── buildbot.tac +#     └── master.cfg +# +# Non autogen masters load from for now. +base_dir = os.path.abspath(f'{cfg_dir}/../') # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: + exec(file.read(), config, {}) + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + ####### PROJECT IDENTITY diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index abe3a8bc..c22ebdd3 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -14,6 +14,7 @@ from twisted.internet import defer import docker import os import sys +import yaml sys.setrecursionlimit(10000) @@ -26,14 +27,27 @@ from utils import * ####### VARIABLES vsWarningPattern = "^.*: warning C.*$" -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} +cfg_dir = os.path.abspath(os.path.dirname(__file__)) + +# Non autogen master. For now the directory structure is: +# +# └── +#    ├── buildbot.tac +#     └── master.cfg +# +# Non autogen masters load from for now. +base_dir = os.path.abspath(f'{cfg_dir}/../') # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: + exec(file.read(), config, {}) + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + ####### PROJECT IDENTITY diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index fb7db83f..5dfa9f15 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -24,14 +24,27 @@ from locks import * from schedulers_definition import SCHEDULERS from common_factories import * -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} +cfg_dir = os.path.abspath(os.path.dirname(__file__)) + +# Non autogen master. For now the directory structure is: +# +# └── +#    ├── buildbot.tac +#     └── master.cfg +# +# Non autogen masters load from for now. +base_dir = os.path.abspath(f'{cfg_dir}/../') # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: + exec(file.read(), config, {}) + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + ####### BUILDBOT SERVICES diff --git a/master-web/master.cfg b/master-web/master.cfg index eaac6d69..a916a6a7 100644 --- a/master-web/master.cfg +++ b/master-web/master.cfg @@ -28,12 +28,29 @@ sys.setrecursionlimit(10000) sys.path.append(os.getcwd() + "/..") from constants import * +cfg_dir = os.path.abspath(os.path.dirname(__file__)) + +# Non autogen master. For now the directory structure is: +# +# └── +#    ├── buildbot.tac +#     └── master.cfg +# +# Non autogen masters load from for now. +base_dir = os.path.abspath(f'{cfg_dir}/../') + +# Load the slave, database passwords and 3rd-party tokens from an external private file, so +# that the rest of the configuration can be public. +config = {"private": {}} +with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: + exec(file.read(), config, {}) + c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external # private file, so that the rest of the configuration can be public. config = {"private": {}} -exec(open("../master-private.cfg").read(), config, {}) +exec(open("master-private.cfg").read(), config, {}) ####### PROJECT IDENTITY diff --git a/master.cfg b/master.cfg index f112868b..1b4bf645 100644 --- a/master.cfg +++ b/master.cfg @@ -1,42 +1,58 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta -from twisted.internet import defer - -import docker import os -import sys -sys.setrecursionlimit(10000) +import yaml -sys.path.insert(0, "/srv/buildbot/master") - -from common_factories import * -from constants import * -from locks import * +from buildbot.plugins import reporters, secrets, steps, util +from buildbot.process.properties import Property +from common_factories import ( + getQuickBuildFactory, + getRpmAutobakeFactory, + getSourceTarball, +) +from constants import ( + BUILDERS_GALERA_MTR, + GITHUB_STATUS_BUILDERS, + SAVED_PACKAGE_BRANCHES, + OS_INFO, +) +from locks import getLocks from schedulers_definition import SCHEDULERS -from utils import * +from utils import ( + canStartBuild, + createDebRepo, + createWorker, + dockerfile, + hasDockerLibrary, + hasPackagesGenerated, + hasInstall, + hasUpgrade, + ls2string, + nextBuild, + printEnv, + savePackageIfBranchMatch, + uploadDebArtifacts, +) with open("master-config.yaml", "r") as f: master_config = yaml.safe_load(f) + +# Load the slave, database passwords and 3rd-party tokens from an external +# private file, so that the rest of the configuration can be public. +config = {"private": {}} +exec(open("master-private.cfg").read(), config, {}) + # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. c = BuildmasterConfig = {} -# Load the slave, database passwords and 3rd-party tokens from an external private file, so -# that the rest of the configuration can be public. -config = {"private": {}} -exec(open("master-private.cfg").read(), config, {}) -####### BUILDBOT SERVICES +####### +# BUILDBOT SERVICES +####### # 'services' is a list of BuildbotService items like reporter targets. The # status of each build will be pushed to these targets. buildbot/reporters/*.py diff --git a/validate_master_cfg.sh b/validate_master_cfg.sh index 62cbef55..c61e8c00 100755 --- a/validate_master_cfg.sh +++ b/validate_master_cfg.sh @@ -69,15 +69,6 @@ command -v python3 >/dev/null || err "python3 command not found" python3 define_masters.py -echo "Checking master.cfg" -# Port is set by generate-config.py (docker-compose), not present in .env -$RUNC run -i -v "$(pwd):/srv/buildbot/master" \ - --env PORT=1234 \ - --env-file <(sed "s/='\([^']*\)'/=\1/" $ENVFILE) \ - -w /srv/buildbot/master \ - $IMAGE \ - buildbot checkconfig master.cfg -echo -e "done\n" # not checking libvirt config file (//TEMP we need to find a solution # to not check ssh connection) for dir in autogen/* \ @@ -91,8 +82,8 @@ for dir in autogen/* \ $RUNC run -i -v "$(pwd):/srv/buildbot/master" \ --env PORT=1234 \ --env-file <(sed "s/='\([^']*\)'/=\1/" $ENVFILE) \ - -w "/srv/buildbot/master/$dir" \ + -w "/srv/buildbot/master/" \ $IMAGE \ - buildbot checkconfig master.cfg + bash -c "buildbot checkconfig $dir/master.cfg" echo -e "done\n" -done \ No newline at end of file +done From f68a6b1f0ab89e0a66381367ae52055ff3e3858f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 8 Jan 2025 11:25:20 +0200 Subject: [PATCH 02/30] Update buildbot.tac and master.cfg for all autogen masters buildbot.tac as the entrypoint for a buildbot master should use relative paths based on the deployment directory structure. Furthermore, the basedir should be set to so that all imports will work relative to , not the location of buildbot.tac or master.cfg. --- buildbot.tac | 39 +++++++++++++++++++++++++++------------ master.cfg | 10 +++++++--- validate_master_cfg.sh | 2 +- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/buildbot.tac b/buildbot.tac index 98055695..79cbe6b1 100644 --- a/buildbot.tac +++ b/buildbot.tac @@ -1,30 +1,45 @@ +# -*- python -*- +# ex: set filetype=python: import os from twisted.application import service +from twisted.python.log import FileLogObserver, ILogObserver +from twisted.python.logfile import LogFile from buildbot.master import BuildMaster -basedir = "." -log_basedir = "/var/log/buildbot/" +# This buildbot.tac file is a basis for all "autogen" masters. +# The folder structure for autogen masters is: +# +# autogen +# └── aarch64-master-0 +#    ├── buildbot.tac +#     ├── master.cfg +#     ├── master-config.yaml +#     └── master-private.cfg +# +# Thus basedir is two levels below this file"s position. +buildbot_tac_dir = os.path.abspath(os.path.dirname(__file__)) +basedir = os.path.abspath(f"{buildbot_tac_dir}/../../") + +# Hard coded as it runs in containers. +# TODO(cvicentiu) this should come as an environment variable. +log_basedir = "/var/log/buildbot" rotateLength = 20000000 maxRotatedFiles = 30 -configfile = "master.cfg" +# Last two directories. autogen and +cfg_from_basedir = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:] + [ + "master.cfg" +] +configfile = os.path.join(*cfg_from_basedir) # Default umask for server umask = None -# if this is a relocatable tac file, get the directory containing the TAC -if basedir == ".": - import os - - basedir = os.path.abspath(os.path.dirname(__file__)) - # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. -application = service.Application('buildmaster') # fmt: skip -from twisted.python.log import FileLogObserver, ILogObserver -from twisted.python.logfile import LogFile +application = service.Application("buildmaster") # fmt: skip logfile = LogFile.fromFullPath( os.path.join(log_basedir, "%s"), diff --git a/master.cfg b/master.cfg index 1b4bf645..1c63ed60 100644 --- a/master.cfg +++ b/master.cfg @@ -36,14 +36,18 @@ from utils import ( uploadDebArtifacts, ) -with open("master-config.yaml", "r") as f: - master_config = yaml.safe_load(f) +cfg_dir = os.path.abspath(os.path.dirname(__file__)) +# Autogen master, see buildbot.tac for why this is the case. +base_dir = os.path.abspath(f'{cfg_dir}/../../') +with open(os.path.join(cfg_dir, "master-config.yaml"), "r") as file: + master_config = yaml.safe_load(file) # Load the slave, database passwords and 3rd-party tokens from an external # private file, so that the rest of the configuration can be public. config = {"private": {}} -exec(open("master-private.cfg").read(), config, {}) +with open(os.path.join(cfg_dir, "master-private.cfg"), "r") as file: + exec(file.read(), config, {}) # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. diff --git a/validate_master_cfg.sh b/validate_master_cfg.sh index c61e8c00..ba2bd236 100755 --- a/validate_master_cfg.sh +++ b/validate_master_cfg.sh @@ -85,5 +85,5 @@ for dir in autogen/* \ -w "/srv/buildbot/master/" \ $IMAGE \ bash -c "buildbot checkconfig $dir/master.cfg" - echo -e "done\n" + echo "done" done From f56e9e4b1a9a22dd6d5822415da646b3bbb02b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 3 Dec 2024 20:07:53 +0200 Subject: [PATCH 03/30] Refactor createWorker from utils.py Simplify and ensure it returns a 3-value tuple. worker_name must not end in dash any more. --- master-docker-nonstandard-2/master.cfg | 22 +++++------ master-docker-nonstandard/master.cfg | 53 ++++++++++++-------------- master-galera/master.cfg | 13 +++---- master-protected-branches/master.cfg | 49 +++++++++++------------- master.cfg | 40 +++++-------------- utils.py | 37 +++++++++--------- 6 files changed, 89 insertions(+), 125 deletions(-) diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index 6b50952a..23c2136f 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -13,6 +13,7 @@ from twisted.internet import defer import docker import os import sys +from collections import defaultdict sys.path.insert(0, "/srv/buildbot/master") sys.setrecursionlimit(10000) @@ -121,7 +122,7 @@ c["schedulers"] = SCHEDULERS # worker name and password must be configured on the worker. c["workers"] = [] -workers = {} +workers = defaultdict(list) def addWorker( @@ -133,7 +134,7 @@ def addWorker( save_packages=False, shm_size="15G", ): - name, instance = createWorker( + base_name, name, instance = createWorker( worker_name_prefix, worker_id, worker_type, @@ -143,21 +144,18 @@ def addWorker( shm_size, ) - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - + workers[base_name].append(name) c["workers"].append(instance) + # Docker workers fqdn = os.environ["BUILDMASTER_WG_IP"] addWorker( "amd-bbw", 1, - "-debian-12-32-bit", + "debian-12-32-bit", os.environ["CONTAINER_REGISTRY_URL"] + "debian12-386", jobs=10, @@ -168,7 +166,7 @@ addWorker( addWorker( "amd-bbw", 2, - "-debian-12-32-bit", + "debian-12-32-bit", os.environ["CONTAINER_REGISTRY_URL"] + "debian12-386", jobs=10, @@ -179,7 +177,7 @@ addWorker( addWorker( "apexis-bbw", 3, - "-debian-12-32-bit", + "debian-12-32-bit", os.environ["CONTAINER_REGISTRY_URL"] + "debian12-386", jobs=10, @@ -190,7 +188,7 @@ addWorker( addWorker( "apexis-bbw", 3, - "-msan-clang-debian-11", + "msan-clang-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan", jobs=20, @@ -200,7 +198,7 @@ addWorker( addWorker( "apexis-bbw", 3, - "-ubuntu-2204-jepsen-mariadb", + "ubuntu-2204-jepsen-mariadb", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04-jepsen-mariadb", jobs=5, diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index a4610764..35c8e6ce 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -13,6 +13,7 @@ from twisted.internet import defer import docker import os import sys +from collections import defaultdict sys.path.insert(0, "/srv/buildbot/master") sys.setrecursionlimit(10000) @@ -121,7 +122,7 @@ c["schedulers"] = SCHEDULERS # worker name and password must be configured on the worker. c["workers"] = [] -workers = {} +workers = defaultdict(list) def addWorker( @@ -133,7 +134,7 @@ def addWorker( save_packages=False, shm_size="15G", ): - name, instance = createWorker( + base_name, name, instance = createWorker( worker_name_prefix, worker_id, worker_type, @@ -143,11 +144,7 @@ def addWorker( shm_size, ) - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - + workers[base_name].append(name) c["workers"].append(instance) @@ -248,7 +245,7 @@ c["workers"].append( addWorker( "hz-bbw", 6, - "-bigtest-ubuntu-2004", + "bigtest-ubuntu-2004", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=20, @@ -260,7 +257,7 @@ for w_name in ["ppc64le-osuosl-bbw"]: addWorker( w_name, 1, - "-ubuntu-2004", + "ubuntu-2004", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, @@ -270,7 +267,7 @@ for w_name in ["ppc64le-osuosl-bbw"]: addWorker( w_name, 1, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=30, @@ -288,7 +285,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( w_name, i, - "-aocc-debian-11", + "aocc-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-aocc", jobs=jobs, @@ -297,7 +294,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( w_name, i, - "-asan-ubuntu-2404", + "asan-ubuntu-2404", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu24.04", jobs=jobs, @@ -306,7 +303,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( w_name, i, - "-icc-ubuntu-2204", + "icc-ubuntu-2204", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04-icc", jobs=jobs, @@ -315,7 +312,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( w_name, i, - "-ubuntu-2004", + "ubuntu-2004", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=jobs, @@ -326,7 +323,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: addWorker( "amd-bbw", 1, - "-valgrind-fedora-40", + "valgrind-fedora-40", os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, @@ -335,7 +332,7 @@ addWorker( addWorker( "amd-bbw", 2, - "-valgrind-fedora-40", + "valgrind-fedora-40", os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, @@ -344,7 +341,7 @@ addWorker( addWorker( "hz-bbw", 6, - "-valgrind-fedora-40", + "valgrind-fedora-40", os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, @@ -354,7 +351,7 @@ addWorker( addWorker( "hz-bbw", 1, - "-msan-clang-16-debian-11", + "msan-clang-16-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=20, @@ -363,7 +360,7 @@ addWorker( addWorker( "hz-bbw", 4, - "-msan-clang-16-debian-11", + "msan-clang-16-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=20, @@ -372,7 +369,7 @@ addWorker( addWorker( "hz-bbw", 5, - "-msan-clang-16-debian-11", + "msan-clang-16-debian-11", os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=30, @@ -382,7 +379,7 @@ addWorker( addWorker( "hz-bbw", 2, - "-debian-12", + "debian-12", os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=20, @@ -391,7 +388,7 @@ addWorker( addWorker( "hz-bbw", 5, - "-debian-12", + "debian-12", os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=20, @@ -401,7 +398,7 @@ addWorker( addWorker( "aarch64-bbw", 6, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=10, @@ -411,7 +408,7 @@ addWorker( addWorker( "hz-bbw", 7, - "-centos-7-bintar", + "centos-7-bintar", os.environ["CONTAINER_REGISTRY_URL"] + "centos7-bintar", jobs=10, @@ -421,7 +418,7 @@ addWorker( addWorker( "hz-bbw", 7, - "-almalinux-8-bintar", + "almalinux-8-bintar", os.environ["CONTAINER_REGISTRY_URL"] + "almalinux8-bintar", jobs=10, @@ -431,7 +428,7 @@ addWorker( addWorker( "s390x-bbw", 1, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, @@ -441,7 +438,7 @@ addWorker( addWorker( "s390x-bbw", 2, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, @@ -451,7 +448,7 @@ addWorker( addWorker( "s390x-bbw", 3, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, diff --git a/master-galera/master.cfg b/master-galera/master.cfg index fe5a2a5d..ab49aa23 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -14,6 +14,7 @@ import docker import yaml from datetime import timedelta +from collections import defaultdict sys.setrecursionlimit(10000) @@ -169,7 +170,7 @@ c["workers"].append( # Docker workers GALERA_PACKAGES = os.environ["GALERA_PACKAGES_DIR"] -workers = {} +workers = defaultdict(list) def addWorker( worker_name_prefix, @@ -180,7 +181,7 @@ def addWorker( save_packages=False, shm_size="15G", ): - name, instance = createWorker( + base_name, name, instance = createWorker( worker_name_prefix, worker_id, worker_type, @@ -196,11 +197,7 @@ def addWorker( ], ) - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - + workers[base_name].append(name) c["workers"].append(instance) @@ -251,7 +248,7 @@ for platform in ALL_PLATFORMS: addWorker( w_name, i, - "-" + os_name, + os_name, quay_name, jobs=jobs, save_packages=True, diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 5dfa9f15..57d582e6 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -1,28 +1,29 @@ # -*- python -*- # ex: set filetype=python: +import os +import sys +from collections import defaultdict +from datetime import timedelta + +import docker from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand +from buildbot.process.properties import Properties, Property +from buildbot.process.remotecommand import RemoteCommand from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver +from buildbot.steps.shell import Compile, SetPropertyFromCommand, ShellCommand, Test from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta from twisted.internet import defer -import docker -import os -import sys - sys.setrecursionlimit(10000) sys.path.insert(0, "/srv/buildbot/master") +from common_factories import * from constants import * -from utils import * from locks import * from schedulers_definition import SCHEDULERS -from common_factories import * +from utils import * cfg_dir = os.path.abspath(os.path.dirname(__file__)) @@ -157,7 +158,7 @@ c["workers"].append( ) ) -workers = {} +workers = defaultdict(list) def addWorker( @@ -169,7 +170,7 @@ def addWorker( save_packages=False, shm_size="15G", ): - name, instance = createWorker( + base_name, name, worker_instance = createWorker( worker_name_prefix, worker_id, worker_type, @@ -179,12 +180,8 @@ def addWorker( shm_size, ) - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - - c["workers"].append(instance) + workers[base_name].append(name) + c["workers"].append(worker_instance) for w_name in ["hz-bbw"]: @@ -193,7 +190,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-debian-11-debug-ps-embed", + "debian-11-debug-ps-embed", os.environ["CONTAINER_REGISTRY_URL"] + "debian11", jobs=14, save_packages=False, @@ -201,7 +198,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-debian-12", + "debian-12", os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=jobs, save_packages=True, @@ -209,7 +206,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-debian-12-debug-embed", + "debian-12-debug-embed", os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=14, save_packages=False, @@ -217,7 +214,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-fedora-40", + "fedora-40", os.environ["CONTAINER_REGISTRY_URL"] + "fedora40", jobs=jobs, save_packages=True, @@ -225,7 +222,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-last-N-failed", + "last-N-failed", os.environ["CONTAINER_REGISTRY_URL"] + "rhel9", jobs=jobs, save_packages=True, @@ -233,7 +230,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-ubuntu-2004-clang", + "ubuntu-2004-clang", "vladbogo/bb:amd64-ubuntu-2004-clang", jobs=jobs, save_packages=True, @@ -241,7 +238,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-ubuntu-2004-debug", + "ubuntu-2004-debug", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=14, save_packages=True, @@ -249,7 +246,7 @@ for w_name in ["hz-bbw"]: addWorker( w_name, i, - "-ubuntu-2204-debug-ps", + "ubuntu-2204-debug-ps", os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04", jobs=14, save_packages=False, diff --git a/master.cfg b/master.cfg index 1c63ed60..d9790994 100644 --- a/master.cfg +++ b/master.cfg @@ -2,6 +2,7 @@ # ex: set filetype=python: import os +from collections import defaultdict import yaml @@ -130,35 +131,7 @@ c["workers"] = [] # Docker workers -workers = {} - - -def addWorker( - worker_name_prefix, - worker_id, - worker_type, - dockerfile, - jobs=5, - save_packages=False, - shm_size="15G", -): - name, instance = createWorker( - worker_name_prefix, - worker_id, - worker_type, - dockerfile, - jobs, - save_packages, - shm_size, - ) - - if name[0] not in workers: - workers[name[0]] = [name[1]] - else: - workers[name[0]].append(name[1]) - - c["workers"].append(instance) - +workers = defaultdict(list) for w_name in master_config["workers"]: jobs = 7 @@ -184,15 +157,20 @@ for w_name in master_config["workers"]: if builder.startswith("x86"): os_name += "-i386" quay_name += "-386" - addWorker( + + base_name, name, worker_instance = createWorker( worker_name, worker_id, - "-" + os_name, + os_name, quay_name, jobs=jobs, save_packages=True, + shm_size="15G", ) + workers[base_name].append(name) + c["workers"].append(worker_instance) + ####### FACTORY CODE f_quick_build = getQuickBuildFactory("nm", mtrDbPool) diff --git a/utils.py b/utils.py index 40f8d088..76c70f33 100644 --- a/utils.py +++ b/utils.py @@ -79,25 +79,22 @@ def createWorker( "/srv/buildbot/packages:/mnt/packages", MASTER_PACKAGES + "/:/packages", ], -) -> Tuple[str, worker.DockerLatentWorker]: +) -> Tuple[str, str, worker.DockerLatentWorker]: worker_name = f"{worker_name_prefix}{worker_id}-docker" - name = f"{worker_name}{worker_type}" - - if worker_name_prefix.startswith("hz"): - b_name = "x64-bbw" - elif worker_name_prefix.startswith("intel"): - b_name = "x64-bbw" - elif worker_name_prefix.startswith("ppc64le"): - b_name = "ppc64le-bbw" - elif worker_name_prefix.startswith("amd"): - b_name = "x64-bbw" - elif worker_name_prefix.startswith("apexis"): - b_name = "x64-bbw" - elif worker_name_prefix.startswith("ns"): - b_name = "x64-bbw" - else: - b_name = worker_name_prefix - base_name = b_name + "-docker" + worker_type + name = f"{worker_name}-{worker_type}{worker_name_suffix}" + + # TODO(cvicentiu) Remove this list when refactoring YAML. + b_name = worker_name_prefix + X64_BUILDER_PREFIXES = ["hz", "intel", "amd", "apexis", "ns"] + PPC64LE_BUILDER_PREFIXES = ["ppc64le"] + for x64_prefix in X64_BUILDER_PREFIXES: + if worker_name_prefix.startswith(x64_prefix): + b_name = "x64-bbw" + for ppc_prefix in PPC64LE_BUILDER_PREFIXES: + if worker_name_prefix.startswith(ppc_prefix): + b_name = "ppc64le-bbw" + + base_name = f"{b_name}-docker-{worker_type}" # Set master FQDN - default to wireguard interface fqdn = os.environ["BUILDMASTER_WG_IP"] @@ -113,7 +110,7 @@ def createWorker( need_pull = False worker_instance = worker.DockerLatentWorker( - name + worker_name_suffix, + name, None, docker_host=private_config["private"]["docker_workers"][worker_name], image=image_str, @@ -135,7 +132,7 @@ def createWorker( volumes=volumes, properties={"jobs": jobs, "save_packages": save_packages}, ) - return ((base_name, name + worker_name_suffix), worker_instance) + return (base_name, name, worker_instance) def printEnv() -> steps.ShellCommand: From 6667789a006d7f3471b4193994932d0c7471d896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 3 Dec 2024 20:24:33 +0200 Subject: [PATCH 04/30] Reorganize master.cfg to simplify config specification --- master.cfg | 136 +++++++++++++++++++++++------------------------------ 1 file changed, 58 insertions(+), 78 deletions(-) diff --git a/master.cfg b/master.cfg index d9790994..da36c46f 100644 --- a/master.cfg +++ b/master.cfg @@ -16,8 +16,8 @@ from common_factories import ( from constants import ( BUILDERS_GALERA_MTR, GITHUB_STATUS_BUILDERS, - SAVED_PACKAGE_BRANCHES, OS_INFO, + SAVED_PACKAGE_BRANCHES, ) from locks import getLocks from schedulers_definition import SCHEDULERS @@ -27,8 +27,8 @@ from utils import ( createWorker, dockerfile, hasDockerLibrary, - hasPackagesGenerated, hasInstall, + hasPackagesGenerated, hasUpgrade, ls2string, nextBuild, @@ -39,7 +39,7 @@ from utils import ( cfg_dir = os.path.abspath(os.path.dirname(__file__)) # Autogen master, see buildbot.tac for why this is the case. -base_dir = os.path.abspath(f'{cfg_dir}/../../') +base_dir = os.path.abspath(f"{cfg_dir}/../../") with open(os.path.join(cfg_dir, "master-config.yaml"), "r") as file: master_config = yaml.safe_load(file) @@ -52,60 +52,63 @@ with open(os.path.join(cfg_dir, "master-private.cfg"), "r") as file: # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - - -####### -# BUILDBOT SERVICES -####### - -# 'services' is a list of BuildbotService items like reporter targets. The -# status of each build will be pushed to these targets. buildbot/reporters/*.py -# has a variety to choose from, like IRC bots. - - -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c["secretsProviders"] = [ - secrets.SecretInAFile(dirname=os.environ["MASTER_CREDENTIALS_DIR"]) -] - -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -c["protocols"] = {"pb": {"port": os.environ["PORT"]}} - -####### DB URL - -c["db"] = { +c = BuildmasterConfig = { + ####### + # PROJECT IDENTITY + ####### + # the 'title' string will appear at the top of this buildbot installation's + "title": os.getenv("TITLE", default="MariaDB CI"), + # home pages (linked to the 'titleURL'). + "titleURL": os.getenv("TITLE_URL", default="https://github.com/MariaDB/server"), + # the 'buildbotURL' string should point to the location where the + # buildbot's internal web server is visible. This typically uses the port + # number set in the 'www' entry below, but with an externally-visible host + # name which the buildbot cannot figure out without some help. + "buildbotURL": os.getenv( + "BUILDMASTER_URL", default="https://buildbot.mariadb.org/" + ), + # 'services' is a list of BuildbotService items like reporter targets. The + # status of each build will be pushed to these targets. buildbot/reporters/*.py + # has a variety to choose from, like IRC bots. + "services": [ + reporters.GitHubStatusPush( + token=config["private"]["gh_mdbci"]["access_token"], + context=util.Interpolate("buildbot/%(prop:buildername)s"), + startDescription="Build started.", + endDescription="Build done.", + verbose=True, + builders=GITHUB_STATUS_BUILDERS, + ) + ], + "secretsProviders": [ + secrets.SecretInAFile(dirname=os.environ["MASTER_CREDENTIALS_DIR"]) + ], + # 'protocols' contains information about protocols which master will use + # for communicating with workers. You must define at least 'port' option + # that workers could connect to your master with this protocol. + # 'port' must match the value configured into the workers (with their + # --master option) + "protocols": {"pb": {"port": os.environ["PORT"]}}, # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] + "db": {"db_url": config["private"]["db_url"]}, + # Disable net usage reports from being sent to buildbot.net + "buildbotNetUsageData": None, + # Configure the Schedulers, which decide how to react to incoming changes. + "schedulers": SCHEDULERS, + "logEncoding": "utf-8", + "multiMaster": True, + "mq": { + # Need to enable multimaster aware mq. Wamp is the only option for now. + "type": "wamp", + "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), + "realm": "realm1", + # valid are: none, critical, error, warn, info, debug, trace + "wamp_debug_level": "info", + }, } +####### DB URL + mtrDbPool = util.EqConnectionPool( "MySQLdb", config["private"]["db_host"], @@ -114,14 +117,6 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS - ####### WORKERS # The 'workers' list defines the set of recognized workers. Each element is @@ -150,10 +145,7 @@ for w_name in master_config["workers"]: if image_tag.startswith("ubuntu"): image_tag = image_tag[:-2] + "." + image_tag[-2:] - quay_name = ( - os.environ["CONTAINER_REGISTRY_URL"] - + image_tag - ) + quay_name = os.environ["CONTAINER_REGISTRY_URL"] + image_tag if builder.startswith("x86"): os_name += "-i386" quay_name += "-386" @@ -370,15 +362,3 @@ for builder in master_config["builders"]: factory=factory_instance, ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -} From 95881c0812aa8f6a0ee524e17e6dc5196a56bf49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 3 Dec 2024 21:29:43 +0200 Subject: [PATCH 05/30] Create base master config This will remove code duplication across master.cfg files. --- .github/workflows/bbm_deploy.yml | 1 + master.cfg | 70 ++++------------------------ master_common.py | 79 ++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 60 deletions(-) create mode 100644 master_common.py diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml index dd9d1ad7..61fea84a 100644 --- a/.github/workflows/bbm_deploy.yml +++ b/.github/workflows/bbm_deploy.yml @@ -31,6 +31,7 @@ on: - "locks.py" - "master-**" - "master.cfg" + - "master_common.py" - "os_info.yaml" - "script_templates/**" - "utils.py" diff --git a/master.cfg b/master.cfg index da36c46f..f9a9ad64 100644 --- a/master.cfg +++ b/master.cfg @@ -6,7 +6,7 @@ from collections import defaultdict import yaml -from buildbot.plugins import reporters, secrets, steps, util +from buildbot.plugins import steps, util from buildbot.process.properties import Property from common_factories import ( getQuickBuildFactory, @@ -20,7 +20,7 @@ from constants import ( SAVED_PACKAGE_BRANCHES, ) from locks import getLocks -from schedulers_definition import SCHEDULERS +from master_common import base_master_config from utils import ( canStartBuild, createDebRepo, @@ -52,63 +52,12 @@ with open(os.path.join(cfg_dir, "master-private.cfg"), "r") as file: # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = { - ####### - # PROJECT IDENTITY - ####### - # the 'title' string will appear at the top of this buildbot installation's - "title": os.getenv("TITLE", default="MariaDB CI"), - # home pages (linked to the 'titleURL'). - "titleURL": os.getenv("TITLE_URL", default="https://github.com/MariaDB/server"), - # the 'buildbotURL' string should point to the location where the - # buildbot's internal web server is visible. This typically uses the port - # number set in the 'www' entry below, but with an externally-visible host - # name which the buildbot cannot figure out without some help. - "buildbotURL": os.getenv( - "BUILDMASTER_URL", default="https://buildbot.mariadb.org/" - ), - # 'services' is a list of BuildbotService items like reporter targets. The - # status of each build will be pushed to these targets. buildbot/reporters/*.py - # has a variety to choose from, like IRC bots. - "services": [ - reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=util.Interpolate("buildbot/%(prop:buildername)s"), - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, - ) - ], - "secretsProviders": [ - secrets.SecretInAFile(dirname=os.environ["MASTER_CREDENTIALS_DIR"]) - ], - # 'protocols' contains information about protocols which master will use - # for communicating with workers. You must define at least 'port' option - # that workers could connect to your master with this protocol. - # 'port' must match the value configured into the workers (with their - # --master option) - "protocols": {"pb": {"port": os.environ["PORT"]}}, - # This specifies what database buildbot uses to store its state. - "db": {"db_url": config["private"]["db_url"]}, - # Disable net usage reports from being sent to buildbot.net - "buildbotNetUsageData": None, - # Configure the Schedulers, which decide how to react to incoming changes. - "schedulers": SCHEDULERS, - "logEncoding": "utf-8", - "multiMaster": True, - "mq": { - # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", - }, -} - -####### DB URL +c = BuildmasterConfig = base_master_config(config) + +####### +# DB URL +####### mtrDbPool = util.EqConnectionPool( "MySQLdb", config["private"]["db_host"], @@ -117,7 +66,9 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### WORKERS +######### +# WORKERS +######### # The 'workers' list defines the set of recognized workers. Each element is # a Worker object, specifying a unique worker name and password. The same @@ -168,7 +119,6 @@ for w_name in master_config["workers"]: f_quick_build = getQuickBuildFactory("nm", mtrDbPool) f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool) -## f_deb_autobake f_deb_autobake = util.BuildFactory() f_deb_autobake.addStep(printEnv()) f_deb_autobake.addStep( diff --git a/master_common.py b/master_common.py new file mode 100644 index 00000000..c996961f --- /dev/null +++ b/master_common.py @@ -0,0 +1,79 @@ +import os + +from buildbot.plugins import reporters, secrets, util +from constants import GITHUB_STATUS_BUILDERS +from schedulers_definition import SCHEDULERS + + +def base_master_config( + config: dict, + title=os.environ["TITLE"], + title_url=os.environ["TITLE_URL"], + buildbot_url=os.environ["BUILDMASTER_URL"], + secrets_provider_file=os.environ["MASTER_CREDENTIALS_DIR"], + master_port=os.environ["PORT"], + mq_router_url=os.environ["MQ_ROUTER_URL"], +): + # TODO(cvicentiu) either move this to environ or all other params to config + # file. + github_access_token = (config["private"]["gh_mdbci"]["access_token"],) + db_url = config["private"]["db_url"] + + return { + ####### + # PROJECT IDENTITY + ####### + # the 'title' string will appear at the top of this buildbot + # installation's + "title": title, + # home pages (linked to the 'titleURL'). + "titleURL": title_url, + # the 'buildbotURL' string should point to the location where the + # buildbot's internal web server is visible. This typically uses the + # port number set in the 'www' entry below, but with an + # externally-visible host name which the buildbot cannot figure out + # without some help. + "buildbotURL": buildbot_url, + # 'services' is a list of BuildbotService items like reporter targets. + # The status of each build will be pushed to these targets. + # buildbot/reporters/*.py has a variety to choose from, like IRC bots. + "services": [ + reporters.GitHubStatusPush( + token=github_access_token, + context=util.Interpolate("buildbot/%(prop:buildername)s"), + startDescription="Build started.", + endDescription="Build done.", + verbose=True, + builders=GITHUB_STATUS_BUILDERS, + ) + ], + "secretsProviders": [secrets.SecretInAFile(dirname=secrets_provider_file)], + # 'protocols' contains information about protocols which master will + # use for communicating with workers. You must define at least 'port' + # option that workers could connect to your master with this protocol. + # 'port' must match the value configured into the workers (with their + # --master option) + "protocols": { + "pb": {"port": master_port}, + }, + # This specifies what database buildbot uses to store its state. + "db": { + "db_url": db_url, + }, + # Disable net usage reports from being sent to buildbot.net + "buildbotNetUsageData": None, + # Configure the Schedulers, which decide how to react to incoming + # changes. + "schedulers": SCHEDULERS, + "logEncoding": "utf-8", + "multiMaster": True, + "mq": { + # Need to enable multimaster aware mq. Wamp is the only option for + # now. + "type": "wamp", + "router_url": mq_router_url, + "realm": "realm1", + # valid are: none, critical, error, warn, info, debug, trace + "wamp_debug_level": "info", + }, + } From a309ab38b6ee5e16c8b1b541bd2426cb0dcf4eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 3 Dec 2024 21:40:13 +0200 Subject: [PATCH 06/30] Move master-docker-nonstandard to use common master class --- master-docker-nonstandard/master.cfg | 199 ++++++++------------------- 1 file changed, 58 insertions(+), 141 deletions(-) diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 35c8e6ce..3a439967 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -1,28 +1,38 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta -from twisted.internet import defer - -import docker import os -import sys from collections import defaultdict -sys.path.insert(0, "/srv/buildbot/master") -sys.setrecursionlimit(10000) - -from common_factories import * -from constants import * -from locks import * -from schedulers_definition import SCHEDULERS -from utils import * +import docker +from buildbot.plugins import steps, util, worker +from buildbot.process.properties import Property +from common_factories import ( + addTests, + getHTMLLogString, + getQuickBuildFactory, + getRpmAutobakeFactory, + getSourceTarball, +) +from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES +from locks import getLocks +from master_common import base_master_config +from utils import ( + canStartBuild, + createVar, + createWorker, + dockerfile, + filterBranch, + hasAutobake, + hasBigtest, + hasFailed, + moveMTRLogs, + mtrJobsMultiplier, + nextBuild, + printEnv, + saveLogs, + savePackageIfBranchMatch, +) FQDN = os.environ["BUILDMASTER_WG_IP"] @@ -35,7 +45,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__)) #     └── master.cfg # # Non autogen masters load from for now. -base_dir = os.path.abspath(f'{cfg_dir}/../') +base_dir = os.path.abspath(f"{cfg_dir}/../") # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. @@ -43,61 +53,11 @@ config = {"private": {}} with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: exec(file.read(), config, {}) + # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - - -####### BUILDBOT SERVICES - -# 'services' is a list of BuildbotService items like reporter targets. The -# status of each build will be pushed to these targets. buildbot/reporters/*.py -# has a variety to choose from, like IRC bots. - - -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c["secretsProviders"] = [ - secrets.SecretInAFile( - dirname=os.environ["MASTER_CREDENTIALS_DIR"] - ) -] -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.environ["PORT"]) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} +c = BuildmasterConfig = base_master_config(config) + mtrDbPool = util.EqConnectionPool( "MySQLdb", @@ -107,14 +67,6 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS - ####### WORKERS # The 'workers' list defines the set of recognized workers. Each element is @@ -149,8 +101,6 @@ def addWorker( # Docker workers -fqdn = os.environ["BUILDMASTER_WG_IP"] - ## hz-bbw2-docker c["workers"].append( worker.DockerLatentWorker( @@ -159,7 +109,7 @@ c["workers"].append( docker_host=config["private"]["docker_workers"]["hz-bbw2-docker"], dockerfile=open("dockerfiles/eco-php-ubuntu-2004.dockerfile").read(), followStartupLogs=False, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "6G", "ulimits": [ @@ -182,7 +132,7 @@ c["workers"].append( "dockerfiles/eco-pymysql-python-3-9-slim-buster.dockerfile" ).read(), followStartupLogs=False, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "6G", "ulimits": [ @@ -203,7 +153,7 @@ c["workers"].append( docker_host=config["private"]["docker_workers"]["hz-bbw2-docker"], dockerfile=open("dockerfiles/eco-mysqljs-nodejs15-buster.dockerfile").read(), followStartupLogs=False, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "6G", "ulimits": [ @@ -224,12 +174,11 @@ c["workers"].append( "bm-bbw1-docker-ubuntu-2004", None, docker_host=config["private"]["docker_workers"]["bm-bbw1-docker"], - image=os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + image=os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", followStartupLogs=False, autopull=True, alwaysPull=True, - masterFQDN=fqdn, + masterFQDN=FQDN, hostconfig={ "shm_size": "20G", "ulimits": [ @@ -246,8 +195,7 @@ addWorker( "hz-bbw", 6, "bigtest-ubuntu-2004", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=20, save_packages=False, ) @@ -258,8 +206,7 @@ for w_name in ["ppc64le-osuosl-bbw"]: w_name, 1, "ubuntu-2004", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, save_packages=True, shm_size="20G", @@ -268,8 +215,7 @@ for w_name in ["ppc64le-osuosl-bbw"]: w_name, 1, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=30, save_packages=True, ) @@ -286,8 +232,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "aocc-debian-11", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian11-aocc", + os.environ["CONTAINER_REGISTRY_URL"] + "debian11-aocc", jobs=jobs, save_packages=False, ) @@ -295,8 +240,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "asan-ubuntu-2404", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu24.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu24.04", jobs=jobs, save_packages=False, ) @@ -304,8 +248,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "icc-ubuntu-2204", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu22.04-icc", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04-icc", jobs=jobs, save_packages=False, ) @@ -313,8 +256,7 @@ for w_name in ["ns-x64-bbw", "apexis-bbw"]: w_name, i, "ubuntu-2004", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=jobs, save_packages=True, ) @@ -324,8 +266,7 @@ addWorker( "amd-bbw", 1, "valgrind-fedora-40", - os.environ["CONTAINER_REGISTRY_URL"] - + "fedora40-valgrind", + os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, save_packages=False, ) @@ -333,8 +274,7 @@ addWorker( "amd-bbw", 2, "valgrind-fedora-40", - os.environ["CONTAINER_REGISTRY_URL"] - + "fedora40-valgrind", + os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, save_packages=False, ) @@ -342,8 +282,7 @@ addWorker( "hz-bbw", 6, "valgrind-fedora-40", - os.environ["CONTAINER_REGISTRY_URL"] - + "fedora40-valgrind", + os.environ["CONTAINER_REGISTRY_URL"] + "fedora40-valgrind", jobs=20, save_packages=False, ) @@ -352,8 +291,7 @@ addWorker( "hz-bbw", 1, "msan-clang-16-debian-11", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian11-msan-clang-16", + os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=20, save_packages=False, ) @@ -361,8 +299,7 @@ addWorker( "hz-bbw", 4, "msan-clang-16-debian-11", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian11-msan-clang-16", + os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=20, save_packages=False, ) @@ -370,8 +307,7 @@ addWorker( "hz-bbw", 5, "msan-clang-16-debian-11", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian11-msan-clang-16", + os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan-clang-16", jobs=30, save_packages=False, ) @@ -380,8 +316,7 @@ addWorker( "hz-bbw", 2, "debian-12", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian12", + os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=20, save_packages=False, ) @@ -389,8 +324,7 @@ addWorker( "hz-bbw", 5, "debian-12", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian12", + os.environ["CONTAINER_REGISTRY_URL"] + "debian12", jobs=20, save_packages=False, ) @@ -399,8 +333,7 @@ addWorker( "aarch64-bbw", 6, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=10, save_packages=True, ) @@ -429,8 +362,7 @@ addWorker( "s390x-bbw", 1, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, save_packages=False, ) @@ -439,8 +371,7 @@ addWorker( "s390x-bbw", 2, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, save_packages=False, ) @@ -449,8 +380,7 @@ addWorker( "s390x-bbw", 3, "ubuntu-2004-debug", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu20.04", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu20.04", jobs=7, save_packages=False, ) @@ -461,8 +391,7 @@ c["workers"].append( "release-prep-docker", None, docker_host=config["private"]["docker_workers"]["release-prep-docker"], - image=os.environ["CONTAINER_REGISTRY_URL"] - + "debian12-release", + image=os.environ["CONTAINER_REGISTRY_URL"] + "debian12-release", followStartupLogs=False, autopull=True, alwaysPull=True, @@ -1756,15 +1685,3 @@ c["builders"].append( factory=f_prep_local, ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -} From 9a3bce014af26947d1f2427f7390d4b9733ea867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 4 Dec 2024 15:44:07 +0200 Subject: [PATCH 07/30] Move master-docker-nonstandard-2 to use common master class --- master-docker-nonstandard-2/master.cfg | 146 +++++++------------------ 1 file changed, 37 insertions(+), 109 deletions(-) diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index 23c2136f..43c5d078 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -1,30 +1,38 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta -from twisted.internet import defer - -import docker import os -import sys from collections import defaultdict -sys.path.insert(0, "/srv/buildbot/master") -sys.setrecursionlimit(10000) - -from common_factories import * -from constants import * -from locks import * -from schedulers_definition import SCHEDULERS -from utils import * - -FQDN = os.environ["BUILDMASTER_WG_IP"] +from buildbot.plugins import steps, util +from buildbot.process.properties import Property +from common_factories import ( + getQuickBuildFactory, + getRpmAutobakeFactory, + getSourceTarball, +) +from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES +from locks import getLocks +from master_common import base_master_config +from utils import ( + canStartBuild, + createVar, + createWorker, + dockerfile, + filterBranch, + getHTMLLogString, + hasAutobake, + hasBigtest, + hasEco, + hasFailed, + moveMTRLogs, + mtrJobsMultiplier, + nextBuild, + printEnv, + read_template, + saveLogs, + savePackageIfBranchMatch, +) cfg_dir = os.path.abspath(os.path.dirname(__file__)) @@ -35,7 +43,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__)) #     └── master.cfg # # Non autogen masters load from for now. -base_dir = os.path.abspath(f'{cfg_dir}/../') +base_dir = os.path.abspath(f"{cfg_dir}/../") # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. @@ -43,61 +51,10 @@ config = {"private": {}} with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: exec(file.read(), config, {}) + # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - - -####### BUILDBOT SERVICES - -# 'services' is a list of BuildbotService items like reporter targets. The -# status of each build will be pushed to these targets. buildbot/reporters/*.py -# has a variety to choose from, like IRC bots. - - -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c["secretsProviders"] = [ - secrets.SecretInAFile( - dirname=os.environ["MASTER_CREDENTIALS_DIR"] - ) -] -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.environ["PORT"]) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} +c = BuildmasterConfig = base_master_config(config) mtrDbPool = util.EqConnectionPool( "MySQLdb", @@ -107,14 +64,6 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS - ####### WORKERS # The 'workers' list defines the set of recognized workers. Each element is @@ -148,16 +97,11 @@ def addWorker( c["workers"].append(instance) - -# Docker workers -fqdn = os.environ["BUILDMASTER_WG_IP"] - addWorker( "amd-bbw", 1, "debian-12-32-bit", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian12-386", + os.environ["CONTAINER_REGISTRY_URL"] + "debian12-386", jobs=10, save_packages=False, shm_size="30G", @@ -167,8 +111,7 @@ addWorker( "amd-bbw", 2, "debian-12-32-bit", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian12-386", + os.environ["CONTAINER_REGISTRY_URL"] + "debian12-386", jobs=10, save_packages=False, shm_size="30G", @@ -178,8 +121,7 @@ addWorker( "apexis-bbw", 3, "debian-12-32-bit", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian12-386", + os.environ["CONTAINER_REGISTRY_URL"] + "debian12-386", jobs=10, save_packages=False, shm_size="30G", @@ -189,8 +131,7 @@ addWorker( "apexis-bbw", 3, "msan-clang-debian-11", - os.environ["CONTAINER_REGISTRY_URL"] - + "debian11-msan", + os.environ["CONTAINER_REGISTRY_URL"] + "debian11-msan", jobs=20, save_packages=False, ) @@ -199,8 +140,7 @@ addWorker( "apexis-bbw", 3, "ubuntu-2204-jepsen-mariadb", - os.environ["CONTAINER_REGISTRY_URL"] - + "ubuntu22.04-jepsen-mariadb", + os.environ["CONTAINER_REGISTRY_URL"] + "ubuntu22.04-jepsen-mariadb", jobs=5, save_packages=False, ) @@ -986,15 +926,3 @@ c["builders"].append( factory=f_jepsen_mariadb, ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -} From 48ea0c4c5464e578c42d3660473c09217b9003b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 4 Dec 2024 15:50:57 +0200 Subject: [PATCH 08/30] Move master-protected-branches to use common master class --- master-protected-branches/master.cfg | 105 ++++----------------------- 1 file changed, 16 insertions(+), 89 deletions(-) diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 57d582e6..f6de257c 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -2,28 +2,23 @@ # ex: set filetype=python: import os -import sys from collections import defaultdict -from datetime import timedelta -import docker -from buildbot.plugins import * -from buildbot.process.properties import Properties, Property -from buildbot.process.remotecommand import RemoteCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.shell import Compile, SetPropertyFromCommand, ShellCommand, Test -from buildbot.steps.source.github import GitHub -from twisted.internet import defer - -sys.setrecursionlimit(10000) - -sys.path.insert(0, "/srv/buildbot/master") - -from common_factories import * -from constants import * -from locks import * -from schedulers_definition import SCHEDULERS -from utils import * +from buildbot.plugins import steps, util, worker +from buildbot.process.properties import Property +from common_factories import getLastNFailedBuildsFactory, getQuickBuildFactory +from locks import getLocks +from master_common import base_master_config +from utils import ( + canStartBuild, + createWorker, + dockerfile, + isJepsenBranch, + isStagingBranch, + ls2list, + nextBuild, + waitIfStaging, +) cfg_dir = os.path.abspath(os.path.dirname(__file__)) @@ -44,55 +39,7 @@ with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - - -####### BUILDBOT SERVICES - -# 'services' is a list of BuildbotService items like reporter targets. The -# status of each build will be pushed to these targets. buildbot/reporters/*.py -# has a variety to choose from, like IRC bots. - - -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.environ["MASTER_CREDENTIALS_DIR"])] -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.environ["PORT"]) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} +c = BuildmasterConfig = base_master_config(config) mtrDbPool = util.EqConnectionPool( "MySQLdb", @@ -102,14 +49,6 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS - ####### WORKERS # The 'workers' list defines the set of recognized workers. Each element is @@ -571,15 +510,3 @@ c["builders"].append( factory=getLastNFailedBuildsFactory("debug", mtrDbPool), ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -} From 94c663533c36d62deb2d7281079c48f8ec7c58d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 4 Dec 2024 16:10:34 +0200 Subject: [PATCH 09/30] Move master-libvirt to use common master class --- master-libvirt/master.cfg | 83 +++++++-------------------------------- 1 file changed, 14 insertions(+), 69 deletions(-) diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 1614780e..19d303df 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -1,25 +1,13 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from datetime import timedelta -from twisted.internet import defer - -import docker import os -import sys -sys.setrecursionlimit(10000) - -sys.path.insert(0, "/srv/buildbot/master") - -from utils import * -from constants import OS_INFO +from buildbot.plugins import steps, util, worker +from buildbot.steps.shell import Test +from constants import BUILDERS_INSTALL, OS_INFO +from master_common import base_master_config +from utils import canStartBuild, envFromProperties, getScript, nextBuild, printEnv cfg_dir = os.path.abspath(os.path.dirname(__file__)) @@ -30,7 +18,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__)) #     └── master.cfg # # Non autogen masters load from for now. -base_dir = os.path.abspath(f'{cfg_dir}/../') +base_dir = os.path.abspath(f"{cfg_dir}/../") # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. @@ -40,41 +28,10 @@ with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - +c = BuildmasterConfig = base_master_config(config) -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] artifactsURL = os.environ["ARTIFACTS_URL"] -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.environ["PORT"]) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} - -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None - ####### UTILS def getRpmUpgradeStep(): @@ -133,6 +90,7 @@ def getRpmInstallStep(): command=["./rpm-install.sh"], ) + def getDebUpgradeStep(): return Test( name="upgrade", @@ -198,6 +156,7 @@ def getMajorVersionStep(): ), ) + def getPAMTestStep(): return Test( name="PAM authentication test", @@ -212,9 +171,8 @@ def getPAMTestStep(): command=["./pam-test.sh"], ) -# FACTORY -## f_deb_install +# FACTORY f_deb_install = util.BuildFactory() f_deb_install.addStep(printEnv()) f_deb_install.addStep(getScript("deb-install.sh")) @@ -222,14 +180,12 @@ f_deb_install.addStep(getDebInstallStep()) f_deb_install.addStep(getScript("pam-test.sh")) f_deb_install.addStep(getPAMTestStep()) -## f_deb_upgrade f_deb_upgrade = util.BuildFactory() f_deb_upgrade.addStep(printEnv()) f_deb_upgrade.addStep(getMajorVersionStep()) f_deb_upgrade.addStep(getScript("deb-upgrade.sh")) f_deb_upgrade.addStep(getDebUpgradeStep()) -## f_rpm_install f_rpm_install = util.BuildFactory() f_rpm_install.addStep(printEnv()) f_rpm_install.addStep(getScript("rpm-install.sh")) @@ -237,13 +193,13 @@ f_rpm_install.addStep(getRpmInstallStep()) f_rpm_install.addStep(getScript("pam-test.sh")) f_rpm_install.addStep(getPAMTestStep()) -## f_rpm_upgrade f_rpm_upgrade = util.BuildFactory() f_rpm_upgrade.addStep(printEnv()) f_rpm_upgrade.addStep(getMajorVersionStep()) f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh")) f_rpm_upgrade.addStep(getRpmUpgradeStep()) + ####### WORKERS and BUILDERS # The 'workers' list defines the set of recognized workers. Each element is @@ -291,7 +247,9 @@ for builder_name in BUILDERS_INSTALL: elif builder_type == "rpm": factory_install = f_rpm_install factory_upgrade = f_rpm_upgrade - build_arch = os_name + str(OS_INFO[os_info_name]["version_name"]) + "-" + platform + build_arch = ( + os_name + str(OS_INFO[os_info_name]["version_name"]) + "-" + platform + ) # FIXME - all RPM's should follow the same conventions! if os_name == "centos" and OS_INFO[os_info_name]["version_name"] >= 9: @@ -299,7 +257,6 @@ for builder_name in BUILDERS_INSTALL: platform = "x86_64" build_arch = f"centos/{OS_INFO[os_info_name]['version_name']}/{platform}" - c["builders"].append( util.BuilderConfig( name=builder_name, @@ -396,15 +353,3 @@ for builder_name in BUILDERS_INSTALL: factory=factory_upgrade, ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -} From 381313aecadf8bdcfb8ad5bb609e6da209317b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 4 Dec 2024 16:24:29 +0200 Subject: [PATCH 10/30] Move master-nonlatent to use common master class * Also fixup unescaped backslashes in strings --- master-nonlatent/master.cfg | 146 ++++++++++-------------------------- 1 file changed, 38 insertions(+), 108 deletions(-) diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index c22ebdd3..da3c138f 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -1,9 +1,8 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties from buildbot.process.results import SUCCESS, FAILURE, WARNINGS +from buildbot.process.properties import Property, Properties from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver from buildbot.steps.source.github import GitHub @@ -13,16 +12,26 @@ from twisted.internet import defer import docker import os -import sys -import yaml - -sys.setrecursionlimit(10000) +from datetime import timedelta -sys.path.insert(0, "/srv/buildbot/master") -from common_factories import * -from locks import * -from schedulers_definition import SCHEDULERS -from utils import * +from buildbot.plugins import steps, util, worker +from buildbot.process.properties import Property +from common_factories import addWinTests, getQuickBuildFactory +from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES +from master_common import base_master_config +from utils import ( + canStartBuild, + createVar, + getHTMLLogString, + getSourceTarball, + hasFailed, + ls2list, + moveMTRLogs, + mtrJobsMultiplier, + nextBuild, + prioritizeBuilders, + savePackageIfBranchMatch, +) ####### VARIABLES vsWarningPattern = "^.*: warning C.*$" @@ -36,7 +45,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__)) #     └── master.cfg # # Non autogen masters load from for now. -base_dir = os.path.abspath(f'{cfg_dir}/../') +base_dir = os.path.abspath(f"{cfg_dir}/../") # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. @@ -46,36 +55,7 @@ with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - - -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.environ["PORT"]) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} +c = BuildmasterConfig = base_master_config(config) mtrDbPool = util.EqConnectionPool( "MySQLdb", @@ -85,33 +65,10 @@ mtrDbPool = util.EqConnectionPool( config["private"]["db_mtr_db"], ) -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None -####### Services -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c["secretsProviders"] = [ - secrets.SecretInAFile( - dirname=os.environ["MASTER_CREDENTIALS_DIR"] - ) -] ####### Builder priority c["prioritizeBuilders"] = prioritizeBuilders -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS ####### WORKERS @@ -178,9 +135,6 @@ f_windows_env = { } f_windows_env.update(MTR_ENV) -# f_quick_build = getQuickBuildFactory(mtrDbPool) -f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool) - ## f_windows f_windows = util.BuildFactory() f_windows.addStep( @@ -259,7 +213,7 @@ f_windows.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -A %(kw:arch_cmake)s -DPLUGIN_ROCKSDB=NO -DMYSQL_MAINTAINER_MODE=ERR -Wno-dev', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -A %(kw:arch_cmake)s -DPLUGIN_ROCKSDB=NO -DMYSQL_MAINTAINER_MODE=ERR -Wno-dev', arch=util.Property("arch", default="x64"), arch_cmake=util.Property("arch_cmake", default="x64"), ), @@ -276,7 +230,7 @@ f_windows.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config Debug -- -m', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config Debug -- -m', arch=util.Property("arch", default="x64"), ), '"', @@ -317,14 +271,8 @@ f_windows.addStep( + "%(prop:buildername)s" ), url=util.Interpolate( - f'{os.environ["ARTIFACTS_URL"]}' - "/" - "%(prop:tarbuildnum)s" - "/" - "logs" - "/" - "%(prop:buildername)s" - "/" + os.environ["ARTIFACTS_URL"], + "/" "%(prop:tarbuildnum)s" "/" "logs" "/" "%(prop:buildername)s" "/", ), ) ) @@ -337,7 +285,7 @@ f_windows.addStep( "powershell", "-command", "Remove-Item", - '"$pwd\*"', + '"$pwd\\*"', "-Recurse", "-Force", '"', @@ -387,7 +335,7 @@ f_windows_msi.addStep( "powershell", "-command", "Remove-Item", - '"$pwd\*"', + '"$pwd\\*"', "-Recurse", "-Force", '"', @@ -439,7 +387,7 @@ f_windows_msi.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -G "Visual Studio 17 2022" -A %(kw:arch_cmake)s -DBUILD_CONFIG=mysql_release -DWITH_THIRD_PARTY=HeidiSQL -DWITH_EMBEDDED_SERVER=0 -DSIGNCODE=ON -DSIGNTOOL_PARAMETERS="/fd;SHA256;/a;/t;http://timestamp.globalsign.com/?signature=sha2" -DWITH_UNIT_TESTS=0 -DMYSQL_MAINTAINER_MODE=ERR', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake . -G "Visual Studio 17 2022" -A %(kw:arch_cmake)s -DBUILD_CONFIG=mysql_release -DWITH_THIRD_PARTY=HeidiSQL -DWITH_EMBEDDED_SERVER=0 -DSIGNCODE=ON -DSIGNTOOL_PARAMETERS="/fd;SHA256;/a;/t;http://timestamp.globalsign.com/?signature=sha2" -DWITH_UNIT_TESTS=0 -DMYSQL_MAINTAINER_MODE=ERR', arch=util.Property("arch", default="x64"), arch_cmake=util.Property("arch_cmake", default="x64"), ), @@ -456,7 +404,7 @@ f_windows_msi.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config RelWithDebInfo -- -m', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --verbose --config RelWithDebInfo -- -m', arch=util.Property("arch", default="x64"), ), '"', @@ -473,7 +421,7 @@ f_windows_msi.addStep( "dojob", '"', util.Interpolate( - '"C:\Program Files (x86)\Microsoft Visual Studio\\2022\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --config RelWithDebInfo --target win_package && cmake --build . --config RelWithDebInfo --target MSI', + '"C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat" -arch=%(kw:arch)s && cmake --build . --config RelWithDebInfo --target win_package && cmake --build . --config RelWithDebInfo --target MSI', arch=util.Property("arch", default="x64"), ), '"', @@ -531,7 +479,7 @@ f_windows_msi.addStep( command=[ "powershell", "-command", - 'Get-ChildItem .\* -Include @("*.msi", "*.zip") | Get-FileHash | Select-Object Hash, @{Name="Name";Expression={[System.IO.Path]::GetFileName($_.Path)}} | Format-Table -HideTableHeaders | Out-File sha256sums.txt', + 'Get-ChildItem .\\* -Include @("*.msi", "*.zip") | Get-FileHash | Select-Object Hash, @{Name="Name";Expression={[System.IO.Path]::GetFileName($_.Path)}} | Format-Table -HideTableHeaders | Out-File sha256sums.txt', ], ) ) @@ -593,7 +541,7 @@ f_windows_msi.addStep( "powershell", "-command", "Remove-Item", - '"$pwd\*"', + '"$pwd\\*"', "-Recurse", "-Force", '"', @@ -692,7 +640,7 @@ f_wordpress.addStep( ) # f_dockerlibrary -DECODE_RC={0:SUCCESS,1:FAILURE,2:WARNINGS} +DECODE_RC = {0: SUCCESS, 1: FAILURE, 2: WARNINGS} f_dockerlibrary = util.BuildFactory() f_dockerlibrary.addStep( @@ -710,9 +658,7 @@ f_dockerlibrary.addStep( f_dockerlibrary.addStep( steps.ShellCommand( name="building MariaDB docker library test image", - env={ - "ARTIFACTS_URL": os.environ["ARTIFACTS_URL"] - }, + env={"ARTIFACTS_URL": os.environ["ARTIFACTS_URL"]}, decodeRC=DECODE_RC, command=[ "bash", @@ -982,7 +928,7 @@ def get_macos_factory(compile_only=False): util.Interpolate( """ cd mysql-test && - exec perl mysql-test-run.pl --verbose-restart --force --retry=3 --max-save-core=2 --max-save-datadir=10 --max-test-fail=20 --parallel=$(expr %(kw:jobs)s \* 2) %(kw:mtr_additional_args)s + exec perl mysql-test-run.pl --verbose-restart --force --retry=3 --max-save-core=2 --max-save-datadir=10 --max-test-fail=20 --parallel=$(expr %(kw:jobs)s \\* 2) %(kw:mtr_additional_args)s """, mtr_additional_args=util.Property( "mtr_additional_args", default="" @@ -1070,9 +1016,7 @@ c["builders"].append( name=os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"] .replace("bb", "amd64") .replace("docker", "wordpress"), - workernames=[ - os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"] - ], + workernames=[os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"]], tags=["RHEL"], collapseRequests=True, nextBuild=nextBuild, @@ -1085,9 +1029,7 @@ c["builders"].append( util.BuilderConfig( name=os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"].replace("bb", "amd64") + "library", - workernames=[ - os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"] - ], + workernames=[os.environ["MASTER_NONLATENT_DOCKERLIBRARY_WORKER"]], tags=["RHEL"], collapseRequests=True, nextBuild=nextBuild, @@ -1175,15 +1117,3 @@ c["builders"].append( # Add a Janitor configurator that removes old logs c["configurators"] = [util.JanitorConfigurator(logHorizon=timedelta(weeks=6), hour=23)] - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -} From 96f8fde5974f472ac56845ba1ee624eb2b32eb0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Fri, 6 Dec 2024 08:28:18 +0200 Subject: [PATCH 11/30] Move master-galera to use common master class --- master-galera/master.cfg | 109 +++++++++++---------------------------- 1 file changed, 29 insertions(+), 80 deletions(-) diff --git a/master-galera/master.cfg b/master-galera/master.cfg index ab49aa23..1563f514 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -1,26 +1,17 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from twisted.internet import defer -import sys +# git branch filter using fnmatch +import fnmatch import os -import docker -import yaml -from datetime import timedelta from collections import defaultdict -sys.setrecursionlimit(10000) +from buildbot.plugins import schedulers, steps, util, worker +from constants import ALL_PLATFORMS, BUILDERS_GALERA, OS_INFO +from master_common import base_master_config +from utils import createWorker, savePackageIfBranchMatch, waitIfStaging, nextBuild -sys.path.append(os.getcwd() + "/..") -from constants import * -from utils import * cfg_dir = os.path.abspath(os.path.dirname(__file__)) @@ -31,7 +22,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__)) #     └── master.cfg # # Non autogen masters load from for now. -base_dir = os.path.abspath(f'{cfg_dir}/../') +base_dir = os.path.abspath(f"{cfg_dir}/../") # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. @@ -41,41 +32,10 @@ with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. -c = BuildmasterConfig = {} - +c = BuildmasterConfig = base_master_config(config) FQDN = os.environ["BUILDMASTER_WG_IP"] -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.environ["PORT"]) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} - -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None ####### SCHEDULERS @@ -84,9 +44,6 @@ c["buildbotNetUsageData"] = None BRANCHES_MAIN = ["mariadb-3.x", "mariadb-4.x", "bb-*"] SAVED_PACKAGE_BRANCHES_GALERA = ["mariadb-3.x", "mariadb-4.x", "bb-*"] -# git branch filter using fnmatch -import fnmatch - def upstream_branch_fn(branch): return ( @@ -98,9 +55,11 @@ def upstream_branch_fn(branch): ) - +# Override schedulers. +# TODO(cvicentiu): Move this to base_master_config maybe? c["schedulers"] = [] + schedulerTrigger = schedulers.AnyBranchScheduler( name="s_upstream_galera", change_filter=util.ChangeFilter( @@ -172,6 +131,7 @@ GALERA_PACKAGES = os.environ["GALERA_PACKAGES_DIR"] workers = defaultdict(list) + def addWorker( worker_name_prefix, worker_id, @@ -256,7 +216,7 @@ for platform in ALL_PLATFORMS: def dpkgDeb(): - return ShellCommand( + return steps.ShellCommand( name="apt-ftparchive", haltOnFailure=True, command=[ @@ -275,14 +235,14 @@ def dpkgDeb(): """ ), ], - doStepIf=(lambda step: - savePackageIfBranchMatch(step, - SAVED_PACKAGE_BRANCHES_GALERA)), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA) + ), ) def rpmSave(): - return ShellCommand( + return steps.ShellCommand( name="move rpm files", haltOnFailure=True, command=[ @@ -292,13 +252,13 @@ def rpmSave(): """set -e mkdir -p rpms srpms cp `find *.rpm -maxdepth 1 -type f` rpms - find rpms -type f -exec sha256sum {} \; | sort > sha256sums.txt + find rpms -type f -exec sha256sum {} \\; | sort > sha256sums.txt """ ), ], - doStepIf=(lambda step: - savePackageIfBranchMatch(step, - SAVED_PACKAGE_BRANCHES_GALERA)), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA) + ), ) @@ -380,9 +340,9 @@ EOF """, url=os.environ["ARTIFACTS_URL"], ), - doStepIf=(lambda step: - savePackageIfBranchMatch(step, - SAVED_PACKAGE_BRANCHES_GALERA)), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA) + ), ) ) f_deb_build.addStep( @@ -439,9 +399,9 @@ EOF """, url=os.environ["ARTIFACTS_URL"], ), - doStepIf=(lambda step: - savePackageIfBranchMatch(step, - SAVED_PACKAGE_BRANCHES_GALERA)), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES_GALERA) + ), ) ) f_rpm_build.addStep( @@ -450,6 +410,7 @@ f_rpm_build.addStep( ) ) + ####### BUILDERS LIST c["builders"] = [] @@ -467,6 +428,7 @@ c["builders"].append( ) ) + for os_i in OS_INFO: if "install_only" in OS_INFO[os_i] and OS_INFO[os_i]["install_only"]: continue @@ -488,7 +450,6 @@ for os_i in OS_INFO: env = {} if os_i == "ubuntu-2004": - print("using gcc/++-10") env = {"CC": "gcc-10", "CXX": "g++-10"} c["builders"].append( @@ -502,15 +463,3 @@ for os_i in OS_INFO: factory=factory, ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -} From 2362c6e464dc6b1db20c76301e0ae6ffae2fcab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Fri, 6 Dec 2024 09:00:04 +0200 Subject: [PATCH 12/30] Move master-web to use master_common class * Also simplify branch matching functions --- constants.py | 17 +++++++ master-web/master.cfg | 101 ++++++++---------------------------------- utils.py | 14 ++++-- 3 files changed, 46 insertions(+), 86 deletions(-) diff --git a/constants.py b/constants.py index 463d1eb4..6b0311a8 100644 --- a/constants.py +++ b/constants.py @@ -21,6 +21,23 @@ "main", ] +# Branches with special prefixes that invoke a BB run. +BB_TEST_BRANCHES = [ + "bb-*", + "st-*", + "prot-*", + "refs/pull/*", + "preview-1[0-9].*", + "jpsn-*", +] + +# A list of all branches that invoke a buildbot run. +ALL_BB_TEST_BRANCHES = BRANCHES_MAIN + BB_TEST_BRANCHES + +STAGING_PROT_TEST_BRANCHES = [ + "prot-st-*", +] + # Defines what builders report status to GitHub GITHUB_STATUS_BUILDERS = [ "aarch64-macos-compile-only", diff --git a/master-web/master.cfg b/master-web/master.cfg index a916a6a7..bb2ad46d 100644 --- a/master-web/master.cfg +++ b/master-web/master.cfg @@ -1,32 +1,13 @@ # -*- python -*- # ex: set filetype=python: -from __future__ import absolute_import -from __future__ import print_function - +# git branch filter using fnmatch import os -import sys -import time -import requests - -from buildbot.plugins import * -from buildbot.process.properties import Property, Properties -from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.source.github import GitHub -from buildbot.process.remotecommand import RemoteCommand -from twisted.internet import defer -from datetime import timedelta - -from flask import Flask -from flask import render_template - -from buildbot.process.results import statusToString - -sys.setrecursionlimit(10000) -sys.path.append(os.getcwd() + "/..") -from constants import * +from buildbot.plugins import schedulers, util +from flask import Flask, render_template +from master_common import base_master_config +from utils import upstream_branch_fn cfg_dir = os.path.abspath(os.path.dirname(__file__)) @@ -37,7 +18,7 @@ cfg_dir = os.path.abspath(os.path.dirname(__file__)) #     └── master.cfg # # Non autogen masters load from for now. -base_dir = os.path.abspath(f'{cfg_dir}/../') +base_dir = os.path.abspath(f"{cfg_dir}/../") # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. @@ -45,26 +26,22 @@ config = {"private": {}} with open(os.path.join(base_dir, "master-private.cfg"), "r") as file: exec(file.read(), config, {}) -c = BuildmasterConfig = {} -# Load the slave, database passwords and 3rd-party tokens from an external -# private file, so that the rest of the configuration can be public. -config = {"private": {}} -exec(open("master-private.cfg").read(), config, {}) - -####### PROJECT IDENTITY +c = BuildmasterConfig = base_master_config(config) -c["title"] = os.environ["TITLE"] -c["titleURL"] = os.environ["TITLE_URL"] -c["buildbotURL"] = os.environ["BUILDMASTER_URL"] -port = int(os.environ["PORT"]) +# No slaves connect to this master, only meant for WEB. +c["protocols"] = {} # minimalistic config to activate web UI -c["www"] = dict( - port=port, - plugins=dict(waterfall_view={}, console_view={}, grid_view={}), - custom_templates_dir="templates", -) +c["www"] = { + "port": os.environ["PORT"], + "plugins": { + "waterfall_view": {}, + "console_view": {}, + "grid_view": {}, + }, + "custom_templates_dir": os.path.join(cfg_dir, "templates"), +} # Github Auth, allow control for MariaDB affiliated accounts c["www"]["authz"] = util.Authz( @@ -102,11 +79,6 @@ c["www"]["plugins"]["wsgi_dashboards"] = ( ] ) -####### DB URL -c["db"] = {"db_url": config["private"]["db_url"]} - -####### Disable net usage reports from being sent to buildbot.net -c["buildbotNetUsageData"] = None ####### GitHub hooks @@ -123,43 +95,8 @@ c["www"]["ui_default_config"] = { "Grid.buildFetchLimit": 50, } -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -# Need to enable multimaster aware mq. Wamp is the only option for now. -c["mq"] = { - "type": "wamp", - "router_url": os.environ["MQ_ROUTER_URL"], - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -} - -# git branch filter using fnmatch -import fnmatch - - -def upstream_branch_fn(branch): - return ( - branch in BRANCHES_MAIN - or fnmatch.fnmatch(branch, "bb-*") - or fnmatch.fnmatch(branch, "st-*") - or fnmatch.fnmatch(branch, "prot-*") - or fnmatch.fnmatch(branch, "refs/pull/*") - or fnmatch.fnmatch(branch, "preview-1[0-9].*") - or fnmatch.fnmatch(branch, "jpsn-*") - ) - - -def staging_branch_fn(branch): - return fnmatch.fnmatch(branch, "st-*") - - -def fnmatch_any(s, list_of_patterns): - return any(fnmatch.fnmatch(s, p) for p in list_of_patterns) - +# Override schedulers c["schedulers"] = [] # upstream scheduling diff --git a/utils.py b/utils.py index 76c70f33..6f2ad43f 100644 --- a/utils.py +++ b/utils.py @@ -17,6 +17,7 @@ from buildbot.process.workerforbuilder import AbstractWorkerForBuilder from buildbot.worker import AbstractWorker from constants import ( + ALL_BB_TEST_BRANCHES, BUILDERS_AUTOBAKE, BUILDERS_BIG, BUILDERS_ECO, @@ -28,6 +29,7 @@ MTR_ENV, RELEASE_BRANCHES, SAVED_PACKAGE_BRANCHES, + STAGING_PROT_TEST_BRANCHES, ) private_config = {"private": {}} @@ -259,14 +261,18 @@ def uploadDebArtifacts() -> steps.ShellCommand: ) -def staging_branch_fn(branch: str) -> bool: - return fnmatch.fnmatch(branch, "prot-st-*") - - def fnmatch_any(branch: str, patterns: list[str]) -> bool: return any(fnmatch.fnmatch(branch, pattern) for pattern in patterns) +def upstream_branch_fn(branch): + return fnmatch_any(branch, ALL_BB_TEST_BRANCHES) + + +def staging_branch_fn(branch: str) -> bool: + return fnmatch_any(branch, STAGING_PROT_TEST_BRANCHES) + + # Priority filter based on saved package branches def nextBuild(builder: Builder, requests: list[BuildRequest]) -> BuildRequest: def build_request_sort_key(request: BuildRequest): From 8f1c9c5999d6d27fb54bac3eeef0d1cbc0bf13dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Fri, 6 Dec 2024 09:11:43 +0200 Subject: [PATCH 13/30] Remove dead code: master.cfg mtr_additional_args not present in os_info --- master.cfg | 7 ------- 1 file changed, 7 deletions(-) diff --git a/master.cfg b/master.cfg index f9a9ad64..f0034383 100644 --- a/master.cfg +++ b/master.cfg @@ -236,11 +236,6 @@ for builder in master_config["builders"]: arch = splits[0] os_name = "-".join(splits[1:]) - mtr_additional_args = None - if "mtr_additional_args" in OS_INFO[os_name]: - if arch in OS_INFO[os_name]["mtr_additional_args"]: - mtr_additional_args = OS_INFO[os_name]["mtr_additional_args"][arch] - if arch == "amd64": arch = "x64" worker_name = arch + "-bbw-docker-" + os_name @@ -275,8 +270,6 @@ for builder in master_config["builders"]: if arch == "ppc64le": properties["verbose_build"] = "VERBOSE=1" - if mtr_additional_args is not None: - properties["mtr_additional_args"] = mtr_additional_args if build_type == "rpm": properties["rpm_type"] = "".join(os_name.split("-")) factory_instance = f_rpm_autobake From 67a5f828f551fa0c0fbda74ddccc35a966bbcf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 12 Jan 2025 14:28:43 +0200 Subject: [PATCH 14/30] tests/ -> make into module --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b From 7f42cba5140ec868128d8212078f1f0974d92902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 5 Feb 2025 01:30:39 +0200 Subject: [PATCH 15/30] cleanup define_masters.py and os_info.yaml This is a larger commit that does the following: 1. os_info now specifically mentions the image_tag to use for the OS. 2. os_info specifies tags per os. 3. define_masters now reads os_info and groups by arch. 4. Make autogen buildbot.tac allow relative import 5. buildbot.tac no longer a "template" with log name filled in by master name, but a file that gets copied for each master. 6. Validate master updated to change the way each master should be started. --- buildbot.tac | 10 +-- define_masters.py | 47 ++++++------- master.cfg | 167 +++++++++++++++++++++------------------------- os_info.yaml | 82 +++++++++++++++++++++++ 4 files changed, 187 insertions(+), 119 deletions(-) diff --git a/buildbot.tac b/buildbot.tac index 79cbe6b1..d58d0cbe 100644 --- a/buildbot.tac +++ b/buildbot.tac @@ -28,10 +28,11 @@ log_basedir = "/var/log/buildbot" rotateLength = 20000000 maxRotatedFiles = 30 + +last_two_dirs = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:] +master_name = last_two_dirs[-1] # Last two directories. autogen and -cfg_from_basedir = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:] + [ - "master.cfg" -] +cfg_from_basedir = last_two_dirs + ["master.cfg"] configfile = os.path.join(*cfg_from_basedir) # Default umask for server @@ -41,8 +42,9 @@ umask = None # directory; do not edit it. application = service.Application("buildmaster") # fmt: skip +# This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, "%s"), + os.path.join(log_basedir, f"{master_name}.log"), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) diff --git a/define_masters.py b/define_masters.py index 4b62abed..29273932 100755 --- a/define_masters.py +++ b/define_masters.py @@ -2,26 +2,33 @@ import os import shutil +from collections import defaultdict import yaml BASE_PATH = "autogen/" config = {"private": {}} -exec(open("master-private.cfg").read(), config, {}) +with open("master-private.cfg", "r") as file: + exec(file.read(), config, {}) + +master_variables = config["private"]["master-variables"] with open("os_info.yaml", encoding="utf-8") as file: OS_INFO = yaml.safe_load(file) -platforms = {} +platforms = defaultdict(dict) for os_name in OS_INFO: + # TODO(cvicentiu) this should be removed, hack. if "install_only" in OS_INFO[os_name] and OS_INFO[os_name]["install_only"]: continue + for arch in OS_INFO[os_name]["arch"]: - builder_name = arch + "-" + os_name - if arch not in platforms: - platforms[arch] = [] - platforms[arch].append(builder_name) + builder_name = f"{arch}-{os_name}" + platforms[arch][os_name] = { + "image_tag": OS_INFO[os_name]["image_tag"], + "tags": OS_INFO[os_name]["tags"], + } # Clear old configurations if os.path.exists(BASE_PATH): @@ -32,33 +39,23 @@ # If for a given architecture there are more than "max_builds" builds, # create multiple masters # "max_builds" is defined is master-private.py - num_masters = ( - int(len(platforms[arch]) / config["private"]["master-variables"]["max_builds"]) - + 1 - ) + num_masters = int(len(platforms[arch]) / master_variables["max_builds"]) + 1 for master_id in range(num_masters): - dir_path = BASE_PATH + arch + "-master-" + str(master_id) + dir_path = f"{BASE_PATH}{arch}-master-{master_id}" os.makedirs(dir_path) - master_config = {} - master_config["builders"] = platforms[arch] - master_config["workers"] = config["private"]["master-variables"]["workers"][ - arch - ] - master_config["log_name"] = ( - "master-docker-" + arch + "-" + str(master_id) + ".log" - ) + master_config = { + "builders": {arch: platforms[arch]}, + "workers": master_variables["workers"][arch], + "log_name": f"master-docker-{arch}-{master_id}.log", + } - with open(dir_path + "/master-config.yaml", mode="w", encoding="utf-8") as file: + with open(f"{dir_path}/master-config.yaml", mode="w", encoding="utf-8") as file: yaml.dump(master_config, file) shutil.copyfile("master.cfg", dir_path + "/master.cfg") shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg") + shutil.copyfile("buildbot.tac", dir_path + "/buildbot.tac") - buildbot_tac = ( - open("buildbot.tac", encoding="utf-8").read() % master_config["log_name"] - ) - with open(dir_path + "/buildbot.tac", mode="w", encoding="utf-8") as f: - f.write(buildbot_tac) print(arch, len(master_config["builders"])) diff --git a/master.cfg b/master.cfg index f0034383..e645d7e9 100644 --- a/master.cfg +++ b/master.cfg @@ -14,7 +14,6 @@ from common_factories import ( getSourceTarball, ) from constants import ( - BUILDERS_GALERA_MTR, GITHUB_STATUS_BUILDERS, OS_INFO, SAVED_PACKAGE_BRANCHES, @@ -79,40 +78,45 @@ c["workers"] = [] workers = defaultdict(list) +# For each worker in master_config ['aarch64-bbw1', 2, 3, 4] for w_name in master_config["workers"]: jobs = 7 + worker_name = w_name[:-1] # aarch64-bbw + worker_id = w_name[-1] # 1, 2, 3, 4 + + for arch in master_config["builders"]: + builders = master_config["builders"][arch] + for os_name in builders: + os_definition = builders[os_name] + image_tag = os_definition['image_tag'] + + # Skip s390x non-SLES builders on SLES host (bbw2) + if ("s390x" in arch + and (worker_id == "2") + and ("sles" not in os_name)): + continue + + if image_tag.startswith("ubuntu"): + image_tag = image_tag[:-2] + "." + image_tag[-2:] + + quay_name = f'{os.environ["CONTAINER_REGISTRY_URL"]}{image_tag}' + if arch.startswith("x86"): + os_name += "-i386" + quay_name += "-386" + + base_name, name, worker_instance = createWorker( + worker_name, + worker_id, + os_name, + quay_name, + jobs=jobs, + save_packages=True, + shm_size="15G", + ) - for builder in master_config["builders"]: - worker_name = w_name[:-1] - worker_id = w_name[-1] - - os_name = "-".join(builder.split("-")[1:]) - image_tag = "".join(os_name.split("-")) - - # Skip s390x non-SLES builders on SLES host (bbw2) - if ("s390x" in builder) and (worker_id == "2") and ("sles" not in os_name): - continue - - if image_tag.startswith("ubuntu"): - image_tag = image_tag[:-2] + "." + image_tag[-2:] - - quay_name = os.environ["CONTAINER_REGISTRY_URL"] + image_tag - if builder.startswith("x86"): - os_name += "-i386" - quay_name += "-386" - - base_name, name, worker_instance = createWorker( - worker_name, - worker_id, - os_name, - quay_name, - jobs=jobs, - save_packages=True, - shm_size="15G", - ) + workers[base_name].append(name) + c["workers"].append(worker_instance) - workers[base_name].append(name) - c["workers"].append(worker_instance) ####### FACTORY CODE @@ -231,77 +235,60 @@ f_deb_autobake.addStep( c["builders"] = [] -for builder in master_config["builders"]: - splits = builder.split("-") - arch = splits[0] - os_name = "-".join(splits[1:]) +for arch in master_config["builders"]: + builders_group = master_config["builders"][arch] + for os_name in builders_group: + worker_prefix = arch + worker_suffix = '' + + if arch == "amd64": + worker_prefix = "x64" - if arch == "amd64": - arch = "x64" - worker_name = arch + "-bbw-docker-" + os_name + if arch == "x86": + worker_prefix = 'x64' + worker_suffix = '-i386' + worker_name = f'{worker_prefix}-bbw-docker-{os_name}{worker_suffix}' - if arch == "x86": - worker_name = "x64-bbw-docker-" + os_name + "-i386" + build_type = OS_INFO[os_name]["type"] - build_type = OS_INFO[os_name]["type"] + builder = f'{arch}-{os_name}' - # Add builder only if it's not a protected branches one - if builder not in GITHUB_STATUS_BUILDERS: tags = [os_name] - if arch == "s390x" and builder in BUILDERS_GALERA_MTR: - tags += ["experimental"] - if "sid" in builder or "stream-9" in builder: - tags += ["bleeding-edge"] + # Add builder only if it's not a protected branches one + if builder not in GITHUB_STATUS_BUILDERS: + c["builders"].append( + util.BuilderConfig( + name=builder, + workernames=workers[worker_name], + tags=tags, + collapseRequests=True, + nextBuild=nextBuild, + canStartBuild=canStartBuild, + locks=getLocks, + factory=f_quick_build, + ) + ) + + factory_instance = ( + f_deb_autobake if build_type != "rpm" else f_rpm_autobake + ) + properties = { + "verbose_build": "VERBOSE=1" if arch == "ppc4le" else None, + "rpm_type": "".join(os_name.split("-")) if build_type == "rpm" else None + } + + tags += [build_type, "autobake"] + c["builders"].append( util.BuilderConfig( - name=builder, + name=f"{builder}-{build_type}-autobake", workernames=workers[worker_name], tags=tags, collapseRequests=True, nextBuild=nextBuild, canStartBuild=canStartBuild, locks=getLocks, - factory=f_quick_build, + properties=properties, + factory=factory_instance, ) ) - - factory_instance = f_deb_autobake - properties = {} - - if arch == "ppc64le": - properties["verbose_build"] = "VERBOSE=1" - if build_type == "rpm": - properties["rpm_type"] = "".join(os_name.split("-")) - factory_instance = f_rpm_autobake - tags = [os_name, build_type, "autobake"] - # From mariadb.org-tools/release/prep - under - # Dirs for buildbot.mariadb.org - if builder in [ - "aarch64-openeuler-2403", - "amd64-openeuler-2403", - "s390x-ubuntu-2004", - "s390x-rhel-8", - "s390x-sles-15", - "ppc64le-rhel-9", - "s390x-rhel-9", - "ppc64le-ubuntu-2204", - "s390x-ubuntu-2204", - "amd64-debian-sid", - "aarch64-debian-sid", - "ppc64le-debian-sid", - "amd64-opensuse-1506", - ]: - tags += ["release_packages"] - c["builders"].append( - util.BuilderConfig( - name=builder + "-" + build_type + "-autobake", - workernames=workers[worker_name], - tags=tags, - collapseRequests=True, - nextBuild=nextBuild, - canStartBuild=canStartBuild, - locks=getLocks, - properties=properties, - factory=factory_instance, - ) - ) diff --git a/os_info.yaml b/os_info.yaml index c976284e..9ce437db 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -1,33 +1,53 @@ --- almalinux-8: + image_tag: almalinux-8 version_name: 8 + tags: + - release_packages + - autobake arch: - amd64 - aarch64 type: rpm install_only: True almalinux-9: + image_tag: almalinux-9 version_name: 9 + tags: + - release_packages + - autobake arch: - amd64 - aarch64 type: rpm install_only: True centos-stream9: + image_tag: centos-stream9 version_name: 9 + tags: + - release_packages + - autobake arch: - amd64 - aarch64 - ppc64le type: rpm debian-11: + image_tag: debian-11 version_name: bullseye + tags: + - release_packages + - autobake arch: - amd64 - aarch64 type: deb debian-12: + image_tag: debian-12 version_name: bookworm + tags: + - release_packages + - autobake arch: - amd64 - aarch64 @@ -35,7 +55,12 @@ debian-12: - x86 type: deb debian-sid: + image_tag: debian-sid version_name: sid + tags: + - release_packages + - autobake + - bleeding_edge arch: - amd64 - aarch64 @@ -43,35 +68,62 @@ debian-sid: - x86 type: deb fedora-39: + image_tag: fedora-39 version_name: 39 arch: - amd64 - aarch64 type: rpm + tags: + - autobake + - bleeding_edge fedora-40: + image_tag: fedora-40 + tags: + - release_packages + - autobake version_name: 40 arch: - amd64 - aarch64 type: rpm fedora-41: + image_tag: fedora-41 + tags: + - release_packages + - autobake + - bleeding_edge version_name: 41 arch: - amd64 - aarch64 type: rpm openeuler-2403: + image_tag: openeuler-2403 + tags: + - release_packages + - autobake + - bleeding_edge version_name: 24.03 arch: - amd64 - aarch64 type: rpm opensuse-1506: + image_tag: opensuse-1506 + tags: + - release_packages + - autobake + - bleeding_edge version_name: 156 arch: - amd64 type: rpm rhel-8: + image_tag: rhel-8 + tags: + - release_packages + - autobake version_name: 8 arch: - amd64 @@ -80,6 +132,10 @@ rhel-8: - s390x type: rpm rhel-9: + image_tag: rhel-9 + tags: + - release_packages + - autobake version_name: 9 arch: - amd64 @@ -88,24 +144,38 @@ rhel-9: - s390x type: rpm rockylinux-8: + image_tag: rockylinux-8 + tags: + - autobake version_name: 8 arch: - amd64 type: rpm install_only: True rockylinux-9: + image_tag: rockylinux-9 + tags: + - autobake version_name: 9 arch: - amd64 type: rpm install_only: True sles-1506: + image_tag: sles-1506 + tags: + - release_packages + - autobake version_name: 15 arch: - amd64 - s390x type: rpm ubuntu-2004: + image_tag: ubuntu-2004 + tags: + - release_packages + - autobake version_name: focal arch: - amd64 @@ -114,6 +184,10 @@ ubuntu-2004: - s390x type: deb ubuntu-2204: + image_tag: ubuntu-2204 + tags: + - release_packages + - autobake version_name: jammy arch: - amd64 @@ -122,6 +196,10 @@ ubuntu-2204: - s390x type: deb ubuntu-2404: + image_tag: ubuntu-2404 + tags: + - release_packages + - autobake version_name: noble arch: - amd64 @@ -130,6 +208,10 @@ ubuntu-2404: - s390x type: deb ubuntu-2410: + image_tag: ubuntu-2410 + tags: + - release_packages + - autobake version_name: oracular arch: - amd64 From 89e5742f8b0572902f5ea35ed26907399e605d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 5 Feb 2025 02:00:11 +0200 Subject: [PATCH 16/30] getRpmAutobakeFactory does not take in mtrDbPool --- common_factories.py | 2 +- master-docker-nonstandard-2/master.cfg | 10 ---------- master-docker-nonstandard/master.cfg | 5 +---- master.cfg | 2 +- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/common_factories.py b/common_factories.py index d6242baf..365874d8 100644 --- a/common_factories.py +++ b/common_factories.py @@ -626,7 +626,7 @@ def getTests(props): return addPostTests(f) -def getRpmAutobakeFactory(mtrDbPool): +def getRpmAutobakeFactory(): f_rpm_autobake = util.BuildFactory() f_rpm_autobake.addStep(printEnv()) f_rpm_autobake.addStep( diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index 43c5d078..f9ff573f 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -5,10 +5,7 @@ import os from collections import defaultdict from buildbot.plugins import steps, util -from buildbot.process.properties import Property from common_factories import ( - getQuickBuildFactory, - getRpmAutobakeFactory, getSourceTarball, ) from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES @@ -21,9 +18,6 @@ from utils import ( dockerfile, filterBranch, getHTMLLogString, - hasAutobake, - hasBigtest, - hasEco, hasFailed, moveMTRLogs, mtrJobsMultiplier, @@ -147,10 +141,6 @@ addWorker( ####### FACTORY CODE -f_quick_build = getQuickBuildFactory("nm", mtrDbPool) -f_quick_debug = getQuickBuildFactory("debug", mtrDbPool) -f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool) - ## f_asan_ubsan_build f_asan_ubsan_build = util.BuildFactory() f_asan_ubsan_build.addStep(printEnv()) diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 3a439967..1c605d7f 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -6,7 +6,6 @@ from collections import defaultdict import docker from buildbot.plugins import steps, util, worker -from buildbot.process.properties import Property from common_factories import ( addTests, getHTMLLogString, @@ -23,8 +22,6 @@ from utils import ( createWorker, dockerfile, filterBranch, - hasAutobake, - hasBigtest, hasFailed, moveMTRLogs, mtrJobsMultiplier, @@ -407,7 +404,7 @@ c["workers"].append( f_quick_build = getQuickBuildFactory("nm", mtrDbPool) f_quick_debug = getQuickBuildFactory("debug", mtrDbPool) -f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool) +f_rpm_autobake = getRpmAutobakeFactory() ## f_asan_ubsan_build f_asan_ubsan_build = util.BuildFactory() diff --git a/master.cfg b/master.cfg index e645d7e9..20f73332 100644 --- a/master.cfg +++ b/master.cfg @@ -121,7 +121,7 @@ for w_name in master_config["workers"]: ####### FACTORY CODE f_quick_build = getQuickBuildFactory("nm", mtrDbPool) -f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool) +f_rpm_autobake = getRpmAutobakeFactory() f_deb_autobake = util.BuildFactory() f_deb_autobake.addStep(printEnv()) From 17082419adc988aa48a88825b5ae7e5c47ad7ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 5 Feb 2025 02:00:40 +0200 Subject: [PATCH 17/30] Move getDebAutobakeFactory to common_factories --- common_factories.py | 116 ++++++++++++++++++++++++++++++++++++++++++ master.cfg | 120 +------------------------------------------- 2 files changed, 118 insertions(+), 118 deletions(-) diff --git a/common_factories.py b/common_factories.py index 365874d8..f7db180d 100644 --- a/common_factories.py +++ b/common_factories.py @@ -12,6 +12,7 @@ # Local from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES, test_type_to_mtr_arg from utils import ( + createDebRepo, createVar, dockerfile, getArch, @@ -36,6 +37,7 @@ printEnv, saveLogs, savePackageIfBranchMatch, + uploadDebArtifacts, ) @@ -826,3 +828,117 @@ def getRpmAutobakeFactory(): ) ) return f_rpm_autobake + + +def getDebAutobakeFactory() -> util.BuildFactory: + f_deb_autobake = util.BuildFactory() + f_deb_autobake.addStep(printEnv()) + f_deb_autobake.addStep( + steps.SetProperty( + property="dockerfile", + value=util.Interpolate("%(kw:url)s", url=dockerfile), + description="dockerfile", + ) + ) + f_deb_autobake.addStep(getSourceTarball()) + # build steps + f_deb_autobake.addStep( + steps.Compile( + logfiles={"CMakeCache.txt": "./builddir/CMakeCache.txt"}, + command=["debian/autobake-deb.sh"], + env={ + "CCACHE_DIR": "/mnt/ccache", + "DEB_BUILD_OPTIONS": util.Interpolate( + "parallel=%(kw:jobs)s", + jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"), + ), + }, + description="autobake-deb.sh", + ) + ) + # upload artifacts + f_deb_autobake.addStep( + steps.SetPropertyFromCommand( + command="find .. -maxdepth 1 -type f", extract_fn=ls2string + ) + ) + f_deb_autobake.addStep(createDebRepo()) + f_deb_autobake.addStep(uploadDebArtifacts()) + + f_deb_autobake.addStep( + steps.Trigger( + name="dockerlibrary", + schedulerNames=["s_dockerlibrary"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + }, + doStepIf=lambda step: hasDockerLibrary(step), + ) + ) + f_deb_autobake.addStep( + steps.Trigger( + name="release preparation", + schedulerNames=["s_release_prep"], + waitForFinish=True, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + }, + doStepIf=( + lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) + ), + ) + ) + f_deb_autobake.addStep( + steps.Trigger( + name="install", + schedulerNames=["s_install"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + }, + doStepIf=( + lambda step: hasInstall(step) + and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) + and hasPackagesGenerated(step) + ), + ) + ) + f_deb_autobake.addStep( + steps.Trigger( + name="major-minor-upgrade", + schedulerNames=["s_upgrade"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("tarbuildnum"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + "parentbuildername": Property("buildername"), + }, + doStepIf=( + lambda step: hasUpgrade(step) + and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) + and hasPackagesGenerated(step) + ), + ) + ) + f_deb_autobake.addStep( + steps.ShellCommand( + name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True + ) + ) + + return f_deb_autobake diff --git a/master.cfg b/master.cfg index 20f73332..aa2710fe 100644 --- a/master.cfg +++ b/master.cfg @@ -10,30 +10,20 @@ from buildbot.plugins import steps, util from buildbot.process.properties import Property from common_factories import ( getQuickBuildFactory, + getDebAutobakeFactory, getRpmAutobakeFactory, getSourceTarball, ) from constants import ( GITHUB_STATUS_BUILDERS, OS_INFO, - SAVED_PACKAGE_BRANCHES, ) from locks import getLocks from master_common import base_master_config from utils import ( canStartBuild, - createDebRepo, createWorker, - dockerfile, - hasDockerLibrary, - hasInstall, - hasPackagesGenerated, - hasUpgrade, - ls2string, nextBuild, - printEnv, - savePackageIfBranchMatch, - uploadDebArtifacts, ) cfg_dir = os.path.abspath(os.path.dirname(__file__)) @@ -122,114 +112,8 @@ for w_name in master_config["workers"]: f_quick_build = getQuickBuildFactory("nm", mtrDbPool) f_rpm_autobake = getRpmAutobakeFactory() +f_deb_autobake = getDebAutobakeFactory() -f_deb_autobake = util.BuildFactory() -f_deb_autobake.addStep(printEnv()) -f_deb_autobake.addStep( - steps.SetProperty( - property="dockerfile", - value=util.Interpolate("%(kw:url)s", url=dockerfile), - description="dockerfile", - ) -) -f_deb_autobake.addStep(getSourceTarball()) -# build steps -f_deb_autobake.addStep( - steps.Compile( - logfiles={"CMakeCache.txt": "./builddir/CMakeCache.txt"}, - command=["debian/autobake-deb.sh"], - env={ - "CCACHE_DIR": "/mnt/ccache", - "DEB_BUILD_OPTIONS": util.Interpolate( - "parallel=%(kw:jobs)s", - jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"), - ), - }, - description="autobake-deb.sh", - ) -) -# upload artifacts -f_deb_autobake.addStep( - steps.SetPropertyFromCommand( - command="find .. -maxdepth 1 -type f", extract_fn=ls2string - ) -) -f_deb_autobake.addStep(createDebRepo()) -f_deb_autobake.addStep(uploadDebArtifacts()) - -f_deb_autobake.addStep( - steps.Trigger( - name="dockerlibrary", - schedulerNames=["s_dockerlibrary"], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - "tarbuildnum": Property("tarbuildnum"), - "mariadb_version": Property("mariadb_version"), - "master_branch": Property("master_branch"), - "parentbuildername": Property("buildername"), - }, - doStepIf=lambda step: hasDockerLibrary(step), - ) -) -f_deb_autobake.addStep( - steps.Trigger( - name="release preparation", - schedulerNames=["s_release_prep"], - waitForFinish=True, - updateSourceStamp=False, - set_properties={ - "tarbuildnum": Property("tarbuildnum"), - "mariadb_version": Property("mariadb_version"), - "master_branch": Property("master_branch"), - "parentbuildername": Property("buildername"), - }, - doStepIf=(lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)), - ) -) -f_deb_autobake.addStep( - steps.Trigger( - name="install", - schedulerNames=["s_install"], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - "tarbuildnum": Property("tarbuildnum"), - "mariadb_version": Property("mariadb_version"), - "master_branch": Property("master_branch"), - "parentbuildername": Property("buildername"), - }, - doStepIf=( - lambda step: hasInstall(step) - and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) - and hasPackagesGenerated(step) - ), - ) -) -f_deb_autobake.addStep( - steps.Trigger( - name="major-minor-upgrade", - schedulerNames=["s_upgrade"], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - "tarbuildnum": Property("tarbuildnum"), - "mariadb_version": Property("mariadb_version"), - "master_branch": Property("master_branch"), - "parentbuildername": Property("buildername"), - }, - doStepIf=( - lambda step: hasUpgrade(step) - and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES) - and hasPackagesGenerated(step) - ), - ) -) -f_deb_autobake.addStep( - steps.ShellCommand( - name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True - ) -) ####### BUILDERS LIST From 018da9f95e6492b30548e15b110e0b050ea114f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 5 Feb 2025 02:20:10 +0200 Subject: [PATCH 18/30] start.sh for masters no longer CDs to where master.cfg Instead only cd to base srcdir. This will allow all relative imports to work accordingly. --- docker-compose/start.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose/start.sh b/docker-compose/start.sh index f9ebb2d3..49f7f8a0 100755 --- a/docker-compose/start.sh +++ b/docker-compose/start.sh @@ -11,7 +11,7 @@ err() { } # Make sure to pass the master name as the first argument -cd "/srv/buildbot/master/$1" || err "cd /srv/buildbot/master/$1" +cd "/srv/buildbot/master/" || err "cd /srv/buildbot/master/" # shellcheck disable=SC2226 [[ -f master-private.cfg ]] || ln -s ../master-private.cfg @@ -41,4 +41,4 @@ fi # loop for debug # while true; do date && sleep 30; done -exec buildbot start --nodaemon \ No newline at end of file +exec buildbot start --nodaemon "$1" From 3b9afa2450296915c52a5712ad6f7a630d6c8d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 5 Feb 2025 02:00:40 +0200 Subject: [PATCH 19/30] Move getDebAutobakeFactory to common_factories --- master.cfg | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/master.cfg b/master.cfg index aa2710fe..583288e6 100644 --- a/master.cfg +++ b/master.cfg @@ -6,13 +6,11 @@ from collections import defaultdict import yaml -from buildbot.plugins import steps, util -from buildbot.process.properties import Property +from buildbot.plugins import util from common_factories import ( getQuickBuildFactory, getDebAutobakeFactory, getRpmAutobakeFactory, - getSourceTarball, ) from constants import ( GITHUB_STATUS_BUILDERS, From 5ce398702d1ca00645f87b9d49df4224b2b65465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 5 Feb 2025 14:42:16 +0200 Subject: [PATCH 20/30] All buildbot.tac files refactor - use Path Autogen buildbot.tac sets the basedir as two folders up, while other buildbot.tac files set basedir as one folder up. All the files are the same otherwise. * Small side effect, make maxRotatedFiles set to 30 across the board. --- buildbot.tac | 27 +++++++------ master-docker-nonstandard-2/buildbot.tac | 50 ++++++++++++++++-------- master-docker-nonstandard/buildbot.tac | 50 ++++++++++++++++-------- master-galera/buildbot.tac | 48 ++++++++++++++++------- master-libvirt/buildbot.tac | 48 ++++++++++++++++------- master-nonlatent/buildbot.tac | 48 ++++++++++++++++------- master-protected-branches/buildbot.tac | 50 ++++++++++++++++-------- master-web/buildbot.tac | 48 ++++++++++++++++------- 8 files changed, 248 insertions(+), 121 deletions(-) diff --git a/buildbot.tac b/buildbot.tac index d58d0cbe..6fa2de76 100644 --- a/buildbot.tac +++ b/buildbot.tac @@ -1,6 +1,6 @@ # -*- python -*- # ex: set filetype=python: -import os +from pathlib import Path from twisted.application import service from twisted.python.log import FileLogObserver, ILogObserver @@ -18,39 +18,40 @@ from buildbot.master import BuildMaster #     ├── master-config.yaml #     └── master-private.cfg # -# Thus basedir is two levels below this file"s position. -buildbot_tac_dir = os.path.abspath(os.path.dirname(__file__)) -basedir = os.path.abspath(f"{buildbot_tac_dir}/../../") +# Thus basedir is two levels below this file's position. +buildbot_tac_dir = Path(__file__).resolve().parent +basedir = buildbot_tac_dir.parent.parent # Hard coded as it runs in containers. # TODO(cvicentiu) this should come as an environment variable. -log_basedir = "/var/log/buildbot" +log_basedir_path = Path("/var/log/buildbot/") +log_basedir = log_basedir_path.as_posix() # Kept in case buildbot uses it. -rotateLength = 20000000 +rotateLength = 10000000 maxRotatedFiles = 30 -last_two_dirs = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:] -master_name = last_two_dirs[-1] +master_name = buildbot_tac_dir.name # Last two directories. autogen and -cfg_from_basedir = last_two_dirs + ["master.cfg"] +cfg_from_basedir = (buildbot_tac_dir / "master.cfg").relative_to(basedir) + +configfile = cfg_from_basedir.as_posix() -configfile = os.path.join(*cfg_from_basedir) # Default umask for server umask = None # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. -application = service.Application("buildmaster") # fmt: skip +application = service.Application('buildmaster') # fmt: skip # This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, f"{master_name}.log"), + str(log_basedir_path / f"{master_name}.log"), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) application.setComponent(ILogObserver, FileLogObserver(logfile).emit) -m = BuildMaster(basedir, configfile, umask) +m = BuildMaster(str(basedir), configfile, umask) m.setServiceParent(application) m.log_rotation.rotateLength = rotateLength m.log_rotation.maxRotatedFiles = maxRotatedFiles diff --git a/master-docker-nonstandard-2/buildbot.tac b/master-docker-nonstandard-2/buildbot.tac index af4db63b..98e0381a 100644 --- a/master-docker-nonstandard-2/buildbot.tac +++ b/master-docker-nonstandard-2/buildbot.tac @@ -1,39 +1,57 @@ -import os +# -*- python -*- +# ex: set filetype=python: +from pathlib import Path from twisted.application import service +from twisted.python.log import FileLogObserver, ILogObserver +from twisted.python.logfile import LogFile from buildbot.master import BuildMaster -basedir = "." -log_basedir = "/var/log/buildbot/" - -rotateLength = 20000000 +# This buildbot.tac file is a basis for non-autogen masters. +# The folder structure for autogen masters is: +# +# +# └── master-xxxxxx +#    ├── buildbot.tac +#     ├── master.cfg +#     ├── master-config.yaml +#     └── master-private.cfg +# +# Thus basedir is one levels below this file's position. +buildbot_tac_dir = Path(__file__).resolve().parent +basedir = buildbot_tac_dir.parent + +# Hard coded as it runs in containers. +# TODO(cvicentiu) this should come as an environment variable. +log_basedir_path = Path("/var/log/buildbot/") +log_basedir = log_basedir_path.as_posix() # Kept in case buildbot uses it. + +rotateLength = 10000000 maxRotatedFiles = 30 -configfile = "master.cfg" -# Default umask for server -umask = None +master_name = buildbot_tac_dir.name +# Last two directories. autogen and +cfg_from_basedir = (buildbot_tac_dir / "master.cfg").relative_to(basedir) -# if this is a relocatable tac file, get the directory containing the TAC -if basedir == ".": - import os +configfile = cfg_from_basedir.as_posix() - basedir = os.path.abspath(os.path.dirname(__file__)) +# Default umask for server +umask = None # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. application = service.Application('buildmaster') # fmt: skip -from twisted.python.log import FileLogObserver, ILogObserver -from twisted.python.logfile import LogFile +# This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, "master-docker-non-standard-2.log"), + str(log_basedir_path / f"{master_name}.log"), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) application.setComponent(ILogObserver, FileLogObserver(logfile).emit) -m = BuildMaster(basedir, configfile, umask) +m = BuildMaster(str(basedir), configfile, umask) m.setServiceParent(application) m.log_rotation.rotateLength = rotateLength m.log_rotation.maxRotatedFiles = maxRotatedFiles diff --git a/master-docker-nonstandard/buildbot.tac b/master-docker-nonstandard/buildbot.tac index 0993d9a5..98e0381a 100644 --- a/master-docker-nonstandard/buildbot.tac +++ b/master-docker-nonstandard/buildbot.tac @@ -1,39 +1,57 @@ -import os +# -*- python -*- +# ex: set filetype=python: +from pathlib import Path from twisted.application import service +from twisted.python.log import FileLogObserver, ILogObserver +from twisted.python.logfile import LogFile from buildbot.master import BuildMaster -basedir = "." -log_basedir = "/var/log/buildbot/" - -rotateLength = 20000000 +# This buildbot.tac file is a basis for non-autogen masters. +# The folder structure for autogen masters is: +# +# +# └── master-xxxxxx +#    ├── buildbot.tac +#     ├── master.cfg +#     ├── master-config.yaml +#     └── master-private.cfg +# +# Thus basedir is one levels below this file's position. +buildbot_tac_dir = Path(__file__).resolve().parent +basedir = buildbot_tac_dir.parent + +# Hard coded as it runs in containers. +# TODO(cvicentiu) this should come as an environment variable. +log_basedir_path = Path("/var/log/buildbot/") +log_basedir = log_basedir_path.as_posix() # Kept in case buildbot uses it. + +rotateLength = 10000000 maxRotatedFiles = 30 -configfile = "master.cfg" -# Default umask for server -umask = None +master_name = buildbot_tac_dir.name +# Last two directories. autogen and +cfg_from_basedir = (buildbot_tac_dir / "master.cfg").relative_to(basedir) -# if this is a relocatable tac file, get the directory containing the TAC -if basedir == ".": - import os +configfile = cfg_from_basedir.as_posix() - basedir = os.path.abspath(os.path.dirname(__file__)) +# Default umask for server +umask = None # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. application = service.Application('buildmaster') # fmt: skip -from twisted.python.log import FileLogObserver, ILogObserver -from twisted.python.logfile import LogFile +# This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, "master-docker-non-standard.log"), + str(log_basedir_path / f"{master_name}.log"), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) application.setComponent(ILogObserver, FileLogObserver(logfile).emit) -m = BuildMaster(basedir, configfile, umask) +m = BuildMaster(str(basedir), configfile, umask) m.setServiceParent(application) m.log_rotation.rotateLength = rotateLength m.log_rotation.maxRotatedFiles = maxRotatedFiles diff --git a/master-galera/buildbot.tac b/master-galera/buildbot.tac index d48bb688..98e0381a 100644 --- a/master-galera/buildbot.tac +++ b/master-galera/buildbot.tac @@ -1,39 +1,57 @@ -import os +# -*- python -*- +# ex: set filetype=python: +from pathlib import Path from twisted.application import service +from twisted.python.log import FileLogObserver, ILogObserver +from twisted.python.logfile import LogFile from buildbot.master import BuildMaster -basedir = "." -log_basedir = "/var/log/buildbot/" +# This buildbot.tac file is a basis for non-autogen masters. +# The folder structure for autogen masters is: +# +# +# └── master-xxxxxx +#    ├── buildbot.tac +#     ├── master.cfg +#     ├── master-config.yaml +#     └── master-private.cfg +# +# Thus basedir is one levels below this file's position. +buildbot_tac_dir = Path(__file__).resolve().parent +basedir = buildbot_tac_dir.parent + +# Hard coded as it runs in containers. +# TODO(cvicentiu) this should come as an environment variable. +log_basedir_path = Path("/var/log/buildbot/") +log_basedir = log_basedir_path.as_posix() # Kept in case buildbot uses it. rotateLength = 10000000 -maxRotatedFiles = 10 -configfile = "master.cfg" +maxRotatedFiles = 30 -# Default umask for server -umask = None +master_name = buildbot_tac_dir.name +# Last two directories. autogen and +cfg_from_basedir = (buildbot_tac_dir / "master.cfg").relative_to(basedir) -# if this is a relocatable tac file, get the directory containing the TAC -if basedir == ".": - import os +configfile = cfg_from_basedir.as_posix() - basedir = os.path.abspath(os.path.dirname(__file__)) +# Default umask for server +umask = None # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. application = service.Application('buildmaster') # fmt: skip -from twisted.python.log import FileLogObserver, ILogObserver -from twisted.python.logfile import LogFile +# This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, "master-galera.log"), + str(log_basedir_path / f"{master_name}.log"), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) application.setComponent(ILogObserver, FileLogObserver(logfile).emit) -m = BuildMaster(basedir, configfile, umask) +m = BuildMaster(str(basedir), configfile, umask) m.setServiceParent(application) m.log_rotation.rotateLength = rotateLength m.log_rotation.maxRotatedFiles = maxRotatedFiles diff --git a/master-libvirt/buildbot.tac b/master-libvirt/buildbot.tac index 10360bf8..98e0381a 100644 --- a/master-libvirt/buildbot.tac +++ b/master-libvirt/buildbot.tac @@ -1,39 +1,57 @@ -import os +# -*- python -*- +# ex: set filetype=python: +from pathlib import Path from twisted.application import service +from twisted.python.log import FileLogObserver, ILogObserver +from twisted.python.logfile import LogFile from buildbot.master import BuildMaster -basedir = "." -log_basedir = "/var/log/buildbot/" +# This buildbot.tac file is a basis for non-autogen masters. +# The folder structure for autogen masters is: +# +# +# └── master-xxxxxx +#    ├── buildbot.tac +#     ├── master.cfg +#     ├── master-config.yaml +#     └── master-private.cfg +# +# Thus basedir is one levels below this file's position. +buildbot_tac_dir = Path(__file__).resolve().parent +basedir = buildbot_tac_dir.parent + +# Hard coded as it runs in containers. +# TODO(cvicentiu) this should come as an environment variable. +log_basedir_path = Path("/var/log/buildbot/") +log_basedir = log_basedir_path.as_posix() # Kept in case buildbot uses it. rotateLength = 10000000 -maxRotatedFiles = 10 -configfile = "master.cfg" +maxRotatedFiles = 30 -# Default umask for server -umask = None +master_name = buildbot_tac_dir.name +# Last two directories. autogen and +cfg_from_basedir = (buildbot_tac_dir / "master.cfg").relative_to(basedir) -# if this is a relocatable tac file, get the directory containing the TAC -if basedir == ".": - import os +configfile = cfg_from_basedir.as_posix() - basedir = os.path.abspath(os.path.dirname(__file__)) +# Default umask for server +umask = None # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. application = service.Application('buildmaster') # fmt: skip -from twisted.python.log import FileLogObserver, ILogObserver -from twisted.python.logfile import LogFile +# This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, "master-libvirt.log"), + str(log_basedir_path / f"{master_name}.log"), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) application.setComponent(ILogObserver, FileLogObserver(logfile).emit) -m = BuildMaster(basedir, configfile, umask) +m = BuildMaster(str(basedir), configfile, umask) m.setServiceParent(application) m.log_rotation.rotateLength = rotateLength m.log_rotation.maxRotatedFiles = maxRotatedFiles diff --git a/master-nonlatent/buildbot.tac b/master-nonlatent/buildbot.tac index 107d3918..98e0381a 100644 --- a/master-nonlatent/buildbot.tac +++ b/master-nonlatent/buildbot.tac @@ -1,39 +1,57 @@ -import os +# -*- python -*- +# ex: set filetype=python: +from pathlib import Path from twisted.application import service +from twisted.python.log import FileLogObserver, ILogObserver +from twisted.python.logfile import LogFile from buildbot.master import BuildMaster -basedir = "." -log_basedir = "/var/log/buildbot/" +# This buildbot.tac file is a basis for non-autogen masters. +# The folder structure for autogen masters is: +# +# +# └── master-xxxxxx +#    ├── buildbot.tac +#     ├── master.cfg +#     ├── master-config.yaml +#     └── master-private.cfg +# +# Thus basedir is one levels below this file's position. +buildbot_tac_dir = Path(__file__).resolve().parent +basedir = buildbot_tac_dir.parent + +# Hard coded as it runs in containers. +# TODO(cvicentiu) this should come as an environment variable. +log_basedir_path = Path("/var/log/buildbot/") +log_basedir = log_basedir_path.as_posix() # Kept in case buildbot uses it. rotateLength = 10000000 -maxRotatedFiles = 10 -configfile = "master.cfg" +maxRotatedFiles = 30 -# Default umask for server -umask = None +master_name = buildbot_tac_dir.name +# Last two directories. autogen and +cfg_from_basedir = (buildbot_tac_dir / "master.cfg").relative_to(basedir) -# if this is a relocatable tac file, get the directory containing the TAC -if basedir == ".": - import os +configfile = cfg_from_basedir.as_posix() - basedir = os.path.abspath(os.path.dirname(__file__)) +# Default umask for server +umask = None # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. application = service.Application('buildmaster') # fmt: skip -from twisted.python.log import FileLogObserver, ILogObserver -from twisted.python.logfile import LogFile +# This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, "master-nonlatent.log"), + str(log_basedir_path / f"{master_name}.log"), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) application.setComponent(ILogObserver, FileLogObserver(logfile).emit) -m = BuildMaster(basedir, configfile, umask) +m = BuildMaster(str(basedir), configfile, umask) m.setServiceParent(application) m.log_rotation.rotateLength = rotateLength m.log_rotation.maxRotatedFiles = maxRotatedFiles diff --git a/master-protected-branches/buildbot.tac b/master-protected-branches/buildbot.tac index 6c8546b0..98e0381a 100644 --- a/master-protected-branches/buildbot.tac +++ b/master-protected-branches/buildbot.tac @@ -1,39 +1,57 @@ -import os +# -*- python -*- +# ex: set filetype=python: +from pathlib import Path from twisted.application import service +from twisted.python.log import FileLogObserver, ILogObserver +from twisted.python.logfile import LogFile from buildbot.master import BuildMaster -basedir = "." -log_basedir = "/var/log/buildbot/" - -rotateLength = 20000000 +# This buildbot.tac file is a basis for non-autogen masters. +# The folder structure for autogen masters is: +# +# +# └── master-xxxxxx +#    ├── buildbot.tac +#     ├── master.cfg +#     ├── master-config.yaml +#     └── master-private.cfg +# +# Thus basedir is one levels below this file's position. +buildbot_tac_dir = Path(__file__).resolve().parent +basedir = buildbot_tac_dir.parent + +# Hard coded as it runs in containers. +# TODO(cvicentiu) this should come as an environment variable. +log_basedir_path = Path("/var/log/buildbot/") +log_basedir = log_basedir_path.as_posix() # Kept in case buildbot uses it. + +rotateLength = 10000000 maxRotatedFiles = 30 -configfile = "master.cfg" -# Default umask for server -umask = None +master_name = buildbot_tac_dir.name +# Last two directories. autogen and +cfg_from_basedir = (buildbot_tac_dir / "master.cfg").relative_to(basedir) -# if this is a relocatable tac file, get the directory containing the TAC -if basedir == ".": - import os +configfile = cfg_from_basedir.as_posix() - basedir = os.path.abspath(os.path.dirname(__file__)) +# Default umask for server +umask = None # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. application = service.Application('buildmaster') # fmt: skip -from twisted.python.log import FileLogObserver, ILogObserver -from twisted.python.logfile import LogFile +# This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, "master-protected-branches.log"), + str(log_basedir_path / f"{master_name}.log"), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) application.setComponent(ILogObserver, FileLogObserver(logfile).emit) -m = BuildMaster(basedir, configfile, umask) +m = BuildMaster(str(basedir), configfile, umask) m.setServiceParent(application) m.log_rotation.rotateLength = rotateLength m.log_rotation.maxRotatedFiles = maxRotatedFiles diff --git a/master-web/buildbot.tac b/master-web/buildbot.tac index 0e513b97..98e0381a 100644 --- a/master-web/buildbot.tac +++ b/master-web/buildbot.tac @@ -1,39 +1,57 @@ -import os +# -*- python -*- +# ex: set filetype=python: +from pathlib import Path from twisted.application import service +from twisted.python.log import FileLogObserver, ILogObserver +from twisted.python.logfile import LogFile from buildbot.master import BuildMaster -basedir = "." -log_basedir = "/var/log/buildbot/" +# This buildbot.tac file is a basis for non-autogen masters. +# The folder structure for autogen masters is: +# +# +# └── master-xxxxxx +#    ├── buildbot.tac +#     ├── master.cfg +#     ├── master-config.yaml +#     └── master-private.cfg +# +# Thus basedir is one levels below this file's position. +buildbot_tac_dir = Path(__file__).resolve().parent +basedir = buildbot_tac_dir.parent + +# Hard coded as it runs in containers. +# TODO(cvicentiu) this should come as an environment variable. +log_basedir_path = Path("/var/log/buildbot/") +log_basedir = log_basedir_path.as_posix() # Kept in case buildbot uses it. rotateLength = 10000000 -maxRotatedFiles = 10 -configfile = "master.cfg" +maxRotatedFiles = 30 -# Default umask for server -umask = None +master_name = buildbot_tac_dir.name +# Last two directories. autogen and +cfg_from_basedir = (buildbot_tac_dir / "master.cfg").relative_to(basedir) -# if this is a relocatable tac file, get the directory containing the TAC -if basedir == ".": - import os +configfile = cfg_from_basedir.as_posix() - basedir = os.path.abspath(os.path.dirname(__file__)) +# Default umask for server +umask = None # note: this line is matched against to check that this is a buildmaster # directory; do not edit it. application = service.Application('buildmaster') # fmt: skip -from twisted.python.log import FileLogObserver, ILogObserver -from twisted.python.logfile import LogFile +# This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, "master-web.log"), + str(log_basedir_path / f"{master_name}.log"), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) application.setComponent(ILogObserver, FileLogObserver(logfile).emit) -m = BuildMaster(basedir, configfile, umask) +m = BuildMaster(str(basedir), configfile, umask) m.setServiceParent(application) m.log_rotation.rotateLength = rotateLength m.log_rotation.maxRotatedFiles = maxRotatedFiles From 230e572f4ecc46b160eb87664f50ab380b6de731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 5 Feb 2025 16:51:58 +0200 Subject: [PATCH 21/30] Correct image tags for ubuntu No need to modify them in the master.cfg, just define them correctly in os_info.yaml --- master.cfg | 3 --- os_info.yaml | 8 ++++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/master.cfg b/master.cfg index 583288e6..8ca5d23c 100644 --- a/master.cfg +++ b/master.cfg @@ -84,9 +84,6 @@ for w_name in master_config["workers"]: and ("sles" not in os_name)): continue - if image_tag.startswith("ubuntu"): - image_tag = image_tag[:-2] + "." + image_tag[-2:] - quay_name = f'{os.environ["CONTAINER_REGISTRY_URL"]}{image_tag}' if arch.startswith("x86"): os_name += "-i386" diff --git a/os_info.yaml b/os_info.yaml index 9ce437db..67da1197 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -172,7 +172,7 @@ sles-1506: - s390x type: rpm ubuntu-2004: - image_tag: ubuntu-2004 + image_tag: ubuntu20.04 tags: - release_packages - autobake @@ -184,7 +184,7 @@ ubuntu-2004: - s390x type: deb ubuntu-2204: - image_tag: ubuntu-2204 + image_tag: ubuntu22.04 tags: - release_packages - autobake @@ -196,7 +196,7 @@ ubuntu-2204: - s390x type: deb ubuntu-2404: - image_tag: ubuntu-2404 + image_tag: ubuntu24.04 tags: - release_packages - autobake @@ -208,7 +208,7 @@ ubuntu-2404: - s390x type: deb ubuntu-2410: - image_tag: ubuntu-2410 + image_tag: ubuntu24.10 tags: - release_packages - autobake From 9f094ec2e226577067a68cc1f5da4baab27dcfe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 5 Feb 2025 17:00:18 +0200 Subject: [PATCH 22/30] Do not add autobake tag in os_info.yaml * autobake tag gets added to autobake builders during master.cfg --- os_info.yaml | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/os_info.yaml b/os_info.yaml index 67da1197..8db4c26d 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -4,7 +4,6 @@ almalinux-8: version_name: 8 tags: - release_packages - - autobake arch: - amd64 - aarch64 @@ -15,7 +14,6 @@ almalinux-9: version_name: 9 tags: - release_packages - - autobake arch: - amd64 - aarch64 @@ -26,7 +24,6 @@ centos-stream9: version_name: 9 tags: - release_packages - - autobake arch: - amd64 - aarch64 @@ -37,7 +34,6 @@ debian-11: version_name: bullseye tags: - release_packages - - autobake arch: - amd64 - aarch64 @@ -47,7 +43,6 @@ debian-12: version_name: bookworm tags: - release_packages - - autobake arch: - amd64 - aarch64 @@ -59,7 +54,6 @@ debian-sid: version_name: sid tags: - release_packages - - autobake - bleeding_edge arch: - amd64 @@ -75,13 +69,11 @@ fedora-39: - aarch64 type: rpm tags: - - autobake - bleeding_edge fedora-40: image_tag: fedora-40 tags: - release_packages - - autobake version_name: 40 arch: - amd64 @@ -91,7 +83,6 @@ fedora-41: image_tag: fedora-41 tags: - release_packages - - autobake - bleeding_edge version_name: 41 arch: @@ -102,7 +93,6 @@ openeuler-2403: image_tag: openeuler-2403 tags: - release_packages - - autobake - bleeding_edge version_name: 24.03 arch: @@ -113,7 +103,6 @@ opensuse-1506: image_tag: opensuse-1506 tags: - release_packages - - autobake - bleeding_edge version_name: 156 arch: @@ -123,7 +112,6 @@ rhel-8: image_tag: rhel-8 tags: - release_packages - - autobake version_name: 8 arch: - amd64 @@ -135,7 +123,6 @@ rhel-9: image_tag: rhel-9 tags: - release_packages - - autobake version_name: 9 arch: - amd64 @@ -145,8 +132,7 @@ rhel-9: type: rpm rockylinux-8: image_tag: rockylinux-8 - tags: - - autobake + tags: [] version_name: 8 arch: - amd64 @@ -154,8 +140,7 @@ rockylinux-8: install_only: True rockylinux-9: image_tag: rockylinux-9 - tags: - - autobake + tags: [] version_name: 9 arch: - amd64 @@ -165,7 +150,6 @@ sles-1506: image_tag: sles-1506 tags: - release_packages - - autobake version_name: 15 arch: - amd64 @@ -175,7 +159,6 @@ ubuntu-2004: image_tag: ubuntu20.04 tags: - release_packages - - autobake version_name: focal arch: - amd64 @@ -187,7 +170,6 @@ ubuntu-2204: image_tag: ubuntu22.04 tags: - release_packages - - autobake version_name: jammy arch: - amd64 @@ -199,7 +181,6 @@ ubuntu-2404: image_tag: ubuntu24.04 tags: - release_packages - - autobake version_name: noble arch: - amd64 @@ -211,7 +192,6 @@ ubuntu-2410: image_tag: ubuntu24.10 tags: - release_packages - - autobake version_name: oracular arch: - amd64 From 57e45527538fcb2ada535721399534bf3202f068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 5 Feb 2025 17:22:02 +0200 Subject: [PATCH 23/30] cleanup: master.cfg simplify worker name suffix/prefix construction No logic changes, just reorder the code to make it shorter. --- master.cfg | 20 +++++--------------- os_info.yaml | 2 +- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/master.cfg b/master.cfg index 8ca5d23c..4d29064a 100644 --- a/master.cfg +++ b/master.cfg @@ -74,8 +74,7 @@ for w_name in master_config["workers"]: for arch in master_config["builders"]: builders = master_config["builders"][arch] - for os_name in builders: - os_definition = builders[os_name] + for os_name, os_definition in builders.items(): image_tag = os_definition['image_tag'] # Skip s390x non-SLES builders on SLES host (bbw2) @@ -111,26 +110,17 @@ f_deb_autobake = getDebAutobakeFactory() ####### BUILDERS LIST - c["builders"] = [] for arch in master_config["builders"]: + worker_prefix = "x64" if arch in ["amd64", "x86"] else arch + worker_suffix = "-i386" if arch == "x86" else "" + builders_group = master_config["builders"][arch] for os_name in builders_group: - worker_prefix = arch - worker_suffix = '' - - if arch == "amd64": - worker_prefix = "x64" - - if arch == "x86": - worker_prefix = 'x64' - worker_suffix = '-i386' worker_name = f'{worker_prefix}-bbw-docker-{os_name}{worker_suffix}' - - build_type = OS_INFO[os_name]["type"] - builder = f'{arch}-{os_name}' + build_type = OS_INFO[os_name]["type"] tags = [os_name] # Add builder only if it's not a protected branches one diff --git a/os_info.yaml b/os_info.yaml index 8db4c26d..9c4c60cd 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -104,7 +104,7 @@ opensuse-1506: tags: - release_packages - bleeding_edge - version_name: 156 + version_name: 15 arch: - amd64 type: rpm From 5cab4fef4c85941f175e56b19ab225ba148df458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Thu, 6 Feb 2025 13:09:31 +0200 Subject: [PATCH 24/30] Fixup tags for all autogen builders * Typo in ppc64le * Use tags from master-config.yml, not from OS_INFO. * Have define_masters add tags to master-config.yml --- define_masters.py | 1 + master.cfg | 13 ++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/define_masters.py b/define_masters.py index 29273932..15c0dd56 100755 --- a/define_masters.py +++ b/define_masters.py @@ -28,6 +28,7 @@ platforms[arch][os_name] = { "image_tag": OS_INFO[os_name]["image_tag"], "tags": OS_INFO[os_name]["tags"], + "type": OS_INFO[os_name]["type"], } # Clear old configurations diff --git a/master.cfg b/master.cfg index 4d29064a..e4044787 100644 --- a/master.cfg +++ b/master.cfg @@ -14,7 +14,6 @@ from common_factories import ( ) from constants import ( GITHUB_STATUS_BUILDERS, - OS_INFO, ) from locks import getLocks from master_common import base_master_config @@ -120,9 +119,9 @@ for arch in master_config["builders"]: for os_name in builders_group: worker_name = f'{worker_prefix}-bbw-docker-{os_name}{worker_suffix}' builder = f'{arch}-{os_name}' - build_type = OS_INFO[os_name]["type"] + build_type = builders_group[os_name]["type"] - tags = [os_name] + tags = builders_group[os_name]["tags"] # Add builder only if it's not a protected branches one if builder not in GITHUB_STATUS_BUILDERS: c["builders"].append( @@ -142,17 +141,17 @@ for arch in master_config["builders"]: f_deb_autobake if build_type != "rpm" else f_rpm_autobake ) properties = { - "verbose_build": "VERBOSE=1" if arch == "ppc4le" else None, - "rpm_type": "".join(os_name.split("-")) if build_type == "rpm" else None + "verbose_build": "VERBOSE=1" if arch == "ppc64le" else "", + "rpm_type": "".join(os_name.split("-")) if build_type == "rpm" else "" } - tags += [build_type, "autobake"] + autobake_tags = tags + [build_type, "autobake"] c["builders"].append( util.BuilderConfig( name=f"{builder}-{build_type}-autobake", workernames=workers[worker_name], - tags=tags, + tags=autobake_tags, collapseRequests=True, nextBuild=nextBuild, canStartBuild=canStartBuild, From 8a26e94f3a785d2ecbe046889cd426e6bcff5541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Thu, 6 Feb 2025 13:11:10 +0200 Subject: [PATCH 25/30] Fix github_access_token, it should be a string, not a list Accidental tuple was created because of a stray `,`. --- master_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master_common.py b/master_common.py index c996961f..eef9bba4 100644 --- a/master_common.py +++ b/master_common.py @@ -16,7 +16,7 @@ def base_master_config( ): # TODO(cvicentiu) either move this to environ or all other params to config # file. - github_access_token = (config["private"]["gh_mdbci"]["access_token"],) + github_access_token = config["private"]["gh_mdbci"]["access_token"] db_url = config["private"]["db_url"] return { From 694521380a472825638a55e34844704768f0689c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Thu, 6 Feb 2025 13:11:53 +0200 Subject: [PATCH 26/30] Correct image tag names in os_info.yaml --- os_info.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/os_info.yaml b/os_info.yaml index 9c4c60cd..2cc3f248 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -1,6 +1,6 @@ --- almalinux-8: - image_tag: almalinux-8 + image_tag: almalinux8 version_name: 8 tags: - release_packages @@ -10,7 +10,7 @@ almalinux-8: type: rpm install_only: True almalinux-9: - image_tag: almalinux-9 + image_tag: almalinux9 version_name: 9 tags: - release_packages @@ -20,7 +20,7 @@ almalinux-9: type: rpm install_only: True centos-stream9: - image_tag: centos-stream9 + image_tag: centosstream9 version_name: 9 tags: - release_packages @@ -30,7 +30,7 @@ centos-stream9: - ppc64le type: rpm debian-11: - image_tag: debian-11 + image_tag: debian11 version_name: bullseye tags: - release_packages @@ -39,7 +39,7 @@ debian-11: - aarch64 type: deb debian-12: - image_tag: debian-12 + image_tag: debian12 version_name: bookworm tags: - release_packages @@ -50,7 +50,7 @@ debian-12: - x86 type: deb debian-sid: - image_tag: debian-sid + image_tag: debiansid version_name: sid tags: - release_packages @@ -62,7 +62,7 @@ debian-sid: - x86 type: deb fedora-39: - image_tag: fedora-39 + image_tag: fedora39 version_name: 39 arch: - amd64 @@ -71,7 +71,7 @@ fedora-39: tags: - bleeding_edge fedora-40: - image_tag: fedora-40 + image_tag: fedora40 tags: - release_packages version_name: 40 @@ -80,7 +80,7 @@ fedora-40: - aarch64 type: rpm fedora-41: - image_tag: fedora-41 + image_tag: fedora41 tags: - release_packages - bleeding_edge @@ -90,7 +90,7 @@ fedora-41: - aarch64 type: rpm openeuler-2403: - image_tag: openeuler-2403 + image_tag: openeuler2403 tags: - release_packages - bleeding_edge @@ -100,7 +100,7 @@ openeuler-2403: - aarch64 type: rpm opensuse-1506: - image_tag: opensuse-1506 + image_tag: opensuse1506 tags: - release_packages - bleeding_edge @@ -109,7 +109,7 @@ opensuse-1506: - amd64 type: rpm rhel-8: - image_tag: rhel-8 + image_tag: rhel8 tags: - release_packages version_name: 8 @@ -120,7 +120,7 @@ rhel-8: - s390x type: rpm rhel-9: - image_tag: rhel-9 + image_tag: rhel9 tags: - release_packages version_name: 9 @@ -131,7 +131,7 @@ rhel-9: - s390x type: rpm rockylinux-8: - image_tag: rockylinux-8 + image_tag: rockylinux8 tags: [] version_name: 8 arch: @@ -139,7 +139,7 @@ rockylinux-8: type: rpm install_only: True rockylinux-9: - image_tag: rockylinux-9 + image_tag: rockylinux9 tags: [] version_name: 9 arch: @@ -147,7 +147,7 @@ rockylinux-9: type: rpm install_only: True sles-1506: - image_tag: sles-1506 + image_tag: sles1506 tags: - release_packages version_name: 15 From c13fdcb6b60ec1867e5c41e2ef2d24133d787df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Thu, 6 Feb 2025 15:22:10 +0200 Subject: [PATCH 27/30] Cleanup define masters printing --- define_masters.py | 7 ++++++- master_common.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/define_masters.py b/define_masters.py index 15c0dd56..9a4f727e 100755 --- a/define_masters.py +++ b/define_masters.py @@ -59,4 +59,9 @@ shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg") shutil.copyfile("buildbot.tac", dir_path + "/buildbot.tac") - print(arch, len(master_config["builders"])) + print( + "{:<20} {}".format( + f"Master {master_id} {arch}", + f'{len(master_config["builders"][arch])} builders', + ) + ) diff --git a/master_common.py b/master_common.py index eef9bba4..ca18c67e 100644 --- a/master_common.py +++ b/master_common.py @@ -54,7 +54,7 @@ def base_master_config( # 'port' must match the value configured into the workers (with their # --master option) "protocols": { - "pb": {"port": master_port}, + "pb": {"port": int(master_port)}, # master_port must be int }, # This specifies what database buildbot uses to store its state. "db": { From 73c5647be26c6271a1566356f12d9e4a54692c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Thu, 6 Feb 2025 15:23:36 +0200 Subject: [PATCH 28/30] Fix bleeding-edge tags --- os_info.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/os_info.yaml b/os_info.yaml index 2cc3f248..bacefa64 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -24,6 +24,7 @@ centos-stream9: version_name: 9 tags: - release_packages + - bleeding-edge arch: - amd64 - aarch64 @@ -54,7 +55,7 @@ debian-sid: version_name: sid tags: - release_packages - - bleeding_edge + - bleeding-edge arch: - amd64 - aarch64 @@ -68,8 +69,7 @@ fedora-39: - amd64 - aarch64 type: rpm - tags: - - bleeding_edge + tags: [] fedora-40: image_tag: fedora40 tags: @@ -83,7 +83,6 @@ fedora-41: image_tag: fedora41 tags: - release_packages - - bleeding_edge version_name: 41 arch: - amd64 @@ -93,7 +92,6 @@ openeuler-2403: image_tag: openeuler2403 tags: - release_packages - - bleeding_edge version_name: 24.03 arch: - amd64 @@ -103,8 +101,7 @@ opensuse-1506: image_tag: opensuse1506 tags: - release_packages - - bleeding_edge - version_name: 15 + version_name: 156 arch: - amd64 type: rpm From b7ec41c0ea5aab28cf0eb16239de77ecab0563a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Thu, 6 Feb 2025 15:23:47 +0200 Subject: [PATCH 29/30] Append os_name tag to all builders --- master.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/master.cfg b/master.cfg index e4044787..17a1555d 100644 --- a/master.cfg +++ b/master.cfg @@ -122,6 +122,9 @@ for arch in master_config["builders"]: build_type = builders_group[os_name]["type"] tags = builders_group[os_name]["tags"] + # For easier searching in Builders tab for all sub-builders, like + # autobake, install, upgrade from the same OS. + tags.append(os_name) # Add builder only if it's not a protected branches one if builder not in GITHUB_STATUS_BUILDERS: c["builders"].append( From fc72186f0825d6edaed70989649f6e22dbc2fcb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Thu, 6 Feb 2025 17:48:07 +0200 Subject: [PATCH 30/30] Fixup master-web * No longer run buildbot upgrade-master. The script is not compatible with our deployment format. * Fix paths in start-bbm-web.sh * www port must be an int. --- docker-compose/start-bbm-web.sh | 5 ++--- master-web/master.cfg | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docker-compose/start-bbm-web.sh b/docker-compose/start-bbm-web.sh index 19031fd7..7e26068e 100755 --- a/docker-compose/start-bbm-web.sh +++ b/docker-compose/start-bbm-web.sh @@ -10,7 +10,7 @@ err() { exit 1 } -cd /srv/buildbot/master/master-web || err "cd /srv/buildbot/master/master-web" +cd /srv/buildbot/master/ || err "cd /srv/buildbot/master/" # # loop for debug # while true; do date && sleep 30; done @@ -29,5 +29,4 @@ while ! nc -z 127.0.0.1 8080; do sleep 0.1 done echo "Crossbar started" -buildbot upgrade-master /srv/buildbot/master/master-web -exec buildbot start --nodaemon +exec buildbot start --nodaemon master-web diff --git a/master-web/master.cfg b/master-web/master.cfg index bb2ad46d..f664f76a 100644 --- a/master-web/master.cfg +++ b/master-web/master.cfg @@ -34,7 +34,7 @@ c = BuildmasterConfig = base_master_config(config) c["protocols"] = {} # minimalistic config to activate web UI c["www"] = { - "port": os.environ["PORT"], + "port": int(os.environ["PORT"]), "plugins": { "waterfall_view": {}, "console_view": {},