Refactor master cfg

buildbot.tac

@@ -1,33 +1,50 @@
+# -*- python -*-
+# ex: set filetype=python:
 import os
 
 from twisted.application import service
+from twisted.python.log import FileLogObserver, ILogObserver
+from twisted.python.logfile import LogFile
 
 from buildbot.master import BuildMaster
 
-basedir = "."
-log_basedir = "/var/log/buildbot/"
+# This buildbot.tac file is a basis for all "autogen" masters.
+# The folder structure for autogen masters is:
+#
+# autogen
+# └── aarch64-master-0
+#     ├── buildbot.tac
+#     ├── master.cfg
+#     ├── master-config.yaml
+#     └── master-private.cfg
+#
+# Thus basedir is two levels below this file"s position.
+buildbot_tac_dir = os.path.abspath(os.path.dirname(__file__))
+basedir = os.path.abspath(f"{buildbot_tac_dir}/../../")
+
+# Hard coded as it runs in containers.
+# TODO(cvicentiu) this should come as an environment variable.
+log_basedir = "/var/log/buildbot"
 
 rotateLength = 20000000
 maxRotatedFiles = 30
-configfile = "master.cfg"
 
+last_two_dirs = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:]
+master_name = last_two_dirs[-1]
+# Last two directories. autogen and <master-name>
+cfg_from_basedir = last_two_dirs + ["master.cfg"]
+
+configfile = os.path.join(*cfg_from_basedir)
 # Default umask for server
 umask = None
 
-# if this is a relocatable tac file, get the directory containing the TAC
-if basedir == ".":
-    import os
-
-    basedir = os.path.abspath(os.path.dirname(__file__))
-
 # note: this line is matched against to check that this is a buildmaster
 # directory; do not edit it.
-application = service.Application('buildmaster')  # fmt: skip
-from twisted.python.log import FileLogObserver, ILogObserver
-from twisted.python.logfile import LogFile
+application = service.Application("buildmaster")  # fmt: skip
 
+# This logfile is monitored. It must end in .log.
 logfile = LogFile.fromFullPath(
-    os.path.join(log_basedir, "%s"),
+    os.path.join(log_basedir, f'{master_name}.log'),
     rotateLength=rotateLength,
     maxRotatedFiles=maxRotatedFiles,
 )

constants.py

@@ -21,6 +21,23 @@
     "main",
 ]
 
+# Branches with special prefixes that invoke a BB run.
+BB_TEST_BRANCHES = [
+    "bb-*",
+    "st-*",
+    "prot-*",
+    "refs/pull/*",
+    "preview-1[0-9].*",
+    "jpsn-*",
+]
+
+# A list of all branches that invoke a buildbot run.
+ALL_BB_TEST_BRANCHES = BRANCHES_MAIN + BB_TEST_BRANCHES
+
+STAGING_PROT_TEST_BRANCHES = [
+    "prot-st-*",
+]
+
 # Defines what builders report status to GitHub
 GITHUB_STATUS_BUILDERS = [
     "aarch64-macos-compile-only",

define_masters.py

@@ -3,25 +3,31 @@
 import os
 import shutil
 
+from collections import defaultdict
+
 import yaml
 
 BASE_PATH = "autogen/"
 config = {"private": {}}
-exec(open("master-private.cfg").read(), config, {})
+with open("master-private.cfg", "r") as file:
+    exec(file.read(), config, {})
+
+master_variables = config["private"]["master-variables"]
 
 with open("os_info.yaml", encoding="utf-8") as file:
     OS_INFO = yaml.safe_load(file)
 
-platforms = {}
+platforms = defaultdict(dict)
 
 for os_name in OS_INFO:
     if "install_only" in OS_INFO[os_name] and OS_INFO[os_name]["install_only"]:
         continue
     for arch in OS_INFO[os_name]["arch"]:
-        builder_name = arch + "-" + os_name
-        if arch not in platforms:
-            platforms[arch] = []
-        platforms[arch].append(builder_name)
+        builder_name = f"{arch}-{os_name}"
+        platforms[arch][os_name] = {
+            'image_tag': OS_INFO[os_name]['image_tag'],
+            'tags': OS_INFO[os_name]['tags'] if 'tags' in OS_INFO[os_name] else []
+        }
 
 # Clear old configurations
 if os.path.exists(BASE_PATH):
@@ -33,32 +39,25 @@
     # create multiple masters
     # "max_builds" is defined is master-private.py
     num_masters = (
-        int(len(platforms[arch]) / config["private"]["master-variables"]["max_builds"])
-        + 1
+        int(len(platforms[arch]) / master_variables["max_builds"]) + 1
     )
 
     for master_id in range(num_masters):
-        dir_path = BASE_PATH + arch + "-master-" + str(master_id)
+        dir_path = f'{BASE_PATH}{arch}-master-{master_id}'
         os.makedirs(dir_path)
 
-        master_config = {}
-        master_config["builders"] = platforms[arch]
-        master_config["workers"] = config["private"]["master-variables"]["workers"][
-            arch
-        ]
-        master_config["log_name"] = (
-            "master-docker-" + arch + "-" + str(master_id) + ".log"
-        )
+        master_config = {
+            'builders': {arch: platforms[arch]},
+            'workers': master_variables["workers"][arch],
+            'log_name': f'master-docker-{arch}-{master_id}.log'
+        }
 
-        with open(dir_path + "/master-config.yaml", mode="w", encoding="utf-8") as file:
+        with open(f"{dir_path}/master-config.yaml", mode="w",
+                  encoding="utf-8") as file:
             yaml.dump(master_config, file)
 
         shutil.copyfile("master.cfg", dir_path + "/master.cfg")
         shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg")
+        shutil.copyfile("buildbot.tac", dir_path + "/buildbot.tac")
 
-        buildbot_tac = (
-            open("buildbot.tac", encoding="utf-8").read() % master_config["log_name"]
-        )
-        with open(dir_path + "/buildbot.tac", mode="w", encoding="utf-8") as f:
-            f.write(buildbot_tac)
     print(arch, len(master_config["builders"]))

master-bintars/master.cfg

@@ -1,91 +1,39 @@
 # -*- python -*-
 # ex: set filetype=python:
 
-from buildbot.plugins import *
-from buildbot.process.properties import Property, Properties
-from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand
-from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
-from buildbot.steps.source.github import GitHub
-from buildbot.process.remotecommand import RemoteCommand
-from datetime import timedelta
-from twisted.internet import defer
-
-import docker
 import os
-import sys
 import json
 
-sys.path.insert(0, "/srv/buildbot/master")
-sys.setrecursionlimit(10000)
+from buildbot.plugins import util, worker, steps
 
-from common_factories import *
-from constants import *
-from locks import *
-from schedulers_definition import SCHEDULERS
-from utils import *
 
-FQDN = os.environ["BUILDMASTER_WG_IP"]
+from constants import SAVED_PACKAGE_BRANCHES
+from utils import getSourceTarball, read_template, savePackageIfBranchMatch
+from master_common import base_master_config
 
-# This is the dictionary that the buildmaster pays attention to. We also use
-# a shorter alias to save typing.
-c = BuildmasterConfig = {}
+cfg_dir = os.path.abspath(os.path.dirname(__file__))
+
+# Non autogen master. For now the directory structure is:
+# <srcdir>
+# └── <master-name>
+#     ├── buildbot.tac
+#     └── master.cfg
+#
+# Non autogen masters load from <srcdir> for now.
+base_dir = os.path.abspath(f'{cfg_dir}/../')
 
 # Load the slave, database passwords and 3rd-party tokens from an external private file, so
 # that the rest of the configuration can be public.
 config = {"private": {}}
-exec(open("../master-private.cfg").read(), config, {})
-
-####### BUILDBOT SERVICES
-
-# 'services' is a list of BuildbotService items like reporter targets. The
-# status of each build will be pushed to these targets. buildbot/reporters/*.py
-# has a variety to choose from, like IRC bots.
+with open(os.path.join(base_dir, "master-private.cfg"), "r") as file:
+    exec(file.read(), config, {})
 
 
-c["services"] = []
-context = util.Interpolate("buildbot/%(prop:buildername)s")
-gs = reporters.GitHubStatusPush(
-    token=config["private"]["gh_mdbci"]["access_token"],
-    context=context,
-    startDescription="Build started.",
-    endDescription="Build done.",
-    verbose=True,
-    builders=GITHUB_STATUS_BUILDERS,
-)
-c["services"].append(gs)
-c["secretsProviders"] = [
-    secrets.SecretInAFile(
-        dirname=os.environ["MASTER_CREDENTIALS_DIR"]
-    )
-]
-
-####### PROJECT IDENTITY
-
-# the 'title' string will appear at the top of this buildbot installation's
-# home pages (linked to the 'titleURL').
-c["title"] = os.environ["TITLE"]
-c["titleURL"] = os.environ["TITLE_URL"]
-
-# the 'buildbotURL' string should point to the location where the buildbot's
-# internal web server is visible. This typically uses the port number set in
-# the 'www' entry below, but with an externally-visible host name which the
-# buildbot cannot figure out without some help.
-c["buildbotURL"] = os.environ["BUILDMASTER_URL"]
-
-# 'protocols' contains information about protocols which master will use for
-# communicating with workers. You must define at least 'port' option that workers
-# could connect to your master with this protocol.
-# 'port' must match the value configured into the workers (with their
-# --master option)
-port = int(os.environ["MASTER_NONLATENT_BINTARS_WORKER_PORT"])
-c["protocols"] = {"pb": {"port": port}}
-
-####### DB URL
-
-c["db"] = {
-    # This specifies what database buildbot uses to store its state.
-    "db_url": config["private"]["db_url"]
-}
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = base_master_config(
+    config,
+    master_port=os.environ["MASTER_NONLATENT_BINTARS_WORKER_PORT"])
 
 mtrDbPool = util.EqConnectionPool(
     "MySQLdb",
@@ -95,17 +43,10 @@ mtrDbPool = util.EqConnectionPool(
     config["private"]["db_mtr_db"],
 )
 
-####### Disable net usage reports from being sent to buildbot.net
-c["buildbotNetUsageData"] = None
-
-####### SCHEDULERS
-
-# Configure the Schedulers, which decide how to react to incoming changes.
-c["schedulers"] = SCHEDULERS
-
-####### WORKERS
-
 
+#######
+# WORKERS
+#######
 def mkWorker(name, **kwargs):
     return worker.Worker(name, config["private"]["worker_pass"][name], **kwargs)
 
@@ -182,8 +123,6 @@ def getBintarFactory(
 
     kvm_image_test = kvm_image.replace("vm-centos5", "vm-centos6")
 
-    arch = "x86_64" if "amd64" in name else "i686"
-
     # Step 1: Cleaning build directory
     bin_fact.addStep(
         steps.ShellCommand(
@@ -331,7 +270,6 @@ def getBintarFactory(
 
 
 ####### BUILDERS LIST
-
 c["builders"] = []
 
 builder_definitions = {
@@ -364,15 +302,3 @@ for b in builder_definitions:
             factory=f,
         )
     )
-
-c["logEncoding"] = "utf-8"
-
-c["multiMaster"] = True
-
-c["mq"] = {  # Need to enable multimaster aware mq. Wamp is the only option for now.
-    "type": "wamp",
-    "router_url": os.environ["MQ_ROUTER_URL"],
-    "realm": "realm1",
-    # valid are: none, critical, error, warn, info, debug, trace
-    "wamp_debug_level": "info",
-}