From 3dd50bf856adc8aabcf08f4b90cf82001850614e Mon Sep 17 00:00:00 2001 From: Faustin Lammler Date: Thu, 20 Jun 2024 09:41:05 +0200 Subject: [PATCH] Jepsen tests implementation Based on https://github.com/vlad-lesin/jepsen-mysql/tree/main/mariadb-docker --- master-docker-nonstandard-2/master.cfg | 88 ++++++++++++++++++++++++++ master-protected-branches/master.cfg | 13 ++++ master-web/master.cfg | 36 ++++++----- utils.py | 5 +- 4 files changed, 125 insertions(+), 17 deletions(-) diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index b3b3ed97..c8aeac3e 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -175,6 +175,15 @@ addWorker( save_packages=False, ) +addWorker( + "apexis-bbw", + 3, + "-ubuntu-2204-jepsen-mysql", + "quay.io/mariadb-foundation/bb-worker:ubuntu22.04-jepsen-mysql", + jobs=5, + save_packages=False, +) + ####### FACTORY CODE f_quick_build = getQuickBuildFactory("nm", mtrDbPool) @@ -726,6 +735,73 @@ f_big_test.addStep( ) ) +## f_jepsen_mysql +f_jepsen_mysql = util.BuildFactory() +f_jepsen_mysql.addStep(printEnv()) +f_jepsen_mysql.addStep( + steps.ShellCommand( + name="create html log file", + command=[ + "bash", + "-c", + util.Interpolate( + getHTMLLogString(), + jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"), + ), + ], + ) +) +# get the source tarball and extract it +f_jepsen_mysql.addStep( + steps.FileDownload( + mastersrc=util.Interpolate( + "/srv/buildbot/packages/" + + "%(prop:tarbuildnum)s" + + "/" + + "%(prop:mariadb_version)s" + + ".tar.gz" + ), + workerdest=util.Interpolate("%(prop:mariadb_version)s" + ".tar.gz"), + ) +) +f_jepsen_mysql.addStep( + steps.ShellCommand( + command=util.Interpolate( + "tar -xzf " + "%(prop:mariadb_version)s" + ".tar.gz --strip-components=1" + ) + ) +) +# build steps +f_jepsen_mysql.addStep( + steps.Compile( + command=[ + "sh", + "-c", + util.Interpolate( + "export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH && cmake . -DPLUGIN_{ARCHIVE,TOKUDB,MROONGA,OQGRAPH,ROCKSDB,CONNECT,SPIDER,SPHINX,COLUMNSTORE,PERFSCHEMA,XPAND}=NO -DWITH_SAFEMALLOC=OFF -DCMAKE_BUILD_TYPE=RelWithDebinfo -DCMAKE_INSTALL_PREFIX=$PREFIX && make -j%(kw:jobs)s install", + jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"), + ), + ], + env={"CCACHE_DIR": "/mnt/ccache"}, + ) +) +f_jepsen_mysql.addStep( + steps.ShellCommand( + name="test", + command=[ + "bash", + "-c", + read_template("jepsen_mysql"), + ], + workdir="/home/buildbot/jepsen-mysql", + ) +) +f_jepsen_mysql.addStep( + steps.ShellCommand( + name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True + ) +) + # Define a function to add test steps to a factory def add_test_steps(factory, test_types): @@ -1300,6 +1376,18 @@ c["builders"].append( ) ) +c["builders"].append( + util.BuilderConfig( + name="amd64-ubuntu-2204-jepsen-mysql", + workernames=workers["x64-bbw-docker-ubuntu-2204-jepsen-mysql"], + tags=["Ubuntu", "jepsen-mysql"], + collapseRequests=True, + nextBuild=nextBuild, + canStartBuild=canStartBuild, + factory=f_jepsen_mysql, + ) +) + c["logEncoding"] = "utf-8" c["multiMaster"] = True diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 340e3ca1..6e0dd42a 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -377,6 +377,19 @@ f_tarball.addStep( }, ) ) +f_tarball.addStep( + steps.Trigger( + schedulerNames=["s_jepsen"], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + "tarbuildnum": Property("buildnumber"), + "mariadb_version": Property("mariadb_version"), + "master_branch": Property("master_branch"), + }, + doStepIf=lambda step: isJepsenBranch(step), + ) +) f_tarball.addStep( steps.SetPropertyFromCommand( command=util.Interpolate("echo " + "prot-" + "%(prop:master_branch)s"), diff --git a/master-web/master.cfg b/master-web/master.cfg index 35f4b4ce..3e5a691a 100644 --- a/master-web/master.cfg +++ b/master-web/master.cfg @@ -59,28 +59,31 @@ c["www"]["auth"] = util.GitHubAuth( ) # Sponsor plugin -sponsorapp = Flask('Sponsors', root_path=os.path.dirname(__file__)) +sponsorapp = Flask("Sponsors", root_path=os.path.dirname(__file__)) # this allows to work on the template without having to restart Buildbot -sponsorapp.config['TEMPLATES_AUTO_RELOAD'] = True +sponsorapp.config["TEMPLATES_AUTO_RELOAD"] = True @sponsorapp.route("/index.html") def main(): # sponsor.html is a template inside the template directory - return render_template('sponsor.html') - -c['www']['plugins']['wsgi_dashboards'] = [ # This is a list of dashboards, you can create several - { - 'name': 'sponsor', # as used in URLs - 'caption': 'Sponsors', # Title displayed in the UI' - 'app': sponsorapp, - # priority of the dashboard in the left menu (lower is higher in the - # menu) - 'order': 20, - # available icon list can be found at http://fontawesome.io/icons/ - 'icon': 'share-alt-square' - } -] + return render_template("sponsor.html") + + +c["www"]["plugins"]["wsgi_dashboards"] = ( + [ # This is a list of dashboards, you can create several + { + "name": "sponsor", # as used in URLs + "caption": "Sponsors", # Title displayed in the UI' + "app": sponsorapp, + # priority of the dashboard in the left menu (lower is higher in the + # menu) + "order": 20, + # available icon list can be found at http://fontawesome.io/icons/ + "icon": "share-alt-square", + } + ] +) ####### DB URL c["db"] = {"db_url": config["private"]["db_url"]} @@ -128,6 +131,7 @@ def upstream_branch_fn(branch): or fnmatch.fnmatch(branch, "prot-*") or fnmatch.fnmatch(branch, "refs/pull/*") or fnmatch.fnmatch(branch, "preview-1[0-9].*") + or fnmatch.fnmatch(branch, "jpsn-*") ) diff --git a/utils.py b/utils.py index f07fe06d..7532c5b6 100644 --- a/utils.py +++ b/utils.py @@ -273,7 +273,6 @@ def nextBuild(bldr, requests): return requests[0] - def canStartBuild(builder, wfb, request): worker = wfb.worker if not "s390x" in worker.name: @@ -704,3 +703,7 @@ def getMetric(hostname, metric): def read_template(template_name): with open(f"/srv/buildbot/master/script_templates/{template_name}.sh", "r") as f: return f.read() + + +def isJepsenBranch(step): + return step.getProperty("branch").startswith("jpsn")