Jepsen tests implementation

Based on https://github.com/vlad-lesin/jepsen-mysql/tree/main/mariadb-docker
fauust · Jun 20, 2024 · 3dd50bf · 3dd50bf
1 parent 28b429d
commit 3dd50bf
Show file tree

Hide file tree

Showing 4 changed files with 125 additions and 17 deletions.
diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg
@@ -175,6 +175,15 @@ addWorker(
     save_packages=False,
 )
 
+addWorker(
+    "apexis-bbw",
+    3,
+    "-ubuntu-2204-jepsen-mysql",
+    "quay.io/mariadb-foundation/bb-worker:ubuntu22.04-jepsen-mysql",
+    jobs=5,
+    save_packages=False,
+)
+
 ####### FACTORY CODE
 
 f_quick_build = getQuickBuildFactory("nm", mtrDbPool)
@@ -726,6 +735,73 @@ f_big_test.addStep(
     )
 )
 
+## f_jepsen_mysql
+f_jepsen_mysql = util.BuildFactory()
+f_jepsen_mysql.addStep(printEnv())
+f_jepsen_mysql.addStep(
+    steps.ShellCommand(
+        name="create html log file",
+        command=[
+            "bash",
+            "-c",
+            util.Interpolate(
+                getHTMLLogString(),
+                jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"),
+            ),
+        ],
+    )
+)
+# get the source tarball and extract it
+f_jepsen_mysql.addStep(
+    steps.FileDownload(
+        mastersrc=util.Interpolate(
+            "/srv/buildbot/packages/"
+            + "%(prop:tarbuildnum)s"
+            + "/"
+            + "%(prop:mariadb_version)s"
+            + ".tar.gz"
+        ),
+        workerdest=util.Interpolate("%(prop:mariadb_version)s" + ".tar.gz"),
+    )
+)
+f_jepsen_mysql.addStep(
+    steps.ShellCommand(
+        command=util.Interpolate(
+            "tar -xzf " + "%(prop:mariadb_version)s" + ".tar.gz --strip-components=1"
+        )
+    )
+)
+# build steps
+f_jepsen_mysql.addStep(
+    steps.Compile(
+        command=[
+            "sh",
+            "-c",
+            util.Interpolate(
+                "export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH && cmake . -DPLUGIN_{ARCHIVE,TOKUDB,MROONGA,OQGRAPH,ROCKSDB,CONNECT,SPIDER,SPHINX,COLUMNSTORE,PERFSCHEMA,XPAND}=NO -DWITH_SAFEMALLOC=OFF -DCMAKE_BUILD_TYPE=RelWithDebinfo -DCMAKE_INSTALL_PREFIX=$PREFIX && make -j%(kw:jobs)s install",
+                jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"),
+            ),
+        ],
+        env={"CCACHE_DIR": "/mnt/ccache"},
+    )
+)
+f_jepsen_mysql.addStep(
+    steps.ShellCommand(
+        name="test",
+        command=[
+            "bash",
+            "-c",
+            read_template("jepsen_mysql"),
+        ],
+        workdir="/home/buildbot/jepsen-mysql",
+    )
+)
+f_jepsen_mysql.addStep(
+    steps.ShellCommand(
+        name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True
+    )
+)
+
 
 # Define a function to add test steps to a factory
 def add_test_steps(factory, test_types):
@@ -1300,6 +1376,18 @@ c["builders"].append(
     )
 )
 
+c["builders"].append(
+    util.BuilderConfig(
+        name="amd64-ubuntu-2204-jepsen-mysql",
+        workernames=workers["x64-bbw-docker-ubuntu-2204-jepsen-mysql"],
+        tags=["Ubuntu", "jepsen-mysql"],
+        collapseRequests=True,
+        nextBuild=nextBuild,
+        canStartBuild=canStartBuild,
+        factory=f_jepsen_mysql,
+    )
+)
+
 c["logEncoding"] = "utf-8"
 
 c["multiMaster"] = True

diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg
@@ -377,6 +377,19 @@ f_tarball.addStep(
         },
     )
 )
+f_tarball.addStep(
+    steps.Trigger(
+        schedulerNames=["s_jepsen"],
+        waitForFinish=False,
+        updateSourceStamp=False,
+        set_properties={
+            "tarbuildnum": Property("buildnumber"),
+            "mariadb_version": Property("mariadb_version"),
+            "master_branch": Property("master_branch"),
+        },
+        doStepIf=lambda step: isJepsenBranch(step),
+    )
+)
 f_tarball.addStep(
     steps.SetPropertyFromCommand(
         command=util.Interpolate("echo " + "prot-" + "%(prop:master_branch)s"),

diff --git a/master-web/master.cfg b/master-web/master.cfg
@@ -59,28 +59,31 @@ c["www"]["auth"] = util.GitHubAuth(
 )
 
 # Sponsor plugin
-sponsorapp = Flask('Sponsors', root_path=os.path.dirname(__file__))
+sponsorapp = Flask("Sponsors", root_path=os.path.dirname(__file__))
 # this allows to work on the template without having to restart Buildbot
-sponsorapp.config['TEMPLATES_AUTO_RELOAD'] = True
+sponsorapp.config["TEMPLATES_AUTO_RELOAD"] = True
 
 
 @sponsorapp.route("/index.html")
 def main():
     # sponsor.html is a template inside the template directory
-    return render_template('sponsor.html')
-
-c['www']['plugins']['wsgi_dashboards'] = [  # This is a list of dashboards, you can create several
-    {
-        'name': 'sponsor',  # as used in URLs
-        'caption': 'Sponsors',  # Title displayed in the UI'
-        'app': sponsorapp,
-        # priority of the dashboard in the left menu (lower is higher in the
-        # menu)
-        'order': 20,
-        # available icon list can be found at http://fontawesome.io/icons/
-        'icon': 'share-alt-square'
-    }
-]
+    return render_template("sponsor.html")
+
+
+c["www"]["plugins"]["wsgi_dashboards"] = (
+    [  # This is a list of dashboards, you can create several
+        {
+            "name": "sponsor",  # as used in URLs
+            "caption": "Sponsors",  # Title displayed in the UI'
+            "app": sponsorapp,
+            # priority of the dashboard in the left menu (lower is higher in the
+            # menu)
+            "order": 20,
+            # available icon list can be found at http://fontawesome.io/icons/
+            "icon": "share-alt-square",
+        }
+    ]
+)
 
 ####### DB URL
 c["db"] = {"db_url": config["private"]["db_url"]}
@@ -128,6 +131,7 @@ def upstream_branch_fn(branch):
         or fnmatch.fnmatch(branch, "prot-*")
         or fnmatch.fnmatch(branch, "refs/pull/*")
         or fnmatch.fnmatch(branch, "preview-1[0-9].*")
+        or fnmatch.fnmatch(branch, "jpsn-*")
     )
 
 

diff --git a/utils.py b/utils.py
@@ -273,7 +273,6 @@ def nextBuild(bldr, requests):
     return requests[0]
 
 
-
 def canStartBuild(builder, wfb, request):
     worker = wfb.worker
     if not "s390x" in worker.name:
@@ -704,3 +703,7 @@ def getMetric(hostname, metric):
 def read_template(template_name):
     with open(f"/srv/buildbot/master/script_templates/{template_name}.sh", "r") as f:
         return f.read()
+
+
+def isJepsenBranch(step):
+    return step.getProperty("branch").startswith("jpsn")