diff --git a/.github/workflows/bbm_deploy.yml b/.github/workflows/bbm_deploy.yml index ede823e1..6702066d 100644 --- a/.github/workflows/bbm_deploy.yml +++ b/.github/workflows/bbm_deploy.yml @@ -14,6 +14,7 @@ on: - "master-**" - "master.cfg" - "os_info.yaml" + - "script_templates/**" - "utils.py" - "validate_master_cfg.sh" pull_request: @@ -28,6 +29,7 @@ on: - "master-**" - "master.cfg" - "os_info.yaml" + - "script_templates/**" - "utils.py" - "validate_master_cfg.sh" diff --git a/.github/workflows/bbw_build_container.yml b/.github/workflows/bbw_build_container.yml index d80b9e5b..b10e1fa1 100644 --- a/.github/workflows/bbw_build_container.yml +++ b/.github/workflows/bbw_build_container.yml @@ -82,10 +82,6 @@ jobs: image: ubuntu:22.04 branch: 10.11 platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x - - dockerfile: debian.Dockerfile - image: ubuntu:23.04 - branch: 10.11 - platforms: linux/amd64, linux/arm64/v8 - dockerfile: debian.Dockerfile image: ubuntu:23.10 branch: 10.11 diff --git a/constants.py b/constants.py index a251baf3..3032b1f6 100644 --- a/constants.py +++ b/constants.py @@ -7,8 +7,6 @@ "10.4", "10.5", "10.6", - "10.8", - "10.10", "10.11", "11.0", "11.1", @@ -23,6 +21,7 @@ # Defines what builders report status to GitHub github_status_builders = [ + "aarch64-macos", "amd64-centos-7", "amd64-centos-7-rpm-autobake", "amd64-debian-10", @@ -50,7 +49,6 @@ builders_galera_mtr = [ "aarch64-debian-12", "amd64-fedora-39", - "amd64-ubuntu-2304", "s390x-ubuntu-2004", "s390x-ubuntu-2204", "ppc64le-ubuntu-2004", @@ -104,9 +102,20 @@ "ppc64le-ubuntu-2004", "ppc64le-ubuntu-2004-debug", "ppc64le-ubuntu-2004-without-server", + "x86-debian-12-fulltest", +] + +supportedPlatforms["10.5"] = supportedPlatforms["10.4"].copy() + +# Add only 10.4 supported platforms +supportedPlatforms["10.4"] += [ + "amd64-kvm-centos-6-bintar", + "amd64-kvm-ubuntu-1210-bintar", + "x86-kvm-centos-6-bintar", + "x86-kvm-ubuntu-1210-bintar", ] -supportedPlatforms["10.5"] = [ +supportedPlatforms["10.5"] += [ "aarch64-centos-stream9", "aarch64-debian-11", "aarch64-fedora-38", @@ -115,6 +124,7 @@ "amd64-centos-stream9", "amd64-debian-11", "amd64-debian-11-msan", + "amd64-debian-11-msan-clang-16", "amd64-debian-12-asan-ubsan", "amd64-debian-12-rocksdb", "amd64-fedora-38", @@ -132,9 +142,18 @@ "s390x-sles-15", "s390x-ubuntu-2004", ] -supportedPlatforms["10.5"] += supportedPlatforms["10.4"] -supportedPlatforms["10.6"] = [ +supportedPlatforms["10.6"] = supportedPlatforms["10.5"].copy() + +# Add only 10.5 supported platforms +supportedPlatforms["10.5"] += [ + "amd64-kvm-centos-6-bintar", + "amd64-kvm-ubuntu-1604-bintar", + "x86-kvm-centos-6-bintar", + "x86-kvm-ubuntu-1604-bintar", +] + +supportedPlatforms["10.6"] += [ "aarch64-ubuntu-2204", "amd64-almalinux-8", "amd64-almalinux-9", @@ -145,7 +164,6 @@ "s390x-ubuntu-2204", "x86-debian-12", ] -supportedPlatforms["10.6"] += supportedPlatforms["10.5"] supportedPlatforms["10.9"] = supportedPlatforms["10.6"].copy() @@ -157,11 +175,9 @@ supportedPlatforms["10.11"] = [ "aarch64-debian-12", "aarch64-debian-sid", - "aarch64-ubuntu-2304", "aarch64-ubuntu-2310", "amd64-debian-12", "amd64-debian-sid", - "amd64-ubuntu-2304", "amd64-ubuntu-2310", "ppc64le-debian-sid", "x86-debian-sid", diff --git a/cross-reference/crossreference/cr/models.py b/cross-reference/crossreference/cr/models.py index e7168a16..af0f55e2 100644 --- a/cross-reference/crossreference/cr/models.py +++ b/cross-reference/crossreference/cr/models.py @@ -145,12 +145,12 @@ def select_test_failures(filters, include_failures=True): ], 'test_name': [ { - 'pattern': '^[a-zA-Z0-9_.-]*$', + 'pattern': '^[/a-zA-Z0-9_.-]*$', 'filter': [('test_name__exact', 'AND')], 'replace': False }, { - 'pattern': '^\*\.[a-zA-Z0-9_.-]*$', + 'pattern': '^\*\.[/a-zA-Z0-9_.-]*$', 'filter': [('test_name__icontains', 'AND')], 'replace': True } diff --git a/docker-compose/.env b/docker-compose/.env index 80f6814b..9f42277b 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -8,3 +8,4 @@ ARTIFACTS_URL="https://ci.mariadb.org" NGINX_ARTIFACTS_VHOST="ci.mariadb.org" ENVIRON="PROD" BRANCH="main" +MASTER_NONLATENT_DOCKERLIBRARY_WORKER="bb-rhel8-docker" diff --git a/docker-compose/.env.dev b/docker-compose/.env.dev index 5d66d880..d98abcff 100644 --- a/docker-compose/.env.dev +++ b/docker-compose/.env.dev @@ -9,3 +9,4 @@ ARTIFACTS_URL="https://ci.dev.mariadb.org" NGINX_ARTIFACTS_VHOST="ci.dev.mariadb.org" ENVIRON="DEV" BRANCH="dev" +MASTER_NONLATENT_DOCKERLIBRARY_WORKER="bb-rhel9-docker" diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 4635fd04..d4b01593 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -413,7 +413,7 @@ services: - MASTER_PACKAGES_DIR - MQ_ROUTER_URL - BUILDMASTER_URL - - PORT=10004 + - PORT=10007 hostname: master-docker-nonstandard-2 volumes: - ./logs:/var/log/buildbot @@ -498,6 +498,39 @@ services: depends_on: - mariadb - crossbar + master-bintars: + image: quay.io/mariadb-foundation/bb-master:master + restart: unless-stopped + container_name: master-bintars + environment: + - GALERA_PACKAGES_DIR + - ENVIRON + - TITLE + - BRANCH + - ARTIFACTS_URL + - TITLE_URL + - BUILDMASTER_WG_IP + - NGINX_ARTIFACTS_VHOST + - MASTER_PACKAGES_DIR + - MQ_ROUTER_URL + - BUILDMASTER_URL + - PORT=10004 + hostname: master-bintars + volumes: + - ./logs:/var/log/buildbot + - ./buildbot/:/srv/buildbot/master + entrypoint: + - /bin/bash + - -c + - "/srv/buildbot/master/docker-compose/start.sh master-bintars" + networks: + net_front: + net_back: + ports: + - "100.64.101.1:10008:10008" + depends_on: + - mariadb + - crossbar networks: net_front: diff --git a/docker-compose/generate-config.py b/docker-compose/generate-config.py index 45b72fbd..dbeb4ace 100755 --- a/docker-compose/generate-config.py +++ b/docker-compose/generate-config.py @@ -20,6 +20,7 @@ "master-galera", "master-protected-branches", "master-docker-nonstandard-2", + "master-bintars", ] START_TEMPLATE = """ diff --git a/master-bintars/buildbot.tac b/master-bintars/buildbot.tac new file mode 100644 index 00000000..40027360 --- /dev/null +++ b/master-bintars/buildbot.tac @@ -0,0 +1,33 @@ +import os + +from twisted.application import service +from buildbot.master import BuildMaster + +basedir = '.' +log_basedir = '/var/log/buildbot/' + +rotateLength = 20000000 +maxRotatedFiles = 30 +configfile = 'master.cfg' + +# Default umask for server +umask = None + +# if this is a relocatable tac file, get the directory containing the TAC +if basedir == '.': + import os + basedir = os.path.abspath(os.path.dirname(__file__)) + +# note: this line is matched against to check that this is a buildmaster +# directory; do not edit it. +application = service.Application('buildmaster') +from twisted.python.logfile import LogFile +from twisted.python.log import ILogObserver, FileLogObserver +logfile = LogFile.fromFullPath(os.path.join(log_basedir, "master-bintars.log"), rotateLength=rotateLength, + maxRotatedFiles=maxRotatedFiles) +application.setComponent(ILogObserver, FileLogObserver(logfile).emit) + +m = BuildMaster(basedir, configfile, umask) +m.setServiceParent(application) +m.log_rotation.rotateLength = rotateLength +m.log_rotation.maxRotatedFiles = maxRotatedFiles diff --git a/master-bintars/master.cfg b/master-bintars/master.cfg new file mode 100644 index 00000000..aff39a12 --- /dev/null +++ b/master-bintars/master.cfg @@ -0,0 +1,376 @@ +# -*- python -*- +# ex: set filetype=python: + +from buildbot.plugins import * +from buildbot.process.properties import Property, Properties +from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand +from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver +from buildbot.steps.source.github import GitHub +from buildbot.process.remotecommand import RemoteCommand +from datetime import timedelta +from twisted.internet import defer + +import docker +import os +import sys + +sys.path.insert(0, "/srv/buildbot/master") +sys.setrecursionlimit(10000) + +from common_factories import * +from constants import * +from locks import * +from schedulers_definition import * +from utils import * + +FQDN = os.getenv("BUILDMASTER_WG_IP", default="100.64.100.1") + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +# Load the slave, database passwords and 3rd-party tokens from an external private file, so +# that the rest of the configuration can be public. +config = {"private": {}} +exec(open("../master-private.cfg").read(), config, {}) + +####### BUILDBOT SERVICES + +# 'services' is a list of BuildbotService items like reporter targets. The +# status of each build will be pushed to these targets. buildbot/reporters/*.py +# has a variety to choose from, like IRC bots. + + +c["services"] = [] +context = util.Interpolate("buildbot/%(prop:buildername)s") +gs = reporters.GitHubStatusPush( + token=config["private"]["gh_mdbci"]["access_token"], + context=context, + startDescription="Build started.", + endDescription="Build done.", + verbose=True, + builders=github_status_builders, +) +c["services"].append(gs) + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot installation's +# home pages (linked to the 'titleURL'). +c["title"] = os.getenv("TITLE", default="MariaDB CI") +c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server") + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server is visible. This typically uses the port number set in +# the 'www' entry below, but with an externally-visible host name which the +# buildbot cannot figure out without some help. +c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/") + +# 'protocols' contains information about protocols which master will use for +# communicating with workers. You must define at least 'port' option that workers +# could connect to your master with this protocol. +# 'port' must match the value configured into the workers (with their +# --master option) +port = int(os.getenv("PORT", default="9992")) +c["protocols"] = {"pb": {"port": port}} + +####### DB URL + +c["db"] = { + # This specifies what database buildbot uses to store its state. + "db_url": config["private"]["db_url"] +} + +mtrDbPool = util.EqConnectionPool( + "MySQLdb", + config["private"]["db_host"], + config["private"]["db_user"], + config["private"]["db_password"], + config["private"]["db_mtr_db"], +) + +####### Disable net usage reports from being sent to buildbot.net +c["buildbotNetUsageData"] = None + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. +c["schedulers"] = getSchedulers() + +####### WORKERS + + +def mkWorker(name, **kwargs): + return worker.Worker(name, config["private"]["worker_pass"][name], **kwargs) + + +# The 'workers' list defines the set of recognized workers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. +c["workers"] = [] + +c["workers"].append(mkWorker("monty-bbw1-x64", max_builds=1, properties={"jobs": 12})) + + +####### FACTORY CODE +def memory_def(): + # TODO Dummy implementation, replace with actual logic to define memory + return "8192" + + +gccWarningPattern = "^(.*?):([0-9]+):(?:[0-9]+:)? [Ww]arning: (.*)$" +kvm_scpopt = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PubkeyAcceptedKeyTypes=ssh-dss -o HostKeyAlgorithms=ssh-dss" + + +def ScpSourceIntoVM(port): + return util.Interpolate( + "= scp -P " + + port + + " " + + kvm_scpopt + + " /mnt/packages/%(prop:tarbuildnum)s_%(prop:mariadb_version)s.tar.gz buildbot@localhost:buildbot/build" + ) + + +def getMTR(**kwargs): + return steps.MTR( + dbpool=mtrDbPool, + warningPattern="MTR's internal check of the test case '.*' failed|Failing test\(s\):.*|\*\*\*Warnings generated in error logs.*|Test warning:.*", + **kwargs + ) + + +def getBintarFactory( + name, + kvm_image, + cpu, + ramdisk=True, + catg="main", + suffix="", + port="10000", +): + bin_fact = util.BuildFactory() + + args = [ + "--port=" + port, + "--memory=" + memory_def(), + "--user=buildbot", + "--cpu=" + cpu, + "--smp=8", + "--logfile=kernel_" + port + ".log", + ] + if ramdisk: + args.append("--work-image=/dev/shm/vm-tmp-" + port + ".qcow2") + + kvm_image_test = kvm_image.replace("vm-centos5", "vm-centos6") + + arch = "x86_64" if "amd64" in name else "i686" + + # Step 1: Cleaning build directory + bin_fact.addStep( + steps.ShellCommand( + description=["cleaning", "build", "dir"], + command=["sh", "-c", "rm -Rf ../build/*"], + ) + ) + + # Step 2: Downloading source tarball (Adapt this to your source download mechanism) + bin_fact.addStep(downloadSourceTarball()) + + # Custom Compile Step + # Step 5: Compile (Simplified example, adapt as needed) + bin_fact.addStep( + steps.Compile( + description=["making", "bintar"], + descriptionDone=["make", "bintar"], + timeout=3600, + logfiles={"kernel": "kernel_" + port + ".log"}, + warningPattern=gccWarningPattern, + env={"TERM": "vt102"}, + command=["runvm"] + + args + + [ + "--base-image=/kvm/vms/" + kvm_image + "-build.qcow2", + "vm-tmp-" + port + ".qcow2", + "rm -Rf buildbot && mkdir -p buildbot/build", + ScpSourceIntoVM(port), + util.Interpolate( + "cd buildbot/build && tar -xzf %(prop:tarbuildnum)s_%(prop:mariadb_version)s.tar.gz --strip-components=1 && rm %(prop:tarbuildnum)s_%(prop:mariadb_version)s.tar.gz" + ), + util.Interpolate(read_template("bintar_compile")), + util.Interpolate( + "= scp -P " + + port + + " " + + kvm_scpopt + + " 'buildbot@localhost:buildbot/build/mariadb-*.tar.gz' ." + ), + "= scp -P " + + port + + " " + + kvm_scpopt + + " buildbot@localhost:buildbot/bindistname.txt .", + ], + ) + ) + + bin_fact.addStep( + steps.Test( + name="providers", + description=["checking", "compression", "providers"], + descriptionDone=["check", "compression", "providers"], + timeout=1200, + env={"TERM": "vt102"}, + command=["runvm"] + + args + + [ + "--logfile=kernel_" + port + ".log", + "vm-tmp-" + port + ".qcow2", + util.Interpolate(read_template("bintar_providers")), + ], + ) + ) + + bin_fact.addStep( + steps.SetPropertyFromCommand( + command=["cat", "bindistname.txt"], + property="bindistname", + ) + ) + bin_fact.addStep( + steps.ShellCommand( + name="save_packages", + timeout=7200, + haltOnFailure=True, + command=util.Interpolate( + "mkdir -p " + + "/packages/" + + "%(prop:tarbuildnum)s" + + "/" + + "%(prop:buildername)s" + + " && sha256sum %(prop:bindistname)s >> sha256sums.txt && cp " + + "%(prop:bindistname)s sha256sums.txt" + + " /packages/" + + "%(prop:tarbuildnum)s" + + "/" + + "%(prop:buildername)s" + + "/" + + " && sync /packages/" + + "%(prop:tarbuildnum)s" + ), + doStepIf=savePackage, + ) + ) + + bin_fact.addStep( + getMTR( + name="mtr", + description=["testing", "bintar"], + descriptionDone=["test", "bintar"], + logfiles={"kernel": "kernel_" + port + ".log"}, + mtr_subdir=".", # Where we scp vardir to + env={"TERM": "vt102"}, + command=["runvm"] + + args + + [ + "--base-image=/kvm/vms/" + kvm_image_test + "-install.qcow2", + "vm-tmp-" + port + ".qcow2", + "rm -Rf buildbot && mkdir buildbot", + util.Interpolate( + "= scp -P " + + port + + " " + + kvm_scpopt + + " %(prop:bindistname)s.tar.gz buildbot@localhost:buildbot/" + ), + util.Interpolate(read_template("bintar_mtr")), + util.Interpolate( + "! sudo chmod -R go+rX /usr/local/%(prop:bindistname)s/mysql-test/var/ || :" + ), + util.Interpolate( + "!= scp -rp -P " + + port + + " " + + kvm_scpopt + + " buildbot@localhost:/usr/local/%(prop:bindistname)s/mysql-test/var/ . || :" + ), + ], + ) + ) + + bin_fact.addStep( + getMTR( + name="mtr-rocksdb", + description=["testing", "rocksdb"], + descriptionDone=["test", "rocksdb"], + logfiles={"kernel": "kernel_" + port + ".log"}, + mtr_subdir=".", # Where we scp vardir to + env={"TERM": "vt102"}, + command=["runvm"] + + args + + [ + "vm-tmp-" + port + ".qcow2", + util.Interpolate(read_template("bintar_mtr_rocksdb")), + util.Interpolate( + "! sudo chmod -R go+rX /usr/local/%(prop:bindistname)s/mysql-test/var/ || :" + ), + util.Interpolate( + "!= scp -rp -P " + + port + + " " + + kvm_scpopt + + " buildbot@localhost:/usr/local/%(prop:bindistname)s/mysql-test/var/ . || :" + ), + ], + ) + ) + + return bin_fact + + +####### BUILDERS LIST + +c["builders"] = [] + +builder_definitions = { + "ubuntu-1210": "quantal", + "ubuntu-1604": "trusty", + "centos-6": "centos6", +} + +current_port = 10000 +for b in builder_definitions: + for arch in ["amd64", "i386"]: + if arch == "i386": + builder_name = "x86" + else: + builder_name = arch + + f = getBintarFactory( + name=builder_name + "-kvm-" + b + "-bintar", + kvm_image="vm-" + builder_definitions[b] + "-" + arch, + cpu="host", + ramdisk=False, + port=str(current_port), + ) + current_port += 1 + + c["builders"].append( + util.BuilderConfig( + name=builder_name + "-kvm-" + b + "-bintar", + workernames=["monty-bbw1-x64"], + factory=f, + ) + ) + +c["logEncoding"] = "utf-8" + +c["multiMaster"] = True + +c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. + "type": "wamp", + "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), + "realm": "realm1", + # valid are: none, critical, error, warn, info, debug, trace + "wamp_debug_level": "info", +} diff --git a/master-config.yaml-sample b/master-config.yaml-sample index a73d1e09..8ee90a67 100644 --- a/master-config.yaml-sample +++ b/master-config.yaml-sample @@ -10,7 +10,6 @@ builders: - aarch64-rhel-9 - aarch64-ubuntu-2004 - aarch64-ubuntu-2204 -- aarch64-ubuntu-2304 log_name: master-docker-aarch64-0.log port: 9998 workers: diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index 5c27a819..bcd8b3bf 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -775,7 +775,7 @@ def add_test_steps(factory, test_types): extra_args = "--ps-protocol" elif test_type == "emb-ps": extra_args = "--ps --embedded" - elif test_type == "nm_func_1_2_stress_jp_with_big": + elif test_type == "nm_func_1_2": extra_args = "--suite=funcs_1,funcs_2,stress,jp --big --mysqld=--open-files-limit=0 --mysqld=--log-warnings=1" elif test_type == "nm_engines": extra_args = "--suite=spider,spider/bg,engines/funcs,engines/iuds --big --mysqld=--open-files-limit=0 --mysqld=--log-warnings=1" @@ -852,23 +852,11 @@ f_full_test.addStep( description="dockerfile", ) ) -# get the source tarball and extract it -f_full_test.addStep( - steps.FileDownload( - mastersrc=util.Interpolate( - "/srv/buildbot/packages/" - + "%(prop:tarbuildnum)s" - + "/" - + "%(prop:mariadb_version)s" - + ".tar.gz" - ), - workerdest=util.Interpolate("%(prop:mariadb_version)s" + ".tar.gz"), - ) -) +f_full_test.addStep(downloadSourceTarball()) f_full_test.addStep( steps.ShellCommand( command=util.Interpolate( - "tar -xvzf " + "%(prop:mariadb_version)s" + ".tar.gz --strip-components=1" + "tar -xvzf /mnt/packages/%(prop:tarbuildnum)s_%(prop:mariadb_version)s.tar.gz --strip-components=1" ) ) ) @@ -893,7 +881,7 @@ full_test_configs = [ "nm", "ps", "emb-ps", - "nm_func_1_2_stress_jp_with_big", + "nm_func_1_2", "nm_engines", "view", ] diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 35bb5474..f606cdcf 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -1028,7 +1028,7 @@ def add_test_steps(factory, test_types): extra_args = "--ps-protocol" elif test_type == "emb-ps": extra_args = "--ps --embedded" - elif test_type == "nm_func_1_2_stress_jp_with_big": + elif test_type == "nm_func_1_2": extra_args = "--suite=funcs_1,funcs_2,stress,jp --big --mysqld=--open-files-limit=0 --mysqld=--log-warnings=1" elif test_type == "nm_engines": extra_args = "--suite=spider,spider/bg,engines/funcs,engines/iuds --big --mysqld=--open-files-limit=0 --mysqld=--log-warnings=1" @@ -1105,23 +1105,11 @@ f_full_test.addStep( description="dockerfile", ) ) -# get the source tarball and extract it -f_full_test.addStep( - steps.FileDownload( - mastersrc=util.Interpolate( - "/srv/buildbot/packages/" - + "%(prop:tarbuildnum)s" - + "/" - + "%(prop:mariadb_version)s" - + ".tar.gz" - ), - workerdest=util.Interpolate("%(prop:mariadb_version)s" + ".tar.gz"), - ) -) +f_full_test.addStep(downloadSourceTarball()) f_full_test.addStep( steps.ShellCommand( command=util.Interpolate( - "tar -xvzf " + "%(prop:mariadb_version)s" + ".tar.gz --strip-components=1" + "tar -xvzf /mnt/packages/%(prop:tarbuildnum)s_%(prop:mariadb_version)s.tar.gz --strip-components=1" ) ) ) @@ -1146,7 +1134,7 @@ full_test_configs = [ "nm", "ps", "emb-ps", - "nm_func_1_2_stress_jp_with_big", + "nm_func_1_2", "nm_engines", "view", ] diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index f6da58c8..50c213dc 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -114,7 +114,7 @@ c["workers"].append(aix_worker) # Docker Library dockerlibrary_worker = mkWorker( - "bb-rhel8-docker", properties={"jobs": 1, "save_packages": True} + os.getenv("MASTER_NONLATENT_DOCKERLIBRARY_WORKER", default="bb-rhel8-docker"), properties={"jobs": 1, "save_packages": True} ) c["workers"].append(dockerlibrary_worker) @@ -149,8 +149,8 @@ c["workers"].append(freebds_worker) ####### FACTORY CODE f_windows_env = { - "TMP": util.Interpolate("C:\\Buildbot\\%(prop:buildername)s\\build\\tmpdir"), - "TEMP": util.Interpolate("C:\\Buildbot\\%(prop:buildername)s\\build\\tmpdir"), + "TMP": util.Interpolate("C:\\Buildbot\\%(prop:buildername)s\\build\\tmpdir"), + "TEMP": util.Interpolate("C:\\Buildbot\\%(prop:buildername)s\\build\\tmpdir"), } f_windows_env.update(MTR_ENV) @@ -894,7 +894,9 @@ f_macos.addStep( ), ], timeout=950, - haltOnFailure="true", + haltOnFailure=False, + flunkOnFailure=False, + warnOnFailure=True, parallel=mtrJobsMultiplier, dbpool=mtrDbPool, autoCreateTables=True, @@ -972,7 +974,7 @@ c["builders"] = [] c["builders"].append( util.BuilderConfig( name="amd64-rhel8-wordpress", - workernames=["bb-rhel8-docker"], + workernames=[os.getenv("MASTER_NONLATENT_DOCKERLIBRARY_WORKER", default="bb-rhel8-docker")], tags=["RHEL"], collapseRequests=True, nextBuild=nextBuild, @@ -984,7 +986,7 @@ c["builders"].append( c["builders"].append( util.BuilderConfig( name="amd64-rhel8-dockerlibrary", - workernames=["bb-rhel8-docker"], + workernames=[os.getenv("MASTER_NONLATENT_DOCKERLIBRARY_WORKER", default="bb-rhel8-docker")], tags=["RHEL"], collapseRequests=True, nextBuild=nextBuild, diff --git a/master-private.cfg-sample b/master-private.cfg-sample index d17f9222..ed925f9f 100644 --- a/master-private.cfg-sample +++ b/master-private.cfg-sample @@ -52,6 +52,7 @@ private["worker_pass"]= { "bbw1-windows":"1234", "bbw2-windows":"1234", "bb-rhel8-docker":"1234", + "monty-bbw1-x64": "1234", "s390x-rhel8":"1234", "libvirt": "1234", } diff --git a/os_info.yaml b/os_info.yaml index 83bc2e6a..2a83315e 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -140,13 +140,6 @@ ubuntu-2204: - ppc64le - s390x type: deb -ubuntu-2304: - version_name: lunar - arch: - - amd64 - - aarch64 - type: deb - has_install: False ubuntu-2310: version_name: mantic arch: diff --git a/script_templates/bintar_compile.sh b/script_templates/bintar_compile.sh new file mode 100644 index 00000000..ef3c3732 --- /dev/null +++ b/script_templates/bintar_compile.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# This is a template file for a shell script that is used to compile a binary tarball. +# The script CANNOT BE EXECUTED DIRECTLY, it is a template for the buildbot to use. + +set -o errexit +set -o pipefail +set -o posix + +err() { + echo >&2 "ERROR: $*" + exit 1 +} + +#distdirname="%(prop:distdirname)s" +branch="%(prop:branch)s" +cmake="%(prop:additional_args)s" +extra="" + +# TODO handle galera part +#case $distdirname in +# *-10.[23].*) +# # centos5 sort doesn't support -V +# latest=$(wget --no-check-certificate https://hasky.askmonty.org/galera/ -O -|grep -o galera-25'\.[0-9]\.[0-9][0-9]' |sort|tail -1) +# ;; +# *) +# latest=$(wget --no-check-certificate https://hasky.askmonty.org/galera/ -O -|grep -o galera-26'\.[0-9]\+\.[0-9]\+' |sort -V|tail -1) +# ;; +#esac +#if [ -n "$latest" ] ; then +# wget --no-check-certificate https://hasky.askmonty.org/galera/$latest/bintar/$latest""" + suffix + "-" + arch + """.tar.gz +# if [ -r galera-*.tar.gz ] ; then +# tar xf galera-*.tar.gz +# p=$(echo $HOME/galera-*/usr) +# extra="-DEXTRA_FILES=$p/lib/libgalera_smm.so=lib;$p/lib/galera/libgalera_smm.so=lib/galera" +# for f in $p/bin/*; do +# extra="$extra;$f=bin" +# done +# fi +#fi +cd buildbot/build || err "cd buildbot/build" +mkdir mkbin +cd mkbin +echo "$PATH" +echo "$SHELL" + +if [[ -d "$HOME"/local/lib ]]; then + export CMAKE_LIBRARY_PATH="$HOME/local/lib" +fi +if [[ -d "$HOME"/local/share/pkgconfig ]]; then + export PKG_CONFIG_PATH="$HOME/local/share/pkgconfig" +fi +case $branch in + preview-*) + EV=$branch + EV=-DEXTRA_VERSION=-${EV#preview-*.*-} + ;; +esac +export JAVA_HOME=/usr/lib/jvm/java +cmake -DBUILD_CONFIG=mysql_release -DWITH_READLINE=1 "$EV $extra $cmake" .. +make -j"$(nproc)" package VERBOSE=1 +basename mariadb-*.tar.gz .tar.gz >../../bindistname.txt +mv "$(cat ../../bindistname.txt).tar.gz" ../ diff --git a/script_templates/bintar_mtr.sh b/script_templates/bintar_mtr.sh new file mode 100644 index 00000000..b324d2eb --- /dev/null +++ b/script_templates/bintar_mtr.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -ex + +err() { + echo >&2 "ERROR: $*" + exit 1 +} + +bindistname="%(prop:bindistname)s" + +df -kT +cd buildbot +sudo rm -Rf /usr/local/"$bindistname" +sudo tar zxf "$bindistname".tar.gz -C /usr/local/ +cd /usr/local/"$bindistname" +sudo /usr/sbin/useradd mysql +sudo sudo chown -R mysql . +sudo chgrp -R mysql . +sudo bin/mysql --version +sudo scripts/mysql_install_db --no-defaults --user=mysql +sudo chown -R root . +sudo chown -R mysql data + +sudo chown -R mysql mysql-test || + sudo chown -R mysql mariadb-test + +echo "Test for MDEV-18563, MDEV-18526" + +set +e +for p in /bin /sbin /usr/bin /usr/sbin /usr/local/bin /usr/local/sbin /usr/local/"$bindistname"/scripts; do + if [[ -x "$p"/mysql_install_db ]]; then + sudo "$p"/mysql_install_db --no-defaults --user=mysql --plugin-maturity=unknown + else + echo "$p/mysql_install_db does not exist" + fi +done +sudo scripts/mysql_install_db --no-defaults --user=mysql --plugin-maturity=unknown + +if ldd lib/plugin/ha_connect.so | grep libodbc.so.1 | grep 'not found'; then + if [ -e /usr/lib64/libodbc.so.2 ]; then + sudo ln -s /usr/lib64/libodbc.so.2 /usr/lib64/libodbc.so.1 + elif [ -e /usr/lib/libodbc.so.2 ]; then + sudo ln -s /usr/lib/libodbc.so.2 /usr/lib/libodbc.so.1 + fi +fi + +cd mysql-test || cd mariadb-test +if test -f suite/plugins/pam/pam_mariadb_mtr.so; then + for p in /lib*/security /lib*/*/security; do + test -f "$p"/pam_unix.so && sudo cp -v suite/plugins/pam/pam_mariadb_mtr.so "$p"/ + done + sudo cp -v suite/plugins/pam/mariadb_mtr /etc/pam.d/ +fi +perl mysql-test-run.pl --verbose-restart --force --parallel=4 \ + --retry=3 --vardir="$(readlink -f /dev/shm/var)" \ + --max-save-core=0 --max-save-datadir=1 diff --git a/script_templates/bintar_mtr_rocksdb.sh b/script_templates/bintar_mtr_rocksdb.sh new file mode 100644 index 00000000..58b093bc --- /dev/null +++ b/script_templates/bintar_mtr_rocksdb.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +bindistname="%(prop:bindistname)s" + +cd /usr/local/"$bindistname"/mysql-test || + cd /usr/local/"$bindistname"/mariadb-test || + err /usr/local/"$bindistname"/mariadb-test +if ! sudo su -s /bin/sh -c "perl mysql-test-run.pl --mem rocksdb.1st 2>&1" mysql | grep -E 'RocksDB is not compiled|Could not find'; then + sudo su -s /bin/sh -c "perl mysql-test-run.pl --suite=rocksdb* --skip-test=rocksdb_hotbackup* --verbose-restart --force --parallel=4 --retry=3 --mem --max-save-core=0 --max-save-datadir=1" mysql +fi diff --git a/script_templates/bintar_providers.sh b/script_templates/bintar_providers.sh new file mode 100644 index 00000000..bb500a3c --- /dev/null +++ b/script_templates/bintar_providers.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -ex + +err() { + echo >&2 "ERROR: $*" + exit 1 +} + +buildername="%(prop:buildername)s" + +cd buildbot/build || err "cd buildbot/build" +# shellcheck disable=SC2010 +num_providers_expected=$(ls plugin/* | grep -cE "^provider") +if ((num_providers_expected == 0)); then + echo "No expected providers found, skipping the test" + exit 0 +fi +if [[ $buildername == "kvm-rpm-rhel8-ppc64le" ]]; then + echo "Test warning: This builder cannot build all providers due to MDEV-28738" + num_providers_expected=2 +fi + +find mkbin/plugin/ -name "provider*.so" +num_providers_built=$(find mkbin/plugin/ -name "provider*.so" | wc -l) + +if ((num_providers_built != num_providers_expected)); then + err "found $num_providers_built provider libraries, expected $num_providers_expected" +fi diff --git a/utils.py b/utils.py index edcdad28..ae5a6fd0 100644 --- a/utils.py +++ b/utils.py @@ -681,3 +681,8 @@ def getMetric(hostname, metric): assert elapsed_from_last < 80 return last_value + + +def read_template(template_name): + with open(f"/srv/buildbot/master/script_templates/{template_name}.sh", "r") as f: + return f.read() diff --git a/validate_master_cfg.sh b/validate_master_cfg.sh index ce1e676c..2c0b2ede 100755 --- a/validate_master_cfg.sh +++ b/validate_master_cfg.sh @@ -29,7 +29,7 @@ $RUNC run -i -v "$(pwd):/srv/buildbot/master" -w /srv/buildbot/master quay.io/ma echo -e "done\n" # not checking libvirt config file (//TEMP we need to find a solution # to not check ssh connection) -for dir in master-docker-nonstandard master-docker-nonstandard-2 master-galera master-nonlatent master-web master-protected-branches autogen/*; do +for dir in master-bintars master-docker-nonstandard master-docker-nonstandard-2 master-galera master-nonlatent master-web master-protected-branches autogen/*; do echo "Checking $dir/master.cfg" $RUNC run -i -v "$(pwd):/srv/buildbot/master" -w /srv/buildbot/master quay.io/mariadb-foundation/bb-master:master bash -c "cd $dir && buildbot checkconfig master.cfg" echo -e "done\n"