From 685cda5413740984b99064f0157aa90fa3ebddb1 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Dec 2023 23:02:10 +0100 Subject: [PATCH 1/5] Migrate -debbuilders to DSL gazebo_libs Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 36 +++++++++++++++++++++-- jenkins-scripts/dsl/gz-collections.yaml | 6 ++++ jenkins-scripts/dsl/ignition.dsl | 39 ------------------------- 3 files changed, 40 insertions(+), 41 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 9adcad88f..7b4e68a3d 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -281,6 +281,33 @@ String generate_brew_install(src_name, lib_name, arch) return job_name } +def generate_debbuilder_job(src_name, lib_name, pkg_config) +{ + def pre_setup_script = pkg_config.pre_setup_script_hook?.get(lib_name)?.join('\n') + def extra_cmd = pre_setup_script ?: "" + + def build_pkg_job = job("${src_name}-debbuilder") + OSRFLinuxBuildPkg.create(build_pkg_job) + build_pkg_job.with + { + concurrentBuild(true) + + throttleConcurrentBuilds { + maxPerNode(1) + maxTotal(8) + } + + steps { + shell("""\ + #!/bin/bash -xe + ${GLOBAL_SHELL_CMD} + ${extra_cmd} + /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash + """.stripIndent()) + } + } +} + def ciconf_per_lib_index = [:].withDefault { [:] } def pkgconf_per_src_index = [:].withDefault { [:] } generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index) @@ -444,7 +471,12 @@ pkgconf_per_src_index.each { pkg_src, pkg_src_configs -> if (pkg_config.exclude?.contains(canonical_lib_name)) return def pkg_system = pkg_config.system - // -------------------------------------------------------------- + // - DEBBUILD jobs ------------------------------------------------- + generate_debbuilder_job(pkg_src, + canonical_lib_name, + pkg_config + ) + // - SOURCE jobs --------------------------------------------------- def gz_source_job = job("${pkg_src}-source") OSRFSourceCreation.create(gz_source_job, [ PACKAGE: pkg_src, @@ -452,7 +484,7 @@ pkgconf_per_src_index.each { pkg_src, pkg_src_configs -> OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, 'repository_uploader_packages', '_releasepy') - // -------------------------------------------------------------- + // - CI-INSTALL jobs ------------------------------------------------ pkg_system.arch.each { arch -> def linux_install_job_name = generate_linux_install( pkg_src, diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index 61e845b23..d548bee9b 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -588,6 +588,9 @@ packaging_configs: version: focal arch: - amd64 + pre_setup_script_hook: + gz-physics: + - "export MAKE_JOBS=1" - name: jammy system: so: linux @@ -595,5 +598,8 @@ packaging_configs: version: jammy arch: - amd64 + pre_setup_script_hook: + gz-physics: + - "export MAKE_JOBS=1" exclude: - __upcoming__ diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 643e6043a..1cf0d26fd 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -391,45 +391,6 @@ gz_software.each { gz_sw -> } // end of arch } // end of gz_software -// -------------------------------------------------------------- -// DEBBUILD: linux package builder -all_debbuilders().each { debbuilder_name -> - extra_str = "" - if (debbuilder_name.contains("gazebo") || debbuilder_name == "transport7") - extra_str="export NEED_C17_COMPILER=true" - - // Gazebo physics consumes huge amount of memory making arm node to FAIL - // Force here to use one compilation thread - if (debbuilder_name.contains("-physics")) - extra_str += '\nexport MAKE_JOBS=1' - - def build_pkg_job = job("${debbuilder_name}-debbuilder") - OSRFLinuxBuildPkg.create(build_pkg_job) - build_pkg_job.with - { - - concurrentBuild(true) - - throttleConcurrentBuilds { - maxPerNode(1) - maxTotal(8) - } - - steps { - shell("""\ - #!/bin/bash -xe - - ${GLOBAL_SHELL_CMD} - ${extra_str} - /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash - """.stripIndent()) - } - } -} - -// -------------------------------------------------------------- -// WINDOWS: CI job - // Main CI workflow gz_software.each { gz_sw -> if (gz_sw == 'sim') From 96cc7635f43be215a0cbc438acfa896eefa59990 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 28 Dec 2023 19:23:33 +0100 Subject: [PATCH 2/5] Use single linux generation for -source and -debbuilders Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 57 +++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 7b4e68a3d..d0c3d4ef4 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -84,7 +84,7 @@ boolean are_cmake_warnings_enabled(lib_name, ci_config) * index[gz-cmake][jammy] -> [ branch: gz-cmake3, collection: garden , * branch: gz-cmake3, collection: harmonic] * - * # pkgconf_per_src_inde index structure: + * # pkgconf_per_src index structure: * pkg_src_name : [ packaging_config_name : [ .lib_name .collection ] ] * * The index main keys are package source names (i.e gz-cmake3 or gz-harmonic and associated them @@ -281,10 +281,10 @@ String generate_brew_install(src_name, lib_name, arch) return job_name } -def generate_debbuilder_job(src_name, lib_name, pkg_config) +def generate_debbuilder_job(src_name, ArrayList pre_setup_script_hooks) { - def pre_setup_script = pkg_config.pre_setup_script_hook?.get(lib_name)?.join('\n') - def extra_cmd = pre_setup_script ?: "" + def extra_cmd = pre_setup_script_hooks.join('\n') + assert extra_cmd instanceof String def build_pkg_job = job("${src_name}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) @@ -463,18 +463,36 @@ ciconf_per_lib_index.each { lib_name, lib_configs -> } // end of lib pkgconf_per_src_index.each { pkg_src, pkg_src_configs -> - pkg_src_configs.each { pkg_src_config -> - def config_name = pkg_src_config.getKey() - def pkg_config = gz_collections_yaml.packaging_configs.find{ it.name == config_name } - // lib_names are the same in all the entries - def canonical_lib_name = pkg_src_config.getValue()[0].lib_name - if (pkg_config.exclude?.contains(canonical_lib_name)) - return - def pkg_system = pkg_config.system + // For each entry in the index perform two steps: + // 1. Generate Linux builders artifacts (-source and -debbuilders) + // 2. Generate all -ci-install- jobs looping in the index entries + + // 1. Generate Linux builders + // All entries are the same for canonical_lib_name, pick the first + def canonical_lib_name = pkg_src_configs.values()[0].lib_name[0] + def linux_ciconfigs = gz_collections_yaml.packaging_configs.findAll{ + it.name in pkg_src_configs.keySet() && + it.system.so == 'linux'} + // Collect the pre_setup_script_hooks defined in all the linux distributions + // Each distribution commands are joined in a signle item. + // Unique + findall { it } should clean up all null values + def pre_setup_script_hooks = linux_ciconfigs.findAll { + it.keySet().contains('pre_setup_script_hook') + }.collect { + it -> it.pre_setup_script_hook.get(canonical_lib_name) + }.flatten().unique().findAll { it } + + def exclusion_list = linux_ciconfigs.findAll { + it.keySet().contains('exclude') + }.collect{ + it -> it.exclude + } + + if (linux_ciconfigs && !exclusion_list.contains(canonical_lib_name)) + { // - DEBBUILD jobs ------------------------------------------------- generate_debbuilder_job(pkg_src, - canonical_lib_name, - pkg_config + pre_setup_script_hooks ) // - SOURCE jobs --------------------------------------------------- def gz_source_job = job("${pkg_src}-source") @@ -484,7 +502,16 @@ pkgconf_per_src_index.each { pkg_src, pkg_src_configs -> OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, 'repository_uploader_packages', '_releasepy') - // - CI-INSTALL jobs ------------------------------------------------ + } + + // 2. Generate all -ci-install jobs + pkg_src_configs.each { pkg_src_config -> + def config_name = pkg_src_config.getKey() + def pkg_config = gz_collections_yaml.packaging_configs.find{ it.name == config_name } + if (pkg_config.exclude?.contains(canonical_lib_name)) + return + def pkg_system = pkg_config.system + // - CI-INSTALL jobs ------------------------------------------------ pkg_system.arch.each { arch -> def linux_install_job_name = generate_linux_install( pkg_src, From 75d57b6091491f70776189b3019752458c2b3db0 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 28 Dec 2023 19:24:31 +0100 Subject: [PATCH 3/5] Force CI: revert Signed-off-by: Jose Luis Rivero --- .github/workflows/ci-dsl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-dsl.yaml b/.github/workflows/ci-dsl.yaml index de37d7f96..fae8ba9b5 100644 --- a/.github/workflows/ci-dsl.yaml +++ b/.github/workflows/ci-dsl.yaml @@ -1,7 +1,7 @@ name: DSL logs and checks on: - pull_request: + push: paths: - 'jenkins-scripts/dsl/**' From f1431985af9d2c7cfeadac6347dd28b6067a4524 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 10 Jan 2024 17:40:42 +0100 Subject: [PATCH 4/5] Add test and use always gz- prefix for -source and -debbuilders Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 22 ++++++++++++++++++++++ release.py | 6 ++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index a7c07a222..dbef90108 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -15,6 +15,16 @@ exec_releasepy_test() gz-foo 1.2.3 token ${test_params}"" } +exec_ignition_releasepy_test() +{ + test_params=${1} + + ./release.py \ + --dry-run \ + --no-sanity-checks \ + ign-foo 1.2.3 token ${test_params}"" +} + expect_job_run() { output="${1}" job="${2}" @@ -81,3 +91,15 @@ expect_job_not_run "${bump_linux_test}" "generic-release-homebrew_pull_request_u expect_job_not_run "${bump_linux_test}" "gz-foo-source" expect_number_of_jobs "${bump_linux_test}" "6" expect_param "${bump_linux_test}" "RELEASE_VERSION=2" + +ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.org/gazebosim/gz-foo") +expect_job_run "${ignition_test}" "gz-foo-source" +expect_job_not_run "${ignition_test}" "ignition-foo-source" +expect_number_of_jobs "${ignition_test}" "1" + +ignition_source_tarball_uri_test=$(exec_ignition_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz") +expect_job_run "${ignition_source_tarball_uri_test}" "gz-foo-debbuilder" +expect_job_run "${ignition_source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater" +expect_job_not_run "${ignition_source_tarball_uri_test}" "gz-foo-source" +expect_number_of_jobs "${ignition_source_tarball_uri_test}" "7" +expect_param "${ignition_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz" diff --git a/release.py b/release.py index 56ca9c5b5..60a8cc87d 100755 --- a/release.py +++ b/release.py @@ -521,6 +521,8 @@ def go(argv): if not args.release_version: args.release_version = 1 + package_alias_force_gz = args.package_alias.replace('ignition-','gz-') + print(f"Downloading releasing info for {args.package}") # Sanity checks and dicover supported distributions before proceed. repo_dir, args.release_repo_branch = download_release_repository(args.package, args.release_repo_branch) @@ -604,7 +606,7 @@ def go(argv): # "-()" do not work even in the web UI directly. Real # string should be: # f"{args.version}-{args.release_version}({l}/{d}::{a})") - call_jenkins_build(f"{args.package_alias}-debbuilder", + call_jenkins_build(f'{package_alias_force_gz}-debbuilder', linux_platform_params, f"{l} {d}/{a}", f"{args.version}-{args.release_version}") @@ -632,7 +634,7 @@ def go(argv): params['SOURCE_REPO_REF'] = tag_repo(args) \ if not args.source_repo_ref else args.source_repo_ref - call_jenkins_build(f"{args.package_alias}-source", + call_jenkins_build(f'{package_alias_force_gz}-source', params, 'Source', args.version) From a33d65e275bf49161a4419b24f203532f600fa20 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 10 Jan 2024 18:14:45 +0100 Subject: [PATCH 5/5] Revert "Force CI: revert" This reverts commit 75d57b6091491f70776189b3019752458c2b3db0. --- .github/workflows/ci-dsl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-dsl.yaml b/.github/workflows/ci-dsl.yaml index fae8ba9b5..de37d7f96 100644 --- a/.github/workflows/ci-dsl.yaml +++ b/.github/workflows/ci-dsl.yaml @@ -1,7 +1,7 @@ name: DSL logs and checks on: - push: + pull_request: paths: - 'jenkins-scripts/dsl/**'