From 85a32b0d93deaef286a633ec965bfed36370a09e Mon Sep 17 00:00:00 2001 From: Constantin Nicolai Date: Wed, 25 Oct 2023 18:54:52 +0200 Subject: [PATCH 1/2] new Jenkinsfile, jenkins lib file --- Jenkinsfile | 435 ++++++++++++++--------------------------- pipeline_config.groovy | 5 + 2 files changed, 155 insertions(+), 285 deletions(-) create mode 100644 pipeline_config.groovy diff --git a/Jenkinsfile b/Jenkinsfile index 6f584c18..7c3af51f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,306 +1,171 @@ pipeline { - agent {label 'main'} - options { - timeout(time: 30, unit: 'MINUTES') - } - tools { - maven 'Maven 3' - } - - environment { - REPO_NAME = sh(returnStdout: true, script: 'basename `git remote get-url origin` .git').trim() - VERSION = sh(returnStdout: true, script: 'mvn --batch-mode org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -Ev "(^\\[|Download\\w+)"').trim() - LATEST_AUTHOR = sh(returnStdout: true, script: 'git show -s --pretty=%an').trim() - LATEST_COMMIT_ID = sh(returnStdout: true, script: 'git describe --tags --long --always').trim() - - MAVEN_GENERAL_OPTIONS = '--batch-mode --update-snapshots' - // START CUSTOM ohsome API - MAVEN_TEST_OPTIONS = '-Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty=--database.db=/opt/data/heidelberg-v1.0-beta.oshdb' - // END CUSTOM ohsome API - SNAPSHOT_BRANCH_REGEX = /(^master$)/ - RELEASE_REGEX = /^([0-9]+(\.[0-9]+)*)(-(RC|beta-|alpha-)[0-9]+)?$/ - RELEASE_DEPLOY = false - SNAPSHOT_DEPLOY = false - } - - stages { - stage ('Build and Test') { - steps { - script { - env.MAVEN_HOME = '/usr/share/maven' - - echo REPO_NAME - echo LATEST_AUTHOR - echo LATEST_COMMIT_ID - - echo env.BRANCH_NAME - echo env.BUILD_NUMBER - echo env.TAG_NAME - - if (!(VERSION ==~ RELEASE_REGEX || VERSION ==~ /.*-SNAPSHOT$/)) { - echo 'Version:' - echo VERSION - error 'The version declaration is invalid. It is neither a release nor a snapshot. Maybe an error occured while fetching the parent pom using maven?' - } - } - script { - withCredentials([string(credentialsId: 'gpg-signing-key-passphrase', variable: 'PASSPHRASE')]) { - sh 'mvn $MAVEN_GENERAL_OPTIONS clean compile javadoc:jar source:jar verify -P jacoco,sign,git -Dmaven.repo.local=.m2 $MAVEN_TEST_OPTIONS -Dgpg.passphrase=$PASSPHRASE' - } - } - } - post { - failure { - rocketSend channel: 'jenkinsohsome', emoji: ':sob:' , message: "*${REPO_NAME}*-build nr. ${env.BUILD_NUMBER} *failed* on Branch - ${env.BRANCH_NAME} (<${env.BUILD_URL}|Open Build in Jenkins>). Latest commit from ${LATEST_AUTHOR}. Review the code!" , rawMessage: true - } - } + agent { label 'worker' } + options { + timeout(time: 30, unit: 'MINUTES') } - - stage ('Reports and Statistics') { - steps { - script { - withSonarQubeEnv('sonarcloud GIScience/ohsome') { - SONAR_CLI_PARAMETER = "" - if (env.CHANGE_ID) { - SONAR_CLI_PARAMETER += " " + - "-Dsonar.pullrequest.key=${env.CHANGE_ID} " + - "-Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} " + - "-Dsonar.pullrequest.base=${env.CHANGE_TARGET}" - } else { - SONAR_CLI_PARAMETER += " " + - "-Dsonar.branch.name=${env.BRANCH_NAME}" - } - sh "mvn $MAVEN_GENERAL_OPTIONS sonar:sonar ${SONAR_CLI_PARAMETER}" - } - report_basedir = "/srv/reports/${REPO_NAME}/${VERSION}_${env.BRANCH_NAME}/${env.BUILD_NUMBER}_${LATEST_COMMIT_ID}" - - // jacoco - report_dir = report_basedir + "/jacoco/" - - jacoco( - execPattern : '**/target/jacoco.exec', - classPattern : '**/target/classes', - sourcePattern : '**/src/main/java', - inclusionPattern : 'org/heigit/**' - ) - sh "mkdir -p ${report_dir} && rm -Rf ${report_dir}* && find . -path '*/target/site/jacoco' -exec cp -R --parents {} ${report_dir} \\; && find ${report_dir} -path '*/target/site/jacoco' | while read line; do echo \$line; neu=\${line/target\\/site\\/jacoco/} ; mv \$line/* \$neu ; done && find ${report_dir} -type d -empty -delete" - - // warnings plugin - sh 'mvn $MAVEN_GENERAL_OPTIONS -V -e compile checkstyle:checkstyle pmd:pmd pmd:cpd spotbugs:spotbugs -Dmaven.repo.local=.m2 $MAVEN_TEST_OPTIONS' - - recordIssues enabledForFailure: true, tools: [mavenConsole(), java(), javaDoc()] - recordIssues enabledForFailure: true, tool: checkStyle() - recordIssues enabledForFailure: true, tool: spotBugs() - recordIssues enabledForFailure: true, tool: cpd(pattern: '**/target/cpd.xml') - recordIssues enabledForFailure: true, tool: pmdParser(pattern: '**/target/pmd.xml') - } - } - post { - failure { - rocketSend channel: 'jenkinsohsome', emoji: ':disappointed:', message: "Reporting of *${REPO_NAME}*-build nr. ${env.BUILD_NUMBER} *failed* on Branch - ${env.BRANCH_NAME} (<${env.BUILD_URL}|Open Build in Jenkins>). Latest commit from ${LATEST_AUTHOR}." , rawMessage: true - } - } + tools { + maven 'Maven 3' } - stage ('Deploy Snapshot') { - when { - expression { - return env.BRANCH_NAME ==~ SNAPSHOT_BRANCH_REGEX && VERSION ==~ /.*-SNAPSHOT$/ - } - } - steps { - script { - withCredentials([ - file(credentialsId: 'nexus-settings', variable: 'settingsFile'), - string(credentialsId: 'gpg-signing-key-passphrase', variable: 'PASSPHRASE') - ]) { - sh 'mvn $MAVEN_GENERAL_OPTIONS clean compile -s $settingsFile javadoc:jar source:jar deploy -P sign,git -Dmaven.repo.local=.m2 $MAVEN_TEST_OPTIONS -Dgpg.passphrase=$PASSPHRASE -DskipTests=true' - } - SNAPSHOT_DEPLOY = true - } - } - post { - failure { - rocketSend channel: 'jenkinsohsome', emoji: ':disappointed:', message: "Deployment of *${REPO_NAME}*-build nr. ${env.BUILD_NUMBER} *failed* on Branch - ${env.BRANCH_NAME} (<${env.BUILD_URL}|Open Build in Jenkins>). Latest commit from ${LATEST_AUTHOR}. Is Artifactory running?" , rawMessage: true - } - } + environment { + // START CUSTOM ohsome API + MAVEN_TEST_OPTIONS = '-Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty=--database.db=/data/heidelberg-v1.0-beta.oshdb' + // END CUSTOM ohsome API + SNAPSHOT_BRANCH_REGEX = /(^master$)/ + RELEASE_REGEX = /^([0-9]+(\.[0-9]+)*)(-(RC|beta-|alpha-)[0-9]+)?$/ + RELEASE_DEPLOY = false + SNAPSHOT_DEPLOY = false } - stage ('Deploy Release') { - when { - expression { - return VERSION ==~ RELEASE_REGEX && env.TAG_NAME ==~ RELEASE_REGEX - } - } - steps { - script { - withCredentials([ - file(credentialsId: 'nexus-settings', variable: 'settingsFile'), - string(credentialsId: 'gpg-signing-key-passphrase', variable: 'PASSPHRASE') - ]) { - sh 'mvn $MAVEN_GENERAL_OPTIONS clean compile -s $settingsFile javadoc:jar source:jar deploy -P sign,git -Dmaven.repo.local=.m2 $MAVEN_TEST_OPTIONS -Dgpg.passphrase=$PASSPHRASE -DskipTests=true' - } - RELEASE_DEPLOY = true - } - withCredentials([ - file(credentialsId: 'ossrh-settings', variable: 'settingsFile'), - string(credentialsId: 'gpg-signing-key-passphrase', variable: 'PASSPHRASE') - ]) { - sh 'mvn $MAVEN_GENERAL_OPTIONS clean compile -s $settingsFile javadoc:jar source:jar deploy -P sign,git,deploy-central -Dmaven.repo.local=.m2 $MAVEN_TEST_OPTIONS -Dgpg.passphrase=$PASSPHRASE -DskipTests=true' - } - } - post { - failure { - rocketSend channel: 'jenkinsohsome', emoji: ':disappointed:', message: "Deployment of *${REPO_NAME}*-build nr. ${env.BUILD_NUMBER} *failed* on Branch - ${env.BRANCH_NAME} (<${env.BUILD_URL}|Open Build in Jenkins>). Latest commit from ${LATEST_AUTHOR}. Is Artifactory running?" , rawMessage: true - } - } - } + stages { + stage('Build and Test') { + steps { + // setting up a few basic env variables like REPO_NAME and LATEST_AUTHOR + setup_basic_env() - stage ('Publish Javadoc') { - when { - anyOf { - equals expected: true, actual: RELEASE_DEPLOY - equals expected: true, actual: SNAPSHOT_DEPLOY + mavenbuild('clean compile javadoc:jar source:jar verify -P jacoco,sign,git') + } + post { + failure { + rocket_buildfail() + rocket_testfail() + } + } } - } - steps { - script { - // load dependencies to artifactory - sh 'mvn $MAVEN_GENERAL_OPTIONS org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version -Dmaven.repo.local=.m2 $MAVEN_TEST_OPTIONS' - javadc_dir = "/srv/javadoc/java/" + REPO_NAME + "/" + VERSION + "/" - echo javadc_dir - - sh 'mvn $MAVEN_GENERAL_OPTIONS clean javadoc:javadoc -Dmaven.repo.local=.m2 $MAVEN_TEST_OPTIONS' - sh "echo ${javadc_dir}" - // make sure jenkins uses bash not dash! - sh "mkdir -p ${javadc_dir} && rm -Rf ${javadc_dir}* && find . -path '*/target/site/apidocs' -exec cp -R --parents {} ${javadc_dir} \\; && find ${javadc_dir} -path '*/target/site/apidocs' | while read line; do echo \$line; neu=\${line/target\\/site\\/apidocs/} ; mv \$line/* \$neu ; done && find ${javadc_dir} -type d -empty -delete" - } - } - post { - failure { - rocketSend channel: 'jenkinsohsome', emoji: ':disappointed:', message: "Deployment of javadoc *${REPO_NAME}*-build nr. ${env.BUILD_NUMBER} *failed* on Branch - ${env.BRANCH_NAME} (<${env.BUILD_URL}|Open Build in Jenkins>). Latest commit from ${LATEST_AUTHOR}." , rawMessage: true + stage('Reports and Statistics') { + steps { + reports_sonar_jacoco() + } } - } - } - // START CUSTOM ohsome API - stage ('Publish API Docs') { - when { - anyOf { - equals expected: true, actual: RELEASE_DEPLOY - equals expected: true, actual: SNAPSHOT_DEPLOY + stage('Deploy Snapshot') { + when { + expression { + return env.BRANCH_NAME ==~ SNAPSHOT_BRANCH_REGEX && VERSION ==~ /.*-SNAPSHOT$/ + } + } + steps { + deploy_snapshot('clean compile javadoc:jar source:jar deploy -P sign,git') + script { + SNAPSHOT_DEPLOY = true + } + } + post { + failure { + rocket_snapshotdeployfail() + } + } } - } - steps { - script { - DOC_RELEASE_REGEX = /^([0-9]+(\.[0-9]+)*)$/ - DOCS_DEPLOYMENT = "development" - API_DOCS_PATH = "development" - if (VERSION ==~ DOC_RELEASE_REGEX) { - DOCS_DEPLOYMENT = "release" - API_DOCS_PATH = sh(returnStdout: true, script: 'cd docs && python3 get_pom_metadata.py | awk \'/^Path:/{ print $2 }\'').trim() - } - publish_dir = "/srv/javadoc/${REPO_NAME}/${API_DOCS_PATH}/" - venv_dir = sh(returnStdout: true, script: 'mktemp -d --suffix .sphinx-docs').trim() + "/venv" - - if (!fileExists("${venv_dir}")) { - sh "python3 -m venv ${venv_dir}" - } + stage('Deploy Release') { + when { + expression { + return VERSION ==~ RELEASE_REGEX && env.TAG_NAME ==~ RELEASE_REGEX + } + } + steps { + deploy_release('clean compile javadoc:jar source:jar deploy -P sign,git') - sh """ - source ${venv_dir}/bin/activate - cd docs - python3 -m pip install -U pip - python3 -m pip install -r requirements.txt - DOCS_DEPLOYMENT=${DOCS_DEPLOYMENT} make clean html - """ - sh "mkdir -p ${publish_dir} && rm -rf ${publish_dir}* && cp -r docs/_build/html/* ${publish_dir}" - sh "rm -rf ${venv_dir}" - } - } - post { - failure { - sh "rm -rf ${venv_dir}" - rocketSend channel: 'jenkinsohsome', emoji: ':unamused:', message: "Deployment of api docs ${REPO_NAME}-build nr. ${env.BUILD_NUMBER} *failed* on Branch - ${env.BRANCH_NAME} (<${env.BUILD_URL}|Open Build in Jenkins>). Latest commit from ${LATEST_AUTHOR}." , rawMessage: true + deploy_release_central('clean compile javadoc:jar source:jar deploy -P sign,git,deploy-central') + script { + RELEASE_DEPLOY = true + } + } + post { + failure { + rocket_releasedeployfail() + } + } } - } - } - // END CUSTOM ohsome API - stage ('Check Dependencies') { - when { - expression { - if ((currentBuild.number > 1) && (env.BRANCH_NAME ==~ SNAPSHOT_BRANCH_REGEX)) { - month_pre = new Date(currentBuild.previousBuild.rawBuild.getStartTimeInMillis())[Calendar.MONTH] - echo month_pre.toString() - month_now = new Date(currentBuild.rawBuild.getStartTimeInMillis())[Calendar.MONTH] - echo month_now.toString() - return month_pre != month_now - } - return false - } - } - steps { - script { - try { - update_notify = sh(returnStdout: true, script: 'mvn $MAVEN_GENERAL_OPTIONS versions:display-dependency-updates | grep -Pzo "(?s)The following dependencies([^\\n]*\\S\\n)*[^\\n]*\\s\\n"').trim() - echo update_notify - rocketSend channel: 'jenkinsohsome', emoji: ':wave:' , message: "Check your dependencies in *${REPO_NAME}*. You might have updates: ${update_notify}" , rawMessage: true - } catch (err) { - echo "No maven dependency upgrades found." - } - } - script { - try { - update_notify = sh(returnStdout: true, script: 'mvn $MAVEN_GENERAL_OPTIONS versions:display-plugin-updates | grep -Pzo "(?s)The following plugin update([^\\n]*\\S\\n)*[^\\n]*\\s\\n"').trim() - echo update_notify - rocketSend channel: 'jenkinsohsome', emoji: ':wave:' , message: "Check your maven plugins in *${REPO_NAME}*. You might have updates: ${update_notify}" , rawMessage: true - } catch (err) { - echo "No maven plugin upgrades found." - } - } - } - } + // START CUSTOM ohsome API + stage('Compile API Docs') { + when { + anyOf { + equals expected: true, actual: RELEASE_DEPLOY + equals expected: true, actual: SNAPSHOT_DEPLOY + } + } + steps { + script { + DOC_RELEASE_REGEX = /^([0-9]+(\.[0-9]+)*)$/ + DOCS_DEPLOYMENT = 'development' + API_DOCS_PATH = 'development' + if (VERSION ==~ DOC_RELEASE_REGEX) { + DOCS_DEPLOYMENT = 'release' + API_DOCS_PATH = sh(returnStdout: true, script: 'cd docs && python3 get_pom_metadata.py | awk \'/^Path:/{ print $2 }\'').trim() + } + + if (!fileExists('venv')) { + sh 'python3 -m venv venv' + } + + sh ''' + . venv/bin/activate + venv/bin/pip install --upgrade pip + venv/bin/pip install -r docs/requirements.txt + cd docs + ../venv/bin/sphinx-build -b html . _build + ''' + sh 'rm -rf venv' + } + } + post { + failure { + rocket_basicsend('Compile of API Docs failed on ${env.BRANCH_NAME}') + } + } + } - stage ('Encourage') { - when { - expression { - if (currentBuild.number > 1) { - date_pre = new Date(currentBuild.previousBuild.rawBuild.getStartTimeInMillis()).clearTime() - echo date_pre.format( 'yyyyMMdd' ) - date_now = new Date(currentBuild.rawBuild.getStartTimeInMillis()).clearTime() - echo date_now.format( 'yyyyMMdd' ) - return date_pre.numberAwareCompareTo(date_now) < 0 - } - return false - } - } - steps { - rocketSend channel: 'jenkinsohsome', emoji: ':wink:', message: "Hey, this is just your daily notice that Jenkins is still working for you on *${REPO_NAME}* Branch ${env.BRANCH_NAME}! Happy and for free! Keep it up!" , rawMessage: true - } - post { - failure { - rocketSend channel: 'jenkinsohsome', emoji: ':disappointed:', message: "Reporting of *${REPO_NAME}*-build nr. ${env.BUILD_NUMBER} *failed* on Branch - ${env.BRANCH_NAME} (<${env.BUILD_URL}|Open Build in Jenkins>). Latest commit from ${LATEST_AUTHOR}." , rawMessage: true - } - } - } + stage('Publish API Docs') { + when { + anyOf { + equals expected: true, actual: RELEASE_DEPLOY + equals expected: true, actual: SNAPSHOT_DEPLOY + } + } + agent { + label 'builtin' + } + steps { + script { + working_dir = sh(returnStdout: true, script: 'basename $(pwd)').trim() + publish_dir = "/var/lib/jenkins/apidocs/${REPO_NAME}/${API_DOCS_PATH}/" + sh """ + rm -rf ${publish_dir} + mkdir -p ${publish_dir} + cp -r /var/lib/jenkins/workspace/${working_dir}/docs/_build/* ${publish_dir} + """ + } + } + post { + failure { + rocket_basicsend('Publishing of API Docs failed on ${env.BRANCH_NAME}') + } + } + } + // END CUSTOM ohsome API - stage ('Report Status Change') { - when { - expression { - return ((currentBuild.number > 1) && (currentBuild.getPreviousBuild().result == 'FAILURE')) + stage('Check Dependencies') { + when { + expression { + if (currentBuild.number > 1) { + return (((currentBuild.getStartTimeInMillis() - currentBuild.previousBuild.getStartTimeInMillis()) > 2592000000000) && (env.BRANCH_NAME ==~ SNAPSHOT_BRANCH_REGEX)) //2592000000000 one month in milliseconds + } + return false + } + } + steps { + check_dependencies() + } } - } - steps { - rocketSend channel: 'jenkinsohsome', emoji: ':sunglasses:', message: "We had some problems, but we are BACK TO NORMAL! Nice debugging: *${REPO_NAME}*-build-nr. ${env.BUILD_NUMBER} *succeeded* on Branch - ${env.BRANCH_NAME} (<${env.BUILD_URL}|Open Build in Jenkins>). Latest commit from ${LATEST_AUTHOR}." , rawMessage: true - } - post { - failure { - rocketSend channel: 'jenkinsohsome', emoji: ':disappointed:', message: "Reporting of *${REPO_NAME}*-build nr. ${env.BUILD_NUMBER} *failed* on Branch - ${env.BRANCH_NAME} (<${env.BUILD_URL}|Open Build in Jenkins>). Latest commit from ${LATEST_AUTHOR}." , rawMessage: true + + stage('Wrapping Up') { + steps { + encourage() + status_change() + } } - } } - } -} +} \ No newline at end of file diff --git a/pipeline_config.groovy b/pipeline_config.groovy new file mode 100644 index 00000000..1abbbb1b --- /dev/null +++ b/pipeline_config.groovy @@ -0,0 +1,5 @@ +libraries{ + maven + misc + rocketchat +} \ No newline at end of file From ef27a0df0f0f126f4e13f4d4bd660f3c5581c3b6 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Tue, 31 Oct 2023 14:20:34 +0100 Subject: [PATCH 2/2] tidy up Jenkinsfile --- Jenkinsfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7c3af51f..961e5e48 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,7 @@ pipeline { - agent { label 'worker' } + agent { + label 'worker' + } options { timeout(time: 30, unit: 'MINUTES') } @@ -11,6 +13,7 @@ pipeline { // START CUSTOM ohsome API MAVEN_TEST_OPTIONS = '-Dport_get=8081 -Dport_post=8082 -Dport_data=8083 -DdbFilePathProperty=--database.db=/data/heidelberg-v1.0-beta.oshdb' // END CUSTOM ohsome API + // this regex determines which branch is deployed as a snapshot SNAPSHOT_BRANCH_REGEX = /(^master$)/ RELEASE_REGEX = /^([0-9]+(\.[0-9]+)*)(-(RC|beta-|alpha-)[0-9]+)?$/ RELEASE_DEPLOY = false @@ -47,9 +50,11 @@ pipeline { } steps { deploy_snapshot('clean compile javadoc:jar source:jar deploy -P sign,git') + // START CUSTOM ohsome API script { SNAPSHOT_DEPLOY = true } + // END CUSTOM ohsome API } post { failure { @@ -68,9 +73,11 @@ pipeline { deploy_release('clean compile javadoc:jar source:jar deploy -P sign,git') deploy_release_central('clean compile javadoc:jar source:jar deploy -P sign,git,deploy-central') + // START CUSTOM ohsome API script { RELEASE_DEPLOY = true } + // END CUSTOM ohsome API } post { failure { @@ -151,7 +158,7 @@ pipeline { when { expression { if (currentBuild.number > 1) { - return (((currentBuild.getStartTimeInMillis() - currentBuild.previousBuild.getStartTimeInMillis()) > 2592000000000) && (env.BRANCH_NAME ==~ SNAPSHOT_BRANCH_REGEX)) //2592000000000 one month in milliseconds + return (((currentBuild.getStartTimeInMillis() - currentBuild.previousBuild.getStartTimeInMillis()) > 2592000000) && (env.BRANCH_NAME ==~ SNAPSHOT_BRANCH_REGEX)) //2592000000 30 days in milliseconds } return false } @@ -168,4 +175,4 @@ pipeline { } } } -} \ No newline at end of file +}