From 9e6f34fc55f2dc5b0eb19f2a1f73ca65aaea235b Mon Sep 17 00:00:00 2001 From: Morgan Davies Date: Mon, 8 Feb 2021 13:23:11 +0000 Subject: [PATCH] Superceed local file over network download * Local platform configs will now superceed network locations * Build will fail if a local file doesn't exist and PLATFORM_CONFIG_LOCATION is unset Signed-off-by: Morgan Davies --- .../set-platform-specific-configurations.sh | 97 +++++++++++-------- docs/UsingOurScripts.md | 2 +- 2 files changed, 57 insertions(+), 42 deletions(-) diff --git a/build-farm/set-platform-specific-configurations.sh b/build-farm/set-platform-specific-configurations.sh index 10aae2eee..6bf802fff 100755 --- a/build-farm/set-platform-specific-configurations.sh +++ b/build-farm/set-platform-specific-configurations.sh @@ -25,60 +25,75 @@ fi export VARIANT_ARG="--build-variant ${VARIANT}" -# Create placeholders for curl --output -if [ ! -d "$SCRIPT_DIR/platform-specific-configurations" ] -then - mkdir "$SCRIPT_DIR/platform-specific-configurations" -fi -if [ ! -f "$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh" ] +# If a user file doesn't exist, pull from an online source. To always pull from an online source, ensure platform config files have been deleted from your local clone. +PLATFORM_CONFIG_FILEPATH="$SCRIPT_DIR/platform-specific-configurations/${OPERATING_SYSTEM}.sh" +if [ ! -f "${PLATFORM_CONFIG_FILEPATH}" ] then + # Fail build if a local file doesn't exist and network location hasn't been set + if [ -z "${PLATFORM_CONFIG_LOCATION}" ] + then + echo "[ERROR] No local file detected at ${PLATFORM_CONFIG_FILEPATH} and PLATFORM_CONFIG_LOCATION is not set. Please set PLATFORM_CONFIG_LOCATION to a repository path of a platform config file (e.g. AdoptOpenJDK/openjdk-build/master/build-farm/platform-specific-configurations)." + exit 3 + fi + + # Create placeholders for curl --output + if [ ! -d "$SCRIPT_DIR/platform-specific-configurations" ] + then + mkdir "$SCRIPT_DIR/platform-specific-configurations" + fi + + # Reset config file to execute + rm -f "$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh" touch "$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh" -fi -PLATFORM_CONFIG_FILEPATH="$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh" -# Setup for platform config download -rawGithubSource="https://raw.githubusercontent.com" -ret=0 -fileContents="" + PLATFORM_CONFIG_FILEPATH="$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh" -# Uses curl to download the platform config file -# param 1: LOCATION - Repo path to where the file is located -# param 2: SUFFIX - Operating system to append -function downloadPlatformConfigFile () { - echo "Attempting to download platform configuration file from ${rawGithubSource}/$1/$2" - # make-adopt-build-farm.sh has 'set -e'. We need to disable that for the fallback mechanism, as downloading might fail - set +e - curl "${rawGithubSource}/$1/$2" > "${PLATFORM_CONFIG_FILEPATH}" - ret=$? - # A download will succeed if location is a directory, so we also check the contents are valid - fileContents=$(cat $PLATFORM_CONFIG_FILEPATH) - set -e -} + # Setup for platform config download + rawGithubSource="https://raw.githubusercontent.com" + ret=0 + fileContents="" -# Attempt to download and source the user's custom platform config -downloadPlatformConfigFile "${PLATFORM_CONFIG_LOCATION}" "" -# Regex to spot github api error messages similar to "404: Not Found" -contentsErrorRegex="#!/bin/bash" + # Uses curl to download the platform config file + # param 1: LOCATION - Repo path to where the file is located + # param 2: SUFFIX - Operating system to append + function downloadPlatformConfigFile () { + echo "Attempting to download platform configuration file from ${rawGithubSource}/$1/$2" + # make-adopt-build-farm.sh has 'set -e'. We need to disable that for the fallback mechanism, as downloading might fail + set +e + curl "${rawGithubSource}/$1/$2" > "${PLATFORM_CONFIG_FILEPATH}" + ret=$? + # A download will succeed if location is a directory, so we also check the contents are valid + fileContents=$(cat $PLATFORM_CONFIG_FILEPATH) + set -e + } -if [ $ret -ne 0 ] || [[ ! $fileContents =~ $contentsErrorRegex ]] -then - # Check to make sure that a OS file doesn't exist if we can't find a config file from the direct link - echo "[WARNING] Failed to find a user configuration file, ${rawGithubSource}/${PLATFORM_CONFIG_LOCATION} is likely a directory so we will try and search for a ${OPERATING_SYSTEM}.sh file." - downloadPlatformConfigFile "${PLATFORM_CONFIG_LOCATION}" "${OPERATING_SYSTEM}.sh" + # Attempt to download and source the user's custom platform config + downloadPlatformConfigFile "${PLATFORM_CONFIG_LOCATION}" "" + # Regex to spot github api error messages similar to "404: Not Found" + contentsErrorRegex="#!/bin/bash" if [ $ret -ne 0 ] || [[ ! $fileContents =~ $contentsErrorRegex ]] then - # If there is no user platform config, use adopt's as a default instead - echo "[WARNING] Failed to download a user platform configuration file. Downloading Adopt's ${OPERATING_SYSTEM}.sh configuration file instead." - downloadPlatformConfigFile "${ADOPT_PLATFORM_CONFIG_LOCATION}" "${OPERATING_SYSTEM}.sh" + # Check to make sure that a OS file doesn't exist if we can't find a config file from the direct link + echo "[WARNING] Failed to find a user configuration file, ${rawGithubSource}/${PLATFORM_CONFIG_LOCATION} is likely a directory so we will try and search for a ${OPERATING_SYSTEM}.sh file." + downloadPlatformConfigFile "${PLATFORM_CONFIG_LOCATION}" "${OPERATING_SYSTEM}.sh" if [ $ret -ne 0 ] || [[ ! $fileContents =~ $contentsErrorRegex ]] then - echo "[ERROR] Failed to download a platform configuration file from User and Adopt's repositories" - exit 2 + # If there is no user platform config, use adopt's as a default instead + echo "[WARNING] Failed to download a user platform configuration file. Downloading Adopt's ${OPERATING_SYSTEM}.sh configuration file instead." + downloadPlatformConfigFile "${ADOPT_PLATFORM_CONFIG_LOCATION}" "${OPERATING_SYSTEM}.sh" + + if [ $ret -ne 0 ] || [[ ! $fileContents =~ $contentsErrorRegex ]] + then + echo "[ERROR] Failed to download a platform configuration file from User and Adopt's repositories" + exit 2 + fi fi fi -fi -echo "[SUCCESS] Config file downloaded successfully to ${PLATFORM_CONFIG_FILEPATH}" + echo "[SUCCESS] Config file downloaded successfully to ${PLATFORM_CONFIG_FILEPATH}" +else + echo "[SUCCESS] Executing local file at ${PLATFORM_CONFIG_FILEPATH}" +fi source "${PLATFORM_CONFIG_FILEPATH}" diff --git a/docs/UsingOurScripts.md b/docs/UsingOurScripts.md index af21b9722..96028c2d0 100644 --- a/docs/UsingOurScripts.md +++ b/docs/UsingOurScripts.md @@ -113,7 +113,7 @@ Once it has been approved and merged, update your scripts and/or jenkins jobs to - [pipelines/build/regeneration/jdkx_regeneration_pipeline.groovy](https://github.com/AdoptOpenJDK/openjdk-build/blob/master/pipelines/build/regeneration/jdk8_regeneration_pipeline.groovy) - Main downstream generator files. These are what the [pipeline_jobs_generator_jdk8u jenkins jobs](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/pipeline_jobs_generator_jdk8u/) execute on build, generating the [downstream jobs](https://ci.adoptopenjdk.net/job/build-scripts/job/jobs/) via `pipelines/build/common/config_regeneration.groovy` (see below). - [pipelines/build/common/config_regeneration.groovy](https://github.com/AdoptOpenJDK/openjdk-build/blob/master/pipelines/build/common/config_regeneration.groovy) - Base downstream script file. These are what the [pipeline_jobs_generator_jdk8u jenkins jobs](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/pipeline_jobs_generator_jdk8u/) execute after `jdkx_regeneration_pipeline.groovy`, calling the dsl template `pipelines/build/common/create_job_from_template.groovy`. - [pipelines/build/common/create_job_from_template.groovy](https://github.com/AdoptOpenJDK/openjdk-build/blob/master/pipelines/build/common/create_job_from_template.groovy) - Downstream jobs dsl. This is the dsl job framework of the [downstream jobs]((https://ci.adoptopenjdk.net/job/build-scripts/job/jobs/)). -2. Create a User JSON file containing your default constants that the build scripts will use (see [#the defaults.json](#defaults.json)) +2. Create a User JSON file containing your default constants that the build scripts will use (see [#the defaults.json](#defaults.json)). Delete any of Adopt's files and directories that you have not altered in step 1. 3. Copy the [build-pipeline-generator](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/build-pipeline-generator/) and [pipeline_jobs_generator_jdk8u](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/pipeline_jobs_generator_jdk8u/) jobs to your Jenkins instance (replace `jdk8u` with whichever version you intend to build, there should be one job for each jdk version). 4. Execute the copied `build-pipeline-generator`. Make sure you have filled in the parameters that are not covered by your `defaults.json` (e.g. `DEFAULTS_URL`, `CHECKOUT_CREDENTIALS`). You should now see that the nightly and weekly pipeline jobs have been successfully created in whatever folder was entered into `JOB_ROOT` 5. Execute the copied `pipeline_jobs_generator_jdkxx` jobs. Again, make sure you have filled in the parameters that are not covered by your `defaults.json`. You should now see that the `jobs/jdkxx-platform-arch-variant` jobs have been successfully created in whatever folder was entered into `JOB_ROOT`