Skip to content

Commit

Permalink
Superceed local file over network download (#2458)
Browse files Browse the repository at this point in the history 
* Local platform configs will now superceed network locations
* Build will fail if a local file doesn't exist and PLATFORM_CONFIG_LOCATION is unset

Signed-off-by: Morgan Davies <morgandavies2020@gmail.com>
Morgan Davies authored Feb 8, 2021
1 parent dd651a5 commit 93b479a
Showing 2 changed files with 57 additions and 42 deletions.
97 changes: 56 additions & 41 deletions build-farm/set-platform-specific-configurations.sh
View file
Original file line number Diff line number Diff line change
@@ -25,60 +25,75 @@ fi

export VARIANT_ARG="--build-variant ${VARIANT}"

# Create placeholders for curl --output
if [ ! -d "$SCRIPT_DIR/platform-specific-configurations" ]
then
mkdir "$SCRIPT_DIR/platform-specific-configurations"
fi
if [ ! -f "$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh" ]
# If a user file doesn't exist, pull from an online source. To always pull from an online source, ensure platform config files have been deleted from your local clone.
PLATFORM_CONFIG_FILEPATH="$SCRIPT_DIR/platform-specific-configurations/${OPERATING_SYSTEM}.sh"
if [ ! -f "${PLATFORM_CONFIG_FILEPATH}" ]
then
# Fail build if a local file doesn't exist and network location hasn't been set
if [ -z "${PLATFORM_CONFIG_LOCATION}" ]
then
echo "[ERROR] No local file detected at ${PLATFORM_CONFIG_FILEPATH} and PLATFORM_CONFIG_LOCATION is not set. Please set PLATFORM_CONFIG_LOCATION to a repository path of a platform config file (e.g. AdoptOpenJDK/openjdk-build/master/build-farm/platform-specific-configurations)."
exit 3
fi

# Create placeholders for curl --output
if [ ! -d "$SCRIPT_DIR/platform-specific-configurations" ]
then
mkdir "$SCRIPT_DIR/platform-specific-configurations"
fi

# Reset config file to execute
rm -f "$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh"
touch "$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh"
fi
PLATFORM_CONFIG_FILEPATH="$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh"

# Setup for platform config download
rawGithubSource="https://raw.githubusercontent.com"
ret=0
fileContents=""
PLATFORM_CONFIG_FILEPATH="$SCRIPT_DIR/platform-specific-configurations/platformConfigFile.sh"

# Uses curl to download the platform config file
# param 1: LOCATION - Repo path to where the file is located
# param 2: SUFFIX - Operating system to append
function downloadPlatformConfigFile () {
echo "Attempting to download platform configuration file from ${rawGithubSource}/$1/$2"
# make-adopt-build-farm.sh has 'set -e'. We need to disable that for the fallback mechanism, as downloading might fail
set +e
curl "${rawGithubSource}/$1/$2" > "${PLATFORM_CONFIG_FILEPATH}"
ret=$?
# A download will succeed if location is a directory, so we also check the contents are valid
fileContents=$(cat $PLATFORM_CONFIG_FILEPATH)
set -e
}
# Setup for platform config download
rawGithubSource="https://raw.githubusercontent.com"
ret=0
fileContents=""

# Attempt to download and source the user's custom platform config
downloadPlatformConfigFile "${PLATFORM_CONFIG_LOCATION}" ""
# Regex to spot github api error messages similar to "404: Not Found"
contentsErrorRegex="#!/bin/bash"
# Uses curl to download the platform config file
# param 1: LOCATION - Repo path to where the file is located
# param 2: SUFFIX - Operating system to append
function downloadPlatformConfigFile () {
echo "Attempting to download platform configuration file from ${rawGithubSource}/$1/$2"
# make-adopt-build-farm.sh has 'set -e'. We need to disable that for the fallback mechanism, as downloading might fail
set +e
curl "${rawGithubSource}/$1/$2" > "${PLATFORM_CONFIG_FILEPATH}"
ret=$?
# A download will succeed if location is a directory, so we also check the contents are valid
fileContents=$(cat $PLATFORM_CONFIG_FILEPATH)
set -e
}

if [ $ret -ne 0 ] || [[ ! $fileContents =~ $contentsErrorRegex ]]
then
# Check to make sure that a OS file doesn't exist if we can't find a config file from the direct link
echo "[WARNING] Failed to find a user configuration file, ${rawGithubSource}/${PLATFORM_CONFIG_LOCATION} is likely a directory so we will try and search for a ${OPERATING_SYSTEM}.sh file."
downloadPlatformConfigFile "${PLATFORM_CONFIG_LOCATION}" "${OPERATING_SYSTEM}.sh"
# Attempt to download and source the user's custom platform config
downloadPlatformConfigFile "${PLATFORM_CONFIG_LOCATION}" ""
# Regex to spot github api error messages similar to "404: Not Found"
contentsErrorRegex="#!/bin/bash"

if [ $ret -ne 0 ] || [[ ! $fileContents =~ $contentsErrorRegex ]]
then
# If there is no user platform config, use adopt's as a default instead
echo "[WARNING] Failed to download a user platform configuration file. Downloading Adopt's ${OPERATING_SYSTEM}.sh configuration file instead."
downloadPlatformConfigFile "${ADOPT_PLATFORM_CONFIG_LOCATION}" "${OPERATING_SYSTEM}.sh"
# Check to make sure that a OS file doesn't exist if we can't find a config file from the direct link
echo "[WARNING] Failed to find a user configuration file, ${rawGithubSource}/${PLATFORM_CONFIG_LOCATION} is likely a directory so we will try and search for a ${OPERATING_SYSTEM}.sh file."
downloadPlatformConfigFile "${PLATFORM_CONFIG_LOCATION}" "${OPERATING_SYSTEM}.sh"

if [ $ret -ne 0 ] || [[ ! $fileContents =~ $contentsErrorRegex ]]
then
echo "[ERROR] Failed to download a platform configuration file from User and Adopt's repositories"
exit 2
# If there is no user platform config, use adopt's as a default instead
echo "[WARNING] Failed to download a user platform configuration file. Downloading Adopt's ${OPERATING_SYSTEM}.sh configuration file instead."
downloadPlatformConfigFile "${ADOPT_PLATFORM_CONFIG_LOCATION}" "${OPERATING_SYSTEM}.sh"

if [ $ret -ne 0 ] || [[ ! $fileContents =~ $contentsErrorRegex ]]
then
echo "[ERROR] Failed to download a platform configuration file from User and Adopt's repositories"
exit 2
fi
fi
fi
fi

echo "[SUCCESS] Config file downloaded successfully to ${PLATFORM_CONFIG_FILEPATH}"
echo "[SUCCESS] Config file downloaded successfully to ${PLATFORM_CONFIG_FILEPATH}"
else
echo "[SUCCESS] Executing local file at ${PLATFORM_CONFIG_FILEPATH}"
fi
source "${PLATFORM_CONFIG_FILEPATH}"
2 changes: 1 addition & 1 deletion docs/UsingOurScripts.md
View file
Original file line number Diff line number Diff line change
@@ -113,7 +113,7 @@ Once it has been approved and merged, update your scripts and/or jenkins jobs to
- [pipelines/build/regeneration/jdkx_regeneration_pipeline.groovy](https://github.com/AdoptOpenJDK/openjdk-build/blob/master/pipelines/build/regeneration/jdk8_regeneration_pipeline.groovy) - Main downstream generator files. These are what the [pipeline_jobs_generator_jdk8u jenkins jobs](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/pipeline_jobs_generator_jdk8u/) execute on build, generating the [downstream jobs](https://ci.adoptopenjdk.net/job/build-scripts/job/jobs/) via `pipelines/build/common/config_regeneration.groovy` (see below).
- [pipelines/build/common/config_regeneration.groovy](https://github.com/AdoptOpenJDK/openjdk-build/blob/master/pipelines/build/common/config_regeneration.groovy) - Base downstream script file. These are what the [pipeline_jobs_generator_jdk8u jenkins jobs](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/pipeline_jobs_generator_jdk8u/) execute after `jdkx_regeneration_pipeline.groovy`, calling the dsl template `pipelines/build/common/create_job_from_template.groovy`.
- [pipelines/build/common/create_job_from_template.groovy](https://github.com/AdoptOpenJDK/openjdk-build/blob/master/pipelines/build/common/create_job_from_template.groovy) - Downstream jobs dsl. This is the dsl job framework of the [downstream jobs]((https://ci.adoptopenjdk.net/job/build-scripts/job/jobs/)).
2. Create a User JSON file containing your default constants that the build scripts will use (see [#the defaults.json](#defaults.json))
2. Create a User JSON file containing your default constants that the build scripts will use (see [#the defaults.json](#defaults.json)). Delete any of Adopt's files and directories that you have not altered in step 1.
3. Copy the [build-pipeline-generator](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/build-pipeline-generator/) and [pipeline_jobs_generator_jdk8u](https://ci.adoptopenjdk.net/job/build-scripts/job/utils/job/pipeline_jobs_generator_jdk8u/) jobs to your Jenkins instance (replace `jdk8u` with whichever version you intend to build, there should be one job for each jdk version).
4. Execute the copied `build-pipeline-generator`. Make sure you have filled in the parameters that are not covered by your `defaults.json` (e.g. `DEFAULTS_URL`, `CHECKOUT_CREDENTIALS`). You should now see that the nightly and weekly pipeline jobs have been successfully created in whatever folder was entered into `JOB_ROOT`
5. Execute the copied `pipeline_jobs_generator_jdkxx` jobs. Again, make sure you have filled in the parameters that are not covered by your `defaults.json`. You should now see that the `jobs/jdkxx-platform-arch-variant` jobs have been successfully created in whatever folder was entered into `JOB_ROOT`

0 comments on commit 93b479a

Please sign in to comment.