From 5a0cd1274c1d566d3b13b6b06b9709097fe6e3a1 Mon Sep 17 00:00:00 2001 From: Eddie Webb Date: Mon, 13 Jan 2020 19:21:26 -0500 Subject: [PATCH 1/6] parallelism --- .circleci/config.yml | 29 ++++++++++++++++++++++++++--- src/jobs/block_workflow.yml | 1 + test/test_expansion.bats | 24 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1d1e83..f0b8520 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ workflows: branches: ignore: - master - - test: + - publish_dev: requires: - validate filters: @@ -23,6 +23,14 @@ workflows: only: - staging - trying + - test: + requires: + - publish_dev + filters: + branches: + only: + - staging + - trying - publish: requires: - test @@ -39,7 +47,7 @@ jobs: - checkout - install-circleci - pack-and-validate - test: + publish_dev: docker: - image: cimg/base:2019.08 working_directory: ~/repo @@ -56,11 +64,24 @@ jobs: echo "export ORB_VERSION=\"${ORB_VERSION}\"" >> $BASH_ENV echo $ORB_VERSION echo "export PR_MESSAGE=\"BotComment: *Development* version of orb available for manual validation - \\\`${ORB_VERSION}\\\`\"" >> $BASH_ENV + + test: + docker: + - image: cimg/base:2019.08 + working_directory: ~/repo + parallelism: 4 + steps: + - checkout + - install-circleci + - pack-and-validate + - pr-info - install-bats - run: name: Import Tests using BATS command: | export BATS_IMPORT_DEV_ORB="eddiewebb/<>@dev:${PR_NUMBER}" + export BATS_TEST_LIST=$(bats -l test | cirleci split) + echo "This node will run ${BATS_TEST_LIST}" bats test - pr-comment @@ -101,7 +122,9 @@ commands: - run: name: Install BATS (bash testing) command: | - cd /tmp && git clone https://github.com/bats-core/bats-core.git && cd bats-core + # my fork includes tets list for splitting + cd /tmp && git clone https://github.com/eddiewebb/bats-core.git && cd bats-core + cp ./install.sh /usr/local - run: name: Install YQ diff --git a/src/jobs/block_workflow.yml b/src/jobs/block_workflow.yml index 508525d..be988dd 100644 --- a/src/jobs/block_workflow.yml +++ b/src/jobs/block_workflow.yml @@ -35,6 +35,7 @@ resource_class: small steps: - until_front_of_line: consider-branch: <> + block-workflow: <> time: <> dont-quit: <> only-on-branch: <> diff --git a/test/test_expansion.bats b/test/test_expansion.bats index 9e05df2..94dee33 100755 --- a/test/test_expansion.bats +++ b/test/test_expansion.bats @@ -51,6 +51,30 @@ function setup { } + +@test "Default job sets block workflow properly" { + # given + process_config_with test/inputs/fulljob.yml + + # when + assert_jq_match '.jobs | length' 1 #only 1 job + assert_jq_match '.jobs["Single File"].steps | length' 1 #only 1 steps + + jq -r '.jobs["Single File"].steps[0].run.command' $JSON_PROJECT_CONFIG > ${BATS_TMPDIR}/script-${BATS_TEST_NUMBER}.bash + + export CIRCLECI_API_KEY="madethisup" + export CIRCLE_BUILD_NUM="2" + export CIRCLE_JOB="singlejob" + export CIRCLE_PROJECT_USERNAME="madethisup" + export CIRCLE_PROJECT_REPONAME="madethisup" + export CIRCLE_REPOSITORY_URL="madethisup" + export CIRCLE_BRANCH="madethisup" + + run bash ${BATS_TMPDIR}/script-${BATS_TEST_NUMBER}.bash + assert_contains_text "Orb parameter block-workflow is true." +} + + # See https://github.com/eddiewebb/circleci-queue/issues/26 for explanation of race condition @test "Race condition on previous workflow does not fool us" { # given From a54b84ad766b23e308ca98a715791bd692c1db06 Mon Sep 17 00:00:00 2001 From: Eddie Webb Date: Mon, 13 Jan 2020 20:15:06 -0500 Subject: [PATCH 2/6] fix splitting --- .circleci/config.yml | 3 +-- test/test_expansion.bats | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f0b8520..78a3d1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,7 +80,7 @@ jobs: name: Import Tests using BATS command: | export BATS_IMPORT_DEV_ORB="eddiewebb/<>@dev:${PR_NUMBER}" - export BATS_TEST_LIST=$(bats -l test | cirleci split) + export BATS_TEST_LIST=$(bats -l test | cirleci tests split | tr '\n' ' ') echo "This node will run ${BATS_TEST_LIST}" bats test - pr-comment @@ -124,7 +124,6 @@ commands: command: | # my fork includes tets list for splitting cd /tmp && git clone https://github.com/eddiewebb/bats-core.git && cd bats-core - cp ./install.sh /usr/local - run: name: Install YQ diff --git a/test/test_expansion.bats b/test/test_expansion.bats index 94dee33..48822f6 100755 --- a/test/test_expansion.bats +++ b/test/test_expansion.bats @@ -50,8 +50,6 @@ function setup { } - - @test "Default job sets block workflow properly" { # given process_config_with test/inputs/fulljob.yml From c17f02461407d6bcafe2897a9da9f8daf2fc4ca6 Mon Sep 17 00:00:00 2001 From: Eddie Webb Date: Mon, 13 Jan 2020 20:19:57 -0500 Subject: [PATCH 3/6] fix splitting --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 78a3d1a..fd4888b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,7 +80,7 @@ jobs: name: Import Tests using BATS command: | export BATS_IMPORT_DEV_ORB="eddiewebb/<>@dev:${PR_NUMBER}" - export BATS_TEST_LIST=$(bats -l test | cirleci tests split | tr '\n' ' ') + export BATS_TEST_LIST=$(bats -l test | circleci tests split | tr '\n' ' ') echo "This node will run ${BATS_TEST_LIST}" bats test - pr-comment From 6caaff45d3a223c387cda314286d0df2dca5a8aa Mon Sep 17 00:00:00 2001 From: Eddie Webb Date: Mon, 13 Jan 2020 21:58:26 -0500 Subject: [PATCH 4/6] export timing data to circleci --- .circleci/config.yml | 8 +- src/parse-taps.bash | 27 +++++ src/queue.bash | 207 +++++++++++++++++++++++++++++++++++++++ test/test_expansion.bats | 1 - 4 files changed, 240 insertions(+), 3 deletions(-) create mode 100755 src/parse-taps.bash create mode 100644 src/queue.bash diff --git a/.circleci/config.yml b/.circleci/config.yml index fd4888b..7ea134b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,10 +80,14 @@ jobs: name: Import Tests using BATS command: | export BATS_IMPORT_DEV_ORB="eddiewebb/<>@dev:${PR_NUMBER}" - export BATS_TEST_LIST=$(bats -l test | circleci tests split | tr '\n' ' ') + export BATS_TEST_LIST=$(bats -l test | circleci tests split --split-by=timings | tr '\n' ' ') echo "This node will run ${BATS_TEST_LIST}" - bats test + mkdir -p reports + bats test | tee reports/test-report + bash src/parse-taps.bash reports/test-report > reports/bats_junit_report.xml - pr-comment + - store_test_results: + path: reports publish: docker: diff --git a/src/parse-taps.bash b/src/parse-taps.bash new file mode 100755 index 0000000..f061a34 --- /dev/null +++ b/src/parse-taps.bash @@ -0,0 +1,27 @@ +#!/bin/bash + +head='\n' +foot='' + +test='' + +parse_it(){ + $line=$1 + NAME=$(expr "$line" : '.*ok [0-9]* \(.*\) #time.*') + TIME=$(expr "$line" : '.*ok [0-9]*.*\#time=\(.*\)') + printf "${test}" "${NAME}" ${TIME} +} + + +printf "$head" + +while read -r line;do + case $line in + 1..*) continue ;; + ok*) parse_it $line;; + \#*) continue;; + *) echo "unkon line" ;; + esac +done < "$1" + +printf "\n$foot" \ No newline at end of file diff --git a/src/queue.bash b/src/queue.bash new file mode 100644 index 0000000..12968a2 --- /dev/null +++ b/src/queue.bash @@ -0,0 +1,207 @@ + +# +#.This query builds and determine our place in queue +# +main_loop(){ + load_variables + max_time=<< parameters.time >> + echo "This build will block until all previous builds complete." + echo "Max Queue Time: ${max_time} minutes." + wait_time=0 + loop_time=10 + max_time_seconds=$((max_time * 60)) + + + load_current_job_details #gives us our pipeline ID & Order + + + + #get recent pipeline for this project (optionally filtering on branch) + # for each pipeline, is ID lower than ours? + # if lower, get all workflows + # for each workflow, are they running? + # if job-specific running, do they contain my job? + # v2 api sucks. + + + + + # + # Queue Loop + # + confidence=0 + while true; do + update_comparables + echo "This Workflow Timestamp: $my_commit_time" + echo "Oldest Workflow Timestamp: $oldest_commit_time" + if [[ "$oldest_commit_time" > "$my_commit_time" ]] || [[ "$oldest_commit_time" = "$my_commit_time" ]] ; then + # API returns Y-M-D HH:MM (with 24 hour clock) so alphabetical string compare is accurate to timestamp compare as well + # recent-jobs API does not include pending, so it is posisble we queried in between a workfow transition, and we;re NOT really front of line. + if [ $confidence -lt <> ];then + # To grow confidence, we check again with a delay. + confidence=$((confidence+1)) + else + echo "Front of the line, WooHoo!, Build continuing" + break + fi + else + echo "This build (${CIRCLE_BUILD_NUM}) is queued, waiting for build number (${oldest_running_build_num}) to complete." + echo "Total Queue time: ${wait_time} seconds." + fi + + if [ $wait_time -ge $max_time_seconds ]; then + echo "Max wait time exceeded, considering response." + if [ "<>" == "true" ];then + echo "Orb parameter dont-quit is set to true, letting this job proceed!" + exit 0 + else + cancel_current_build + sleep 10 # wait for API to cancel this job, rather than showing as failure + exit 1 # but just in case, fail job + fi + fi + + sleep $loop_time + wait_time=$(( loop_time + wait_time )) + done + +} + + + +load_variables(){ + # just confirm our required variables are present + : ${CIRCLE_BUILD_NUM:?"Required Env Variable not found!"} + : ${CIRCLE_PROJECT_USERNAME:?"Required Env Variable not found!"} + : ${CIRCLE_PROJECT_REPONAME:?"Required Env Variable not found!"} + : ${CIRCLE_REPOSITORY_URL:?"Required Env Variable not found!"} + : ${CIRCLE_JOB:?"Required Env Variable not found!"} + : ${CIRCLE_WORKFLOW_ID:?"Required Env Variable not found!"} + # Only needed for private projects + if [ -z "$CIRCLECI_API_KEY" ]; then + echo "ERROR: CIRCLECI_API_KEY not set. API will be inaccessible." >&2 + exit 1 + fi + VCS_TYPE="<>" +} + +load_current_pipeline_info(){ + #get_api_payload "https://circleci.com/api/v2/project/${VCS_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/job/${CIRCLE_BUILD_NUM}" /tmp/current_workflow.json + get_api_payload "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}" /tmp/current_workflow.json + CIRCLE_PIPELINE_ID=$(jq '.pipeline_id' /tmp/current_workflow.json) #UUID + CIRCLE_PIPELINE_NUMBER=$(jq '.pipeline_number' /tmp/current_workflow.json) #RelativeOrder +} + + +get_api_payload(){ + url=$1 + target=$2 + curl -X GET "${url}" -H "Accept: application/json" -H "Circle-Token: ${CIRCLECI_API_KEY}" > ${target} + if [ $? -ne 0 ];then + echo "ERROR: Curl command to ${url} failed. Response below." + cat $target + exit 1 + fi +} + + +fetch_filtered_active_builds(){ + if [ "<>" != "true" ];then + echo "Orb parameter 'consider-branch' is false, will block previous builds on any branch." + jobs_api_url_template="https://circleci.com/api/v1.1/project/${VCS_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}?circle-token=${CIRCLECI_API_KEY}&filter=running" + else + echo "Only blocking execution if running previous jobs on branch: ${CIRCLE_BRANCH}" + : ${CIRCLE_BRANCH:?"Required Env Variable not found!"} + jobs_api_url_template="https://circleci.com/api/v1.1/project/${VCS_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/tree/${CIRCLE_BRANCH}?circle-token=${CIRCLECI_API_KEY}&filter=running" + fi + + if [ ! -z $TESTING_MOCK_RESPONSE ] && [ -f $TESTING_MOCK_RESPONSE ];then + echo "Using test mock response" + cat $TESTING_MOCK_RESPONSE > /tmp/jobstatus.json + else + echo "Attempting to access CircleCI api. If the build process fails after this step, ensure your CIRCLECI_API_KEY is set." + curl -f -s $jobs_api_url_template > /tmp/jobstatus.json + echo "API access successful" + fi +} + +fetch_active_workflows(){ + cp /tmp/jobstatus.json /tmp/augmented_jobstatus.json + for workflow in `jq -r ".[] | .workflows.workflow_id" /tmp/augmented_jobstatus.json | uniq` + do + echo "Checking time of workflow: ${workflow}" + workflow_file=/tmp/workflow-${workflow}.json + if [ ! -z $TESTING_MOCK_WORKFLOW_RESPONSES ] && [ -f $TESTING_MOCK_WORKFLOW_RESPONSES/${workflow}.json ]; then + echo "Using test mock workflow response" + cat $TESTING_MOCK_WORKFLOW_RESPONSES/${workflow}.json > ${workflow_file} + else + curl -f -s "https://circleci.com/api/v2/workflow/${workflow}?circle-token=${CIRCLECI_API_KEY}" > ${workflow_file} + fi + created_at=`jq -r '.created_at' ${workflow_file}` + echo "Workflow was created at: ${created_at}" + cat /tmp/augmented_jobstatus.json | jq --arg created_at "${created_at}" --arg workflow "${workflow}" '(.[] | select(.workflows.workflow_id == $workflow) | .workflows) |= . + {created_at:$created_at}' > /tmp/augmented_jobstatus-${workflow}.json + #DEBUG echo "new augmented_jobstatus:" + #DEBUG cat /tmp/augmented_jobstatus-${workflow}.json + mv /tmp/augmented_jobstatus-${workflow}.json /tmp/augmented_jobstatus.json + done +} + +update_comparables(){ + fetch_filtered_active_builds + + fetch_active_workflows + + load_current_workflow_values + + # falsey parameters are empty strings, so always compare against 'true' + if [ "<>" != "true" ] || [ "<>" = "true" ] ;then + echo "Orb parameter block-workflow is true." + echo "This job will block until no previous workflows have *any* jobs running." + oldest_running_build_num=`jq 'sort_by(.workflows.created_at)| .[0].build_num' /tmp/augmented_jobstatus.json` + oldest_commit_time=`jq 'sort_by(.workflows.created_at)| .[0].workflows.created_at' /tmp/augmented_jobstatus.json` + else + echo "Orb parameter block-workflow is false." + echo "Only blocking execution if running previous jobs matching this job: ${CIRCLE_JOB}" + oldest_running_build_num=`jq ". | map(select(.build_parameters.CIRCLE_JOB==\"${CIRCLE_JOB}\")) | sort_by(.workflows.created_at)| .[0].build_num" /tmp/augmented_jobstatus.json` + oldest_commit_time=`jq ". | map(select(.build_parameters.CIRCLE_JOB==\"${CIRCLE_JOB}\")) | sort_by(.workflows.created_at)| .[0].workflows.created_at" /tmp/augmented_jobstatus.json` + fi + echo "Oldest job: $oldest_running_build_num" + if [ -z $oldest_commit_time ];then + echo "API Call for existing jobs failed, failing this build. Please check API token" + echo "All running jobs:" + cat /tmp/jobstatus.json || exit 0 + echo "All running jobs with created_at:" + cat /tmp/augmented_jobstatus.json || exit 0 + echo "All worfklow details." + cat /tmp/workflow-*.json + exit 1 + fi +} + +load_current_workflow_values(){ + my_commit_time=`jq '.[] | select( .build_num == '"${CIRCLE_BUILD_NUM}"').workflows.created_at' /tmp/augmented_jobstatus.json` +} + +cancel_current_build(){ + echo "Cancelleing build ${CIRCLE_BUILD_NUM}" + cancel_api_url_template="https://circleci.com/api/v1.1/project/${VCS_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}/cancel?circle-token=${CIRCLECI_API_KEY}" + curl -s -X POST $cancel_api_url_template > /dev/null +} + + + +# +# We can skip a few use cases without calling API +# +if [ ! -z "$CIRCLE_PR_REPONAME" ]; then + echo "Queueing on forks is not supported. Skipping queue..." + # It's important that we not fail here because it could cause issues on the main repo's branch + exit 0 +fi +if [ "<>" = "*" ] || [ "<>" = "${CIRCLE_BRANCH}" ]; then + echo "${CIRCLE_BRANCH} queueable" +else + echo "Queueing only happens on <> branch, skipping queue" + exit 0 +fi + diff --git a/test/test_expansion.bats b/test/test_expansion.bats index 48822f6..26c15f8 100755 --- a/test/test_expansion.bats +++ b/test/test_expansion.bats @@ -32,7 +32,6 @@ function setup { # when assert_jq_match '.jobs | length' 1 #only 1 job assert_jq_match '.jobs["Single File"].steps | length' 1 #only 1 steps - } From 8bfcda97d374a343e43283f76365ba65c5d554ee Mon Sep 17 00:00:00 2001 From: Eddie Webb Date: Mon, 13 Jan 2020 22:18:31 -0500 Subject: [PATCH 5/6] hope for better timing --- .circleci/config.yml | 2 ++ test/test_expansion.bats | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ea134b..3d6caa6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,6 +88,8 @@ jobs: - pr-comment - store_test_results: path: reports + - store_artifacts: + path: reports publish: docker: diff --git a/test/test_expansion.bats b/test/test_expansion.bats index 26c15f8..043a51c 100755 --- a/test/test_expansion.bats +++ b/test/test_expansion.bats @@ -71,8 +71,6 @@ function setup { assert_contains_text "Orb parameter block-workflow is true." } - -# See https://github.com/eddiewebb/circleci-queue/issues/26 for explanation of race condition @test "Race condition on previous workflow does not fool us" { # given process_config_with test/inputs/command-defaults.yml From 5bdd396aa0a099dee2b639234264cf5ffbe4258f Mon Sep 17 00:00:00 2001 From: Eddie Webb Date: Mon, 13 Jan 2020 22:19:02 -0500 Subject: [PATCH 6/6] hope for better timing --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d6caa6..c0efe6d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ jobs: - store_test_results: path: reports - store_artifacts: - path: reports + path: reports publish: docker: