Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[github](action) optimize trigger teamcity pipeline #27257

Merged
merged 6 commits into from
Nov 22, 2023
Merged

[github](action) optimize trigger teamcity pipeline #27257

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
23ca2d4
[github](action) optimize triiger teamcity pipeline
Nov 20, 2023
5a7ca61
1121
Nov 21, 2023
aa3ad96
disable old trigger
Nov 21, 2023
40bb847
2
Nov 22, 2023
ed1592e
Merge branch 'master' into 1120-fast-action
hello-stephen Nov 22, 2023
8a242d6
2
Nov 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 111 additions & 111 deletions .github/workflows/auto_trigger_teamcity.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,129 +15,129 @@
# specific language governing permissions and limitations
# under the License.

name: Auto trigger teamcity
# name: Auto trigger teamcity

on:
issue_comment:
types: [created, edited]
pull_request_review_comment:
types: [created, edited]
# on:
# issue_comment:
# types: [created, edited]
# pull_request_review_comment:
# types: [created, edited]

env:
PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}')
LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json)
TEAMCITY_URL: '-H \"Content-Type:text/plain\" -H \"Accept: application/json\" -u OneMoreChance:OneMoreChance http://43.132.222.7:8111'
# env:
# PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}')
# LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json)
# TEAMCITY_URL: '-H \"Content-Type:text/plain\" -H \"Accept: application/json\" -u OneMoreChance:OneMoreChance http://43.132.222.7:8111'

jobs:
run_teamcity_pipeline:
if: (contains(github.event.comment.body, 'buildall') || contains(github.event.comment.body, 'p0') || contains(github.event.comment.body, 'pipelinex_p0') || contains(github.event.comment.body, 'p1') || contains(github.event.comment.body, 'feut') || contains(github.event.comment.body, 'beut') || contains(github.event.comment.body, 'compile') || contains(github.event.comment.body, 'clickbench') || contains(github.event.comment.body, 'arm') || contains(github.event.comment.body, 'external') || contains(github.event.comment.body, 'just_for_test')) && contains(github.event.comment.body, 'run') && !contains(github.event.comment.body, 'Thanks for your contribution')
# jobs:
# run_teamcity_pipeline:
# if: (contains(github.event.comment.body, 'buildall') || contains(github.event.comment.body, 'p0') || contains(github.event.comment.body, 'pipelinex_p0') || contains(github.event.comment.body, 'p1') || contains(github.event.comment.body, 'feut') || contains(github.event.comment.body, 'beut') || contains(github.event.comment.body, 'compile') || contains(github.event.comment.body, 'clickbench') || contains(github.event.comment.body, 'arm') || contains(github.event.comment.body, 'external') || contains(github.event.comment.body, 'just_for_test')) && contains(github.event.comment.body, 'run') && !contains(github.event.comment.body, 'Thanks for your contribution')

runs-on: ubuntu-latest
# runs-on: ubuntu-latest

env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ISSUE_COMMENT: ${{ github.event.comment.body }}
steps:
- name: Run pipeline by teamcity restful
run: |
if [ "_xx""${{ github.event.issue.pull_request.url }}" != "_xx" ]; then
echo "Comment was made on pull request: $(echo ${{ github.event.issue.pull_request.url }} | awk -F/ '{print $NF}')"
else
echo "Comment was made on an issue, not a pull request."
fi
pull_request_num=${{ env.PULL_REQUEST_NUM }}
encoded_string=$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$ISSUE_COMMENT")
comment_message="$ISSUE_COMMENT"
latest_commit_id="${{ env.LATEST_COMMIT }}"
teamcity_url="${{ env.TEAMCITY_URL }}"
trigger_pipelines=""
same_build_sign="false"
echo "pull_request_num : ${pull_request_num}"
echo "encoded_string : ${encoded_string}"
echo "latest_commit_id : ${{ env.LATEST_COMMIT }}"
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# ISSUE_COMMENT: ${{ github.event.comment.body }}
# steps:
# - name: Run pipeline by teamcity restful
# run: |
# if [ "_xx""${{ github.event.issue.pull_request.url }}" != "_xx" ]; then
# echo "Comment was made on pull request: $(echo ${{ github.event.issue.pull_request.url }} | awk -F/ '{print $NF}')"
# else
# echo "Comment was made on an issue, not a pull request."
# fi
# pull_request_num=${{ env.PULL_REQUEST_NUM }}
# encoded_string=$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$ISSUE_COMMENT")
# comment_message="$ISSUE_COMMENT"
# latest_commit_id="${{ env.LATEST_COMMIT }}"
# teamcity_url="${{ env.TEAMCITY_URL }}"
# trigger_pipelines=""
# same_build_sign="false"
# echo "pull_request_num : ${pull_request_num}"
# echo "encoded_string : ${encoded_string}"
# echo "latest_commit_id : ${{ env.LATEST_COMMIT }}"

set -x
reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*"
comment_trigger_type="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)"
comment_repeat_times="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)"
if [[ "${comment_trigger_type}" == "buildall" ]]; then
trigger_pipelines="Doris_Doris_FeUt Doris_DorisBeUt_BeUt Doris_DorisCompile_Compile Doris_Performance_Clickbench_ClickbenchNew Doris_ArmPipeline_P0Regression"
elif [[ "${comment_trigger_type}" == "compile" ]]; then
trigger_pipelines="Doris_DorisCompile_Compile"
elif [[ "${comment_trigger_type}" == "p0" ]]; then
trigger_pipelines="Doris_DorisRegression_P0Regression"
elif [[ "${comment_trigger_type}" == "p1" ]]; then
trigger_pipelines="Doris_DorisRegression_P1Regression"
elif [[ "${comment_trigger_type}" == "feut" ]]; then
trigger_pipelines="Doris_Doris_FeUt"
elif [[ "${comment_trigger_type}" == "beut" ]]; then
trigger_pipelines="Doris_DorisBeUt_BeUt"
elif [[ "${comment_trigger_type}" == "external" ]]; then
trigger_pipelines="Doris_External_Regression"
elif [[ "${comment_trigger_type}" == "clickbench" ]]; then
trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew"
elif [[ "${comment_trigger_type}" == "pipelinex_p0" ]]; then
trigger_pipelines="Doris_DorisRegression_P0RegressionPipelineX"
elif [[ "${comment_trigger_type}" == "arm" ]]; then
trigger_pipelines="Doris_ArmPipeline_P0Regression"
elif [[ "${comment_trigger_type}" == "tpch" ]]; then
trigger_pipelines="Tpch_TpchSf100"
fi
if [ -z "${trigger_pipelines}" ];then
echo "Just a general comment that doesn't match any pipeline rules"
fi
set +x
# set -x
# reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*"
# comment_trigger_type="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)"
# comment_repeat_times="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)"
# if [[ "${comment_trigger_type}" == "buildall" ]]; then
# trigger_pipelines="Doris_Doris_FeUt Doris_DorisBeUt_BeUt Doris_DorisCompile_Compile Doris_Performance_Clickbench_ClickbenchNew Doris_ArmPipeline_P0Regression"
# elif [[ "${comment_trigger_type}" == "compile" ]]; then
# trigger_pipelines="Doris_DorisCompile_Compile"
# elif [[ "${comment_trigger_type}" == "p0" ]]; then
# trigger_pipelines="Doris_DorisRegression_P0Regression"
# elif [[ "${comment_trigger_type}" == "p1" ]]; then
# trigger_pipelines="Doris_DorisRegression_P1Regression"
# elif [[ "${comment_trigger_type}" == "feut" ]]; then
# trigger_pipelines="Doris_Doris_FeUt"
# elif [[ "${comment_trigger_type}" == "beut" ]]; then
# trigger_pipelines="Doris_DorisBeUt_BeUt"
# elif [[ "${comment_trigger_type}" == "external" ]]; then
# trigger_pipelines="Doris_External_Regression"
# elif [[ "${comment_trigger_type}" == "clickbench" ]]; then
# trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew"
# elif [[ "${comment_trigger_type}" == "pipelinex_p0" ]]; then
# trigger_pipelines="Doris_DorisRegression_P0RegressionPipelineX"
# elif [[ "${comment_trigger_type}" == "arm" ]]; then
# trigger_pipelines="Doris_ArmPipeline_P0Regression"
# elif [[ "${comment_trigger_type}" == "tpch" ]]; then
# trigger_pipelines="Tpch_TpchSf100"
# fi
# if [ -z "${trigger_pipelines}" ];then
# echo "Just a general comment that doesn't match any pipeline rules"
# fi
# set +x

echo "need run pipelines: ${trigger_pipelines}"
# echo "need run pipelines: ${trigger_pipelines}"

for pipeline in ${trigger_pipelines}
do
same_build_sign="false"
echo "-----------------------------------------------------------------"
running_builds_command="curl -s -X GET ${teamcity_url}/app/rest/builds\?locator\=buildType\:${pipeline}\,branch:pull/${pull_request_num}\,running:true"
echo "${running_builds_command}"
running_builds_list=$(eval "${running_builds_command}" > running_builds.json && jq -r '.build[].id' running_builds.json)
echo "running_builds_list : ${running_builds_list}"
# for pipeline in ${trigger_pipelines}
# do
# same_build_sign="false"
# echo "-----------------------------------------------------------------"
# running_builds_command="curl -s -X GET ${teamcity_url}/app/rest/builds\?locator\=buildType\:${pipeline}\,branch:pull/${pull_request_num}\,running:true"
# echo "${running_builds_command}"
# running_builds_list=$(eval "${running_builds_command}" > running_builds.json && jq -r '.build[].id' running_builds.json)
# echo "running_builds_list : ${running_builds_list}"

queue_builds_command="curl -s -X GET ${teamcity_url}/app/rest/buildQueue\?locator\=buildType\:${pipeline}"
echo "${queue_builds_command}"
eval "${queue_builds_command}" > queue_builds.json
queue_builds_list=$(cat queue_builds.json | jq ".build[] | select(.branchName == \"pull/${pull_request_num}\") | .id" )
echo "queue builds list: ${queue_builds_list}"
# queue_builds_command="curl -s -X GET ${teamcity_url}/app/rest/buildQueue\?locator\=buildType\:${pipeline}"
# echo "${queue_builds_command}"
# eval "${queue_builds_command}" > queue_builds.json
# queue_builds_list=$(cat queue_builds.json | jq ".build[] | select(.branchName == \"pull/${pull_request_num}\") | .id" )
# echo "queue builds list: ${queue_builds_list}"

merged_list=("${running_builds_list[@]} ${queue_builds_list[@]}")
echo "merged_list : ${merged_list}"
# merged_list=("${running_builds_list[@]} ${queue_builds_list[@]}")
# echo "merged_list : ${merged_list}"

for build in ${merged_list}; do
running_commit_command="curl -s -X GET ${teamcity_url}/app/rest/builds/${build}"
echo "${running_commit_command}"
eval "${running_commit_command}" > build_commit_info.json
running_commit_id=$(jq -r '.revisions.revision[0].version' build_commit_info.json)
running_env_commit_id=$(cat build_commit_info.json | jq ".properties.property[] | select(.name == \"env.latest_commit_id\") | .value")
# for build in ${merged_list}; do
# running_commit_command="curl -s -X GET ${teamcity_url}/app/rest/builds/${build}"
# echo "${running_commit_command}"
# eval "${running_commit_command}" > build_commit_info.json
# running_commit_id=$(jq -r '.revisions.revision[0].version' build_commit_info.json)
# running_env_commit_id=$(cat build_commit_info.json | jq ".properties.property[] | select(.name == \"env.latest_commit_id\") | .value")

if [[ "_"${latest_commit_id} == "_"${running_commit_id} || _"\"${latest_commit_id}\"" == "_"${running_env_commit_id} ]];then
echo "the same pr_commit build already exist, this build ${build} is running or in queue!"
same_build_sign="true"
break
fi
done
# if [[ "_"${latest_commit_id} == "_"${running_commit_id} || _"\"${latest_commit_id}\"" == "_"${running_env_commit_id} ]];then
# echo "the same pr_commit build already exist, this build ${build} is running or in queue!"
# same_build_sign="true"
# break
# fi
# done

if [ "_""${same_build_sign}" == "_false" ];then
sleep 10s
echo "there is no running build or queue build, so trigger a new !"
echo "comment_repeat_times: ${comment_repeat_times}"
if [[ -n "${comment_repeat_times}" ]]; then
execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}\&name=env.repeat_times\&value=${comment_repeat_times}"
else
execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}"
fi
echo "${execute_command}"
eval "${execute_command}"
echo "-----------------------------------------------------------------"
else
echo "there is same pr commit task in queue,so skip trigger !"
echo "-----------------------------------------------------------------"
fi
done
# if [ "_""${same_build_sign}" == "_false" ];then
# sleep 10s
# echo "there is no running build or queue build, so trigger a new !"
# echo "comment_repeat_times: ${comment_repeat_times}"
# if [[ -n "${comment_repeat_times}" ]]; then
# execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}\&name=env.repeat_times\&value=${comment_repeat_times}"
# else
# execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}"
# fi
# echo "${execute_command}"
# eval "${execute_command}"
# echo "-----------------------------------------------------------------"
# else
# echo "there is same pr commit task in queue,so skip trigger !"
# echo "-----------------------------------------------------------------"
# fi
# done


Loading