diff --git a/.github/scripts/sql-correctness/current_correctness.txt b/.github/scripts/sql-correctness/current_correctness.txt new file mode 100644 index 00000000000..b54871a4ae4 --- /dev/null +++ b/.github/scripts/sql-correctness/current_correctness.txt @@ -0,0 +1 @@ +99.997743 \ No newline at end of file diff --git a/.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh b/.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh index 1c1160ce0d1..8fa604df282 100755 --- a/.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh +++ b/.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh @@ -3,7 +3,7 @@ set -e if [ "$#" -lt 5 ]; then - echo "Usage: ./get-dolt-correctness-job-json.sh " + echo "Usage: ./get-dolt-correctness-job-json.sh " exit 1 fi @@ -14,6 +14,7 @@ actorprefix="$4" format="$5" nomsBinFormat="$6" issueNumber="$7" +regressComp="$8" precision="6" @@ -25,6 +26,12 @@ if [ -n "$issueNumber" ]; then issueNumber="\"--issue-number=$issueNumber\"," fi +regressPrec="" +if [ -n "$regressComp" ]; then + regressComp="\"--regress-compare=$regressComp\"," + regressPrec="\"--regress-precision=$precision\"," +fi + resultCountQuery="select version, result, count(*) as total from results where result != 'skipped' group by result;" testCountQuery="select version, count(*) as total_tests from results where result != 'skipped';" correctnessQuery="select ROUND(100.0 * (cast(ok_results.total as decimal) / (cast(all_results.total as decimal) + .000001)), $precision) as correctness_percentage from (select count(*) as total from results where result = 'ok') as ok_results join (select count(*) as total from results where result != 'skipped') as all_results" @@ -72,6 +79,8 @@ echo ' "--version='$version'", '"$nomsBinFormat"' '"$issueNumber"' + '"$regressComp"' + '"$regressPrec"' "--bucket=sql-correctness-github-actions-results", "--region=us-west-2", "--results-dir='$timeprefix'", diff --git a/.github/scripts/sql-correctness/run-correctness.sh b/.github/scripts/sql-correctness/run-correctness.sh index 96624c568ff..ddd767988d6 100755 --- a/.github/scripts/sql-correctness/run-correctness.sh +++ b/.github/scripts/sql-correctness/run-correctness.sh @@ -32,6 +32,13 @@ if [ -z "$MODE" ]; then exit 1 fi +if [ -n "$PR_NUMBER" ]; then + if [ -z "$REGRESS_COMP" ]; then + echo "Must set REGRESS_COMP for PR correctness comparisons" + exit 1 + fi +fi + # use first 8 characters of VERSION to differentiate # jobs short=${VERSION:0:8} @@ -71,7 +78,8 @@ source \ "$actorprefix" \ "$format" \ "$NOMS_BIN_FORMAT" \ - "$issuenumber" > job.json + "$issuenumber" \ + "$REGRESS_COMP" > job.json # delete existing job with same name if this is a pr job if [ -n "$PR_NUMBER" ]; then diff --git a/.github/workflows/k8s-sql-correctness.yaml b/.github/workflows/k8s-sql-correctness.yaml index 69dec05e421..1a5a063fca1 100644 --- a/.github/workflows/k8s-sql-correctness.yaml +++ b/.github/workflows/k8s-sql-correctness.yaml @@ -39,6 +39,7 @@ jobs: - name: Create SQL Correctness K8s Job run: ./.github/scripts/sql-correctness/run-correctness.sh env: + REGRESS_COMP: ${{ github.event.client_payload.regress_comp }} PR_NUMBER: ${{ github.event.client_payload.issue_number }} VERSION: ${{ github.event.client_payload.version }} MODE: ${{ github.event.client_payload.mode }} diff --git a/.github/workflows/pull-report-test.yaml b/.github/workflows/pull-report-test.yaml new file mode 100644 index 00000000000..c9f1a6115a7 --- /dev/null +++ b/.github/workflows/pull-report-test.yaml @@ -0,0 +1,55 @@ +on: + push: + branches: + - 'db/compare' + +jobs: + report-pull-request: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.2.0 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Remove Passing Labels if regression detected + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { ACTOR, ISSUE_NUMBER, GITHUB_WORKSPACE } = process.env; + const issue_number = parseInt(ISSUE_NUMBER, 10); + const { owner, repo } = context.repo; + try { + const res = await github.rest.issues.listLabelsOnIssue({ + issue_number, + owner, + repo, + }); + + if (res.data) { + const labels = res.data; + + for (const label of labels) { + if (label.name === LABEL) { + console.log("found label: ", LABEL) + await github.rest.issues.removeLabel({ + issue_number, + owner, + repo, + name: label.name, + }); + } + } + } + + } catch(e) { + console.error(e) + } + env: + ACTOR: ${{ github.event.client_payload.actor }} + ISSUE_NUMBER: '7309' + LABEL: 'correctness_approved' diff --git a/.github/workflows/pull-report.yaml b/.github/workflows/pull-report.yaml index 983e9fa46ef..83643df12cd 100644 --- a/.github/workflows/pull-report.yaml +++ b/.github/workflows/pull-report.yaml @@ -48,3 +48,27 @@ jobs: ACTOR: ${{ github.event.client_payload.actor }} ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }} FORMAT: ${{ github.event.client_payload.noms_bin_format }} + - name: Remove Passing Labels if regression detected + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { ACTOR, ISSUE_NUMBER, GITHUB_WORKSPACE } = process.env; + const issue_number = parseInt(ISSUE_NUMBER, 10); + const { owner, repo } = context.repo; + try { + const labels = github.rest.issues.createComment({ + issue_number, + owner, + repo, + body: `@${ACTOR} ${FORMAT}\n ${data}` + }); + } catch(e) { + console.error(e) + } + env: + ACTOR: ${{ github.event.client_payload.actor }} + ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }} + # if this is a regression + # first remove all correctness_approved labels from the pr + # if this diff --git a/.github/workflows/sql-regressions.yaml b/.github/workflows/sql-regressions.yaml index 85f4010d48a..87393d3d1c8 100644 --- a/.github/workflows/sql-regressions.yaml +++ b/.github/workflows/sql-regressions.yaml @@ -9,9 +9,20 @@ jobs: name: Set Version and Actor runs-on: ubuntu-22.04 outputs: + regress_comp: ${{ steps.regress-comp.outputs.regress_comp }} version: ${{ steps.set-vars.outputs.version }} actor: ${{ steps.set-vars.outputs.actor }} steps: + - name: Checkout PR HEAD REF + uses: actions/checkout@v3 + with: + ref: ${{ github.base_ref }} + - name: Get Current Correctness + id: regress-comp + working-directory: ./.github/scripts/sql-correctness + run: | + out=$(cat current_correctness.txt) + echo "regress_comp=$out" >> $GITHUB_OUTPUT - name: Checkout PR HEAD REF uses: actions/checkout@v3 with: @@ -41,4 +52,4 @@ jobs: with: token: ${{ secrets.REPO_ACCESS_TOKEN }} event-type: sql-correctness - client-payload: '{"issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", "version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh"}' + client-payload: '{"issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", "regress_comp": "${{ needs.set-version-actor.outputs.regress_comp }}", "version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh"}'