Merge pull request #795 from srvrco/release-and-package-workflow

Release and package workflow, plus misc fixes for the automated tests
srvrco · Feb 23, 2023 · 02fe6cf · 02fe6cf
2 parents b70ffea + 73b21eb
commit 02fe6cf
Show file tree

Hide file tree

Showing 7 changed files with 155 additions and 75 deletions.
diff --git a/.github/workflows/release-and-package.yml b/.github/workflows/release-and-package.yml
@@ -1,5 +1,19 @@
-# Hacked together from https://github.com/qemu-riscv-xpack/.github/workflows/build-all.yml
-# and https://github.com/palmetto/palm-cli/blob/develop/.github/workflows/pypi-deploy.yaml
+# ==========================
+# Can test locally using act (https://github.com/nektos/act)
+# ==========================
+# ./bin/act -s GITHUB_TOKEN=<fine-grained-token> --directory runner --workflows "../.github/workflows/" -e ../payloads.json --no-skip-checkout -j deploy
+#
+# where payloads.json is:
+# {
+# 	"inputs": {
+#         "tags": "2.47"
+#     }
+# }
+#
+# ==========================
+# Can debug remotely on github actions instance by uncommenting the 'tmate' section below
+# ==========================
+
 
 name: Deploy getssl
 
@@ -15,15 +29,33 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+      - name: prepare
+        # Keep the outputs persistent outside the docker container to use for the other steps
+        run: |
+          mkdir -p ${{ github.workspace }}/bin
+          mkdir -p ${{ github.workspace }}/debbuild/BUILD
+          mkdir -p ${{ github.workspace }}/debbuild/DEBS/all
+          mkdir -p ${{ github.workspace }}/debbuild/SDEBS
+          mkdir -p ${{ github.workspace }}/debbuild/SOURCES
+          mkdir -p ${{ github.workspace }}/debbuild/SPECS
+          mkdir -p ${{ github.workspace }}/rpmbuild/SOURCES
+          mkdir -p ${{ github.workspace }}/rpmbuild/RPMS/noarch
+          mkdir -p ${{ github.workspace }}/rpmbuild/RPMS/SRPMS
+
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          path: "getssl"
+          path: source
 
       - name: Get version number
         id: get_version
         run: |
-          echo ::set-output name=VERSION::$(bash ./getssl/getssl --version)
+          echo "VERSION=$(bash ${{ github.workspace }}/source/getssl --version)" >> $GITHUB_OUTPUT
+
+      - name: Get release
+        id: get_release
+        run: |
+          echo "RELEASE=$(grep Release source/getssl.spec | awk '{ print $2 }')" >> $GITHUB_OUTPUT
 
       - name: Check version matches tag
         run: |
@@ -35,40 +67,47 @@ jobs:
       - name: build .deb package
         id: build_deb
         run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential devscripts debhelper pax liblocale-gettext-perl
+          sudo apt-get update -qq
+          sudo apt-get install --no-install-recommends -qq -y build-essential devscripts debhelper pax liblocale-gettext-perl wget
           wget https://github.com/debbuild/debbuild/releases/download/22.02.1/debbuild_22.02.1-0ubuntu20.04_all.deb
           sudo dpkg --install debbuild_22.02.1-0ubuntu20.04_all.deb
           # Line 1959 has an extra ")" bracket
-          sudo patch /usr/bin/debbuild < ./getssl/debbuild.patch
-          mkdir -p /root/debbuild/BUILD
-          mkdir -p /root/debbuild/DEBS/all
-          mkdir -p /root/debbuild/SDEBS
-          mkdir -p /root/debbuild/SOURCES
-          mkdir -p /root/debbuild/SPECS
-          tar -czf /root/debbuild/SOURCES/getssl-${{ github.event.inputs.tags }}.tar.gz ./getssl/* --transform "s/getssl\//getssl-${{ github.event.inputs.tags }}\//"
-          tar --append -f /root/getssl-${{ github.event.inputs.tags }}.sdeb /root/debbuild/SOURCES/getssl-${{ github.event.inputs.tags }}.tar.gz --transform 's,\.,SOURCES,'
-          tar --append -f /root/getssl-${{ github.event.inputs.tags }}.sdeb -C ./getssl getssl.crontab getssl.logrotate --transform 's,^,SOURCES/,'
-          tar --append -f /root/getssl-${{ github.event.inputs.tags }}.sdeb -C ./getssl getssl.spec --transform 's,^,SPECS/,'
-          debbuild --install /root/getssl-${{ github.event.inputs.tags }}.sdeb
-          debbuild -vv -ba /root/debbuild/SPECS/getssl.spec
+          sudo chmod +w /usr/bin/debbuild
+          sudo patch /usr/bin/debbuild < ${GITHUB_WORKSPACE}/source/debbuild.patch
+          tar --absolute-names -czf ${GITHUB_WORKSPACE}/getssl-${{ github.event.inputs.tags }}.tar.gz ${GITHUB_WORKSPACE}/source/* --transform "s,${GITHUB_WORKSPACE}/source,getssl-${{ github.event.inputs.tags }},"
+          tar --absolute-names -cf ${GITHUB_WORKSPACE}/debbuild/SDEBS/getssl-${{ github.event.inputs.tags }}.sdeb ${GITHUB_WORKSPACE}/getssl-${{ github.event.inputs.tags }}.tar.gz --transform "s,${GITHUB_WORKSPACE},SOURCES,"
+          tar --append -f ${GITHUB_WORKSPACE}/debbuild/SDEBS/getssl-${{ github.event.inputs.tags }}.sdeb -C ${GITHUB_WORKSPACE}/source getssl.crontab getssl.logrotate --transform 's,^,SOURCES/,'
+          tar --append -f ${GITHUB_WORKSPACE}/debbuild/SDEBS/getssl-${{ github.event.inputs.tags }}.sdeb -C ${GITHUB_WORKSPACE}/source getssl.spec --transform 's,^,SPECS/,'
+          ln -s ${GITHUB_WORKSPACE}/debbuild ${HOME}/debbuild
+          /usr/bin/debbuild -vv --install ${GITHUB_WORKSPACE}/debbuild/SDEBS/getssl-${{ github.event.inputs.tags }}.sdeb
+          /usr/bin/debbuild -vv -ba ${GITHUB_WORKSPACE}/debbuild/SPECS/getssl.spec
+          echo "getssl_deb=${GITHUB_WORKSPACE}/debbuild/DEBS/all/getssl_${{ github.event.inputs.tags }}-${{ steps.get_release.outputs.RELEASE }}_all.deb" >> $GITHUB_OUTPUT
+
+      # *** Uncomment this to debug remotely ***
+      # - name: Setup tmate session
+      #   if: ${{ failure() }}
+      #   uses: mxschmitt/action-tmate@v3
 
       - name: build .rpm package
         id: build_rpm
         if: ${{ success() }}
         uses: addnab/docker-run-action@v3
         with:
           image: rockylinux:8
-          options: -v ${{ github.workspace }}:/root/getssl -e GITHUB_REF=${{ github.ref }}
+          options: -v ${{ github.workspace }}:/root -e GITHUB_REF=${{ github.ref }}
           run: |
-            sudo yum install -y rpm-build make
-            mkdir -p /root/rpmbuild/SOURCES /root/rpmbuild/RPMS/SRPMS /root/rpmbuild/RPMS/noarch
-            tar -czf /root/rpmbuild/SOURCES/getssl-${{ github.event.inputs.tags }}.tar.gz /root/getssl/* --transform "s/root\/getssl\//getssl-${{ github.event.inputs.tags }}\//"
-            cp /root/getssl/getssl.crontab /root/rpmbuild/SOURCES
-            cp /root/getssl/getssl.logrotate /root/rpmbuild/SOURCES
-            rpmbuild -ba /root/getssl/getssl.spec
-            cp /root/rpmbuild/SRPMS/getssl-*.rpm /root/getssl
-            cp /root/rpmbuild/RPMS/noarch/getssl-*.rpm /root/getssl
+            yum install -y rpm-build make
+            tar -czf /root/rpmbuild/SOURCES/getssl-${{ github.event.inputs.tags }}.tar.gz /root/source/* --transform "s/root\/source\//getssl-${{ github.event.inputs.tags }}\//"
+            cp /root/source/getssl.crontab /root/rpmbuild/SOURCES
+            cp /root/source/getssl.logrotate /root/rpmbuild/SOURCES
+            rpmbuild -ba /root/source/getssl.spec
+
+      - name: output .rpm packages
+        id: output_rpm
+        if: ${{ success() }}
+        run: |
+          echo "getssl_rpm=${GITHUB_WORKSPACE}/rpmbuild/RPMS/noarch/getssl-${{ github.event.inputs.tags }}-${{ steps.get_release.outputs.RELEASE }}.noarch.rpm" >> $GITHUB_OUTPUT
+          echo "getssl_srpm=${GITHUB_WORKSPACE}/rpmbuild/SRPMS/getssl-${{ github.event.inputs.tags }}-${{ steps.get_release.outputs.RELEASE }}.src.rpm" >> $GITHUB_OUTPUT
 
       - name: create_release
         id: create_release
@@ -77,12 +116,12 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          tag: ${{ github.ref }}
-          name: Draft Release ${{ github.ref }}
+          tag: ${{ github.event.inputs.tags }}
+          name: Draft Release ${{ github.event.inputs.tags }}
           generateReleaseNotes: true
           draft: true
           prerelease: false
           artifacts: |
-            /root/debbuild/DEBS/all/getssl_${{ github.event.inputs.tags }}-*_all.deb
-            /root/getssl-${{ github.event.inputs.tags }}-*.src.rpm
-            /root/getssl-${{ github.event.inputs.tags }}-*.noarch.rpm
+            ${{ steps.build_deb.outputs.getssl_deb }}
+            ${{ steps.output_rpm.outputs.getssl_rpm }}
+            ${{ steps.output_rpm.outputs.getssl_srpm }}
diff --git a/RELEASE.md b/RELEASE.md
@@ -0,0 +1,32 @@
+# How to do a release of getssl
+
+## Update the version and tag the release
+
+1. git pull
+2. git branch -c release_2_nn
+3. git switch release_2_nn
+4. update VERSION in `getssl` and `getssl.spec`
+5. git commit -m"Update version to v2.nn"
+6. git tag -a v2.nn
+7. git push origin release_2_nn
+8. git push --tags
+
+## The github release-and-package action should:
+
+1. Build the .deb and .rpm packages
+2. create a draft release containing the packages and the release note
+
+## Can test the .deb file using the following steps:
+
+1. Change the status from draft to pre-release
+2. Test that the package can be installed using a cloud instance
+   1. Start an Ubuntu ec2 instance from AWS Console (or Azure or Google Cloud)
+   2. Or use the instant-ec2.sh script from my Github gist to start an Ubuntu ec2 instance
+      1. git clone [email protected]:12c297e0645920c413273c9d15edbc68.git instant-ec2
+      2. ./instant-ec2/instant-ec2.sh
+3. download the deb package
+   `wget https://github.com/srvrco/getssl/releases/download/v2.nn/getssl_2.nn-1_all.deb`
+4. install the deb package
+   `dpkg -i getssl_2.nn-1_all.deb`
+5. Check it's installed correctly
+   `getssl --version`
diff --git a/getssl b/getssl
@@ -568,7 +568,7 @@ check_challenge_completion_dns() { # perform validation via DNS challenge
 
     # add +noidnout if idn-domain so search for domain in results works
     if [[ "${d}" == xn--* || "${d}" == *".xn--"* ]]; then
-      if [[ "$DNS_CHECK_FUNC" == "nslookup" || "$DNS_CHECK_FUNC" == "host" || ("$DNS_CHECK_FUNC" == "dig" && "$DIG_SUPPORTS_NOIDNOUT" == "false") ]]; then
+      if [[ "$DNS_CHECK_FUNC" == "nslookup" || "$DNS_CHECK_FUNC" == "host" || ("$DNS_CHECK_FUNC" == "$HAS_DIG_OR_DRILL" && "$DIG_SUPPORTS_NOIDNOUT" == "false") ]]; then
         info "Info: idn domain but $DNS_CHECK_FUNC doesn't support +noidnout"
       else
         debug "adding +noidnout to DNS_CHECK_OPTIONS"
@@ -1090,12 +1090,12 @@ copy_file_to_location() { # copies a file, using scp, sftp or ftp if required.
         # shellcheck disable=SC2086
         debug curl ${_NOMETER} $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftps://${ftphost}${SFTP_PORT}/${ftpdirn}/"
         # shellcheck disable=SC2086
-        curl ${_NOMETER} $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftps://${ftphost}${SFTP_PORT}/${ftpdirn}/"
+        curl ${_NOMETER} $FTPS_OPTIONS --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftps://${ftphost}${SFTP_PORT}/${ftpdirn}/"
       else
         # shellcheck disable=SC2086
         debug curl ${_NOMETER} $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${SFTP_PORT}/${ftpdirn}/"
         # shellcheck disable=SC2086
-        curl ${_NOMETER} $FTPS_OPTIONS --ftp-ssl --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${SFTP_PORT}/${ftpdirn}/"
+        curl ${_NOMETER} $FTPS_OPTIONS --ftp-ssl-reqd -u "${ftpuser}:${ftppass}" -T "${fromdir}/${fromfile}" "ftp://${ftphost}${SFTP_PORT}/${ftpdirn}/"
       fi
     else
       if ! mkdir -p "$(dirname "$to")" ; then
@@ -1418,6 +1418,8 @@ for d in "${alldomains[@]}"; do
         # get the token and uri from the dns-01 component
         token=$(json_get "$response" "challenges" "type" "dns-01" "token")
         uri=$(json_get "$response" "challenges" "type" "dns-01" "url")
+        # when using pebble this sometimes appears to have a newline which causes problems in send_signed_request
+        uri=$(echo "$uri" | tr -d '\r')
         debug uri "$uri"
       fi
 
@@ -2516,7 +2518,7 @@ send_signed_request() { # Sends a request to the ACME server, signed with your p
 
     code="500"
     loop_limit=5
-    while [[ "$code" -eq 500 ]]; do
+    while [[ "$code" == 5* ]]; do
       if [[ "$outfile" ]] ; then
         $CURL -X POST -H "Content-Type: application/jose+json" --data "$body" "$url" > "$outfile"
         errcode=$?
@@ -2562,13 +2564,13 @@ send_signed_request() { # Sends a request to the ACME server, signed with your p
         fi
       fi
       debug "response status = $response_status"
-      if [[ "$code" -eq 500 ]]; then
-        info "_error on acme server - trying again ...."
+      if [[ "$code" == 5* ]]; then
+        info "_error on acme server - waiting 30s then trying again ...."
         debug "loop_limit = $loop_limit"
-        sleep 5
+        sleep 30
         loop_limit=$((loop_limit - 1))
         if [[ $loop_limit -lt 1 ]]; then
-          error_exit "500 error from ACME server:  $response"
+          error_exit "$code error from ACME server: $response"
         fi
       fi
     done

diff --git a/getssl.spec b/getssl.spec
@@ -18,7 +18,7 @@ Source1:          getssl.crontab
 Source2:          getssl.logrotate
 BuildArch:        noarch
 
-Requires:         bash cronie
+Requires:         bash
 BuildRequires:    bash
 
 %description

diff --git a/test/0-test-usage.bats b/test/0-test-usage.bats
@@ -40,6 +40,9 @@ setup() {
     if [ -n "$STAGING" ]; then
         skip "Using staging server, skipping internal test"
     fi
+
+    # Feb-23 Getting semi-repeatable "can't check for upgrades: ''" errors which are because the limit is being exceeded (re-use of github action ip?)
+    check_github_quota 7
     run ${CODE_DIR}/getssl --upgrade
     refute_output
     assert_success

diff --git a/test/32-test-upgrade.bats b/test/32-test-upgrade.bats
@@ -4,39 +4,6 @@ load '/bats-support/load.bash'
 load '/bats-assert/load.bash'
 load '/getssl/test/test_helper.bash'
 
-LIMIT_API="https://api.github.com/rate_limit"
-
-# Quota generally shouldn't be an issue - except for tests
-# Rate limits are per-IP address
-check_github_quota() {
-  local need remaining reset limits now
-  need="$1"
-  while true ; do
-    limits="$(curl ${_NOMETER:---silent} --user-agent "$CURL_USERAGENT" -H 'Accept: application/vnd.github.v3+json' "$LIMIT_API" | sed -e's/\("[^:]*": *\("[^""]*",\|[^,]*[,}]\)\)/\r\n\1/g' | sed -ne'/"core":/,/}/p')"
-    errcode=$?
-    if [[ $errcode -eq 60 ]]; then
-      error_exit "curl needs updating, your version does not support SNI (multiple SSL domains on a single IP)"
-    elif [[ $errcode -gt 0 ]]; then
-      error_exit "curl error checking releases: $errcode"
-    fi
-    limits="$(sed -e's/^ *//g' <<<"${limits}")"
-    remaining="$(sed -e'/^"remaining": *[0-9]/!d;s/^"remaining": *\([0-9][0-9]*\).*$/\1/' <<<"${limits}")"
-    reset="$(sed -e'/^"reset": *[0-9]/!d;s/^"reset": *\([0-9][0-9]*\).*$/\1/' <<<"${limits}")"
-    if [[ "$remaining" -ge "$need" ]] ; then return 0 ; fi
-    limit="$(sed -e'/^"limit": *[0-9]/!d;s/^"limit": *\([0-9][0-9]*\).*$/\1/' <<<"${limits}")"
-    if [[ "$limit" -lt "$need" ]] ; then
-      error_exit "GitHub API request $need exceeds limit $limit"
-    fi
-    now="$(date +%s)"
-    while [[ "$now" -lt "$reset" ]] ; do
-      info "sleeping $(( "$reset" - "$now" )) seconds for GitHub quota"
-      sleep "$(( "$reset" - "$now" ))"
-      now="$(date +%s)"
-   done
-  done
-}
-
-
 setup_file() {
     if [ -n "$STAGING" ]; then
         echo "Using staging server, skipping internal test" >&3

diff --git a/test/test_helper.bash b/test/test_helper.bash
@@ -1,5 +1,6 @@
 INSTALL_DIR=/root
 CODE_DIR=/getssl
+LIMIT_API="https://api.github.com/rate_limit"
 
 check_certificates()
 {
@@ -8,6 +9,42 @@ check_certificates()
   assert [ -e "${INSTALL_DIR}/.getssl/${GETSSL_CMD_HOST}/${GETSSL_CMD_HOST}.crt" ]
 }
 
+# Quota generally shouldn't be an issue - except for tests
+# Rate limits are per-IP address
+check_github_quota() {
+  local need remaining reset limits now
+  need="$1"
+  echo "# Checking github limits"
+  while true ; do
+    limits="$(curl ${_NOMETER:---silent} --user-agent "srvrco/getssl/github-actions" -H 'Accept: application/vnd.github.v3+json' "$LIMIT_API")"
+    echo "# limits = $limits"
+    errcode=$?
+    if [[ $errcode -eq 60 ]]; then
+      echo "curl needs updating, your version does not support SNI (multiple SSL domains on a single IP)"
+      exit 1
+    elif [[ $errcode -gt 0 ]]; then
+      echo "curl error checking releases: $errcode"
+      exit 1
+    fi
+    remaining="$(jq -r '.resources.core.remaining' <<<"$limits")"
+    echo "# Remaining: $remaining"
+    reset="$(jq -r '.resources.core.reset' <<<"$limits")"
+    if [[ "$remaining" -ge "$need" ]] ; then return 0 ; fi
+    limit="$(jq -r '.resources.core.limit' <<<"$limits")"
+    echo "# Limit: $limit"
+    if [[ "$limit" -lt "$need" ]] ; then
+      echo "GitHub API request $need exceeds limit $limit"
+      exit 1
+    fi
+    now="$(date +%s)"
+    while [[ "$now" -lt "$reset" ]] ; do
+      echo "# sleeping $(( reset - now )) seconds for GitHub quota"
+      sleep "$(( reset - now ))"
+      now="$(date +%s)"
+   done
+  done
+}
+
 # Only nginx > 1.11.0 support dual certificates in a single configuration file
 # https://unix.stackexchange.com/questions/285924/how-to-compare-a-programs-version-in-a-shell-script
 check_nginx() {
@@ -24,7 +61,7 @@ check_nginx() {
 
 check_output_for_errors() {
   refute_output --regexp '[Ff][Aa][Ii][Ll][Ee][Dd]'
-  refute_output --regexp '[^_][Ee][Rr][Rr][Oo][Rr][^:nonce]'
+  refute_output --regexp '[^_][Ee][Rr][Rr][Oo][Rr][^:badNonce]'
   refute_output --regexp '[^_][Ww][Aa][Rr][Nn][Ii][Nn][Gg]'
   refute_line --partial 'command not found'
 }