From fd008f0d8069abb8bba74412da8fd320d0754604 Mon Sep 17 00:00:00 2001 From: Fraser Tweedale Date: Tue, 6 Sep 2022 12:05:32 +1000 Subject: [PATCH] Implement GitHub matrix job limit check GitHub has a limit of 256 jobs. Validate that the size of the matrix does not exceed this limit. --- fixtures/matrix-extra-toomany.args | 2 + fixtures/matrix-extra-toomany.bash | 520 ++++++++++++++++++++++++++ fixtures/matrix-extra-toomany.github | 2 + fixtures/matrix-extra-toomany.project | 1 + fixtures/matrix-extra-toomany.travis | 298 +++++++++++++++ src/HaskellCI/GitHub.hs | 13 + test/Tests.hs | 1 + 7 files changed, 837 insertions(+) create mode 100644 fixtures/matrix-extra-toomany.args create mode 100644 fixtures/matrix-extra-toomany.bash create mode 100644 fixtures/matrix-extra-toomany.github create mode 100644 fixtures/matrix-extra-toomany.project create mode 100644 fixtures/matrix-extra-toomany.travis diff --git a/fixtures/matrix-extra-toomany.args b/fixtures/matrix-extra-toomany.args new file mode 100644 index 00000000..6b69595d --- /dev/null +++ b/fixtures/matrix-extra-toomany.args @@ -0,0 +1,2 @@ +--matrix-extra=abc:1,2,3 +--matrix-extra=foo:bar,baz diff --git a/fixtures/matrix-extra-toomany.bash b/fixtures/matrix-extra-toomany.bash new file mode 100644 index 00000000..a2982a09 --- /dev/null +++ b/fixtures/matrix-extra-toomany.bash @@ -0,0 +1,520 @@ +# SUCCESS +# *INFO* Generating Bash script for testing for GHC versions: 7.0.1 7.0.2 7.0.3 7.0.4 7.2.1 7.2.2 7.4.1 7.4.2 7.6.1 7.6.2 7.6.3 7.8.1 7.8.2 7.8.3 7.8.4 7.10.1 7.10.2 7.10.3 8.0.1 8.0.2 8.2.1 8.2.2 8.4.1 8.4.2 8.4.3 8.4.4 8.6.1 8.6.2 8.6.3 8.6.4 8.6.5 8.8.1 8.8.2 8.8.3 8.8.4 8.10.1 8.10.2 8.10.3 8.10.4 8.10.5 8.10.6 8.10.7 9.0.1 9.0.2 9.2.1 9.2.2 9.2.3 9.2.4 9.4.1 9.4.2 ghcjs-8.4 +#!/bin/bash +# shellcheck disable=SC2086,SC2016,SC2046 +# REGENDATA ["--matrix-extra=abc:1,2,3","--matrix-extra=foo:bar,baz","bash","matrix-extra-toomany.project"] + +set -o pipefail + +# Mode +############################################################################## + +if [ "$1" = "indocker" ]; then + INDOCKER=true + shift +else + INDOCKER=false +fi + +# Run configuration +############################################################################## + +CFG_CABAL_STORE_CACHE="" +CFG_CABAL_REPO_CACHE="" +CFG_JOBS="9.4.2 9.4.1 9.2.4 9.2.3 9.2.2 9.2.1 9.0.2 9.0.1 8.10.7 8.10.6 8.10.5 8.10.4 8.10.3 8.10.2 8.10.1 8.8.4 8.8.3 8.8.2 8.8.1 8.6.5 8.6.4 8.6.3 8.6.2 8.6.1 8.4.4 8.4.3 8.4.2 8.4.1 8.2.2 8.2.1 8.0.2 8.0.1 7.10.3 7.10.2 7.10.1 7.8.4 7.8.3 7.8.2 7.8.1 7.6.3 7.6.2 7.6.1 7.4.2 7.4.1 7.2.2 7.2.1 7.0.4 7.0.3 7.0.2 7.0.1" +CFG_CABAL_UPDATE=false + +SCRIPT_NAME=$(basename "$0") +START_TIME="$(date +'%s')" + +XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config} + +# Job configuration +############################################################################## + +GHC_VERSION="non-existing" +CABAL_VERSION=3.2 +HEADHACKAGE=false + +# Locale +############################################################################## + +export LC_ALL=C.UTF-8 + +# Utilities +############################################################################## + +SGR_RED='\033[1;31m' +SGR_GREEN='\033[1;32m' +SGR_BLUE='\033[1;34m' +SGR_CYAN='\033[1;96m' +SGR_RESET='\033[0m' # No Color + +put_info() { + printf "$SGR_CYAN%s$SGR_RESET\n" "### $*" +} + +put_error() { + printf "$SGR_RED%s$SGR_RESET\n" "!!! $*" +} + +run_cmd() { + local PRETTYCMD="$*" + local PROMPT + if $INDOCKER; then + PROMPT="$(pwd) >>>" + else + PROMPT=">>>" + fi + + printf "$SGR_BLUE%s %s$SGR_RESET\n" "$PROMPT" "$PRETTYCMD" + + local start_time end_time cmd_duration total_duration + start_time=$(date +'%s') + + "$@" + local RET=$? + + end_time=$(date +'%s') + cmd_duration=$((end_time - start_time)) + total_duration=$((end_time - START_TIME)) + + cmd_min=$((cmd_duration / 60)) + cmd_sec=$((cmd_duration % 60)) + + total_min=$((total_duration / 60)) + total_sec=$((total_duration % 60)) + + if [ $RET -eq 0 ]; then + printf "$SGR_GREEN%s$SGR_RESET (%dm%02ds; %dm%02ds)\n" "<<< $PRETTYCMD" "$cmd_min" "$cmd_sec" "$total_min" "$total_sec" + else + printf "$SGR_RED%s$SGR_RESET\n" "!!! $PRETTYCMD" + exit 1 + fi +} + +run_cmd_if() { + local COND=$1 + shift + + if [ $COND -eq 1 ]; then + run_cmd "$@" + else + local PRETTYCMD="$*" + local PROMPT + PROMPT="$(pwd) (skipping) >>>" + + printf "$SGR_BLUE%s %s$SGR_RESET\n" "$PROMPT" "$PRETTYCMD" + fi +} + +run_cmd_unchecked() { + local PRETTYCMD="$*" + local PROMPT + if $INDOCKER; then + PROMPT="$(pwd) >>>" + else + PROMPT=">>>" + fi + + printf "$SGR_BLUE%s %s$SGR_RESET\n" "$PROMPT" "$PRETTYCMD" + + local start_time end_time cmd_duration total_duration cmd_min cmd_sec total_min total_sec + start_time=$(date +'%s') + + "$@" + + end_time=$(date +'%s') + cmd_duration=$((end_time - start_time)) + total_duration=$((end_time - START_TIME)) + + cmd_min=$((cmd_duration / 60)) + cmd_sec=$((cmd_duration % 60)) + + total_min=$((total_duration / 60)) + total_sec=$((total_duration % 60)) + + printf "$SGR_GREEN%s$SGR_RESET (%dm%02ds; %dm%02ds)\n" "<<< $PRETTYCMD" "$cmd_min" "$cmd_sec" "$total_min" "$total_sec" +} + +change_dir() { + local DIR=$1 + if [ -d "$DIR" ]; then + printf "$SGR_BLUE%s$SGR_RESET\n" "change directory to $DIR" + cd "$DIR" || exit 1 + else + printf "$SGR_RED%s$SGR_RESET\n" "!!! cd $DIR" + exit 1 + fi +} + +change_dir_if() { + local COND=$1 + local DIR=$2 + + if [ $COND -ne 0 ]; then + change_dir "$DIR" + fi +} + +echo_to() { + local DEST=$1 + local CONTENTS=$2 + + echo "$CONTENTS" >> "$DEST" +} + +echo_if_to() { + local COND=$1 + local DEST=$2 + local CONTENTS=$3 + + if [ $COND -ne 0 ]; then + echo_to "$DEST" "$CONTENTS" + fi +} + +install_cabalplan() { + put_info "installing cabal-plan" + + if [ ! -e $CABAL_REPOCACHE/downloads/cabal-plan ]; then + curl -L https://github.com/haskell-hvr/cabal-plan/releases/download/v0.6.2.0/cabal-plan-0.6.2.0-x86_64-linux.xz > /tmp/cabal-plan.xz || exit 1 + (cd /tmp && echo "de73600b1836d3f55e32d80385acc055fd97f60eaa0ab68a755302685f5d81bc cabal-plan.xz" | sha256sum -c -)|| exit 1 + mkdir -p $CABAL_REPOCACHE/downloads + xz -d < /tmp/cabal-plan.xz > $CABAL_REPOCACHE/downloads/cabal-plan || exit 1 + chmod a+x $CABAL_REPOCACHE/downloads/cabal-plan || exit 1 + fi + + mkdir -p $CABAL_DIR/bin || exit 1 + ln -s $CABAL_REPOCACHE/downloads/cabal-plan $CABAL_DIR/bin/cabal-plan || exit 1 +} + +# Help +############################################################################## + +show_usage() { +cat < $BUILDDIR/cabal/config <= 80200)) cabal.project "package splitmix" +echo_if_to $((GHCJSARITH || ! GHCJSARITH && HCNUMVER >= 80200)) cabal.project " ghc-options: -Werror=missing-methods" +cat >> cabal.project <> cabal.project.local +run_cmd cat cabal.project +run_cmd cat cabal.project.local + +# dump install plan +put_info "dump install plan" +run_cmd $CABAL v2-build $ARG_COMPILER $ARG_TESTS $ARG_BENCH --dry-run all +run_cmd cabal-plan + +# install dependencies +put_info "install dependencies" +run_cmd $CABAL v2-build $ARG_COMPILER --disable-tests --disable-benchmarks --dependencies-only -j all +run_cmd $CABAL v2-build $ARG_COMPILER $ARG_TESTS $ARG_BENCH --dependencies-only -j all + +# build w/o tests +put_info "build w/o tests" +run_cmd $CABAL v2-build $ARG_COMPILER --disable-tests --disable-benchmarks all + +# build +put_info "build" +run_cmd $CABAL v2-build $ARG_COMPILER $ARG_TESTS $ARG_BENCH all + +# tests +put_info "tests" +run_cmd_if $((! GHCJSARITH)) $CABAL v2-test $ARG_COMPILER $ARG_TESTS $ARG_BENCH all --test-show-details=direct + +# cabal check +put_info "cabal check" +change_dir "${PKGDIR_splitmix}" +run_cmd ${CABAL} -vnormal check +change_dir "$BUILDDIR" + +# haddock +put_info "haddock" +run_cmd_if $((! GHCJSARITH)) $CABAL v2-haddock --haddock-all $ARG_COMPILER --with-haddock $HADDOCK $ARG_TESTS $ARG_BENCH all + +# unconstrained build +put_info "unconstrained build" +run_cmd rm -f cabal.project.local +run_cmd $CABAL v2-build $ARG_COMPILER --disable-tests --disable-benchmarks all + + +# Done +run_cmd echo OK diff --git a/fixtures/matrix-extra-toomany.github b/fixtures/matrix-extra-toomany.github new file mode 100644 index 00000000..acab2e75 --- /dev/null +++ b/fixtures/matrix-extra-toomany.github @@ -0,0 +1,2 @@ +# FAILURE +# *ERROR* The matrix has 306 jobs, exceeding the limit of 256. diff --git a/fixtures/matrix-extra-toomany.project b/fixtures/matrix-extra-toomany.project new file mode 100644 index 00000000..b1f6a74e --- /dev/null +++ b/fixtures/matrix-extra-toomany.project @@ -0,0 +1 @@ +packages: splitmix diff --git a/fixtures/matrix-extra-toomany.travis b/fixtures/matrix-extra-toomany.travis new file mode 100644 index 00000000..c70073f4 --- /dev/null +++ b/fixtures/matrix-extra-toomany.travis @@ -0,0 +1,298 @@ +# SUCCESS +# *INFO* Generating Travis-CI config for testing for GHC versions: 7.0.1 7.0.2 7.0.3 7.0.4 7.2.1 7.2.2 7.4.1 7.4.2 7.6.1 7.6.2 7.6.3 7.8.1 7.8.2 7.8.3 7.8.4 7.10.1 7.10.2 7.10.3 8.0.1 8.0.2 8.2.1 8.2.2 8.4.1 8.4.2 8.4.3 8.4.4 8.6.1 8.6.2 8.6.3 8.6.4 8.6.5 8.8.1 8.8.2 8.8.3 8.8.4 8.10.1 8.10.2 8.10.3 8.10.4 8.10.5 8.10.6 8.10.7 9.0.1 9.0.2 9.2.1 9.2.2 9.2.3 9.2.4 9.4.1 9.4.2 ghcjs-8.4 +# This Travis job script has been generated by a script via +# +# haskell-ci '--matrix-extra=abc:1,2,3' '--matrix-extra=foo:bar,baz' 'travis' 'matrix-extra-toomany.project' +# +# To regenerate the script (for example after adjusting tested-with) run +# +# haskell-ci regenerate +# +# For more information, see https://github.com/haskell-CI/haskell-ci +# +version: ~> 1.0 +language: c +os: linux +dist: bionic +git: + # whether to recursively clone submodules + submodules: false +cache: + directories: + - $HOME/.cabal/packages + - $HOME/.cabal/store + - $HOME/.hlint +before_cache: + - rm -fv $CABALHOME/packages/hackage.haskell.org/build-reports.log + # remove files that are regenerated by 'cabal update' + - rm -fv $CABALHOME/packages/hackage.haskell.org/00-index.* + - rm -fv $CABALHOME/packages/hackage.haskell.org/*.json + - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.cache + - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar + - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar.idx + - rm -rfv $CABALHOME/packages/head.hackage +jobs: + include: + - compiler: ghcjs-8.4 + addons: {"apt":{"packages":["ghcjs-8.4","cabal-install-3.4","ghc-8.4.4","nodejs"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"},{"sourceline":"deb http://ppa.launchpad.net/hvr/ghcjs/ubuntu bionic main"},{"key_url":"https://deb.nodesource.com/gpgkey/nodesource.gpg.key","sourceline":"deb https://deb.nodesource.com/node_10.x bionic main"}]}} + os: linux + - compiler: ghc-9.4.2 + addons: {"apt":{"packages":["ghc-9.4.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-9.4.1 + addons: {"apt":{"packages":["ghc-9.4.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-9.2.4 + addons: {"apt":{"packages":["ghc-9.2.4","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-9.2.3 + addons: {"apt":{"packages":["ghc-9.2.3","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-9.2.2 + addons: {"apt":{"packages":["ghc-9.2.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-9.2.1 + addons: {"apt":{"packages":["ghc-9.2.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-9.0.2 + addons: {"apt":{"packages":["ghc-9.0.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-9.0.1 + addons: {"apt":{"packages":["ghc-9.0.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.10.7 + addons: {"apt":{"packages":["ghc-8.10.7","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.10.6 + addons: {"apt":{"packages":["ghc-8.10.6","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.10.5 + addons: {"apt":{"packages":["ghc-8.10.5","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.10.4 + addons: {"apt":{"packages":["ghc-8.10.4","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.10.3 + addons: {"apt":{"packages":["ghc-8.10.3","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.10.2 + addons: {"apt":{"packages":["ghc-8.10.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.10.1 + addons: {"apt":{"packages":["ghc-8.10.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.8.4 + addons: {"apt":{"packages":["ghc-8.8.4","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.8.3 + addons: {"apt":{"packages":["ghc-8.8.3","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.8.2 + addons: {"apt":{"packages":["ghc-8.8.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.8.1 + addons: {"apt":{"packages":["ghc-8.8.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.6.5 + addons: {"apt":{"packages":["ghc-8.6.5","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.6.4 + addons: {"apt":{"packages":["ghc-8.6.4","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.6.3 + addons: {"apt":{"packages":["ghc-8.6.3","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.6.2 + addons: {"apt":{"packages":["ghc-8.6.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.6.1 + addons: {"apt":{"packages":["ghc-8.6.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.4.4 + addons: {"apt":{"packages":["ghc-8.4.4","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.4.3 + addons: {"apt":{"packages":["ghc-8.4.3","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.4.2 + addons: {"apt":{"packages":["ghc-8.4.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.4.1 + addons: {"apt":{"packages":["ghc-8.4.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.2.2 + addons: {"apt":{"packages":["ghc-8.2.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.2.1 + addons: {"apt":{"packages":["ghc-8.2.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.0.2 + addons: {"apt":{"packages":["ghc-8.0.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-8.0.1 + addons: {"apt":{"packages":["ghc-8.0.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.10.3 + addons: {"apt":{"packages":["ghc-7.10.3","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.10.2 + addons: {"apt":{"packages":["ghc-7.10.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.10.1 + addons: {"apt":{"packages":["ghc-7.10.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.8.4 + addons: {"apt":{"packages":["ghc-7.8.4","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.8.3 + addons: {"apt":{"packages":["ghc-7.8.3","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.8.2 + addons: {"apt":{"packages":["ghc-7.8.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.8.1 + addons: {"apt":{"packages":["ghc-7.8.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.6.3 + addons: {"apt":{"packages":["ghc-7.6.3","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.6.2 + addons: {"apt":{"packages":["ghc-7.6.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.6.1 + addons: {"apt":{"packages":["ghc-7.6.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.4.2 + addons: {"apt":{"packages":["ghc-7.4.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.4.1 + addons: {"apt":{"packages":["ghc-7.4.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.2.2 + addons: {"apt":{"packages":["ghc-7.2.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.2.1 + addons: {"apt":{"packages":["ghc-7.2.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.0.4 + addons: {"apt":{"packages":["ghc-7.0.4","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.0.3 + addons: {"apt":{"packages":["ghc-7.0.3","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.0.2 + addons: {"apt":{"packages":["ghc-7.0.2","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux + - compiler: ghc-7.0.1 + addons: {"apt":{"packages":["ghc-7.0.1","cabal-install-3.6"],"sources":[{"key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286","sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu bionic main"}]}} + os: linux +before_install: + - | + if echo $CC | grep -q ghcjs; then + GHCJS=true; GHCJSARITH=1; + else + GHCJS=false; GHCJSARITH=0; + fi + - HC=$(echo "/opt/$CC/bin/ghc" | sed 's/-/\//') + - WITHCOMPILER="-w $HC" + - if [ $((GHCJSARITH)) -ne 0 ] ; then HC=${HC}js ; fi + - if [ $((GHCJSARITH)) -ne 0 ] ; then WITHCOMPILER="--ghcjs ${WITHCOMPILER}js" ; fi + - HADDOCK=$(echo "/opt/$CC/bin/haddock" | sed 's/-/\//') + - if [ $((GHCJSARITH)) -ne 0 ] ; then PATH="/opt/ghc/8.4.4/bin:$PATH" ; fi + - HCPKG="$HC-pkg" + - unset CC + - CABAL=/opt/ghc/bin/cabal + - CABALHOME=$HOME/.cabal + - export PATH="$CABALHOME/bin:$PATH" + - TOP=$(pwd) + - "HCNUMVER=$(${HC} --numeric-version|perl -ne '/^(\\d+)\\.(\\d+)\\.(\\d+)(\\.(\\d+))?$/; print(10000 * $1 + 100 * $2 + ($3 == 0 ? $5 != 1 : $3))')" + - echo $HCNUMVER + - CABAL="$CABAL -vnormal+nowrap" + - set -o pipefail + - TEST=--enable-tests + - BENCH=--enable-benchmarks + - HEADHACKAGE=false + - rm -f $CABALHOME/config + - | + echo "verbose: normal +nowrap +markoutput" >> $CABALHOME/config + echo "remote-build-reporting: anonymous" >> $CABALHOME/config + echo "write-ghc-environment-files: never" >> $CABALHOME/config + echo "remote-repo-cache: $CABALHOME/packages" >> $CABALHOME/config + echo "logs-dir: $CABALHOME/logs" >> $CABALHOME/config + echo "world-file: $CABALHOME/world" >> $CABALHOME/config + echo "extra-prog-path: $CABALHOME/bin" >> $CABALHOME/config + echo "symlink-bindir: $CABALHOME/bin" >> $CABALHOME/config + echo "installdir: $CABALHOME/bin" >> $CABALHOME/config + echo "build-summary: $CABALHOME/logs/build.log" >> $CABALHOME/config + echo "store-dir: $CABALHOME/store" >> $CABALHOME/config + echo "install-dirs user" >> $CABALHOME/config + echo " prefix: $CABALHOME" >> $CABALHOME/config + echo "repository hackage.haskell.org" >> $CABALHOME/config + echo " url: http://hackage.haskell.org/" >> $CABALHOME/config +install: + - ${CABAL} --version + - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]" + - node --version + - echo $GHCJS + - | + echo "program-default-options" >> $CABALHOME/config + echo " ghc-options: $GHCJOBS +RTS -M6G -RTS" >> $CABALHOME/config + - cat $CABALHOME/config + - rm -fv cabal.project cabal.project.local cabal.project.freeze + - travis_retry ${CABAL} v2-update -v + # Generate cabal.project + - rm -rf cabal.project cabal.project.local cabal.project.freeze + - touch cabal.project + - | + echo "packages: splitmix" >> cabal.project + - if [ $((GHCJSARITH || ! GHCJSARITH && HCNUMVER >= 80200)) -ne 0 ] ; then echo 'package splitmix' >> cabal.project ; fi + - "if [ $((GHCJSARITH || ! GHCJSARITH && HCNUMVER >= 80200)) -ne 0 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi" + - "" + - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(splitmix)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done" + - cat cabal.project || true + - cat cabal.project.local || true + - if [ -f "splitmix/configure.ac" ]; then (cd "splitmix" && autoreconf -i); fi + - ${CABAL} v2-freeze $WITHCOMPILER ${TEST} ${BENCH} + - "cat cabal.project.freeze | sed -E 's/^(constraints: *| *)//' | sed 's/any.//'" + - rm cabal.project.freeze + - travis_wait 40 ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} --dep -j2 all + - travis_wait 40 ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --dep -j2 all +script: + - DISTDIR=$(mktemp -d /tmp/dist-test.XXXX) + # Packaging... + - ${CABAL} v2-sdist all + # Unpacking... + - mv dist-newstyle/sdist/*.tar.gz ${DISTDIR}/ + - cd ${DISTDIR} || false + - find . -maxdepth 1 -type f -name '*.tar.gz' -exec tar -xvf '{}' \; + - find . -maxdepth 1 -type f -name '*.tar.gz' -exec rm '{}' \; + - PKGDIR_splitmix="$(find . -maxdepth 1 -type d -regex '.*/splitmix-[0-9.]*')" + # Generate cabal.project + - rm -rf cabal.project cabal.project.local cabal.project.freeze + - touch cabal.project + - | + echo "packages: ${PKGDIR_splitmix}" >> cabal.project + - if [ $((GHCJSARITH || ! GHCJSARITH && HCNUMVER >= 80200)) -ne 0 ] ; then echo 'package splitmix' >> cabal.project ; fi + - "if [ $((GHCJSARITH || ! GHCJSARITH && HCNUMVER >= 80200)) -ne 0 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi" + - "" + - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(splitmix)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done" + - cat cabal.project || true + - cat cabal.project.local || true + # Building... + # this builds all libraries and executables (without tests/benchmarks) + - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all + # Building with tests and benchmarks... + # build & run tests, build benchmarks + - ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} all --write-ghc-environment-files=always + # Testing... + - if [ $((! GHCJSARITH)) -ne 0 ] ; then ${CABAL} v2-test $WITHCOMPILER ${TEST} ${BENCH} all --test-show-details=direct ; fi + # cabal check... + - (cd ${PKGDIR_splitmix} && ${CABAL} -vnormal check) + # haddock... + - if [ $((! GHCJSARITH)) -ne 0 ] ; then ${CABAL} v2-haddock --haddock-all $WITHCOMPILER --with-haddock $HADDOCK ${TEST} ${BENCH} all ; fi + # Building without installed constraints for packages in global-db... + - rm -f cabal.project.local + - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all + +# REGENDATA ["--matrix-extra=abc:1,2,3","--matrix-extra=foo:bar,baz","travis","matrix-extra-toomany.project"] +# EOF diff --git a/src/HaskellCI/GitHub.hs b/src/HaskellCI/GitHub.hs index bd05937e..939a7dec 100644 --- a/src/HaskellCI/GitHub.hs +++ b/src/HaskellCI/GitHub.hs @@ -113,6 +113,13 @@ makeGitHub _argv config@Config {..} gitconfig prj jobs@JobVersions {..} = do [ "Using submodules on the GitHub Actions backend requires" , "Ubuntu 20.04 (Focal Fossa) or later." ] + when (length matrix > maxMatrixJobs) $ + throwErr $ ValidationError $ unwords + [ "The matrix has" + , show (length matrix) + , "jobs, exceeding the limit of" + , show maxMatrixJobs ++ "." + ] steps <- sequence $ buildList $ do -- This have to be first, since the packages we install depend on @@ -911,3 +918,9 @@ parseGitHubRepo t = -- runners support. ghcRunsOnVer :: String ghcRunsOnVer = "ubuntu-20.04" + +-- | GitHub has a limit of 256 jobs. +-- See https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#using-a-matrix-strategy +-- +maxMatrixJobs :: Int +maxMatrixJobs = 256 diff --git a/test/Tests.hs b/test/Tests.hs index ae950bb6..ce63bfa1 100644 --- a/test/Tests.hs +++ b/test/Tests.hs @@ -31,6 +31,7 @@ main = do , fixtureGoldenTest "travis-patch" , fixtureGoldenTest "enabled-jobs" , fixtureGoldenTest "matrix-extra" + , fixtureGoldenTest "matrix-extra-toomany" , testGroup "copy-fields" [ fixtureGoldenTest "copy-fields-all" , fixtureGoldenTest "copy-fields-some"