CI: Run a single job on Julia nightly (but skip the incremental=false tests on Julia 1.12)

.github/workflows/ci.nightly.yml

@@ -0,0 +1,62 @@
+name: Nightly CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - 'master'
+      - 'release-*'
+    tags: '*'
+  merge_group: # GitHub Merge Queue
+
+concurrency:
+  # Skip intermediate builds: all builds except for builds on the `master` branch
+  # Cancel intermediate builds: only pull request builds
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/master' || github.run_number }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  test-nightly:
+    timeout-minutes: 90
+    runs-on: ${{ matrix.github-runner }}
+    strategy:
+      max-parallel: 5 # leave space for other runs in the JuliaLang org, given these tests are long
+      fail-fast: false
+      matrix:
+        include:
+          # 1. We intentionally put nightly CI into a separate YAML
+          #    file (`ci.nightly.yml` versus the main `ci.yml` file).
+          #    This allows us to enable and disable the nightly CI
+          #    workflow from the GitHub web UI, without having any
+          #    impact on the main CI.
+          # 2. We intentionally only run one CI job on Julia nightly,
+          #    in order to conserve CI resources.
+          # 3. This CI only runs on Julia nightly. It doesn't run on
+          #    Julia pre-releases. For CI on Julia pre-releases, see
+          #    the main CI workflow file.
+          - julia-version: 'nightly'
+            julia-wordsize: '64'
+            github-runner: ubuntu-latest
+            coverage: false
+    steps:
+      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+      - uses: julia-actions/setup-julia@9b79636afcfb07ab02c256cede01fe2db6ba808c # v2.6.0
+        with:
+          version: ${{ matrix.julia-version }}
+          # If `julia-wordsize` is 32, then we set `arch` to `x86`, because we know that
+          # 32-bit builds of Julia are only available for x86.
+          #
+          # If `julia-wordsize` is 64, then we set `arch` to `${{ runner.arch }}`, which
+          # GitHub will automatically expand to the correct value (`x86_64` or `aarch64`)
+          # based on the architecture of the underlying GitHub Runner (virtual machine).
+          arch: ${{ github.ref == '32' && 'x86' || runner.arch }}
+      - uses: julia-actions/cache@824243901fb567ccb490b0d0e2483ccecde46834 # v2.0.5
+      - uses: julia-actions/julia-runtest@d0c4f093badade621cd041bba567d1e832480ac2 # v1.10.0
+        with:
+          coverage: ${{ matrix.coverage }}
+      - uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38 # v1.2.2
+      - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
+        with:
+          file: lcov.info
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

test/runtests.jl

@@ -18,6 +18,7 @@ const is_slow_ci = is_ci && Sys.ARCH == :aarch64
 const is_julia_1_6 = VERSION.major == 1 && VERSION.minor == 6
 const is_julia_1_9 = VERSION.major == 1 && VERSION.minor == 9
 const is_julia_1_11 = VERSION.major == 1 && VERSION.minor == 11
+const is_julia_1_12 = VERSION.major == 1 && VERSION.minor == 12
 
 if is_ci
     @show Sys.ARCH
@@ -27,7 +28,14 @@ if is_slow_ci
     @warn "This is \"slow CI\" (`is_ci && Sys.ARCH == :aarch64`). Some tests will be skipped or modified." Sys.ARCH
 end
 
-if any([is_julia_1_6, is_julia_1_9, is_julia_1_11])
+const some_tests_skipped = [
+    is_julia_1_6,
+    is_julia_1_9,
+    is_julia_1_11,
+    is_julia_1_12,
+]
+
+if any(some_tests_skipped)
     @warn "This is Julia $(VERSION.major).$(VERSION.minor). Some tests will be skipped or modified." VERSION
 end
 
@@ -79,10 +87,12 @@ end
     app_compiled_dir = joinpath(tmp, "MyAppCompiled")
     @testset for incremental in (is_slow_ci ? (false,) : (true, false))
         if incremental == false
-            if is_julia_1_11
-                # On Julia 1.11, `incremental=false` is currently broken: https://github.com/JuliaLang/PackageCompiler.jl/issues/976
-                # So, for now, we skip the `incremental=false` tests on Julia 1.11
-                @warn "This is Julia 1.11; skipping incremental=false test due to known bug: https://github.com/JuliaLang/PackageCompiler.jl/issues/976"
+            if is_julia_1_11 || is_julia_1_12
+                # On Julia 1.11 and 1.12, `incremental=false` is currently broken.
+                # 1.11: https://github.com/JuliaLang/PackageCompiler.jl/issues/976
+                # 1.12: No GitHub issue yet.
+                # So, for now, we skip the `incremental=false` tests on Julia 1.11 and 1.12
+                @warn "This is Julia $(VERSION.major).$(VERSION.minor); skipping incremental=false test due to known bug: #976 (for 1.11), issue TODO (for 1.12)"
                 @test_skip false
                 continue
             end
@@ -186,30 +196,52 @@ end
     end # testset
 
     if !is_slow_ci
-        # Test library creation
-        lib_source_dir = joinpath(@__DIR__, "..", "examples/MyLib")
-        lib_target_dir = joinpath(tmp, "MyLibCompiled")
-
-        incremental = false
-        filter = true
-        lib_name = "inc"
-
-        tmp_lib_src_dir = joinpath(tmp, "MyLib")
-        cp(lib_source_dir, tmp_lib_src_dir)
-        create_library(tmp_lib_src_dir, lib_target_dir; incremental=incremental, force=true, filter_stdlibs=filter,
-                    precompile_execution_file=joinpath(lib_source_dir, "build", "generate_precompile.jl"),
-                    precompile_statements_file=joinpath(lib_source_dir, "build", "additional_precompile.jl"),
-                    lib_name=lib_name, version=v"1.0.0")
-        rm(tmp_lib_src_dir; recursive=true)
+        if is_julia_1_12
+            # On Julia 1.12, `incremental=false` is currently broken when doing `create_library()`.
+            # 1.12: No GitHub issue yet.
+            # So, for now, we skip the `incremental=false` tests on Julia 1.12 when doing `create_library()`.
+            @warn "This is Julia $(VERSION.major).$(VERSION.minor); skipping incremental=false test when doing `create_library()` due to known bug: issue TODO (for 1.12)"
+            @test_skip false
+        else
+            # Test library creation
+            lib_source_dir = joinpath(@__DIR__, "..", "examples/MyLib")
+            lib_target_dir = joinpath(tmp, "MyLibCompiled")
+
+            # This is why we have to skip this test on 1.12:
+            incremental = false
+
+            filter = true
+            lib_name = "inc"
+
+            tmp_lib_src_dir = joinpath(tmp, "MyLib")
+            cp(lib_source_dir, tmp_lib_src_dir)
+            create_library(tmp_lib_src_dir, lib_target_dir; incremental=incremental, force=true, filter_stdlibs=filter,
+                        precompile_execution_file=joinpath(lib_source_dir, "build", "generate_precompile.jl"),
+                        precompile_statements_file=joinpath(lib_source_dir, "build", "additional_precompile.jl"),
+                        lib_name=lib_name, version=v"1.0.0")
+            rm(tmp_lib_src_dir; recursive=true)
+        end
     end
 
     # Test creating an empty sysimage
     if !is_slow_ci
-        tmp = mktempdir()
-        sysimage_path = joinpath(tmp, "empty." * Libdl.dlext)
-        foreach(x -> touch(joinpath(tmp, x)), ["Project.toml", "Manifest.toml"])
-        create_sysimage(String[]; sysimage_path=sysimage_path, incremental=false, filter_stdlibs=true, project=tmp)
-        hello = read(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'print("hello, world")'`, String)
-        @test hello == "hello, world"
+        if is_julia_1_12
+            # On Julia 1.12, `incremental=false` is currently broken when doing `create_library()`.
+            # 1.12: No GitHub issue yet.
+            # So, for now, we skip the `incremental=false` tests on Julia 1.12 when doing `create_library()`.
+            @warn "This is Julia $(VERSION.major).$(VERSION.minor); skipping incremental=false test when doing `create_library()` due to known bug: issue TODO (for 1.12)"
+            @test_skip false
+        else
+            tmp = mktempdir()
+            sysimage_path = joinpath(tmp, "empty." * Libdl.dlext)
+            foreach(x -> touch(joinpath(tmp, x)), ["Project.toml", "Manifest.toml"])
+
+            # This is why we need to skip this test on 1.12:
+            incremental=false
+
+            create_sysimage(String[]; sysimage_path=sysimage_path, incremental=incremental, filter_stdlibs=true, project=tmp)
+            hello = read(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'print("hello, world")'`, String)
+            @test hello == "hello, world"
+        end
     end
 end