AMReX-Combustion · marchdf · Oct 4, 2023 · Oct 4, 2023 · Oct 4, 2023 · Oct 4, 2023
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -137,11 +137,27 @@ jobs:
           echo "IGNDELAY_WORKING_DIRECTORY=${{github.workspace}}/PelePhysics-${{matrix.comp}}/Testing/Exec/IgnitionDelay" >> $GITHUB_ENV
           echo "JAC_WORKING_DIRECTORY=${{github.workspace}}/PelePhysics-${{matrix.comp}}/Testing/Exec/Jacobian" >> $GITHUB_ENV
           echo "NPROCS=$(nproc)" >> $GITHUB_ENV
+          echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
+          echo "CCACHE_COMPRESSLEVEL=5" >> $GITHUB_ENV
+          echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
           if [ "${{matrix.comp}}" == 'cuda' ]; then \
             echo "CUDA_HOME=/usr/local/cuda-11.2" >> $GITHUB_ENV; \
             echo "LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV; \
             echo "/usr/local/cuda-11.2/bin" >> $GITHUB_PATH; \
           fi
+      - name: Install Ccache
+        run: |
+          wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz
+          tar xvf ccache-4.8-linux-x86_64.tar.xz
+          sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/
+      - name: Set Up Ccache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/ccache
+          key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.comp}}-git-${{github.sha}}
+          restore-keys: |
+               ccache-${{github.workflow}}-${{github.job}}-${{matrix.comp}}-git-
       - name: Dependencies
         working-directory: ${{env.TRANSPORT_WORKING_DIRECTORY}}
         run: |
@@ -174,39 +190,55 @@ jobs:
           echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json"
           if [ "${{matrix.comp}}" == 'hip' ]; then source /etc/profile.d/rocm.sh; fi;
           if [ "${{matrix.comp}}" == 'sycl' ]; then source /opt/intel/oneapi/setvars.sh || true; fi;
+          set -eu -o pipefail
+          ccache -z
           for TYPE in Constant Simple Sutherland; do \
             printf "\n-------- ${TYPE} --------\n"; \
-            make -j ${{env.NPROCS}} Transport_Model=${TYPE} TINY_PROFILE=TRUE ${{matrix.amrex_build_args}}; \
+            make -j ${{env.NPROCS}} Transport_Model=${TYPE} TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}; \
             if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
               ./Pele2d.${{matrix.comp}}.TPROF.ex inputs.2d_${TYPE}; \
             fi; \
             make realclean; \
             if [ $? -ne 0 ]; then exit 1; fi; \
           done
+      - name: Transport ccache report
+        working-directory: ${{env.TRANSPORT_WORKING_DIRECTORY}}
+        run: |
+          ccache -s
+          du -hs ${HOME}/.cache/ccache
       - name: Test EOS
         working-directory: ${{env.EOS_WORKING_DIRECTORY}}
         run: |
           echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json"
           if [ "${{matrix.comp}}" == 'hip' ]; then source /etc/profile.d/rocm.sh; fi;
           if [ "${{matrix.comp}}" == 'sycl' ]; then source /opt/intel/oneapi/setvars.sh || true; fi;
+          set -eu -o pipefail
+          ccache -z
           for TYPE in Fuego GammaLaw Soave-Redlich-Kwong; do \
             if [ "${TYPE}" == 'Fuego' ] || [ "${TYPE}" == 'Soave-Redlich-Kwong' ]; then CHEMISTRY=LiDryer; else CHEMISTRY=Null; fi; \
             if [ "${TYPE}" == 'Soave-Redlich-Kwong' ]; then TRANSPORT=Simple; else TRANSPORT=Constant; fi; \
             printf "\n-------- ${TYPE} --------\n"; \
-            make -j ${{env.NPROCS}} Eos_Model=${TYPE} Chemistry_Model=${CHEMISTRY} Transport_Model=${TRANSPORT} TINY_PROFILE=TRUE ${{matrix.amrex_build_args}}; \
+            make -j ${{env.NPROCS}} Eos_Model=${TYPE} Chemistry_Model=${CHEMISTRY} Transport_Model=${TRANSPORT} TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}; \
             if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
               ./Pele2d.${{matrix.comp}}.TPROF.ex inputs.2d; \
             fi; \
             make realclean; \
             if [ $? -ne 0 ]; then exit 1; fi; \
           done
+      - name: EOS ccache report
+        working-directory: ${{env.EOS_WORKING_DIRECTORY}}
+        run: |
+          ccache -s
+          du -hs ${HOME}/.cache/ccache
       - name: Test Integration
         working-directory: ${{env.REACT_WORKING_DIRECTORY}}
         run: |
           echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json"
           if [ "${{matrix.comp}}" == 'hip' ]; then source /etc/profile.d/rocm.sh; fi;
           if [ "${{matrix.comp}}" == 'sycl' ]; then source /opt/intel/oneapi/setvars.sh || true; fi;
-          make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=drm19 TINY_PROFILE=TRUE ${{matrix.amrex_build_args}}
+          set -eu -o pipefail
+          ccache -z
+          make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=drm19 TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}
           if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
             for TYPE in Cvode Arkode RK64; do \
               printf "\n-------- ${TYPE} --------\n"; \
@@ -216,7 +248,7 @@ jobs:
             done \
           fi;
           make realclean
-          make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=LiDryer TINY_PROFILE=TRUE ${{matrix.amrex_build_args}}
+          make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=LiDryer TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}
           if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
             for TYPE in Cvode Arkode RK64; do \
               printf "\n-------- ${TYPE} --------\n"; \
@@ -226,30 +258,49 @@ jobs:
             done \
           fi
           make realclean
+      - name: Integration ccache report
+        working-directory: ${{env.REACT_WORKING_DIRECTORY}}
+        run: |
+          ccache -s
+          du -hs ${HOME}/.cache/ccache
       - name: Test Ignition delay
         working-directory: ${{env.IGNDELAY_WORKING_DIRECTORY}}
         run: |
           echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json"
           if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
               python -m pip install --upgrade pip
               pip install numpy
-              make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu TINY_PROFILE=TRUE ${{matrix.amrex_build_args}}
+              set -eu -o pipefail
+              ccache -z
+              make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}
               bash exec_ignDelay.sh
               python check_ignDelay.py
               if [ $? -ne 0 ]; then exit 1; fi; \
           fi;
           make realclean
+      - name: Ignition delay ccache report
+        working-directory: ${{env.IGNDELAY_WORKING_DIRECTORY}}
+        run: |
+          ccache -s
+          du -hs ${HOME}/.cache/ccache
       - name: Test Jacobian
         working-directory: ${{env.JAC_WORKING_DIRECTORY}}
         run: |
           echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json"
           if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
+              set -eu -o pipefail
+              ccache -z
               cd ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/ceptr
               poetry run qssa -f ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/Mechanism/Models/dodecane_lu_qss/skeletal.yaml -n ${{github.workspace}}/Support/Mechanism/Models/dodecane_lu_qss/non_qssa_list.yaml
               poetry run convert -f ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/Mechanism/Models/dodecane_lu_qss/qssa.yaml --qss_format_input ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/Mechanism/Models/dodecane_lu_qss/qssa_input_non_optimized.toml --qss_symbolic_jacobian
               cd ${{env.JAC_WORKING_DIRECTORY}}
-              make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu_qss DEBUG=TRUE TINY_PROFILE=TRUE ${{matrix.amrex_build_args}}
+              make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu_qss DEBUG=TRUE TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}
               ./Pele3d.${{matrix.comp}}.DEBUG.TPROF.ex; \
               if [ $? -ne 0 ]; then exit 1; fi; \
           fi;
           make realclean
+      - name: Jacobian ccache report
+        working-directory: ${{env.JAC_WORKING_DIRECTORY}}
+        run: |
+          ccache -s
+          du -hs ${HOME}/.cache/ccache
diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml
@@ -0,0 +1,37 @@
+name: CleanUpCachePostPR
+
+on:
+  workflow_run:
+    workflows: [PostPR]
+    types:
+      - completed
+
+jobs:
+  CleanUpCcacheCachePostPR:
+    name: Clean Up Ccahe Cache Post PR
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Clean up ccahe
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          gh run download ${{ github.event.workflow_run.id }} -n pr_number
+          pr_number=`cat pr_number.txt`
+          BRANCH=refs/pull/${pr_number}/merge
+
+          # Setting this to not fail the workflow while deleting cache keys.
+          set +e
+
+          keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1)
+          for k in $keys
+          do
+            gh actions-cache delete $k -R $REPO -B $BRANCH --confirm
+          done
diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml
@@ -0,0 +1,60 @@
+name: CleanUpCache
+
+on:
+  workflow_run:
+    workflows: [PelePhysics-CI]
+    types:
+      - completed
+
+jobs:
+  CleanUpCcacheCache:
+    name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }}
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Clean up ccahe
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          # push or pull_request or schedule or ...
+          EVENT=${{ github.event.workflow_run.event }}
+
+          # Triggering workflow run name (e.g., LinuxClang)
+          WORKFLOW_NAME=${{ github.event.workflow_run.name }}
+
+          if [[ $EVENT == "pull_request" ]]; then
+            gh run download ${{ github.event.workflow_run.id }} -n pr_number
+            pr_number=`cat pr_number.txt`
+            BRANCH=refs/pull/${pr_number}/merge
+          else
+            BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }}
+          fi
+
+          # Setting this to not fail the workflow while deleting cache keys.
+          set +e
+
+          # In our cache keys, substring after `-git-` is git hash, substring
+          # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`).
+          # The goal is to keep the last used key of each job and delete all others.
+
+          # something like ccache-LinuxClang-
+          keyprefix=ccache-${WORKFLOW_NAME}-
+
+          cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key $keyprefix | awk -F '-git-' '{print $1}' | sort | uniq)
+
+          # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d".
+          for j in $cached_jobs
+          do
+            old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key ${j}-git- --sort last-used | cut -f 1 | tail -n +2)
+            for k in $old_keys
+            do
+              gh actions-cache delete $k -R $REPO -B $BRANCH --confirm
+            done
+          done
diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml
@@ -0,0 +1,20 @@
+name: PostPR
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v3
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1