From 3732ca2f005889a3b636a6dbc848a41ec4336da7 Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Mon, 6 Jan 2025 15:15:49 -0800 Subject: [PATCH] add github actions caching Summary: X-link: https://github.com/facebook/sapling/pull/1005 Speed up github builds with github actions CI caching based on the cache-key mechanism already present in getdeps. Cached fizz windows build completed in 22% of original time, linux build in 54% of the original time, and mac in 72% of original time. This saves a lot of waiting around for PR builds on OSS changes as windows signal is usually the lagging one. Speed ups will vary across the different Meta OSS projects. Windows benefits most from caching dependencies as we don't use system packages there at all, linux next as its default runner is slower than mac, and then finaly mac (the strongest hardware of the default github runners). Github allows [up to 10GB cache per repo](https://github.blog/changelog/2021-11-23-github-actions-cache-size-is-now-increased-to-10gb-per-repository/), expiring data over capacity. You can see the size of the caches generated in the [caches view of githhub actions UX](https://github.com/ahornby/fizz/actions/caches?query=sort%3Asize-desc), looks like our usage is pretty small so far. More background: Github actions caching decides its own compression format (currently it prefers zstd), but they are free to change that in future, hence no .zstd suffix or similar on the cache keys. Github actions caches from main are used from feature branches but not vice versa, hence a PR can't pollute cache for the trunk build. This allows us to benefit from caching on main and PRs where dependency versions match. The final item being built, and dependencies inside the same repo as the final are not cached. A different mechanism would be necessary for those (e.g. using a tool like ccache/sccache or caching cargo/buck2 output). This means that when building EdenFS, sapling could not be cached (its in the same repo), but folly could be as we have a key for it based on the folly-rev.txt file. When there is a cache hit the build step is skipped using extension of the conditionals introduced in previous diff D67839708 Reviewed By: bigfootjon Differential Revision: D67839730 fbshipit-source-id: c384a216eb27ccd3f816e3c31b167232bda571a6 --- .github/workflows/rust-shed_linux.yml | 525 ++++++++++++++++++++++++-- build/fbcode_builder/getdeps.py | 56 ++- build/fbcode_builder/getdeps/cargo.py | 12 +- 3 files changed, 552 insertions(+), 41 deletions(-) diff --git a/.github/workflows/rust-shed_linux.yml b/.github/workflows/rust-shed_linux.yml index eba900e6a..2e489e9ce 100644 --- a/.github/workflows/rust-shed_linux.yml +++ b/.github/workflows/rust-shed_linux.yml @@ -132,111 +132,566 @@ jobs: - name: Fetch fb303 if: ${{ steps.paths.outputs.fb303_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fb303 - - name: Build xxhash + - name: Restore xxhash from cache + id: restore_xxhash if: ${{ steps.paths.outputs.xxhash_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.xxhash_INSTALL }} + key: ${{ steps.paths.outputs.xxhash_CACHE_KEY }}-install + - name: Build xxhash + if: ${{ steps.paths.outputs.xxhash_SOURCE && ! steps.restore_xxhash.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xxhash - - name: Build ninja + - name: Save xxhash to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.xxhash_SOURCE && ! steps.restore_xxhash.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.xxhash_INSTALL }} + key: ${{ steps.paths.outputs.xxhash_CACHE_KEY }}-install + - name: Restore ninja from cache + id: restore_ninja if: ${{ steps.paths.outputs.ninja_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.ninja_INSTALL }} + key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install + - name: Build ninja + if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja - - name: Build cmake + - name: Save ninja to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.ninja_INSTALL }} + key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install + - name: Restore cmake from cache + id: restore_cmake if: ${{ steps.paths.outputs.cmake_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.cmake_INSTALL }} + key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install + - name: Build cmake + if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake - - name: Build fmt + - name: Save cmake to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.cmake_INSTALL }} + key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install + - name: Restore fmt from cache + id: restore_fmt if: ${{ steps.paths.outputs.fmt_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.fmt_INSTALL }} + key: ${{ steps.paths.outputs.fmt_CACHE_KEY }}-install + - name: Build fmt + if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt - - name: Build googletest + - name: Save fmt to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.fmt_INSTALL }} + key: ${{ steps.paths.outputs.fmt_CACHE_KEY }}-install + - name: Restore googletest from cache + id: restore_googletest if: ${{ steps.paths.outputs.googletest_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.googletest_INSTALL }} + key: ${{ steps.paths.outputs.googletest_CACHE_KEY }}-install + - name: Build googletest + if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest - - name: Build zstd + - name: Save googletest to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.googletest_INSTALL }} + key: ${{ steps.paths.outputs.googletest_CACHE_KEY }}-install + - name: Restore zstd from cache + id: restore_zstd if: ${{ steps.paths.outputs.zstd_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.zstd_INSTALL }} + key: ${{ steps.paths.outputs.zstd_CACHE_KEY }}-install + - name: Build zstd + if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd - - name: Build boost + - name: Save zstd to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.zstd_INSTALL }} + key: ${{ steps.paths.outputs.zstd_CACHE_KEY }}-install + - name: Restore boost from cache + id: restore_boost if: ${{ steps.paths.outputs.boost_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.boost_INSTALL }} + key: ${{ steps.paths.outputs.boost_CACHE_KEY }}-install + - name: Build boost + if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost - - name: Build double-conversion + - name: Save boost to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.boost_INSTALL }} + key: ${{ steps.paths.outputs.boost_CACHE_KEY }}-install + - name: Restore double-conversion from cache + id: restore_double-conversion if: ${{ steps.paths.outputs.double-conversion_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.double-conversion_INSTALL }} + key: ${{ steps.paths.outputs.double-conversion_CACHE_KEY }}-install + - name: Build double-conversion + if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion - - name: Build fast_float + - name: Save double-conversion to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.double-conversion_INSTALL }} + key: ${{ steps.paths.outputs.double-conversion_CACHE_KEY }}-install + - name: Restore fast_float from cache + id: restore_fast_float if: ${{ steps.paths.outputs.fast_float_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.fast_float_INSTALL }} + key: ${{ steps.paths.outputs.fast_float_CACHE_KEY }}-install + - name: Build fast_float + if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fast_float - - name: Build gflags + - name: Save fast_float to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.fast_float_INSTALL }} + key: ${{ steps.paths.outputs.fast_float_CACHE_KEY }}-install + - name: Restore gflags from cache + id: restore_gflags if: ${{ steps.paths.outputs.gflags_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.gflags_INSTALL }} + key: ${{ steps.paths.outputs.gflags_CACHE_KEY }}-install + - name: Build gflags + if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags - - name: Build glog + - name: Save gflags to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.gflags_INSTALL }} + key: ${{ steps.paths.outputs.gflags_CACHE_KEY }}-install + - name: Restore glog from cache + id: restore_glog if: ${{ steps.paths.outputs.glog_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.glog_INSTALL }} + key: ${{ steps.paths.outputs.glog_CACHE_KEY }}-install + - name: Build glog + if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog - - name: Build libdwarf + - name: Save glog to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.glog_INSTALL }} + key: ${{ steps.paths.outputs.glog_CACHE_KEY }}-install + - name: Restore libdwarf from cache + id: restore_libdwarf if: ${{ steps.paths.outputs.libdwarf_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.libdwarf_INSTALL }} + key: ${{ steps.paths.outputs.libdwarf_CACHE_KEY }}-install + - name: Build libdwarf + if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libdwarf - - name: Build libevent + - name: Save libdwarf to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.libdwarf_INSTALL }} + key: ${{ steps.paths.outputs.libdwarf_CACHE_KEY }}-install + - name: Restore libevent from cache + id: restore_libevent if: ${{ steps.paths.outputs.libevent_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.libevent_INSTALL }} + key: ${{ steps.paths.outputs.libevent_CACHE_KEY }}-install + - name: Build libevent + if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent - - name: Build lz4 + - name: Save libevent to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.libevent_INSTALL }} + key: ${{ steps.paths.outputs.libevent_CACHE_KEY }}-install + - name: Restore lz4 from cache + id: restore_lz4 if: ${{ steps.paths.outputs.lz4_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.lz4_INSTALL }} + key: ${{ steps.paths.outputs.lz4_CACHE_KEY }}-install + - name: Build lz4 + if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests lz4 - - name: Build snappy + - name: Save lz4 to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.lz4_INSTALL }} + key: ${{ steps.paths.outputs.lz4_CACHE_KEY }}-install + - name: Restore snappy from cache + id: restore_snappy if: ${{ steps.paths.outputs.snappy_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.snappy_INSTALL }} + key: ${{ steps.paths.outputs.snappy_CACHE_KEY }}-install + - name: Build snappy + if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy - - name: Build zlib + - name: Save snappy to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.snappy_INSTALL }} + key: ${{ steps.paths.outputs.snappy_CACHE_KEY }}-install + - name: Restore zlib from cache + id: restore_zlib if: ${{ steps.paths.outputs.zlib_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.zlib_INSTALL }} + key: ${{ steps.paths.outputs.zlib_CACHE_KEY }}-install + - name: Build zlib + if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zlib - - name: Build bz2 + - name: Save zlib to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.zlib_INSTALL }} + key: ${{ steps.paths.outputs.zlib_CACHE_KEY }}-install + - name: Restore bz2 from cache + id: restore_bz2 if: ${{ steps.paths.outputs.bz2_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.bz2_INSTALL }} + key: ${{ steps.paths.outputs.bz2_CACHE_KEY }}-install + - name: Build bz2 + if: ${{ steps.paths.outputs.bz2_SOURCE && ! steps.restore_bz2.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests bz2 - - name: Build openssl + - name: Save bz2 to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.bz2_SOURCE && ! steps.restore_bz2.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.bz2_INSTALL }} + key: ${{ steps.paths.outputs.bz2_CACHE_KEY }}-install + - name: Restore openssl from cache + id: restore_openssl if: ${{ steps.paths.outputs.openssl_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.openssl_INSTALL }} + key: ${{ steps.paths.outputs.openssl_CACHE_KEY }}-install + - name: Build openssl + if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests openssl - - name: Build liboqs + - name: Save openssl to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.openssl_INSTALL }} + key: ${{ steps.paths.outputs.openssl_CACHE_KEY }}-install + - name: Restore liboqs from cache + id: restore_liboqs if: ${{ steps.paths.outputs.liboqs_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.liboqs_INSTALL }} + key: ${{ steps.paths.outputs.liboqs_CACHE_KEY }}-install + - name: Build liboqs + if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests liboqs - - name: Build autoconf + - name: Save liboqs to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.liboqs_INSTALL }} + key: ${{ steps.paths.outputs.liboqs_CACHE_KEY }}-install + - name: Restore autoconf from cache + id: restore_autoconf if: ${{ steps.paths.outputs.autoconf_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.autoconf_INSTALL }} + key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install + - name: Build autoconf + if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf - - name: Build automake + - name: Save autoconf to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.autoconf_INSTALL }} + key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install + - name: Restore automake from cache + id: restore_automake if: ${{ steps.paths.outputs.automake_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.automake_INSTALL }} + key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install + - name: Build automake + if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake - - name: Build libtool + - name: Save automake to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.automake_INSTALL }} + key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install + - name: Restore libtool from cache + id: restore_libtool if: ${{ steps.paths.outputs.libtool_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.libtool_INSTALL }} + key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install + - name: Build libtool + if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool - - name: Build libsodium + - name: Save libtool to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.libtool_INSTALL }} + key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install + - name: Restore libsodium from cache + id: restore_libsodium if: ${{ steps.paths.outputs.libsodium_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.libsodium_INSTALL }} + key: ${{ steps.paths.outputs.libsodium_CACHE_KEY }}-install + - name: Build libsodium + if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium - - name: Build libiberty + - name: Save libsodium to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.libsodium_INSTALL }} + key: ${{ steps.paths.outputs.libsodium_CACHE_KEY }}-install + - name: Restore libiberty from cache + id: restore_libiberty if: ${{ steps.paths.outputs.libiberty_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.libiberty_INSTALL }} + key: ${{ steps.paths.outputs.libiberty_CACHE_KEY }}-install + - name: Build libiberty + if: ${{ steps.paths.outputs.libiberty_SOURCE && ! steps.restore_libiberty.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libiberty - - name: Build libunwind + - name: Save libiberty to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.libiberty_SOURCE && ! steps.restore_libiberty.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.libiberty_INSTALL }} + key: ${{ steps.paths.outputs.libiberty_CACHE_KEY }}-install + - name: Restore libunwind from cache + id: restore_libunwind if: ${{ steps.paths.outputs.libunwind_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.libunwind_INSTALL }} + key: ${{ steps.paths.outputs.libunwind_CACHE_KEY }}-install + - name: Build libunwind + if: ${{ steps.paths.outputs.libunwind_SOURCE && ! steps.restore_libunwind.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libunwind - - name: Build xz + - name: Save libunwind to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.libunwind_SOURCE && ! steps.restore_libunwind.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.libunwind_INSTALL }} + key: ${{ steps.paths.outputs.libunwind_CACHE_KEY }}-install + - name: Restore xz from cache + id: restore_xz if: ${{ steps.paths.outputs.xz_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.xz_INSTALL }} + key: ${{ steps.paths.outputs.xz_CACHE_KEY }}-install + - name: Build xz + if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xz - - name: Build folly + - name: Save xz to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.xz_INSTALL }} + key: ${{ steps.paths.outputs.xz_CACHE_KEY }}-install + - name: Restore folly from cache + id: restore_folly if: ${{ steps.paths.outputs.folly_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.folly_INSTALL }} + key: ${{ steps.paths.outputs.folly_CACHE_KEY }}-install + - name: Build folly + if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly - - name: Build fizz + - name: Save folly to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.folly_INSTALL }} + key: ${{ steps.paths.outputs.folly_CACHE_KEY }}-install + - name: Restore fizz from cache + id: restore_fizz if: ${{ steps.paths.outputs.fizz_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.fizz_INSTALL }} + key: ${{ steps.paths.outputs.fizz_CACHE_KEY }}-install + - name: Build fizz + if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz - - name: Build mvfst + - name: Save fizz to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.fizz_INSTALL }} + key: ${{ steps.paths.outputs.fizz_CACHE_KEY }}-install + - name: Restore mvfst from cache + id: restore_mvfst if: ${{ steps.paths.outputs.mvfst_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.mvfst_INSTALL }} + key: ${{ steps.paths.outputs.mvfst_CACHE_KEY }}-install + - name: Build mvfst + if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests mvfst - - name: Build libffi + - name: Save mvfst to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.mvfst_INSTALL }} + key: ${{ steps.paths.outputs.mvfst_CACHE_KEY }}-install + - name: Restore libffi from cache + id: restore_libffi if: ${{ steps.paths.outputs.libffi_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.libffi_INSTALL }} + key: ${{ steps.paths.outputs.libffi_CACHE_KEY }}-install + - name: Build libffi + if: ${{ steps.paths.outputs.libffi_SOURCE && ! steps.restore_libffi.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libffi - - name: Build ncurses + - name: Save libffi to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.libffi_SOURCE && ! steps.restore_libffi.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.libffi_INSTALL }} + key: ${{ steps.paths.outputs.libffi_CACHE_KEY }}-install + - name: Restore ncurses from cache + id: restore_ncurses if: ${{ steps.paths.outputs.ncurses_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.ncurses_INSTALL }} + key: ${{ steps.paths.outputs.ncurses_CACHE_KEY }}-install + - name: Build ncurses + if: ${{ steps.paths.outputs.ncurses_SOURCE && ! steps.restore_ncurses.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ncurses - - name: Build python + - name: Save ncurses to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.ncurses_SOURCE && ! steps.restore_ncurses.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.ncurses_INSTALL }} + key: ${{ steps.paths.outputs.ncurses_CACHE_KEY }}-install + - name: Restore python from cache + id: restore_python if: ${{ steps.paths.outputs.python_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.python_INSTALL }} + key: ${{ steps.paths.outputs.python_CACHE_KEY }}-install + - name: Build python + if: ${{ steps.paths.outputs.python_SOURCE && ! steps.restore_python.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python - - name: Build wangle + - name: Save python to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.python_SOURCE && ! steps.restore_python.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.python_INSTALL }} + key: ${{ steps.paths.outputs.python_CACHE_KEY }}-install + - name: Restore wangle from cache + id: restore_wangle if: ${{ steps.paths.outputs.wangle_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.wangle_INSTALL }} + key: ${{ steps.paths.outputs.wangle_CACHE_KEY }}-install + - name: Build wangle + if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests wangle - - name: Build fbthrift + - name: Save wangle to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.wangle_INSTALL }} + key: ${{ steps.paths.outputs.wangle_CACHE_KEY }}-install + - name: Restore fbthrift from cache + id: restore_fbthrift if: ${{ steps.paths.outputs.fbthrift_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.fbthrift_INSTALL }} + key: ${{ steps.paths.outputs.fbthrift_CACHE_KEY }}-install + - name: Build fbthrift + if: ${{ steps.paths.outputs.fbthrift_SOURCE && ! steps.restore_fbthrift.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fbthrift - - name: Build fb303 + - name: Save fbthrift to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.fbthrift_SOURCE && ! steps.restore_fbthrift.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.fbthrift_INSTALL }} + key: ${{ steps.paths.outputs.fbthrift_CACHE_KEY }}-install + - name: Restore fb303 from cache + id: restore_fb303 if: ${{ steps.paths.outputs.fb303_SOURCE }} + uses: actions/cache/restore@v4 + with: + path: ${{ steps.paths.outputs.fb303_INSTALL }} + key: ${{ steps.paths.outputs.fb303_CACHE_KEY }}-install + - name: Build fb303 + if: ${{ steps.paths.outputs.fb303_SOURCE && ! steps.restore_fb303.outputs.cache-hit }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fb303 + - name: Save fb303 to cache + uses: actions/cache/save@v4 + if: ${{ steps.paths.outputs.fb303_SOURCE && ! steps.restore_fb303.outputs.cache-hit }} + with: + path: ${{ steps.paths.outputs.fb303_INSTALL }} + key: ${{ steps.paths.outputs.fb303_CACHE_KEY }}-install - name: Build rust-shed run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. rust-shed --project-install-prefix rust-shed:/usr/local - name: Copy artifacts diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 138e729e7..5efd2efc6 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -245,16 +245,16 @@ def __init__(self, cache, loader, m): self.loader = loader self.cache = cache - self.cache_file_name = "-".join( + self.cache_key = "-".join( ( m.name, self.ctx.get("os"), self.ctx.get("distro") or "none", self.ctx.get("distro_vers") or "none", self.project_hash, - "buildcache.tgz", ) ) + self.cache_file_name = self.cache_key + "-buildcache.tgz" def is_cacheable(self): """We only cache third party projects""" @@ -560,6 +560,7 @@ def run_project_cmd(self, args, loader, manifest): else: manifests = [manifest] + cache = cache_module.create_cache() for m in manifests: fetcher = loader.create_fetcher(m) if isinstance(fetcher, SystemPackageFetcher): @@ -569,6 +570,10 @@ def run_project_cmd(self, args, loader, manifest): continue src_dir = fetcher.get_src_dir() print(f"{m.name}_SOURCE={src_dir}") + inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) + print(f"{m.name}_INSTALL={inst_dir}") + cached_project = CachedProject(cache, loader, m) + print(f"{m.name}_CACHE_KEY={cached_project.cache_key}") def setup_project_cmd_parser(self, parser): parser.add_argument( @@ -1235,16 +1240,50 @@ def write_job_for_platform(self, platform, args): # noqa: C901 src_dir_arg = "--src-dir=. " has_same_repo_dep = True - out.write(" - name: Build %s\n" % m.name) - if not src_dir_arg: - # only run the step if needed + if args.use_build_cache and not src_dir_arg: + out.write(f" - name: Restore {m.name} from cache\n") + out.write(f" id: restore_{m.name}\n") + # only need to restore if would build it out.write( f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n" ) + out.write(" uses: actions/cache/restore@v4\n") + out.write(" with:\n") + out.write( + f" path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n" + ) + out.write( + f" key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n" + ) + + out.write(" - name: Build %s\n" % m.name) + if not src_dir_arg: + if args.use_build_cache: + out.write( + f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n" + ) + else: + out.write( + f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n" + ) out.write( f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n" ) + if args.use_build_cache and not src_dir_arg: + out.write(f" - name: Save {m.name} to cache\n") + out.write(" uses: actions/cache/save@v4\n") + out.write( + f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n" + ) + out.write(" with:\n") + out.write( + f" path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n" + ) + out.write( + f" key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n" + ) + out.write(" - name: Build %s\n" % manifest.name) project_prefix = "" @@ -1363,6 +1402,13 @@ def setup_project_cmd_parser(self, parser): action="store", default=None, ) + parser.add_argument( + "--no-build-cache", + action="store_false", + default=True, + dest="use_build_cache", + help="Do not attempt to use the build cache.", + ) def get_arg_var_name(args): diff --git a/build/fbcode_builder/getdeps/cargo.py b/build/fbcode_builder/getdeps/cargo.py index 5bb2ada85..c1d6dae91 100644 --- a/build/fbcode_builder/getdeps/cargo.py +++ b/build/fbcode_builder/getdeps/cargo.py @@ -82,6 +82,14 @@ def recreate_dir(self, src, dst) -> None: shutil.rmtree(dst) simple_copytree(src, dst) + def recreate_linked_dir(self, src, dst) -> None: + if os.path.isdir(dst): + if os.path.islink(dst): + os.remove(dst) + elif os.path.isdir(dst): + shutil.rmtree(dst) + os.symlink(src, dst) + def cargo_config_file(self): build_source_dir = self.build_dir if self.cargo_config_file_subdir: @@ -191,7 +199,9 @@ def _build(self, reconfigure) -> None: ], ) - self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source")) + self.recreate_linked_dir( + build_source_dir, os.path.join(self.inst_dir, "source") + ) def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> None: if test_filter: