-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ceph: add basic build of binaries and OCI
In order to meet our supply chain protection standards, we have to build our own version of Ceph to use with Rook. This commit brings in rules_foreign_cc, a set of bazel rules to more easily run cmake builds, and then uses those rules to build Ceph using cmake. Of course, we end up needing to patch both Ceph and rules_foreign_cc to make them work correctly together, and we end up needing to install many more packages into the build chroot.
- Loading branch information
Showing
9 changed files
with
279 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| load("@rules_foreign_cc//tools/build_defs:cmake.bzl", "cmake_external") | ||
| load("//debian:debian.bzl", "debinstall") | ||
| load("//bazel:package.bzl", "homeworld_oci") | ||
| load("//ceph:unpack.bzl", "basename", "unpack_filegroup") | ||
|
|
||
| BINARIES = [ | ||
| "ceph-authtool", | ||
| "ceph-bluestore-tool", | ||
| "ceph-conf", | ||
| "ceph-dencoder", | ||
| "ceph-mgr", | ||
| "ceph-mon", | ||
| "ceph-objectstore-tool", | ||
| "ceph-osd", | ||
| "ceph-syn", | ||
| "rados", | ||
| "radosgw", | ||
| "radosgw-admin", | ||
| "radosgw-es", | ||
| "radosgw-object-expirer", | ||
| "radosgw-token", | ||
| "rbd", | ||
| "rbd-mirror", | ||
| "rbd-nbd", | ||
| "rbd-replay", | ||
| "rbd-replay-prep", | ||
| ] | ||
|
|
||
| LIBRARIES = [ | ||
| "libcephfs.so.2", | ||
| "librados.so.2", | ||
| "libradosstriper.so.1", | ||
| "librbd.so.1", | ||
| "librgw.so.2", | ||
| "librgw_admin_user.so.0", | ||
| "ceph/libceph-common.so.0", | ||
| "ceph/compressor/libceph_lz4.so.2", | ||
| "ceph/compressor/libceph_snappy.so.2", | ||
| "ceph/compressor/libceph_zlib.so.2", | ||
| "ceph/compressor/libceph_zstd.so.2", | ||
| "ceph/crypto/libceph_crypto_isal.so.1", | ||
| "ceph/crypto/libceph_crypto_openssl.so", | ||
| "ceph/erasure-code/libec_clay.so", | ||
| "ceph/erasure-code/libec_isa.so", | ||
| "ceph/erasure-code/libec_jerasure.so", | ||
| "ceph/erasure-code/libec_jerasure_generic.so", | ||
| "ceph/erasure-code/libec_jerasure_sse3.so", | ||
| "ceph/erasure-code/libec_jerasure_sse4.so", | ||
| "ceph/erasure-code/libec_lrc.so", | ||
| "ceph/erasure-code/libec_shec.so", | ||
| "ceph/erasure-code/libec_shec_generic.so", | ||
| "ceph/erasure-code/libec_shec_sse3.so", | ||
| "ceph/erasure-code/libec_shec_sse4.so", | ||
| "rados-classes/libcls_cas.so.1", | ||
| "rados-classes/libcls_hello.so.1", | ||
| "rados-classes/libcls_journal.so.1", | ||
| "rados-classes/libcls_kvs.so.1", | ||
| "rados-classes/libcls_lock.so.1", | ||
| "rados-classes/libcls_log.so.1", | ||
| "rados-classes/libcls_lua.so.1", | ||
| "rados-classes/libcls_numops.so.1", | ||
| "rados-classes/libcls_otp.so.1", | ||
| "rados-classes/libcls_rbd.so.1", | ||
| "rados-classes/libcls_refcount.so.1", | ||
| "rados-classes/libcls_rgw.so.1", | ||
| "rados-classes/libcls_sdk.so.1", | ||
| "rados-classes/libcls_timeindex.so.1", | ||
| "rados-classes/libcls_user.so.1", | ||
| "rados-classes/libcls_version.so.1", | ||
| ] | ||
|
|
||
| cmake_external( | ||
| name = "ceph", | ||
| binaries = BINARIES, | ||
| cache_entries = { | ||
| "WITH_MANPAGE": "off", | ||
| "WITH_PYTHON3": "off", | ||
| "WITH_LTTNG": "off", | ||
| "WITH_EMBEDDED": "off", | ||
| "WITH_TESTS": "off", | ||
| "WITH_CEPHFS": "off", | ||
| # using system boost shaves off about 25% of the build time in my tests | ||
| "WITH_SYSTEM_BOOST": "on", | ||
| "WITH_RADOSGW_BEAST_FRONTEND": "on", | ||
| "WITH_FUSE": "off", | ||
| # disabled because the dashboard requires npm, which requires nodeenv, | ||
| # which requires working pip in a virtualenv, which is not working for | ||
| # some reason relating to DNS resolution. might be able to be fixed at | ||
| # a later time. | ||
| "WITH_MGR_DASHBOARD_FRONTEND": "off", | ||
| }, | ||
| lib_source = "@ceph//:source", | ||
| make_commands = [ | ||
| # we cannot just use -j1, because that will take hours, but bazel and | ||
| # make both trying to run their own multithreading subsystems will | ||
| # cause conflicts, so we need to make sure that not too many make | ||
| # threads are used. and we also need to make sure that we never use | ||
| # less than -j1, even if there's only one processor. | ||
| # | ||
| # so we compute the number of processors available, and divide by two | ||
| # rounding up. | ||
| # | ||
| # if this ever accidentally becomes 'make -j' due to the expr commands | ||
| # failing, then it's going to be terrible and (from experience) | ||
| # probably hang the entire build machine for several minutes before the | ||
| # OOM killer gets fed up with Bazel, but I can't come up with a good | ||
| # way around that, and hopefully this code is correct enough that it | ||
| # never happens. | ||
| "N=\"$(expr '(' 1 + $(cut -f 1 </proc/cpuinfo | grep -E '^processor$' | wc -l) ')' / 2)\"; if [ \"$N\" = '' ]; then make -j1; else make -j$N; fi", | ||
| "make install", | ||
| ], | ||
| shared_libraries = LIBRARIES, | ||
| ) | ||
|
|
||
| unpack_filegroup( | ||
| src = ":ceph", | ||
| names = BINARIES + LIBRARIES, | ||
| ) | ||
|
|
||
| oci_include = { | ||
| ":" + binary: "/usr/bin/" + binary | ||
| for binary in BINARIES | ||
| } | ||
|
|
||
| oci_include.update({ | ||
| ":" + library: "/usr/lib/" + basename(library) | ||
| for library in LIBRARIES | ||
| }) | ||
|
|
||
| homeworld_oci( | ||
| name = "oci", | ||
| bin = oci_include, | ||
| visibility = ["//visibility:public"], | ||
| deps = [ | ||
| ":debian-for-ceph.tgz", | ||
| ], | ||
| ) | ||
|
|
||
| debinstall( | ||
| name = "debian-for-ceph.tgz", | ||
| base = "//debian:debian-mini.tgz", | ||
| packages = [ | ||
| "libaio1", | ||
| "libbabeltrace1", | ||
| "libboost-atomic1.67.0", | ||
| "libboost-chrono1.67.0", | ||
| "libboost-context1.67.0", | ||
| "libboost-coroutine1.67.0", | ||
| "libboost-date-time1.67.0", | ||
| "libboost-iostreams1.67.0", | ||
| "libboost-program-options1.67.0", | ||
| "libboost-python1.67.0", | ||
| "libboost-random1.67.0", | ||
| "libboost-regex1.67.0", | ||
| "libboost-system1.67.0", | ||
| "libboost-thread1.67.0", | ||
| "libcurl4", | ||
| "libexpat1", | ||
| "libgoogle-perftools4", | ||
| "libibverbs1", | ||
| "libleveldb1d", | ||
| "libncurses6", | ||
| "libnss3", | ||
| "liboath0", | ||
| "libpython2.7", | ||
| "librabbitmq4", | ||
| "librdmacm1", | ||
| "libssl1.1", | ||
| ], | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") | ||
| load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") | ||
|
|
||
| VERSION = "14.2.9" | ||
| SHA256 = "349e099292f6e2bbfc3b25d2b114b30a814d47694261ea8a72e1a13d840a707e" | ||
|
|
||
| def ceph_dependencies(): | ||
| rules_foreign_cc_dependencies() | ||
| http_archive( | ||
| name = "ceph", | ||
| url = "https://download.ceph.com/tarballs/ceph_" + VERSION + ".orig.tar.gz", | ||
| sha256 = SHA256, | ||
| patch_cmds = [ | ||
| # remove symlinks that create cycles (which break glob) | ||
| "find -name '.qa' -type l -delete", | ||
| # remove filenames with ":" in them (which are disallowed in filegroups) | ||
| "rm src/test/common/test_blkdev_sys_block/sys/dev/block/8:0 src/test/common/test_blkdev_sys_block/sys/dev/block/9:0", | ||
| # remove filenames with special characters (which break bazel for some reason) | ||
| "rm src/boost/libs/wave/test/testwave/testfiles/utf8-test-*/file.hpp", | ||
| "rmdir src/boost/libs/wave/test/testwave/testfiles/utf8-test-*", | ||
| # error out if any other filenames with ":" in them exist | ||
| "find -name '*:*' | ( ! grep -q . )", | ||
| ], | ||
| strip_prefix = "ceph-" + VERSION + "/", | ||
| build_file_content = """filegroup(name = "source", srcs = glob(["**"]), visibility = ["//visibility:public"])""", | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") | ||
|
|
||
| def ceph_dependencies_early(): | ||
| git_repository( | ||
| name = "rules_foreign_cc", | ||
| remote = "https://github.com/bazelbuild/rules_foreign_cc", | ||
| commit = "ed3db61a55c13da311d875460938c42ee8bbc2a5", | ||
| patches = [ | ||
| # we need this patch so that bazel's -D__DATE__="redacted" CFLAG doesn't become | ||
| # -D__DATE__=redacted, which causes code that uses __DATE__ to break. | ||
| "//ceph:foreign_cc/0001-fix-date-quoting.patch", | ||
| # (see https://github.com/bazelbuild/rules_foreign_cc/issues/239 | ||
| # and https://github.com/bazelbuild/rules_foreign_cc/pull/362) | ||
|
|
||
| # we need this so that we can correctly reference the generated libraries | ||
| "//ceph:foreign_cc/0002-more-output-groups.patch", | ||
| # see https://github.com/bazelbuild/rules_foreign_cc/issues/376 | ||
| ], | ||
| shallow_since = "1574792034 +0100", | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| diff --git tools/build_defs/cmake_script.bzl tools/build_defs/cmake_script.bzl | ||
| index e1b0c13..393368a 100644 | ||
| --- tools/build_defs/cmake_script.bzl | ||
| +++ tools/build_defs/cmake_script.bzl | ||
| @@ -65,7 +65,7 @@ def create_cmake_script( | ||
| if not params.cache.get("CMAKE_RANLIB"): | ||
| params.cache.update({"CMAKE_RANLIB": ""}) | ||
|
|
||
| - set_env_vars = " ".join([key + "=\"" + params.env[key] + "\"" for key in params.env]) | ||
| + set_env_vars = " ".join([key + "='" + params.env[key] + "'" for key in params.env]) | ||
| str_cmake_cache_entries = " ".join(["-D" + key + "=\"" + params.cache[key] + "\"" for key in params.cache]) | ||
| cmake_call = " ".join([ | ||
| set_env_vars, | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| diff --git tools/build_defs/framework.bzl tools/build_defs/framework.bzl | ||
| index df37124..9f12a59 100644 | ||
| --- tools/build_defs/framework.bzl | ||
| +++ tools/build_defs/framework.bzl | ||
| @@ -289,7 +289,7 @@ def cc_external_rule_impl(ctx, attrs): | ||
| lib_dir_name = attrs.out_lib_dir, | ||
| include_dir_name = attrs.out_include_dir, | ||
| ) | ||
| - output_groups = _declare_output_groups(installdir_copy.file, outputs.out_binary_files) | ||
| + output_groups = _declare_output_groups(installdir_copy.file, outputs.declared_outputs) | ||
| wrapped_files = [ | ||
| wrapped_outputs.script_file, | ||
| wrapped_outputs.log_file, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| def basename(name): | ||
| name_fragments = name.split("/") | ||
| return name_fragments[-1] | ||
|
|
||
| # for unpacking results from foreign_cc cmake builds | ||
| def unpack_filegroup(names, src, visibility = None): | ||
| for name in names: | ||
| native.filegroup( | ||
| name = name, | ||
| srcs = [src], | ||
| output_group = basename(name), | ||
| visibility = visibility, | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters