ceph: add basic build of binaries and OCI

In order to meet our supply chain protection standards, we have to build our own version of Ceph to use with Rook. This commit brings in rules_foreign_cc, a set of bazel rules to more easily run cmake builds, and then uses those rules to build Ceph using cmake. Of course, we end up needing to patch both Ceph and rules_foreign_cc to make them work correctly together, and we end up needing to install many more packages into the build chroot.
sipb · Feb 29, 2020 · 16435fd · 16435fd
1 parent 8c51394
commit 16435fd
Show file tree

Hide file tree

Showing 9 changed files with 255 additions and 1 deletion.
diff --git a/build-chroot/packages.list b/build-chroot/packages.list
@@ -1,3 +1,16 @@
+
+# ceph
+bc btrfs-progs cmake cpio cryptsetup-bin cython cython3 gdisk git gperf jq libaio-dev libbabeltrace-ctf-dev
+libbabeltrace-dev libblkid-dev libexpat1-dev libgoogle-perftools-dev libibverbs-dev libkeyutils-dev
+libldap2-dev libleveldb-dev liblttng-ust-dev libleveldb-dev liblttng-ust-dev libnss3-dev libsnappy-dev libssl-dev
+libtool libudev-dev libxml2-dev lsb-release parted pkg-config python python-all-dev python-cherrypy3 python-nose
+python-pecan python-prettytable python-setuptools python-sphinx python-werkzeug python3-all-dev python3-setuptools
+socat uuid-runtime virtualenv xfslibs-dev xfsprogs xmlstarlet yasm zlib1g-dev librabbitmq-dev libpython-dev
+libcunit1-dev liboath-dev libncurses-dev libcurl4-openssl-dev librdmacm-dev liblz4-dev
+
+# ceph boost deps
+libboost-atomic-dev libboost-chrono-dev libboost-context-dev libboost-coroutine-dev libboost-date-time-dev libboost-iostreams-dev libboost-program-options-dev libboost-python-dev libboost-random-dev libboost-regex-dev libboost-system-dev libboost-thread-dev
+
 # general dependencies
 debhelper
 

diff --git a/platform/WORKSPACE b/platform/WORKSPACE
@@ -22,6 +22,14 @@ load("//bazel:deps.bzl", "bazel_dependencies")
 
 bazel_dependencies()
 
+load("//ceph:deps_early.bzl", "ceph_dependencies_early")
+
+ceph_dependencies_early()
+
+load("//ceph:deps.bzl", "ceph_dependencies")
+
+ceph_dependencies()
+
 load("//cni-plugins:deps.bzl", "cni_plugins_dependencies")
 
 cni_plugins_dependencies()

diff --git a/platform/ceph/BUILD.bazel b/platform/ceph/BUILD.bazel
@@ -0,0 +1,150 @@
+load("@rules_foreign_cc//tools/build_defs:cmake.bzl", "cmake_external")
+load("//debian:debian.bzl", "debinstall")
+load("//bazel:package.bzl", "homeworld_oci")
+load("//ceph:unpack.bzl", "basename", "unpack_filegroup")
+
+BINARIES = [
+    "ceph-authtool",
+    "ceph-bluestore-tool",
+    "ceph-conf",
+    "ceph-dencoder",
+    "ceph-mgr",
+    "ceph-mon",
+    "ceph-objectstore-tool",
+    "ceph-osd",
+    "ceph-syn",
+    "rados",
+    "radosgw",
+    "radosgw-admin",
+    "radosgw-es",
+    "radosgw-object-expirer",
+    "radosgw-token",
+    "rbd",
+    "rbd-mirror",
+    "rbd-nbd",
+    "rbd-replay",
+    "rbd-replay-prep",
+]
+
+LIBRARIES = [
+    "libcephfs.so.2",
+    "librados.so.2",
+    "libradosstriper.so.1",
+    "librbd.so.1",
+    "librgw.so.2",
+    "librgw_admin_user.so.0",
+    "ceph/libceph-common.so.0",
+    "ceph/compressor/libceph_lz4.so.2",
+    "ceph/compressor/libceph_snappy.so.2",
+    "ceph/compressor/libceph_zlib.so.2",
+    "ceph/compressor/libceph_zstd.so.2",
+    "ceph/crypto/libceph_crypto_isal.so.1",
+    "ceph/crypto/libceph_crypto_openssl.so",
+    "ceph/erasure-code/libec_clay.so",
+    "ceph/erasure-code/libec_isa.so",
+    "ceph/erasure-code/libec_jerasure.so",
+    "ceph/erasure-code/libec_jerasure_generic.so",
+    "ceph/erasure-code/libec_jerasure_sse3.so",
+    "ceph/erasure-code/libec_jerasure_sse4.so",
+    "ceph/erasure-code/libec_lrc.so",
+    "ceph/erasure-code/libec_shec.so",
+    "ceph/erasure-code/libec_shec_generic.so",
+    "ceph/erasure-code/libec_shec_sse3.so",
+    "ceph/erasure-code/libec_shec_sse4.so",
+    "rados-classes/libcls_cas.so.1",
+    "rados-classes/libcls_hello.so.1",
+    "rados-classes/libcls_journal.so.1",
+    "rados-classes/libcls_kvs.so.1",
+    "rados-classes/libcls_lock.so.1",
+    "rados-classes/libcls_log.so.1",
+    "rados-classes/libcls_lua.so.1",
+    "rados-classes/libcls_numops.so.1",
+    "rados-classes/libcls_otp.so.1",
+    "rados-classes/libcls_rbd.so.1",
+    "rados-classes/libcls_refcount.so.1",
+    "rados-classes/libcls_rgw.so.1",
+    "rados-classes/libcls_sdk.so.1",
+    "rados-classes/libcls_timeindex.so.1",
+    "rados-classes/libcls_user.so.1",
+    "rados-classes/libcls_version.so.1",
+]
+
+cmake_external(
+    name = "ceph",
+    binaries = BINARIES,
+    shared_libraries = LIBRARIES,
+    cache_entries = {
+        "WITH_MANPAGE": "off",
+        "WITH_PYTHON3": "off",
+        "WITH_LTTNG": "off",
+        "WITH_EMBEDDED": "off",
+        "WITH_TESTS": "off",
+        "WITH_CEPHFS": "off",
+        # using system boost shaves off about 25% of the build time in my tests
+        "WITH_SYSTEM_BOOST": "on",
+        "WITH_RADOSGW_BEAST_FRONTEND": "on",
+        "WITH_FUSE": "off",
+        # disabled because the dashboard requires npm, which requires nodeenv,
+        # which requires working pip in a virtualenv, which is not working for
+        # some reason relating to DNS resolution.
+        # (and it's too much hell to make it functional for me to want to deal
+        # with this mess right now.)
+        "WITH_MGR_DASHBOARD_FRONTEND": "off",
+    },
+    lib_source = "@ceph//:source",
+    make_commands = ["make -j12", "make install"],
+)
+
+unpack_filegroup(
+    names = BINARIES + LIBRARIES,
+    src = ":ceph",
+)
+
+oci_include = {
+    ":" + binary: "/usr/bin/" + binary for binary in BINARIES
+}
+oci_include.update({
+    ":" + library: "/usr/lib/" + basename(library) for library in LIBRARIES
+})
+
+homeworld_oci(
+    name = "oci",
+    bin = oci_include,
+    deps = [
+        ":debian-for-ceph.tgz",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+debinstall(
+    name = "debian-for-ceph.tgz",
+    base = "//debian:debian-mini.tgz",
+    packages = [
+        "libaio1",
+        "libbabeltrace1",
+        "libboost-atomic1.67.0",
+        "libboost-chrono1.67.0",
+        "libboost-context1.67.0",
+        "libboost-coroutine1.67.0",
+        "libboost-date-time1.67.0",
+        "libboost-iostreams1.67.0",
+        "libboost-program-options1.67.0",
+        "libboost-python1.67.0",
+        "libboost-random1.67.0",
+        "libboost-regex1.67.0",
+        "libboost-system1.67.0",
+        "libboost-thread1.67.0",
+        "libcurl4",
+        "libexpat1",
+        "libgoogle-perftools4",
+        "libibverbs1",
+        "libleveldb1d",
+        "libncurses6",
+        "libnss3",
+        "liboath0",
+        "libpython2.7",
+        "librabbitmq4",
+        "librdmacm1",
+        "libssl1.1",
+    ],
+)
diff --git a/platform/ceph/deps.bzl b/platform/ceph/deps.bzl
@@ -0,0 +1,21 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
+
+VERSION = "14.2.4"
+SHA256 = "7180b6afcac57d858f1b7cf49615e8903fbf1a701ae00fb956a5656d608fd0f8"
+
+def ceph_dependencies():
+    rules_foreign_cc_dependencies()
+    http_archive(
+        name = "ceph",
+        url = "https://download.ceph.com/tarballs/ceph_" + VERSION + ".orig.tar.gz",
+        sha256 = SHA256,
+        patch_cmds = [
+            # remove symlinks that create cycles (which break glob)
+            "find -name '.qa' -delete",
+            # remove filenames with ":" in them (which are disallowed in filegroups)
+            "rm src/test/common/test_blkdev_sys_block/sys/dev/block/8:0 src/test/common/test_blkdev_sys_block/sys/dev/block/9:0",
+        ],
+        strip_prefix = "ceph-" + VERSION + "/",
+        build_file_content = """filegroup(name = "source", srcs = glob(["**"]), visibility = ["//visibility:public"])""",
+    )
diff --git a/platform/ceph/deps_early.bzl b/platform/ceph/deps_early.bzl
@@ -0,0 +1,21 @@
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+def ceph_dependencies_early():
+    git_repository(
+        name = "rules_foreign_cc",
+        remote = "https://github.com/bazelbuild/rules_foreign_cc",
+        commit = "ed3db61a55c13da311d875460938c42ee8bbc2a5",
+
+        patches = [
+            # we need this patch so that bazel's -D__DATE__="redacted" CFLAG doesn't become
+            # -D__DATE__=redacted, which causes code that uses __DATE__ to break.
+            "//ceph:foreign_cc/0001-fix-date-quoting.patch",
+            # (see https://github.com/bazelbuild/rules_foreign_cc/issues/239
+            # and https://github.com/bazelbuild/rules_foreign_cc/pull/362)
+
+            # we need this so that we can correctly reference the generated libraries
+            "//ceph:foreign_cc/0002-more-output-groups.patch",
+        ],
+
+        shallow_since = "1574792034 +0100"
+    )
diff --git a/platform/ceph/foreign_cc/0001-fix-date-quoting.patch b/platform/ceph/foreign_cc/0001-fix-date-quoting.patch
@@ -0,0 +1,14 @@
+diff --git tools/build_defs/cmake_script.bzl tools/build_defs/cmake_script.bzl
+index e1b0c13..393368a 100644
+--- tools/build_defs/cmake_script.bzl
++++ tools/build_defs/cmake_script.bzl
+@@ -65,7 +65,7 @@ def create_cmake_script(
+     if not params.cache.get("CMAKE_RANLIB"):
+         params.cache.update({"CMAKE_RANLIB": ""})
+
+-    set_env_vars = " ".join([key + "=\"" + params.env[key] + "\"" for key in params.env])
++    set_env_vars = " ".join([key + "='" + params.env[key] + "'" for key in params.env])
+     str_cmake_cache_entries = " ".join(["-D" + key + "=\"" + params.cache[key] + "\"" for key in params.cache])
+     cmake_call = " ".join([
+         set_env_vars,
+
diff --git a/platform/ceph/foreign_cc/0002-more-output-groups.patch b/platform/ceph/foreign_cc/0002-more-output-groups.patch
@@ -0,0 +1,13 @@
+diff --git tools/build_defs/framework.bzl tools/build_defs/framework.bzl
+index df37124..9f12a59 100644
+--- tools/build_defs/framework.bzl
++++ tools/build_defs/framework.bzl
+@@ -289,7 +289,7 @@ def cc_external_rule_impl(ctx, attrs):
+         lib_dir_name = attrs.out_lib_dir,
+         include_dir_name = attrs.out_include_dir,
+     )
+-    output_groups = _declare_output_groups(installdir_copy.file, outputs.out_binary_files)
++    output_groups = _declare_output_groups(installdir_copy.file, outputs.declared_outputs)
+     wrapped_files = [
+         wrapped_outputs.script_file,
+         wrapped_outputs.log_file,
diff --git a/platform/ceph/unpack.bzl b/platform/ceph/unpack.bzl
@@ -0,0 +1,13 @@
+def basename(name):
+    name_fragments = name.split("/")
+    return name_fragments[-1]
+
+
+# for unpacking results from foreign_cc cmake builds
+def unpack_filegroup(names, src):
+    for name in names:
+        native.filegroup(
+            name = name,
+            srcs = [src],
+            output_group = basename(name),
+        )
diff --git a/platform/upload/BUILD.bazel b/platform/upload/BUILD.bazel
@@ -4,14 +4,15 @@ load(":container_push.bzl", "oci_pusher")
 load("//python:resources.bzl", "py_resources")
 
 ocis = {
-    "pullcheck": "//pull-monitor/pullcheck:oci",
+    "ceph": "//ceph:oci",
     "dns-monitor": "//dns-monitor:oci",
     "dnsmasq-nanny": "//dnsmasq:dnsmasq-nanny",
     "flannel": "//flannel:oci",
     "flannel-monitor": "//flannel-monitor:oci",
     "kube-dns-main": "//kube-dns:kube-dns-main",
     "kube-dns-sidecar": "//kube-dns:kube-dns-sidecar",
     "pause": "//cri-o/pause:oci",
+    "pullcheck": "//pull-monitor/pullcheck:oci",
     "user-grant": "//user-grant:oci",
 }