Brotli streaming compression (netdata#16287)

* initial brotli implementation

* configuration for brotli

* default brotli compression to 6

* ordering of compression algorithms, always enables all compressions

* brotli is the default, with compression level 3

* updated stream.conf

* final brotli configuration

* core re-org

* more practical speed test

Makefile.am

@@ -667,6 +667,8 @@ STREAMING_PLUGIN_FILES = \
     streaming/rrdpush.c \
     streaming/compression.c \
     streaming/compression.h \
+    streaming/compression_brotli.c \
+    streaming/compression_brotli.h \
     streaming/compression_gzip.c \
     streaming/compression_gzip.h \
     streaming/compression_lz4.c \
@@ -1151,6 +1153,8 @@ NETDATA_COMMON_LIBS = \
     $(OPTIONAL_UV_LIBS) \
     $(OPTIONAL_LZ4_LIBS) \
     $(OPTIONAL_ZSTD_LIBS) \
+    $(OPTIONAL_BROTLIENC_LIBS) \
+    $(OPTIONAL_BROTLIDEC_LIBS) \
     $(OPTIONAL_DATACHANNEL_LIBS) \
     libjudy.a \
     $(OPTIONAL_SSL_LIBS) \

configure.ac

@@ -577,6 +577,27 @@ if test "x$LIBZSTD_FOUND" = "xyes"; then
     OPTIONAL_ZSTD_LIBS="-lzstd"
 fi
 
+# -----------------------------------------------------------------------------
+# brotli
+
+AC_CHECK_LIB([brotlienc], [BrotliEncoderCreateInstance, BrotliEncoderCompressStream],
+             [LIBBROTLIENC_FOUND=yes],
+             [LIBBROTLIENC_FOUND=no])
+
+if test "x$LIBBROTLIENC_FOUND" = "xyes"; then
+    AC_DEFINE([ENABLE_BROTLIENC], [1], [libbrotlienc usability])
+    OPTIONAL_BROTLIENC_LIBS="-lbrotlienc"
+fi
+
+AC_CHECK_LIB([brotlidec], [BrotliDecoderCreateInstance, BrotliDecoderDecompressStream],
+             [LIBBROTLIDEC_FOUND=yes],
+             [LIBBROTLIDEC_FOUND=no])
+
+if test "x$LIBBROTLIDEC_FOUND" = "xyes"; then
+    AC_DEFINE([ENABLE_BROTLIDEC], [1], [libbrotlidec usability])
+    OPTIONAL_BROTLIDEC_LIBS="-lbrotlidec"
+fi
+
 # -----------------------------------------------------------------------------
 # zlib
 
@@ -1912,6 +1933,8 @@ AC_SUBST([OPTIONAL_MATH_LIBS])
 AC_SUBST([OPTIONAL_DATACHANNEL_LIBS])
 AC_SUBST([OPTIONAL_UV_LIBS])
 AC_SUBST([OPTIONAL_LZ4_LIBS])
+AC_SUBST([OPTIONAL_BROTLIENC_LIBS])
+AC_SUBST([OPTIONAL_BROTLIDEC_LIBS])
 AC_SUBST([OPTIONAL_ZSTD_LIBS])
 AC_SUBST([OPTIONAL_SSL_LIBS])
 AC_SUBST([OPTIONAL_JSONC_LIBS])

daemon/buildinfo.c

@@ -1115,6 +1115,9 @@ __attribute__((constructor)) void initialize_build_info(void) {
 
     build_info_set_status(BIB_FEATURE_STREAMING_COMPRESSION, true);
 
+#ifdef ENABLE_BROTLI
+    build_info_append_value(BIB_FEATURE_STREAMING_COMPRESSION, "brotli");
+#endif
 #ifdef ENABLE_ZSTD
     build_info_append_value(BIB_FEATURE_STREAMING_COMPRESSION, "zstd");
 #endif

libnetdata/libnetdata.h

@@ -11,6 +11,10 @@ extern "C" {
 #include <config.h>
 #endif
 
+#if defined(ENABLE_BROTLIENC) && defined(ENABLE_BROTLIDEC)
+#define ENABLE_BROTLI 1
+#endif
+
 #ifdef ENABLE_OPENSSL
 #define ENABLE_HTTPS 1
 #endif