diff --git a/CMakeLists.txt b/CMakeLists.txt index 15c422665c0466..e3ab47ea192028 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1473,8 +1473,8 @@ set(RRD_PLUGIN_FILES src/database/sqlite/vendored/sqlite3recover.c src/database/sqlite/vendored/sqlite3recover.h src/database/sqlite/vendored/dbdata.c - src/database/KolmogorovSmirnovDist.c - src/database/KolmogorovSmirnovDist.h + src/web/api/queries/KolmogorovSmirnovDist.c + src/web/api/queries/KolmogorovSmirnovDist.h src/database/rrdfunctions-inflight.c src/database/rrdfunctions-inflight.h src/database/rrdfunctions-exporters.c @@ -1503,6 +1503,16 @@ set(RRD_PLUGIN_FILES src/database/rrddim-backfill.h src/database/rrddim-collection.c src/database/rrddim-collection.h + src/database/rrdset-type.c + src/database/rrdset-type.h + src/database/rrdhost-slots.c + src/database/rrdhost-slots.h + src/database/rrd-algorithm.c + src/database/rrd-algorithm.h + src/database/rrdhost-labels.c + src/database/rrdhost-labels.h + src/database/rrdhost-collection.c + src/database/rrdhost-collection.h ) if(ENABLE_DBENGINE) diff --git a/src/collectors/cgroups.plugin/tests/test_doubles.c b/src/collectors/cgroups.plugin/tests/test_doubles.c index 53fefa9c2472bc..2e6d810b1a79fc 100644 --- a/src/collectors/cgroups.plugin/tests/test_doubles.c +++ b/src/collectors/cgroups.plugin/tests/test_doubles.c @@ -47,7 +47,7 @@ void mountinfo_free_all(struct mountinfo *mi) RRDSET *rrdset_create_custom( RRDHOST *host, const char *type, const char *id, const char *name, const char *family, const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every, - RRDSET_TYPE chart_type, RRD_MEMORY_MODE memory_mode, long history_entries) + RRDSET_TYPE chart_type, RRD_DB_MODE memory_mode, long history_entries) { UNUSED(host); UNUSED(type); @@ -70,7 +70,7 @@ RRDSET *rrdset_create_custom( RRDDIM *rrddim_add_custom( RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, - RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode) + RRD_ALGORITHM algorithm, RRD_DB_MODE memory_mode) { UNUSED(st); UNUSED(id); diff --git a/src/collectors/statsd.plugin/statsd.c b/src/collectors/statsd.plugin/statsd.c index 0c87df9c2a2ebb..f065b4fa525a53 100644 --- a/src/collectors/statsd.plugin/statsd.c +++ b/src/collectors/statsd.plugin/statsd.c @@ -220,7 +220,7 @@ typedef struct statsd_app { const char *name; SIMPLE_PATTERN *metrics; STATS_METRIC_OPTIONS default_options; - RRD_MEMORY_MODE rrd_memory_mode; + RRD_DB_MODE rrd_memory_mode; int32_t rrd_history_entries; DICTIONARY *dict; diff --git a/src/daemon/config/netdata-conf-db.c b/src/daemon/config/netdata-conf-db.c index 2f3baa04442fb2..98e20c92a0abf7 100644 --- a/src/daemon/config/netdata-conf-db.c +++ b/src/daemon/config/netdata-conf-db.c @@ -87,9 +87,9 @@ static void netdata_conf_dbengine_pre_logs(void) { // } #else - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead."); - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE) { + error_report("RRD_DB_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead."); + default_rrd_memory_mode = RRD_DB_MODE_RAM; } #endif } @@ -381,7 +381,7 @@ void netdata_conf_section_db(void) { // ------------------------------------------------------------------------ // get default database size - if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && default_rrd_memory_mode != RRD_MEMORY_MODE_NONE) { + if(default_rrd_memory_mode != RRD_DB_MODE_DBENGINE && default_rrd_memory_mode != RRD_DB_MODE_NONE) { default_rrd_history_entries = (int)config_get_duration_seconds( CONFIG_SECTION_DB, "retention", align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES)); diff --git a/src/daemon/main.c b/src/daemon/main.c index bf60d30766192b..0ef243ee1ba916 100644 --- a/src/daemon/main.c +++ b/src/daemon/main.c @@ -213,7 +213,7 @@ int unittest_prepare_rrd(const char **user) { netdata_conf_section_global_run_as_user(user); netdata_conf_section_global(); nd_profile.update_every = 1; - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; + default_rrd_memory_mode = RRD_DB_MODE_RAM; health_plugin_disable(); nd_profile.storage_tiers = 1; registry_init(); diff --git a/src/daemon/pulse/pulse-db-dbengine.c b/src/daemon/pulse/pulse-db-dbengine.c index 0a08169369adf4..961b6b6a323a30 100644 --- a/src/daemon/pulse/pulse-db-dbengine.c +++ b/src/daemon/pulse/pulse-db-dbengine.c @@ -1490,7 +1490,7 @@ void pulse_dbengine_do(bool extended) { /* get localhost's DB engine's statistics for each tier */ for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++) { - if(host->db[tier].mode != RRD_MEMORY_MODE_DBENGINE) continue; + if(host->db[tier].mode != RRD_DB_MODE_DBENGINE) continue; if(!host->db[tier].si) continue; if(counted_multihost_db[tier]) diff --git a/src/daemon/service.c b/src/daemon/service.c index 5975712ee8dfc9..e10a0cfc9b1a59 100644 --- a/src/daemon/service.c +++ b/src/daemon/service.c @@ -34,7 +34,7 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) { rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED); rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE); - if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + if (rd->rrd_memory_mode == RRD_DB_MODE_DBENGINE) { /* only a collector can mark a chart as obsolete, so we must remove the reference */ if (!rrddim_finalize_collection_and_check_retention(rd)) { /* This metric has no data and no references */ diff --git a/src/daemon/unit_test.c b/src/daemon/unit_test.c index 963e3b1c394d42..07d0486d74bb6f 100644 --- a/src/daemon/unit_test.c +++ b/src/daemon/unit_test.c @@ -1268,7 +1268,7 @@ int run_test(struct test *test) { fprintf(stderr, "\nRunning test '%s':\n%s\n", test->name, test->description); - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC; + default_rrd_memory_mode = RRD_DB_MODE_ALLOC; nd_profile.update_every = test->update_every; char name[101]; @@ -1537,7 +1537,7 @@ int unit_test(long delay, long shift) snprintfz(name, sizeof(name) - 1, "unittest-%d-%ld-%ld", repeat, delay, shift); //debug_flags = 0xffffffff; - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC; + default_rrd_memory_mode = RRD_DB_MODE_ALLOC; nd_profile.update_every = 1; int do_abs = 1; diff --git a/src/database/contexts/contexts-loading.c b/src/database/contexts/contexts-loading.c index a43b9370ac5c3a..b8e52f01bf1e27 100644 --- a/src/database/contexts/contexts-loading.c +++ b/src/database/contexts/contexts-loading.c @@ -121,7 +121,7 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) { if(host->rrdctx.contexts) return; rrdhost_create_rrdcontexts(host); - if (host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + if (host->rrd_memory_mode != RRD_DB_MODE_DBENGINE) return; ignored_metrics = 0; diff --git a/src/database/contexts/worker.c b/src/database/contexts/worker.c index c7e497e586cb4a..b698f33750d90d 100644 --- a/src/database/contexts/worker.c +++ b/src/database/contexts/worker.c @@ -264,7 +264,7 @@ void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc) { rc->hub.units = string2str(rc->units); rc->hub.family = string2str(rc->family); - if (rc->rrdhost->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + if (rc->rrdhost->rrd_memory_mode != RRD_DB_MODE_DBENGINE) return; // delete it from SQL @@ -762,7 +762,7 @@ void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused rrdcontext_delete_from_sql_unsafe(rc); else { - if (rc->rrdhost->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + if (rc->rrdhost->rrd_memory_mode != RRD_DB_MODE_DBENGINE) return; if (ctx_store_context(&rc->rrdhost->host_id.uuid, &rc->hub) != 0) netdata_log_error( diff --git a/src/database/engine/dbengine-stresstest.c b/src/database/engine/dbengine-stresstest.c index a68b87dbba5951..61d63b09131659 100644 --- a/src/database/engine/dbengine-stresstest.c +++ b/src/database/engine/dbengine-stresstest.c @@ -20,7 +20,7 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) { NETDATA_VERSION, nd_profile.update_every, default_rrd_history_entries, - RRD_MEMORY_MODE_DBENGINE, + RRD_DB_MODE_DBENGINE, health_plugin_enabled(), stream_send.enabled, stream_send.parents.destination, @@ -140,7 +140,7 @@ void generate_dbengine_dataset(unsigned history_seconds) int i; time_t time_present; - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; + default_rrd_memory_mode = RRD_DB_MODE_DBENGINE; default_rrdeng_page_cache_mb = 128; // Worst case for uncompressible data default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * history_seconds) / @@ -333,7 +333,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi if (PAGE_CACHE_MB < RRDENG_MIN_PAGE_CACHE_SIZE_MB) PAGE_CACHE_MB = RRDENG_MIN_PAGE_CACHE_SIZE_MB; - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; + default_rrd_memory_mode = RRD_DB_MODE_DBENGINE; default_rrdeng_page_cache_mb = PAGE_CACHE_MB; if (DISK_SPACE_MB) { fprintf(stderr, "By setting disk space limit data are allowed to be deleted. " diff --git a/src/database/engine/dbengine-unittest.c b/src/database/engine/dbengine-unittest.c index 937335f949b24c..d25ec0697f428c 100644 --- a/src/database/engine/dbengine-unittest.c +++ b/src/database/engine/dbengine-unittest.c @@ -106,7 +106,7 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) { NETDATA_VERSION, nd_profile.update_every, default_rrd_history_entries, - RRD_MEMORY_MODE_DBENGINE, + RRD_DB_MODE_DBENGINE, health_plugin_enabled(), stream_send.enabled, stream_send.parents.destination, @@ -362,7 +362,7 @@ int test_dbengine(void) { nd_log_limits_unlimited(); fprintf(stderr, "\nRunning DB-engine test\n"); - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; + default_rrd_memory_mode = RRD_DB_MODE_DBENGINE; fprintf(stderr, "Initializing localhost with hostname 'unittest-dbengine'"); RRDHOST *host = dbengine_rrdhost_find_or_create("unittest-dbengine"); if(!host) diff --git a/src/database/rrd-algorithm.c b/src/database/rrd-algorithm.c new file mode 100644 index 00000000000000..40f8dfc53ac56d --- /dev/null +++ b/src/database/rrd-algorithm.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rrd-algorithm.h" + +RRD_ALGORITHM rrd_algorithm_id(const char *name) { + if(strcmp(name, RRD_ALGORITHM_INCREMENTAL_NAME) == 0) + return RRD_ALGORITHM_INCREMENTAL; + + else if(strcmp(name, RRD_ALGORITHM_ABSOLUTE_NAME) == 0) + return RRD_ALGORITHM_ABSOLUTE; + + else if(strcmp(name, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME) == 0) + return RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL; + + else if(strcmp(name, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME) == 0) + return RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL; + + else + return RRD_ALGORITHM_ABSOLUTE; +} + +const char *rrd_algorithm_name(RRD_ALGORITHM algorithm) { + switch(algorithm) { + case RRD_ALGORITHM_ABSOLUTE: + default: + return RRD_ALGORITHM_ABSOLUTE_NAME; + + case RRD_ALGORITHM_INCREMENTAL: + return RRD_ALGORITHM_INCREMENTAL_NAME; + + case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL: + return RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME; + + case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: + return RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME; + } +} diff --git a/src/database/rrd-algorithm.h b/src/database/rrd-algorithm.h new file mode 100644 index 00000000000000..7e2caf52815a71 --- /dev/null +++ b/src/database/rrd-algorithm.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRD_ALGORITHM_H +#define NETDATA_RRD_ALGORITHM_H + +#include "libnetdata/libnetdata.h" + +typedef enum __attribute__ ((__packed__)) rrd_algorithm { + RRD_ALGORITHM_ABSOLUTE = 0, + RRD_ALGORITHM_INCREMENTAL = 1, + RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL = 2, + RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL = 3, + + // this is 8-bit +} RRD_ALGORITHM; + +#define RRD_ALGORITHM_ABSOLUTE_NAME "absolute" +#define RRD_ALGORITHM_INCREMENTAL_NAME "incremental" +#define RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME "percentage-of-incremental-row" +#define RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME "percentage-of-absolute-row" + +RRD_ALGORITHM rrd_algorithm_id(const char *name); +const char *rrd_algorithm_name(RRD_ALGORITHM algorithm); + + +#endif //NETDATA_RRD_ALGORITHM_H diff --git a/src/database/rrd-database-mode.c b/src/database/rrd-database-mode.c index 123b1d021a828f..b4382db75a8185 100644 --- a/src/database/rrd-database-mode.c +++ b/src/database/rrd-database-mode.c @@ -1,19 +1,19 @@ #include "rrd.h" -inline const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) { +inline const char *rrd_memory_mode_name(RRD_DB_MODE id) { switch(id) { - case RRD_MEMORY_MODE_RAM: - return RRD_MEMORY_MODE_RAM_NAME; + case RRD_DB_MODE_RAM: + return RRD_DB_MODE_RAM_NAME; - case RRD_MEMORY_MODE_NONE: - return RRD_MEMORY_MODE_NONE_NAME; + case RRD_DB_MODE_NONE: + return RRD_DB_MODE_NONE_NAME; - case RRD_MEMORY_MODE_ALLOC: - return RRD_MEMORY_MODE_ALLOC_NAME; + case RRD_DB_MODE_ALLOC: + return RRD_DB_MODE_ALLOC_NAME; - case RRD_MEMORY_MODE_DBENGINE: - return RRD_MEMORY_MODE_DBENGINE_NAME; + case RRD_DB_MODE_DBENGINE: + return RRD_DB_MODE_DBENGINE_NAME; } STORAGE_ENGINE* eng = storage_engine_get(id); @@ -21,14 +21,14 @@ inline const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) { return eng->name; } - return RRD_MEMORY_MODE_RAM_NAME; + return RRD_DB_MODE_RAM_NAME; } -RRD_MEMORY_MODE rrd_memory_mode_id(const char *name) { +RRD_DB_MODE rrd_memory_mode_id(const char *name) { STORAGE_ENGINE* eng = storage_engine_find(name); if (eng) { return eng->id; } - return RRD_MEMORY_MODE_RAM; + return RRD_DB_MODE_RAM; } diff --git a/src/database/rrd-database-mode.h b/src/database/rrd-database-mode.h index fa8238d1996bbc..207464c555481b 100644 --- a/src/database/rrd-database-mode.h +++ b/src/database/rrd-database-mode.h @@ -4,22 +4,22 @@ #define NETDATA_RRD_DATABASE_MODE_H typedef enum __attribute__ ((__packed__)) rrd_memory_mode { - RRD_MEMORY_MODE_NONE = 0, - RRD_MEMORY_MODE_RAM = 1, - RRD_MEMORY_MODE_ALLOC = 4, - RRD_MEMORY_MODE_DBENGINE = 5, + RRD_DB_MODE_NONE = 0, + RRD_DB_MODE_RAM = 1, + RRD_DB_MODE_ALLOC = 4, + RRD_DB_MODE_DBENGINE = 5, // this is 8-bit -} RRD_MEMORY_MODE; +} RRD_DB_MODE; -#define RRD_MEMORY_MODE_NONE_NAME "none" -#define RRD_MEMORY_MODE_RAM_NAME "ram" -#define RRD_MEMORY_MODE_ALLOC_NAME "alloc" -#define RRD_MEMORY_MODE_DBENGINE_NAME "dbengine" +#define RRD_DB_MODE_NONE_NAME "none" +#define RRD_DB_MODE_RAM_NAME "ram" +#define RRD_DB_MODE_ALLOC_NAME "alloc" +#define RRD_DB_MODE_DBENGINE_NAME "dbengine" -extern RRD_MEMORY_MODE default_rrd_memory_mode; +extern RRD_DB_MODE default_rrd_memory_mode; -const char *rrd_memory_mode_name(RRD_MEMORY_MODE id); -RRD_MEMORY_MODE rrd_memory_mode_id(const char *name); +const char *rrd_memory_mode_name(RRD_DB_MODE id); +RRD_DB_MODE rrd_memory_mode_id(const char *name); #endif //NETDATA_RRD_DATABASE_MODE_H diff --git a/src/database/rrd.c b/src/database/rrd.c index 754345e1cf45bb..2c952d6bcb5fd7 100644 --- a/src/database/rrd.c +++ b/src/database/rrd.c @@ -3,7 +3,7 @@ #define RRDHOST_INTERNALS #include "rrd.h" -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // globals /* @@ -14,51 +14,14 @@ int rrd_delete_unupdated_dimensions = 0; */ #ifdef ENABLE_DBENGINE -RRD_MEMORY_MODE default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; +RRD_DB_MODE default_rrd_memory_mode = RRD_DB_MODE_DBENGINE; #else -RRD_MEMORY_MODE default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; +RRD_DB_MODE default_rrd_memory_mode = RRD_DB_MODE_RAM; #endif int gap_when_lost_iterations_above = 1; -// ---------------------------------------------------------------------------- -// RRD - algorithms types - -RRD_ALGORITHM rrd_algorithm_id(const char *name) { - if(strcmp(name, RRD_ALGORITHM_INCREMENTAL_NAME) == 0) - return RRD_ALGORITHM_INCREMENTAL; - - else if(strcmp(name, RRD_ALGORITHM_ABSOLUTE_NAME) == 0) - return RRD_ALGORITHM_ABSOLUTE; - - else if(strcmp(name, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME) == 0) - return RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL; - - else if(strcmp(name, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME) == 0) - return RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL; - - else - return RRD_ALGORITHM_ABSOLUTE; -} - -const char *rrd_algorithm_name(RRD_ALGORITHM algorithm) { - switch(algorithm) { - case RRD_ALGORITHM_ABSOLUTE: - default: - return RRD_ALGORITHM_ABSOLUTE_NAME; - - case RRD_ALGORITHM_INCREMENTAL: - return RRD_ALGORITHM_INCREMENTAL_NAME; - - case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL: - return RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME; - - case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: - return RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME; - } -} - -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // RRD - string management STRING *rrd_string_strdupz(const char *s) { @@ -71,16 +34,16 @@ STRING *rrd_string_strdupz(const char *s) { return ret; } -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- -inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) { - if(mode == RRD_MEMORY_MODE_DBENGINE) return 0; - if(mode == RRD_MEMORY_MODE_NONE) return 5; +inline long align_entries_to_pagesize(RRD_DB_MODE mode, long entries) { + if(mode == RRD_DB_MODE_DBENGINE) return 0; + if(mode == RRD_DB_MODE_NONE) return 5; if(entries < 5) entries = 5; if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX; - if(mode == RRD_MEMORY_MODE_RAM) { + if(mode == RRD_DB_MODE_RAM) { long header_size = 0; long page = (long)sysconf(_SC_PAGESIZE); @@ -97,12 +60,15 @@ inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) { return entries; } +// -------------------------------------------------------------------------------------------------------------------- + void api_v1_management_init(void); + int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool unittest) { rrdhost_init(); if (unlikely(sql_init_meta_database(DB_CHECK_NONE, system_info ? 0 : 1))) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE) { set_late_analytics_variables(system_info); fatal("Failed to initialize SQLite"); } @@ -119,7 +85,7 @@ int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool dbengine_enabled = true; } else { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || stream_conf_receiver_needs_dbengine()) { + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE || stream_conf_receiver_needs_dbengine()) { nd_log(NDLS_DAEMON, NDLP_DEBUG, "DBENGINE: Initializing ..."); @@ -137,12 +103,12 @@ int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool nd_profile.storage_tiers = 1; } - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE) { nd_log(NDLS_DAEMON, NDLP_WARNING, "dbengine is not enabled, but it has been given as the default db mode. " "Resetting db mode to alloc"); - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC; + default_rrd_memory_mode = RRD_DB_MODE_ALLOC; } } } diff --git a/src/database/rrd.h b/src/database/rrd.h index eca688e0e78d09..0f7dacd61fc918 100644 --- a/src/database/rrd.h +++ b/src/database/rrd.h @@ -31,26 +31,6 @@ typedef enum __attribute__ ((__packed__)) { QUERY_SOURCE_UNITTEST, } QUERY_SOURCE; -// -------------------------------------------------------------------------------------------------------------------- -// algorithms types - -typedef enum __attribute__ ((__packed__)) rrd_algorithm { - RRD_ALGORITHM_ABSOLUTE = 0, - RRD_ALGORITHM_INCREMENTAL = 1, - RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL = 2, - RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL = 3, - - // this is 8-bit -} RRD_ALGORITHM; - -#define RRD_ALGORITHM_ABSOLUTE_NAME "absolute" -#define RRD_ALGORITHM_INCREMENTAL_NAME "incremental" -#define RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME "percentage-of-incremental-row" -#define RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME "percentage-of-absolute-row" - -RRD_ALGORITHM rrd_algorithm_id(const char *name); -const char *rrd_algorithm_name(RRD_ALGORITHM algorithm); - // -------------------------------------------------------------------------------------------------------------------- typedef struct rrdcalc RRDCALC; @@ -101,7 +81,7 @@ extern netdata_rwlock_t rrd_rwlock; STRING *rrd_string_strdupz(const char *s); #include "rrd-database-mode.h" -long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries); +long align_entries_to_pagesize(RRD_DB_MODE mode, long entries); static inline uint32_t get_uint32_id() { return now_realtime_sec() & UINT32_MAX; diff --git a/src/database/rrddim-backfill.c b/src/database/rrddim-backfill.c index 86e6a20104448b..52543b18320f9b 100644 --- a/src/database/rrddim-backfill.c +++ b/src/database/rrddim-backfill.c @@ -6,8 +6,6 @@ // ---------------------------------------------------------------------------- // fill the gap of a tier -void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut); - bool backfill_tier_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s) { if(unlikely(tier >= nd_profile.storage_tiers)) return false; #ifdef ENABLE_DBENGINE diff --git a/src/database/rrddim-collection.c b/src/database/rrddim-collection.c index 70c2976a7a1727..7abb9ed53cb61c 100644 --- a/src/database/rrddim-collection.c +++ b/src/database/rrddim-collection.c @@ -72,3 +72,80 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG } } } + +#ifdef NETDATA_LOG_COLLECTION_ERRORS +void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function) { +#else // !NETDATA_LOG_COLLECTION_ERRORS +void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) { +#endif // !NETDATA_LOG_COLLECTION_ERRORS + + static __thread struct log_stack_entry lgs[] = { + [0] = ND_LOG_FIELD_STR(NDF_NIDL_DIMENSION, NULL), + [1] = ND_LOG_FIELD_END(), + }; + lgs[0].str = rd->id; + log_stack_push(lgs); + +#ifdef NETDATA_LOG_COLLECTION_ERRORS + rd->rrddim_store_metric_count++; + + if(likely(rd->rrddim_store_metric_count > 1)) { + usec_t expected = rd->rrddim_store_metric_last_ut + rd->update_every * USEC_PER_SEC; + + if(point_end_time_ut != rd->rrddim_store_metric_last_ut) { + internal_error(true, + "%s COLLECTION: 'host:%s/chart:%s/dim:%s' granularity %d, collection %zu, expected to store at tier 0 a value at %llu, but it gave %llu [%s%llu usec] (called from %s(), previously by %s())", + (point_end_time_ut < rd->rrddim_store_metric_last_ut) ? "**PAST**" : "GAP", + rrdhost_hostname(rd->rrdset->rrdhost), rrdset_id(rd->rrdset), rrddim_id(rd), + rd->update_every, + rd->rrddim_store_metric_count, + expected, point_end_time_ut, + (point_end_time_ut < rd->rrddim_store_metric_last_ut)?"by -" : "gap ", + expected - point_end_time_ut, + function, + rd->rrddim_store_metric_last_caller?rd->rrddim_store_metric_last_caller:"none"); + } + } + + rd->rrddim_store_metric_last_ut = point_end_time_ut; + rd->rrddim_store_metric_last_caller = function; +#endif // NETDATA_LOG_COLLECTION_ERRORS + + // store the metric on tier 0 + storage_engine_store_metric(rd->tiers[0].sch, point_end_time_ut, + n, 0, 0, + 1, 0, flags); + + rrdset_done_statistics_points_stored_per_tier[0]++; + + time_t now_s = (time_t)(point_end_time_ut / USEC_PER_SEC); + + STORAGE_POINT sp = { + .start_time_s = now_s - rd->rrdset->update_every, + .end_time_s = now_s, + .min = n, + .max = n, + .sum = n, + .count = 1, + .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1, + .flags = flags + }; + + for(size_t tier = 1; tier < nd_profile.storage_tiers;tier++) { + if(unlikely(!rd->tiers[tier].smh)) continue; + + struct rrddim_tier *t = &rd->tiers[tier]; + + if(!rrddim_option_check(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS)) { + // we have not collected this tier before + // let's fill any gap that may exist + backfill_tier_from_smaller_tiers(rd, tier, now_s); + } + + store_metric_at_tier(rd, tier, t, sp, point_end_time_ut); + } + rrddim_option_set(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS); + + rrdcontext_collected_rrddim(rd); + log_stack_pop(&lgs); +} diff --git a/src/database/rrddim-collection.h b/src/database/rrddim-collection.h index 6cb80c2f4adc99..8549488ec942a7 100644 --- a/src/database/rrddim-collection.h +++ b/src/database/rrddim-collection.h @@ -8,4 +8,11 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut); void store_metric_collection_completed(void); +#ifdef NETDATA_LOG_COLLECTION_ERRORS +#define rrddim_store_metric(rd, point_end_time_ut, n, flags) rrddim_store_metric_with_trace(rd, point_end_time_ut, n, flags, __FUNCTION__) +void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function); +#else +void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags); +#endif + #endif //NETDATA_RRDDIM_COLLECTION_H diff --git a/src/database/rrddim.c b/src/database/rrddim.c index daaf76064175c2..8886f27c6052a0 100644 --- a/src/database/rrddim.c +++ b/src/database/rrddim.c @@ -18,7 +18,7 @@ struct rrddim_constructor { collected_number multiplier; collected_number divisor; RRD_ALGORITHM algorithm; - RRD_MEMORY_MODE memory_mode; + RRD_DB_MODE memory_mode; enum { RRDDIM_REACT_NONE = 0, @@ -57,7 +57,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)) rd->collector.counter = 1; - if(ctr->memory_mode == RRD_MEMORY_MODE_RAM) { + if(ctr->memory_mode == RRD_DB_MODE_RAM) { size_t entries = st->db.entries; if(!entries) entries = 5; @@ -68,11 +68,11 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v } else { netdata_log_info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd)); - ctr->memory_mode = RRD_MEMORY_MODE_ALLOC; + ctr->memory_mode = RRD_DB_MODE_ALLOC; } } - if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) { + if(ctr->memory_mode == RRD_DB_MODE_ALLOC || ctr->memory_mode == RRD_DB_MODE_NONE) { size_t entries = st->db.entries; if(entries < 5) entries = 5; @@ -208,7 +208,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v netdata_log_debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd)); - if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_DB_MODE_DBENGINE) { /* This metric has no data and no references */ metaqueue_delete_dimension_uuid(uuidmap_uuid_ptr(rd->uuid)); } @@ -226,7 +226,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v if(rd->db.data) { pulse_db_rrd_memory_sub(rd->db.memsize); - if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM) + if(rd->rrd_memory_mode == RRD_DB_MODE_RAM) nd_munmap(rd->db.data, rd->db.memsize); else freez(rd->db.data); @@ -454,7 +454,8 @@ RRDDIM *rrddim_add_custom(RRDSET *st , collected_number multiplier , collected_number divisor , RRD_ALGORITHM algorithm - , RRD_MEMORY_MODE memory_mode + , + RRD_DB_MODE memory_mode ) { struct rrddim_constructor tmp = { .st = st, diff --git a/src/database/rrddim.h b/src/database/rrddim.h index cfb09a39c3329b..3cf3bfc8b06352 100644 --- a/src/database/rrddim.h +++ b/src/database/rrddim.h @@ -4,6 +4,7 @@ #define NETDATA_RRDDIM_H #include "libnetdata/libnetdata.h" +#include "rrd-algorithm.h" typedef struct rrddim RRDDIM; typedef struct rrddim_acquired RRDDIM_ACQUIRED; @@ -60,7 +61,7 @@ struct rrddim { STRING *name; // the name of this dimension (as presented to user) RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values - RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension + RRD_DB_MODE rrd_memory_mode; // the memory mode for this dimension RRDDIM_FLAGS flags; // run time changing status flags int32_t multiplier; // the multiplier of the collected values @@ -195,7 +196,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st , collected_number multiplier , collected_number divisor , RRD_ALGORITHM algorithm - , RRD_MEMORY_MODE memory_mode + , RRD_DB_MODE memory_mode ); #define rrddim_add(st, id, name, multiplier, divisor, algorithm) \ @@ -223,16 +224,6 @@ collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number collected_number rrddim_set(RRDSET *st, const char *id, collected_number value); bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd); -void rrdset_finalize_collection(RRDSET *st, bool dimensions_too); -void rrdhost_finalize_collection(RRDHOST *host); -void rrd_finalize_collection_for_all_hosts(void); - -#ifdef NETDATA_LOG_COLLECTION_ERRORS -#define rrddim_store_metric(rd, point_end_time_ut, n, flags) rrddim_store_metric_with_trace(rd, point_end_time_ut, n, flags, __FUNCTION__) -void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function); -#else -void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags); -#endif void rrddim_free(RRDSET *st, RRDDIM *rd); diff --git a/src/database/rrdhost-collection.c b/src/database/rrdhost-collection.c new file mode 100644 index 00000000000000..c8f25d3ed01a4a --- /dev/null +++ b/src/database/rrdhost-collection.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rrdhost-collection.h" +#include "rrdset.h" + +void rrd_finalize_collection_for_all_hosts(void) { + RRDHOST *host; + dfe_start_reentrant(rrdhost_root_index, host) { + rrdhost_finalize_collection(host); + } + dfe_done(host); +} + +void rrdhost_finalize_collection(RRDHOST *host) { + ND_LOG_STACK lgs[] = { + ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(host)), + ND_LOG_FIELD_END(), + }; + ND_LOG_STACK_PUSH(lgs); + + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "RRD: 'host:%s' stopping data collection...", + rrdhost_hostname(host)); + + RRDSET *st; + rrdset_foreach_read(st, host) + rrdset_finalize_collection(st, true); + rrdset_foreach_done(st); +} + +bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now) { + time_t first_time_s, last_time_s; + rrdhost_retention(host, now, rrdhost_is_online(host), &first_time_s, &last_time_s); + return query_matches_retention(after, before, first_time_s, last_time_s, 0); +} diff --git a/src/database/rrdhost-collection.h b/src/database/rrdhost-collection.h new file mode 100644 index 00000000000000..c2ce64c7748a36 --- /dev/null +++ b/src/database/rrdhost-collection.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRDHOST_COLLECTION_H +#define NETDATA_RRDHOST_COLLECTION_H + +#include "rrdhost.h" + +void rrdhost_finalize_collection(RRDHOST *host); +void rrd_finalize_collection_for_all_hosts(void); + +#endif //NETDATA_RRDHOST_COLLECTION_H diff --git a/src/database/rrdhost-labels.c b/src/database/rrdhost-labels.c new file mode 100644 index 00000000000000..3f82159e7a4f44 --- /dev/null +++ b/src/database/rrdhost-labels.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rrdhost-labels.h" +#include "rrdhost.h" +#include "streaming/stream.h" + +void rrdhost_set_is_parent_label(void) { + uint32_t count = stream_receivers_currently_connected(); + + if (count == 0 || count == 1) { + RRDLABELS *labels = localhost->rrdlabels; + rrdlabels_add(labels, "_is_parent", (count) ? "true" : "false", RRDLABEL_SRC_AUTO); + + // queue a node info + aclk_queue_node_info(localhost, false); + } +} + +static bool config_label_cb(void *data __maybe_unused, const char *name, const char *value) { + rrdlabels_add(localhost->rrdlabels, name, value, RRDLABEL_SRC_CONFIG); + return true; +} + +static void rrdhost_load_config_labels(void) { + int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL); + if(!status) { + char *filename = CONFIG_DIR "/" CONFIG_FILENAME; + nd_log(NDLS_DAEMON, NDLP_WARNING, + "RRDLABEL: Cannot reload the configuration file '%s', using labels in memory", + filename); + } + + appconfig_foreach_value_in_section(&netdata_config, CONFIG_SECTION_HOST_LABEL, config_label_cb, NULL); +} + +static void rrdhost_load_kubernetes_labels(void) { + char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)]; + sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh"); + + if (unlikely(access(label_script, R_OK) != 0)) { + nd_log(NDLS_DAEMON, NDLP_ERR, + "Kubernetes pod label fetching script %s not found.", + label_script); + + return; + } + + POPEN_INSTANCE *instance = spawn_popen_run(label_script); + if(!instance) return; + + char buffer[1000 + 1]; + while (fgets(buffer, 1000, spawn_popen_stdout(instance)) != NULL) + rrdlabels_add_pair(localhost->rrdlabels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S); + + // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':' + // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null + int rc = spawn_popen_wait(instance); + if(rc) + nd_log(NDLS_DAEMON, NDLP_ERR, + "%s exited abnormally. Failed to get kubernetes labels.", + label_script); +} + +static void rrdhost_load_auto_labels(void) { + RRDLABELS *labels = localhost->rrdlabels; + + rrdhost_system_info_to_rrdlabels(localhost->system_info, labels); + add_aclk_host_labels(); + + // The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf). + // Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084. + + int is_ephemeral = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "is ephemeral node", CONFIG_BOOLEAN_NO); + rrdlabels_add(labels, "_is_ephemeral", is_ephemeral ? "true" : "false", RRDLABEL_SRC_AUTO); + + int has_unstable_connection = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "has unstable connection", CONFIG_BOOLEAN_NO); + rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_AUTO); + + rrdlabels_add(labels, "_is_parent", (stream_receivers_currently_connected() > 0) ? "true" : "false", RRDLABEL_SRC_AUTO); + + rrdlabels_add(labels, "_hostname", string2str(localhost->hostname), RRDLABEL_SRC_AUTO); + rrdlabels_add(labels, "_os", string2str(localhost->os), RRDLABEL_SRC_AUTO); + + if (localhost->stream.snd.destination) + rrdlabels_add(labels, "_streams_to", string2str(localhost->stream.snd.destination), RRDLABEL_SRC_AUTO); +} + +void reload_host_labels(void) { + if(!localhost->rrdlabels) + localhost->rrdlabels = rrdlabels_create(); + + rrdlabels_unmark_all(localhost->rrdlabels); + + // priority is important here + rrdhost_load_config_labels(); + rrdhost_load_kubernetes_labels(); + rrdhost_load_auto_labels(); + + rrdhost_flag_set(localhost,RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE); + + stream_send_host_labels(localhost); +} diff --git a/src/database/rrdhost-labels.h b/src/database/rrdhost-labels.h new file mode 100644 index 00000000000000..6a7b9dd5a30de7 --- /dev/null +++ b/src/database/rrdhost-labels.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRDHOST_LABELS_H +#define NETDATA_RRDHOST_LABELS_H + +#include "libnetdata/libnetdata.h" + +void reload_host_labels(void); +void rrdhost_set_is_parent_label(void); + +#endif //NETDATA_RRDHOST_LABELS_H diff --git a/src/database/rrdhost-slots.c b/src/database/rrdhost-slots.c new file mode 100644 index 00000000000000..87221f8a896a8d --- /dev/null +++ b/src/database/rrdhost-slots.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rrdhost-slots.h" +#include "rrdset.h" + +void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host) { + rrd_slot_memory_removed(host->stream.snd.pluginsd_chart_slots.available.size * sizeof(uint32_t)); + + spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock); + host->stream.snd.pluginsd_chart_slots.available.ignore = true; + freez(host->stream.snd.pluginsd_chart_slots.available.array); + host->stream.snd.pluginsd_chart_slots.available.array = NULL; + host->stream.snd.pluginsd_chart_slots.available.used = 0; + host->stream.snd.pluginsd_chart_slots.available.size = 0; + spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock); + + // zero all the slots on all charts, so that they will not attempt to access the array + RRDSET *st; + rrdset_foreach_read(st, host) { + st->stream.snd.chart_slot = 0; + } + rrdset_foreach_done(st); +} + +void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host) { + rrd_slot_memory_removed(host->stream.rcv.pluginsd_chart_slots.size * sizeof(uint32_t)); + + spinlock_lock(&host->stream.rcv.pluginsd_chart_slots.spinlock); + + if(host->stream.rcv.pluginsd_chart_slots.array) { + for (size_t s = 0; s < host->stream.rcv.pluginsd_chart_slots.size; s++) + rrdset_pluginsd_receive_unslot_and_cleanup(host->stream.rcv.pluginsd_chart_slots.array[s]); + + freez(host->stream.rcv.pluginsd_chart_slots.array); + host->stream.rcv.pluginsd_chart_slots.array = NULL; + host->stream.rcv.pluginsd_chart_slots.size = 0; + } + + spinlock_unlock(&host->stream.rcv.pluginsd_chart_slots.spinlock); +} diff --git a/src/database/rrdhost-slots.h b/src/database/rrdhost-slots.h new file mode 100644 index 00000000000000..6a62350b39e48b --- /dev/null +++ b/src/database/rrdhost-slots.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRDHOST_SLOTS_H +#define NETDATA_RRDHOST_SLOTS_H + +#include "rrdhost.h" + +void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host); +void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host); + + +#endif //NETDATA_RRDHOST_SLOTS_H diff --git a/src/database/rrdhost-status.h b/src/database/rrdhost-status.h index 3dba8f37e50352..0c88a003f6684b 100644 --- a/src/database/rrdhost-status.h +++ b/src/database/rrdhost-status.h @@ -95,7 +95,7 @@ typedef struct rrdhost_status_t { struct { RRDHOST_DB_STATUS status; RRDHOST_DB_LIVENESS liveness; - RRD_MEMORY_MODE mode; + RRD_DB_MODE mode; time_t first_time_s; time_t last_time_s; size_t metrics; diff --git a/src/database/rrdhost.c b/src/database/rrdhost.c index ef0fa6bcf59c39..ecafb40ce507ab 100644 --- a/src/database/rrdhost.c +++ b/src/database/rrdhost.c @@ -187,7 +187,8 @@ static inline void rrdhost_init_timezone(RRDHOST *host, const char *timezone, co host->utc_offset = utc_offset; } -void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, +void set_host_properties(RRDHOST *host, int update_every, + RRD_DB_MODE memory_mode, const char *registry_hostname, const char *os, const char *tzone, const char *abbrev_tzone, int32_t utc_offset, const char *prog_name, const char *prog_version) @@ -238,7 +239,7 @@ static RRDHOST *prepare_host_for_unittest(RRDHOST *host) snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", host->cache_dir); if ((initialized = create_dbengine_directory(host, dbenginepath))) { - host->db[0].mode = RRD_MEMORY_MODE_DBENGINE; + host->db[0].mode = RRD_DB_MODE_DBENGINE; host->db[0].eng = storage_engine_get(host->db[0].mode); host->db[0].tier_grouping = get_tier_grouping(0); @@ -274,20 +275,20 @@ static RRDHOST *prepare_host_for_unittest(RRDHOST *host) } #endif -static void rrdhost_set_replication_parameters(RRDHOST *host, RRD_MEMORY_MODE memory_mode, time_t period, time_t step) { +static void rrdhost_set_replication_parameters(RRDHOST *host, RRD_DB_MODE memory_mode, time_t period, time_t step) { host->stream.replication.period = period; host->stream.replication.step = step; host->stream.rcv.status.replication.percent = 100.0; switch(memory_mode) { default: - case RRD_MEMORY_MODE_ALLOC: - case RRD_MEMORY_MODE_RAM: + case RRD_DB_MODE_ALLOC: + case RRD_DB_MODE_RAM: if(host->stream.replication.period > (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every) host->stream.replication.period = (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every; break; - case RRD_MEMORY_MODE_DBENGINE: + case RRD_DB_MODE_DBENGINE: break; } } @@ -304,7 +305,7 @@ RRDHOST *rrdhost_create( const char *prog_version, int update_every, long entries, - RRD_MEMORY_MODE memory_mode, + RRD_DB_MODE memory_mode, bool health, bool stream, STRING *parents, @@ -317,12 +318,12 @@ RRDHOST *rrdhost_create( int is_localhost, bool archived ) { - if(memory_mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled) { + if(memory_mode == RRD_DB_MODE_DBENGINE && !dbengine_enabled) { nd_log(NDLS_DAEMON, NDLP_ERR, "memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'", hostname); - memory_mode = RRD_MEMORY_MODE_ALLOC; + memory_mode = RRD_DB_MODE_ALLOC; } RRDHOST *host = callocz(1, sizeof(RRDHOST)); @@ -349,7 +350,7 @@ RRDHOST *rrdhost_create( rrdhost_init_hostname(host, hostname, false); host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries); - host->health.enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health; + host->health.enabled = ((memory_mode == RRD_DB_MODE_NONE)) ? 0 : health; spinlock_init(&host->receiver_lock); @@ -385,7 +386,7 @@ RRDHOST *rrdhost_create( rrdcalc_rrdhost_index_init(host); - if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + if (host->rrd_memory_mode == RRD_DB_MODE_DBENGINE) { #ifdef ENABLE_DBENGINE if (unittest_running) { host = prepare_host_for_unittest(host); @@ -394,14 +395,14 @@ RRDHOST *rrdhost_create( } else { for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) { - host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE; + host->db[tier].mode = RRD_DB_MODE_DBENGINE; host->db[tier].eng = storage_engine_get(host->db[tier].mode); host->db[tier].si = (STORAGE_INSTANCE *)multidb_ctx[tier]; host->db[tier].tier_grouping = get_tier_grouping(tier); } } #else - fatal("RRD_MEMORY_MODE_DBENGINE is not supported in this platform."); + fatal("RRD_DB_MODE_DBENGINE is not supported in this platform."); #endif } else { @@ -413,7 +414,7 @@ RRDHOST *rrdhost_create( #ifdef ENABLE_DBENGINE // the first tier is reserved for the non-dbengine modes for(size_t tier = 1; tier < nd_profile.storage_tiers; tier++) { - host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE; + host->db[tier].mode = RRD_DB_MODE_DBENGINE; host->db[tier].eng = storage_engine_get(host->db[tier].mode); host->db[tier].si = (STORAGE_INSTANCE *) multidb_ctx[tier]; host->db[tier].tier_grouping = get_tier_grouping(tier); @@ -519,7 +520,8 @@ static void rrdhost_update(RRDHOST *host , const char *prog_version , int update_every , long history - , RRD_MEMORY_MODE mode + , + RRD_DB_MODE mode , bool health , bool stream , STRING *parents @@ -535,7 +537,7 @@ static void rrdhost_update(RRDHOST *host spinlock_lock(&host->rrdhost_update_lock); - host->health.enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health; + host->health.enabled = (mode == RRD_DB_MODE_NONE) ? 0 : health; { struct rrdhost_system_info *old = host->system_info; @@ -596,7 +598,7 @@ static void rrdhost_update(RRDHOST *host rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode)); - else if(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && host->rrd_history_entries < history) + else if(host->rrd_memory_mode != RRD_DB_MODE_DBENGINE && host->rrd_history_entries < history) nd_log(NDLS_DAEMON, NDLP_WARNING, "Host '%s' has history of %d entries, but the wanted one is %ld entries. " "Restart netdata here to apply the new settings.", @@ -654,7 +656,8 @@ RRDHOST *rrdhost_find_or_create( , const char *prog_version , int update_every , long history - , RRD_MEMORY_MODE mode + , + RRD_DB_MODE mode , bool health , bool stream , STRING *parents @@ -880,131 +883,3 @@ void rrdhost_free_all(void) { rrd_wrunlock(); } - -void rrd_finalize_collection_for_all_hosts(void) { - RRDHOST *host; - dfe_start_reentrant(rrdhost_root_index, host) { - rrdhost_finalize_collection(host); - } - dfe_done(host); -} - -void rrdhost_set_is_parent_label(void) { - uint32_t count = stream_receivers_currently_connected(); - - if (count == 0 || count == 1) { - RRDLABELS *labels = localhost->rrdlabels; - rrdlabels_add(labels, "_is_parent", (count) ? "true" : "false", RRDLABEL_SRC_AUTO); - - // queue a node info - aclk_queue_node_info(localhost, false); - } -} - -static bool config_label_cb(void *data __maybe_unused, const char *name, const char *value) { - rrdlabels_add(localhost->rrdlabels, name, value, RRDLABEL_SRC_CONFIG); - return true; -} - -static void rrdhost_load_config_labels(void) { - int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL); - if(!status) { - char *filename = CONFIG_DIR "/" CONFIG_FILENAME; - nd_log(NDLS_DAEMON, NDLP_WARNING, - "RRDLABEL: Cannot reload the configuration file '%s', using labels in memory", - filename); - } - - appconfig_foreach_value_in_section(&netdata_config, CONFIG_SECTION_HOST_LABEL, config_label_cb, NULL); -} - -static void rrdhost_load_kubernetes_labels(void) { - char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)]; - sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh"); - - if (unlikely(access(label_script, R_OK) != 0)) { - nd_log(NDLS_DAEMON, NDLP_ERR, - "Kubernetes pod label fetching script %s not found.", - label_script); - - return; - } - - POPEN_INSTANCE *instance = spawn_popen_run(label_script); - if(!instance) return; - - char buffer[1000 + 1]; - while (fgets(buffer, 1000, spawn_popen_stdout(instance)) != NULL) - rrdlabels_add_pair(localhost->rrdlabels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S); - - // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':' - // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null - int rc = spawn_popen_wait(instance); - if(rc) - nd_log(NDLS_DAEMON, NDLP_ERR, - "%s exited abnormally. Failed to get kubernetes labels.", - label_script); -} - -static void rrdhost_load_auto_labels(void) { - RRDLABELS *labels = localhost->rrdlabels; - - rrdhost_system_info_to_rrdlabels(localhost->system_info, labels); - add_aclk_host_labels(); - - // The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf). - // Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084. - - int is_ephemeral = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "is ephemeral node", CONFIG_BOOLEAN_NO); - rrdlabels_add(labels, "_is_ephemeral", is_ephemeral ? "true" : "false", RRDLABEL_SRC_AUTO); - - int has_unstable_connection = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "has unstable connection", CONFIG_BOOLEAN_NO); - rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_AUTO); - - rrdlabels_add(labels, "_is_parent", (stream_receivers_currently_connected() > 0) ? "true" : "false", RRDLABEL_SRC_AUTO); - - rrdlabels_add(labels, "_hostname", string2str(localhost->hostname), RRDLABEL_SRC_AUTO); - rrdlabels_add(labels, "_os", string2str(localhost->os), RRDLABEL_SRC_AUTO); - - if (localhost->stream.snd.destination) - rrdlabels_add(labels, "_streams_to", string2str(localhost->stream.snd.destination), RRDLABEL_SRC_AUTO); -} - -void reload_host_labels(void) { - if(!localhost->rrdlabels) - localhost->rrdlabels = rrdlabels_create(); - - rrdlabels_unmark_all(localhost->rrdlabels); - - // priority is important here - rrdhost_load_config_labels(); - rrdhost_load_kubernetes_labels(); - rrdhost_load_auto_labels(); - - rrdhost_flag_set(localhost,RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE); - - stream_send_host_labels(localhost); -} - -void rrdhost_finalize_collection(RRDHOST *host) { - ND_LOG_STACK lgs[] = { - ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(host)), - ND_LOG_FIELD_END(), - }; - ND_LOG_STACK_PUSH(lgs); - - nd_log(NDLS_DAEMON, NDLP_DEBUG, - "RRD: 'host:%s' stopping data collection...", - rrdhost_hostname(host)); - - RRDSET *st; - rrdset_foreach_read(st, host) - rrdset_finalize_collection(st, true); - rrdset_foreach_done(st); -} - -bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now) { - time_t first_time_s, last_time_s; - rrdhost_retention(host, now, rrdhost_is_online(host), &first_time_s, &last_time_s); - return query_matches_retention(after, before, first_time_s, last_time_s, 0); -} diff --git a/src/database/rrdhost.h b/src/database/rrdhost.h index f9f2498deeda3f..efe09eee1778d8 100644 --- a/src/database/rrdhost.h +++ b/src/database/rrdhost.h @@ -12,13 +12,14 @@ typedef struct rrdhost RRDHOST; typedef struct ml_host rrd_ml_host_t; typedef struct rrdhost_acquired RRDHOST_ACQUIRED; -#include "streaming/stream-traffic-types.h" +//#include "streaming/stream-traffic-types.h" #include "streaming/stream-sender-commit.h" -#include "streaming/stream-replication-tracking.h" +#include "rrd-database-mode.h" +//#include "streaming/stream-replication-tracking.h" #include "streaming/stream-parents.h" #include "streaming/stream-path.h" #include "storage-engine.h" -#include "streaming/stream-traffic-types.h" +//#include "streaming/stream-traffic-types.h" #include "rrdlabels.h" #include "health/health-alert-log.h" @@ -143,13 +144,13 @@ struct rrdhost { int32_t rrd_update_every; // the update frequency of the host int32_t rrd_history_entries; // the number of history entries for the host's charts - RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host - // the actual per tier is at .db[tier].mode + RRD_DB_MODE rrd_memory_mode; // the configured memory more for the charts of this host + // the actual per tier is at .db[tier].mode char *cache_dir; // the directory to save RRD cache files struct { - RRD_MEMORY_MODE mode; // the db mode for this tier + RRD_DB_MODE mode; // the db mode for this tier STORAGE_ENGINE *eng; // the storage engine API for this tier STORAGE_INSTANCE *si; // the db instance for this tier uint32_t tier_grouping; // tier 0 iterations aggregated on this tier @@ -393,7 +394,7 @@ RRDHOST *rrdhost_create( const char *prog_version, int update_every, long entries, - RRD_MEMORY_MODE memory_mode, + RRD_DB_MODE memory_mode, bool health, bool stream, STRING *parents, @@ -422,7 +423,7 @@ RRDHOST *rrdhost_find_or_create( const char *prog_version, int update_every, long history, - RRD_MEMORY_MODE mode, + RRD_DB_MODE mode, bool health, bool stream, STRING *parents, @@ -441,11 +442,9 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force); bool rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s); bool rrdhost_should_run_health(RRDHOST *host); -void reload_host_labels(void); -void rrdhost_set_is_parent_label(void); - void set_host_properties( - RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *registry_hostname, + RRDHOST *host, int update_every, + RRD_DB_MODE memory_mode, const char *registry_hostname, const char *os, const char *tzone, const char *abbrev_tzone, int32_t utc_offset, const char *prog_name, const char *prog_version); @@ -463,10 +462,11 @@ static inline void rrdhost_retention(RRDHOST *host, time_t now, bool online, tim *to = online ? now : last_time_s; } -void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host); -void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host); - extern time_t rrdhost_free_orphan_time_s; extern time_t rrdhost_free_ephemeral_time_s; +#include "rrdhost-collection.h" +#include "rrdhost-slots.h" +#include "rrdhost-labels.h" + #endif //NETDATA_RRDHOST_H diff --git a/src/database/rrdset-collection.c b/src/database/rrdset-collection.c index fb36abc2aafb76..ac79802ccd295a 100644 --- a/src/database/rrdset-collection.c +++ b/src/database/rrdset-collection.c @@ -273,83 +273,6 @@ static inline void rrdset_init_last_updated_time(RRDSET *st) { __thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS]; -#ifdef NETDATA_LOG_COLLECTION_ERRORS -void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function) { -#else // !NETDATA_LOG_COLLECTION_ERRORS -void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) { -#endif // !NETDATA_LOG_COLLECTION_ERRORS - - static __thread struct log_stack_entry lgs[] = { - [0] = ND_LOG_FIELD_STR(NDF_NIDL_DIMENSION, NULL), - [1] = ND_LOG_FIELD_END(), - }; - lgs[0].str = rd->id; - log_stack_push(lgs); - -#ifdef NETDATA_LOG_COLLECTION_ERRORS - rd->rrddim_store_metric_count++; - - if(likely(rd->rrddim_store_metric_count > 1)) { - usec_t expected = rd->rrddim_store_metric_last_ut + rd->update_every * USEC_PER_SEC; - - if(point_end_time_ut != rd->rrddim_store_metric_last_ut) { - internal_error(true, - "%s COLLECTION: 'host:%s/chart:%s/dim:%s' granularity %d, collection %zu, expected to store at tier 0 a value at %llu, but it gave %llu [%s%llu usec] (called from %s(), previously by %s())", - (point_end_time_ut < rd->rrddim_store_metric_last_ut) ? "**PAST**" : "GAP", - rrdhost_hostname(rd->rrdset->rrdhost), rrdset_id(rd->rrdset), rrddim_id(rd), - rd->update_every, - rd->rrddim_store_metric_count, - expected, point_end_time_ut, - (point_end_time_ut < rd->rrddim_store_metric_last_ut)?"by -" : "gap ", - expected - point_end_time_ut, - function, - rd->rrddim_store_metric_last_caller?rd->rrddim_store_metric_last_caller:"none"); - } - } - - rd->rrddim_store_metric_last_ut = point_end_time_ut; - rd->rrddim_store_metric_last_caller = function; -#endif // NETDATA_LOG_COLLECTION_ERRORS - - // store the metric on tier 0 - storage_engine_store_metric(rd->tiers[0].sch, point_end_time_ut, - n, 0, 0, - 1, 0, flags); - - rrdset_done_statistics_points_stored_per_tier[0]++; - - time_t now_s = (time_t)(point_end_time_ut / USEC_PER_SEC); - - STORAGE_POINT sp = { - .start_time_s = now_s - rd->rrdset->update_every, - .end_time_s = now_s, - .min = n, - .max = n, - .sum = n, - .count = 1, - .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1, - .flags = flags - }; - - for(size_t tier = 1; tier < nd_profile.storage_tiers;tier++) { - if(unlikely(!rd->tiers[tier].smh)) continue; - - struct rrddim_tier *t = &rd->tiers[tier]; - - if(!rrddim_option_check(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS)) { - // we have not collected this tier before - // let's fill any gap that may exist - backfill_tier_from_smaller_tiers(rd, tier, now_s); - } - - store_metric_at_tier(rd, tier, t, sp, point_end_time_ut); - } - rrddim_option_set(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS); - - rrdcontext_collected_rrddim(rd); - log_stack_pop(&lgs); -} - // caching of dimensions rrdset_done() and rrdset_done_interpolate() loop through struct rda_item { const DICTIONARY_ITEM *item; @@ -360,7 +283,7 @@ struct rda_item { static __thread struct rda_item *thread_rda = NULL; static __thread size_t thread_rda_entries = 0; -struct rda_item *rrdset_thread_rda_get(size_t *dimensions) { +static struct rda_item *rrdset_thread_rda_get(size_t *dimensions) { if(unlikely(!thread_rda || (*dimensions) > thread_rda_entries)) { size_t old_mem = thread_rda_entries * sizeof(struct rda_item); @@ -665,7 +588,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next) // check if we will re-write the entire data set if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->db.entries * update_every_ut && - st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) { + st->rrd_memory_mode != RRD_DB_MODE_DBENGINE)) { nd_log_daemon(NDLP_DEBUG, "'%s': too old data (last updated at %" PRId64 ".%" PRId64 ", last collected at %" PRId64 ".%" PRId64 "). " "Resetting it. Will not store the next entry.", rrdset_id(st), diff --git a/src/database/rrdset-collection.h b/src/database/rrdset-collection.h index 5bef081d455259..67e4a56f122e4d 100644 --- a/src/database/rrdset-collection.h +++ b/src/database/rrdset-collection.h @@ -7,6 +7,14 @@ extern __thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS]; +void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds); +void rrdset_next_usec(RRDSET *st, usec_t microseconds); +void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t microseconds); +#define rrdset_next(st) rrdset_next_usec(st, 0ULL) + +void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next); +void rrdset_done(RRDSET *st); + void rrdset_finalize_collection(RRDSET *st, bool dimensions_too); time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s); diff --git a/src/database/rrdset-index-id.c b/src/database/rrdset-index-id.c index e16f28607cf9aa..b5a963519eb985 100644 --- a/src/database/rrdset-index-id.c +++ b/src/database/rrdset-index-id.c @@ -28,7 +28,7 @@ struct rrdset_constructor { long priority; int update_every; RRDSET_TYPE chart_type; - RRD_MEMORY_MODE memory_mode; + RRD_DB_MODE memory_mode; long history_entries; enum { @@ -69,7 +69,7 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v st->module_name = rrd_string_strdupz(ctr->module); st->priority = ctr->priority; - st->db.entries = (ctr->memory_mode != RRD_MEMORY_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5; + st->db.entries = (ctr->memory_mode != RRD_DB_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5; st->update_every = ctr->update_every; st->rrd_memory_mode = ctr->memory_mode; @@ -378,7 +378,8 @@ RRDSET *rrdset_create_custom( , long priority , int update_every , RRDSET_TYPE chart_type - , RRD_MEMORY_MODE memory_mode + , + RRD_DB_MODE memory_mode , long history_entries ) { if (host != localhost) diff --git a/src/database/rrdset-index-id.h b/src/database/rrdset-index-id.h index 1862d5e9974bc6..0a95d07f041fb4 100644 --- a/src/database/rrdset-index-id.h +++ b/src/database/rrdset-index-id.h @@ -19,7 +19,8 @@ RRDSET *rrdset_create_custom( , long priority , int update_every , RRDSET_TYPE chart_type - , RRD_MEMORY_MODE memory_mode + , + RRD_DB_MODE memory_mode , long history_entries ); @@ -59,8 +60,6 @@ static inline localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type); } -int rrdset_reset_name(RRDSET *st, const char *name); - void rrdset_index_init(RRDHOST *host); void rrdset_index_destroy(RRDHOST *host); @@ -74,4 +73,18 @@ RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id); void rrdset_acquired_release(RRDSET_ACQUIRED *rsa); RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa); +#define rrdset_find_localhost(id) rrdset_find(localhost, id) +/* This will not return charts that are archived */ +static inline RRDSET *rrdset_find_active_localhost(const char *id) { + RRDSET *st = rrdset_find_localhost(id); + return st; +} + +#define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id) +/* This will not return charts that are archived */ +static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id) { + RRDSET *st = rrdset_find_bytype_localhost(type, id); + return st; +} + #endif //NETDATA_RRDSET_INDEX_ID_H diff --git a/src/database/rrdset-index-name.c b/src/database/rrdset-index-name.c index f42976c6055bb1..9c8b918ae73776 100644 --- a/src/database/rrdset-index-name.c +++ b/src/database/rrdset-index-name.c @@ -4,6 +4,8 @@ #include "rrdset-index-id.h" #include "rrdset-slots.h" +static RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name); + STRING *rrdset_fix_name(RRDHOST *host, const char *chart_full_id, const char *type, const char *current_name, const char *name) { if(!name || !*name) return NULL; @@ -74,7 +76,7 @@ static void rrdset_name_delete_callback(const DICTIONARY_ITEM *item __maybe_unus rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME); } -RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name) { +static RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name) { if (unlikely(!host->rrdset_root_index_name)) return NULL; return dictionary_get(host->rrdset_root_index_name, name); @@ -101,7 +103,7 @@ void rrdset_index_del_name(RRDHOST *host, RRDSET *st) { dictionary_del(host->rrdset_root_index_name, rrdset_name(st)); } -inline RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) { +RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) { netdata_log_debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, rrdhost_hostname(host)); RRDSET *st = rrdset_index_find_name(host, name); return(st); diff --git a/src/database/rrdset-index-name.h b/src/database/rrdset-index-name.h index f9cc97918fe083..7d31944c594969 100644 --- a/src/database/rrdset-index-name.h +++ b/src/database/rrdset-index-name.h @@ -11,7 +11,15 @@ void rrdset_index_byname_init(RRDHOST *host); void rrdset_index_add_name(RRDHOST *host, RRDSET *st); void rrdset_index_del_name(RRDHOST *host, RRDSET *st); -RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name); +extern RRDHOST *localhost; +RRDSET *rrdset_find_byname(RRDHOST *host, const char *name); +#define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name) + +/* This will not return charts that are archived */ +static inline RRDSET *rrdset_find_active_byname_localhost(const char *name) { + RRDSET *st = rrdset_find_byname_localhost(name); + return st; +} int rrdset_reset_name(RRDSET *st, const char *name); STRING *rrdset_fix_name(RRDHOST *host, const char *chart_full_id, const char *type, const char *current_name, const char *name); diff --git a/src/database/rrdset-slots.c b/src/database/rrdset-slots.c index 34c6cb25472503..ece2a8efd6ee61 100644 --- a/src/database/rrdset-slots.c +++ b/src/database/rrdset-slots.c @@ -41,25 +41,6 @@ void rrdset_stream_send_chart_slot_release(RRDSET *st) { spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock); } -void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host) { - rrd_slot_memory_removed(host->stream.snd.pluginsd_chart_slots.available.size * sizeof(uint32_t)); - - spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock); - host->stream.snd.pluginsd_chart_slots.available.ignore = true; - freez(host->stream.snd.pluginsd_chart_slots.available.array); - host->stream.snd.pluginsd_chart_slots.available.array = NULL; - host->stream.snd.pluginsd_chart_slots.available.used = 0; - host->stream.snd.pluginsd_chart_slots.available.size = 0; - spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock); - - // zero all the slots on all charts, so that they will not attempt to access the array - RRDSET *st; - rrdset_foreach_read(st, host) { - st->stream.snd.chart_slot = 0; - } - rrdset_foreach_done(st); -} - void rrdset_pluginsd_receive_unslot(RRDSET *st) { for(size_t i = 0; i < st->pluginsd.size ;i++) { rrddim_acquired_release(st->pluginsd.prd_array[i].rda); // can be NULL @@ -105,20 +86,3 @@ void rrdset_pluginsd_receive_slots_initialize(RRDSET *st) { spinlock_init(&st->pluginsd.spinlock); st->pluginsd.last_slot = -1; } - -void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host) { - rrd_slot_memory_removed(host->stream.rcv.pluginsd_chart_slots.size * sizeof(uint32_t)); - - spinlock_lock(&host->stream.rcv.pluginsd_chart_slots.spinlock); - - if(host->stream.rcv.pluginsd_chart_slots.array) { - for (size_t s = 0; s < host->stream.rcv.pluginsd_chart_slots.size; s++) - rrdset_pluginsd_receive_unslot_and_cleanup(host->stream.rcv.pluginsd_chart_slots.array[s]); - - freez(host->stream.rcv.pluginsd_chart_slots.array); - host->stream.rcv.pluginsd_chart_slots.array = NULL; - host->stream.rcv.pluginsd_chart_slots.size = 0; - } - - spinlock_unlock(&host->stream.rcv.pluginsd_chart_slots.spinlock); -} diff --git a/src/database/rrdset-slots.h b/src/database/rrdset-slots.h index 1fb57600fbbcfe..12f5f9a3adae02 100644 --- a/src/database/rrdset-slots.h +++ b/src/database/rrdset-slots.h @@ -8,10 +8,8 @@ void rrdset_stream_send_chart_slot_assign(RRDSET *st); void rrdset_stream_send_chart_slot_release(RRDSET *st); -void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host); void rrdset_pluginsd_receive_unslot(RRDSET *st); void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st); void rrdset_pluginsd_receive_slots_initialize(RRDSET *st); -void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host); #endif //NETDATA_RRDSET_SLOTS_H diff --git a/src/database/rrdset-type.c b/src/database/rrdset-type.c new file mode 100644 index 00000000000000..a22c22965dd071 --- /dev/null +++ b/src/database/rrdset-type.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rrdset-type.h" + +RRDSET_TYPE rrdset_type_id(const char *name) { + if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0)) + return RRDSET_TYPE_AREA; + + else if(unlikely(strcmp(name, RRDSET_TYPE_STACKED_NAME) == 0)) + return RRDSET_TYPE_STACKED; + + else if(unlikely(strcmp(name, RRDSET_TYPE_HEATMAP_NAME) == 0)) + return RRDSET_TYPE_HEATMAP; + + else // if(unlikely(strcmp(name, RRDSET_TYPE_LINE_NAME) == 0)) + return RRDSET_TYPE_LINE; +} + +const char *rrdset_type_name(RRDSET_TYPE chart_type) { + switch(chart_type) { + case RRDSET_TYPE_LINE: + default: + return RRDSET_TYPE_LINE_NAME; + + case RRDSET_TYPE_AREA: + return RRDSET_TYPE_AREA_NAME; + + case RRDSET_TYPE_STACKED: + return RRDSET_TYPE_STACKED_NAME; + + case RRDSET_TYPE_HEATMAP: + return RRDSET_TYPE_HEATMAP_NAME; + } +} diff --git a/src/database/rrdset-type.h b/src/database/rrdset-type.h new file mode 100644 index 00000000000000..e771a3ffaff9a4 --- /dev/null +++ b/src/database/rrdset-type.h @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRDSET_TYPE_H +#define NETDATA_RRDSET_TYPE_H + +#include "libnetdata/libnetdata.h" + +typedef enum __attribute__ ((__packed__)) rrdset_type { + RRDSET_TYPE_LINE = 0, + RRDSET_TYPE_AREA = 1, + RRDSET_TYPE_STACKED = 2, + RRDSET_TYPE_HEATMAP = 3, +} RRDSET_TYPE; + +#define RRDSET_TYPE_LINE_NAME "line" +#define RRDSET_TYPE_AREA_NAME "area" +#define RRDSET_TYPE_STACKED_NAME "stacked" +#define RRDSET_TYPE_HEATMAP_NAME "heatmap" + +RRDSET_TYPE rrdset_type_id(const char *name); +const char *rrdset_type_name(RRDSET_TYPE chart_type); + +#endif //NETDATA_RRDSET_TYPE_H diff --git a/src/database/rrdset.c b/src/database/rrdset.c index 585d4f9933e2c9..9a7bbc26d522b4 100644 --- a/src/database/rrdset.c +++ b/src/database/rrdset.c @@ -9,41 +9,6 @@ void rrdset_metadata_updated(RRDSET *st) { } // ---------------------------------------------------------------------------- -// RRD - chart types - -inline RRDSET_TYPE rrdset_type_id(const char *name) { - if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0)) - return RRDSET_TYPE_AREA; - - else if(unlikely(strcmp(name, RRDSET_TYPE_STACKED_NAME) == 0)) - return RRDSET_TYPE_STACKED; - - else if(unlikely(strcmp(name, RRDSET_TYPE_HEATMAP_NAME) == 0)) - return RRDSET_TYPE_HEATMAP; - - else // if(unlikely(strcmp(name, RRDSET_TYPE_LINE_NAME) == 0)) - return RRDSET_TYPE_LINE; -} - -const char *rrdset_type_name(RRDSET_TYPE chart_type) { - switch(chart_type) { - case RRDSET_TYPE_LINE: - default: - return RRDSET_TYPE_LINE_NAME; - - case RRDSET_TYPE_AREA: - return RRDSET_TYPE_AREA_NAME; - - case RRDSET_TYPE_STACKED: - return RRDSET_TYPE_STACKED_NAME; - - case RRDSET_TYPE_HEATMAP: - return RRDSET_TYPE_HEATMAP_NAME; - } -} - -// ---------------------------------------------------------------------------- -// RRDSET - rename charts // get the timestamp of the last entry in the round-robin database time_t rrdset_last_entry_s(RRDSET *st) { @@ -148,7 +113,7 @@ void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_ *last_time_s = db_last_entry_s; } -inline void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st) { +void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st) { if(!st) return; rrdset_pluginsd_receive_unslot(st); @@ -171,7 +136,7 @@ inline void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st) { } } -inline void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st) { +void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st) { if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) { // netdata_log_info("Clearing obsolete flag on chart 'host:%s/chart:%s'", @@ -188,7 +153,7 @@ inline void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st) { } } -inline void rrdset_update_heterogeneous_flag(RRDSET *st) { +void rrdset_update_heterogeneous_flag(RRDSET *st) { RRDHOST *host = st->rrdhost; (void)host; diff --git a/src/database/rrdset.h b/src/database/rrdset.h index fa6877fd9fc0ac..9c8d70c9083128 100644 --- a/src/database/rrdset.h +++ b/src/database/rrdset.h @@ -9,28 +9,12 @@ typedef struct rrdset RRDSET; typedef struct rrdset_acquired RRDSET_ACQUIRED; typedef struct ml_chart rrd_ml_chart_t; +#include "rrdset-type.h" #include "rrdlabels.h" #include "rrd-database-mode.h" // -------------------------------------------------------------------------------------------------------------------- -typedef enum __attribute__ ((__packed__)) rrdset_type { - RRDSET_TYPE_LINE = 0, - RRDSET_TYPE_AREA = 1, - RRDSET_TYPE_STACKED = 2, - RRDSET_TYPE_HEATMAP = 3, -} RRDSET_TYPE; - -#define RRDSET_TYPE_LINE_NAME "line" -#define RRDSET_TYPE_AREA_NAME "area" -#define RRDSET_TYPE_STACKED_NAME "stacked" -#define RRDSET_TYPE_HEATMAP_NAME "heatmap" - -RRDSET_TYPE rrdset_type_id(const char *name); -const char *rrdset_type_name(RRDSET_TYPE chart_type); - -// -------------------------------------------------------------------------------------------------------------------- - struct rrdhost; struct rrdcalc; struct pluginsd_rrddim; @@ -128,7 +112,7 @@ struct rrdset { // operational state members RRDSET_FLAGS flags; // flags - RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset + RRD_DB_MODE rrd_memory_mode; // the db mode of this rrdset DICTIONARY *rrddim_root_index; // dimensions index @@ -295,69 +279,33 @@ void rrdset_metadata_updated(RRDSET *st); // -------------------------------------------------------------------------------------------------------------------- #ifdef NETDATA_INTERNAL_CHECKS -#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \ - netdata_logger(NDLS_DEBUG, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); } while(0) +#define rrdset_debug(st, fmt, args...) do { \ + if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \ + netdata_logger(NDLS_DEBUG, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); \ + } while(0) #else #define rrdset_debug(st, fmt, args...) debug_dummy() #endif // -------------------------------------------------------------------------------------------------------------------- -#include "rrdset-index-name.h" -#include "rrdset-index-id.h" -#include "rrdset-slots.h" -#include "rrdset-collection.h" - void rrdset_update_heterogeneous_flag(RRDSET *st); -time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s); - -RRDSET *rrdset_find(RRDHOST *host, const char *id); - -RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id); -RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa); -void rrdset_acquired_release(RRDSET_ACQUIRED *rsa); - -#define rrdset_find_localhost(id) rrdset_find(localhost, id) -/* This will not return charts that are archived */ -static inline RRDSET *rrdset_find_active_localhost(const char *id) -{ - RRDSET *st = rrdset_find_localhost(id); - return st; -} - -RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id); -#define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id) -/* This will not return charts that are archived */ -static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id) -{ - RRDSET *st = rrdset_find_bytype_localhost(type, id); - return st; -} - -RRDSET *rrdset_find_byname(RRDHOST *host, const char *name); -#define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name) -/* This will not return charts that are archived */ -static inline RRDSET *rrdset_find_active_byname_localhost(const char *name) -{ - RRDSET *st = rrdset_find_byname_localhost(name); - return st; -} - -void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds); -void rrdset_next_usec(RRDSET *st, usec_t microseconds); -void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t microseconds); -#define rrdset_next(st) rrdset_next_usec(st, 0ULL) - -void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next); -void rrdset_done(RRDSET *st); - void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st); void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st); // checks if the RRDSET should be offered to viewers -#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE) -#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st)) +#define rrdset_is_available_for_viewers(st) ( \ + !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && \ + !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && \ + rrdset_number_of_dimensions(st) && \ + (st)->rrd_memory_mode != RRD_DB_MODE_NONE \ + ) + +#define rrdset_is_available_for_exporting_and_alarms(st) ( \ + !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && \ + rrdset_number_of_dimensions(st) \ + ) time_t rrdset_first_entry_s(RRDSET *st); time_t rrdset_first_entry_s_of_tier(RRDSET *st, size_t tier); @@ -368,9 +316,9 @@ void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_ void rrdset_update_rrdlabels(RRDSET *st, RRDLABELS *new_rrdlabels); -void rrdset_free(RRDSET *st); - -void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st); -void rrdset_pluginsd_receive_unslot(RRDSET *st); +#include "rrdset-index-name.h" +#include "rrdset-index-id.h" +#include "rrdset-slots.h" +#include "rrdset-collection.h" #endif //NETDATA_RRDSET_H diff --git a/src/database/sqlite/sqlite_functions.h b/src/database/sqlite/sqlite_functions.h index ac802d18f9ed99..11b9ceee37cc65 100644 --- a/src/database/sqlite/sqlite_functions.h +++ b/src/database/sqlite/sqlite_functions.h @@ -51,7 +51,7 @@ void analytics_set_data_str(char **name, const char *value); #define REQUIRE_DB(db) \ ({ \ if (unlikely(!(db))) { \ - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) \ + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE) \ error_report("Database has not been initialized in %s", __FUNCTION__); \ } \ (db) != NULL; \ diff --git a/src/database/storage-engine.c b/src/database/storage-engine.c index b52ec5b7d7467a..c0501e50fde1dc 100644 --- a/src/database/storage-engine.c +++ b/src/database/storage-engine.c @@ -8,8 +8,8 @@ static STORAGE_ENGINE engines[] = { { - .id = RRD_MEMORY_MODE_NONE, - .name = RRD_MEMORY_MODE_NONE_NAME, + .id = RRD_DB_MODE_NONE, + .name = RRD_DB_MODE_NONE_NAME, .seb = STORAGE_ENGINE_BACKEND_RRDDIM, .api = { .metric_get_by_id = rrddim_metric_get_by_id, @@ -22,8 +22,8 @@ static STORAGE_ENGINE engines[] = { } }, { - .id = RRD_MEMORY_MODE_RAM, - .name = RRD_MEMORY_MODE_RAM_NAME, + .id = RRD_DB_MODE_RAM, + .name = RRD_DB_MODE_RAM_NAME, .seb = STORAGE_ENGINE_BACKEND_RRDDIM, .api = { .metric_get_by_id = rrddim_metric_get_by_id, @@ -36,8 +36,8 @@ static STORAGE_ENGINE engines[] = { } }, { - .id = RRD_MEMORY_MODE_ALLOC, - .name = RRD_MEMORY_MODE_ALLOC_NAME, + .id = RRD_DB_MODE_ALLOC, + .name = RRD_DB_MODE_ALLOC_NAME, .seb = STORAGE_ENGINE_BACKEND_RRDDIM, .api = { .metric_get_by_id = rrddim_metric_get_by_id, @@ -51,8 +51,8 @@ static STORAGE_ENGINE engines[] = { }, #ifdef ENABLE_DBENGINE { - .id = RRD_MEMORY_MODE_DBENGINE, - .name = RRD_MEMORY_MODE_DBENGINE_NAME, + .id = RRD_DB_MODE_DBENGINE, + .name = RRD_DB_MODE_DBENGINE_NAME, .seb = STORAGE_ENGINE_BACKEND_DBENGINE, .api = { .metric_get_by_id = rrdeng_metric_get_by_id, @@ -65,7 +65,7 @@ static STORAGE_ENGINE engines[] = { } }, #endif - { .id = RRD_MEMORY_MODE_NONE, .name = NULL } + { .id = RRD_DB_MODE_NONE, .name = NULL } }; STORAGE_ENGINE* storage_engine_find(const char* name) @@ -77,7 +77,7 @@ STORAGE_ENGINE* storage_engine_find(const char* name) return NULL; } -STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode) +STORAGE_ENGINE* storage_engine_get(RRD_DB_MODE mmode) { for (STORAGE_ENGINE* it = engines; it->name; it++) { if (it->id == mmode) diff --git a/src/database/storage-engine.h b/src/database/storage-engine.h index bdc5eb02ceb47d..5331d3f1668725 100644 --- a/src/database/storage-engine.h +++ b/src/database/storage-engine.h @@ -70,12 +70,12 @@ typedef struct storage_engine_api { typedef struct storage { STORAGE_ENGINE_BACKEND seb; - RRD_MEMORY_MODE id; + RRD_DB_MODE id; const char* name; STORAGE_ENGINE_API api; } STORAGE_ENGINE; -STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode); +STORAGE_ENGINE* storage_engine_get(RRD_DB_MODE mmode); STORAGE_ENGINE* storage_engine_find(const char* name); // Iterator over existing engines diff --git a/src/exporting/check_filters.c b/src/exporting/check_filters.c index 19eecc9bc9b638..bfa4b7b3dc2d86 100644 --- a/src/exporting/check_filters.c +++ b/src/exporting/check_filters.c @@ -79,7 +79,7 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st) return 0; } - if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) { + if(unlikely(st->rrd_memory_mode == RRD_DB_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) { netdata_log_debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", rrdset_id(st), rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode)); return 0; } diff --git a/src/exporting/prometheus/prometheus.c b/src/exporting/prometheus/prometheus.c index f74453d886827a..be042341dc20fb 100644 --- a/src/exporting/prometheus/prometheus.c +++ b/src/exporting/prometheus/prometheus.c @@ -66,7 +66,7 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN } if (unlikely( - st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && + st->rrd_memory_mode == RRD_DB_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) { netdata_log_debug( D_EXPORTING, diff --git a/src/ml/ml_config.cc b/src/ml/ml_config.cc index 44bdc7abef0083..9c5d60a210d72a 100644 --- a/src/ml/ml_config.cc +++ b/src/ml/ml_config.cc @@ -144,7 +144,7 @@ void ml_config_load(ml_config_t *cfg) { cfg->enable_statistics_charts = enable_statistics_charts; - if (cfg->enable_anomaly_detection == CONFIG_BOOLEAN_AUTO && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) { + if (cfg->enable_anomaly_detection == CONFIG_BOOLEAN_AUTO && default_rrd_memory_mode != RRD_DB_MODE_DBENGINE) { Cfg.enable_anomaly_detection = 0; config_set_boolean(config_section_ml, "enabled", CONFIG_BOOLEAN_NO); return; diff --git a/src/streaming/stream-conf.c b/src/streaming/stream-conf.c index abfc8f84342a4c..7672aca47b5fcf 100644 --- a/src/streaming/stream-conf.c +++ b/src/streaming/stream-conf.c @@ -233,7 +233,7 @@ void stream_conf_receiver_config(struct receiver_state *rpt, struct stream_recei appconfig_get(&stream_config, api_key, "db", rrd_memory_mode_name(default_rrd_memory_mode)))); - if (unlikely(config->mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled)) { + if (unlikely(config->mode == RRD_DB_MODE_DBENGINE && !dbengine_enabled)) { netdata_log_error("STREAM RCV '%s' [from [%s]:%s]: " "dbengine is not enabled, falling back to default." , rpt->hostname diff --git a/src/streaming/stream-conf.h b/src/streaming/stream-conf.h index 6a2956b61a4e6e..d65e6e1a646e68 100644 --- a/src/streaming/stream-conf.h +++ b/src/streaming/stream-conf.h @@ -52,7 +52,7 @@ struct _stream_receive { extern struct _stream_receive stream_receive; struct stream_receiver_config { - RRD_MEMORY_MODE mode; + RRD_DB_MODE mode; bool ephemeral; int history; int update_every; diff --git a/src/streaming/stream-parents.h b/src/streaming/stream-parents.h index 24eda84d1eec96..3611033718a7ab 100644 --- a/src/streaming/stream-parents.h +++ b/src/streaming/stream-parents.h @@ -17,17 +17,11 @@ typedef struct rrdhost_stream_parents { } RRDHOST_STREAM_PARENTS; #include "stream-handshake.h" -#include "database/rrdhost-status.h" void rrdhost_stream_parent_ssl_init(struct sender_state *s); int stream_info_to_json_v1(BUFFER *wb, const char *machine_guid); -void rrdhost_stream_parents_reset(RRDHOST *host, STREAM_HANDSHAKE reason); - -void rrdhost_stream_parents_update_from_destination(RRDHOST *host); -void rrdhost_stream_parents_free(struct rrdhost *host, bool having_write_lock); - bool stream_parent_connect_to_one( ND_SOCK *sender_sock, struct rrdhost *host, @@ -46,6 +40,11 @@ bool stream_parent_is_ssl(STREAM_PARENT *d); usec_t stream_parent_handshake_error_to_json(BUFFER *wb, struct rrdhost *host); -void rrdhost_stream_parents_init(RRDHOST *host); +void rrdhost_stream_parents_reset(struct rrdhost *host, STREAM_HANDSHAKE reason); + +void rrdhost_stream_parents_update_from_destination(struct rrdhost *host); +void rrdhost_stream_parents_free(struct rrdhost *host, bool having_write_lock); + +void rrdhost_stream_parents_init(struct rrdhost *host); #endif //NETDATA_STREAM_PARENTS_H diff --git a/src/streaming/stream.h b/src/streaming/stream.h index 56e4e6329a7c14..67bc7e41fb6e86 100644 --- a/src/streaming/stream.h +++ b/src/streaming/stream.h @@ -21,7 +21,7 @@ void stream_sender_start_host(struct rrdhost *host); void stream_sender_signal_to_stop_and_wait(struct rrdhost *host, STREAM_HANDSHAKE reason, bool wait); // managing host sender structures -void stream_sender_structures_init(RRDHOST *host, bool stream, STRING *parents, STRING *api_key, STRING *send_charts_matching); +void stream_sender_structures_init(struct rrdhost *host, bool stream, STRING *parents, STRING *api_key, STRING *send_charts_matching); void stream_sender_structures_free(struct rrdhost *host); // querying host sender information diff --git a/src/database/KolmogorovSmirnovDist.c b/src/web/api/queries/KolmogorovSmirnovDist.c similarity index 100% rename from src/database/KolmogorovSmirnovDist.c rename to src/web/api/queries/KolmogorovSmirnovDist.c diff --git a/src/database/KolmogorovSmirnovDist.h b/src/web/api/queries/KolmogorovSmirnovDist.h similarity index 100% rename from src/database/KolmogorovSmirnovDist.h rename to src/web/api/queries/KolmogorovSmirnovDist.h diff --git a/src/web/api/queries/weights.c b/src/web/api/queries/weights.c index 8e7d76d172901a..8d8ce6055c94cb 100644 --- a/src/web/api/queries/weights.c +++ b/src/web/api/queries/weights.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "daemon/common.h" -#include "database/KolmogorovSmirnovDist.h" +#include "KolmogorovSmirnovDist.h" #define MAX_POINTS 10000 int metric_correlations_version = 1;