diff --git a/CMakeLists.txt b/CMakeLists.txt index 283cec70d22332..5ca7e36dd15ac3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -768,6 +768,14 @@ set(LIBNETDATA_FILES src/libnetdata/os/windows-perflib/perflib.h src/libnetdata/os/windows-perflib/perflib-names.c src/libnetdata/os/windows-perflib/perflib-dump.c + src/libnetdata/os/system-maps/cached-uid-username.c + src/libnetdata/os/system-maps/cached-uid-username.h + src/libnetdata/os/system-maps/cached-sid-username.c + src/libnetdata/os/system-maps/cached-sid-username.h + src/libnetdata/os/system-maps/cached-gid-groupname.c + src/libnetdata/os/system-maps/cached-gid-groupname.h + src/libnetdata/os/system-maps/cache-host-users-and-groups.c + src/libnetdata/os/system-maps/cache-host-users-and-groups.h src/libnetdata/spawn_server/spawn_server_nofork.c src/libnetdata/spawn_server/spawn_server.h src/libnetdata/spawn_server/spawn_popen.c @@ -845,6 +853,7 @@ set(LIBNETDATA_FILES src/libnetdata/spawn_server/log-forwarder.h src/libnetdata/log/nd_log-common.h src/libnetdata/log/nd_log-to-windows-common.h + src/libnetdata/common.h ) if(ENABLE_PLUGIN_EBPF) @@ -1332,9 +1341,7 @@ set(SYSTEMD_JOURNAL_PLUGIN_FILES src/collectors/systemd-journal.plugin/systemd-journal-fstat.c src/collectors/systemd-journal.plugin/systemd-journal-watcher.c src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c - src/libnetdata/maps/system-users.h - src/libnetdata/maps/system-groups.h - src/libnetdata/maps/system-services.h + src/libnetdata/os/system-maps/system-services.h src/collectors/systemd-journal.plugin/systemd-journal-sampling.h ) @@ -1464,8 +1471,6 @@ set(WINDOWS_EVENTS_PLUGIN_FILES src/collectors/windows-events.plugin/windows-events-query.c src/collectors/windows-events.plugin/windows-events-sources.c src/collectors/windows-events.plugin/windows-events-sources.h - src/collectors/windows-events.plugin/windows-events-sid.c - src/collectors/windows-events.plugin/windows-events-sid.h src/collectors/windows-events.plugin/windows-events-unicode.c src/collectors/windows-events.plugin/windows-events-unicode.h src/collectors/windows-events.plugin/windows-events-xml.c @@ -2020,7 +2025,6 @@ if(ENABLE_PLUGIN_APPS) src/collectors/apps.plugin/apps_plugin.h src/collectors/apps.plugin/apps_functions.c src/collectors/apps.plugin/apps_targets.c - src/collectors/apps.plugin/apps_users_and_groups.c src/collectors/apps.plugin/apps_output.c src/collectors/apps.plugin/apps_pid_files.c src/collectors/apps.plugin/apps_pid.c @@ -2356,7 +2360,7 @@ endif() if(ENABLE_PLUGIN_LOCAL_LISTENERS) set(LOCAL_LISTENERS_FILES src/collectors/utils/local_listeners.c - src/libnetdata/maps/local-sockets.h + src/libnetdata/local-sockets/local-sockets.h ) add_executable(local-listeners ${LOCAL_LISTENERS_FILES}) @@ -2375,8 +2379,7 @@ endif() if(ENABLE_PLUGIN_NETWORK_VIEWER) set(NETWORK_VIEWER_FILES - src/libnetdata/maps/local-sockets.h - src/libnetdata/maps/system-users.h + src/libnetdata/local-sockets/local-sockets.h src/collectors/network-viewer.plugin/network-viewer.c ) diff --git a/src/collectors/apps.plugin/apps_aggregations.c b/src/collectors/apps.plugin/apps_aggregations.c index e5d8da715c7e70..d8846d6e7c0287 100644 --- a/src/collectors/apps.plugin/apps_aggregations.c +++ b/src/collectors/apps.plugin/apps_aggregations.c @@ -169,6 +169,9 @@ void aggregate_processes_to_targets(void) { #if (PROCESSES_HAVE_GID == 1) zero_all_targets(groups_root_target); #endif +#if (PROCESSES_HAVE_SID == 1) + zero_all_targets(sids_root_target); +#endif // this has to be done, before the cleanup struct target *w = NULL, *o = NULL; @@ -187,6 +190,8 @@ void aggregate_processes_to_targets(void) { // user target #if (PROCESSES_HAVE_UID == 1) + update_cached_host_users(); + o = p->uid_target; if(likely(p->uid_target && p->uid_target->uid == p->uid)) w = p->uid_target; @@ -204,6 +209,8 @@ void aggregate_processes_to_targets(void) { // user group target #if (PROCESSES_HAVE_GID == 1) + update_cached_host_users(); + o = p->gid_target; if(likely(p->gid_target && p->gid_target->gid == p->gid)) w = p->gid_target; @@ -214,6 +221,19 @@ void aggregate_processes_to_targets(void) { w = p->gid_target = get_gid_target(p->gid); } + aggregate_pid_on_target(w, p, o); +#endif + + // -------------------------------------------------------------------- + // sid target + +#if (PROCESSES_HAVE_SID == 1) + o = p->sid_target; + if(likely(p->sid_target && p->sid_target->sid_name == p->sid_name)) + w = p->sid_target; + else + w = p->sid_target = get_sid_target(p->sid_name); + aggregate_pid_on_target(w, p, o); #endif diff --git a/src/collectors/apps.plugin/apps_functions.c b/src/collectors/apps.plugin/apps_functions.c index 96f3cf2f6f452a..6f8d1dc38dc21e 100644 --- a/src/collectors/apps.plugin/apps_functions.c +++ b/src/collectors/apps.plugin/apps_functions.c @@ -27,7 +27,7 @@ static void apps_plugin_function_processes_help(const char *transaction) { " parent:NAME\n" " Shows only processes that are aggregated under parent `NAME`\n" "\n" -#if (PROCESSES_HAVE_UID == 1) +#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1) " user:NAME\n" " Shows only processes that are running as user name `NAME`.\n" "\n" @@ -89,6 +89,12 @@ void function_processes(const char *transaction, char *function, size_t num_words = quoted_strings_splitter_whitespace(function, words, PLUGINSD_MAX_WORDS); struct target *category = NULL, *user = NULL, *group = NULL; (void)category; (void)user; (void)group; +#if (PROCESSES_HAVE_UID == 1) + struct target *users_sid_root = users_root_target; +#endif +#if (PROCESSES_HAVE_SID == 1) + struct target *users_sid_root = sids_root_target; +#endif const char *process_name = NULL; pid_t pid = 0; uid_t uid = 0; (void)uid; @@ -110,9 +116,9 @@ void function_processes(const char *transaction, char *function, return; } } -#if (PROCESSES_HAVE_UID == 1) +#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1) else if(!user && strncmp(keyword, PROCESS_FILTER_USER, strlen(PROCESS_FILTER_USER)) == 0) { - user = find_target_by_name(users_root_target, &keyword[strlen(PROCESS_FILTER_USER)]); + user = find_target_by_name(users_sid_root, &keyword[strlen(PROCESS_FILTER_USER)]); if(!user) { pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found."); @@ -271,6 +277,11 @@ void function_processes(const char *transaction, char *function, continue; #endif +#if (PROCESSES_HAVE_SID == 1) + if(user && p->sid_target != user) + continue; +#endif + if(process_name && ((strcmp(pid_stat_comm(p), process_name) != 0 && !p->parent) || (p->parent && strcmp(pid_stat_comm(p), process_name) != 0 && strcmp(pid_stat_comm(p->parent), process_name) != 0))) continue; @@ -323,6 +334,10 @@ void function_processes(const char *transaction, char *function, // uid buffer_json_add_array_item_uint64(wb, p->uid); #endif +#if (PROCESSES_HAVE_SID == 1) + // account + buffer_json_add_array_item_string(wb, p->sid_target ? string2str(p->sid_target->name) : "-"); +#endif #if (PROCESSES_HAVE_GID == 1) // group @@ -493,12 +508,14 @@ void function_processes(const char *transaction, char *function, RRDF_FIELD_FILTER_MULTISELECT, RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_STICKY, NULL); -#if (PROCESSES_HAVE_UID == 1) +#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1) buffer_rrdf_table_add_field(wb, field_id++, "User", "User Owner", RRDF_FIELD_TYPE_STRING, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_MULTISELECT, RRDF_FIELD_OPTS_VISIBLE, NULL); +#endif +#if (PROCESSES_HAVE_UID == 1) buffer_rrdf_table_add_field(wb, field_id++, "Uid", "User ID", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NUMBER, 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_COUNT, @@ -1079,7 +1096,7 @@ void function_processes(const char *transaction, char *function, } buffer_json_object_close(wb); -#if (PROCESSES_HAVE_UID == 1) +#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1) // group by User buffer_json_member_add_object(wb, "User"); { diff --git a/src/collectors/apps.plugin/apps_os_windows.c b/src/collectors/apps.plugin/apps_os_windows.c index d2c30de76cfa8a..6c2cabc50adc98 100644 --- a/src/collectors/apps.plugin/apps_os_windows.c +++ b/src/collectors/apps.plugin/apps_os_windows.c @@ -525,7 +525,7 @@ static char *ansi_to_utf8(LPCSTR str) { static __thread WCHAR unicode[PATH_MAX]; // Step 1: Convert ANSI string (LPSTR) to wide string (UTF-16) - size_t count = any_to_utf16(CP_ACP, unicode, _countof(unicode), str, -1); + size_t count = any_to_utf16(CP_ACP, unicode, _countof(unicode), str, -1, NULL); if (!count) return NULL; return wchar_to_utf8(unicode); @@ -664,6 +664,45 @@ static WCHAR *executable_path_from_cmdline(WCHAR *cmdline) { return NULL; } +static BOOL GetProcessUserSID(HANDLE hProcess, PSID *ppSid) { + HANDLE hToken; + BOOL result = FALSE; + DWORD dwSize = 0; + PTOKEN_USER pTokenUser = NULL; + + if (!OpenProcessToken(hProcess, TOKEN_QUERY, &hToken)) + return FALSE; + + GetTokenInformation(hToken, TokenUser, NULL, 0, &dwSize); + if (dwSize == 0) { + CloseHandle(hToken); + return FALSE; + } + + pTokenUser = (PTOKEN_USER)LocalAlloc(LPTR, dwSize); + if (pTokenUser == NULL) { + CloseHandle(hToken); + return FALSE; + } + + if (GetTokenInformation(hToken, TokenUser, pTokenUser, dwSize, &dwSize)) { + DWORD sidSize = GetLengthSid(pTokenUser->User.Sid); + *ppSid = (PSID)LocalAlloc(LPTR, sidSize); + if (*ppSid) { + if (CopySid(sidSize, *ppSid, pTokenUser->User.Sid)) { + result = TRUE; + } else { + LocalFree(*ppSid); + *ppSid = NULL; + } + } + } + + LocalFree(pTokenUser); + CloseHandle(hToken); + return result; +} + void GetAllProcessesInfo(void) { static __thread wchar_t unicode[PATH_MAX]; static __thread DWORD unicode_size = sizeof(unicode) / sizeof(*unicode); @@ -723,6 +762,14 @@ void GetAllProcessesInfo(void) { } } + if(!p->sid_name) { + PSID pSid = NULL; + if (GetProcessUserSID(hProcess, &pSid)) + p->sid_name = cached_sid_fullname_or_sid_str(pSid); + else + p->sid_name = string_strdupz("Unknown"); + } + CloseHandle(hProcess); char *comm = wchar_to_utf8(pe32.szExeFile); diff --git a/src/collectors/apps.plugin/apps_pid.c b/src/collectors/apps.plugin/apps_pid.c index 88626ba5804ba4..509aeadb5db2b4 100644 --- a/src/collectors/apps.plugin/apps_pid.c +++ b/src/collectors/apps.plugin/apps_pid.c @@ -134,6 +134,10 @@ void del_pid_entry(pid_t pid) { freez(p->fds); #endif +#if (PROCESSES_HAVE_SID == 1) + string_freez(p->sid_name); +#endif + string_freez(p->comm_orig); string_freez(p->comm); string_freez(p->cmdline); diff --git a/src/collectors/apps.plugin/apps_plugin.c b/src/collectors/apps.plugin/apps_plugin.c index 5cdeef178c0663..f6633ca83472c6 100644 --- a/src/collectors/apps.plugin/apps_plugin.c +++ b/src/collectors/apps.plugin/apps_plugin.c @@ -74,10 +74,10 @@ uint32_t // the metrics. This results in utilization that exceeds the total utilization // of the system. // -// During normalization, we align the per-process utilization, to the total of -// the system. We first consume the exited children utilization and it the -// collected values is above the total, we proportionally scale each reported -// metric. +// During normalization, we align the per-process utilization to the global +// utilization of the system. We first consume the exited children utilization +// and it the collected values is above the total, we proportionally scale each +// reported metric. // the total system time, as reported by /proc/stat #if (ALL_PIDS_ARE_READ_INSTANTLY == 0) @@ -211,9 +211,10 @@ void print_hierarchy(struct pid_stat *root) { static void normalize_utilization(struct target *root) { struct target *w; - // childs processing introduces spikes - // here we try to eliminate them by disabling childs processing either for specific dimensions - // or entirely. Of course, either way, we disable it just a single iteration. + // children processing introduces spikes, + // here we try to eliminate them by disabling children processing either + // for specific dimensions or entirely. + // of course, either way, we disable it just for a single iteration. kernel_uint_t max_time = os_get_system_cpus() * NSEC_PER_SEC; kernel_uint_t utime = 0, cutime = 0, stime = 0, cstime = 0, gtime = 0, cgtime = 0, minflt = 0, cminflt = 0, majflt = 0, cmajflt = 0; @@ -249,7 +250,7 @@ static void normalize_utilization(struct target *root) { cgtime_fix_ratio = 1.0; //(NETDATA_DOUBLE)(global_utime + global_stime) / (NETDATA_DOUBLE)(utime + cutime + stime + cstime); } else if((global_utime + global_stime > utime + stime) && (cutime || cstime)) { - // children resources are too high + // children resources are too high, // lower only the children resources utime_fix_ratio = stime_fix_ratio = @@ -481,7 +482,7 @@ static void parse_args(int argc, char **argv) } #endif -#if (PROCESSES_HAVE_UID == 1) +#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1) if(strcmp("no-users", argv[i]) == 0 || strcmp("without-users", argv[i]) == 0) { enable_users_charts = 0; continue; @@ -544,7 +545,7 @@ static void parse_args(int argc, char **argv) " (default is enabled)\n" "\n" #endif -#if (PROCESSES_HAVE_UID == 1) +#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1) " without-users disable reporting per user charts\n" "\n" #endif @@ -734,7 +735,18 @@ int main(int argc, char **argv) { netdata_log_info("started on pid %d", getpid()); - apps_users_and_groups_init(); +#if (PROCESSES_HAVE_UID == 1) + cached_usernames_init(); +#endif + +#if (PROCESSES_HAVE_GID == 1) + cached_groupnames_init(); +#endif + +#if (PROCESSES_HAVE_SID == 1) + cached_sid_username_init(); +#endif + apps_pids_init(); OS_FUNCTION(apps_os_init)(); @@ -823,6 +835,13 @@ int main(int argc, char **argv) { } #endif +#if (PROCESSES_HAVE_SID == 1) + if (enable_users_charts) { + send_charts_updates_to_netdata(sids_root_target, "user", "user", "User Processes"); + send_collected_data_to_netdata(sids_root_target, "user", dt); + } +#endif + fflush(stdout); debug_log("done Loop No %zu", global_iterations_counter); diff --git a/src/collectors/apps.plugin/apps_plugin.h b/src/collectors/apps.plugin/apps_plugin.h index 391b7ff90a99b3..771e9eb6da3c7c 100644 --- a/src/collectors/apps.plugin/apps_plugin.h +++ b/src/collectors/apps.plugin/apps_plugin.h @@ -22,6 +22,7 @@ #define PROCESSES_HAVE_IO_CALLS 0 #define PROCESSES_HAVE_UID 1 #define PROCESSES_HAVE_GID 1 +#define PROCESSES_HAVE_SID 0 #define PROCESSES_HAVE_MAJFLT 1 #define PROCESSES_HAVE_CHILDREN_FLTS 1 #define PROCESSES_HAVE_VMSWAP 0 @@ -65,6 +66,7 @@ struct pid_info { #define PROCESSES_HAVE_IO_CALLS 0 #define PROCESSES_HAVE_UID 1 #define PROCESSES_HAVE_GID 1 +#define PROCESSES_HAVE_SID 0 #define PROCESSES_HAVE_MAJFLT 1 #define PROCESSES_HAVE_CHILDREN_FLTS 0 #define PROCESSES_HAVE_VMSWAP 0 @@ -96,6 +98,7 @@ struct pid_info { #define PROCESSES_HAVE_IO_CALLS 1 #define PROCESSES_HAVE_UID 0 #define PROCESSES_HAVE_GID 0 +#define PROCESSES_HAVE_SID 1 #define PROCESSES_HAVE_MAJFLT 0 #define PROCESSES_HAVE_CHILDREN_FLTS 0 #define PROCESSES_HAVE_VMSWAP 1 @@ -125,6 +128,7 @@ struct pid_info { #define PROCESSES_HAVE_IO_CALLS 1 #define PROCESSES_HAVE_UID 1 #define PROCESSES_HAVE_GID 1 +#define PROCESSES_HAVE_SID 0 #define PROCESSES_HAVE_MAJFLT 1 #define PROCESSES_HAVE_CHILDREN_FLTS 1 #define PROCESSES_HAVE_VMSWAP 1 @@ -149,6 +153,10 @@ extern int max_fds_cache_seconds; #error "Unsupported operating system" #endif +#if (PROCESSES_HAVE_UID == 1) && (PROCESSES_HAVE_SID == 1) +#error "Do not enable SID and UID at the same time" +#endif + // -------------------------------------------------------------------------------------------------------------------- #define MAX_SYSTEM_FD_TO_ALLOW_FILES_PROCESSING 100000 @@ -205,12 +213,12 @@ extern size_t pagesize; extern netdata_mutex_t apps_and_stdout_mutex; -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // string lengths #define MAX_CMDLINE 65536 -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // to avoid reallocating too frequently when we add file descriptors, // we double the allocation at every increase request. @@ -218,7 +226,7 @@ static inline uint32_t fds_new_size(uint32_t old_size, uint32_t new_fd) { return MAX(old_size * 2, new_fd + 1); // 1 space always } -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // some variables for keeping track of processes count by states #if (PROCESSES_HAVE_STATE == 1) typedef enum { @@ -234,7 +242,7 @@ extern proc_state proc_state_count[PROC_STATUS_END]; extern const char *proc_states[]; #endif -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // the rates we are going to send to netdata will have this detail a value of: // - 1 will send just integer parts to netdata // - 100 will send 2 decimal points @@ -257,7 +265,7 @@ struct openfds { #define pid_openfds_sum(p) ((p)->openfds.files + (p)->openfds.pipes + (p)->openfds.sockets + (p)->openfds.inotifies + (p)->openfds.eventfds + (p)->openfds.timerfds + (p)->openfds.signalfds + (p)->openfds.eventpolls + (p)->openfds.other) #endif -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // target // // target is the structure that processes are aggregated to be reported @@ -278,6 +286,9 @@ typedef enum __attribute__((packed)) { #endif #if (PROCESSES_HAVE_GID == 1) TARGET_TYPE_GID, +#endif +#if (PROCESSES_HAVE_SID == 1) + TARGET_TYPE_SID, #endif TARGET_TYPE_TREE, } TARGET_TYPE; @@ -384,6 +395,9 @@ struct target { #if (PROCESSES_HAVE_GID == 1) gid_t gid; #endif +#if (PROCESSES_HAVE_SID == 1) + STRING *sid_name; +#endif kernel_uint_t values[PDF_MAX]; @@ -405,7 +419,7 @@ struct target { struct target *next; }; -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // internal flags // handled in code (automatically set) @@ -421,7 +435,7 @@ typedef enum __attribute__((packed)) { PID_LOG_LIMITS_DETAIL = (1 << 6), } PID_LOG; -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // pid_stat // // structure to store data for each process running @@ -460,8 +474,8 @@ struct pid_fd { #endif }; -#define pid_stat_comm(p) (string2str(p->comm)) -#define pid_stat_cmdline(p) (string2str(p->cmdline)) +#define pid_stat_comm(p) (string2str((p)->comm)) +#define pid_stat_cmdline(p) (string2str((p)->cmdline)) uint32_t all_files_len_get(void); struct pid_stat { @@ -485,10 +499,13 @@ struct pid_stat { #if (PROCESSES_HAVE_GID == 1) struct target *gid_target; // gid based targets #endif +#if (PROCESSES_HAVE_SID == 1) + struct target *sid_target; // sid based targets +#endif STRING *comm_orig; // the command, as-collected STRING *comm; // the command, sanitized - STRING *name; // the command name if any, sanitized + STRING *name; // the command name, if any, sanitized STRING *cmdline; // the full command line of the program #if defined(OS_WINDOWS) @@ -505,6 +522,9 @@ struct pid_stat { #if (PROCESSES_HAVE_GID == 1) gid_t gid; #endif +#if (PROCESSES_HAVE_SID == 1) + STRING *sid_name; +#endif #if (ALL_PIDS_ARE_READ_INSTANTLY == 0) uint32_t sortlist; // higher numbers = top on the process tree @@ -521,8 +541,8 @@ struct pid_stat { uint32_t fds_size; // the size of the fds array #endif - uint32_t children_count; // number of processes directly referencing this - // it is absorbed by apps_groups.conf inheritance + uint32_t children_count; // the number of processes directly referencing this. + // used internally for apps_groups.conf inheritance. // don't rely on it for anything else. uint32_t keeploops; // increases by 1 every time keep is 1 and updated 0 @@ -566,32 +586,11 @@ struct pid_stat { #endif }; -// ---------------------------------------------------------------------------- - -#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_GID == 1) -struct user_or_group_id { - avl_t avl; - - union { -#if (PROCESSES_HAVE_UID == 1) - uid_t uid; -#endif -#if (PROCESSES_HAVE_GID == 1) - gid_t gid; -#endif - } id; - - char *name; - - int updated; - - struct user_or_group_id * next; -}; -#endif +// -------------------------------------------------------------------------------------------------------------------- extern int update_every; -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // debugging static inline void debug_log_int(const char *fmt, ... ) { @@ -618,14 +617,14 @@ static inline void debug_log_dummy(void) {} bool managed_log(struct pid_stat *p, PID_LOG log, bool status); void sanitize_apps_plugin_chart_meta(char *buf); -// ---------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // macro to calculate the incremental rate of a value // each parameter is accessed only ONCE - so it is safe to pass function calls // or other macros as parameters #define incremental_rate(rate_variable, last_kernel_variable, new_kernel_value, collected_usec, last_collected_usec, multiplier) do { \ kernel_uint_t _new_tmp = new_kernel_value; \ - (rate_variable) = (_new_tmp - (last_kernel_variable)) * (USEC_PER_SEC * multiplier) / ((collected_usec) - (last_collected_usec)); \ + (rate_variable) = (_new_tmp - (last_kernel_variable)) * (USEC_PER_SEC * (multiplier)) / ((collected_usec) - (last_collected_usec)); \ (last_kernel_variable) = _new_tmp; \ } while(0) @@ -637,7 +636,6 @@ void sanitize_apps_plugin_chart_meta(char *buf); incremental_rate(p->values[idx], p->raw[idx], value, p->type##_collected_usec, p->last_##type##_collected_usec, CPU_TO_NANOSECONDCORES) void apps_managers_and_aggregators_init(void); -void apps_users_and_groups_init(void); void apps_pids_init(void); #if (PROCESSES_HAVE_CMDLINE == 1) @@ -700,13 +698,16 @@ void aggregate_processes_to_targets(void); #if (PROCESSES_HAVE_UID == 1) extern struct target *users_root_target; struct target *get_uid_target(uid_t uid); -struct user_or_group_id *user_id_find(struct user_or_group_id *user_id_to_find); #endif #if (PROCESSES_HAVE_GID == 1) extern struct target *groups_root_target; struct target *get_gid_target(gid_t gid); -struct user_or_group_id *group_id_find(struct user_or_group_id *group_id_to_find); +#endif + +#if (PROCESSES_HAVE_SID == 1) +extern struct target *sids_root_target; +struct target *get_sid_target(STRING *sid_name); #endif extern struct target *apps_groups_root_target; diff --git a/src/collectors/apps.plugin/apps_targets.c b/src/collectors/apps.plugin/apps_targets.c index 283f4788101593..bdcb40a14dd2f4 100644 --- a/src/collectors/apps.plugin/apps_targets.c +++ b/src/collectors/apps.plugin/apps_targets.c @@ -228,24 +228,10 @@ struct target *get_uid_target(uid_t uid) { w->uid = uid; w->id = get_numeric_string(uid); - struct user_or_group_id user_id_to_find = { - .id = { - .uid = uid, - } - }; - struct user_or_group_id *user_or_group_id = user_id_find(&user_id_to_find); - - if(user_or_group_id && user_or_group_id->name && *user_or_group_id->name) - w->name = string_strdupz(user_or_group_id->name); - else { - struct passwd *pw = getpwuid(uid); - if(!pw || !pw->pw_name || !*pw->pw_name) - w->name = get_numeric_string(uid); - else - w->name = string_strdupz(pw->pw_name); - } - + CACHED_USERNAME cu = cached_username_get_by_uid(uid); + w->name = string_dup(cu.username); w->clean_name = get_clean_name(w->name); + cached_username_release(cu); w->next = users_root_target; users_root_target = w; @@ -272,24 +258,10 @@ struct target *get_gid_target(gid_t gid) { w->gid = gid; w->id = get_numeric_string(gid); - struct user_or_group_id group_id_to_find = { - .id = { - .gid = gid, - } - }; - struct user_or_group_id *group_id = group_id_find(&group_id_to_find); - - if(group_id && group_id->name) - w->name = string_strdupz(group_id->name); - else { - struct group *gr = getgrgid(gid); - if(!gr || !gr->gr_name || !*gr->gr_name) - w->name = get_numeric_string(gid); - else - w->name = string_strdupz(gr->gr_name); - } - + CACHED_GROUPNAME cg = cached_groupname_get_by_gid(gid); + w->name = string_dup(cg.groupname); w->clean_name = get_clean_name(w->name); + cached_groupname_release(cg); w->next = groups_root_target; groups_root_target = w; @@ -300,6 +272,33 @@ struct target *get_gid_target(gid_t gid) { } #endif +// -------------------------------------------------------------------------------------------------------------------- +// SID + +#if (PROCESSES_HAVE_SID == 1) +struct target *sids_root_target = NULL; + +struct target *get_sid_target(STRING *sid_name) { + struct target *w; + for(w = sids_root_target ; w ; w = w->next) + if(w->sid_name == sid_name) return w; + + w = callocz(sizeof(struct target), 1); + w->type = TARGET_TYPE_SID; + w->sid_name = string_dup(sid_name); + w->id = string_dup(sid_name); + w->name = string_dup(sid_name); + w->clean_name = get_clean_name(w->name); + + w->next = sids_root_target; + sids_root_target = w; + + debug_log("added uid %s ('%s') target", string2str(w->sid_name), string2str(w->name)); + + return w; +} +#endif + // -------------------------------------------------------------------------------------------------------------------- // apps_groups.conf @@ -425,7 +424,7 @@ int read_apps_groups_conf(const char *path, const char *file) { managed_list_add(&tree.interpreters, s); } - // done with managers, proceed to next line + // done with managers, proceed to the next line continue; } diff --git a/src/collectors/apps.plugin/apps_users_and_groups.c b/src/collectors/apps.plugin/apps_users_and_groups.c deleted file mode 100644 index 8a8b50ff54176f..00000000000000 --- a/src/collectors/apps.plugin/apps_users_and_groups.c +++ /dev/null @@ -1,224 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "apps_plugin.h" - -#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_GID == 1) - -// ---------------------------------------------------------------------------- -// read users and groups from files - -enum user_or_group_id_type { - USER_ID, - GROUP_ID -}; - -struct user_or_group_ids { - enum user_or_group_id_type type; - - avl_tree_type index; - struct user_or_group_id *root; - - char filename[FILENAME_MAX + 1]; -}; - -#if (PROCESSES_HAVE_UID == 1) -static int user_id_compare(void* a, void* b) { - if(((struct user_or_group_id *)a)->id.uid < ((struct user_or_group_id *)b)->id.uid) - return -1; - - else if(((struct user_or_group_id *)a)->id.uid > ((struct user_or_group_id *)b)->id.uid) - return 1; - - else - return 0; -} - -static struct user_or_group_ids all_user_ids = { - .type = USER_ID, - - .index = { - NULL, - user_id_compare - }, - - .root = NULL, - - .filename = "", -}; -#endif - -#if (PROCESSES_HAVE_GID == 1) -static int group_id_compare(void* a, void* b) { - if(((struct user_or_group_id *)a)->id.gid < ((struct user_or_group_id *)b)->id.gid) - return -1; - - else if(((struct user_or_group_id *)a)->id.gid > ((struct user_or_group_id *)b)->id.gid) - return 1; - - else - return 0; -} - -static struct user_or_group_ids all_group_ids = { - .type = GROUP_ID, - - .index = { - NULL, - group_id_compare - }, - - .root = NULL, - - .filename = "", -}; -#endif - -static int file_changed(const struct stat *statbuf __maybe_unused, struct timespec *last_modification_time __maybe_unused) { -#if defined(OS_MACOS) || defined(OS_WINDOWS) - return 0; -#else - if(likely(statbuf->st_mtim.tv_sec == last_modification_time->tv_sec && - statbuf->st_mtim.tv_nsec == last_modification_time->tv_nsec)) return 0; - - last_modification_time->tv_sec = statbuf->st_mtim.tv_sec; - last_modification_time->tv_nsec = statbuf->st_mtim.tv_nsec; - - return 1; -#endif -} - -static int read_user_or_group_ids(struct user_or_group_ids *ids, struct timespec *last_modification_time) { - struct stat statbuf; - if(unlikely(stat(ids->filename, &statbuf))) - return 1; - else - if(likely(!file_changed(&statbuf, last_modification_time))) return 0; - - procfile *ff = procfile_open(ids->filename, " :\t", PROCFILE_FLAG_DEFAULT); - if(unlikely(!ff)) return 1; - - ff = procfile_readall(ff); - if(unlikely(!ff)) return 1; - - size_t line, lines = procfile_lines(ff); - - for(line = 0; line < lines ;line++) { - size_t words = procfile_linewords(ff, line); - if(unlikely(words < 3)) continue; - - char *name = procfile_lineword(ff, line, 0); - if(unlikely(!name || !*name)) continue; - - char *id_string = procfile_lineword(ff, line, 2); - if(unlikely(!id_string || !*id_string)) continue; - - - struct user_or_group_id *user_or_group_id = callocz(1, sizeof(struct user_or_group_id)); - -#if (PROCESSES_HAVE_UID == 1) - if(ids->type == USER_ID) - user_or_group_id->id.uid = (uid_t) str2ull(id_string, NULL); -#endif -#if (PROCESSES_HAVE_GID == 1) - if(ids->type == GROUP_ID) - user_or_group_id->id.gid = (uid_t) str2ull(id_string, NULL); -#endif - - user_or_group_id->name = strdupz(name); - user_or_group_id->updated = 1; - - struct user_or_group_id *existing_user_id = NULL; - - if(likely(ids->root)) - existing_user_id = (struct user_or_group_id *)avl_search(&ids->index, (avl_t *) user_or_group_id); - - if(unlikely(existing_user_id)) { - freez(existing_user_id->name); - existing_user_id->name = user_or_group_id->name; - existing_user_id->updated = 1; - freez(user_or_group_id); - } - else { - if(unlikely(avl_insert(&ids->index, (avl_t *) user_or_group_id) != (void *) user_or_group_id)) { - netdata_log_error("INTERNAL ERROR: duplicate indexing of id during realloc"); - } - - user_or_group_id->next = ids->root; - ids->root = user_or_group_id; - } - } - - procfile_close(ff); - - // remove unused ids - struct user_or_group_id *user_or_group_id = ids->root, *prev_user_id = NULL; - - while(user_or_group_id) { - if(unlikely(!user_or_group_id->updated)) { - if(unlikely((struct user_or_group_id *)avl_remove(&ids->index, (avl_t *) user_or_group_id) != user_or_group_id)) - netdata_log_error("INTERNAL ERROR: removal of unused id from index, removed a different id"); - - if(prev_user_id) - prev_user_id->next = user_or_group_id->next; - else - ids->root = user_or_group_id->next; - - freez(user_or_group_id->name); - freez(user_or_group_id); - - if(prev_user_id) - user_or_group_id = prev_user_id->next; - else - user_or_group_id = ids->root; - } - else { - user_or_group_id->updated = 0; - - prev_user_id = user_or_group_id; - user_or_group_id = user_or_group_id->next; - } - } - - return 0; -} - -#if (PROCESSES_HAVE_UID == 1) -struct user_or_group_id *user_id_find(struct user_or_group_id *user_id_to_find) { - if(*netdata_configured_host_prefix) { - static struct timespec last_passwd_modification_time; - int ret = read_user_or_group_ids(&all_user_ids, &last_passwd_modification_time); - - if(likely(!ret && all_user_ids.index.root)) - return (struct user_or_group_id *)avl_search(&all_user_ids.index, (avl_t *)user_id_to_find); - } - - return NULL; -} -#endif - -#if (PROCESSES_HAVE_GID == 1) -struct user_or_group_id *group_id_find(struct user_or_group_id *group_id_to_find) { - if(*netdata_configured_host_prefix) { - static struct timespec last_group_modification_time; - int ret = read_user_or_group_ids(&all_group_ids, &last_group_modification_time); - - if(likely(!ret && all_group_ids.index.root)) - return (struct user_or_group_id *)avl_search(&all_group_ids.index, (avl_t *) &group_id_to_find); - } - - return NULL; -} -#endif -#endif - -void apps_users_and_groups_init(void) { -#if (PROCESSES_HAVE_UID == 1) - snprintfz(all_user_ids.filename, FILENAME_MAX, "%s/etc/passwd", netdata_configured_host_prefix); - debug_log("passwd file: '%s'", all_user_ids.filename); -#endif - -#if (PROCESSES_HAVE_GID == 1) - snprintfz(all_group_ids.filename, FILENAME_MAX, "%s/etc/group", netdata_configured_host_prefix); - debug_log("group file: '%s'", all_group_ids.filename); -#endif -} diff --git a/src/collectors/network-viewer.plugin/network-viewer.c b/src/collectors/network-viewer.plugin/network-viewer.c index 2bfb9db9963a64..6892287f2b29fe 100644 --- a/src/collectors/network-viewer.plugin/network-viewer.c +++ b/src/collectors/network-viewer.plugin/network-viewer.c @@ -23,9 +23,8 @@ static SPAWN_SERVER *spawn_srv = NULL; } aggregated_key; \ } network_viewer; -#include "libnetdata/maps/local-sockets.h" -#include "libnetdata/maps/system-users.h" -#include "libnetdata/maps/system-services.h" +#include "libnetdata/local-sockets/local-sockets.h" +#include "libnetdata/os/system-maps/system-services.h" #define NETWORK_CONNECTIONS_VIEWER_FUNCTION "network-connections" #define NETWORK_CONNECTIONS_VIEWER_HELP "Network connections explorer" @@ -36,7 +35,6 @@ static SPAWN_SERVER *spawn_srv = NULL; netdata_mutex_t stdout_mutex = NETDATA_MUTEX_INITIALIZER; static bool plugin_should_exit = false; -static USERNAMES_CACHE *uc; static SERVICENAMES_CACHE *sc; ENUM_STR_MAP_DEFINE(SOCKET_DIRECTION) = { @@ -151,9 +149,9 @@ static void local_socket_to_json_array(struct sockets_stats *st, const LOCAL_SOC } else { // buffer_json_add_array_item_uint64(wb, n->uid); - STRING *u = system_usernames_cache_lookup_uid(uc, n->uid); - buffer_json_add_array_item_string(wb, string2str(u)); - string_freez(u); + CACHED_USERNAME cu = cached_username_get_by_uid(n->uid); + buffer_json_add_array_item_string(wb, string2str(cu.username)); + cached_username_release(cu); } const struct socket_endpoint *server_endpoint; @@ -975,7 +973,8 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) { } #endif - uc = system_usernames_cache_init(); + cached_usernames_init(); + update_cached_host_users(); sc = system_servicenames_cache_init(); // ---------------------------------------------------------------------------------------------------------------- diff --git a/src/collectors/systemd-journal.plugin/systemd-journal-annotations.c b/src/collectors/systemd-journal.plugin/systemd-journal-annotations.c index c5b708714fce18..c9ade0a33a676a 100644 --- a/src/collectors/systemd-journal.plugin/systemd-journal-annotations.c +++ b/src/collectors/systemd-journal.plugin/systemd-journal-annotations.c @@ -2,18 +2,6 @@ #include "systemd-internals.h" -// ---------------------------------------------------------------------------- -#include "libnetdata/maps/system-users.h" -#include "libnetdata/maps/system-groups.h" - -static struct { - USERNAMES_CACHE *uc; - GROUPNAMES_CACHE *gc; -} systemd_annotations_globals = { - .uc = NULL, - .gc = NULL, -}; - // ---------------------------------------------------------------------------- const char *errno_map[] = { @@ -369,9 +357,9 @@ void netdata_systemd_journal_transform_uid(FACETS *facets __maybe_unused, BUFFER const char *v = buffer_tostring(wb); if(*v && isdigit(*v)) { uid_t uid = str2i(buffer_tostring(wb)); - STRING *u = system_usernames_cache_lookup_uid(systemd_annotations_globals.uc, uid); - buffer_contents_replace(wb, string2str(u), string_strlen(u)); - string_freez(u); + CACHED_USERNAME cu = cached_username_get_by_uid(uid); + buffer_contents_replace(wb, string2str(cu.username), string_strlen(cu.username)); + cached_username_release(cu); } } @@ -382,9 +370,9 @@ void netdata_systemd_journal_transform_gid(FACETS *facets __maybe_unused, BUFFER const char *v = buffer_tostring(wb); if(*v && isdigit(*v)) { gid_t gid = str2i(buffer_tostring(wb)); - STRING *g = system_groupnames_cache_lookup_gid(systemd_annotations_globals.gc, gid); - buffer_contents_replace(wb, string2str(g), string_strlen(g)); - string_freez(g); + CACHED_GROUPNAME cg = cached_groupname_get_by_gid(gid); + buffer_contents_replace(wb, string2str(cg.groupname), string_strlen(cg.groupname)); + cached_groupname_release(cg); } } @@ -650,8 +638,10 @@ void netdata_systemd_journal_transform_message_id(FACETS *facets __maybe_unused, // ---------------------------------------------------------------------------- void netdata_systemd_journal_annotations_init(void) { - systemd_annotations_globals.uc = system_usernames_cache_init(); - systemd_annotations_globals.gc = system_groupnames_cache_init(); + cached_usernames_init(); + cached_groupnames_init(); + update_cached_host_users(); + update_cached_host_groups(); netdata_systemd_journal_message_ids_init(); } diff --git a/src/collectors/utils/local_listeners.c b/src/collectors/utils/local_listeners.c index 501c4b5864f994..a2e8968fff1c28 100644 --- a/src/collectors/utils/local_listeners.c +++ b/src/collectors/utils/local_listeners.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "libnetdata/libnetdata.h" -#include "libnetdata/maps/local-sockets.h" +#include "libnetdata/local-sockets/local-sockets.h" #include "libnetdata/required_dummies.h" // -------------------------------------------------------------------------------------------------------------------- diff --git a/src/collectors/windows-events.plugin/windows-events-providers.c b/src/collectors/windows-events.plugin/windows-events-providers.c index a0400cab937959..68045127aecf09 100644 --- a/src/collectors/windows-events.plugin/windows-events-providers.c +++ b/src/collectors/windows-events.plugin/windows-events-providers.c @@ -78,7 +78,8 @@ static struct { .spinlock = NETDATA_SPINLOCK_INITIALIZER, }; -static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id); +static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, + TXT_UTF16 *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id); const char *provider_get_name(PROVIDER_META_HANDLE *p) { return (p && p->provider && p->provider->name) ? p->provider->name : "__UNKNOWN PROVIDER__"; @@ -226,7 +227,7 @@ PROVIDER_META_HANDLE *provider_get(ND_UUID uuid, LPCWSTR providerName) { if(load_it) { WEVT_VARIANT content = { 0 }; WEVT_VARIANT property = { 0 }; - TXT_UNICODE unicode = { 0 }; + TXT_UTF16 unicode = { 0 }; provider_detect_platform(h, &content); provider_load_list(h, &content, &property, &unicode, &p->keyword, EvtPublisherMetadataKeywords); @@ -234,7 +235,7 @@ PROVIDER_META_HANDLE *provider_get(ND_UUID uuid, LPCWSTR providerName) { provider_load_list(h, &content, &property, &unicode, &p->opcodes, EvtPublisherMetadataOpcodes); provider_load_list(h, &content, &property, &unicode, &p->tasks, EvtPublisherMetadataTasks); - txt_unicode_cleanup(&unicode); + txt_utf16_cleanup(&unicode); wevt_variant_cleanup(&content); wevt_variant_cleanup(&property); } @@ -365,7 +366,8 @@ static int compare_ascending(const void *a, const void *b) { // return 0; //} -static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, TXT_UNICODE *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) { +static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, WEVT_VARIANT *property, + TXT_UTF16 *dst, struct provider_list *l, EVT_PUBLISHER_METADATA_PROPERTY_ID property_id) { if(!h || !h->hMetadata) return; EVT_PUBLISHER_METADATA_PROPERTY_ID name_id, message_id, value_id; @@ -477,7 +479,7 @@ static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, W if (messageID != (uint32_t)-1) { if (EvtFormatMessage_utf16(dst, hMetadata, NULL, messageID, EvtFormatMessageId)) { size_t len; - d->name = unicode2utf8_strdupz(dst->data, &len); + d->name = utf16_to_utf8_strdupz(dst->data, &len); d->len = len; } } @@ -487,7 +489,7 @@ static void provider_load_list(PROVIDER_META_HANDLE *h, WEVT_VARIANT *content, W if (!d->name && wevt_get_property_from_array(property, hArray, i, name_id)) { fatal_assert(property->data->Type == EvtVarTypeString); size_t len; - d->name = unicode2utf8_strdupz(property->data->StringVal, &len); + d->name = utf16_to_utf8_strdupz(property->data->StringVal, &len); d->len = len; } diff --git a/src/collectors/windows-events.plugin/windows-events-query-evt-variant.c b/src/collectors/windows-events.plugin/windows-events-query-evt-variant.c index 9d18f81fcf9a75..ee3aa382befd2a 100644 --- a/src/collectors/windows-events.plugin/windows-events-query-evt-variant.c +++ b/src/collectors/windows-events.plugin/windows-events-query-evt-variant.c @@ -21,13 +21,14 @@ static inline void append_utf16(BUFFER *b, LPCWSTR utf16Str, const char *separat remaining = b->size - b->len; } - size_t used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1); - if(used >= remaining) { - // oops, we need to resize - size_t needed = utf16_to_utf8(NULL, 0, utf16Str, -1); // find the size needed + bool truncated = false; + size_t used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1, &truncated); + if(truncated) { + // we need to resize + size_t needed = utf16_to_utf8(NULL, 0, utf16Str, -1, NULL); // find the size needed buffer_need_bytes(b, needed); remaining = b->size - b->len; - used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1); + used = utf16_to_utf8(&b->buffer[b->len], remaining, utf16Str, -1, NULL); } if(used) { @@ -113,7 +114,7 @@ static inline void append_filetime(BUFFER *b, FILETIME *ft, const char *separato } static inline void append_sid(BUFFER *b, PSID sid, const char *separator) { - buffer_sid_to_sid_str_and_name(sid, b, separator); + cached_sid_to_buffer_append(sid, b, separator); } static inline void append_sbyte(BUFFER *b, INT8 n, const char *separator) { diff --git a/src/collectors/windows-events.plugin/windows-events-query.c b/src/collectors/windows-events.plugin/windows-events-query.c index 12734fe9d538ad..fefa72829eaa87 100644 --- a/src/collectors/windows-events.plugin/windows-events-query.c +++ b/src/collectors/windows-events.plugin/windows-events-query.c @@ -29,7 +29,8 @@ static const char *EvtGetExtendedStatus_utf8(void) { // -------------------------------------------------------------------------------------------------------------------- -bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags) { +bool EvtFormatMessage_utf16( + TXT_UTF16 *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags) { dst->used = 0; DWORD size = 0; @@ -39,7 +40,7 @@ bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE h // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() to get message size failed."); goto cleanup; } - txt_unicode_resize(dst, size, false); + txt_utf16_resize(dst, size, false); } // First, try to get the message using the existing buffer @@ -50,7 +51,7 @@ bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE h } // Try again with the resized buffer - txt_unicode_resize(dst, size, false); + txt_utf16_resize(dst, size, false); if (!EvtFormatMessage(hMetadata, hEvent, dwMessageId, 0, NULL, flags, dst->size, dst->data, &size)) { // nd_log(NDLS_COLLECTORS, NDLP_ERR, "EvtFormatMessage() failed after resizing buffer."); goto cleanup; @@ -75,23 +76,23 @@ bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE h } static bool EvtFormatMessage_utf8( - TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, + TXT_UTF16 *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst, EVT_FORMAT_MESSAGE_FLAGS flags) { dst->src = TXT_SOURCE_EVENT_LOG; if(EvtFormatMessage_utf16(tmp, provider_handle(p), hEvent, 0, flags)) - return wevt_str_unicode_to_utf8(dst, tmp); + return txt_utf16_to_utf8(dst, tmp); - wevt_utf8_empty(dst); + txt_utf8_empty(dst); return false; } -bool EvtFormatMessage_Event_utf8(TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) { +bool EvtFormatMessage_Event_utf8(TXT_UTF16 *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) { return EvtFormatMessage_utf8(tmp, p, hEvent, dst, EvtFormatMessageEvent); } -bool EvtFormatMessage_Xml_utf8(TXT_UNICODE *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) { +bool EvtFormatMessage_Xml_utf8(TXT_UTF16 *tmp, PROVIDER_META_HANDLE *p, EVT_HANDLE hEvent, TXT_UTF8 *dst) { return EvtFormatMessage_utf8(tmp, p, hEvent, dst, EvtFormatMessageXml); } @@ -130,7 +131,7 @@ static void wevt_get_level(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE * TXT_UTF8 *dst = &log->ops.level; uint64_t value = ev->level; - wevt_utf8_empty(dst); + txt_utf8_empty(dst); EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageLevel; WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_LEVEL; @@ -182,7 +183,7 @@ static void wevt_get_opcode(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE TXT_UTF8 *dst = &log->ops.opcode; uint64_t value = ev->opcode; - wevt_utf8_empty(dst); + txt_utf8_empty(dst); EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageOpcode; WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_OPCODE; @@ -224,7 +225,7 @@ static void wevt_get_task(WEVT_LOG *log, WEVT_EVENT *ev, PROVIDER_META_HANDLE *h TXT_UTF8 *dst = &log->ops.task; uint64_t value = ev->task; - wevt_utf8_empty(dst); + txt_utf8_empty(dst); EVT_FORMAT_MESSAGE_FLAGS flags = EvtFormatMessageTask; WEVT_FIELD_TYPE cache_type = WEVT_FIELD_TYPE_TASK; @@ -273,7 +274,7 @@ static uint64_t wevt_keyword_handle_reserved(uint64_t value, TXT_UTF8 *dst) { SET_BITS(WEVT_KEYWORD_RESPONSE_TIME, WEVT_KEYWORD_NAME_RESPONSE_TIME), }; - wevt_utf8_empty(dst); + txt_utf8_empty(dst); for(size_t i = 0; i < sizeof(bits) / sizeof(bits[0]) ;i++) { if((value & bits[i].mask) == bits[i].mask) { @@ -592,7 +593,7 @@ void wevt_closelog6(WEVT_LOG *log) { wevt_variant_cleanup(&log->ops.raw.system); wevt_variant_cleanup(&log->ops.raw.user); - txt_unicode_cleanup(&log->ops.unicode); + txt_utf16_cleanup(&log->ops.unicode); txt_utf8_cleanup(&log->ops.channel); txt_utf8_cleanup(&log->ops.provider); txt_utf8_cleanup(&log->ops.computer); diff --git a/src/collectors/windows-events.plugin/windows-events-query.h b/src/collectors/windows-events.plugin/windows-events-query.h index c43c3c9b5044a1..3136b23dfc328f 100644 --- a/src/collectors/windows-events.plugin/windows-events-query.h +++ b/src/collectors/windows-events.plugin/windows-events-query.h @@ -83,7 +83,7 @@ typedef struct wevt_log { // every string operation overwrites it, multiple times per event log entry // it can be used within any function, for its own purposes, // but never share between functions - TXT_UNICODE unicode; + TXT_UTF16 unicode; // string attributes of the current event log entry // valid until another event if fetched @@ -136,10 +136,11 @@ void wevt_query_done(WEVT_LOG *log); bool wevt_get_next_event(WEVT_LOG *log, WEVT_EVENT *ev); -bool EvtFormatMessage_utf16(TXT_UNICODE *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags); +bool EvtFormatMessage_utf16( + TXT_UTF16 *dst, EVT_HANDLE hMetadata, EVT_HANDLE hEvent, DWORD dwMessageId, EVT_FORMAT_MESSAGE_FLAGS flags); -bool EvtFormatMessage_Event_utf8(TXT_UNICODE *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst); -bool EvtFormatMessage_Xml_utf8(TXT_UNICODE *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst); +bool EvtFormatMessage_Event_utf8(TXT_UTF16 *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst); +bool EvtFormatMessage_Xml_utf8(TXT_UTF16 *tmp, struct provider_meta_handle *p, EVT_HANDLE hEvent, TXT_UTF8 *dst); void evt_variant_to_buffer(BUFFER *b, EVT_VARIANT *ev, const char *separator); @@ -152,7 +153,7 @@ static inline void wevt_variant_resize(WEVT_VARIANT *v, size_t required_size) { return; wevt_variant_cleanup(v); - v->size = compute_new_size(v->size, required_size); + v->size = txt_compute_new_size(v->size, required_size); v->data = mallocz(v->size); } @@ -202,25 +203,25 @@ static inline uint64_t wevt_field_get_uint64_hex(EVT_VARIANT *ev) { static inline bool wevt_field_get_string_utf8(EVT_VARIANT *ev, TXT_UTF8 *dst) { if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) { - wevt_utf8_empty(dst); + txt_utf8_empty(dst); return false; } fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeString); - return wevt_str_wchar_to_utf8(dst, ev->StringVal, -1); + return wchar_to_txt_utf8(dst, ev->StringVal, -1); } -bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str); +bool cached_sid_to_account_domain_sidstr(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str); static inline bool wevt_field_get_sid(EVT_VARIANT *ev, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) { if((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeNull) { - wevt_utf8_empty(dst_account); - wevt_utf8_empty(dst_domain); - wevt_utf8_empty(dst_sid_str); + txt_utf8_empty(dst_account); + txt_utf8_empty(dst_domain); + txt_utf8_empty(dst_sid_str); return false; } fatal_assert((ev->Type & EVT_VARIANT_TYPE_MASK) == EvtVarTypeSid); - return wevt_convert_user_id_to_name(ev->SidVal, dst_account, dst_domain, dst_sid_str); + return cached_sid_to_account_domain_sidstr(ev->SidVal, dst_account, dst_domain, dst_sid_str); } static inline uint64_t wevt_field_get_filetime_to_ns(EVT_VARIANT *ev) { diff --git a/src/collectors/windows-events.plugin/windows-events-sid.h b/src/collectors/windows-events.plugin/windows-events-sid.h deleted file mode 100644 index aca8e77e34c1fe..00000000000000 --- a/src/collectors/windows-events.plugin/windows-events-sid.h +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_WINDOWS_EVENTS_SID_H -#define NETDATA_WINDOWS_EVENTS_SID_H - -#include "windows-events.h" - -struct wevt_log; -bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str); -bool buffer_sid_to_sid_str_and_name(PSID sid, BUFFER *dst, const char *prefix); -void sid_cache_init(void); - -#endif //NETDATA_WINDOWS_EVENTS_SID_H diff --git a/src/collectors/windows-events.plugin/windows-events-unicode.c b/src/collectors/windows-events.plugin/windows-events-unicode.c index f84418dc76ee9c..81da311073bb73 100644 --- a/src/collectors/windows-events.plugin/windows-events-unicode.c +++ b/src/collectors/windows-events.plugin/windows-events-unicode.c @@ -21,24 +21,6 @@ inline void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src) { strncpyz(dst, "[null]", dst_size - 1); } -char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len) { - int size = WideCharToMultiByte(CP_UTF8, 0, src, -1, NULL, 0, NULL, NULL); - if (size > 0) { - char *dst = mallocz(size); - WideCharToMultiByte(CP_UTF8, 0, src, -1, dst, size, NULL, NULL); - - if(utf8_len) - *utf8_len = size - 1; - - return dst; - } - - if(utf8_len) - *utf8_len = 0; - - return NULL; -} - wchar_t *channel2unicode(const char *utf8str) { static __thread wchar_t buffer[1024]; utf82unicode(buffer, _countof(buffer), utf8str); @@ -51,18 +33,6 @@ char *channel2utf8(const wchar_t *channel) { return buffer; } -char *account2utf8(const wchar_t *user) { - static __thread char buffer[1024]; - unicode2utf8(buffer, sizeof(buffer), user); - return buffer; -} - -char *domain2utf8(const wchar_t *domain) { - static __thread char buffer[1024]; - unicode2utf8(buffer, sizeof(buffer), domain); - return buffer; -} - char *query2utf8(const wchar_t *query) { static __thread char buffer[16384]; unicode2utf8(buffer, sizeof(buffer), query); @@ -74,71 +44,3 @@ char *provider2utf8(const wchar_t *provider) { unicode2utf8(buffer, sizeof(buffer), provider); return buffer; } - -bool wevt_str_wchar_to_utf8(TXT_UTF8 *dst, const wchar_t *src, int src_len_with_null) { - if(!src || !src_len_with_null) - goto cleanup; - - // make sure the input is null terminated at the exact point we need it - // (otherwise, the output will not be null terminated either) - fatal_assert(src_len_with_null == -1 || (src_len_with_null >= 1 && src[src_len_with_null - 1] == 0)); - - // Try to convert using the existing buffer (if it exists, otherwise get the required buffer size) - int size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, dst->data, (int)dst->size, NULL, NULL); - if(size <= 0 || !dst->data) { - // we have to set a buffer, or increase it - - if(dst->data) { - // we need to increase it the buffer size - - if(GetLastError() != ERROR_INSUFFICIENT_BUFFER) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed."); - goto cleanup; - } - - // we have to find the required buffer size - size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, NULL, 0, NULL, NULL); - if(size <= 0) - goto cleanup; - } - - // Retry conversion with the new buffer - txt_utf8_resize(dst, size, false); - size = WideCharToMultiByte(CP_UTF8, 0, src, src_len_with_null, dst->data, (int)dst->size, NULL, NULL); - if (size <= 0) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, "WideCharToMultiByte() failed after resizing."); - goto cleanup; - } - } - - // Make sure it is not zero padded at the end - while(size >= 2 && dst->data[size - 2] == 0) - size--; - - dst->used = (size_t)size; - - internal_fatal(strlen(dst->data) + 1 != dst->used, - "Wrong UTF8 string length"); - - return true; - -cleanup: - txt_utf8_resize(dst, 128, false); - if(src) - dst->used = snprintfz(dst->data, dst->size, "[failed conv.]") + 1; - else { - dst->data[0] = '\0'; - dst->used = 1; - } - - return false; -} - -bool wevt_str_unicode_to_utf8(TXT_UTF8 *dst, TXT_UNICODE *unicode) { - fatal_assert(dst && ((dst->data && dst->size) || (!dst->data && !dst->size))); - fatal_assert(unicode && ((unicode->data && unicode->size) || (!unicode->data && !unicode->size))); - - // pass the entire unicode size, including the null terminator - // so that the resulting utf8 message will be null terminated too. - return wevt_str_wchar_to_utf8(dst, unicode->data, (int)unicode->used); -} diff --git a/src/collectors/windows-events.plugin/windows-events-unicode.h b/src/collectors/windows-events.plugin/windows-events-unicode.h index fa2f4a49ebb375..e0d3b95361827c 100644 --- a/src/collectors/windows-events.plugin/windows-events-unicode.h +++ b/src/collectors/windows-events.plugin/windows-events-unicode.h @@ -7,87 +7,6 @@ #include #include -typedef enum __attribute__((packed)) { - TXT_SOURCE_UNKNOWN = 0, - TXT_SOURCE_PROVIDER, - TXT_SOURCE_FIELD_CACHE, - TXT_SOURCE_EVENT_LOG, - TXT_SOURCE_HARDCODED, - - // terminator - TXT_SOURCE_MAX, -} TXT_SOURCE; - -static inline size_t compute_new_size(size_t old_size, size_t required_size) { - size_t size = (required_size % 2048 == 0) ? required_size : required_size + 2048; - size = (size / 2048) * 2048; - - if(size < old_size * 2) - size = old_size * 2; - - return size; -} - -// -------------------------------------------------------------------------------------------------------------------- -// TXT_UTF8 - -typedef struct { - char *data; - size_t size; // the allocated size of data buffer - size_t used; // the used size of the data buffer (including null terminators, if any) - TXT_SOURCE src; -} TXT_UTF8; - -static inline void txt_utf8_cleanup(TXT_UTF8 *dst) { - freez(dst->data); - dst->data = NULL; - dst->used = 0; -} - -static inline void txt_utf8_resize(TXT_UTF8 *dst, size_t required_size, bool keep) { - if(required_size <= dst->size) - return; - - size_t new_size = compute_new_size(dst->size, required_size); - - if(keep && dst->data) - dst->data = reallocz(dst->data, new_size); - else { - txt_utf8_cleanup(dst); - dst->data = mallocz(new_size); - dst->used = 0; - } - - dst->size = new_size; -} - -static inline void wevt_utf8_empty(TXT_UTF8 *dst) { - txt_utf8_resize(dst, 1, false); - dst->data[0] = '\0'; - dst->used = 1; -} - -static inline void txt_utf8_set(TXT_UTF8 *dst, const char *txt, size_t txt_len) { - txt_utf8_resize(dst, dst->used + txt_len + 1, true); - memcpy(dst->data, txt, txt_len); - dst->used = txt_len + 1; - dst->data[dst->used - 1] = '\0'; -} - -static inline void txt_utf8_append(TXT_UTF8 *dst, const char *txt, size_t txt_len) { - if(dst->used <= 1) { - // the destination is empty - txt_utf8_set(dst, txt, txt_len); - } - else { - // there is something already in the buffer - txt_utf8_resize(dst, dst->used + txt_len, true); - memcpy(&dst->data[dst->used - 1], txt, txt_len); - dst->used += txt_len; // the null was already counted - dst->data[dst->used - 1] = '\0'; - } -} - #define WINEVENT_NAME_KEYWORDS_SEPARATOR ", " static inline void txt_utf8_add_keywords_separator_if_needed(TXT_UTF8 *dst) { if(dst->used > 1) @@ -110,75 +29,16 @@ static inline void txt_utf8_set_hex_if_empty(TXT_UTF8 *dst, const char *prefix, } } -// -------------------------------------------------------------------------------------------------------------------- -// TXT_UNICODE - -typedef struct { - wchar_t *data; - size_t size; // the allocated size of data buffer - size_t used; // the used size of the data buffer (including null terminators, if any) -} TXT_UNICODE; - -static inline void txt_unicode_cleanup(TXT_UNICODE *dst) { - freez(dst->data); -} - -static inline void txt_unicode_resize(TXT_UNICODE *dst, size_t required_size, bool keep) { - if(required_size <= dst->size) - return; - - size_t new_size = compute_new_size(dst->size, required_size); - - if (keep && dst->data) { - dst->data = reallocz(dst->data, new_size * sizeof(wchar_t)); - } else { - txt_unicode_cleanup(dst); - dst->data = mallocz(new_size * sizeof(wchar_t)); - dst->used = 0; - } - - dst->size = new_size; -} - -static inline void txt_unicode_set(TXT_UNICODE *dst, const wchar_t *txt, size_t txt_len) { - txt_unicode_resize(dst, dst->used + txt_len + 1, true); - memcpy(dst->data, txt, txt_len * sizeof(wchar_t)); - dst->used = txt_len + 1; - dst->data[dst->used - 1] = '\0'; -} - -static inline void txt_unicode_append(TXT_UNICODE *dst, const wchar_t *txt, size_t txt_len) { - if(dst->used <= 1) { - // the destination is empty - txt_unicode_set(dst, txt, txt_len); - } - else { - // there is something already in the buffer - txt_unicode_resize(dst, dst->used + txt_len, true); - memcpy(&dst->data[dst->used - 1], txt, txt_len * sizeof(wchar_t)); - dst->used += txt_len; // the null was already counted - dst->data[dst->used - 1] = '\0'; - } -} - // -------------------------------------------------------------------------------------------------------------------- // conversions -bool wevt_str_unicode_to_utf8(TXT_UTF8 *dst, TXT_UNICODE *unicode); -bool wevt_str_wchar_to_utf8(TXT_UTF8 *dst, const wchar_t *src, int src_len_with_null); - void unicode2utf8(char *dst, size_t dst_size, const wchar_t *src); void utf82unicode(wchar_t *dst, size_t dst_size, const char *src); -char *account2utf8(const wchar_t *user); -char *domain2utf8(const wchar_t *domain); - char *channel2utf8(const wchar_t *channel); wchar_t *channel2unicode(const char *utf8str); char *query2utf8(const wchar_t *query); char *provider2utf8(const wchar_t *provider); -char *unicode2utf8_strdupz(const wchar_t *src, size_t *utf8_len); - #endif //NETDATA_WINDOWS_EVENTS_UNICODE_H diff --git a/src/collectors/windows-events.plugin/windows-events.c b/src/collectors/windows-events.plugin/windows-events.c index 14ca9d6ceab408..ea816c9c79e38a 100644 --- a/src/collectors/windows-events.plugin/windows-events.c +++ b/src/collectors/windows-events.plugin/windows-events.c @@ -1299,7 +1299,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) { wevt_sources_init(); provider_cache_init(); - sid_cache_init(); + cached_sid_username_init(); field_cache_init(); if(!EnableWindowsPrivilege(SE_SECURITY_NAME)) diff --git a/src/collectors/windows-events.plugin/windows-events.h b/src/collectors/windows-events.plugin/windows-events.h index 4cb5b50cadc446..85b5b8890f7d05 100644 --- a/src/collectors/windows-events.plugin/windows-events.h +++ b/src/collectors/windows-events.plugin/windows-events.h @@ -126,7 +126,6 @@ typedef enum { #include "windows-events-sources.h" #include "windows-events-unicode.h" -#include "windows-events-sid.h" #include "windows-events-xml.h" #include "windows-events-providers.h" #include "windows-events-fields-cache.h" diff --git a/src/collectors/windows.plugin/perflib-storage.c b/src/collectors/windows.plugin/perflib-storage.c index 537f7c4eca8f1e..d2db61fd26b445 100644 --- a/src/collectors/windows.plugin/perflib-storage.c +++ b/src/collectors/windows.plugin/perflib-storage.c @@ -6,7 +6,6 @@ #define _COMMON_PLUGIN_NAME PLUGIN_WINDOWS_NAME #define _COMMON_PLUGIN_MODULE_NAME "PerflibStorage" #include "../common-contexts/common-contexts.h" - #include "libnetdata/os/windows-wmi/windows-wmi.h" struct logical_disk { diff --git a/src/database/engine/cache.h b/src/database/engine/cache.h index b6f81bcc2df977..ef96520289e19e 100644 --- a/src/database/engine/cache.h +++ b/src/database/engine/cache.h @@ -248,4 +248,15 @@ struct aral_statistics *pgc_aral_statistics(void); size_t pgc_aral_structures(void); size_t pgc_aral_overhead(void); +static inline size_t indexing_partition(Word_t ptr, Word_t modulo) __attribute__((const)); +static inline size_t indexing_partition(Word_t ptr, Word_t modulo) { +#ifdef ENV64BIT + uint64_t hash = murmur64(ptr); + return hash % modulo; +#else + uint32_t hash = murmur32(ptr); + return hash % modulo; +#endif +} + #endif // DBENGINE_CACHE_H diff --git a/src/libnetdata/buffer/buffer.h b/src/libnetdata/buffer/buffer.h index 01a24be35a8839..78ee49d54bd802 100644 --- a/src/libnetdata/buffer/buffer.h +++ b/src/libnetdata/buffer/buffer.h @@ -3,6 +3,8 @@ #ifndef NETDATA_WEB_BUFFER_H #define NETDATA_WEB_BUFFER_H 1 +#include "../uuid/uuid.h" +#include "../http/content_type.h" #include "../string/utf8.h" #include "../libnetdata.h" diff --git a/src/libnetdata/common.h b/src/libnetdata/common.h new file mode 100644 index 00000000000000..b959929d2aa51b --- /dev/null +++ b/src/libnetdata/common.h @@ -0,0 +1,394 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef LIBNETDATA_COMMON_H +#define LIBNETDATA_COMMON_H + +# ifdef __cplusplus +extern "C" { +# endif + +#include "config.h" + +#if defined(NETDATA_DEV_MODE) && !defined(NETDATA_INTERNAL_CHECKS) +#define NETDATA_INTERNAL_CHECKS 1 +#endif + +#ifndef SIZEOF_VOID_P +#error SIZEOF_VOID_P is not defined +#endif + +#if SIZEOF_VOID_P == 4 +#define ENV32BIT 1 +#else +#define ENV64BIT 1 +#endif + +#ifdef HAVE_LIBDATACHANNEL +#define ENABLE_WEBRTC 1 +#endif + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) + +// -------------------------------------------------------------------------------------------------------------------- +// NETDATA_OS_TYPE + +#if defined(__FreeBSD__) +#include +#define NETDATA_OS_TYPE "freebsd" +#elif defined(__APPLE__) +#define NETDATA_OS_TYPE "macos" +#elif defined(OS_WINDOWS) +#define NETDATA_OS_TYPE "windows" +#else +#define NETDATA_OS_TYPE "linux" +#endif /* __FreeBSD__, __APPLE__*/ + +// -------------------------------------------------------------------------------------------------------------------- +// memory allocators + +/* select the memory allocator, based on autoconf findings */ +#if defined(ENABLE_JEMALLOC) + +#if defined(HAVE_JEMALLOC_JEMALLOC_H) +#include +#else // !defined(HAVE_JEMALLOC_JEMALLOC_H) +#include +#endif // !defined(HAVE_JEMALLOC_JEMALLOC_H) + +#elif defined(ENABLE_TCMALLOC) + +#include + +#else /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ + +#if !(defined(__FreeBSD__) || defined(__APPLE__)) +#include +#endif /* __FreeBSD__ || __APPLE__ */ + +#endif /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ + +// -------------------------------------------------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ARPA_INET_H +#include +#endif + +#ifdef HAVE_NETINET_TCP_H +#include +#endif + +#ifdef HAVE_SYS_IOCTL_H +#include +#endif + +#ifdef HAVE_GRP_H +#include +#else +typedef uint32_t gid_t; +#endif + +#ifdef HAVE_PWD_H +#include +#else +typedef uint32_t uid_t; +#endif + +#ifdef HAVE_NET_IF_H +#include +#endif + +#ifdef HAVE_POLL_H +#include +#endif + +#ifdef HAVE_SYSLOG_H +#include +#else +/* priorities */ +#define LOG_EMERG 0 /* system is unusable */ +#define LOG_ALERT 1 /* action must be taken immediately */ +#define LOG_CRIT 2 /* critical conditions */ +#define LOG_ERR 3 /* error conditions */ +#define LOG_WARNING 4 /* warning conditions */ +#define LOG_NOTICE 5 /* normal but significant condition */ +#define LOG_INFO 6 /* informational */ +#define LOG_DEBUG 7 /* debug-level messages */ + +/* facility codes */ +#define LOG_KERN (0<<3) /* kernel messages */ +#define LOG_USER (1<<3) /* random user-level messages */ +#define LOG_MAIL (2<<3) /* mail system */ +#define LOG_DAEMON (3<<3) /* system daemons */ +#define LOG_AUTH (4<<3) /* security/authorization messages */ +#define LOG_SYSLOG (5<<3) /* messages generated internally by syslogd */ +#define LOG_LPR (6<<3) /* line printer subsystem */ +#define LOG_NEWS (7<<3) /* network news subsystem */ +#define LOG_UUCP (8<<3) /* UUCP subsystem */ +#define LOG_CRON (9<<3) /* clock daemon */ +#define LOG_AUTHPRIV (10<<3) /* security/authorization messages (private) */ +#define LOG_FTP (11<<3) /* ftp daemon */ + +/* other codes through 15 reserved for system use */ +#define LOG_LOCAL0 (16<<3) /* reserved for local use */ +#define LOG_LOCAL1 (17<<3) /* reserved for local use */ +#define LOG_LOCAL2 (18<<3) /* reserved for local use */ +#define LOG_LOCAL3 (19<<3) /* reserved for local use */ +#define LOG_LOCAL4 (20<<3) /* reserved for local use */ +#define LOG_LOCAL5 (21<<3) /* reserved for local use */ +#define LOG_LOCAL6 (22<<3) /* reserved for local use */ +#define LOG_LOCAL7 (23<<3) /* reserved for local use */ +#endif + +#ifdef HAVE_SYS_MMAN_H +#include +#endif + +#ifdef HAVE_SYS_RESOURCE_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif + +#ifdef HAVE_SYS_WAIT_H +#include +#endif + +#ifdef HAVE_SYS_UN_H +#include +#endif + +#ifdef HAVE_SPAWN_H +#include +#endif + +#ifdef HAVE_NETINET_IN_H +#include +#endif + +#ifdef HAVE_RESOLV_H +#include +#endif + +#ifdef HAVE_NETDB_H +#include +#endif + +#ifdef HAVE_SYS_PRCTL_H +#include +#endif + +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#ifdef HAVE_SYS_VFS_H +#include +#endif + +#ifdef HAVE_SYS_STATFS_H +#include +#endif + +#ifdef HAVE_LINUX_MAGIC_H +#include +#endif + +#ifdef HAVE_SYS_MOUNT_H +#include +#endif + +#ifdef HAVE_SYS_STATVFS_H +#include +#endif + +// #1408 +#ifdef MAJOR_IN_MKDEV +#include +#endif +#ifdef MAJOR_IN_SYSMACROS +#include +#endif + +#include +#include + +#if defined(HAVE_INTTYPES_H) +#include +#elif defined(HAVE_STDINT_H) +#include +#endif + +#include + +#ifdef HAVE_SYS_CAPABILITY_H +#include +#endif + +#define XXH_INLINE_ALL +#include "xxhash.h" + +// -------------------------------------------------------------------------------------------------------------------- +// OpenSSL + +#define OPENSSL_VERSION_095 0x00905100L +#define OPENSSL_VERSION_097 0x0907000L +#define OPENSSL_VERSION_110 0x10100000L +#define OPENSSL_VERSION_111 0x10101000L +#define OPENSSL_VERSION_300 0x30000000L + +#include +#include +#include +#include +#include +#if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097) && (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) +#include +#endif + +#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_300 +#include +#include +#endif + +// -------------------------------------------------------------------------------------------------------------------- + +#ifndef O_CLOEXEC +#define O_CLOEXEC (0) +#endif + +// -------------------------------------------------------------------------------------------------------------------- +// FUNCTION ATTRIBUTES + +#define _cleanup_(x) __attribute__((__cleanup__(x))) + +#ifdef HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL +#define NEVERNULL __attribute__((returns_nonnull)) +#else +#define NEVERNULL +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_NOINLINE +#define NOINLINE __attribute__((noinline)) +#else +#define NOINLINE +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_MALLOC +#define MALLOCLIKE __attribute__((malloc)) +#else +#define MALLOCLIKE +#endif + +#if defined(HAVE_FUNC_ATTRIBUTE_FORMAT_GNU_PRINTF) +#define PRINTFLIKE(f, a) __attribute__ ((format(gnu_printf, f, a))) +#elif defined(HAVE_FUNC_ATTRIBUTE_FORMAT_PRINTF) +#define PRINTFLIKE(f, a) __attribute__ ((format(printf, f, a))) +#else +#define PRINTFLIKE(f, a) +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_NORETURN +#define NORETURN __attribute__ ((noreturn)) +#else +#define NORETURN +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT +#define WARNUNUSED __attribute__ ((warn_unused_result)) +#else +#define WARNUNUSED +#endif + +#define UNUSED(x) (void)(x) + +#ifdef __GNUC__ +#define UNUSED_FUNCTION(x) __attribute__((unused)) UNUSED_##x +#else +#define UNUSED_FUNCTION(x) UNUSED_##x +#endif + +// -------------------------------------------------------------------------------------------------------------------- +// fix for alpine linux + +#if !defined(RUSAGE_THREAD) && defined(RUSAGE_CHILDREN) +#define RUSAGE_THREAD RUSAGE_CHILDREN +#endif + +// -------------------------------------------------------------------------------------------------------------------- +// HELPFUL MACROS + +#define ABS(x) (((x) < 0)? (-(x)) : (x)) +#define MIN(a,b) (((a)<(b))?(a):(b)) +#define MAX(a,b) (((a)>(b))?(a):(b)) +#define SWAP(a, b) do { \ + typeof(a) _tmp = b; \ + b = a; \ + a = _tmp; \ +} while(0) + +// -------------------------------------------------------------------------------------------------------------------- +// NETDATA CLOUD + +// BEWARE: this exists in alarm-notify.sh +#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" + +// -------------------------------------------------------------------------------------------------------------------- +// DBENGINE + +#define RRD_STORAGE_TIERS 5 + +// -------------------------------------------------------------------------------------------------------------------- +// PIPES + +#define PIPE_READ 0 +#define PIPE_WRITE 1 + +// -------------------------------------------------------------------------------------------------------------------- +// UUIDs + +#define GUID_LEN 36 + +// -------------------------------------------------------------------------------------------------------------------- +// Macro-only includes + +#include "linked-lists.h" + +// -------------------------------------------------------------------------------------------------------------------- + +// Taken from linux kernel +#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) + +// -------------------------------------------------------------------------------------------------------------------- + +# ifdef __cplusplus +} +# endif + +#endif //LIBNETDATA_COMMON_H diff --git a/src/libnetdata/inlined.h b/src/libnetdata/inlined.h index 624be84919039d..50bc5e269bb813 100644 --- a/src/libnetdata/inlined.h +++ b/src/libnetdata/inlined.h @@ -106,17 +106,6 @@ static inline uint64_t murmur64(uint64_t k) { return k; } -static inline size_t indexing_partition(Word_t ptr, Word_t modulo) __attribute__((const)); -static inline size_t indexing_partition(Word_t ptr, Word_t modulo) { -#ifdef ENV64BIT - uint64_t hash = murmur64(ptr); - return hash % modulo; -#else - uint32_t hash = murmur32(ptr); - return hash % modulo; -#endif -} - static inline unsigned int str2u(const char *s) { unsigned int n = 0; diff --git a/src/libnetdata/libnetdata.c b/src/libnetdata/libnetdata.c index fad00c3a7fe2d4..e21bf119dcb0eb 100644 --- a/src/libnetdata/libnetdata.c +++ b/src/libnetdata/libnetdata.c @@ -2,6 +2,10 @@ #include "libnetdata.h" +#define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2) +#define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED) +#define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED) + #if !defined(MADV_DONTFORK) #define MADV_DONTFORK 0 #endif diff --git a/src/libnetdata/libnetdata.h b/src/libnetdata/libnetdata.h index 969e677cb77655..2ba64b95bdc5c4 100644 --- a/src/libnetdata/libnetdata.h +++ b/src/libnetdata/libnetdata.h @@ -7,323 +7,17 @@ extern "C" { # endif -#include "config.h" - -#ifdef HAVE_LIBDATACHANNEL -#define ENABLE_WEBRTC 1 -#endif - -#define STRINGIFY(x) #x -#define TOSTRING(x) STRINGIFY(x) +#include "common.h" #define JUDYHS_INDEX_SIZE_ESTIMATE(key_bytes) (((key_bytes) + sizeof(Word_t) - 1) / sizeof(Word_t) * 4) -#if defined(NETDATA_DEV_MODE) && !defined(NETDATA_INTERNAL_CHECKS) -#define NETDATA_INTERNAL_CHECKS 1 -#endif - -#ifndef SIZEOF_VOID_P -#error SIZEOF_VOID_P is not defined -#endif - -#if SIZEOF_VOID_P == 4 -#define ENV32BIT 1 -#else -#define ENV64BIT 1 -#endif - // NETDATA_TRACE_ALLOCATIONS does not work under musl libc, so don't enable it //#if defined(NETDATA_INTERNAL_CHECKS) && !defined(NETDATA_TRACE_ALLOCATIONS) //#define NETDATA_TRACE_ALLOCATIONS 1 //#endif -#define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2) -#define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED) -#define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED) - -// ---------------------------------------------------------------------------- -// system include files for all netdata C programs - -/* select the memory allocator, based on autoconf findings */ -#if defined(ENABLE_JEMALLOC) - -#if defined(HAVE_JEMALLOC_JEMALLOC_H) -#include -#else // !defined(HAVE_JEMALLOC_JEMALLOC_H) -#include -#endif // !defined(HAVE_JEMALLOC_JEMALLOC_H) - -#elif defined(ENABLE_TCMALLOC) - -#include - -#else /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ - -#if !(defined(__FreeBSD__) || defined(__APPLE__)) -#include -#endif /* __FreeBSD__ || __APPLE__ */ - -#endif /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ - -// ---------------------------------------------------------------------------- - -#if defined(__FreeBSD__) -#include -#define NETDATA_OS_TYPE "freebsd" -#elif defined(__APPLE__) -#define NETDATA_OS_TYPE "macos" -#elif defined(OS_WINDOWS) -#define NETDATA_OS_TYPE "windows" -#else -#define NETDATA_OS_TYPE "linux" -#endif /* __FreeBSD__, __APPLE__*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_ARPA_INET_H -#include -#endif - -#ifdef HAVE_NETINET_TCP_H -#include -#endif - -#ifdef HAVE_SYS_IOCTL_H -#include -#endif - -#ifdef HAVE_GRP_H -#include -#else -typedef uint32_t gid_t; -#endif - -#ifdef HAVE_PWD_H -#include -#else -typedef uint32_t uid_t; -#endif - -#ifdef HAVE_NET_IF_H -#include -#endif - -#ifdef HAVE_POLL_H -#include -#endif - -#ifdef HAVE_SYSLOG_H -#include -#else -/* priorities */ -#define LOG_EMERG 0 /* system is unusable */ -#define LOG_ALERT 1 /* action must be taken immediately */ -#define LOG_CRIT 2 /* critical conditions */ -#define LOG_ERR 3 /* error conditions */ -#define LOG_WARNING 4 /* warning conditions */ -#define LOG_NOTICE 5 /* normal but significant condition */ -#define LOG_INFO 6 /* informational */ -#define LOG_DEBUG 7 /* debug-level messages */ - -/* facility codes */ -#define LOG_KERN (0<<3) /* kernel messages */ -#define LOG_USER (1<<3) /* random user-level messages */ -#define LOG_MAIL (2<<3) /* mail system */ -#define LOG_DAEMON (3<<3) /* system daemons */ -#define LOG_AUTH (4<<3) /* security/authorization messages */ -#define LOG_SYSLOG (5<<3) /* messages generated internally by syslogd */ -#define LOG_LPR (6<<3) /* line printer subsystem */ -#define LOG_NEWS (7<<3) /* network news subsystem */ -#define LOG_UUCP (8<<3) /* UUCP subsystem */ -#define LOG_CRON (9<<3) /* clock daemon */ -#define LOG_AUTHPRIV (10<<3) /* security/authorization messages (private) */ -#define LOG_FTP (11<<3) /* ftp daemon */ - -/* other codes through 15 reserved for system use */ -#define LOG_LOCAL0 (16<<3) /* reserved for local use */ -#define LOG_LOCAL1 (17<<3) /* reserved for local use */ -#define LOG_LOCAL2 (18<<3) /* reserved for local use */ -#define LOG_LOCAL3 (19<<3) /* reserved for local use */ -#define LOG_LOCAL4 (20<<3) /* reserved for local use */ -#define LOG_LOCAL5 (21<<3) /* reserved for local use */ -#define LOG_LOCAL6 (22<<3) /* reserved for local use */ -#define LOG_LOCAL7 (23<<3) /* reserved for local use */ -#endif - -#ifdef HAVE_SYS_MMAN_H -#include -#endif - -#ifdef HAVE_SYS_RESOURCE_H -#include -#endif - -#ifdef HAVE_SYS_SOCKET_H -#include -#endif - -#ifdef HAVE_SYS_WAIT_H -#include -#endif - -#ifdef HAVE_SYS_UN_H -#include -#endif - -#ifdef HAVE_SPAWN_H -#include -#endif - -#ifdef HAVE_NETINET_IN_H -#include -#endif - -#ifdef HAVE_RESOLV_H -#include -#endif - -#ifdef HAVE_NETDB_H -#include -#endif - -#ifdef HAVE_SYS_PRCTL_H -#include -#endif - -#ifdef HAVE_SYS_STAT_H -#include -#endif - -#ifdef HAVE_SYS_VFS_H -#include -#endif - -#ifdef HAVE_SYS_STATFS_H -#include -#endif - -#ifdef HAVE_LINUX_MAGIC_H -#include -#endif - -#ifdef HAVE_SYS_MOUNT_H -#include -#endif - -#ifdef HAVE_SYS_STATVFS_H -#include -#endif - -// #1408 -#ifdef MAJOR_IN_MKDEV -#include -#endif -#ifdef MAJOR_IN_SYSMACROS -#include -#endif - -#include -#include - -#if defined(HAVE_INTTYPES_H) -#include -#elif defined(HAVE_STDINT_H) -#include -#endif - -#include - -#ifdef HAVE_SYS_CAPABILITY_H -#include -#endif - - -#ifndef O_CLOEXEC -#define O_CLOEXEC (0) -#endif - -// ---------------------------------------------------------------------------- -// netdata common definitions - -#define _cleanup_(x) __attribute__((__cleanup__(x))) - -#ifdef HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL -#define NEVERNULL __attribute__((returns_nonnull)) -#else -#define NEVERNULL -#endif - -#ifdef HAVE_FUNC_ATTRIBUTE_NOINLINE -#define NOINLINE __attribute__((noinline)) -#else -#define NOINLINE -#endif - -#ifdef HAVE_FUNC_ATTRIBUTE_MALLOC -#define MALLOCLIKE __attribute__((malloc)) -#else -#define MALLOCLIKE -#endif - -#if defined(HAVE_FUNC_ATTRIBUTE_FORMAT_GNU_PRINTF) -#define PRINTFLIKE(f, a) __attribute__ ((format(gnu_printf, f, a))) -#elif defined(HAVE_FUNC_ATTRIBUTE_FORMAT_PRINTF) -#define PRINTFLIKE(f, a) __attribute__ ((format(printf, f, a))) -#else -#define PRINTFLIKE(f, a) -#endif - -#ifdef HAVE_FUNC_ATTRIBUTE_NORETURN -#define NORETURN __attribute__ ((noreturn)) -#else -#define NORETURN -#endif - -#ifdef HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT -#define WARNUNUSED __attribute__ ((warn_unused_result)) -#else -#define WARNUNUSED -#endif - #include "libjudy/judy-malloc.h" -#define ABS(x) (((x) < 0)? (-(x)) : (x)) -#define MIN(a,b) (((a)<(b))?(a):(b)) -#define MAX(a,b) (((a)>(b))?(a):(b)) -#define SWAP(a, b) do { \ - typeof(a) _tmp = b; \ - b = a; \ - a = _tmp; \ -} while(0) - -#define GUID_LEN 36 - -#define PIPE_READ 0 -#define PIPE_WRITE 1 - -#include "linked-lists.h" #include "storage-point.h" #include "paths/paths.h" @@ -389,31 +83,11 @@ extern volatile sig_atomic_t netdata_exit; char *read_by_filename(const char *filename, long *file_size); char *find_and_replace(const char *src, const char *find, const char *replace, const char *where); -/* fix for alpine linux */ -#ifndef RUSAGE_THREAD -#ifdef RUSAGE_CHILDREN -#define RUSAGE_THREAD RUSAGE_CHILDREN -#endif -#endif - #define BITS_IN_A_KILOBIT 1000 #define KILOBITS_IN_A_MEGABIT 1000 -/* misc. */ - -#define UNUSED(x) (void)(x) - -#ifdef __GNUC__ -#define UNUSED_FUNCTION(x) __attribute__((unused)) UNUSED_##x -#else -#define UNUSED_FUNCTION(x) UNUSED_##x -#endif - #define error_report(x, args...) do { errno_clear(); netdata_log_error(x, ##args); } while(0) -// Taken from linux kernel -#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) - #include "bitmap64.h" #define COMPRESSION_MAX_CHUNK 0x4000 @@ -432,47 +106,55 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re extern const char *netdata_configured_host_prefix; -#include "os/os.h" +// safe includes before O/S specific functions +#include "template-enum.h" +#include "libjudy/src/Judy.h" +#include "july/july.h" -#define XXH_INLINE_ALL -#include "xxhash.h" +#include "string/string.h" +#include "buffer/buffer.h" #include "uuid/uuid.h" -#include "template-enum.h" -#include "http/http_access.h" #include "http/content_type.h" -#include "config/dyncfg.h" -#include "libjudy/src/Judy.h" -#include "july/july.h" +#include "http/http_access.h" + +#include "inlined.h" +#include "parsers/parsers.h" + #include "threads/threads.h" -#include "buffer/buffer.h" -#include "ringbuffer/ringbuffer.h" -#include "c_rhash/c_rhash.h" #include "locks/locks.h" -#include "circular_buffer/circular_buffer.h" +#include "completion/completion.h" +#include "clocks/clocks.h" +#include "simple_pattern/simple_pattern.h" +#include "libnetdata/log/nd_log.h" + +#include "socket/security.h" // must be before windows.h + +// this may include windows.h +#include "os/os.h" + +#include "socket/socket.h" #include "avl/avl.h" -#include "inlined.h" + #include "line_splitter/line_splitter.h" -#include "clocks/clocks.h" -#include "parsers/parsers.h" +#include "c_rhash/c_rhash.h" +#include "ringbuffer/ringbuffer.h" +#include "circular_buffer/circular_buffer.h" +#include "buffered_reader/buffered_reader.h" #include "datetime/iso8601.h" #include "datetime/rfc3339.h" #include "datetime/rfc7231.h" -#include "completion/completion.h" -#include "libnetdata/log/nd_log.h" +#include "sanitizers/sanitizers.h" + +#include "config/dyncfg.h" +#include "config/appconfig.h" #include "spawn_server/spawn_server.h" #include "spawn_server/spawn_popen.h" -#include "simple_pattern/simple_pattern.h" -#include "socket/security.h" -#include "socket/socket.h" -#include "config/appconfig.h" -#include "log/systemd-journal-helpers.h" -#include "buffered_reader/buffered_reader.h" #include "procfile/procfile.h" -#include "string/string.h" #include "dictionary/dictionary.h" #include "dictionary/thread-cache.h" -#include "sanitizers/sanitizers.h" + +#include "log/systemd-journal-helpers.h" #if defined(HAVE_LIBBPF) && !defined(__cplusplus) #include "ebpf/ebpf.h" @@ -494,11 +176,6 @@ extern const char *netdata_configured_host_prefix; #include "functions_evloop/functions_evloop.h" #include "query_progress/progress.h" -// BEWARE: this exists in alarm-notify.sh -#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" - -#define RRD_STORAGE_TIERS 5 - static inline size_t struct_natural_alignment(size_t size) __attribute__((const)); #define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2) @@ -644,6 +321,8 @@ bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_ int netdata_base64_decode(unsigned char *out, const unsigned char *in, int in_len); int netdata_base64_encode(unsigned char *encoded, const unsigned char *input, size_t input_size); +// -------------------------------------------------------------------------------------------------------------------- + static inline void freez_charp(char **p) { freez(*p); } diff --git a/src/libnetdata/maps/local-sockets.h b/src/libnetdata/local-sockets/local-sockets.h similarity index 100% rename from src/libnetdata/maps/local-sockets.h rename to src/libnetdata/local-sockets/local-sockets.h diff --git a/src/libnetdata/maps/system-groups.h b/src/libnetdata/maps/system-groups.h deleted file mode 100644 index f164c0be5aa8dd..00000000000000 --- a/src/libnetdata/maps/system-groups.h +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_SYSTEM_GROUPS_H -#define NETDATA_SYSTEM_GROUPS_H - -#include "libnetdata/libnetdata.h" - -// -------------------------------------------------------------------------------------------------------------------- -// hashtable for caching uid to username mappings -// key is the uid, value is username (STRING) - -#define SIMPLE_HASHTABLE_VALUE_TYPE STRING -#define SIMPLE_HASHTABLE_NAME _GROUPNAMES_CACHE -#include "libnetdata/simple_hashtable.h" - -typedef struct groupnames_cache { - SPINLOCK spinlock; - SIMPLE_HASHTABLE_GROUPNAMES_CACHE ht; -} GROUPNAMES_CACHE; - -static inline STRING *system_groupnames_cache_lookup_gid(GROUPNAMES_CACHE *gc, gid_t gid) { - spinlock_lock(&gc->spinlock); - - XXH64_hash_t hash = XXH3_64bits(&gid, sizeof(gid)); - SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_get_slot_GROUPNAMES_CACHE(&gc->ht, hash, &gid, true); - STRING *g = SIMPLE_HASHTABLE_SLOT_DATA(sl); - if(!g) { - char tmp[1024 + 1]; - struct group grp, *result = NULL; - - if (getgrgid_r(gid, &grp, tmp, sizeof(tmp), &result) != 0 || !result || !grp.gr_name || !(*grp.gr_name)) { - char name[50]; - snprintfz(name, sizeof(name), "%u", gid); - g = string_strdupz(name); - } - else - g = string_strdupz(grp.gr_name); - - simple_hashtable_set_slot_GROUPNAMES_CACHE(&gc->ht, sl, hash, g); - } - - g = string_dup(g); - spinlock_unlock(&gc->spinlock); - return g; -} - -static inline GROUPNAMES_CACHE *system_groupnames_cache_init(void) { - GROUPNAMES_CACHE *gc = callocz(1, sizeof(*gc)); - spinlock_init(&gc->spinlock); - simple_hashtable_init_GROUPNAMES_CACHE(&gc->ht, 100); - return gc; -} - -static inline void system_groupnames_cache_destroy(GROUPNAMES_CACHE *gc) { - spinlock_lock(&gc->spinlock); - - for(SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_first_read_only_GROUPNAMES_CACHE(&gc->ht); - sl; - sl = simple_hashtable_next_read_only_GROUPNAMES_CACHE(&gc->ht, sl)) { - STRING *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); - string_freez(u); - } - - simple_hashtable_destroy_GROUPNAMES_CACHE(&gc->ht); - freez(gc); -} - -#endif //NETDATA_SYSTEM_GROUPS_H diff --git a/src/libnetdata/maps/system-users.h b/src/libnetdata/maps/system-users.h deleted file mode 100644 index f8fbb64ad58ac6..00000000000000 --- a/src/libnetdata/maps/system-users.h +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_SYSTEM_USERS_H -#define NETDATA_SYSTEM_USERS_H - -#include "libnetdata/libnetdata.h" - -// -------------------------------------------------------------------------------------------------------------------- -// hashtable for caching uid to username mappings -// key is the uid, value is username (STRING) - -#define SIMPLE_HASHTABLE_VALUE_TYPE STRING -#define SIMPLE_HASHTABLE_NAME _USERNAMES_CACHE -#include "libnetdata/simple_hashtable.h" - -typedef struct usernames_cache { - SPINLOCK spinlock; - SIMPLE_HASHTABLE_USERNAMES_CACHE ht; -} USERNAMES_CACHE; - -static inline STRING *system_usernames_cache_lookup_uid(USERNAMES_CACHE *uc, uid_t uid) { - spinlock_lock(&uc->spinlock); - - XXH64_hash_t hash = XXH3_64bits(&uid, sizeof(uid)); - SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_get_slot_USERNAMES_CACHE(&uc->ht, hash, &uid, true); - STRING *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); - if(!u) { - char tmp[1024 + 1]; - struct passwd pw, *result = NULL; - - if (getpwuid_r(uid, &pw, tmp, sizeof(tmp), &result) != 0 || !result || !pw.pw_name || !(*pw.pw_name)) { - char name[50]; - snprintfz(name, sizeof(name), "%u", uid); - u = string_strdupz(name); - } - else - u = string_strdupz(pw.pw_name); - - simple_hashtable_set_slot_USERNAMES_CACHE(&uc->ht, sl, hash, u); - } - - u = string_dup(u); - spinlock_unlock(&uc->spinlock); - return u; -} - -static inline USERNAMES_CACHE *system_usernames_cache_init(void) { - USERNAMES_CACHE *uc = callocz(1, sizeof(*uc)); - spinlock_init(&uc->spinlock); - simple_hashtable_init_USERNAMES_CACHE(&uc->ht, 100); - return uc; -} - -static inline void system_usernames_cache_destroy(USERNAMES_CACHE *uc) { - spinlock_lock(&uc->spinlock); - - for(SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_first_read_only_USERNAMES_CACHE(&uc->ht); - sl; - sl = simple_hashtable_next_read_only_USERNAMES_CACHE(&uc->ht, sl)) { - STRING *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); - string_freez(u); - } - - simple_hashtable_destroy_USERNAMES_CACHE(&uc->ht); - freez(uc); -} - -#endif //NETDATA_SYSTEM_USERS_H diff --git a/src/libnetdata/os/os.h b/src/libnetdata/os/os.h index 04bbaf7d1300ff..77d530b01b9584 100644 --- a/src/libnetdata/os/os.h +++ b/src/libnetdata/os/os.h @@ -23,8 +23,16 @@ #include "os-freebsd-wrappers.h" #include "os-macos-wrappers.h" #include "os-windows-wrappers.h" +#include "system-maps/cached-uid-username.h" +#include "system-maps/cached-gid-groupname.h" +#include "system-maps/cache-host-users-and-groups.h" +#include "system-maps/cached-sid-username.h" #include "windows-perflib/perflib.h" +// this includes windows.h to the whole of netdata +// so various conflicts arise +// #include "windows-wmi/windows-wmi.h" + // ===================================================================================================================== // common defs for Apple/FreeBSD/Linux diff --git a/src/libnetdata/os/system-maps/cache-host-users-and-groups.c b/src/libnetdata/os/system-maps/cache-host-users-and-groups.c new file mode 100644 index 00000000000000..53825fd35b75fc --- /dev/null +++ b/src/libnetdata/os/system-maps/cache-host-users-and-groups.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "libnetdata/libnetdata.h" + +static bool file_changed(const struct stat *statbuf __maybe_unused, struct timespec *last_modification_time __maybe_unused) { +#if defined(OS_MACOS) || defined(OS_WINDOWS) + return false; +#else + if(likely(statbuf->st_mtim.tv_sec == last_modification_time->tv_sec && + statbuf->st_mtim.tv_nsec == last_modification_time->tv_nsec)) return false; + + last_modification_time->tv_sec = statbuf->st_mtim.tv_sec; + last_modification_time->tv_nsec = statbuf->st_mtim.tv_nsec; + + return true; +#endif +} + +static size_t read_passwd_or_group(const char *filename, struct timespec *last_modification_time, void (*cb)(uint32_t gid, const char *name, uint32_t version), uint32_t version) { + struct stat statbuf; + if(unlikely(stat(filename, &statbuf) || !file_changed(&statbuf, last_modification_time))) + return 0; + + procfile *ff = procfile_open(filename, " :\t", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 0; + + ff = procfile_readall(ff); + if(unlikely(!ff)) return 0; + + size_t line, lines = procfile_lines(ff); + + size_t added = 0; + for(line = 0; line < lines ;line++) { + size_t words = procfile_linewords(ff, line); + if(unlikely(words < 3)) continue; + + char *name = procfile_lineword(ff, line, 0); + if(unlikely(!name || !*name)) continue; + + char *id_string = procfile_lineword(ff, line, 2); + if(unlikely(!id_string || !*id_string)) continue; + + uint32_t id = str2ull(id_string, NULL); + + cb(id, name, version); + added++; + } + + procfile_close(ff); + return added; +} + +void update_cached_host_users(void) { + if(!netdata_configured_host_prefix || !*netdata_configured_host_prefix) return; + + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER; + if(!spinlock_trylock(&spinlock)) return; + + char filename[FILENAME_MAX]; + static bool initialized = false; + + size_t added = 0; + + if(!initialized) { + initialized = true; + cached_usernames_init(); + } + + static uint32_t passwd_version = 0; + static struct timespec passwd_ts = { 0 }; + snprintfz(filename, FILENAME_MAX, "%s/etc/passwd", netdata_configured_host_prefix); + added = read_passwd_or_group(filename, &passwd_ts, cached_username_populate_by_uid, ++passwd_version); + if(added) cached_usernames_delete_old_versions(passwd_version); + + spinlock_unlock(&spinlock); +} + +void update_cached_host_groups(void) { + if(!netdata_configured_host_prefix || !*netdata_configured_host_prefix) return; + + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER; + if(!spinlock_trylock(&spinlock)) return; + + char filename[FILENAME_MAX]; + static bool initialized = false; + + size_t added = 0; + + if(!initialized) { + initialized = true; + cached_groupnames_init(); + } + + static uint32_t group_version = 0; + static struct timespec group_ts = { 0 }; + snprintfz(filename, FILENAME_MAX, "%s/etc/group", netdata_configured_host_prefix); + added = read_passwd_or_group(filename, &group_ts, cached_groupname_populate_by_gid, ++group_version); + if(added) cached_groupnames_delete_old_versions(group_version); + + spinlock_unlock(&spinlock); +} diff --git a/src/libnetdata/os/system-maps/cache-host-users-and-groups.h b/src/libnetdata/os/system-maps/cache-host-users-and-groups.h new file mode 100644 index 00000000000000..7a84bcadffa32d --- /dev/null +++ b/src/libnetdata/os/system-maps/cache-host-users-and-groups.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CACHE_HOST_USERS_AND_GROUPS_H +#define NETDATA_CACHE_HOST_USERS_AND_GROUPS_H + +void update_cached_host_users(void); +void update_cached_host_groups(void); + +#endif //NETDATA_CACHE_HOST_USERS_AND_GROUPS_H diff --git a/src/libnetdata/os/system-maps/cached-gid-groupname.c b/src/libnetdata/os/system-maps/cached-gid-groupname.c new file mode 100644 index 00000000000000..be34b3efec5cd3 --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-gid-groupname.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "cached-gid-groupname.h" + +// -------------------------------------------------------------------------------------------------------------------- +// hashtable for caching gid to groupname mappings +// key is the gid, value is groupname (STRING) + +#define SIMPLE_HASHTABLE_KEY_TYPE gid_t +#define SIMPLE_HASHTABLE_VALUE_TYPE CACHED_GROUPNAME +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION cached_groupname_to_gid_ptr +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION compar_gid_ptr +#define SIMPLE_HASHTABLE_NAME _GROUPNAMES_CACHE +#include "libnetdata/simple_hashtable.h" + +static struct { + bool initialized; + SPINLOCK spinlock; + SIMPLE_HASHTABLE_GROUPNAMES_CACHE ht; +} uc = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .ht = { 0 }, +}; + +static gid_t *cached_groupname_to_gid_ptr(CACHED_GROUPNAME *cu) { + return &cu->gid; +} + +static bool compar_gid_ptr(gid_t *a, gid_t *b) { + return *a == *b; +} + +void cached_groupname_populate_by_gid(gid_t gid, const char *groupname, uint32_t version) { + internal_fatal(!uc.initialized, "system-users cache needs to be initialized"); + if(!groupname || !*groupname) return; + + spinlock_lock(&uc.spinlock); + + XXH64_hash_t hash = XXH3_64bits(&gid, sizeof(gid)); + SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_get_slot_GROUPNAMES_CACHE(&uc.ht, hash, &gid, true); + CACHED_GROUPNAME *cg = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!cg || (cg->version && version > cg->version)) { + internal_fatal(cg && cg->gid != gid, "invalid gid matched from cache"); + + if(cg) + string_freez(cg->groupname); + else + cg = callocz(1, sizeof(*cg)); + + cg->version = version; + cg->gid = gid; + cg->groupname = string_strdupz(groupname); + simple_hashtable_set_slot_GROUPNAMES_CACHE(&uc.ht, sl, hash, cg); + } + + spinlock_unlock(&uc.spinlock); +} + +CACHED_GROUPNAME cached_groupname_get_by_gid(gid_t gid) { + internal_fatal(!uc.initialized, "system-users cache needs to be initialized"); + + spinlock_lock(&uc.spinlock); + + XXH64_hash_t hash = XXH3_64bits(&gid, sizeof(gid)); + SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_get_slot_GROUPNAMES_CACHE(&uc.ht, hash, &gid, true); + CACHED_GROUPNAME *cg = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!cg) { + cg = callocz(1, sizeof(*cg)); + + static char tmp[1024]; // we are inside a global spinlock - it is ok to be static + struct group gr, *result = NULL; + + if (getgrgid_r(gid, &gr, tmp, sizeof(tmp), &result) != 0 || !result || !gr.gr_name || !(*gr.gr_name)) { + char name[UINT64_MAX_LENGTH]; + print_uint64(name, gid); + cg->groupname = string_strdupz(name); + } + else + cg->groupname = string_strdupz(gr.gr_name); + + cg->gid = gid; + simple_hashtable_set_slot_GROUPNAMES_CACHE(&uc.ht, sl, hash, cg); + } + + internal_fatal(cg->gid != gid, "invalid gid matched from cache"); + + CACHED_GROUPNAME rc = { + .version = cg->version, + .gid = cg->gid, + .groupname = string_dup(cg->groupname), + }; + + spinlock_unlock(&uc.spinlock); + return rc; +} + +void cached_groupname_release(CACHED_GROUPNAME cg) { + string_freez(cg.groupname); +} + +void cached_groupnames_init(void) { + if(uc.initialized) return; + uc.initialized = true; + + spinlock_init(&uc.spinlock); + simple_hashtable_init_GROUPNAMES_CACHE(&uc.ht, 100); +} + +void cached_groupnames_destroy(void) { + if(!uc.initialized) return; + + spinlock_lock(&uc.spinlock); + + for(SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_first_read_only_GROUPNAMES_CACHE(&uc.ht); + sl; + sl = simple_hashtable_next_read_only_GROUPNAMES_CACHE(&uc.ht, sl)) { + CACHED_GROUPNAME *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(u) { + string_freez(u->groupname); + freez(u); + // simple_hashtable_del_slot_GROUPNAMES_CACHE(&uc.ht, sl); + } + } + + simple_hashtable_destroy_GROUPNAMES_CACHE(&uc.ht); + uc.initialized = false; +} + +void cached_groupnames_delete_old_versions(uint32_t version) { + if(!uc.initialized) return; + + spinlock_lock(&uc.spinlock); + + for(SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_first_read_only_GROUPNAMES_CACHE(&uc.ht); + sl; + sl = simple_hashtable_next_read_only_GROUPNAMES_CACHE(&uc.ht, sl)) { + CACHED_GROUPNAME *cg = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(cg && cg->version && cg->version < version) { + string_freez(cg->groupname); + freez(cg); + simple_hashtable_del_slot_GROUPNAMES_CACHE(&uc.ht, sl); + } + } + + simple_hashtable_destroy_GROUPNAMES_CACHE(&uc.ht); + uc.initialized = false; +} diff --git a/src/libnetdata/os/system-maps/cached-gid-groupname.h b/src/libnetdata/os/system-maps/cached-gid-groupname.h new file mode 100644 index 00000000000000..81a62523e47ae8 --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-gid-groupname.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CACHED_UID_GROUPNAME_H +#define NETDATA_CACHED_UID_GROUPNAME_H + +#include "libnetdata/libnetdata.h" + +struct netdata_string; + +typedef struct { + uint32_t version; + gid_t gid; + struct netdata_string *groupname; +} CACHED_GROUPNAME; + +void cached_groupname_populate_by_gid(gid_t gid, const char *groupname, uint32_t version); +CACHED_GROUPNAME cached_groupname_get_by_gid(gid_t gid); +void cached_groupname_release(CACHED_GROUPNAME cg); +void cached_groupnames_delete_old_versions(uint32_t version); + +void cached_groupnames_init(void); +void cached_groupnames_destroy(void); + +#endif //NETDATA_CACHED_UID_GROUPNAME_H diff --git a/src/collectors/windows-events.plugin/windows-events-sid.c b/src/libnetdata/os/system-maps/cached-sid-username.c similarity index 57% rename from src/collectors/windows-events.plugin/windows-events-sid.c rename to src/libnetdata/os/system-maps/cached-sid-username.c index 8f92fb495d169c..466ca62c639e47 100644 --- a/src/collectors/windows-events.plugin/windows-events-sid.c +++ b/src/libnetdata/os/system-maps/cached-sid-username.c @@ -1,6 +1,10 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "windows-events-sid.h" +#include "../../libnetdata.h" + +#if defined(OS_WINDOWS) +#include "cached-sid-username.h" +#include #include typedef struct { @@ -12,15 +16,10 @@ typedef struct { // IMPORTANT: // This is malloc'd ! You have to manually set fields to zero. - const char *account; - const char *domain; - const char *full; - const char *sid_str; - - uint32_t account_len; - uint32_t domain_len; - uint32_t full_len; - uint32_t sid_str_len; + STRING *account; + STRING *domain; + STRING *full; + STRING *sid_str; // this needs to be last, because of its variable size SID_KEY key; @@ -49,11 +48,25 @@ static inline bool sid_cache_compar(SID_KEY *a, SID_KEY *b) { return a->len == b->len && memcmp(&a->sid, &b->sid, a->len) == 0; } -void sid_cache_init(void) { +void cached_sid_username_init(void) { simple_hashtable_init_SID(&sid_globals.hashtable, 100); } -static void lookup_user(SID_VALUE *sv) { +static char *account2utf8(const wchar_t *user) { + static __thread char buffer[256]; + if(utf16_to_utf8(buffer, sizeof(buffer), user, -1, NULL) == 0) + buffer[0] = '\0'; + return buffer; +} + +static char *domain2utf8(const wchar_t *domain) { + static __thread char buffer[256]; + if(utf16_to_utf8(buffer, sizeof(buffer), domain, -1, NULL) == 0) + buffer[0] = '\0'; + return buffer; +} + +static void lookup_user_in_system(SID_VALUE *sv) { static __thread wchar_t account_unicode[256]; static __thread wchar_t domain_unicode[256]; static __thread char tmp[512 + 2]; @@ -66,28 +79,21 @@ static void lookup_user(SID_VALUE *sv) { const char *account = account2utf8(account_unicode); const char *domain = domain2utf8(domain_unicode); snprintfz(tmp, sizeof(tmp), "%s\\%s", domain, account); - sv->domain = strdupz(domain); sv->domain_len = strlen(sv->domain); - sv->account = strdupz(account); sv->account_len = strlen(sv->account); - sv->full = strdupz(tmp); sv->full_len = strlen(sv->full); + sv->domain = string_strdupz(domain); + sv->account = string_strdupz(account); + sv->full = string_strdupz(tmp); } else { sv->domain = NULL; sv->account = NULL; sv->full = NULL; - sv->domain_len = 0; - sv->account_len = 0; - sv->full_len = 0; } wchar_t *sid_string = NULL; - if (ConvertSidToStringSidW(sv->key.sid, &sid_string)) { - sv->sid_str = strdupz(account2utf8(sid_string)); - sv->sid_str_len = strlen(sv->sid_str); - } - else { + if (ConvertSidToStringSidW(sv->key.sid, &sid_string)) + sv->sid_str = string_strdupz(account2utf8(sid_string)); + else sv->sid_str = NULL; - sv->sid_str_len = 0; - } } static SID_VALUE *lookup_or_convert_user_id_to_name_lookup(PSID sid) { @@ -112,7 +118,7 @@ static SID_VALUE *lookup_or_convert_user_id_to_name_lookup(PSID sid) { found = mallocz(tmp_size); memcpy(found, buf, tmp_size); - lookup_user(found); + lookup_user_in_system(found); // add it to the cache spinlock_lock(&sid_globals.spinlock); @@ -122,41 +128,44 @@ static SID_VALUE *lookup_or_convert_user_id_to_name_lookup(PSID sid) { return found; } -bool wevt_convert_user_id_to_name(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) { +bool cached_sid_to_account_domain_sidstr(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) { SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid); if(found) { if (found->account) { - txt_utf8_resize(dst_account, found->account_len + 1, false); - memcpy(dst_account->data, found->account, found->account_len + 1); - dst_account->used = found->account_len + 1; + txt_utf8_resize(dst_account, string_strlen(found->account) + 1, false); + memcpy(dst_account->data, string2str(found->account), string_strlen(found->account) + 1); + dst_account->used = string_strlen(found->account) + 1; } - else wevt_utf8_empty(dst_account); + else + txt_utf8_empty(dst_account); if (found->domain) { - txt_utf8_resize(dst_domain, found->domain_len + 1, false); - memcpy(dst_domain->data, found->domain, found->domain_len + 1); - dst_domain->used = found->domain_len + 1; + txt_utf8_resize(dst_domain, string_strlen(found->domain) + 1, false); + memcpy(dst_domain->data, string2str(found->domain), string_strlen(found->domain) + 1); + dst_domain->used = string_strlen(found->domain) + 1; } - else wevt_utf8_empty(dst_domain); + else + txt_utf8_empty(dst_domain); if (found->sid_str) { - txt_utf8_resize(dst_sid_str, found->sid_str_len + 1, false); - memcpy(dst_sid_str->data, found->sid_str, found->sid_str_len + 1); - dst_sid_str->used = found->sid_str_len + 1; + txt_utf8_resize(dst_sid_str, string_strlen(found->sid_str) + 1, false); + memcpy(dst_sid_str->data, string2str(found->sid_str), string_strlen(found->sid_str) + 1); + dst_sid_str->used = string_strlen(found->sid_str) + 1; } - else wevt_utf8_empty(dst_sid_str); + else + txt_utf8_empty(dst_sid_str); return true; } - wevt_utf8_empty(dst_account); - wevt_utf8_empty(dst_domain); - wevt_utf8_empty(dst_sid_str); + txt_utf8_empty(dst_account); + txt_utf8_empty(dst_domain); + txt_utf8_empty(dst_sid_str); return false; } -bool buffer_sid_to_sid_str_and_name(PSID sid, BUFFER *dst, const char *prefix) { +bool cached_sid_to_buffer_append(PSID sid, BUFFER *dst, const char *prefix) { SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid); size_t added = 0; @@ -165,17 +174,28 @@ bool buffer_sid_to_sid_str_and_name(PSID sid, BUFFER *dst, const char *prefix) { if (prefix && *prefix) buffer_strcat(dst, prefix); - buffer_fast_strcat(dst, found->full, found->full_len); + buffer_fast_strcat(dst, string2str(found->full), string_strlen(found->full)); added++; } if (found->sid_str) { if (prefix && *prefix) buffer_strcat(dst, prefix); - buffer_fast_strcat(dst, found->sid_str, found->sid_str_len); + buffer_fast_strcat(dst, string2str(found->sid_str), string_strlen(found->sid_str)); added++; } } return added > 0; } + +STRING *cached_sid_fullname_or_sid_str(PSID sid) { + SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid); + if(found) { + if(found->full) return string_dup(found->full); + return string_dup(found->sid_str); + } + return NULL; +} + +#endif \ No newline at end of file diff --git a/src/libnetdata/os/system-maps/cached-sid-username.h b/src/libnetdata/os/system-maps/cached-sid-username.h new file mode 100644 index 00000000000000..4077cad11d91dd --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-sid-username.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CACHED_SID_USERNAME_H +#define NETDATA_CACHED_SID_USERNAME_H + +#include "../../libnetdata.h" + +#if defined(OS_WINDOWS) +#include "../../string/utf8.h" + +bool cached_sid_to_account_domain_sidstr(void *sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str); +bool cached_sid_to_buffer_append(void *sid, BUFFER *dst, const char *prefix); +void cached_sid_username_init(void); +STRING *cached_sid_fullname_or_sid_str(void *sid); +#endif + +#endif //NETDATA_CACHED_SID_USERNAME_H diff --git a/src/libnetdata/os/system-maps/cached-uid-username.c b/src/libnetdata/os/system-maps/cached-uid-username.c new file mode 100644 index 00000000000000..2de4ad9c5f9a96 --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-uid-username.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "cached-uid-username.h" + +// -------------------------------------------------------------------------------------------------------------------- +// hashtable for caching uid to username mappings +// key is the uid, value is username (STRING) + +#define SIMPLE_HASHTABLE_KEY_TYPE uid_t +#define SIMPLE_HASHTABLE_VALUE_TYPE CACHED_USERNAME +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION cached_username_to_uid_ptr +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION compar_uid_ptr +#define SIMPLE_HASHTABLE_NAME _USERNAMES_CACHE +#include "libnetdata/simple_hashtable.h" + +static struct { + uint32_t version; + bool initialized; + SPINLOCK spinlock; + SIMPLE_HASHTABLE_USERNAMES_CACHE ht; +} uc = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .ht = { 0 }, +}; + +static uid_t *cached_username_to_uid_ptr(CACHED_USERNAME *cu) { + return &cu->uid; +} + +static bool compar_uid_ptr(uid_t *a, uid_t *b) { + return *a == *b; +} + +void cached_username_populate_by_uid(uid_t uid, const char *username, uint32_t version) { + internal_fatal(!uc.initialized, "system-users cache needs to be initialized"); + if(!username || !*username) return; + + spinlock_lock(&uc.spinlock); + + XXH64_hash_t hash = XXH3_64bits(&uid, sizeof(uid)); + SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_get_slot_USERNAMES_CACHE(&uc.ht, hash, &uid, true); + CACHED_USERNAME *cu = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!cu || (cu->version && version > cu->version)) { + internal_fatal(cu && cu->uid != uid, "invalid uid matched from cache"); + + if(cu) + string_freez(cu->username); + else + cu = callocz(1, sizeof(*cu)); + + cu->version = version; + cu->uid = uid; + cu->username = string_strdupz(username); + simple_hashtable_set_slot_USERNAMES_CACHE(&uc.ht, sl, hash, cu); + } + + spinlock_unlock(&uc.spinlock); +} + +CACHED_USERNAME cached_username_get_by_uid(uid_t uid) { + internal_fatal(!uc.initialized, "system-users cache needs to be initialized"); + + spinlock_lock(&uc.spinlock); + + XXH64_hash_t hash = XXH3_64bits(&uid, sizeof(uid)); + SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_get_slot_USERNAMES_CACHE(&uc.ht, hash, &uid, true); + CACHED_USERNAME *cu = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!cu) { + cu = callocz(1, sizeof(*cu)); + + static char tmp[1024]; // we are inside a global spinlock - it is ok to be static + struct passwd pw, *result = NULL; + + if (getpwuid_r(uid, &pw, tmp, sizeof(tmp), &result) != 0 || !result || !pw.pw_name || !(*pw.pw_name)) { + char name[UINT64_MAX_LENGTH]; + print_uint64(name, uid); + cu->username = string_strdupz(name); + } + else + cu->username = string_strdupz(pw.pw_name); + + cu->uid = uid; + simple_hashtable_set_slot_USERNAMES_CACHE(&uc.ht, sl, hash, cu); + } + + internal_fatal(cu->uid != uid, "invalid uid matched from cache"); + + CACHED_USERNAME rc = { + .version = cu->version, + .uid = cu->uid, + .username = string_dup(cu->username), + }; + + spinlock_unlock(&uc.spinlock); + return rc; +} + +void cached_username_release(CACHED_USERNAME cu) { + string_freez(cu.username); +} + +void cached_usernames_init(void) { + if(uc.initialized) return; + uc.initialized = true; + + spinlock_init(&uc.spinlock); + simple_hashtable_init_USERNAMES_CACHE(&uc.ht, 100); +} + +void cached_usernames_destroy(void) { + if(!uc.initialized) return; + + spinlock_lock(&uc.spinlock); + + for(SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_first_read_only_USERNAMES_CACHE(&uc.ht); + sl; + sl = simple_hashtable_next_read_only_USERNAMES_CACHE(&uc.ht, sl)) { + CACHED_USERNAME *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(u) { + string_freez(u->username); + freez(u); + // simple_hashtable_del_slot_USERNAMES_CACHE(&uc.ht, sl); + } + } + + simple_hashtable_destroy_USERNAMES_CACHE(&uc.ht); + uc.initialized = false; +} + +void cached_usernames_delete_old_versions(uint32_t version) { + if(!uc.initialized) return; + + spinlock_lock(&uc.spinlock); + + for(SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_first_read_only_USERNAMES_CACHE(&uc.ht); + sl; + sl = simple_hashtable_next_read_only_USERNAMES_CACHE(&uc.ht, sl)) { + CACHED_USERNAME *cu = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(cu && cu->version && cu->version < version) { + string_freez(cu->username); + freez(cu); + simple_hashtable_del_slot_USERNAMES_CACHE(&uc.ht, sl); + } + } + + simple_hashtable_destroy_USERNAMES_CACHE(&uc.ht); + uc.initialized = false; +} diff --git a/src/libnetdata/os/system-maps/cached-uid-username.h b/src/libnetdata/os/system-maps/cached-uid-username.h new file mode 100644 index 00000000000000..b7c52c7c4a15d1 --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-uid-username.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CACHED_UID_USERNAME_H +#define NETDATA_CACHED_UID_USERNAME_H + +#include "libnetdata/libnetdata.h" + +struct netdata_string; + +typedef struct { + uint32_t version; + uid_t uid; + struct netdata_string *username; +} CACHED_USERNAME; + +void cached_username_populate_by_uid(uid_t uid, const char *username, uint32_t version); +CACHED_USERNAME cached_username_get_by_uid(uid_t uid); +void cached_username_release(CACHED_USERNAME cu); + +void cached_usernames_init(void); +void cached_usernames_destroy(void); +void cached_usernames_delete_old_versions(uint32_t version); + +#endif //NETDATA_CACHED_UID_USERNAME_H diff --git a/src/libnetdata/maps/system-services.h b/src/libnetdata/os/system-maps/system-services.h similarity index 96% rename from src/libnetdata/maps/system-services.h rename to src/libnetdata/os/system-maps/system-services.h index ff51ed7d514395..f056bc21b76de9 100644 --- a/src/libnetdata/maps/system-services.h +++ b/src/libnetdata/os/system-maps/system-services.h @@ -8,7 +8,7 @@ // -------------------------------------------------------------------------------------------------------------------- // hashtable for caching port and protocol to service name mappings -// key is the combination of protocol and port packed into a uint64_t, value is service name (STRING) +// key is the combination of protocol and port packed into an uint64_t, value is service name (STRING) #define SIMPLE_HASHTABLE_VALUE_TYPE STRING #define SIMPLE_HASHTABLE_NAME _SERVICENAMES_CACHE diff --git a/src/libnetdata/os/windows-perflib/perflib.c b/src/libnetdata/os/windows-perflib/perflib.c index e9e9130775c0e2..3136abaf9db43c 100644 --- a/src/libnetdata/os/windows-perflib/perflib.c +++ b/src/libnetdata/os/windows-perflib/perflib.c @@ -398,7 +398,7 @@ static inline BOOL getEncodedStringToUTF8(char *dst, size_t dst_len, DWORD CodeP } else { tempBuffer = unicode; - charsCopied = any_to_utf16(CodePage, unicode, _countof(unicode), start, (int)length); + charsCopied = any_to_utf16(CodePage, unicode, _countof(unicode), start, (int)length, NULL); if(!charsCopied) return FALSE; } diff --git a/src/libnetdata/socket/security.h b/src/libnetdata/socket/security.h index 4e0b113cf58b07..7deb1d79763ffa 100644 --- a/src/libnetdata/socket/security.h +++ b/src/libnetdata/socket/security.h @@ -1,5 +1,5 @@ #ifndef NETDATA_SECURITY_H -# define NETDATA_SECURITY_H +#define NETDATA_SECURITY_H typedef enum __attribute__((packed)) { NETDATA_SSL_STATE_NOT_SSL = 1, // This connection is not SSL @@ -12,26 +12,6 @@ typedef enum __attribute__((packed)) { #define NETDATA_SSL_STREAMING_SENDER_CTX 1 #define NETDATA_SSL_EXPORTING_CTX 2 -#define OPENSSL_VERSION_095 0x00905100L -#define OPENSSL_VERSION_097 0x0907000L -#define OPENSSL_VERSION_110 0x10100000L -#define OPENSSL_VERSION_111 0x10101000L -#define OPENSSL_VERSION_300 0x30000000L - -# include -# include -# include -# include -# include -# if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097) && (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) -# include -# endif - -#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_300 -#include -#include -#endif - typedef struct netdata_ssl { SSL *conn; // SSL connection NETDATA_SSL_STATE state; // The state for SSL connection diff --git a/src/libnetdata/socket/socket.c b/src/libnetdata/socket/socket.c index c77d80277b2585..d8f417b7ce1a1e 100644 --- a/src/libnetdata/socket/socket.c +++ b/src/libnetdata/socket/socket.c @@ -576,7 +576,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, char buffer2[10 + 1]; snprintfz(buffer2, 10, "%d", default_port); - char *ip = buffer, *port = buffer2, *interface = "", *portconfig; + char *ip = buffer, *port = buffer2, *iface = "", *portconfig; int protocol = IPPROTO_TCP, socktype = SOCK_STREAM; const char *protocol_str = "tcp"; @@ -631,7 +631,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, if(*e == '%') { *e = '\0'; e++; - interface = e; + iface = e; while(*e && *e != ':' && *e != '=') e++; } @@ -668,13 +668,13 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, } uint32_t scope_id = 0; - if(*interface) { - scope_id = if_nametoindex(interface); + if(*iface) { + scope_id = if_nametoindex(iface); if(!scope_id) nd_log(NDLS_DAEMON, NDLP_ERR, "LISTENER: Cannot find a network interface named '%s'. " "Continuing with limiting the network interface", - interface); + iface); } if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all")) @@ -1035,7 +1035,7 @@ int connect_to_this(const char *definition, int default_port, struct timeval *ti char default_service[10 + 1]; snprintfz(default_service, 10, "%d", default_port); - char *host = buffer, *service = default_service, *interface = ""; + char *host = buffer, *service = default_service, *iface = ""; int protocol = IPPROTO_TCP, socktype = SOCK_STREAM; uint32_t scope_id = 0; @@ -1074,7 +1074,7 @@ int connect_to_this(const char *definition, int default_port, struct timeval *ti if(*e == '%') { *e = '\0'; e++; - interface = e; + iface = e; while(*e && *e != ':') e++; } @@ -1092,12 +1092,12 @@ int connect_to_this(const char *definition, int default_port, struct timeval *ti return -1; } - if(*interface) { - scope_id = if_nametoindex(interface); + if(*iface) { + scope_id = if_nametoindex(iface); if(!scope_id) nd_log(NDLS_DAEMON, NDLP_ERR, "Cannot find a network interface named '%s'. Continuing with limiting the network interface", - interface); + iface); } if(!*service) diff --git a/src/libnetdata/string/utf8.c b/src/libnetdata/string/utf8.c index e0a6745da4ffdb..aa982a70a4243b 100644 --- a/src/libnetdata/string/utf8.c +++ b/src/libnetdata/string/utf8.c @@ -13,9 +13,12 @@ * 3. Always return the number of wide characters written, including the null terminator */ -size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len) { +size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len, bool *truncated) { if(!src || src_len == 0) { // invalid input + if(truncated) + *truncated = true; + if(dst && dst_size) *dst = L'\0'; return 0; @@ -23,6 +26,10 @@ size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char if(!dst || !dst_size) { // the caller wants to know the buffer to allocate for the conversion + + if(truncated) + *truncated = true; + int required = MultiByteToWideChar(CodePage, 0, src, src_len, NULL, 0); if(required <= 0) return 0; // error in the conversion @@ -34,6 +41,9 @@ size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char // do the conversion directly to the destination buffer int rc = MultiByteToWideChar(CodePage, 0, src, src_len, dst, (int)dst_size); if(rc <= 0) { + if(truncated) + *truncated = true; + // conversion failed, let's see why... DWORD status = GetLastError(); if(status == ERROR_INSUFFICIENT_BUFFER) { @@ -81,9 +91,17 @@ size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char size_t len = rc; + if(truncated) + *truncated = false; + if(len >= dst_size) { - // truncate it to fit the null - dst[dst_size - 1] = L'\0'; + if(dst[dst_size - 1] != L'\0') { + if (truncated) + *truncated = true; + + // Truncate it to fit the null terminator + dst[dst_size - 1] = L'\0'; + } return dst_size; } @@ -106,16 +124,24 @@ size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char * 3. Always return the number of bytes written, including the null terminator */ -size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len) { +size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len, bool *truncated) { if (!src || src_len == 0) { // invalid input + if(truncated) + *truncated = true; + if(dst && dst_size) - *dst = L'\0'; + *dst = '\0'; + return 0; } if (!dst || dst_size == 0) { // The caller wants to know the buffer size required for the conversion + + if(truncated) + *truncated = true; + int required = WideCharToMultiByte(CP_UTF8, 0, src, src_len, NULL, 0, NULL, NULL); if (required <= 0) return 0; // error in the conversion @@ -126,6 +152,9 @@ size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len // Perform the conversion directly into the destination buffer int rc = WideCharToMultiByte(CP_UTF8, 0, src, src_len, dst, (int)dst_size, NULL, NULL); if (rc <= 0) { + if(truncated) + *truncated = true; + // Conversion failed, let's see why... DWORD status = GetLastError(); if (status == ERROR_INSUFFICIENT_BUFFER) { @@ -173,9 +202,17 @@ size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len size_t len = rc; + if(truncated) + *truncated = false; + if (len >= dst_size) { - // Truncate it to fit the null terminator - dst[dst_size - 1] = '\0'; + if(dst[dst_size - 1] != '\0') { + if (truncated) + *truncated = true; + + // Truncate it to fit the null terminator + dst[dst_size - 1] = '\0'; + } return dst_size; } @@ -189,4 +226,185 @@ size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len // The result is already null-terminated return len; } + +// -------------------------------------------------------------------------------------------------------------------- + +size_t txt_compute_new_size(size_t old_size, size_t required_size) { + size_t size = (required_size % 2048 == 0) ? required_size : required_size + 2048; + size = (size / 2048) * 2048; + + if(size < old_size * 2) + size = old_size * 2; + + return size; +} + +// -------------------------------------------------------------------------------------------------------------------- +// TXT_UTF8 + +void txt_utf8_cleanup(TXT_UTF8 *dst) { + freez(dst->data); + dst->data = NULL; + dst->used = 0; +} + +void txt_utf8_resize(TXT_UTF8 *dst, size_t required_size, bool keep) { + if(required_size <= dst->size) + return; + + size_t new_size = txt_compute_new_size(dst->size, required_size); + + if(keep && dst->data) + dst->data = reallocz(dst->data, new_size); + else { + txt_utf8_cleanup(dst); + dst->data = mallocz(new_size); + dst->used = 0; + } + + dst->size = new_size; +} + +void txt_utf8_empty(TXT_UTF8 *dst) { + txt_utf8_resize(dst, 1, false); + dst->data[0] = '\0'; + dst->used = 1; +} + +void txt_utf8_set(TXT_UTF8 *dst, const char *txt, size_t txt_len) { + txt_utf8_resize(dst, dst->used + txt_len + 1, true); + memcpy(dst->data, txt, txt_len); + dst->used = txt_len + 1; + dst->data[dst->used - 1] = '\0'; +} + +void txt_utf8_append(TXT_UTF8 *dst, const char *txt, size_t txt_len) { + if(dst->used <= 1) { + // the destination is empty + txt_utf8_set(dst, txt, txt_len); + } + else { + // there is something already in the buffer + txt_utf8_resize(dst, dst->used + txt_len, true); + memcpy(&dst->data[dst->used - 1], txt, txt_len); + dst->used += txt_len; // the null was already counted + dst->data[dst->used - 1] = '\0'; + } +} + +// -------------------------------------------------------------------------------------------------------------------- +// TXT_UTF16 + +void txt_utf16_cleanup(TXT_UTF16 *dst) { + freez(dst->data); +} + +void txt_utf16_resize(TXT_UTF16 *dst, size_t required_size, bool keep) { + if(required_size <= dst->size) + return; + + size_t new_size = txt_compute_new_size(dst->size, required_size); + + if (keep && dst->data) { + dst->data = reallocz(dst->data, new_size * sizeof(wchar_t)); + } else { + txt_utf16_cleanup(dst); + dst->data = mallocz(new_size * sizeof(wchar_t)); + dst->used = 0; + } + + dst->size = new_size; +} + +void txt_utf16_set(TXT_UTF16 *dst, const wchar_t *txt, size_t txt_len) { + txt_utf16_resize(dst, dst->used + txt_len + 1, true); + memcpy(dst->data, txt, txt_len * sizeof(wchar_t)); + dst->used = txt_len + 1; + dst->data[dst->used - 1] = '\0'; +} + +void txt_utf16_append(TXT_UTF16 *dst, const wchar_t *txt, size_t txt_len) { + if(dst->used <= 1) { + // the destination is empty + txt_utf16_set(dst, txt, txt_len); + } + else { + // there is something already in the buffer + txt_utf16_resize(dst, dst->used + txt_len, true); + memcpy(&dst->data[dst->used - 1], txt, txt_len * sizeof(wchar_t)); + dst->used += txt_len; // the null was already counted + dst->data[dst->used - 1] = '\0'; + } +} + +// -------------------------------------------------------------------------------------------------------------------- + +bool wchar_to_txt_utf8(TXT_UTF8 *dst, const wchar_t *src, int src_len) { + if(!src || !src_len) { + txt_utf8_empty(dst); + return false; + } + + if(!dst->data && !dst->size) { + size_t size = utf16_to_utf8(NULL, 0, src, src_len, NULL); + if(!size) { + txt_utf8_empty(dst); + return false; + } + + // we +1 here to avoid entering the next condition below + txt_utf8_resize(dst, size, false); + } + + bool truncated = false; + dst->used = utf16_to_utf8(dst->data, dst->size, src, src_len, &truncated); + if(truncated) { + // we need to resize + size_t needed = utf16_to_utf8(NULL, 0, src, src_len, NULL); // find the size needed + if(!needed) { + txt_utf8_empty(dst); + return false; + } + + txt_utf8_resize(dst, needed, false); + dst->used = utf16_to_utf8(dst->data, dst->size, src, src_len, NULL); + } + + // Make sure it is not zero padded at the end + while(dst->used >= 2 && dst->data[dst->used - 2] == 0) + dst->used--; + + internal_fatal(strlen(dst->data) + 1 != dst->used, + "Wrong UTF8 string length"); + + return true; +} + +bool txt_utf16_to_utf8(TXT_UTF8 *utf8, TXT_UTF16 *utf16) { + fatal_assert(utf8 && ((utf8->data && utf8->size) || (!utf8->data && !utf8->size))); + fatal_assert(utf16 && ((utf16->data && utf16->size) || (!utf16->data && !utf16->size))); + + // pass the entire utf16 size, including the null terminator + // so that the resulting utf8 message will be null terminated too. + return wchar_to_txt_utf8(utf8, utf16->data, (int)utf16->used - 1); +} + +char *utf16_to_utf8_strdupz(const wchar_t *src, size_t *dst_len) { + size_t size = utf16_to_utf8(NULL, 0, src, -1, NULL); + if (size) { + char *dst = mallocz(size); + + size = utf16_to_utf8(dst, size, src, -1, NULL); + if(dst_len) + *dst_len = size - 1; + + return dst; + } + + if(dst_len) + *dst_len = 0; + + return NULL; +} + #endif diff --git a/src/libnetdata/string/utf8.h b/src/libnetdata/string/utf8.h index ee9fe12b20a8f7..f27ba5447d2f51 100644 --- a/src/libnetdata/string/utf8.h +++ b/src/libnetdata/string/utf8.h @@ -3,6 +3,8 @@ #ifndef NETDATA_STRING_UTF8_H #define NETDATA_STRING_UTF8_H 1 +#include "../libnetdata.h" + #define IS_UTF8_BYTE(x) ((uint8_t)(x) & (uint8_t)0x80) #define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x) && ((uint8_t)(x) & (uint8_t)0x40)) @@ -15,16 +17,67 @@ // return an always null terminated wide string, truncate to given size if destination is not big enough, // src_len can be -1 use all of it. // returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated). -size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len); +size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len, bool *truncated); // always null terminated, truncated if it does not fit, src_len can be -1 to use all of it. // returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated). -#define utf8_to_utf16(utf16, utf16_count, src, src_len) any_to_utf16(CP_UTF8, utf16, utf16_count, src, src_len) +#define utf8_to_utf16(utf16, utf16_count, src, src_len) any_to_utf16(CP_UTF8, utf16, utf16_count, src, src_len, NULL) // always null terminated, truncated if it does not fit, src_len can be -1 to use all of it. // returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated). -size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len); +size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len, bool *truncated); -#endif +// -------------------------------------------------------------------------------------------------------------------- +// TXT_UTF8 + +typedef enum __attribute__((packed)) { + TXT_SOURCE_UNKNOWN = 0, + TXT_SOURCE_PROVIDER, + TXT_SOURCE_FIELD_CACHE, + TXT_SOURCE_EVENT_LOG, + TXT_SOURCE_HARDCODED, + + // terminator + TXT_SOURCE_MAX, +} TXT_SOURCE; + +typedef struct { + char *data; + uint32_t size; // the allocated size of data buffer + uint32_t used; // the used size of the data buffer (including null terminators, if any) + TXT_SOURCE src; +} TXT_UTF8; + +void txt_utf8_append(TXT_UTF8 *dst, const char *txt, size_t txt_len); +void txt_utf8_set(TXT_UTF8 *dst, const char *txt, size_t txt_len); +void txt_utf8_empty(TXT_UTF8 *dst); +void txt_utf8_resize(TXT_UTF8 *dst, size_t required_size, bool keep); +void txt_utf8_cleanup(TXT_UTF8 *dst); + +// -------------------------------------------------------------------------------------------------------------------- +// TXT_UTF16 + +typedef struct { + wchar_t *data; + uint32_t size; // the allocated size of data buffer + uint32_t used; // the used size of the data buffer (including null terminators, if any) +} TXT_UTF16; + +void txt_utf16_cleanup(TXT_UTF16 *dst); +void txt_utf16_resize(TXT_UTF16 *dst, size_t required_size, bool keep); +void txt_utf16_set(TXT_UTF16 *dst, const wchar_t *txt, size_t txt_len); +void txt_utf16_append(TXT_UTF16 *dst, const wchar_t *txt, size_t txt_len); + +// -------------------------------------------------------------------------------------------------------------------- + +size_t txt_compute_new_size(size_t old_size, size_t required_size); + +bool txt_utf16_to_utf8(TXT_UTF8 *utf8, TXT_UTF16 *utf16); +bool wchar_to_txt_utf8(TXT_UTF8 *dst, const wchar_t *src, int src_len); +char *utf16_to_utf8_strdupz(const wchar_t *src, size_t *dst_len); + +// -------------------------------------------------------------------------------------------------------------------- + +#endif // OS_WINDOWS #endif /* NETDATA_STRING_UTF8_H */