Common O/S Caching Layer for users and groups (netdata#18825)

* added cached-uid-usernames to libnetdata

* added cached-gid-groupnames

* apps.plugin uses the common cache for users and groups

* apps.plugin cleanup

* network-viewer and systemd-journal now use host prefix to ingest usernames and groups

* fix compilation on windows

* move unicode functions from windows-events to libnetdata

* move sid-to-username from windows-events plugin to libnetdata

* common.h in libnetdata for operating system includes and macros

* sid uses STRINGs

* apps.plugin now exposes per user charts on windows

CMakeLists.txt

@@ -768,6 +768,14 @@ set(LIBNETDATA_FILES
         src/libnetdata/os/windows-perflib/perflib.h
         src/libnetdata/os/windows-perflib/perflib-names.c
         src/libnetdata/os/windows-perflib/perflib-dump.c
+        src/libnetdata/os/system-maps/cached-uid-username.c
+        src/libnetdata/os/system-maps/cached-uid-username.h
+        src/libnetdata/os/system-maps/cached-sid-username.c
+        src/libnetdata/os/system-maps/cached-sid-username.h
+        src/libnetdata/os/system-maps/cached-gid-groupname.c
+        src/libnetdata/os/system-maps/cached-gid-groupname.h
+        src/libnetdata/os/system-maps/cache-host-users-and-groups.c
+        src/libnetdata/os/system-maps/cache-host-users-and-groups.h
         src/libnetdata/spawn_server/spawn_server_nofork.c
         src/libnetdata/spawn_server/spawn_server.h
         src/libnetdata/spawn_server/spawn_popen.c
@@ -845,6 +853,7 @@ set(LIBNETDATA_FILES
         src/libnetdata/spawn_server/log-forwarder.h
         src/libnetdata/log/nd_log-common.h
         src/libnetdata/log/nd_log-to-windows-common.h
+        src/libnetdata/common.h
 )
 
 if(ENABLE_PLUGIN_EBPF)
@@ -1332,9 +1341,7 @@ set(SYSTEMD_JOURNAL_PLUGIN_FILES
         src/collectors/systemd-journal.plugin/systemd-journal-fstat.c
         src/collectors/systemd-journal.plugin/systemd-journal-watcher.c
         src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
-        src/libnetdata/maps/system-users.h
-        src/libnetdata/maps/system-groups.h
-        src/libnetdata/maps/system-services.h
+        src/libnetdata/os/system-maps/system-services.h
         src/collectors/systemd-journal.plugin/systemd-journal-sampling.h
 )
 
@@ -1464,8 +1471,6 @@ set(WINDOWS_EVENTS_PLUGIN_FILES
         src/collectors/windows-events.plugin/windows-events-query.c
         src/collectors/windows-events.plugin/windows-events-sources.c
         src/collectors/windows-events.plugin/windows-events-sources.h
-        src/collectors/windows-events.plugin/windows-events-sid.c
-        src/collectors/windows-events.plugin/windows-events-sid.h
         src/collectors/windows-events.plugin/windows-events-unicode.c
         src/collectors/windows-events.plugin/windows-events-unicode.h
         src/collectors/windows-events.plugin/windows-events-xml.c
@@ -2020,7 +2025,6 @@ if(ENABLE_PLUGIN_APPS)
             src/collectors/apps.plugin/apps_plugin.h
             src/collectors/apps.plugin/apps_functions.c
             src/collectors/apps.plugin/apps_targets.c
-            src/collectors/apps.plugin/apps_users_and_groups.c
             src/collectors/apps.plugin/apps_output.c
             src/collectors/apps.plugin/apps_pid_files.c
             src/collectors/apps.plugin/apps_pid.c
@@ -2356,7 +2360,7 @@ endif()
 if(ENABLE_PLUGIN_LOCAL_LISTENERS)
         set(LOCAL_LISTENERS_FILES
                 src/collectors/utils/local_listeners.c
-                src/libnetdata/maps/local-sockets.h
+                src/libnetdata/local-sockets/local-sockets.h
         )
 
         add_executable(local-listeners ${LOCAL_LISTENERS_FILES})
@@ -2375,8 +2379,7 @@ endif()
 
 if(ENABLE_PLUGIN_NETWORK_VIEWER)
         set(NETWORK_VIEWER_FILES
-                src/libnetdata/maps/local-sockets.h
-                src/libnetdata/maps/system-users.h
+                src/libnetdata/local-sockets/local-sockets.h
                 src/collectors/network-viewer.plugin/network-viewer.c
         )
 

src/collectors/apps.plugin/apps_aggregations.c

@@ -169,6 +169,9 @@ void aggregate_processes_to_targets(void) {
 #if (PROCESSES_HAVE_GID == 1)
     zero_all_targets(groups_root_target);
 #endif
+#if (PROCESSES_HAVE_SID == 1)
+    zero_all_targets(sids_root_target);
+#endif
 
     // this has to be done, before the cleanup
     struct target *w = NULL, *o = NULL;
@@ -187,6 +190,8 @@ void aggregate_processes_to_targets(void) {
         // user target
 
 #if (PROCESSES_HAVE_UID == 1)
+        update_cached_host_users();
+
         o = p->uid_target;
         if(likely(p->uid_target && p->uid_target->uid == p->uid))
             w = p->uid_target;
@@ -204,6 +209,8 @@ void aggregate_processes_to_targets(void) {
         // user group target
 
 #if (PROCESSES_HAVE_GID == 1)
+        update_cached_host_users();
+
         o = p->gid_target;
         if(likely(p->gid_target && p->gid_target->gid == p->gid))
             w = p->gid_target;
@@ -214,6 +221,19 @@ void aggregate_processes_to_targets(void) {
             w = p->gid_target = get_gid_target(p->gid);
         }
 
+        aggregate_pid_on_target(w, p, o);
+#endif
+
+        // --------------------------------------------------------------------
+        // sid target
+
+#if (PROCESSES_HAVE_SID == 1)
+        o = p->sid_target;
+        if(likely(p->sid_target && p->sid_target->sid_name == p->sid_name))
+            w = p->sid_target;
+        else
+            w = p->sid_target = get_sid_target(p->sid_name);
+
         aggregate_pid_on_target(w, p, o);
 #endif
 

src/collectors/apps.plugin/apps_functions.c

@@ -27,7 +27,7 @@ static void apps_plugin_function_processes_help(const char *transaction) {
                    "   parent:NAME\n"
                    "      Shows only processes that are aggregated under parent `NAME`\n"
                    "\n"
-#if (PROCESSES_HAVE_UID == 1)
+#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1)
                    "   user:NAME\n"
                    "      Shows only processes that are running as user name `NAME`.\n"
                    "\n"
@@ -89,6 +89,12 @@ void function_processes(const char *transaction, char *function,
     size_t num_words = quoted_strings_splitter_whitespace(function, words, PLUGINSD_MAX_WORDS);
 
     struct target *category = NULL, *user = NULL, *group = NULL; (void)category; (void)user; (void)group;
+#if (PROCESSES_HAVE_UID == 1)
+    struct target *users_sid_root = users_root_target;
+#endif
+#if (PROCESSES_HAVE_SID == 1)
+    struct target *users_sid_root = sids_root_target;
+#endif
     const char *process_name = NULL;
     pid_t pid = 0;
     uid_t uid = 0; (void)uid;
@@ -110,9 +116,9 @@ void function_processes(const char *transaction, char *function,
                 return;
             }
         }
-#if (PROCESSES_HAVE_UID == 1)
+#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1)
         else if(!user && strncmp(keyword, PROCESS_FILTER_USER, strlen(PROCESS_FILTER_USER)) == 0) {
-            user = find_target_by_name(users_root_target, &keyword[strlen(PROCESS_FILTER_USER)]);
+            user = find_target_by_name(users_sid_root, &keyword[strlen(PROCESS_FILTER_USER)]);
             if(!user) {
                 pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,
                                                        "No user with that name found.");
@@ -271,6 +277,11 @@ void function_processes(const char *transaction, char *function,
             continue;
 #endif
 
+#if (PROCESSES_HAVE_SID == 1)
+        if(user && p->sid_target != user)
+            continue;
+#endif
+
         if(process_name && ((strcmp(pid_stat_comm(p), process_name) != 0 && !p->parent) || (p->parent && strcmp(pid_stat_comm(p), process_name) != 0 && strcmp(pid_stat_comm(p->parent), process_name) != 0)))
             continue;
 
@@ -323,6 +334,10 @@ void function_processes(const char *transaction, char *function,
         // uid
         buffer_json_add_array_item_uint64(wb, p->uid);
 #endif
+#if (PROCESSES_HAVE_SID == 1)
+        // account
+        buffer_json_add_array_item_string(wb, p->sid_target ? string2str(p->sid_target->name) : "-");
+#endif
 
 #if (PROCESSES_HAVE_GID == 1)
         // group
@@ -493,12 +508,14 @@ void function_processes(const char *transaction, char *function,
                                     RRDF_FIELD_FILTER_MULTISELECT,
                                     RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_STICKY, NULL);
 
-#if (PROCESSES_HAVE_UID == 1)
+#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1)
         buffer_rrdf_table_add_field(wb, field_id++, "User", "User Owner", RRDF_FIELD_TYPE_STRING,
                                     RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,
                                     RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_COUNT,
                                     RRDF_FIELD_FILTER_MULTISELECT,
                                     RRDF_FIELD_OPTS_VISIBLE, NULL);
+#endif
+#if (PROCESSES_HAVE_UID == 1)
         buffer_rrdf_table_add_field(wb, field_id++, "Uid", "User ID", RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_VISUAL_VALUE,
                                     RRDF_FIELD_TRANSFORM_NUMBER, 0, NULL, NAN,
                                     RRDF_FIELD_SORT_ASCENDING, NULL, RRDF_FIELD_SUMMARY_COUNT,
@@ -1079,7 +1096,7 @@ void function_processes(const char *transaction, char *function,
         }
         buffer_json_object_close(wb);
 
-#if (PROCESSES_HAVE_UID == 1)
+#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1)
         // group by User
         buffer_json_member_add_object(wb, "User");
         {

src/collectors/apps.plugin/apps_os_windows.c

@@ -525,7 +525,7 @@ static char *ansi_to_utf8(LPCSTR str) {
     static __thread WCHAR unicode[PATH_MAX];
 
     // Step 1: Convert ANSI string (LPSTR) to wide string (UTF-16)
-    size_t count = any_to_utf16(CP_ACP, unicode, _countof(unicode), str, -1);
+    size_t count = any_to_utf16(CP_ACP, unicode, _countof(unicode), str, -1, NULL);
     if (!count) return NULL;
 
     return wchar_to_utf8(unicode);
@@ -664,6 +664,45 @@ static WCHAR *executable_path_from_cmdline(WCHAR *cmdline) {
     return NULL;
 }
 
+static BOOL GetProcessUserSID(HANDLE hProcess, PSID *ppSid) {
+    HANDLE hToken;
+    BOOL result = FALSE;
+    DWORD dwSize = 0;
+    PTOKEN_USER pTokenUser = NULL;
+
+    if (!OpenProcessToken(hProcess, TOKEN_QUERY, &hToken))
+        return FALSE;
+
+    GetTokenInformation(hToken, TokenUser, NULL, 0, &dwSize);
+    if (dwSize == 0) {
+        CloseHandle(hToken);
+        return FALSE;
+    }
+
+    pTokenUser = (PTOKEN_USER)LocalAlloc(LPTR, dwSize);
+    if (pTokenUser == NULL) {
+        CloseHandle(hToken);
+        return FALSE;
+    }
+
+    if (GetTokenInformation(hToken, TokenUser, pTokenUser, dwSize, &dwSize)) {
+        DWORD sidSize = GetLengthSid(pTokenUser->User.Sid);
+        *ppSid = (PSID)LocalAlloc(LPTR, sidSize);
+        if (*ppSid) {
+            if (CopySid(sidSize, *ppSid, pTokenUser->User.Sid)) {
+                result = TRUE;
+            } else {
+                LocalFree(*ppSid);
+                *ppSid = NULL;
+            }
+        }
+    }
+
+    LocalFree(pTokenUser);
+    CloseHandle(hToken);
+    return result;
+}
+
 void GetAllProcessesInfo(void) {
     static __thread wchar_t unicode[PATH_MAX];
     static __thread DWORD unicode_size = sizeof(unicode) / sizeof(*unicode);
@@ -723,6 +762,14 @@ void GetAllProcessesInfo(void) {
             }
         }
 
+        if(!p->sid_name) {
+            PSID pSid = NULL;
+            if (GetProcessUserSID(hProcess, &pSid))
+                p->sid_name = cached_sid_fullname_or_sid_str(pSid);
+            else
+                p->sid_name = string_strdupz("Unknown");
+        }
+
         CloseHandle(hProcess);
 
         char *comm = wchar_to_utf8(pe32.szExeFile);

src/collectors/apps.plugin/apps_pid.c

@@ -134,6 +134,10 @@ void del_pid_entry(pid_t pid) {
     freez(p->fds);
 #endif
 
+#if (PROCESSES_HAVE_SID == 1)
+    string_freez(p->sid_name);
+#endif
+
     string_freez(p->comm_orig);
     string_freez(p->comm);
     string_freez(p->cmdline);

src/collectors/apps.plugin/apps_plugin.c

@@ -74,10 +74,10 @@ uint32_t
 // the metrics. This results in utilization that exceeds the total utilization
 // of the system.
 //
-// During normalization, we align the per-process utilization, to the total of
-// the system. We first consume the exited children utilization and it the
-// collected values is above the total, we proportionally scale each reported
-// metric.
+// During normalization, we align the per-process utilization to the global
+// utilization of the system. We first consume the exited children utilization
+// and it the collected values is above the total, we proportionally scale each
+// reported metric.
 
 // the total system time, as reported by /proc/stat
 #if (ALL_PIDS_ARE_READ_INSTANTLY == 0)
@@ -211,9 +211,10 @@ void print_hierarchy(struct pid_stat *root) {
 static void normalize_utilization(struct target *root) {
     struct target *w;
 
-    // childs processing introduces spikes
-    // here we try to eliminate them by disabling childs processing either for specific dimensions
-    // or entirely. Of course, either way, we disable it just a single iteration.
+    // children processing introduces spikes,
+    // here we try to eliminate them by disabling children processing either
+    // for specific dimensions or entirely.
+    // of course, either way, we disable it just for a single iteration.
 
     kernel_uint_t max_time = os_get_system_cpus() * NSEC_PER_SEC;
     kernel_uint_t utime = 0, cutime = 0, stime = 0, cstime = 0, gtime = 0, cgtime = 0, minflt = 0, cminflt = 0, majflt = 0, cmajflt = 0;
@@ -249,7 +250,7 @@ static void normalize_utilization(struct target *root) {
             cgtime_fix_ratio = 1.0; //(NETDATA_DOUBLE)(global_utime + global_stime) / (NETDATA_DOUBLE)(utime + cutime + stime + cstime);
         }
         else if((global_utime + global_stime > utime + stime) && (cutime || cstime)) {
-            // children resources are too high
+            // children resources are too high,
             // lower only the children resources
             utime_fix_ratio  =
             stime_fix_ratio  =
@@ -481,7 +482,7 @@ static void parse_args(int argc, char **argv)
         }
 #endif
 
-#if (PROCESSES_HAVE_UID == 1)
+#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1)
         if(strcmp("no-users", argv[i]) == 0 || strcmp("without-users", argv[i]) == 0) {
             enable_users_charts = 0;
             continue;
@@ -544,7 +545,7 @@ static void parse_args(int argc, char **argv)
                     "                        (default is enabled)\n"
                     "\n"
 #endif
-#if (PROCESSES_HAVE_UID == 1)
+#if (PROCESSES_HAVE_UID == 1) || (PROCESSES_HAVE_SID == 1)
                     " without-users          disable reporting per user charts\n"
                     "\n"
 #endif
@@ -734,7 +735,18 @@ int main(int argc, char **argv) {
 
     netdata_log_info("started on pid %d", getpid());
 
-    apps_users_and_groups_init();
+#if (PROCESSES_HAVE_UID == 1)
+    cached_usernames_init();
+#endif
+
+#if (PROCESSES_HAVE_GID == 1)
+    cached_groupnames_init();
+#endif
+
+#if (PROCESSES_HAVE_SID == 1)
+    cached_sid_username_init();
+#endif
+
     apps_pids_init();
     OS_FUNCTION(apps_os_init)();
 
@@ -823,6 +835,13 @@ int main(int argc, char **argv) {
         }
 #endif
 
+#if (PROCESSES_HAVE_SID == 1)
+        if (enable_users_charts) {
+            send_charts_updates_to_netdata(sids_root_target, "user", "user", "User Processes");
+            send_collected_data_to_netdata(sids_root_target, "user", dt);
+        }
+#endif
+
         fflush(stdout);
 
         debug_log("done Loop No %zu", global_iterations_counter);