Memory: Swap probe_read to kernel or user version

We should always use the probe_read_kernel or probe_read_user helpers
over the probe_read helper (ditto for _str versions).

This commit changes all probe_read to either probe_read_kernel or
probe_read_user (ditto for _str versions).

Signed-off-by: Kevin Sheldrake <[email protected]>

bpf/Makefile

@@ -25,6 +25,11 @@ PROCESS = bpf_execve_event.o bpf_execve_event_v53.o bpf_fork.o bpf_exit.o bpf_ge
 	  bpf_generic_tracepoint_v511.o \
 	  bpf_multi_kprobe_v511.o bpf_multi_retkprobe_v511.o \
 	  bpf_generic_uprobe_v511.o \
+	  bpf_execve_event_v54.o \
+	  bpf_generic_kprobe_v54.o bpf_generic_retkprobe_v54.o \
+	  bpf_generic_tracepoint_v54.o \
+	  bpf_multi_kprobe_v54.o bpf_multi_retkprobe_v54.o \
+	  bpf_generic_uprobe_v54.o \
 	  bpf_loader.o \
 	  bpf_enforcer.o bpf_multi_enforcer.o bpf_fmodret_enforcer.o
 
@@ -71,6 +76,7 @@ endef
 # Generic build targets for each sub-dir
 
 $(eval $(call DEFINE_VARIANT,v53))
+$(eval $(call DEFINE_VARIANT,v54))
 $(eval $(call DEFINE_VARIANT,v511))
 $(eval $(call DEFINE_VARIANT,v61))
 
@@ -123,22 +129,31 @@ $(DEPSDIR)%_v53.d:
 	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -MM -MP -MT $(patsubst $(DEPSDIR)%.d, $(OBJSDIR)%.ll, $@)   $< > $@
 
 objs/bpf_multi_kprobe_v61.ll objs/bpf_multi_retkprobe_v61.ll:
-	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__LARGE_MAP_KEYS -D__V61_BPF_PROG -D__MULTI_KPROBE -c $< -o $@
+	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__PROBE_KERNEL -D__LARGE_MAP_KEYS -D__V61_BPF_PROG -D__MULTI_KPROBE -c $< -o $@
 
 objs/%_v61.ll:
-	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__LARGE_MAP_KEYS -D__V61_BPF_PROG -c $< -o $@
+	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__PROBE_KERNEL -D__LARGE_MAP_KEYS -D__V61_BPF_PROG -c $< -o $@
 
 $(DEPSDIR)%_v61.d:
-	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__LARGE_MAP_KEYS -D__V61_BPF_PROG -MM -MP -MT $(patsubst $(DEPSDIR)%.d, $(OBJSDIR)%.ll, $@)   $< > $@
+	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__PROBE_KERNEL -D__LARGE_MAP_KEYS -D__V61_BPF_PROG -MM -MP -MT $(patsubst $(DEPSDIR)%.d, $(OBJSDIR)%.ll, $@)   $< > $@
 
 objs/bpf_multi_kprobe_v511.ll objs/bpf_multi_retkprobe_v511.ll:
-	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__LARGE_MAP_KEYS -D__MULTI_KPROBE -c $< -o $@
+	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__PROBE_KERNEL -D__LARGE_MAP_KEYS -D__MULTI_KPROBE -c $< -o $@
 
 objs/%_v511.ll:
-	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__LARGE_MAP_KEYS -c $< -o $@
+	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__PROBE_KERNEL -D__LARGE_MAP_KEYS -c $< -o $@
 
 $(DEPSDIR)%_v511.d:
-	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__LARGE_MAP_KEYS -MM -MP -MT $(patsubst $(DEPSDIR)%.d, $(OBJSDIR)%.ll, $@)   $< > $@
+	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__PROBE_KERNEL -D__LARGE_MAP_KEYS -MM -MP -MT $(patsubst $(DEPSDIR)%.d, $(OBJSDIR)%.ll, $@)   $< > $@
+
+objs/bpf_multi_kprobe_v54.ll objs/bpf_multi_retkprobe_v54.ll:
+	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__PROBE_KERNEL -D__MULTI_KPROBE -c $< -o $@
+
+objs/%_v54.ll:
+	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__PROBE_KERNEL -c $< -o $@
+
+$(DEPSDIR)%_v54.d:
+	$(CLANG) $(CLANG_FLAGS) -D__LARGE_BPF_PROG -D__PROBE_KERNEL -MM -MP -MT $(patsubst $(DEPSDIR)%.d, $(OBJSDIR)%.ll, $@)   $< > $@
 
 # BPFTESTDIR
 objs/%.ll: $(BPFTESTDIR)%.c

bpf/cgroup/bpf_cgroup_events.h

@@ -49,7 +49,7 @@ send_cgrp_event(struct bpf_raw_tracepoint_args *ctx,
 	msg->cgrp_data.level = cgrp_track->level;
 	msg->cgrp_data.hierarchy_id = cgrp_track->hierarchy_id;
 	memcpy(&msg->cgrp_data.name, &cgrp_track->name, KN_NAME_LENGTH);
-	probe_read_str(&msg->path, PATH_MAP_SIZE - 1, path);
+	probe_read_kernel_str(&msg->path, PATH_MAP_SIZE - 1, path);
 
 	perf_event_output_metric(ctx, MSG_OP_CGROUP, &tcpmon_map, BPF_F_CURRENT_CPU, msg, size);
 

bpf/include/api.h

@@ -204,6 +204,9 @@ static int BPF_FUNC(fib_lookup, void *ctx, struct bpf_fib_lookup *params, uint32
 static int BPF_FUNC(probe_read, void *dst, uint32_t size, const void *src);
 static int BPF_FUNC(probe_read_str, void *dst, int size, const void *src);
 static int BPF_FUNC(probe_read_kernel, void *dst, uint32_t size, const void *src);
+static int BPF_FUNC(probe_read_kernel_str, void *dst, int size, const void *src);
+static int BPF_FUNC(probe_read_user, void *dst, uint32_t size, const void *src);
+static int BPF_FUNC(probe_read_user_str, void *dst, int size, const void *src);
 
 static uint64_t BPF_FUNC(get_current_task);
 

bpf/lib/bpf_cgroup.h

@@ -9,6 +9,8 @@
 #include "environ_conf.h"
 #include "common.h"
 #include "process.h"
+#include "../process/types/probe_read_kernel_or_user.h"
+#include "bpf_tracing.h"
 
 #define NULL ((void *)0)
 
@@ -109,7 +111,7 @@ __get_cgroup_kn_name(const struct kernfs_node *kn)
 	const char *name = NULL;
 
 	if (kn)
-		probe_read(&name, sizeof(name), _(&kn->name));
+		probe_read_kernel(&name, sizeof(name), _(&kn->name));
 
 	return name;
 }
@@ -139,7 +141,7 @@ __get_cgroup_kn_id(const struct kernfs_node *kn)
 		if (BPF_CORE_READ_INTO(&id, old_kn, id.id) != 0)
 			return 0;
 	} else {
-		probe_read(&id, sizeof(id), _(&kn->id));
+		probe_read_kernel(&id, sizeof(id), _(&kn->id));
 	}
 
 	return id;
@@ -157,7 +159,7 @@ __get_cgroup_kn(const struct cgroup *cgrp)
 	struct kernfs_node *kn = NULL;
 
 	if (cgrp)
-		probe_read(&kn, sizeof(cgrp->kn), _(&cgrp->kn));
+		probe_read_kernel(&kn, sizeof(cgrp->kn), _(&cgrp->kn));
 
 	return kn;
 }
@@ -187,7 +189,7 @@ get_cgroup_hierarchy_id(const struct cgroup *cgrp)
  * @cgrp: target cgroup
  *
  * Returns a pointer to the cgroup node name on success that can
- * be read with probe_read(). NULL on failures.
+ * be read with probe_read_kernel(). NULL on failures.
  */
 static inline __attribute__((always_inline)) const char *
 get_cgroup_name(const struct cgroup *cgrp)
@@ -214,7 +216,7 @@ get_cgroup_level(const struct cgroup *cgrp)
 {
 	__u32 level = 0;
 
-	probe_read(&level, sizeof(level), _(&cgrp->level));
+	probe_read_kernel(&level, sizeof(level), _(&cgrp->level));
 	return level;
 }
 
@@ -264,7 +266,7 @@ get_task_cgroup(struct task_struct *task, __u32 subsys_idx, __u32 *error_flags)
 	struct css_set *cgroups;
 	struct cgroup *cgrp = NULL;
 
-	probe_read(&cgroups, sizeof(cgroups), _(&task->cgroups));
+	probe_read_kernel(&cgroups, sizeof(cgroups), _(&task->cgroups));
 	if (unlikely(!cgroups)) {
 		*error_flags |= EVENT_ERROR_CGROUPS;
 		return cgrp;
@@ -297,13 +299,13 @@ get_task_cgroup(struct task_struct *task, __u32 subsys_idx, __u32 *error_flags)
 	 * support as much as workload as possible. It also reduces errors
 	 * in a significant way.
 	 */
-	probe_read(&subsys, sizeof(subsys), _(&cgroups->subsys[subsys_idx]));
+	probe_read_kernel(&subsys, sizeof(subsys), _(&cgroups->subsys[subsys_idx]));
 	if (unlikely(!subsys)) {
 		*error_flags |= EVENT_ERROR_CGROUP_SUBSYS;
 		return cgrp;
 	}
 
-	probe_read(&cgrp, sizeof(cgrp), _(&subsys->cgroup));
+	probe_read_kernel(&cgrp, sizeof(cgrp), _(&subsys->cgroup));
 	if (!cgrp)
 		*error_flags |= EVENT_ERROR_CGROUP_SUBSYSCGRP;
 
@@ -426,7 +428,7 @@ __init_cgrp_tracking_val_heap(struct cgroup *cgrp, cgroup_state state)
 	kn = __get_cgroup_kn(cgrp);
 	name = __get_cgroup_kn_name(kn);
 	if (name)
-		probe_read_str(&heap->name, KN_NAME_LENGTH - 1, name);
+		probe_read_kernel_str(&heap->name, KN_NAME_LENGTH - 1, name);
 
 	return heap;
 }

bpf/lib/bpf_helpers.h

@@ -43,7 +43,7 @@
  * Following define is to assist VSCode Intellisense so that it treats
  * __builtin_preserve_access_index() as a const void * instead of a
  * simple void (because it doesn't have a definition for it). This stops
- * Intellisense marking all _(P) macros (used in probe_read()) as errors.
+ * Intellisense marking all _(P) macros (used in probe_read_kernel()) as errors.
  * To use this, just define VSCODE in 'C/C++: Edit Configurations (JSON)'
  * in the Command Palette in VSCODE (F1 or View->Command Palette...):
  *    "defines": ["VSCODE"]
@@ -54,15 +54,6 @@ const void *__builtin_preserve_access_index(void *);
 #endif
 #define _(P) (__builtin_preserve_access_index(P))
 
-/*
- * Convenience macro to check that field actually exists in target kernel's.
- * Returns:
- *    1, if matching field is present in target kernel;
- *    0, if no matching field found.
- */
-#define bpf_core_field_exists(field) \
-	__builtin_preserve_field_info(field, BPF_FIELD_EXISTS)
-
 /* second argument to __builtin_preserve_enum_value() built-in */
 enum bpf_enum_value_kind {
 	BPF_ENUMVAL_EXISTS = 0, /* enum value existence in kernel */

bpf/lib/bpf_task.h

@@ -7,6 +7,7 @@
 #include "bpf_event.h"
 #include "bpf_helpers.h"
 #include "generic.h"
+#include "bpf_tracing.h"
 
 /* __d_path_local flags */
 // #define UNRESOLVED_MOUNT_POINTS	   0x01 // (deprecated)
@@ -27,7 +28,7 @@ get_parent(struct task_struct *t)
 	struct task_struct *task;
 
 	/* Read the real parent */
-	probe_read(&task, sizeof(task), _(&t->real_parent));
+	probe_read_kernel(&task, sizeof(task), _(&t->real_parent));
 	if (!task)
 		return 0;
 	return task;
@@ -47,7 +48,7 @@ get_task_from_pid(__u32 pid)
 			i = TASK_PID_LOOP;
 			continue;
 		}
-		probe_read(&cpid, sizeof(cpid), _(&task->tgid));
+		probe_read_kernel(&cpid, sizeof(cpid), _(&task->tgid));
 		if (cpid == pid) {
 			i = TASK_PID_LOOP;
 			continue;
@@ -70,7 +71,7 @@ static inline __attribute__((always_inline)) __u32 get_task_pid_vnr(void)
 
 	thread_pid_exists = bpf_core_field_exists(task->thread_pid);
 	if (thread_pid_exists) {
-		probe_read(&pid, sizeof(pid), _(&task->thread_pid));
+		probe_read_kernel(&pid, sizeof(pid), _(&task->thread_pid));
 		if (!pid)
 			return 0;
 	} else {
@@ -85,16 +86,16 @@ static inline __attribute__((always_inline)) __u32 get_task_pid_vnr(void)
 		if (!thread_pid_exists)
 			link_sz =
 				24; // voodoo magic, hard-code 24 to init stack
-		probe_read(&link, link_sz,
-			   (void *)_(&task->pids) + (PIDTYPE_PID * link_sz));
+		probe_read_kernel(&link, link_sz,
+				  (void *)_(&task->pids) + (PIDTYPE_PID * link_sz));
 		pid = link.pid;
 	}
 	upid_sz = bpf_core_field_size(pid->numbers[0]);
-	probe_read(&level, sizeof(level), _(&pid->level));
+	probe_read_kernel(&level, sizeof(level), _(&pid->level));
 	if (level < 1)
 		return 0;
-	probe_read(&upid, upid_sz,
-		   (void *)_(&pid->numbers) + (level * upid_sz));
+	probe_read_kernel(&upid, upid_sz,
+			  (void *)_(&pid->numbers) + (level * upid_sz));
 	return upid.nr;
 }
 
@@ -106,7 +107,7 @@ event_find_parent_pid(struct task_struct *t)
 
 	if (!task)
 		return 0;
-	probe_read(&pid, sizeof(pid), _(&task->tgid));
+	probe_read_kernel(&pid, sizeof(pid), _(&task->tgid));
 	return pid;
 }
 
@@ -119,10 +120,10 @@ __event_find_parent(struct task_struct *task)
 
 #pragma unroll
 	for (i = 0; i < 4; i++) {
-		probe_read(&task, sizeof(task), _(&task->real_parent));
+		probe_read_kernel(&task, sizeof(task), _(&task->real_parent));
 		if (!task)
 			break;
-		probe_read(&pid, sizeof(pid), _(&task->tgid));
+		probe_read_kernel(&pid, sizeof(pid), _(&task->tgid));
 		value = execve_map_get_noinit(pid);
 		if (value && value->key.ktime != 0)
 			return value;
@@ -164,13 +165,13 @@ event_find_curr(__u32 *ppid, bool *walked)
 
 #pragma unroll
 	for (i = 0; i < 4; i++) {
-		probe_read(&pid, sizeof(pid), _(&task->tgid));
+		probe_read_kernel(&pid, sizeof(pid), _(&task->tgid));
 		value = execve_map_get_noinit(pid);
 		if (value && value->key.ktime != 0)
 			break;
 		value = 0;
 		*walked = 1;
-		probe_read(&task, sizeof(task), _(&task->real_parent));
+		probe_read_kernel(&task, sizeof(task), _(&task->real_parent));
 		if (!task)
 			break;
 	}

bpf/lib/generic.h

@@ -48,6 +48,7 @@ struct msg_generic_kprobe {
 	/* anything above is shared with the userspace so it should match structs MsgGenericKprobe and MsgGenericTracepoint in Go */
 	char args[24000];
 	unsigned long a0, a1, a2, a3, a4;
+	unsigned long ret;
 	long argsoff[MAX_POSSIBLE_ARGS];
 	struct msg_selector_data sel;
 	__u32 idx; // attach cookie index

bpf/lib/process.h

@@ -51,7 +51,7 @@
  * Now we want to read this with call 45 aka probe_read_str as follows,
  * where 'kernel_struct_arg' is the kernel data struct we are reading.
  *
- *   probe_read_str(args[offset], size, kernel_struct_arg)
+ *   probe_read_kernel_str(args[offset], size, kernel_struct_arg)
  *
  * But we have a bit of a problem determining if 'size' is out of array
  * range. The math would be,