update(falco_metrics): rearrange evts and drops prometheus metrics

falco.yaml

@@ -1066,6 +1066,9 @@ syscall_event_drops:
 # counters reflect monotonic values since Falco's start and are exported at a
 # constant stats interval.
 #
+# `kernel_event_counters_per_cpu_enabled`: Detailed kernel event and drop counters
+# per CPU. typically used when debugging and not in production.
+#
 # `libbpf_stats_enabled`: Exposes statistics similar to `bpftool prog show`,
 # providing information such as the number of invocations of each BPF program
 # attached by Falco and the time spent in each program measured in nanoseconds.
@@ -1104,6 +1107,8 @@ metrics:
   resource_utilization_enabled: true
   state_counters_enabled: true
   kernel_event_counters_enabled: true
+  # Enabling `kernel_event_counters_per_cpu_enabled` automatically enables `kernel_event_counters_enabled`
+  kernel_event_counters_per_cpu_enabled: false
   libbpf_stats_enabled: true
   plugins_metrics_enabled: true
   convert_memory_to_mb: true

unit_tests/engine/test_falco_utils.cpp

@@ -73,12 +73,12 @@ TEST(FalcoUtils, parse_prometheus_interval)
 	ASSERT_EQ(falco::utils::parse_prometheus_interval("200"), 0UL);
 }
 
-TEST(FalcoUtils, sanitize_metric_name)
+TEST(FalcoUtils, sanitize_rule_name)
 {
-	ASSERT_EQ(falco::utils::sanitize_metric_name("Testing rule   2 (CVE-2244)"), "Testing_rule_2_CVE_2244");
-	ASSERT_EQ(falco::utils::sanitize_metric_name("Testing rule__:2)"), "Testing_rule_:2");
-	ASSERT_EQ(falco::utils::sanitize_metric_name("This@is_a$test rule123"), "This_is_a_test_rule123");
-	ASSERT_EQ(falco::utils::sanitize_metric_name("RULEwith:special#characters"), "RULEwith:special_characters");
+	ASSERT_EQ(falco::utils::sanitize_rule_name("Testing rule   2 (CVE-2244)"), "Testing_rule_2_CVE_2244");
+	ASSERT_EQ(falco::utils::sanitize_rule_name("Testing rule__:2)"), "Testing_rule_:2");
+	ASSERT_EQ(falco::utils::sanitize_rule_name("This@is_a$test rule123"), "This_is_a_test_rule123");
+	ASSERT_EQ(falco::utils::sanitize_rule_name("RULEwith:special#characters"), "RULEwith:special_characters");
 }
 
 TEST(FalcoUtils, matches_wildcard)

userspace/engine/falco_utils.cpp

@@ -151,7 +151,7 @@ std::string calculate_file_sha256sum(const std::string& filename)
 }
 #endif
 
-std::string sanitize_metric_name(const std::string& name)
+std::string sanitize_rule_name(const std::string& name)
 {
 	std::string sanitized_name = name;
 	RE2::GlobalReplace(&sanitized_name, "[^a-zA-Z0-9_:]", "_");

userspace/engine/falco_utils.h

@@ -31,7 +31,7 @@ uint64_t parse_prometheus_interval(std::string interval_str);
 std::string calculate_file_sha256sum(const std::string& filename);
 #endif
 
-std::string sanitize_metric_name(const std::string& name);
+std::string sanitize_rule_name(const std::string& name);
 
 std::string wrap_text(const std::string& in, uint32_t indent, uint32_t linelen);
 

userspace/falco/configuration.cpp

@@ -581,6 +581,10 @@ void falco_configuration::load_yaml(const std::string& config_name)
 	{
 		m_metrics_flags |= METRICS_V2_KERNEL_COUNTERS;
 	}
+	if (m_config.get_scalar<bool>("metrics.kernel_event_counters_per_cpu_enabled", true))
+	{
+		m_metrics_flags |= METRICS_V2_KERNEL_COUNTERS_PER_CPU;
+	}
 	if (m_config.get_scalar<bool>("metrics.libbpf_stats_enabled", true))
 	{
 		m_metrics_flags |= METRICS_V2_LIBBPF_STATS;

userspace/falco/falco_metrics.cpp

@@ -15,6 +15,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+#include <re2/re2.h>
+
 #include "falco_metrics.h"
 
 #include "falco_utils.h"
@@ -100,13 +102,13 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		for (const auto& item : state.config.get()->m_loaded_rules_filenames_sha256sum)
 		{
 			fs::path fs_path = item.first;
-			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("falco_sha256_rules_files", "falcosecurity", "falco", {{"file_name", fs_path.filename().stem()}, {"sha256", item.second}});
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("sha256_rules_files", "falcosecurity", "falco", {{"file_name", fs_path.filename().stem()}, {"sha256", item.second}});
 		}
 
 		for (const auto& item : state.config.get()->m_loaded_configs_filenames_sha256sum)
 		{
 			fs::path fs_path = item.first;
-			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("falco_sha256_config_files", "falcosecurity", "falco", {{"file_name", fs_path.filename().stem()}, {"sha256", item.second}});
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("sha256_config_files", "falcosecurity", "falco", {{"file_name", fs_path.filename().stem()}, {"sha256", item.second}});
 		}
 
 		static std::string ifinfo_json_escaped;
@@ -213,6 +215,14 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 				auto count = rules_by_id[i]->load();
 				if (count > 0)
 				{
+					/* Examples ...
+						# HELP falcosecurity_falco_rules_counters_total https://falco.org/docs/metrics/
+						# TYPE falcosecurity_falco_rules_counters_total counter
+						falcosecurity_falco_rules_counters_total{priority="4",rule_name="Read sensitive file untrusted",source="syscall",tags="T1555, container, filesystem, host, maturity_stable, mitre_credential_access"} 10
+						# HELP falcosecurity_falco_rules_counters_total https://falco.org/docs/metrics/
+						# TYPE falcosecurity_falco_rules_counters_total counter
+						falcosecurity_falco_rules_counters_total{priority="5",rule_name="Unexpected UDP Traffic",source="syscall",tags="TA0011, container, host, maturity_incubating, mitre_exfiltration, network"} 1
+					*/
 					auto metric = libs::metrics::libsinsp_metrics::new_metric("rules_counters",
 											METRICS_V2_RULE_COUNTERS,
 											METRIC_VALUE_TYPE_U64,
@@ -246,21 +256,87 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		for (auto& metric: metrics_snapshot)
 		{
 			prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
-			std::string namespace_name = "scap";
+			std::string prometheus_subsystem = "scap";
 
-			if (metric.flags & METRICS_V2_RESOURCE_UTILIZATION || metric.flags & METRICS_V2_KERNEL_COUNTERS)
+			if (metric.flags & METRICS_V2_RESOURCE_UTILIZATION)
 			{
-				namespace_name = "falco";
+				prometheus_subsystem = "falco";
 			}
 
 			if (metric.flags & METRICS_V2_PLUGINS)
 			{
-				namespace_name = "plugins";
+				prometheus_subsystem = "plugins";
 			}
 
-			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", namespace_name);
+			// raw incoming in form of for example n_evts_cpu_15 or n_drops_cpu_15
+			if (strncmp(metric.name, "n_evts_cpu", 10) == 0 || strncmp(metric.name, "n_drops_cpu", 11) == 0) // prefix match
+			{
+				std::string name_str(metric.name);
+				re2::RE2 pattern("(\\d+)");
+				std::string cpu_number;
+				if (re2::RE2::PartialMatch(name_str, pattern, &cpu_number))
+				{
+					re2::RE2::GlobalReplace(&name_str, pattern, "");
+					// possible double __ will be sanitized within libs
+					auto metric_new = libs::metrics::libsinsp_metrics::new_metric(name_str.c_str(),
+								METRICS_V2_KERNEL_COUNTERS_PER_CPU,
+								METRIC_VALUE_TYPE_U64,
+								METRIC_VALUE_UNIT_COUNT,
+								METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+								metric.value.u64);
+					const std::map<std::string, std::string>& const_labels = {
+						{"cpu", cpu_number}
+					};
+					/* Examples ...
+						# HELP falcosecurity_scap_n_evts_cpu_total https://falco.org/docs/metrics/
+						# TYPE falcosecurity_scap_n_evts_cpu_total counter
+						falcosecurity_scap_n_evts_cpu_total{cpu="7"} 237
+						# HELP falcosecurity_scap_n_drops_cpu_total https://falco.org/docs/metrics/
+						# TYPE falcosecurity_scap_n_drops_cpu_total counter
+						falcosecurity_scap_n_drops_cpu_total{cpu="7"} 0
+					*/
+					prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric_new, "falcosecurity", prometheus_subsystem, const_labels);
+				}
+			}
+			else if (strcmp(metric.name, "n_drops_buffer_total") == 0)
+			{
+				// Skip the libs aggregate metric since we distinguish between buffer drops using labels similar to the rules_counters
+				continue;
+			}
+			else if (strncmp(metric.name, "n_drops_buffer", 14) == 0) // prefix match
+			{
+				re2::RE2 pattern("n_drops_buffer_([^_]+(?:_[^_]+)*)_(enter|exit)$");
+				std::string drop;
+				std::string dir;
+				std::string name_str(metric.name);
+				if (re2::RE2::FullMatch(name_str, pattern, &drop, &dir))
+				{
+					auto metric_new = libs::metrics::libsinsp_metrics::new_metric("n_drops_buffer",
+								METRICS_V2_KERNEL_COUNTERS,
+								METRIC_VALUE_TYPE_U64,
+								METRIC_VALUE_UNIT_COUNT,
+								METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+								metric.value.u64);
+					const std::map<std::string, std::string>& const_labels = {
+						{"drop", drop},
+						{"dir", dir}
+					};
+					/* Examples ...
+						# HELP falcosecurity_scap_n_drops_buffer_total https://falco.org/docs/metrics/
+						# TYPE falcosecurity_scap_n_drops_buffer_total counter
+						falcosecurity_scap_n_drops_buffer_total{dir="enter",drop="clone_fork"} 0
+						# HELP falcosecurity_scap_n_drops_buffer_total https://falco.org/docs/metrics/
+						# TYPE falcosecurity_scap_n_drops_buffer_total counter
+						falcosecurity_scap_n_drops_buffer_total{dir="exit",drop="clone_fork"} 0
+					*/
+					prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric_new, "falcosecurity", prometheus_subsystem, const_labels);
+				}
+			}
+			else
+			{
+				prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", prometheus_subsystem);
+			}
 		}
-
 	}
 	return prometheus_text;
 }

userspace/falco/stats_writer.cpp

@@ -346,15 +346,15 @@ void stats_writer::collector::get_metrics_output_fields_wrapper(
 	{
 		fs::path fs_path = item.first;
 		std::string metric_name_file_sha256 = fs_path.filename().stem();
-		metric_name_file_sha256 = "falco.sha256_rules_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
+		metric_name_file_sha256 = "falco.sha256_rules_file." + falco::utils::sanitize_rule_name(metric_name_file_sha256);
 		output_fields[metric_name_file_sha256] = item.second;
 	}
 
 	for (const auto& item : m_writer->m_config->m_loaded_configs_filenames_sha256sum)
 	{
 		fs::path fs_path = item.first;
 		std::string metric_name_file_sha256 = fs_path.filename().stem();
-		metric_name_file_sha256 = "falco.sha256_config_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
+		metric_name_file_sha256 = "falco.sha256_config_file." + falco::utils::sanitize_rule_name(metric_name_file_sha256);
 		output_fields[metric_name_file_sha256] = item.second;
 	}
 
@@ -434,7 +434,7 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 				continue;
 			}
 			auto rule = rules.at(i);
-			std::string rules_metric_name = "falco.rules." + falco::utils::sanitize_metric_name(rule->name);
+			std::string rules_metric_name = "falco.rules." + falco::utils::sanitize_rule_name(rule->name);
 			output_fields[rules_metric_name] = rule_count;
 		}
 	}
@@ -470,7 +470,7 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 				m_writer->m_output_rule_metrics_converter->convert_metric_to_unit_convention(metric);
 			}
 			char metric_name[METRIC_NAME_MAX] = "falco.";
-			if((metric.flags & METRICS_V2_LIBBPF_STATS) || (metric.flags & METRICS_V2_KERNEL_COUNTERS) )
+			if((metric.flags & METRICS_V2_LIBBPF_STATS) || (metric.flags & METRICS_V2_KERNEL_COUNTERS) || (metric.flags & METRICS_V2_KERNEL_COUNTERS_PER_CPU) )
 			{
 				strlcpy(metric_name, "scap.", sizeof(metric_name));
 			}
@@ -600,7 +600,7 @@ void stats_writer::collector::collect(const std::shared_ptr<sinsp>& inspector, c
 			// Note: src is static for live captures
 			if (src != falco_common::syscall_source)
 			{
-				flags &= ~(METRICS_V2_KERNEL_COUNTERS | METRICS_V2_STATE_COUNTERS | METRICS_V2_LIBBPF_STATS);
+				flags &= ~(METRICS_V2_KERNEL_COUNTERS | METRICS_V2_KERNEL_COUNTERS_PER_CPU | METRICS_V2_STATE_COUNTERS | METRICS_V2_LIBBPF_STATS);
 
 			}
 			m_writer->m_libs_metrics_collector = std::make_unique<libs::metrics::libs_metrics_collector>(inspector.get(), flags);