Only process PCAPs with httpreplay when tlsdump.log exists (#2315)

These changes prevent PCAPs from being processed by httpreplay (Pcap2 processing module) when there are not TLS keys available. This is because httpreplay processing is very time intensive due to its pure-python implementation.

Httpreplay's core use is to decrypt TLS traffic so it can be processed by Suricata. If there are no TLS keys available, there is no requirement to use it for processing.

For context, when CAPE attempted to process a ~250MB PCAP with httpreplay, it took ~960 seconds. Without httpreplay, it took ~16 seconds.

modules/processing/network.py

@@ -38,6 +38,7 @@
 from lib.cuckoo.common.path_utils import path_delete, path_exists, path_mkdir, path_read_file, path_write_file
 from lib.cuckoo.common.safelist import is_safelisted_domain
 from lib.cuckoo.common.utils import convert_to_printable
+import utils.profiling as profiling
 
 # from lib.cuckoo.common.safelist import is_safelisted_ip
 
@@ -78,6 +79,7 @@
 sys.path.append(CUCKOO_ROOT)
 
 TLS_HANDSHAKE = 22
+PCAP_BYTES_HTTPREPLAY_WARN_LIMIT = 30*1024*1024
 
 Keyed = namedtuple("Keyed", ["key", "obj"])
 Packet = namedtuple("Packet", ["raw", "ts"])
@@ -922,6 +924,11 @@ def run(self):
             log.debug('The PCAP file does not exist at path "%s"', self.pcap_path)
             return {}
 
+        httpreplay_start = profiling.Counter()
+        log.info("starting processing pcap with httpreplay")
+        if os.path.getsize(self.pcap_path) > PCAP_BYTES_HTTPREPLAY_WARN_LIMIT:
+            log.warning("httpreplay processing may timeout due to pcap size")
+
         r = httpreplay.reader.PcapReader(open(self.pcap_path, "rb"))
         r.tcp = httpreplay.smegma.TCPPacketStreamer(r, self.handlers)
 
@@ -1060,6 +1067,8 @@ def run(self):
 
                 results[f"{protocol}_ex"].append(tmp_dict)
 
+        log.info("finished processing pcap with httpreplay")
+        log.debug("httpreplay processing time: %s", (profiling.Counter() - httpreplay_start))
         return results
 
 
@@ -1117,7 +1126,10 @@ def run(self):
 
         if HAVE_HTTPREPLAY:
             try:
-                p2 = Pcap2(self.pcap_path, self.get_tlsmaster(), self.network_path).run()
+                p2 = {}
+                tls_master = self.get_tlsmaster()
+                if tls_master:
+                    p2 = Pcap2(self.pcap_path, tls_master, self.network_path).run()
                 if p2:
                     results.update(p2)
             except Exception:

utils/profiling.py

@@ -0,0 +1,29 @@
+import time
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Counter:
+    """Profiler that counts real and CPU time."""
+    real: float = field(default_factory=time.perf_counter)
+    cpu: float = field(default_factory=time.process_time)
+
+    def __sub__(self, other):
+        real = self.real - other.real
+        cpu = self.cpu - other.cpu
+        return Counter(real, cpu)
+
+    def __add__(self, other):
+        real = self.real + other.real
+        cpu = self.cpu + other.cpu
+        return Counter(real, cpu)
+
+    def __str__(self) -> str:
+        return f"{self.real:.2f}s (cpu {self.cpu:.2f}s)".format(self.real, self.cpu)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        elapsed = (Counter() - self)
+        self.__dict__.update(**elapsed.__dict__)