chg: [export] Export script added

.gitignore

@@ -1,4 +1,6 @@
 .idea/
 ipa.egg-info/
 .venv
-db/
+exports/
+db/
+__pycache__/

bin/export.py

@@ -21,21 +21,40 @@
 import markdown_strings as mds
 import redis
 import os
-
+import time
 
 analyzer_redis_host = os.getenv('D4_ANALYZER_REDIS_HOST', '127.0.0.1')
 analyzer_redis_port = int(os.getenv('D4_ANALYZER_REDIS_PORT', 6405))
 r = redis.Redis(host=analyzer_redis_host, port=analyzer_redis_port)
 
+table_line = '| :--------------- | :--------------- |\n'
+padding = [16, 16]
+
+
+def init_export_dir(path: str):
+    if not os.path.exists(path):
+        os.mkdir(path)
+
 
 def export_icmp_types():
-    res = mds.table_row(['ICMP Type', 'Count'], [10, 10]) + '\n'
-    res += '| :----- | -----: |\n'
+    res = mds.table_row(['ICMP Type', 'Count'], padding) + '\n' + table_line
     redis_dict = r.hgetall('icmp')
     for key in redis_dict:
-        res += mds.table_row([key.decode(), redis_dict[key].decode()], [10, 10]) + '\n'
+        res += mds.table_row([key.decode(), redis_dict[key].decode()], padding) + '\n'
+    return res
+
+
+def export_protocols():
+    res = mds.table_row(['Protocol', 'Count'], padding) + '\n' + table_line
+    redis_list = r.zrange('protocols', 0, -1, withscores=True)
+    for item in redis_list:
+        res += mds.table_row([item[0].decode(), int(item[1])], padding) + '\n'
     return res
 
 
 if __name__ == "__main__":
-    export_icmp_types()
+    pwd = os.getcwd() + '/exports/'
+    init_export_dir(pwd)
+    with open(pwd + str(time.time())[:10] + '-export.md', 'w') as exp_file:
+        exp_file.write(export_icmp_types() + '\n')
+        exp_file.write(export_protocols() + '\n')

install.sh

@@ -7,6 +7,6 @@ export PIPENV_VENV_IN_PROJECT=1
 
 if [ -z "$VIRTUAL_ENV" ]; then
     pipenv install
-    export IPA_HOME=$(pwd)
+    echo export IPA_HOME=$(pwd) >> .venv/bin/activate
 fi
 

lib/analyzer.py

@@ -24,7 +24,7 @@
 import configparser
 import logging
 
-from lib.inspection import get_cap, get_protocol, check_icmp_checksum, get_icmp_payload, get_icmp_ip, \
+from lib.inspection import get_cap, get_raw_cap, get_protocol, check_icmp_checksum, get_icmp_payload, get_icmp_ip, \
     unassigned_icmp_types, deprecated_icmp_types, get_src_port, get_dst_port, list_caps, init_cap_list
 
 
@@ -69,7 +69,7 @@ def __init__(self, dataset_path: str=None):
             self.cap_list = []
             self.logger.info("Adding dataset caps to local queue")
             self.cap_list = init_cap_list(self.dataset)
-            self.logger.info(len(self.cap_list))
+            self.logger.info('Added ' + str(len(self.cap_list)) + ' caps.')
             self.update_queue()
             self.logger.info("Processing...")
             self.process_local()
@@ -79,7 +79,6 @@ def __init__(self, dataset_path: str=None):
             if c == 0:
                 self.enqueue_caps(cap_list=list_caps('scanning', self.r))
                 self.r.delete('scanning')
-                print('[-] Process remaining unfinished caps.')
                 self.process_local()
 
     def enqueue_caps(self, cap_list: list):
@@ -105,7 +104,6 @@ def update_queue(self):
             self.logger.info('Queue updated.')
         else:
             if self.cap_list:
-                self.logger.info('No caps enqueued, initializing...')
                 caps_to_add = self.cap_list
             elif current_caps:
                 return 0
@@ -117,7 +115,7 @@ def parse_cap(self, cap):
         Dissects the cap file to extract info.
         """
         if cap is None:
-            self.logger.info('[X] No caps to parse!')
+            self.logger.info('No caps to parse!')
             return 0
 
         self.logger.info('Parsing cap ' + cap.input_filename[-15:])
@@ -130,7 +128,7 @@ def parse_cap(self, cap):
             icmp_type = str(icmp_layer.type)
             # icmp_code = str(icmp_layer.code)
             protocol = get_protocol(packet)
-            checksum_status = check_icmp_checksum(packet.icmp_raw.value)
+            # checksum_status = check_icmp_checksum(packet.icmp_raw.value)
 
             if protocol == '1 : icmp':
                 payload = get_icmp_payload(packet)
@@ -149,8 +147,8 @@ def parse_cap(self, cap):
             else:
                 pipeline.hincrby('icmp', icmp_type)
 
-            pipeline.hincrby('checksum', 'total')
-            pipeline.hincrby('checksum', checksum_status)
+            # pipeline.hincrby('checksum', 'total')
+            # pipeline.hincrby('checksum', checksum_status)
 
             # entry = str(get_src_port(packet)) + ':' + protocol + ':' + icmp_type + ':' + icmp_code
             # pipeline.zadd(source_ip, {entry: 1}, incr=True)
@@ -172,11 +170,11 @@ def pop_cap(self):
             absolute_path = self.r_d4.rpop(self.queue).decode()
         else:
             absolute_path = self.r.lpop('to_scan').decode()
-        return get_cap(absolute_path)
+        return get_cap(absolute_path), get_raw_cap(absolute_path)
 
     def process_d4(self):
         while True:
-            d4_cap = self.pop_cap()
+            d4_cap, d4_raw_cap = self.pop_cap()
             if d4_cap is None:
                 time.sleep(1)
                 continue
@@ -186,7 +184,7 @@ def process_d4(self):
 
     def process_local(self):
         while self.r.llen(self.queue) != 0:
-            cap = self.pop_cap()
+            cap, raw_cap = self.pop_cap()
             self.r.rpush('scanning', cap.input_filename)
             self.parse_cap(cap)
             self.r.lrem('scanning', 0, cap.input_filename)

lib/inspection.py

@@ -120,10 +120,14 @@
     }
 
 
-def get_cap(path_to_cap):
+def get_raw_cap(path_to_cap: str):
     return FileCapture(input_file=path_to_cap, display_filter='icmp', use_json=True, include_raw=True)
 
 
+def get_cap(path_to_cap: str):
+    return FileCapture(input_file=path_to_cap, display_filter='icmp')
+
+
 def get_files(path) -> list:
     caps = glob(path)
     return caps
@@ -151,20 +155,18 @@ def list_caps(state: str, redis):
 def get_protocol(packet):
     if 'ip_proto' in packet.icmp.field_names:
         protocol = str(packet.icmp.ip_proto)
-        if int(protocol) in range(143, 253):
+        if protocol in unassigned_proto:
             return protocol + ' (unassigned)'
         ip_proto = proto_dict[protocol]
     else:
-        return 'non-backscatter-icmp'
+        return 'nbs-icmp'
     return protocol + ' : ' + str(ip_proto)
 
 
 def get_icmp_payload(packet):
     if 'data' in packet.icmp.field_names:
         return str(packet.icmp.data)
     elif packet.icmp.field_names != ['type', 'code', 'checksum', 'checksum_status', 'ident', 'seq', 'seq_le']:
-        print(packet.icmp.field_names)
-        print(packet.icmp)
         return 'No data'