diff --git a/.gitignore b/.gitignore index 0b87200..e0887e3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .idea/ ipa.egg-info/ .venv -db/ \ No newline at end of file +exports/ +db/ +__pycache__/ \ No newline at end of file diff --git a/bin/export.py b/bin/export.py index b3fbb35..714e905 100644 --- a/bin/export.py +++ b/bin/export.py @@ -21,21 +21,40 @@ import markdown_strings as mds import redis import os - +import time analyzer_redis_host = os.getenv('D4_ANALYZER_REDIS_HOST', '127.0.0.1') analyzer_redis_port = int(os.getenv('D4_ANALYZER_REDIS_PORT', 6405)) r = redis.Redis(host=analyzer_redis_host, port=analyzer_redis_port) +table_line = '| :--------------- | :--------------- |\n' +padding = [16, 16] + + +def init_export_dir(path: str): + if not os.path.exists(path): + os.mkdir(path) + def export_icmp_types(): - res = mds.table_row(['ICMP Type', 'Count'], [10, 10]) + '\n' - res += '| :----- | -----: |\n' + res = mds.table_row(['ICMP Type', 'Count'], padding) + '\n' + table_line redis_dict = r.hgetall('icmp') for key in redis_dict: - res += mds.table_row([key.decode(), redis_dict[key].decode()], [10, 10]) + '\n' + res += mds.table_row([key.decode(), redis_dict[key].decode()], padding) + '\n' + return res + + +def export_protocols(): + res = mds.table_row(['Protocol', 'Count'], padding) + '\n' + table_line + redis_list = r.zrange('protocols', 0, -1, withscores=True) + for item in redis_list: + res += mds.table_row([item[0].decode(), int(item[1])], padding) + '\n' return res if __name__ == "__main__": - export_icmp_types() + pwd = os.getcwd() + '/exports/' + init_export_dir(pwd) + with open(pwd + str(time.time())[:10] + '-export.md', 'w') as exp_file: + exp_file.write(export_icmp_types() + '\n') + exp_file.write(export_protocols() + '\n') diff --git a/install.sh b/install.sh index e1df4aa..0359d33 100755 --- a/install.sh +++ b/install.sh @@ -7,6 +7,6 @@ export PIPENV_VENV_IN_PROJECT=1 if [ -z "$VIRTUAL_ENV" ]; then pipenv install - export IPA_HOME=$(pwd) + echo export IPA_HOME=$(pwd) >> .venv/bin/activate fi diff --git a/lib/__pycache__/inspection.cpython-36.pyc b/lib/__pycache__/inspection.cpython-36.pyc deleted file mode 100644 index 48257e8..0000000 Binary files a/lib/__pycache__/inspection.cpython-36.pyc and /dev/null differ diff --git a/lib/analyzer.py b/lib/analyzer.py index 9dd20b1..8d09076 100644 --- a/lib/analyzer.py +++ b/lib/analyzer.py @@ -24,7 +24,7 @@ import configparser import logging -from lib.inspection import get_cap, get_protocol, check_icmp_checksum, get_icmp_payload, get_icmp_ip, \ +from lib.inspection import get_cap, get_raw_cap, get_protocol, check_icmp_checksum, get_icmp_payload, get_icmp_ip, \ unassigned_icmp_types, deprecated_icmp_types, get_src_port, get_dst_port, list_caps, init_cap_list @@ -69,7 +69,7 @@ def __init__(self, dataset_path: str=None): self.cap_list = [] self.logger.info("Adding dataset caps to local queue") self.cap_list = init_cap_list(self.dataset) - self.logger.info(len(self.cap_list)) + self.logger.info('Added ' + str(len(self.cap_list)) + ' caps.') self.update_queue() self.logger.info("Processing...") self.process_local() @@ -79,7 +79,6 @@ def __init__(self, dataset_path: str=None): if c == 0: self.enqueue_caps(cap_list=list_caps('scanning', self.r)) self.r.delete('scanning') - print('[-] Process remaining unfinished caps.') self.process_local() def enqueue_caps(self, cap_list: list): @@ -105,7 +104,6 @@ def update_queue(self): self.logger.info('Queue updated.') else: if self.cap_list: - self.logger.info('No caps enqueued, initializing...') caps_to_add = self.cap_list elif current_caps: return 0 @@ -117,7 +115,7 @@ def parse_cap(self, cap): Dissects the cap file to extract info. """ if cap is None: - self.logger.info('[X] No caps to parse!') + self.logger.info('No caps to parse!') return 0 self.logger.info('Parsing cap ' + cap.input_filename[-15:]) @@ -130,7 +128,7 @@ def parse_cap(self, cap): icmp_type = str(icmp_layer.type) # icmp_code = str(icmp_layer.code) protocol = get_protocol(packet) - checksum_status = check_icmp_checksum(packet.icmp_raw.value) + # checksum_status = check_icmp_checksum(packet.icmp_raw.value) if protocol == '1 : icmp': payload = get_icmp_payload(packet) @@ -149,8 +147,8 @@ def parse_cap(self, cap): else: pipeline.hincrby('icmp', icmp_type) - pipeline.hincrby('checksum', 'total') - pipeline.hincrby('checksum', checksum_status) + # pipeline.hincrby('checksum', 'total') + # pipeline.hincrby('checksum', checksum_status) # entry = str(get_src_port(packet)) + ':' + protocol + ':' + icmp_type + ':' + icmp_code # pipeline.zadd(source_ip, {entry: 1}, incr=True) @@ -172,11 +170,11 @@ def pop_cap(self): absolute_path = self.r_d4.rpop(self.queue).decode() else: absolute_path = self.r.lpop('to_scan').decode() - return get_cap(absolute_path) + return get_cap(absolute_path), get_raw_cap(absolute_path) def process_d4(self): while True: - d4_cap = self.pop_cap() + d4_cap, d4_raw_cap = self.pop_cap() if d4_cap is None: time.sleep(1) continue @@ -186,7 +184,7 @@ def process_d4(self): def process_local(self): while self.r.llen(self.queue) != 0: - cap = self.pop_cap() + cap, raw_cap = self.pop_cap() self.r.rpush('scanning', cap.input_filename) self.parse_cap(cap) self.r.lrem('scanning', 0, cap.input_filename) diff --git a/lib/inspection.py b/lib/inspection.py index d7b4a47..cac07a1 100644 --- a/lib/inspection.py +++ b/lib/inspection.py @@ -120,10 +120,14 @@ } -def get_cap(path_to_cap): +def get_raw_cap(path_to_cap: str): return FileCapture(input_file=path_to_cap, display_filter='icmp', use_json=True, include_raw=True) +def get_cap(path_to_cap: str): + return FileCapture(input_file=path_to_cap, display_filter='icmp') + + def get_files(path) -> list: caps = glob(path) return caps @@ -151,11 +155,11 @@ def list_caps(state: str, redis): def get_protocol(packet): if 'ip_proto' in packet.icmp.field_names: protocol = str(packet.icmp.ip_proto) - if int(protocol) in range(143, 253): + if protocol in unassigned_proto: return protocol + ' (unassigned)' ip_proto = proto_dict[protocol] else: - return 'non-backscatter-icmp' + return 'nbs-icmp' return protocol + ' : ' + str(ip_proto) @@ -163,8 +167,6 @@ def get_icmp_payload(packet): if 'data' in packet.icmp.field_names: return str(packet.icmp.data) elif packet.icmp.field_names != ['type', 'code', 'checksum', 'checksum_status', 'ident', 'seq', 'seq_le']: - print(packet.icmp.field_names) - print(packet.icmp) return 'No data'