From d89834196cc36833cd5f0bbd38a1b3222db2aa6d Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Fri, 7 Jul 2017 18:10:24 +0530 Subject: [PATCH 01/10] Add new tanner Api function (#166) * Add new tanner api function * apply common filter approach * access code to use api methods * fix errors * apply sub-app approach * add complex statistics * rename methods * fix bug * update method * fix invalid snare_uuid * fix possible owner keyerror * fix and add new tests * update tests * fix invalid filter error * fix snare_uuid key value * fix apply filters * fix tests * fix tests --- tanner/api.py | 154 +++++++++++++++++++++++--- tanner/server.py | 31 +++--- tanner/session.py | 4 +- tanner/session_analyzer.py | 4 +- tanner/tests/test_server.py | 74 +++++++++++-- tanner/tests/test_session_analyzer.py | 2 +- 6 files changed, 228 insertions(+), 41 deletions(-) diff --git a/tanner/api.py b/tanner/api.py index 948eb5ce..1250aa93 100644 --- a/tanner/api.py +++ b/tanner/api.py @@ -1,35 +1,104 @@ import json import logging +import operator -import asyncio_redis - +from aiohttp import web class Api: - def __init__(self): + def __init__(self, redis_client): self.logger = logging.getLogger('tanner.api.Api') + self.redis_client = redis_client - async def handle_api_request(self, query, params, redis_client): - result = None + @staticmethod + def _make_response(msg): + response_message = dict( + version=1, + response=dict(message=msg) + ) + return response_message - if query == 'stats' and not params: - result = await self.return_stats(redis_client) - elif query == 'stats' and 'uuid' in params: - result = await self.return_uuid_stats(params['uuid'], redis_client, 50) - return result + async def handle_index(self, request): + result = 'tanner api' + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def handle_snares(self, request): + result = await self.return_snares() + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def handle_snare_info(self, request): + snare_uuid = request.match_info['snare_uuid'] + result = await self.return_snare_info(snare_uuid, 50) + response_msg = self._make_response(result) + return web.json_response(response_msg) - async def return_stats(self, redis_client): + async def handle_snare_stats(self, request): + snare_uuid = request.match_info['snare_uuid'] + result = await self.return_snare_stats(snare_uuid) + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def handle_sessions(self, request): + params = request.url.query + applied_filters = {} + try: + if 'filters' in params: + applied_filters = {filt.split(':')[0] : filt.split(':')[1] for filt in params['filters'].split()} + if 'start_time' in applied_filters: + applied_filters['start_time'] = float(applied_filters['start_time']) + if 'end_time' in applied_filters: + applied_filters['end_time'] = float(applied_filters['end_time']) + except Exception as e: + self.logger.error('Filter error : %s' % e) + result = 'Invalid filter definition' + else: + result = await self.return_sessions(applied_filters) + + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def handle_session_info(self, request): + sess_uuid = request.match_info['sess_uuid'] + result = await self.return_session_info(sess_uuid) + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def return_snares(self): query_res = [] try: - query_res = await redis_client.smembers('snare_ids') + query_res = await self.redis_client.smembers('snare_ids') query_res = await query_res.asset() except asyncio_redis.NotConnectedError as connection_error: self.logger.error('Can not connect to redis %s', connection_error) return list(query_res) - async def return_uuid_stats(self, uuid, redis_client, count=-1): + async def return_snare_stats(self, snare_uuid): + result = {} + sessions = await self.return_snare_info(snare_uuid) + if sessions == 'Invalid SNARE UUID': + return result + + result['total_sessions'] = len(sessions) + result['total_duration'] = 0 + result['attack_frequency'] = {'sqli' : 0, + 'lfi' : 0, + 'xss' : 0, + 'rfi' : 0, + 'cmd_exec' : 0} + + for sess in sessions: + result['total_duration'] += sess['end_time'] - sess['start_time'] + for attack in sess['attack_types']: + if attack in result['attack_frequency']: + result['attack_frequency'][attack] += 1 + + return result + + async def return_snare_info(self, uuid, count=-1): query_res = [] try: - query_res = await redis_client.lrange_aslist(uuid, 0, count) + query_res = await self.redis_client.lrange_aslist(uuid, 0, count) except asyncio_redis.NotConnectedError as connection_error: self.logger.error('Can not connect to redis %s', connection_error) else: @@ -38,3 +107,60 @@ async def return_uuid_stats(self, uuid, redis_client, count=-1): for (i, val) in enumerate(query_res): query_res[i] = json.loads(val) return query_res + + async def return_session_info(self, sess_uuid, snare_uuid= None): + query_res = [] + if snare_uuid: + snare_uuids = [snare_uuid] + else: + snare_uuids = await self.return_snares() + + for snare_id in snare_uuids: + sessions = await self.return_snare_info(snare_id) + if sessions == 'Invalid SNARE UUID': + continue + for sess in sessions: + if sess['sess_uuid'] == sess_uuid: + return sess + + async def return_sessions(self, filters): + query_res = [] + snare_uuids = await self.return_snares() + + matching_sessions = [] + for snare_id in snare_uuids: + result = await self.return_snare_info(snare_id) + if result == 'Invalid SNARE UUID': + return 'Invalid filter : SNARE UUID' + sessions = result + for sess in sessions: + match_count = 0 + for filter_name, filter_value in filters.items(): + try: + if(self.apply_filter(filter_name, filter_value, sess)): + match_count += 1 + except KeyError: + return 'Invalid filter : %s' % filter_name + + if match_count == len(filters): + matching_sessions.append(sess['sess_uuid']) + + return matching_sessions + + def apply_filter(self, filter_name, filter_value, sess): + available_filters = {'user_agent' : operator.contains, + 'peer_ip' : operator.eq, + 'attack_types' : operator.contains, + 'possible_owners' : operator.contains, + 'start_time' : operator.le, + 'end_time': operator.ge, + 'snare_uuid' : operator.eq + } + + try: + if available_filters[filter_name] is operator.contains: + return available_filters[filter_name](sess[filter_name], filter_value) + else: + return available_filters[filter_name](filter_value, sess[filter_name]) + except KeyError: + raise \ No newline at end of file diff --git a/tanner/server.py b/tanner/server.py index 4b0e4d94..589f7ba8 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -22,7 +22,7 @@ def __init__(self): self.session_manager = session_manager.SessionManager() self.dorks = dorks_manager.DorksManager() - self.api = api.Api() + self.api = None self.base_handler = base.BaseHandler(base_dir, db_name) self.logger = logging.getLogger(__name__) self.redis_client = None @@ -73,15 +73,6 @@ async def handle_event(self, request): lr.create_session(session_data) return web.json_response(response_msg) - async def handle_api(self, request): - api_query = request.match_info.get("api_query") - if api_query is None: - data = "tanner api" - else: - data = await self.api.handle_api_request(api_query, request.url.query, self.redis_client) - response_msg = self._make_response(data) - return web.json_response(response_msg) - async def handle_dorks(self, request): dorks = await self.dorks.choose_dorks(self.redis_client) response_msg = dict(version=1, response=dict(dorks=dorks)) @@ -93,10 +84,21 @@ async def on_shutdown(self, app): def setup_routes(self, app): app.router.add_route('*', '/', self.default_handler) app.router.add_post('/event', self.handle_event) - app.router.add_get('/api', self.handle_api) - app.router.add_get('/api/{api_query}', self.handle_api) app.router.add_get('/dorks', self.handle_dorks) + def setup_api_routes(self, app): + app.router.add_get('/', self.api.handle_index) + app.router.add_get('/snares', self.api.handle_snares) + app.router.add_resource('/snare/{snare_uuid}').add_route('GET', self.api.handle_snare_info) + app.router.add_resource('/snare-stats/{snare_uuid}').add_route('GET', self.api.handle_snare_stats) + app.router.add_resource('/sessions').add_route('GET', self.api.handle_sessions) + app.router.add_resource('/session/{sess_uuid}').add_route('GET', self.api.handle_session_info) + + def create_api_app(self, loop): + api_app = web.Application(loop=loop) + self.setup_api_routes(api_app) + return api_app + def create_app(self, loop): app = web.Application(loop=loop) app.on_shutdown.append(self.on_shutdown) @@ -105,8 +107,11 @@ def create_app(self, loop): def start(self): loop = asyncio.get_event_loop() - tanner_app = self.create_app(loop) self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client()) + self.api = api.Api(self.redis_client) + tanner_app = self.create_app(loop) + api_app = self.create_api_app(loop) + tanner_app.add_subapp('/api/', api_app) host = TannerConfig.get('TANNER', 'host') port = TannerConfig.get('TANNER', 'port') web.run_app(tanner_app, host=host, port=port) diff --git a/tanner/session.py b/tanner/session.py index 6a4ef150..9000dc96 100644 --- a/tanner/session.py +++ b/tanner/session.py @@ -17,7 +17,7 @@ def __init__(self, data): self.ip = data['peer']['ip'] self.port = data['peer']['port'] self.user_agent = data['headers']['user-agent'] - self.sensor = data['uuid'] + self.snare_uuid = data['uuid'] self.paths = [{'path': data['path'], 'timestamp': time.time(), 'response_status': data['status']}] self.cookies = data['cookies'] @@ -47,7 +47,7 @@ def is_expired(self): def to_json(self): sess = dict(peer=dict(ip=self.ip, port=self.port), user_agent=self.user_agent, - sensor=self.sensor, + snare_uuid=self.snare_uuid, sess_uuid=self.sess_uuid.hex, start_time=self.start_timestamp, end_time=self.timestamp, diff --git a/tanner/session_analyzer.py b/tanner/session_analyzer.py index 3a807fe5..bd957a54 100644 --- a/tanner/session_analyzer.py +++ b/tanner/session_analyzer.py @@ -31,7 +31,7 @@ async def analyze(self, session_key, redis_client): async def save_session(self, redis_client): while not self.queue.empty(): session = await self.queue.get() - s_key = session['sensor_uuid'] + s_key = session['snare_uuid'] del_key = session['sess_uuid'] try: await redis_client.lpush(s_key, [json.dumps(session)]) @@ -52,7 +52,7 @@ async def create_stats(self, session, redis_client): peer_ip=session['peer']['ip'], peer_port=session['peer']['port'], user_agent=session['user_agent'], - sensor_uuid=session['sensor'], + snare_uuid=session['snare_uuid'], start_time=session['start_time'], end_time=session['end_time'], requests_in_second=rps, diff --git a/tanner/tests/test_server.py b/tanner/tests/test_server.py index cac865de..36e078fa 100644 --- a/tanner/tests/test_server.py +++ b/tanner/tests/test_server.py @@ -4,7 +4,7 @@ from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop -from tanner import server +from tanner import server, api from tanner.config import TannerConfig @@ -45,6 +45,7 @@ async def choosed(client): redis.close = mock.Mock() self.serv.dorks = dorks self.serv.redis_client = redis + self.serv.api = api.Api(self.serv.redis_client) super(TestServer, self).setUp() @@ -56,6 +57,8 @@ async def coroutine(*args, **kwargs): def get_app(self): app = self.serv.create_app(loop=self.loop) + api_app = self.serv.create_api_app(loop=self.loop) + app.add_subapp('/api/', api_app) return app @unittest_run_loop @@ -93,21 +96,74 @@ async def test_dorks_request(self): self.assertDictEqual(detection, assert_content) @unittest_run_loop - async def test_api_request(self): + async def test_api_index_request(self): assert_content = {"version": 1, "response": {"message": "tanner api"}} - request = await self.client.request("GET", "/api") + request = await self.client.request("GET", "/api/") assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) @unittest_run_loop - async def test_stats_api_request(self): - async def _make_api_coroutine(*args, **kwargs): - return ["1", "2"] + async def test_api_snares_request(self): + async def mock_return_snares(): + return ["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"] - assert_content = {"version": 1, "response": {"message": ["1", "2"]}} - self.serv.api.handle_api_request = _make_api_coroutine - request = await self.client.request("GET", "/api/stats") + assert_content = {"version": 1, "response": {"message": ["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"]}} + self.serv.api.return_snares = mock_return_snares + request = await self.client.request("GET", "/api/snares") assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) + + @unittest_run_loop + async def test_api_snare_info_request(self): + async def mock_return_snare_info(snare_uuid, count): + if snare_uuid == "8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4" and count == 50: + return [{"test_sess1": "sess1_info"}, {"test_sess1": "sess2_info"}] + + assert_content = {"version": 1, "response": {"message": [{"test_sess1": "sess1_info"}, {"test_sess1": "sess2_info"}]}} + self.serv.api.return_snare_info = mock_return_snare_info + request = await self.client.request("GET", "/api/snare/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) + + @unittest_run_loop + async def test_api_snare_stats_request(self): + async def mock_return_snare_stats(snare_uuid): + if snare_uuid == "8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4": + return {"total_sessions": 605, "total_duration": 865.560286283493, "attack_frequency": {"sqli": 0, "lfi": 0, "xss": 0, "rfi": 0, "cmd_exec": 0}} + + assert_content = {"version": 1, "response": {"message": {"total_sessions": 605, "total_duration": 865.560286283493, "attack_frequency": {"sqli": 0, "lfi": 0, "xss": 0, "rfi": 0, "cmd_exec": 0}}}} + self.serv.api.return_snare_stats = mock_return_snare_stats + request = await self.client.request("GET", "/api/snare-stats/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) + + @unittest_run_loop + async def test_api_sessions_request(self): + async def mock_return_sessions(filters): + if type(filters) is dict and filters['peer_ip'] == "127.0.0.1" and \ + filters['start_time'] == 1497890400 and filters['user_agent'] == 'ngnix': + return ["f387d46eaeb1454cadf0713a4a55be49", "e85ae767b0bb4b1f91b421b3a28082ef"] + + assert_content = {"version": 1, "response": {"message": ["f387d46eaeb1454cadf0713a4a55be49", "e85ae767b0bb4b1f91b421b3a28082ef"]}} + self.serv.api.return_sessions = mock_return_sessions + request = await self.client.request("GET", "/api/sessions?filters=peer_ip:127.0.0.1 start_time:1497890400 user_agent:ngnix") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) + + @unittest_run_loop + async def test_api_sessions_info_request(self): + async def mock_return_session_info(sess_uuid): + if sess_uuid == "4afd45d61b994d9eb3ba20faa81a45e1": + return {"test_sess1": "sess1_info"} + + assert_content = {"version": 1, "response": {"message": {"test_sess1": "sess1_info"}}} + self.serv.api.return_session_info = mock_return_session_info + request = await self.client.request("GET", "/api/session/4afd45d61b994d9eb3ba20faa81a45e1") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) \ No newline at end of file diff --git a/tanner/tests/test_session_analyzer.py b/tanner/tests/test_session_analyzer.py index 5c37e7f0..114ea649 100644 --- a/tanner/tests/test_session_analyzer.py +++ b/tanner/tests/test_session_analyzer.py @@ -10,7 +10,7 @@ session = b'{"sess_uuid": "c546114f97f548f982756495f963e280", "start_time": 1466091813.4780173, ' \ b'"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' \ b'Chrome/53.0.2767.4 Safari/537.36", "end_time": 1466091899.9854035, ' \ - b'"sensor": "78e51180-bf0d-4757-8a04-f000e5efa179", "count": 24, ' \ + b'"snare_uuid": "78e51180-bf0d-4757-8a04-f000e5efa179", "count": 24, ' \ b'"paths": [{"timestamp": 1466091813.4779778, "path": "/", "attack_type": "index", "response_status": 200},' \ b'{"timestamp": 1466091842.7088752, "path": "/fluent-python.html", "attack_type": "index", ' \ b'"response_status": 200}, {"timestamp": 1466091858.214475, "path": "/wow-movie.html?exec=/bin/bash", ' \ From fc9dc6f329bf66d627c631a1ec71a2530a802ae0 Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Fri, 7 Jul 2017 18:10:37 +0530 Subject: [PATCH 02/10] Api docs (#169) * add api docs * update docs --- docs/source/api.rst | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/source/api.rst diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 00000000..c55fa831 --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,41 @@ +Tanner API +========== +Tanner api provides various stats related to traffic captured by snare. It can be accessed at ``locahost:8090/api/``. + +api/ +~~~~ +This is the index page which shows ``tanner api``. + +api/snares +~~~~~~~~~~ +This shows all the snares' uuid. + +api/snare/ +~~~~~~~~~~~~~~~~~~~~~~ +Replace ```` with a valid `snare-uuid` and it will show all the sessions related to that ``snare-uuid`` and their details. + +api/snare-stats/ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Replace ```` with a valid `snare-uuid` and it will show some stats. + + * No of sessions in the sanre + * Total duration for which snare remains active + * Attack frequency, which shows no of sessions which face different attacks. + +/api/sessions?filters= +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This shows all the sessions' uuid which follow the filters. +Filters are sepatated by ``white-space`` and name-value pair are separated by ``:``. E.g ``?filters=filter1:value1 filter2:value2``. + +It supports 5 filters: + + * **snare_uuid** -- Sessions related to given snare. E.g ``?filters=snare_uuid:8fa6aa98-4283-4085-bfb9-a1cd3a9e56e7`` + * **peer_ip** -- Sessions with given ip. E.g ``?filters=peer_ip:127.0.0.1`` + * **user-agent** -- Sessions with given user-agent. E.g ``?filters=user-agent:Chrome`` + * **attack_type** -- Sessions with given attack type such as lfi, rfi, xss, cmd_exec, sqli. E.g ``?filters=attack_type:lfi`` + * **possible_owner** -- Sessions with given owner type such as user, tool, crawler, attacker. E.g ``?filters=possible_owner:attacker`` + * **time_interval** -- Sessions which are active during a given time-interval. E.g ``?filters=time_interval:1480560-1480580`` + +/api/session/ +~~~~~~~~~~~~~~~~~~~~~~~~ +It gives all information about the session with given uuid. \ No newline at end of file From 150c05caf945026836c5a9844540e4f93a66a976 Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Mon, 10 Jul 2017 19:05:33 +0530 Subject: [PATCH 03/10] Cleanup docker containers on shutdown (#171) * fix #164 Cleanup docker containers on shutdown * fix tests --- tanner/server.py | 3 ++- tanner/session_manager.py | 32 ++++++++++++++++++++++---------- tanner/tests/test_server.py | 6 +++++- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/tanner/server.py b/tanner/server.py index 589f7ba8..2fef6106 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -79,8 +79,9 @@ async def handle_dorks(self, request): return web.json_response(response_msg) async def on_shutdown(self, app): + await self.session_manager.delete_sessions_on_shutdown(self.redis_client) self.redis_client.close() - + def setup_routes(self, app): app.router.add_route('*', '/', self.default_handler) app.router.add_post('/event', self.handle_event) diff --git a/tanner/session_manager.py b/tanner/session_manager.py index 360c8b89..aee9aeb0 100644 --- a/tanner/session_manager.py +++ b/tanner/session_manager.py @@ -70,13 +70,25 @@ async def delete_old_sessions(self, redis_client): for sess in self.sessions: if not sess.is_expired(): continue - await sess.remove_associated_db() - if sess.associated_env is not None: - await sess.remove_associated_env() - self.sessions.remove(sess) - try: - await redis_client.set(sess.get_uuid(), sess.to_json()) - await self.analyzer.analyze(sess.get_uuid(), redis_client) - except asyncio_redis.NotConnectedError as redis_error: - self.logger.error('Error connect to redis, session stay in memory. %s', redis_error) - self.sessions.append(sess) + is_deleted = await self.delete_session(sess, redis_client) + if is_deleted: + self.sessions.remove(sess) + + async def delete_sessions_on_shutdown(self, redis_client): + for sess in self.sessions: + is_deleted = await self.delete_session(sess, redis_client) + if is_deleted: + self.sessions.remove(sess) + + async def delete_session(self, sess, redis_client): + await sess.remove_associated_db() + if sess.associated_env is not None: + await sess.remove_associated_env() + try: + await redis_client.set(sess.get_uuid(), sess.to_json()) + await self.analyzer.analyze(sess.get_uuid(), redis_client) + except asyncio_redis.NotConnectedError as redis_error: + self.logger.error('Error connect to redis, session stay in memory. %s', redis_error) + return False + else: + return True \ No newline at end of file diff --git a/tanner/tests/test_server.py b/tanner/tests/test_server.py index 36e078fa..7ec6f11b 100644 --- a/tanner/tests/test_server.py +++ b/tanner/tests/test_server.py @@ -32,8 +32,12 @@ async def _add_or_update_mock(data, client): sess.get_uuid = mock.Mock(return_value=str(self.test_uuid)) return sess - self.serv.session_manager.add_or_update_session = _add_or_update_mock + async def _delete_sessions_mock(client): + pass + self.serv.session_manager.add_or_update_session = _add_or_update_mock + self.serv.session_manager.delete_sessions_on_shutdown = _delete_sessions_mock + async def choosed(client): return [x for x in range(10)] From e82d5e49435a0fc073e7693743690aa93dc52bf9 Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Wed, 19 Jul 2017 13:51:38 +0530 Subject: [PATCH 04/10] Make emulator set flexible (#175) * fix #172 Make emulator set flexible using config setting * fix indentation --- tanner/config.py | 4 +++- tanner/emulators/base.py | 29 ++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/tanner/config.py b/tanner/config.py index a6a4f1ac..74d1723e 100644 --- a/tanner/config.py +++ b/tanner/config.py @@ -9,7 +9,9 @@ 'vdocs': '/opt/tanner/data/vdocs.json'}, 'TANNER': {'host': '0.0.0.0', 'port': 8090}, 'REDIS': {'host': 'localhost', 'port': 6379, 'poolsize': 80, 'timeout': 1}, - 'EMULATORS': {'root_dir': '/opt/tanner'}, + 'EMULATORS': {'root_dir': '/opt/tanner', + 'emulator_enabled': {'sqli': True, 'rfi': True, 'lfi': True, 'xss': True, 'cmd_exec': True} + }, 'SQLI': {'type':'SQLITE', 'db_name': 'tanner_db', 'host':'localhost', 'user':'root', 'password':'user_pass'}, 'DOCKER': {'host_image': 'busybox:latest'}, 'LOGGER': {'log_debug': '/opt/tanner/tanner.log', 'log_err': '/opt/tanner/tanner.err'}, diff --git a/tanner/emulators/base.py b/tanner/emulators/base.py index 29546a39..421f4b5a 100644 --- a/tanner/emulators/base.py +++ b/tanner/emulators/base.py @@ -4,17 +4,19 @@ import urllib.parse import yarl +from tanner.config import TannerConfig from tanner.emulators import lfi, rfi, sqli, xss, cmd_exec from tanner.utils import patterns class BaseHandler: def __init__(self, base_dir, db_name, loop=None): + self.emulator_enabled = TannerConfig.get('EMULATORS', 'emulator_enabled') self.emulators = { - 'rfi': rfi.RfiEmulator(base_dir, loop), - 'lfi': lfi.LfiEmulator(), - 'xss': xss.XssEmulator(), - 'sqli': sqli.SqliEmulator(db_name, base_dir), - 'cmd_exec': cmd_exec.CmdExecEmulator() + 'rfi': rfi.RfiEmulator(base_dir, loop) if self.emulator_enabled['rfi'] else None, + 'lfi': lfi.LfiEmulator() if self.emulator_enabled['lfi'] else None, + 'xss': xss.XssEmulator() if self.emulator_enabled['xss'] else None, + 'sqli': sqli.SqliEmulator(db_name, base_dir) if self.emulator_enabled['sqli'] else None, + 'cmd_exec': cmd_exec.CmdExecEmulator() if self.emulator_enabled['cmd_exec'] else None } self.get_emulators = ['sqli', 'rfi', 'lfi', 'xss', 'cmd_exec'] self.post_emulators = ['sqli', 'rfi', 'lfi', 'xss', 'cmd_exec'] @@ -45,13 +47,14 @@ async def get_emulation_result(self, session, data, target_emulators): attack_params = {} for param_id, param_value in data.items(): for emulator in target_emulators: - possible_detection = self.emulators[emulator].scan(param_value) if param_value else None - if possible_detection: - if detection['order'] < possible_detection['order']: - detection = possible_detection - if emulator not in attack_params: - attack_params[emulator] = [] - attack_params[emulator].append(dict(id= param_id, value= param_value)) + if self.emulator_enabled[emulator]: + possible_detection = self.emulators[emulator].scan(param_value) if param_value else None + if possible_detection: + if detection['order'] < possible_detection['order']: + detection = possible_detection + if emulator not in attack_params: + attack_params[emulator] = [] + attack_params[emulator].append(dict(id=param_id, value=param_value)) if detection['name'] in self.emulators: emulation_result = await self.emulators[detection['name']].handle(attack_params[detection['name']], session) @@ -117,4 +120,4 @@ async def emulate(self, data, session): async def handle(self, data, session): detection = await self.emulate(data, session) - return detection \ No newline at end of file + return detection From fc629f030e78e76a120558b265c6f5b8540a1e8b Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Thu, 20 Jul 2017 11:40:57 +0530 Subject: [PATCH 05/10] Web UI (#173) * add index and snares page * add session-info page * add session-filters page * add session-stats * update snare and snares page * add filters input * Update UI list to strings * Add interactive filtering * show filters * add static files * implement inheritance * minor fixes * use congif for port and ip * add filter example on hover * add missing import * start tannerweb easily * add dependencies * fix filter bug * make filter example notable --- bin/tannerweb | 9 +++ requirements.txt | 2 + setup.py | 2 +- tanner/api.py | 1 + tanner/config.py | 1 + tanner/tests/test_config.py | 2 + tanner/web/__init__.py | 0 tanner/web/server.py | 100 ++++++++++++++++++++++++++ tanner/web/static/css/styles.css | 14 ++++ tanner/web/static/js/site.js | 13 ++++ tanner/web/templates/base.html | 17 +++++ tanner/web/templates/index.html | 5 ++ tanner/web/templates/session.html | 92 ++++++++++++++++++++++++ tanner/web/templates/sessions.html | 30 ++++++++ tanner/web/templates/snare-stats.html | 27 +++++++ tanner/web/templates/snare.html | 7 ++ tanner/web/templates/snares.html | 16 +++++ 17 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 bin/tannerweb create mode 100644 tanner/web/__init__.py create mode 100644 tanner/web/server.py create mode 100644 tanner/web/static/css/styles.css create mode 100644 tanner/web/static/js/site.js create mode 100644 tanner/web/templates/base.html create mode 100644 tanner/web/templates/index.html create mode 100644 tanner/web/templates/session.html create mode 100644 tanner/web/templates/sessions.html create mode 100644 tanner/web/templates/snare-stats.html create mode 100644 tanner/web/templates/snare.html create mode 100644 tanner/web/templates/snares.html diff --git a/bin/tannerweb b/bin/tannerweb new file mode 100644 index 00000000..5ceeadc8 --- /dev/null +++ b/bin/tannerweb @@ -0,0 +1,9 @@ +#!/usr/bin/python3.5 +from tanner.web import server + +def main(): + tannerweb = server.TannerWebServer() + tannerweb.start() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 17cdf731..261b4a4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ aiohttp>=2.0 aiomysql +aiohttp_jinja2 docker elizabeth==0.3.27 yarl @@ -8,3 +9,4 @@ asyncio_redis uvloop pymongo pylibinjection +jinja2 \ No newline at end of file diff --git a/setup.py b/setup.py index a022eadd..8ace4ef5 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,6 @@ author_email='glastopf@public.honeynet.org', url='https://github.com/mushorg/tanner', packages=find_packages(exclude=['*.pyc']), - scripts=['bin/tanner'], + scripts=['bin/tanner', 'bin/tannerweb'], data_files=[('/opt/tanner/data/',['tanner/data/dorks.pickle'])] ) \ No newline at end of file diff --git a/tanner/api.py b/tanner/api.py index 1250aa93..d4f5c227 100644 --- a/tanner/api.py +++ b/tanner/api.py @@ -1,6 +1,7 @@ import json import logging import operator +import asyncio_redis from aiohttp import web diff --git a/tanner/config.py b/tanner/config.py index 74d1723e..ee644f58 100644 --- a/tanner/config.py +++ b/tanner/config.py @@ -8,6 +8,7 @@ 'user_dorks': '/opt/tanner/data/user_dorks.pickle', 'vdocs': '/opt/tanner/data/vdocs.json'}, 'TANNER': {'host': '0.0.0.0', 'port': 8090}, + 'WEB': {'host': '0.0.0.0', 'port': 8091}, 'REDIS': {'host': 'localhost', 'port': 6379, 'poolsize': 80, 'timeout': 1}, 'EMULATORS': {'root_dir': '/opt/tanner', 'emulator_enabled': {'sqli': True, 'rfi': True, 'lfi': True, 'xss': True, 'cmd_exec': True} diff --git a/tanner/tests/test_config.py b/tanner/tests/test_config.py index aece0149..bfd39dba 100644 --- a/tanner/tests/test_config.py +++ b/tanner/tests/test_config.py @@ -12,6 +12,7 @@ def setUp(self): 'user_dorks': '/tmp/user_tanner/data/user_dorks.pickle', 'vdocs': '/tmp/user_tanner/data/vdocs.json'}, 'TANNER': {'host': '0.0.0.0', 'port': '9000'}, + 'WEB': {'host': '0.0.0.0', 'port': '9001'}, 'REDIS': {'host': 'localhost', 'port': '1337', 'poolsize': '40', 'timeout': '5'}, 'EMULATORS': {'root_dir': '/tmp/user_tanner'}, 'SQLI': {'type':'SQLITE', 'db_name': 'user_tanner_db', 'host':'localhost', 'user':'user_name', 'password':'user_pass'}, @@ -57,6 +58,7 @@ def test_get_when_file_dont_exists(self): 'user_dorks': '/opt/tanner/data/user_dorks.pickle', 'vdocs': '/opt/tanner/data/vdocs.json'}, 'TANNER': {'host': '0.0.0.0', 'port': 8090}, + 'WEB': {'host': '0.0.0.0', 'port': 8091}, 'REDIS': {'host': 'localhost', 'port': 6379, 'poolsize': 80, 'timeout': 1}, 'EMULATORS': {'root_dir': '/opt/tanner'}, 'SQLI': {'type':'SQLITE', 'db_name': 'tanner_db', 'host':'localhost', 'user':'root', 'password':'user_pass'}, diff --git a/tanner/web/__init__.py b/tanner/web/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tanner/web/server.py b/tanner/web/server.py new file mode 100644 index 00000000..b266e097 --- /dev/null +++ b/tanner/web/server.py @@ -0,0 +1,100 @@ +import asyncio +import aiohttp_jinja2 +import jinja2 +import logging + +from aiohttp import web +from tanner import api, redis_client +from tanner.config import TannerConfig + +class TannerWebServer: + def __init__(self): + self.logger = logging.getLogger('tanner.web.tannerwebserver') + self.api = None + self.redis_client = None + + @aiohttp_jinja2.template('index.html') + async def handle_index(self, request): + return + + @aiohttp_jinja2.template('snares.html') + async def handle_snares(self, request): + snares = await self.api.return_snares() + return { + 'snares' : snares + } + + @aiohttp_jinja2.template('snare.html') + async def handle_snare(self, request): + snare_uuid = request.match_info['snare_uuid'] + return{ + 'snare' : snare_uuid + } + + @aiohttp_jinja2.template('snare-stats.html') + async def handle_snare_stats(self, request): + snare_uuid = request.match_info['snare_uuid'] + snare_stats = await self.api.return_snare_stats(snare_uuid) + return { + 'snare_stats' : snare_stats + } + + @aiohttp_jinja2.template('sessions.html') + async def handle_sessions(self, request): + snare_uuid = request.match_info['snare_uuid'] + params = request.url.query + applied_filters = {'snare_uuid': snare_uuid} + try: + if 'filters' in params: + for filt in params['filters'].split(): + applied_filters[filt.split(':')[0]] = filt.split(':')[1] + if 'start_time' in applied_filters: + applied_filters['start_time'] = float(applied_filters['start_time']) + if 'end_time' in applied_filters: + applied_filters['end_time'] = float(applied_filters['end_time']) + except Exception as e: + self.logger.error('Filter error : %s' % e) + result = 'Invalid filter definition' + else: + sess_uuids = await self.api.return_sessions(applied_filters) + sessions = [] + for sess_uuid in sess_uuids: + sess = await self.api.return_session_info(sess_uuid) + sessions.append(sess) + result = sessions + return { + 'sessions' : result + } + + @aiohttp_jinja2.template('session.html') + async def handle_session_info(self, request): + sess_uuid = request.match_info['sess_uuid'] + session = await self.api.return_session_info(sess_uuid) + return { + 'session' : session + } + + def setup_routes(self, app): + app.router.add_get('/', self.handle_index) + app.router.add_get('/snares', self.handle_snares) + app.router.add_resource('/snare/{snare_uuid}').add_route('GET', self.handle_snare) + app.router.add_resource('/snare-stats/{snare_uuid}').add_route('GET', self.handle_snare_stats) + app.router.add_resource('/session/{sess_uuid}').add_route('GET', self.handle_session_info) + app.router.add_resource('/{snare_uuid}/sessions').add_route('GET', self.handle_sessions) + app.router.add_static('/static/', path='tanner/web/static') + + def create_app(self, loop): + app = web.Application(loop= loop) + aiohttp_jinja2.setup(app, + loader= jinja2.FileSystemLoader('tanner/web/templates')) + self.setup_routes(app) + return app + + def start(self): + loop = asyncio.get_event_loop() + self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client()) + self.api = api.Api(self.redis_client) + app = self.create_app(loop) + host = TannerConfig.get('WEB', 'host') + port = TannerConfig.get('WEB', 'port') + web.run_app(app, host=host, port=port) diff --git a/tanner/web/static/css/styles.css b/tanner/web/static/css/styles.css new file mode 100644 index 00000000..a79ec495 --- /dev/null +++ b/tanner/web/static/css/styles.css @@ -0,0 +1,14 @@ +table { + border-collapse: collapse; + width: 80%; +} + +th, td { + padding: 8px; + text-align: left; + border-bottom: 1px solid #ddd; +} + +tr:hover{ + background-color:#f5f5f5 +} \ No newline at end of file diff --git a/tanner/web/static/js/site.js b/tanner/web/static/js/site.js new file mode 100644 index 00000000..b369b3f9 --- /dev/null +++ b/tanner/web/static/js/site.js @@ -0,0 +1,13 @@ +function findGetParameter(parameterName) { + var result = null, + tmp = []; + location.search + .substr(1) + .split("&") + .forEach(function (item) { + tmp = item.split("="); + if (tmp[0] === parameterName) result = decodeURIComponent(tmp[1]); + }); + return result; +} +document.getElementById('filters').value = findGetParameter("filters"); \ No newline at end of file diff --git a/tanner/web/templates/base.html b/tanner/web/templates/base.html new file mode 100644 index 00000000..fb7d73a1 --- /dev/null +++ b/tanner/web/templates/base.html @@ -0,0 +1,17 @@ + + + + {% block title %}{% endblock %} - Tanner Web + + + +
+ {% block content%} + {% endblock%} +
+ + + \ No newline at end of file diff --git a/tanner/web/templates/index.html b/tanner/web/templates/index.html new file mode 100644 index 00000000..3c87648c --- /dev/null +++ b/tanner/web/templates/index.html @@ -0,0 +1,5 @@ +{% extends "base.html" %} +{% block title %}Home{% endblock %} +{% block content %} + Tanner Web +{% endblock %} \ No newline at end of file diff --git a/tanner/web/templates/session.html b/tanner/web/templates/session.html new file mode 100644 index 00000000..d7751bd5 --- /dev/null +++ b/tanner/web/templates/session.html @@ -0,0 +1,92 @@ +{% extends "base.html" %} +{% block title %}Session({{session.sess_uuid}}){% endblock %} +{% block content %} + +

SESSION INFO

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyValue
UUID{{session.sess_uuid}}
IP{{session.peer_ip}}
Port{{session.peer_port}}
User Agents{{session.user_agent}}
Snare UUID{{session.snare_uuid}}
Start time{{session.start_time}}
End time{{session.end_time}}
Requests/sec{{session.requests_in_second}}
Time between requests{{session.approx_time_between_requests}}
Paths{{session.accepted_paths}}
Errors{{session.errors}}
Hidden Links{{session.hidden_links}}
Attack types + {% for attack in session.attack_types %} + {{attack}}
+ {% endfor %} +
Paths + {% for path in session.paths %} + {{path}}
+ {% endfor %} +
Cookies + {% for key, val in session.cookies.items() %} + {{key}} : {{val}}
+ {% endfor %} +
Possible Owners + {% for owner in session.possible_owners %} + {{owner}}
+ {% endfor %} +
+{% endblock %} \ No newline at end of file diff --git a/tanner/web/templates/sessions.html b/tanner/web/templates/sessions.html new file mode 100644 index 00000000..7efb278b --- /dev/null +++ b/tanner/web/templates/sessions.html @@ -0,0 +1,30 @@ +{% extends "base.html" %} +{% block title %}Snare Sessions{% endblock %} +{% block content %} +

SNARE-SESSIONS

+
+
+ Filters: + How? +
+
+ + + + + + + + {% for sess in sessions %} + + + + + + + {% endfor %} +
NoSession-uuidIPOwner
{{loop.index}}{{sess.sess_uuid}}{{sess.peer_ip}}{{sess.possible_owners|join(',')}}
+ + +{% endblock %} \ No newline at end of file diff --git a/tanner/web/templates/snare-stats.html b/tanner/web/templates/snare-stats.html new file mode 100644 index 00000000..f4dfdccd --- /dev/null +++ b/tanner/web/templates/snare-stats.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} +{% block title %}SNARE-STATS{% endblock %} +{% block content %} +

SNARE-STATS

+ + + + + + + + + + + + + + + + + +
KeyValue
No of Sessions{{snare_stats.total_sessions}}
Total Duration{{snare_stats.total_duration}}
Attack Frequency + {% for key, val in snare_stats.attack_frequency.items() %} + {{key}} : {{val}}
+ {% endfor %} +
+{% endblock %} \ No newline at end of file diff --git a/tanner/web/templates/snare.html b/tanner/web/templates/snare.html new file mode 100644 index 00000000..a44a259c --- /dev/null +++ b/tanner/web/templates/snare.html @@ -0,0 +1,7 @@ +{% extends "base.html" %} +{% block title %}Snare({{snare}}){% endblock %} +{% block content %} +

{{snare}}

+

Snare-Stats

+

Sessions

+{% endblock %} \ No newline at end of file diff --git a/tanner/web/templates/snares.html b/tanner/web/templates/snares.html new file mode 100644 index 00000000..c3435abf --- /dev/null +++ b/tanner/web/templates/snares.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} +{% block title %}Snares{% endblock %} +{% block content %} +

SNARE-UUIDS

+ + + + + + {% for snare in snares %} + + + + + {% endfor %} +{% endblock %} \ No newline at end of file From a88bbdc31c1ebda0c00b57abfc033081e1821845 Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Fri, 21 Jul 2017 12:18:10 +0530 Subject: [PATCH 06/10] add option for poolsize (#176) --- tanner/redis_client.py | 5 +++-- tanner/web/server.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tanner/redis_client.py b/tanner/redis_client.py index cff11085..9641b6c6 100644 --- a/tanner/redis_client.py +++ b/tanner/redis_client.py @@ -10,12 +10,13 @@ class RedisClient: @staticmethod - async def get_redis_client(): + async def get_redis_client(poolsize=None): redis_client = None try: host = TannerConfig.get('REDIS', 'host') port = TannerConfig.get('REDIS', 'port') - poolsize = TannerConfig.get('REDIS', 'poolsize') + if poolsize is None: + poolsize = TannerConfig.get('REDIS', 'poolsize') timeout = TannerConfig.get('REDIS', 'timeout') redis_client = await asyncio.wait_for(asyncio_redis.Pool.create( host=host, port=int(port), poolsize=int(poolsize)), timeout=int(timeout)) diff --git a/tanner/web/server.py b/tanner/web/server.py index b266e097..ff39dd12 100644 --- a/tanner/web/server.py +++ b/tanner/web/server.py @@ -92,7 +92,7 @@ def create_app(self, loop): def start(self): loop = asyncio.get_event_loop() - self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client()) + self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client(poolsize=20)) self.api = api.Api(self.redis_client) app = self.create_app(loop) host = TannerConfig.get('WEB', 'host') From 0297c84b92e344fda5c267a5dc5ab1d83e96192b Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Mon, 24 Jul 2017 00:17:04 +0530 Subject: [PATCH 07/10] New api server (#177) * make new server * make bin file * make new package for api * remove api server tests from tanner server * use config to get host and port * update api name in setup file * reduce poolsize for api --- bin/tannerapi | 9 ++++ setup.py | 2 +- tanner/api/__init__.py | 0 tanner/{ => api}/api.py | 60 +---------------------- tanner/api/server.py | 94 +++++++++++++++++++++++++++++++++++++ tanner/config.py | 1 + tanner/server.py | 21 +-------- tanner/tests/test_config.py | 2 + tanner/tests/test_server.py | 78 +----------------------------- tanner/web/server.py | 3 +- 10 files changed, 114 insertions(+), 156 deletions(-) create mode 100644 bin/tannerapi create mode 100644 tanner/api/__init__.py rename tanner/{ => api}/api.py (65%) create mode 100644 tanner/api/server.py diff --git a/bin/tannerapi b/bin/tannerapi new file mode 100644 index 00000000..0336cd7d --- /dev/null +++ b/bin/tannerapi @@ -0,0 +1,9 @@ +#!/usr/bin/python3.5 +from tanner.api import server + +def main(): + api = server.ApiServer() + api.start() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/setup.py b/setup.py index 8ace4ef5..fa176bde 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,6 @@ author_email='glastopf@public.honeynet.org', url='https://github.com/mushorg/tanner', packages=find_packages(exclude=['*.pyc']), - scripts=['bin/tanner', 'bin/tannerweb'], + scripts=['bin/tanner', 'bin/tannerweb', 'bin/tannerapi'], data_files=[('/opt/tanner/data/',['tanner/data/dorks.pickle'])] ) \ No newline at end of file diff --git a/tanner/api/__init__.py b/tanner/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tanner/api.py b/tanner/api/api.py similarity index 65% rename from tanner/api.py rename to tanner/api/api.py index d4f5c227..6f0548c0 100644 --- a/tanner/api.py +++ b/tanner/api/api.py @@ -1,70 +1,14 @@ import json import logging import operator +import asyncio import asyncio_redis -from aiohttp import web - class Api: def __init__(self, redis_client): self.logger = logging.getLogger('tanner.api.Api') self.redis_client = redis_client - @staticmethod - def _make_response(msg): - response_message = dict( - version=1, - response=dict(message=msg) - ) - return response_message - - async def handle_index(self, request): - result = 'tanner api' - response_msg = self._make_response(result) - return web.json_response(response_msg) - - async def handle_snares(self, request): - result = await self.return_snares() - response_msg = self._make_response(result) - return web.json_response(response_msg) - - async def handle_snare_info(self, request): - snare_uuid = request.match_info['snare_uuid'] - result = await self.return_snare_info(snare_uuid, 50) - response_msg = self._make_response(result) - return web.json_response(response_msg) - - async def handle_snare_stats(self, request): - snare_uuid = request.match_info['snare_uuid'] - result = await self.return_snare_stats(snare_uuid) - response_msg = self._make_response(result) - return web.json_response(response_msg) - - async def handle_sessions(self, request): - params = request.url.query - applied_filters = {} - try: - if 'filters' in params: - applied_filters = {filt.split(':')[0] : filt.split(':')[1] for filt in params['filters'].split()} - if 'start_time' in applied_filters: - applied_filters['start_time'] = float(applied_filters['start_time']) - if 'end_time' in applied_filters: - applied_filters['end_time'] = float(applied_filters['end_time']) - except Exception as e: - self.logger.error('Filter error : %s' % e) - result = 'Invalid filter definition' - else: - result = await self.return_sessions(applied_filters) - - response_msg = self._make_response(result) - return web.json_response(response_msg) - - async def handle_session_info(self, request): - sess_uuid = request.match_info['sess_uuid'] - result = await self.return_session_info(sess_uuid) - response_msg = self._make_response(result) - return web.json_response(response_msg) - async def return_snares(self): query_res = [] try: @@ -164,4 +108,4 @@ def apply_filter(self, filter_name, filter_value, sess): else: return available_filters[filter_name](filter_value, sess[filter_name]) except KeyError: - raise \ No newline at end of file + raise diff --git a/tanner/api/server.py b/tanner/api/server.py new file mode 100644 index 00000000..cb2439f3 --- /dev/null +++ b/tanner/api/server.py @@ -0,0 +1,94 @@ +import asyncio +import logging + +from tanner.api import api +from aiohttp import web +from tanner import redis_client +from tanner.config import TannerConfig + +class ApiServer: + def __init__(self): + self.logger = logging.getLogger('tanner.api.ApiServer') + self.api = None + + @staticmethod + def _make_response(msg): + response_message = dict( + version=1, + response=dict(message=msg) + ) + return response_message + + async def handle_index(self, request): + result = 'tanner api' + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def handle_snares(self, request): + result = await self.api.return_snares() + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def handle_snare_info(self, request): + snare_uuid = request.match_info['snare_uuid'] + result = await self.api.return_snare_info(snare_uuid, 50) + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def handle_snare_stats(self, request): + snare_uuid = request.match_info['snare_uuid'] + result = await self.api.return_snare_stats(snare_uuid) + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def handle_sessions(self, request): + snare_uuid = request.match_info['snare_uuid'] + params = request.url.query + applied_filters = {'snare_uuid': snare_uuid} + try: + if 'filters' in params: + for filt in params['filters'].split(): + applied_filters[filt.split(':')[0]] = filt.split(':')[1] + if 'start_time' in applied_filters: + applied_filters['start_time'] = float(applied_filters['start_time']) + if 'end_time' in applied_filters: + applied_filters['end_time'] = float(applied_filters['end_time']) + except Exception as e: + self.logger.error('Filter error : %s' % e) + result = 'Invalid filter definition' + else: + result = await self.api.return_sessions(applied_filters) + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def handle_session_info(self, request): + sess_uuid = request.match_info['sess_uuid'] + result = await self.api.return_session_info(sess_uuid) + response_msg = self._make_response(result) + return web.json_response(response_msg) + + async def on_shutdown(self, app): + self.redis_client.close() + + def setup_routes(self, app): + app.router.add_get('/', self.handle_index) + app.router.add_get('/snares', self.handle_snares) + app.router.add_resource('/snare/{snare_uuid}').add_route('GET', self.handle_snare_info) + app.router.add_resource('/snare-stats/{snare_uuid}').add_route('GET', self.handle_snare_stats) + app.router.add_resource('/{snare_uuid}/sessions').add_route('GET', self.handle_sessions) + app.router.add_resource('/session/{sess_uuid}').add_route('GET', self.handle_session_info) + + def create_app(self, loop): + app = web.Application(loop=loop) + app.on_shutdown.append(self.on_shutdown) + self.setup_routes(app) + return app + + def start(self): + loop = asyncio.get_event_loop() + self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client(poolsize=20)) + self.api = api.Api(self.redis_client) + app = self.create_app(loop) + host = TannerConfig.get('API', 'host') + port = TannerConfig.get('API', 'port') + web.run_app(app, host=host, port=port) \ No newline at end of file diff --git a/tanner/config.py b/tanner/config.py index ee644f58..34cd1854 100644 --- a/tanner/config.py +++ b/tanner/config.py @@ -9,6 +9,7 @@ 'vdocs': '/opt/tanner/data/vdocs.json'}, 'TANNER': {'host': '0.0.0.0', 'port': 8090}, 'WEB': {'host': '0.0.0.0', 'port': 8091}, + 'API': {'host': '0.0.0.0', 'port': 8092}, 'REDIS': {'host': 'localhost', 'port': 6379, 'poolsize': 80, 'timeout': 1}, 'EMULATORS': {'root_dir': '/opt/tanner', 'emulator_enabled': {'sqli': True, 'rfi': True, 'lfi': True, 'xss': True, 'cmd_exec': True} diff --git a/tanner/server.py b/tanner/server.py index 2fef6106..9deb9250 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -22,7 +22,6 @@ def __init__(self): self.session_manager = session_manager.SessionManager() self.dorks = dorks_manager.DorksManager() - self.api = None self.base_handler = base.BaseHandler(base_dir, db_name) self.logger = logging.getLogger(__name__) self.redis_client = None @@ -87,19 +86,6 @@ def setup_routes(self, app): app.router.add_post('/event', self.handle_event) app.router.add_get('/dorks', self.handle_dorks) - def setup_api_routes(self, app): - app.router.add_get('/', self.api.handle_index) - app.router.add_get('/snares', self.api.handle_snares) - app.router.add_resource('/snare/{snare_uuid}').add_route('GET', self.api.handle_snare_info) - app.router.add_resource('/snare-stats/{snare_uuid}').add_route('GET', self.api.handle_snare_stats) - app.router.add_resource('/sessions').add_route('GET', self.api.handle_sessions) - app.router.add_resource('/session/{sess_uuid}').add_route('GET', self.api.handle_session_info) - - def create_api_app(self, loop): - api_app = web.Application(loop=loop) - self.setup_api_routes(api_app) - return api_app - def create_app(self, loop): app = web.Application(loop=loop) app.on_shutdown.append(self.on_shutdown) @@ -109,10 +95,7 @@ def create_app(self, loop): def start(self): loop = asyncio.get_event_loop() self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client()) - self.api = api.Api(self.redis_client) - tanner_app = self.create_app(loop) - api_app = self.create_api_app(loop) - tanner_app.add_subapp('/api/', api_app) + app = self.create_app(loop) host = TannerConfig.get('TANNER', 'host') port = TannerConfig.get('TANNER', 'port') - web.run_app(tanner_app, host=host, port=port) + web.run_app(app, host=host, port=port) diff --git a/tanner/tests/test_config.py b/tanner/tests/test_config.py index bfd39dba..f4c7f90b 100644 --- a/tanner/tests/test_config.py +++ b/tanner/tests/test_config.py @@ -13,6 +13,7 @@ def setUp(self): 'vdocs': '/tmp/user_tanner/data/vdocs.json'}, 'TANNER': {'host': '0.0.0.0', 'port': '9000'}, 'WEB': {'host': '0.0.0.0', 'port': '9001'}, + 'WEB': {'host': '0.0.0.0', 'port': '9002'}, 'REDIS': {'host': 'localhost', 'port': '1337', 'poolsize': '40', 'timeout': '5'}, 'EMULATORS': {'root_dir': '/tmp/user_tanner'}, 'SQLI': {'type':'SQLITE', 'db_name': 'user_tanner_db', 'host':'localhost', 'user':'user_name', 'password':'user_pass'}, @@ -59,6 +60,7 @@ def test_get_when_file_dont_exists(self): 'vdocs': '/opt/tanner/data/vdocs.json'}, 'TANNER': {'host': '0.0.0.0', 'port': 8090}, 'WEB': {'host': '0.0.0.0', 'port': 8091}, + 'API': {'host': '0.0.0.0', 'port': 8092}, 'REDIS': {'host': 'localhost', 'port': 6379, 'poolsize': 80, 'timeout': 1}, 'EMULATORS': {'root_dir': '/opt/tanner'}, 'SQLI': {'type':'SQLITE', 'db_name': 'tanner_db', 'host':'localhost', 'user':'root', 'password':'user_pass'}, diff --git a/tanner/tests/test_server.py b/tanner/tests/test_server.py index 7ec6f11b..acca42ab 100644 --- a/tanner/tests/test_server.py +++ b/tanner/tests/test_server.py @@ -4,7 +4,7 @@ from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop -from tanner import server, api +from tanner import server from tanner.config import TannerConfig @@ -49,7 +49,6 @@ async def choosed(client): redis.close = mock.Mock() self.serv.dorks = dorks self.serv.redis_client = redis - self.serv.api = api.Api(self.serv.redis_client) super(TestServer, self).setUp() @@ -61,8 +60,6 @@ async def coroutine(*args, **kwargs): def get_app(self): app = self.serv.create_app(loop=self.loop) - api_app = self.serv.create_api_app(loop=self.loop) - app.add_subapp('/api/', api_app) return app @unittest_run_loop @@ -98,76 +95,3 @@ async def test_dorks_request(self): assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) - - @unittest_run_loop - async def test_api_index_request(self): - assert_content = {"version": 1, "response": {"message": "tanner api"}} - request = await self.client.request("GET", "/api/") - assert request.status == 200 - detection = await request.json() - self.assertDictEqual(detection, assert_content) - - @unittest_run_loop - async def test_api_snares_request(self): - async def mock_return_snares(): - return ["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"] - - assert_content = {"version": 1, "response": {"message": ["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"]}} - self.serv.api.return_snares = mock_return_snares - request = await self.client.request("GET", "/api/snares") - assert request.status == 200 - detection = await request.json() - self.assertDictEqual(detection, assert_content) - - @unittest_run_loop - async def test_api_snare_info_request(self): - async def mock_return_snare_info(snare_uuid, count): - if snare_uuid == "8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4" and count == 50: - return [{"test_sess1": "sess1_info"}, {"test_sess1": "sess2_info"}] - - assert_content = {"version": 1, "response": {"message": [{"test_sess1": "sess1_info"}, {"test_sess1": "sess2_info"}]}} - self.serv.api.return_snare_info = mock_return_snare_info - request = await self.client.request("GET", "/api/snare/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4") - assert request.status == 200 - detection = await request.json() - self.assertDictEqual(detection, assert_content) - - @unittest_run_loop - async def test_api_snare_stats_request(self): - async def mock_return_snare_stats(snare_uuid): - if snare_uuid == "8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4": - return {"total_sessions": 605, "total_duration": 865.560286283493, "attack_frequency": {"sqli": 0, "lfi": 0, "xss": 0, "rfi": 0, "cmd_exec": 0}} - - assert_content = {"version": 1, "response": {"message": {"total_sessions": 605, "total_duration": 865.560286283493, "attack_frequency": {"sqli": 0, "lfi": 0, "xss": 0, "rfi": 0, "cmd_exec": 0}}}} - self.serv.api.return_snare_stats = mock_return_snare_stats - request = await self.client.request("GET", "/api/snare-stats/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4") - assert request.status == 200 - detection = await request.json() - self.assertDictEqual(detection, assert_content) - - @unittest_run_loop - async def test_api_sessions_request(self): - async def mock_return_sessions(filters): - if type(filters) is dict and filters['peer_ip'] == "127.0.0.1" and \ - filters['start_time'] == 1497890400 and filters['user_agent'] == 'ngnix': - return ["f387d46eaeb1454cadf0713a4a55be49", "e85ae767b0bb4b1f91b421b3a28082ef"] - - assert_content = {"version": 1, "response": {"message": ["f387d46eaeb1454cadf0713a4a55be49", "e85ae767b0bb4b1f91b421b3a28082ef"]}} - self.serv.api.return_sessions = mock_return_sessions - request = await self.client.request("GET", "/api/sessions?filters=peer_ip:127.0.0.1 start_time:1497890400 user_agent:ngnix") - assert request.status == 200 - detection = await request.json() - self.assertDictEqual(detection, assert_content) - - @unittest_run_loop - async def test_api_sessions_info_request(self): - async def mock_return_session_info(sess_uuid): - if sess_uuid == "4afd45d61b994d9eb3ba20faa81a45e1": - return {"test_sess1": "sess1_info"} - - assert_content = {"version": 1, "response": {"message": {"test_sess1": "sess1_info"}}} - self.serv.api.return_session_info = mock_return_session_info - request = await self.client.request("GET", "/api/session/4afd45d61b994d9eb3ba20faa81a45e1") - assert request.status == 200 - detection = await request.json() - self.assertDictEqual(detection, assert_content) \ No newline at end of file diff --git a/tanner/web/server.py b/tanner/web/server.py index ff39dd12..efff1715 100644 --- a/tanner/web/server.py +++ b/tanner/web/server.py @@ -4,7 +4,8 @@ import logging from aiohttp import web -from tanner import api, redis_client +from tanner.api import api +from tanner import redis_client from tanner.config import TannerConfig class TannerWebServer: From bf04e93fa3bfa1563d9d893def832e4103bc27e4 Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Mon, 24 Jul 2017 23:12:41 +0530 Subject: [PATCH 08/10] Update docs (#178) * update api and web docs * some fixes * some fixes * Update README * Update config * minor updates * Update sessions and startup --- README.md | 8 +++++++ docs/source/api.rst | 26 +++++++++++---------- docs/source/config.rst | 15 +++++++++++- docs/source/index.rst | 2 ++ docs/source/quick-start.rst | 12 +++++++++- docs/source/sessions.rst | 4 ++-- docs/source/web.rst | 46 +++++++++++++++++++++++++++++++++++++ 7 files changed, 97 insertions(+), 16 deletions(-) create mode 100644 docs/source/web.rst diff --git a/README.md b/README.md index a3527376..b8e3285b 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,14 @@ Getting Started 4. Install TANNER: ``python3 setup.py install`` 5. Run TANNER: ``sudo tanner`` +### Run Tanner Api + +Run ``sudo tannerapi`` + +### Run Tanner WebUI + +Run ``sudo tannerweb`` + You obviously want to bind to 0.0.0.0 when running in production and on a different host than SNARE (recommended). [See the docs for more info](docs/source/index.rst) diff --git a/docs/source/api.rst b/docs/source/api.rst index c55fa831..932d0472 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -1,20 +1,20 @@ Tanner API ========== -Tanner api provides various stats related to traffic captured by snare. It can be accessed at ``locahost:8090/api/``. +Tanner api provides various stats related to traffic captured by snare. It can be accessed at ``locahost:8092/``. -api/ +/ ~~~~ This is the index page which shows ``tanner api``. -api/snares +/snares ~~~~~~~~~~ This shows all the snares' uuid. -api/snare/ +/snare/ ~~~~~~~~~~~~~~~~~~~~~~ Replace ```` with a valid `snare-uuid` and it will show all the sessions related to that ``snare-uuid`` and their details. -api/snare-stats/ +/snare-stats/ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Replace ```` with a valid `snare-uuid` and it will show some stats. @@ -22,19 +22,21 @@ Replace ```` with a valid `snare-uuid` and it will show some stats. * Total duration for which snare remains active * Attack frequency, which shows no of sessions which face different attacks. -/api/sessions?filters= +//sessions?filters= ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This shows all the sessions' uuid which follow the filters. Filters are sepatated by ``white-space`` and name-value pair are separated by ``:``. E.g ``?filters=filter1:value1 filter2:value2``. It supports 5 filters: - * **snare_uuid** -- Sessions related to given snare. E.g ``?filters=snare_uuid:8fa6aa98-4283-4085-bfb9-a1cd3a9e56e7`` - * **peer_ip** -- Sessions with given ip. E.g ``?filters=peer_ip:127.0.0.1`` - * **user-agent** -- Sessions with given user-agent. E.g ``?filters=user-agent:Chrome`` - * **attack_type** -- Sessions with given attack type such as lfi, rfi, xss, cmd_exec, sqli. E.g ``?filters=attack_type:lfi`` - * **possible_owner** -- Sessions with given owner type such as user, tool, crawler, attacker. E.g ``?filters=possible_owner:attacker`` - * **time_interval** -- Sessions which are active during a given time-interval. E.g ``?filters=time_interval:1480560-1480580`` + * **peer_ip** -- Sessions with given ip. E.g ``peer_ip:127.0.0.1 `` + * **user-agent** -- Sessions with given user-agent. E.g ``user-agent:Chrome`` + * **attack_types** -- Sessions with given attack type such as lfi, rfi, xss, cmd_exec, sqli. E.g ``attack_types:lfi`` + * **possible_owners** -- Sessions with given owner type such as user, tool, crawler, attacker. E.g ``possible_owners:attacker`` + * **start_time** -- Sessions which started after `start_time`. E.g ``start_time:1480560`` + * **end_time** -- Sessions which ended before `end_time`. E.g ``end_time:1480560`` + +Multiple filters can be applied as ``peer_ip:127.0.0.1 start_time:1480560 possible_owners:attacker`` /api/session/ ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/config.rst b/docs/source/config.rst index 15dd0c21..8c3d58c3 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -13,6 +13,14 @@ There are 8 different sections : :Host: The host at which Tanner is running :Port: The port at which Tanner is running + * **WEB** + + :Host: The host at which Tanner Web UI is running + :Port: The port at which Tanner Web UI is running + * **API** + + :Host: The host at which Tanner API is running + :Port: The port at which Tanner API is running * **REDIS** :Host: The host address at which redis is running @@ -22,6 +30,7 @@ There are 8 different sections : * **EMULATORS** :root_dir: The root directory for emulators that need data storing such as SQLI and LFI. Data will be stored in this directory + :emulator_enabled: This tells which emulators are enabled. * **SQLI** :db_name: THe name of database used in SQLI emulator @@ -53,8 +62,12 @@ If no file is specified, following json will be used as default: 'user_dorks': '/opt/tanner/data/user_dorks.pickle', 'vdocs': '/opt/tanner/data/vdocs.json'}, 'TANNER': {'host': '0.0.0.0', 'port': 8090}, + 'WEB': {'host': '0.0.0.0', 'port': 8091}, + 'API': {'host': '0.0.0.0', 'port': 8092}, 'REDIS': {'host': 'localhost', 'port': 6379, 'poolsize': 80, 'timeout': 1}, - 'EMULATORS': {'root_dir': '/opt/tanner'}, + 'EMULATORS': {'root_dir': '/opt/tanner', + 'emulator_enabled': {'sqli': True, 'rfi': True, 'lfi': True, 'xss': True, 'cmd_exec': True} + }, 'SQLI': {'type':'SQLITE', 'db_name': 'tanner_db', 'host':'localhost', 'user':'root', 'password':'user_pass'}, 'DOCKER': {'host_image': 'busybox:latest'}, 'LOGGER': {'log_file': '/opt/tanner/tanner.log'}, diff --git a/docs/source/index.rst b/docs/source/index.rst index ae57f730..0da547ea 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -17,6 +17,8 @@ Contents: storage dorks config + api + web diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst index 819f1df6..ac32b78d 100644 --- a/docs/source/quick-start.rst +++ b/docs/source/quick-start.rst @@ -33,4 +33,14 @@ Setup and run TANNER #. Go to the tanner source directory ``cd tanner`` #. Install requirements: ``pip3 install -r requirements.txt`` #. Install tanner ``python3 setup.py install`` -#. Run TANNER: ``sudo tanner`` \ No newline at end of file +#. Run TANNER: ``sudo tanner`` + +Run Tanner Api +"""""""""""""" + +#. Run ``sudo tannerapi`` + +Run Tanner WebUI +"""""""""""""""" + +#. Run ``sudo tannerweb`` \ No newline at end of file diff --git a/docs/source/sessions.rst b/docs/source/sessions.rst index 2cd34d2b..040d120f 100644 --- a/docs/source/sessions.rst +++ b/docs/source/sessions.rst @@ -11,7 +11,7 @@ Session class accepts ``data`` as a parameter. The ``data`` came from SNARE and * **ip** -- peer ip address. * **port** -- peer port. * **user_agent** -- peer user agent. - * **sensor** -- SNARE sensor uuid. + * **snare_uuid** -- SNARE sensor uuid. * **paths** -- list of dictionaries. Contains ``path``, ``timestamp``, ``attack_type`` and SNARE ``response status``. * **sess_uuid** -- randomly generated session uuid. * **start_timestamp** -- session start time. @@ -50,7 +50,7 @@ The result contains next fields: * **peer_ip** * **peer_port** * **user_agent** - * **sensor_uuid** + * **snare_uuid** * **start_time** * **cookies** * **end_time** -- last session timestamp diff --git a/docs/source/web.rst b/docs/source/web.rst new file mode 100644 index 00000000..991af656 --- /dev/null +++ b/docs/source/web.rst @@ -0,0 +1,46 @@ +Tanner WEB +========== +Tanner WEB provides various stats related to traffic captured by snare in UI form. It can be accessed at ``locahost:8091/``. + +/ +~~~~ +This is the index page which shows ``Tanner Web``. + +/snares +~~~~~~~~~~ +This shows all the snares' uuid. Each snare object is clickable. Clicking displays the page **/snare/** + +/snare/ +~~~~~~~~~~~~~~~~~~~~~~ +Replace ```` with a valid `snare-uuid` and it will provide two options: + * **Snare-Stats** -- It will move you to **/snare-stats/** + * **Sessions** -- It will move you to **//sessions** + +/snare-stats/ +~~~~~~~~~~~~~~~~~~~~~~~~~ +This page shows some general stats about the snare + + * **No of Sessions** - Total no of sessions of the snare + * **Total Duration** - Total durations during which sessions remain active + * **Attack Frequency** - Frequency of different attacks made on the snare + +//sessions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This shows all the sessions' uuid. Each is clickable. Clicking displays **/session/** +Filters can be on the sessions using the input box and clicking the ``Apply`` button. +Filters are sepatated by ``white-space`` and name-value pair are separated by ``:``. E.g ``filter1:value1 filter2:value2``. + +It supports 5 filters: + * **peer_ip** -- Sessions with given ip. E.g ``peer_ip:127.0.0.1 `` + * **user-agent** -- Sessions with given user-agent. E.g ``user-agent:Chrome`` + * **attack_types** -- Sessions with given attack type such as lfi, rfi, xss, cmd_exec, sqli. E.g ``attack_types:lfi`` + * **possible_owners** -- Sessions with given owner type such as user, tool, crawler, attacker. E.g ``possible_owners:attacker`` + * **start_time** -- Sessions which started after `start_time`. E.g ``start_time:1480560`` + * **end_time** -- Sessions which ended before `end_time`. E.g ``end_time:1480560`` + +Multiple filters can be applied as ``peer_ip:127.0.0.1 start_time:1480560 possible_owners:attacker`` + +/session/ +~~~~~~~~~~~~~~~~~~~~~~~~ +It gives all information about the session with given uuid. Here you may find some of the text clickable such as +``peer_ip``,``possible_owners``, ``start_time``, ``end_time``, ``attack_types``. Clicking on them will display all the sessions will same attribute value. \ No newline at end of file From 29da6bd41974acadce12c3bb1608ef6ae415dcdc Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Wed, 26 Jul 2017 00:56:31 +0530 Subject: [PATCH 09/10] add api server tests (#180) --- tanner/tests/test_api_server.py | 94 +++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 tanner/tests/test_api_server.py diff --git a/tanner/tests/test_api_server.py b/tanner/tests/test_api_server.py new file mode 100644 index 00000000..896e9f17 --- /dev/null +++ b/tanner/tests/test_api_server.py @@ -0,0 +1,94 @@ +import asyncio +from unittest import mock + +from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop + +from tanner.api import server, api + +class TestAPIServer(AioHTTPTestCase): + def setUp(self): + self.serv = server.ApiServer() + + redis = mock.Mock() + redis.close = mock.Mock() + self.serv.redis_client = redis + self.serv.api = api.Api(self.serv.redis_client) + + super(TestAPIServer, self).setUp() + + def get_app(self): + app = self.serv.create_app(loop=self.loop) + return app + + @unittest_run_loop + async def test_api_index_request(self): + assert_content = {"version": 1, "response": {"message": "tanner api"}} + request = await self.client.request("GET", "/") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) + + @unittest_run_loop + async def test_api_snares_request(self): + async def mock_return_snares(): + return ["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"] + + assert_content = {"version": 1, "response": {"message": ["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"]}} + self.serv.api.return_snares = mock_return_snares + request = await self.client.request("GET", "/snares") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) + + @unittest_run_loop + async def test_api_snare_info_request(self): + async def mock_return_snare_info(snare_uuid, count): + if snare_uuid == "8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4" and count == 50: + return [{"test_sess1": "sess1_info"}, {"test_sess1": "sess2_info"}] + + assert_content = {"version": 1, "response": {"message": [{"test_sess1": "sess1_info"}, {"test_sess1": "sess2_info"}]}} + self.serv.api.return_snare_info = mock_return_snare_info + request = await self.client.request("GET", "/snare/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) + + @unittest_run_loop + async def test_api_snare_stats_request(self): + async def mock_return_snare_stats(snare_uuid): + if snare_uuid == "8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4": + return {"total_sessions": 605, "total_duration": 865.560286283493, "attack_frequency": {"sqli": 0, "lfi": 0, "xss": 0, "rfi": 0, "cmd_exec": 0}} + + assert_content = {"version": 1, "response": {"message": {"total_sessions": 605, "total_duration": 865.560286283493, "attack_frequency": {"sqli": 0, "lfi": 0, "xss": 0, "rfi": 0, "cmd_exec": 0}}}} + self.serv.api.return_snare_stats = mock_return_snare_stats + request = await self.client.request("GET", "/snare-stats/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) + + @unittest_run_loop + async def test_api_sessions_request(self): + async def mock_return_sessions(filters): + if type(filters) is dict and filters['peer_ip'] == "127.0.0.1" and \ + filters['start_time'] == 1497890400 and filters['user_agent'] == 'ngnix': + return ["f387d46eaeb1454cadf0713a4a55be49", "e85ae767b0bb4b1f91b421b3a28082ef"] + + assert_content = {"version": 1, "response": {"message": ["f387d46eaeb1454cadf0713a4a55be49", "e85ae767b0bb4b1f91b421b3a28082ef"]}} + self.serv.api.return_sessions = mock_return_sessions + request = await self.client.request("GET", "/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4/sessions?filters=peer_ip:127.0.0.1 start_time:1497890400 user_agent:ngnix") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) + + @unittest_run_loop + async def test_api_sessions_info_request(self): + async def mock_return_session_info(sess_uuid): + if sess_uuid == "4afd45d61b994d9eb3ba20faa81a45e1": + return {"test_sess1": "sess1_info"} + + assert_content = {"version": 1, "response": {"message": {"test_sess1": "sess1_info"}}} + self.serv.api.return_session_info = mock_return_session_info + request = await self.client.request("GET", "/session/4afd45d61b994d9eb3ba20faa81a45e1") + assert request.status == 200 + detection = await request.json() + self.assertDictEqual(detection, assert_content) From 50b19e8bdf0ecf8f89b95e3e37992afc638c11e7 Mon Sep 17 00:00:00 2001 From: Ravinder Nehra Date: Sat, 29 Jul 2017 14:27:25 +0530 Subject: [PATCH 10/10] Improve Web UI (#181) * fix docker freezing * make api server compatible * limit sessions display to 15 on one page and add next & previous buttons * add proper shutdown --- tanner/api/api.py | 2 +- tanner/api/server.py | 4 +++- tanner/tests/test_api_server.py | 2 +- tanner/web/server.py | 34 +++++++++++++++++++++++------- tanner/web/templates/session.html | 10 ++++----- tanner/web/templates/sessions.html | 5 +++++ tanner/web/templates/snare.html | 2 +- 7 files changed, 42 insertions(+), 17 deletions(-) diff --git a/tanner/api/api.py b/tanner/api/api.py index 6f0548c0..11f2adb0 100644 --- a/tanner/api/api.py +++ b/tanner/api/api.py @@ -88,7 +88,7 @@ async def return_sessions(self, filters): return 'Invalid filter : %s' % filter_name if match_count == len(filters): - matching_sessions.append(sess['sess_uuid']) + matching_sessions.append(sess) return matching_sessions diff --git a/tanner/api/server.py b/tanner/api/server.py index cb2439f3..0e05cf5b 100644 --- a/tanner/api/server.py +++ b/tanner/api/server.py @@ -57,7 +57,9 @@ async def handle_sessions(self, request): self.logger.error('Filter error : %s' % e) result = 'Invalid filter definition' else: - result = await self.api.return_sessions(applied_filters) + sessions = await self.api.return_sessions(applied_filters) + sess_uuids = [sess['sess_uuid'] for sess in sessions] + result = sess_uuids response_msg = self._make_response(result) return web.json_response(response_msg) diff --git a/tanner/tests/test_api_server.py b/tanner/tests/test_api_server.py index 896e9f17..c4f56eb4 100644 --- a/tanner/tests/test_api_server.py +++ b/tanner/tests/test_api_server.py @@ -71,7 +71,7 @@ async def test_api_sessions_request(self): async def mock_return_sessions(filters): if type(filters) is dict and filters['peer_ip'] == "127.0.0.1" and \ filters['start_time'] == 1497890400 and filters['user_agent'] == 'ngnix': - return ["f387d46eaeb1454cadf0713a4a55be49", "e85ae767b0bb4b1f91b421b3a28082ef"] + return [{"sess_uuid":"f387d46eaeb1454cadf0713a4a55be49"}, {"sess_uuid":"e85ae767b0bb4b1f91b421b3a28082ef"}] assert_content = {"version": 1, "response": {"message": ["f387d46eaeb1454cadf0713a4a55be49", "e85ae767b0bb4b1f91b421b3a28082ef"]}} self.serv.api.return_sessions = mock_return_sessions diff --git a/tanner/web/server.py b/tanner/web/server.py index efff1715..2bf34f17 100644 --- a/tanner/web/server.py +++ b/tanner/web/server.py @@ -43,6 +43,7 @@ async def handle_snare_stats(self, request): @aiohttp_jinja2.template('sessions.html') async def handle_sessions(self, request): snare_uuid = request.match_info['snare_uuid'] + page_id = int(request.match_info['page_id']) params = request.url.query applied_filters = {'snare_uuid': snare_uuid} try: @@ -57,14 +58,27 @@ async def handle_sessions(self, request): self.logger.error('Filter error : %s' % e) result = 'Invalid filter definition' else: - sess_uuids = await self.api.return_sessions(applied_filters) - sessions = [] - for sess_uuid in sess_uuids: - sess = await self.api.return_session_info(sess_uuid) - sessions.append(sess) - result = sessions + sessions = await self.api.return_sessions(applied_filters) + result = sessions[15*(page_id-1):15*page_id] + next_val = None + pre_val = None + if(page_id*15 <= len(sessions)): + next_val = '/{snare_uuid}/sessions/page/{page_id}'.format(snare_uuid=snare_uuid, + page_id=str(page_id + 1) + ) + if len(applied_filters) > 1: + next_val += '?filters={filters}'.format(filters=params['filters']) + if(page_id > 1): + pre_val = '/{snare_uuid}/sessions/page/{page_id}'.format(snare_uuid=snare_uuid, + page_id=str(page_id - 1) + ) + if len(applied_filters) > 1: + pre_val += '?filters={filters}'.format(filters=params['filters']) + return { - 'sessions' : result + 'sessions' : result, + 'next_val' : next_val, + 'pre_val' : pre_val } @aiohttp_jinja2.template('session.html') @@ -75,19 +89,23 @@ async def handle_session_info(self, request): 'session' : session } + async def on_shutdown(self, app): + self.redis_client.close() + def setup_routes(self, app): app.router.add_get('/', self.handle_index) app.router.add_get('/snares', self.handle_snares) app.router.add_resource('/snare/{snare_uuid}').add_route('GET', self.handle_snare) app.router.add_resource('/snare-stats/{snare_uuid}').add_route('GET', self.handle_snare_stats) app.router.add_resource('/session/{sess_uuid}').add_route('GET', self.handle_session_info) - app.router.add_resource('/{snare_uuid}/sessions').add_route('GET', self.handle_sessions) + app.router.add_resource('/{snare_uuid}/sessions/page/{page_id}').add_route('GET', self.handle_sessions) app.router.add_static('/static/', path='tanner/web/static') def create_app(self, loop): app = web.Application(loop= loop) aiohttp_jinja2.setup(app, loader= jinja2.FileSystemLoader('tanner/web/templates')) + app.on_shutdown.append(self.on_shutdown) self.setup_routes(app) return app diff --git a/tanner/web/templates/session.html b/tanner/web/templates/session.html index d7751bd5..ee0b5386 100644 --- a/tanner/web/templates/session.html +++ b/tanner/web/templates/session.html @@ -14,7 +14,7 @@

SESSION INFO

 - + @@ -30,11 +30,11 @@

SESSION INFO

- + - + @@ -60,7 +60,7 @@

SESSION INFO

@@ -84,7 +84,7 @@

SESSION INFO

diff --git a/tanner/web/templates/sessions.html b/tanner/web/templates/sessions.html index 7efb278b..e20930bd 100644 --- a/tanner/web/templates/sessions.html +++ b/tanner/web/templates/sessions.html @@ -25,6 +25,11 @@

SNARE-SESSIONS

{% endfor %}
NoSnare-uuid
{{loop.index}}{{snare}}
IP{{session.peer_ip}}{{session.peer_ip}}
Port
Start time{{session.start_time}}{{session.start_time}}
End time{{session.end_time}}{{session.end_time}}
Requests/sec Attack types {% for attack in session.attack_types %} - {{attack}}
+ {{attack}}
{% endfor %}
Possible Owners {% for owner in session.possible_owners %} - {{owner}}
+ {{owner}}
{% endfor %}
+
+
+ {% if pre_val %}Previous{% endif %} + {% if next_val %}Next{% endif %} +
{% endblock %} \ No newline at end of file diff --git a/tanner/web/templates/snare.html b/tanner/web/templates/snare.html index a44a259c..6225276c 100644 --- a/tanner/web/templates/snare.html +++ b/tanner/web/templates/snare.html @@ -3,5 +3,5 @@ {% block content %}

{{snare}}

Snare-Stats

-

Sessions

+

Sessions

{% endblock %} \ No newline at end of file