ctu2pic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pandas.core.frame import DataFrame\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import datetime\n",
    "import binascii\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_divided_groups(sorted_dataframe: pd.DataFrame, file_num_list: list) -> list: # return with grouped groups as one list, and get their labels\n",
    "    filename_list = []\n",
    "    itr = 0\n",
    "    _filname_list = np.array(sorted_dataframe.loc[:, ('SrcAddr', 'Sport', 'DstAddr', 'Dport', 'StartTime')]).tolist()\n",
    "    for count in file_num_list:\n",
    "        _filelist = _filname_list[itr:itr + count]\n",
    "        filename_list.append(_filelist)\n",
    "        itr += count\n",
    "    return filename_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_filelist_toread(dir_of_one_proto):\n",
    "    _filelist = os.listdir(dir_of_one_proto)\n",
    "#             filelist = []\n",
    "    ###### get a dataframe of all split filename for further processing\n",
    "    filelist = [os.path.splitext(filename)[0] for filename in _filelist]\n",
    "    split_filename = [filename.split('_', 4) for filename in filelist]\n",
    "    filename_df = pd.DataFrame(split_filename)\n",
    "    filename_df.rename(columns={0: 'SrcAddr', 1: 'Sport',\n",
    "                                2: 'DstAddr', 3: 'Dport',\n",
    "                                4:'StartTime'}, inplace=True)\n",
    "    \n",
    "    # temp GroupByObject of data grouped by ['SrcAddr', 'Sport', 'DstAddr', 'Dport']\n",
    "    divided_group_object = filename_df.groupby(['SrcAddr', 'Sport', 'DstAddr', 'Dport'], sort = False)\n",
    "\n",
    "    # sort data in each group by 'StartTime'\n",
    "    processed_dataflow = divided_group_object.apply(lambda x:x.sort_values('StartTime', ascending = True))\\\n",
    "                                                .reset_index(drop = True)\n",
    "#     processed_dataflow =  divided_group_object.apply(lambda x: x).reset_index(drop = True)\n",
    "\n",
    "    # data number in different groups, used to extract ordered groups from processed_dataflow\n",
    "    processed_group_num = np.array(divided_group_object['SrcAddr'].count()).tolist()\n",
    "    split_filenamelist_in_list = extract_divided_groups(processed_dataflow, processed_group_num)\n",
    "    \n",
    "    ###### pack all filenames with same ['SrcAddr', 'Sport', 'DstAddr', 'Dport'] in one list\n",
    "    ###### each list was sorted by 'StartTime' inside, and pack all filename lists in one list\n",
    "    filenamelist_in_list = []\n",
    "    for split_str_list in split_filenamelist_in_list:\n",
    "        filename_list = [] # filename in one flow\n",
    "        for split_str in split_str_list:\n",
    "            filename = '_'.join(split_str)\n",
    "            filename_list.append(dir_of_one_proto + filename + '.pcap')\n",
    "        filenamelist_in_list.append(filename_list)\n",
    "    \n",
    "    return filenamelist_in_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pcap2hex(filename) -> bytes :\n",
    "    if os.path.exists(filename):\n",
    "        with open(filename, 'rb') as reader:\n",
    "            reader.seek(24, 1) # jump over pcap head\n",
    "            binflow = reader.read()\n",
    "            hexstr = binascii.b2a_hex(binflow)\n",
    "    # no such file\n",
    "    else: return b'' # this is a bytes type string!!!!!!!\n",
    "    return hexstr\n",
    "\n",
    "def readfile_bylist(one_flow_file_list: list): # find pcap file by generated filename\n",
    "    picstr = b'' # bytes string to be written in picture\n",
    "    for filename in one_flow_file_list:\n",
    "        hexstr = pcap2hex(filename)\n",
    "        \n",
    "        if hexstr != b'' and len(picstr) < 2048: # picstr length is more than 1024B\n",
    "            picstr += hexstr\n",
    "        elif hexstr == b'' and len(picstr) < 2048:\n",
    "            continue # no such file matches with generated filename\n",
    "        else: break\n",
    "            \n",
    "    # picstr length >= 2048\n",
    "    if len(picstr) >= 2048:\n",
    "        picstr = picstr[:2048]\n",
    "    else:\n",
    "        picstr = picstr.ljust(2048, b'0')\n",
    "        \n",
    "    return picstr\n",
    "\n",
    "def hex2pic(picstr):\n",
    "    pic_array = np.array([int(picstr[i:i+2], 16) for i in range(0, 2048, 2)]).reshape(32, 32)\n",
    "    img = Image.fromarray(np.uint8(pic_array))\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_label_bypath(file_path):\n",
    "    if 'CTU-13.Dataset-42' in file_path or 'CTU-13.Dataset-43' in file_path \\\n",
    "            or 'CTU-13.Dataset-50' in file_path:\n",
    "        return 'neris'\n",
    "    elif 'CTU-13.Dataset-44' in file_path or 'CTU-13.Dataset-45' in file_path \\\n",
    "            or 'CTU-13.Dataset-51' in file_path or 'CTU-13.Dataset-52' in file_path:\n",
    "        return 'rbot'\n",
    "    elif 'CTU-13.Dataset-46' in file_path or 'CTU-13.Dataset-54' in file_path:\n",
    "        return 'virut'\n",
    "    elif 'CTU-13.Dataset-47' in file_path:\n",
    "        return 'menti'\n",
    "    elif 'CTU-13.Dataset-48' in file_path:\n",
    "        return 'sogou'\n",
    "    elif 'CTU-13.Dataset-49' in file_path:\n",
    "        return 'murlo'\n",
    "    elif 'CTU-13.Dataset-53' in file_path:\n",
    "        return 'nsis.ay'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no such directory: E:\\1IDM\\CTU-13-Backup\\CTU-13.Dataset-48\\botnet\\tcp_nosyn\\\n",
      "no such directory: E:\\1IDM\\CTU-13-Backup\\CTU-13.Dataset-51\\botnet\\udp\\\n",
      "no such directory: E:\\1IDM\\CTU-13-Backup\\CTU-13.Dataset-51\\botnet\\tcp_nosyn\\\n",
      "no such directory: E:\\1IDM\\CTU-13-Backup\\CTU-13.Dataset-51\\botnet\\tcp_syn\\\n",
      "no such directory: E:\\1IDM\\CTU-13-Backup\\CTU-13.Dataset-52\\botnet\\tcp_nosyn\\\n",
      "no such directory: E:\\1IDM\\CTU-13-Backup\\CTU-13.Dataset-53\\botnet\\tcp_nosyn\\\n"
     ]
    }
   ],
   "source": [
    "# dirpath = '/mnt/e/1IDM/CTU-13-Backup/'\n",
    "dirpath = 'E:\\\\1IDM\\\\CTU-13-Backup\\\\'\n",
    "    \n",
    "dataset_list = ['CTU-13.Dataset-42', 'CTU-13.Dataset-43', 'CTU-13.Dataset-44', \n",
    "                    'CTU-13.Dataset-45', 'CTU-13.Dataset-46', 'CTU-13.Dataset-47', \n",
    "                    'CTU-13.Dataset-48', 'CTU-13.Dataset-49', 'CTU-13.Dataset-50',\n",
    "                    'CTU-13.Dataset-51', 'CTU-13.Dataset-52', 'CTU-13.Dataset-53'] # 'CTU-13.Dataset-51' is skipped\n",
    "# proto_list = ['/botnet/udp/', '/botnet/tcp_nosyn/', '/botnet/tcp_syn/'] # skip 'icmp' or other protos\n",
    "proto_list = ['\\\\botnet\\\\udp\\\\', '\\\\botnet\\\\tcp_nosyn\\\\', '\\\\botnet\\\\tcp_syn\\\\'] # skip 'icmp' or other protos\n",
    "\n",
    "all_filename_lists = []\n",
    "    \n",
    "for dataset in dataset_list:\n",
    "    for proto in proto_list:\n",
    "        if os.path.exists(dirpath + dataset + proto):\n",
    "#             print('Processing directory: ' + dirpath + dataset + proto)\n",
    "            filename_toread = generate_filelist_toread(dirpath + dataset + proto)\n",
    "            all_filename_lists.extend(filename_toread)\n",
    "#             print('Processed directory: ' + dirpath + dataset + proto)\n",
    "        else: print('no such directory: ' + dirpath + dataset + proto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 0 pictures\n",
      "2020-03-30 10:44:46.095266\n",
      "Processed 5000 pictures\n",
      "2020-03-30 10:45:41.847674\n",
      "Processed 10000 pictures\n",
      "2020-03-30 10:46:24.802273\n",
      "Processed 15000 pictures\n",
      "2020-03-30 10:47:18.288596\n",
      "Processed 20000 pictures\n",
      "2020-03-30 10:47:58.899513\n",
      "Processed 25000 pictures\n",
      "2020-03-30 10:48:38.549273\n",
      "Processed 30000 pictures\n",
      "2020-03-30 10:49:24.386333\n",
      "Processed 35000 pictures\n",
      "2020-03-30 10:50:10.232309\n",
      "Processed 40000 pictures\n",
      "2020-03-30 10:50:27.870307\n",
      "Processed 45000 pictures\n",
      "2020-03-30 10:50:45.806343\n",
      "Processed 50000 pictures\n",
      "2020-03-30 10:51:07.906655\n",
      "Processed 55000 pictures\n",
      "2020-03-30 10:51:32.113678\n",
      "Processed 60000 pictures\n",
      "2020-03-30 10:51:54.164654\n",
      "Processed 65000 pictures\n",
      "2020-03-30 10:53:11.077475\n",
      "Processed 70000 pictures\n",
      "2020-03-30 10:54:16.475888\n",
      "Processed 75000 pictures\n",
      "2020-03-30 10:55:07.196539\n",
      "Processed 80000 pictures\n",
      "2020-03-30 10:56:02.150497\n",
      "Processed 85000 pictures\n",
      "2020-03-30 11:00:08.106812\n",
      "Processed 90000 pictures\n",
      "2020-03-30 11:03:06.892853\n",
      "Processed 95000 pictures\n",
      "2020-03-30 11:06:35.982131\n",
      "Processed 100000 pictures\n",
      "2020-03-30 11:09:47.639167\n",
      "Processed 105000 pictures\n",
      "2020-03-30 11:12:57.163944\n",
      "Processed 110000 pictures\n",
      "2020-03-30 11:15:50.303780\n",
      "Processed 115000 pictures\n",
      "2020-03-30 11:18:27.171573\n",
      "Processed 120000 pictures\n",
      "2020-03-30 11:21:13.513345\n",
      "Processed 125000 pictures\n",
      "2020-03-30 11:23:46.699448\n",
      "Processed 130000 pictures\n",
      "2020-03-30 11:26:47.412911\n",
      "Processed 135000 pictures\n",
      "2020-03-30 11:29:32.027770\n",
      "Processed 140000 pictures\n",
      "2020-03-30 11:32:28.941195\n",
      "Processed 145000 pictures\n",
      "2020-03-30 11:35:11.126386\n",
      "Processed 150000 pictures\n",
      "2020-03-30 11:37:58.371234\n",
      "Processed 155000 pictures\n",
      "2020-03-30 11:41:01.334698\n",
      "Processed 160000 pictures\n",
      "2020-03-30 11:43:56.417692\n",
      "Processed 165000 pictures\n",
      "2020-03-30 11:46:51.863895\n",
      "Processed 170000 pictures\n",
      "2020-03-30 11:49:23.386986\n",
      "Processed 175000 pictures\n",
      "2020-03-30 11:52:53.256867\n",
      "Processed 179660 pictures at 2020-03-30 11:57:43.839140\n"
     ]
    }
   ],
   "source": [
    "# imagelist = []\n",
    "labellist = []\n",
    "# image_path = '/mnt/e/1IDM/CTU-13-Backup/grayscale/'\n",
    "image_path = 'E:\\\\1IDM\\\\CTU-13-Backup\\\\grayscale\\\\'\n",
    "if os.path.exists(image_path) == False:\n",
    "    os.mkdir(image_path)\n",
    "\n",
    "assert len(os.listdir(image_path)) == 0\n",
    "\n",
    "image_num = len(all_filename_lists)\n",
    "\n",
    "for cnt, filename_list in enumerate(all_filename_lists):\n",
    "    if cnt % 5000 == 0:\n",
    "        print('Processed %s pictures' % cnt)\n",
    "        print(datetime.datetime.now())\n",
    "        \n",
    "    picture_str = readfile_bylist(filename_list)\n",
    "    im = hex2pic(picture_str)\n",
    "#     imagelist.append(im)\n",
    "    im_name = os.path.splitext(os.path.basename(filename_list[0]))[0]\n",
    "    im.save(image_path + im_name + '.bmp')\n",
    "    label = get_label_bypath(filename_list[0])\n",
    "    labellist.append([im_name, label])\n",
    "\n",
    "#     if picture_str != b''.ljust(2048, b'0'):\n",
    "#         im = hex2pic(picture_str)\n",
    "#         imagelist.append(im)\n",
    "#         labellist.append(all_data_label[i])\n",
    "\n",
    "assert len(os.listdir(image_path)) == len(labellist)\n",
    "print('Processed {0} pictures at {1}'.format(image_num, datetime.datetime.now()))\n",
    "\n",
    "label_df = pd.DataFrame(labellist, columns = ['PicName', 'Label'])\n",
    "# label_df.to_csv('/mnt/e/1IDM/CTU-13-Backup/label.csv', encoding = 'utf-8')\n",
    "label_df.to_csv('E:\\\\1IDM\\\\CTU-13-Backup\\\\label.csv', index = 0, encoding = 'utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}