GFW Visualization - Public - pt4 - Interactive Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Wayne\\Dropbox\\Acer Laptop Sync\\Data Science\\GFW Visualization\n",
      "Lightning initialized\n",
      "Connected to server at http://gfwvisualization.herokuapp.com\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Session\n",
       "number: a6248656-40ff-4d19-86ea-927c722c5a78name: gfw_public_vis1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_ipython().magic(u'pwd')\n",
    "get_ipython().magic(u'cd \"C:/Users/Wayne/Dropbox/Acer Laptop Sync/Data Science/GFW Visualization\"')\n",
    "\n",
    "import sqlite3\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "import numpy as np\n",
    "from numpy import random, asarray, linspace, corrcoef\n",
    "from pandas import Series\n",
    "from datetime import datetime, date, time\n",
    "import community\n",
    "import networkx as nx\n",
    "#import jupyter_client\n",
    "#import ipykernel\n",
    "from colorsys import hsv_to_rgb\n",
    "\n",
    "import operator\n",
    "\n",
    "from lightning import Lightning\n",
    "import tsm\n",
    "from twarc import Twarc\n",
    "\n",
    "lgn = Lightning(host=\"http://gfwvisualization.herokuapp.com/\")\n",
    "lgn.create_session()\n",
    "lgn.create_session(\"gfw_public_vis1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Edge list created.\n",
      "Data saved to file \"rt_edgelist.csv\".\n",
      "Your export file is \"rt_edgelist.csv\".\n",
      "Data loaded.\n",
      "Non-directed network created.\n",
      "Community partition complete.\n",
      "Total n of communities: 113\n",
      "Modularity: 0.56\n",
      "Community analysis complete. The top 10 communities in this network account for 64.89 % of all nodes.\n",
      "And 59.33 % of all edges.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<lightning.types.plots.Force at 0x9ead390>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rt_edge_list = tsm.t2e('rawtweets.csv','RTS_ONLY',save_prefix='rt')\n",
    "\n",
    "\n",
    "# Retweet network \n",
    "EDGELIST_matrix = pd.read_csv(\"rt_edgelist.csv\", header = 0)\n",
    "EG = nx.DiGraph()\n",
    "for index, tweet in EDGELIST_matrix.iterrows():\n",
    "    EG.add_edge(tweet[0],tweet[1])\n",
    "\n",
    "#get top nodes\n",
    "g = EG.degree()\n",
    "sorted_nodes = sorted(g.items(), key=operator.itemgetter(1), reverse = True)\n",
    "sorted_nodes = sorted_nodes[0:1000]\n",
    "topnodes = [i[0] for i in sorted_nodes]\n",
    "\n",
    "#get TSM measures\n",
    "partitioned_network = tsm.get_top_communities(rt_edge_list)\n",
    "viewlist = partitioned_network.node_list\n",
    "nodelist = [i[0] for i in viewlist]\n",
    "community_grouping = [int(i[1]) for i in viewlist]\n",
    "in_degree = [int(i[2]) for i in viewlist]\n",
    "    \n",
    "GFW_vis1 = nx.adjacency_matrix(EG, nodelist = topnodes, weight='weight').todense()\n",
    "lgn.force(GFW_vis1, group = community_grouping, labels = in_degree, size=(asarray(in_degree) + 0.7)*.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Edge list created.\n",
      "Data saved to file \"reply_edgelist.csv\".\n",
      "Your export file is \"reply_edgelist.csv\".\n",
      "Data loaded.\n",
      "Non-directed network created.\n",
      "Community partition complete.\n",
      "Total n of communities: 633\n",
      "Modularity: 0.83\n",
      "Community analysis complete. The top 10 communities in this network account for 40.97 % of all nodes.\n",
      "And 52.8 % of all edges.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<lightning.types.plots.Force at 0x9ebca58>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reply_edge_list = tsm.t2e('rawtweets.csv','REPLIES_ONLY',save_prefix='reply')\n",
    "\n",
    "# Retweet network \n",
    "EDGELIST_matrix_reply = pd.read_csv(\"reply_edgelist.csv\", header = 0)\n",
    "EG_reply = nx.DiGraph()\n",
    "for index, tweet in EDGELIST_matrix_reply.iterrows():\n",
    "    EG_reply.add_edge(tweet[0],tweet[1])\n",
    "\n",
    "#get top nodes\n",
    "g_reply = EG_reply.degree()\n",
    "sorted_nodes = sorted(g_reply.items(), key=operator.itemgetter(1), reverse = True)\n",
    "sorted_nodes = sorted_nodes[0:1000]\n",
    "topnodes = [i[0] for i in sorted_nodes]\n",
    "\n",
    "#get TSM measures\n",
    "partitioned_network = tsm.get_top_communities(reply_edge_list)\n",
    "viewlist = partitioned_network.node_list\n",
    "nodelist = [i[0] for i in viewlist]\n",
    "community_grouping = [int(i[1]) for i in viewlist]\n",
    "in_degree = [int(i[2]) for i in viewlist]  \n",
    "GFW_vis2 = nx.adjacency_matrix(EG_reply, nodelist = topnodes, weight='weight').todense()\n",
    "lgn.force(GFW_vis2, group = community_grouping, labels = in_degree, size=(asarray(in_degree) + 0.7)*.1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.3.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}