Centering Resonance Analysis - For Chinese Text - Public.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# set working directory\n",
    "get_ipython().magic(u'pwd')\n",
    "get_ipython().magic(u'cd \"C:/Users/xxxxx\"')\n",
    "\n",
    "import os\n",
    "import glob\n",
    "from glob import glob as gg\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "import re\n",
    "import sqlite3\n",
    "\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "from pandas import Series\n",
    "\n",
    "import numpy as np\n",
    "from numpy import random, asarray, linspace, corrcoef\n",
    "from datetime import datetime, date, time\n",
    "\n",
    "import jieba\n",
    "import jieba.posseg as pseg\n",
    "import jieba.analyse\n",
    "from optparse import OptionParser\n",
    "from __future__ import print_function, unicode_literals\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('max_colwidth',300)\n",
    "\n",
    "#load SQlite database\n",
    "con = sqlite3.connect(\"OccupyHK.sqlite\")\n",
    "cur = con.cursor()\n",
    "df = pd.read_sql_query(\"SELECT * FROM Posts WHERE published_date BETWEEN '2014-12-01' AND '2014-12-31'\", con)\n",
    "con.close()\n",
    "\n",
    "# using the example of Occupy Central Facebook pages\n",
    "df_yellow_official = df.loc[df['org_name'].isin(['學民思潮 Scholarism',\n",
    "                                                 'Occupy Central 佔領中環', \n",
    "                                                 '讓愛與和平佔領中環 Occupy Central with Love and Peace', \n",
    "                                                 '926 平民在政總現場','佔領期間 民間智慧選輯', '傘下爸媽',\n",
    "                                                 '香港天主教正義和平委員會','Hong Kong Democracy Now', '踢爆五毛','香港警權關注組',\n",
    "                                                 '維護香港核心價值', '我係香港人', '香港人 Secrets','保衛香港自由聯盟','9反佔中 HK oic',\n",
    "                                                 'Fluid Occupiers'])] \n",
    "\n",
    "df_yellow_all = df.loc[df['feed_id'].isin([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17])] \n",
    "df_yellow_nonofficial = df_yellow_all[~df_yellow_all.org_name.isin(['學民思潮 Scholarism', \n",
    "                                                                    'Occupy Central 佔領中環', \n",
    "                                                                    '讓愛與和平佔領中環 Occupy Central with Love and Peace', \n",
    "                                                                    '926 平民在政總現場','佔領期間 民間智慧選輯', '傘下爸媽',\n",
    "                                                                    '香港天主教正義和平委員會','Hong Kong Democracy Now', '踢爆五毛','香港警權關注組',\n",
    "                                                                    '維護香港核心價值', '我係香港人', '香港人 Secrets','保衛香港自由聯盟',\n",
    "                                                                    '9反佔中 HK oic','Fluid Occupiers'])]\n",
    "\n",
    "df_blue_official = df.loc[df['org_name'].isin(['幫港出聲 Silent Majority','Speak Out HK 港人講地','藍絲帶運動 The Blue Ribbon Movement','向香港警察致敬',\n",
    "                                               '反佔中苦主大聯盟','泛民不代表我','聲討黃之鋒',\n",
    "                                               '一人一Like救學生，反對中學生衝擊警察及佔中！',\n",
    "                                               '「保普選 反佔中」簽左名揮手區！',\n",
    "                                               '我要真普洱行動','我撐警務處長曾偉雄！',\n",
    "                                               '我係大學生，我愛國愛港'])] \n",
    "\n",
    "\n",
    "df_blue_all = df.loc[df['feed_id'].isin([20,21,22,23,24,25,26,27,28,29,30,31])] \n",
    "\n",
    "df_blue_nonofficial = df_blue_all[~df_blue_all.org_name.isin(['幫港出聲 Silent Majority','Speak Out HK 港人講地','藍絲帶運動 The Blue Ribbon Movement','向香港警察致敬',\n",
    "                                               '反佔中苦主大聯盟','泛民不代表我','聲討黃之鋒',\n",
    "                                               '一人一Like救學生，反對中學生衝擊警察及佔中！',\n",
    "                                               '「保普選 反佔中」簽左名揮手區！',\n",
    "                                               '我要真普洱行動','我撐警務處長曾偉雄！',\n",
    "                                               '我係大學生，我愛國愛港'])]\n",
    "\n",
    "print (\"There are\",len(df_yellow_official), \"OFFICIAL posts sent by anti-establishment civil groups\")\n",
    "print (\"There are\",len(df_yellow_nonofficial), \"UNOFFICIAL posts sent by anti-establishment civil groups\")\n",
    "print (\"There are\",len(df_yellow_all), \"posts sent by anti-establishment civil groups\")\n",
    "print (\"There are\",len(df_blue_official), \"OFFICIAL posts sent by pro-establishment civil groups\")\n",
    "print (\"There are\",len(df_blue_nonofficial), \"UNOFFICIAL posts sent by pro-establishment civil groups\")\n",
    "print (\"There are\",len(df_blue_all), \"posts sent by pro-establishment civil groups\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#convert to corpus\n",
    "def convert2corpus(df):\n",
    "    corpus = []\n",
    "    for index, i in df.iterrows():\n",
    "        corpus.append(i['content'])\n",
    "    corpus = str(corpus)\n",
    "    corpus = corpus.replace(\"[\" and \"]\", \" \")\n",
    "    corpus = corpus.replace(\"\\\\\", \" \")\n",
    "    return corpus\n",
    "\n",
    "processed_corpus = convert2corpus(df_yellow_official) \n",
    "print (\"the length of the processed corpus is:\",len(processed_corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def cnpuncttoen(corpus):\n",
    "    #corpus_new = corpus.replace(\".\", \"\")\n",
    "    corpus_new = corpus.replace(\"。\", \"*.\")\n",
    "    corpus_new = corpus_new.replace(\"？\", \"*?\")\n",
    "    corpus_new = corpus_new.replace(\",\", \"*,\")\n",
    "    corpus_new = corpus_new.replace(\"！\", \"*!\")\n",
    "    return corpus_new\n",
    "    \n",
    "processed_corpus_clean = cnpuncttoen(processed_corpus) \n",
    "#processed_corpus_clean[500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# segment Chinese sentences see http://blog.ourren.com/2015/02/21/nltk_note_token_sentence/\n",
    "\n",
    "#tokenize into sentences\n",
    "def sentence_split(str_centence):\n",
    "    list_ret = list()\n",
    "    for s_str in str_centence.split('*.'):\n",
    "        if '*?' in s_str:\n",
    "            list_ret.extend(s_str.split('*?'))\n",
    "        elif '*!' in s_str:\n",
    "            list_ret.extend(s_str.split('*!'))\n",
    "        else:\n",
    "            list_ret.append(s_str)\n",
    "    return list_ret\n",
    "\n",
    "processed_corpus_sent = sentence_split(processed_corpus_clean)\n",
    "processed_corpus_sent[20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# POS Tagging\n",
    "# apply stop word list\n",
    "jieba.analyse.set_stop_words('C:/Users/xxxxx/cn_stopword.txt')\n",
    "\n",
    "pos_corpus_sent = [[(j.word, j.flag) for j in jieba.posseg.cut(i)] for i in processed_corpus_sent]\n",
    "#pos_corpus_sent[113]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# select only nouns and adjectives\n",
    "matched_pos_corpus_sent = [[token for token, tag in sent if re.match('n|a', tag) and len(token)>1] \n",
    "                for sent in pos_corpus_sent]\n",
    "\n",
    "print (\"the length of matched corpus containing nouns and adjectives is:\", len(matched_pos_corpus_sent))\n",
    "#matched_pos_corpus_sent[113]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create a semantic network with nouns and adjectives as nodes. More refer to http://www.sociology-hacks.org/?p=151\n",
    "\n",
    "import itertools as it\n",
    "edgelist = [edge for phrase in matched_pos_corpus_sent for edge in it.combinations(phrase, 2)]   \n",
    "\n",
    "import networkx as nx\n",
    "word_network = nx.Graph(edgelist)\n",
    "word_network_centrality = nx.betweenness_centrality(word_network, normalized=True)\n",
    "sorted_centrality = sorted(word_network_centrality.items(), key=lambda x:x[1], reverse=True)\n",
    "for word, centr in sorted_centrality[:30]:\n",
    "        print (word)\n",
    "        print (centr)\n",
    "\n",
    "print (nx.number_of_edges(word_network))\n",
    "print (nx.number_of_nodes(word_network))\n",
    "\n",
    "nx.write_graphml(word_network, \"word_network_official_yellow_2014_12.graphml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get simple resonance score to compare similarity of two texts\n",
    "\n",
    "def simple_resonance(index_a, index_b):\n",
    "    sorted_a = sorted(index_a.items(), key=lambda x:x[1], reverse=True)\n",
    "    sorted_b = sorted(index_b.items(), key=lambda x:x[1], reverse=True)\n",
    "    scores = [a[1]*b[1] for a, b in zip(sorted_a, sorted_b) if a[0] == b[0]]\n",
    "    return sum(scores)\n",
    "\n",
    "simple_resonance(word_network_centrality, word_network_centrality_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#get standardized resonance score to compare similarity of two texts\n",
    "\n",
    "import math\n",
    "def standardized_sr(index_a, index_b):\n",
    "    sum_a_squared = sum([centr**2 for word, centr in index_a.items()])\n",
    "    sum_b_squared = sum([centr**2 for word, centr in index_b.items()])\n",
    "    norm = math.sqrt((sum_a_squared * sum_b_squared))\n",
    "    standardized = simple_resonance(index_a, index_b) / norm\n",
    "    return standardized\n",
    "\n",
    "standardized_sr(word_network_centrality, word_network_centrality_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.3.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}