-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathword_doc_graph.py
116 lines (102 loc) · 3.81 KB
/
word_doc_graph.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
import math
def word_coccurrance(df, size = 20):
# word co-occurence with context windows
window_size = size
windows = []
for doc_words in df['Tokens']:
words = doc_words
length = len(words)
if length <= window_size:
windows.append(words)
else:
for j in range(length - window_size + 1):
window = words[j: j + window_size]
windows.append(window)
return windows
def windows_frequency(windows):
word_window_freq = {}
for window in windows:
appeared = set()
for i in range(len(window)):
if window[i] in appeared:
continue
if window[i] in word_window_freq:
word_window_freq[window[i]] += 1
else:
word_window_freq[window[i]] = 1
appeared.add(window[i])
return word_window_freq
def words_pair_count(windows, word_id_map):
word_pair_count = {}
for window in windows:
for i in range(1, len(window)):
for j in range(0, i):
word_i = window[i]
word_i_id = word_id_map[word_i]
word_j = window[j]
word_j_id = word_id_map[word_j]
if word_i_id == word_j_id:
continue
word_pair_str = str(word_i_id) + ',' + str(word_j_id)
if word_pair_str in word_pair_count:
word_pair_count[word_pair_str] += 1
else:
word_pair_count[word_pair_str] = 1
# two orders
word_pair_str = str(word_j_id) + ',' + str(word_i_id)
if word_pair_str in word_pair_count:
word_pair_count[word_pair_str] += 1
else:
word_pair_count[word_pair_str] = 1
return word_pair_count
def pmi_calculator(windows, word_pair_count, word_window_freq, vocab, train_size):
row = []
col = []
weight = []
# pmi as weights
num_window = len(windows)
for key in word_pair_count:
temp = key.split(',')
i = int(temp[0])
j = int(temp[1])
count = word_pair_count[key]
word_freq_i = word_window_freq[vocab[i]]
word_freq_j = word_window_freq[vocab[j]]
pmi = math.log((1.0 * count / num_window) /
(1.0 * word_freq_i * word_freq_j / (num_window * num_window)))
if pmi <= 0:
continue
row.append(train_size + i)
col.append(train_size + j)
weight.append(pmi)
return row, col, weight
def doc_word_frequency(df, word_id_map, train_size, row, vocab_size, col, weight, word_doc_freq, vocab):
doc_word_freq = {}
for doc_id in range(len(df['Tokens'])):
words = df['Tokens'][doc_id]
for word in words:
word_id = word_id_map[word]
doc_word_str = str(doc_id) + ',' + str(word_id)
if doc_word_str in doc_word_freq:
doc_word_freq[doc_word_str] += 1
else:
doc_word_freq[doc_word_str] = 1
for i in range(len(df['Tokens'])):
words = df['Tokens'][i]
doc_word_set = set()
for word in words:
if word in doc_word_set:
continue
j = word_id_map[word]
key = str(i) + ',' + str(j)
freq = doc_word_freq[key]
if i < train_size:
row.append(i)
else:
row.append(i + vocab_size)
col.append(train_size + j)
idf = math.log(1.0 * len(df['Tokens']) /
word_doc_freq[vocab[j]])
weight.append(freq * idf)
doc_word_set.add(word)
return doc_word_freq