唐诗常用高频字分析（文本预处理，筛选中文，dict按值排序）

[ywsswy]

#!/usr/bin/env python3
filename_tangpoetry = 'TangPoetry.txt'

# pre-processing( read from file & cut single word into list
def PreProcessing(filename, total_line):
	word_list = []
	with open(filename, 'r', encoding='utf-8') as f:
		line = 1
		data = f.readlines()
		for i in data:
			for j in i:
				# judge chinese word.(https://blog.csdn.net/zhenyu5211314/article/details/51537778)
				if ord(j) >= 13312 and ord(j) <= 40895:
					word_list.append(j)
			DrawProgressbar(line/total_line)
			line = line + 1
	return word_list
	
# count each word
def CountWord(word_list):
	word_dict = {}
	for i in word_list:
		try:
			word_dict[i] = word_dict[i] + 1
		except:
			word_dict[i] = 1
	return word_dict

		

# get the total number of lines of a file
def CountLine(filename):
	count=0
	thefile=open(filename_tangpoetry)
	while True:
		buffer=thefile.read(1024*8192)
		if not buffer:
			break
		count+=buffer.count('\n')
	thefile.close()
	return count

# draw the ProgressBar
def DrawProgressbar(percent):
	length = 3
	num_ok = int(percent * 100 / 3)
	num_no = (int(100/3) - num_ok)
	print('\r {:>5.1f}%% [{}{}]'.format(percent*100, '◼' * num_ok, '◻' * num_no), end='')
	

def Main():
	print('start preprocessing')
	total_line = CountLine(filename_tangpoetry)
	word_list = PreProcessing(filename_tangpoetry, total_line)
	# print(word_list)
	word_dict = CountWord(word_list)
	# print(word_dict)
	sorted_list = sorted(word_dict.items(), key = lambda word_dict:word_dict[1])
	# the highest frequency word
	print(sorted_list[-150:])

Main()