#!/usr/bin/env python
# -*- coding:utf-8 -*-
import numpy as np
import pandas as pd
import re
df = pd.read_csv("HillaryEmails.csv")
df = df[[\'Id\',\'ExtractedBodyText\']].dropna()#保留这两个信息，其他的扔掉
#文本预处理
def clean_email_text(text):
text = text.replace(\'/n\'," ")#去掉新行
text = re.sub(r\'-\',\' \',text)
text = re.sub(r"\d+/\d+/\d+","",text)
text = re.sub(r"[0-2]?[0-9]:[0-6][0-9]","",text)
text = re.sub(r"[\w]+@[\.\w]+","",text)
text = re.sub(r"/[a-zA-Z]*[:\//\]*[A-Za-z0-9\-_]+\.+[A-Za-z0-9]\.\/"
r"%&=\?\-_]+/i","",text)
pure_text = \'\'
for letter in text:
if letter.isalpha() or letter==\' \': #只留下字母和空格
pure_text +=letter
#去除落单的单词
text = \' \'.join(word for word in pure_text.split() if len(word)>1)
return text
#新建一个colum，把方法跑一遍
docs = df[\'ExtractedBodyText\']
docs = docs.apply(lambda s: clean_email_text(s))
print(docs.head(1).values)
doclist = docs.values
#引入库
from gensim import corpora,models,similarities
import gensim
stoplist = [\'very\', \'ourselves\', \'am\', \'doesn\', \'through\', \'me\', \'against\', \'up\', \'just\', \'her\', \'ours\',
\'couldn\', \'because\', \'is\', \'isn\', \'it\', \'only\', \'in\', \'such\', \'too\', \'mustn\', \'under\', \'their\',
\'if\', \'to\', \'my\', \'himself\', \'after\', \'why\', \'while\', \'can\', \'each\', \'itself\', \'his\', \'all\', \'once\',
\'herself\', \'more\', \'our\', \'they\', \'hasn\', \'on\', \'ma\', \'them\', \'its\', \'where\', \'did\', \'ll\', \'you\',
\'didn\', \'nor\', \'as\', \'now\', \'before\', \'those\', \'yours\', \'from\', \'who\', \'was\', \'m\', \'been\', \'will\',
\'into\', \'same\', \'how\', \'some\', \'of\', \'out\', \'with\', \'s\', \'being\', \'t\', \'mightn\', \'she\', \'again\', \'be\',
\'by\', \'shan\', \'have\', \'yourselves\', \'needn\', \'and\', \'are\', \'o\', \'these\', \'further\', \'most\', \'yourself\',
\'having\', \'aren\', \'here\', \'he\', \'were\', \'but\', \'this\', \'myself\', \'own\', \'we\', \'so\', \'i\', \'does\', \'both\',
\'when\', \'between\', \'d\', \'had\', \'the\', \'y\', \'has\', \'down\', \'off\', \'than\', \'haven\', \'whom\', \'wouldn\',
\'should\', \'ve\', \'over\', \'themselves\', \'few\', \'then\', \'hadn\', \'what\', \'until\', \'won\', \'no\', \'about\',
\'any\', \'that\', \'for\', \'shouldn\', \'don\', \'do\', \'there\', \'doing\', \'an\', \'or\', \'ain\', \'hers\', \'wasn\',
\'weren\', \'above\', \'a\', \'at\', \'your\', \'theirs\', \'below\', \'other\', \'not\', \'re\', \'him\', \'during\', \'which\']
texts = [[word for word in doc.lower().split() if word not in stoplist] for doc in doclist]
print(texts[0])
#建立语料库
dictionary = corpora.Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]
print(corpus[13])
#建立模型
lda = gensim.models.ldamodel.LdaModel(corpus=corpus,id2word=dictionary,num_topics=20)
print(lda.print_topic(10,topn=5))
print(lda.print_topics(num_topics = 10,num_words = 5))
lda_list = [] #doc1这句话属于哪个主题？
doc1 = \'To all the little girls watching never doubt that you are valuable and powerful & deserving of every chance & opportunity in the world\'
for words in doc1:
doc_bow = dictionary.doc2bow(words)
doc_lda = lda[doc_bow]
lda_list.append(doc_lda)
print(lda_list)