文章目录
首先附上本篇文章的流程图
加载和观察数据
第一步是加载和观察训练集和测试集。
导入需要用到的包
import jieba
import pandas as pd
import numpy as np
from sklearn import preprocessing, decomposition, model_selection, metrics, pipeline
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
pandas读取csv数据
train = pd.read_csv('data/input/train.csv')
test = pd.read_csv('data/input/test_public.csv')
sample = pd.read_csv('data/input/submit_example.csv')
查看训练数据和测试数据
train.head()
第一列为id,第二列为评论文本，第三列为类别，第四列为情感词，第5列为分析情感的依据词。
test.head()
这里训练集有9947行 测试集有2364行
要根据content内容预测出其主题和情感。注意可能有多个主题，每个主题的情感可能不同。
合并所有的content
这里合并所有的content内容
train_cutted = train[['content']]
test_cutted = test[['content']]
all1 = train_cutted.append(test_cutted,ignore_index=True)
这里一共有12311行
预测标签转换
类别标签转换使用LabelEncoder()进行转换
lbl_enc = preprocessing.LabelEncoder()
lbl_enc.fit(["动力", "价格", "内饰", "配置","安全性","外观","操控","油耗","空间","舒适性"])
y = lbl_enc.transform(train.subject.values)
分词
def chinese_word_cut(mytext):
return " ".join(jieba.cut(mytext))
all1["content_cutted"] = all1.content.apply(chinese_word_cut)
使用jieba分词进行中文长句子分词
向量化
CountVectorizer()
sklearn中CountVectorizer()可以统计出文本中的词袋模型，并把每句文本转为对应的数值向量。
from sklearn.feature_extraction.text import CountVectorizer
texts=["dog cat fish","dog cat cat","fish bird", 'bird']
cv = CountVectorizer()
cv_fit=cv.fit_transform(texts)
print(cv.get_feature_names())
print(cv_fit.toarray())
#['bird', 'cat', 'dog', 'fish'] 字典中的words
#[[0 1 1 1] 每一句中是否出现对应的单词
# [0 2 1 0]
# [1 0 0 1]
# [1 0 0 0]]
print(cv_fit.toarray().sum(axis=0))
#[2 3 2 2] 计算总体上所有的单词出现的情况
去掉停用词后的向量化
def get_custom_stopwords(stop_words_file):
with open(stop_words_file) as f:
stopwords = f.read()
stopwords_list = stopwords.split('\n')
custom_stopwords_list = [i for i in stopwords_list]
return custom_stopwords_list
stop_words_file = "data/stopwordsHIT.txt"
stopwords = get_custom_stopwords(stop_words_file)
vect = CountVectorizer(stop_words=frozenset(stopwords))
term_matrix_all = pd.DataFrame(vect.fit_transform(all1["content_cutted"]).toarray(), columns=vect.get_feature_names())
这里介绍下停用词，所谓的停用词指的是一些无意义的语气词，如"的、哈哈、哎"等等。去掉停用词之后会使词库更加紧凑，降低维度。
去掉低频词和高频词后 再进行向量化
由于向量化后的输入维度等于词库的大小。这通常是一个很大的数值，而词库中很多的单词出现次数很少，而高频词如汽车等等也对分类没有帮助。因此去掉高频词和低频词后再进行向量化。
max_df = 0.8 # 在超过这一比例的文档中出现的关键词（过于平凡），去除掉。
min_df = 3 # 在低于这一数量的文档中出现的关键词（过于独特），去除掉。
vect = CountVectorizer(max_df = max_df,
min_df = min_df,
token_pattern=u'(?u)\\b[^\\d\\W]\\w+\\b',
stop_words=frozenset(stopwords))
term_matrix_all = pd.DataFrame(vect.fit_transform(all1["content_cutted"]).toarray(), columns=vect.get_feature_names())
原始词库中有17823个单词，去掉停用词后有17611。而去掉高低频词后，词典数量减少到了6056。
model
划分测试集和训练集
X_train = term_matrix_all.iloc[:9947]
Y_train = y
X_test = term_matrix_all.iloc[9947:]
X_train.shape, Y_train.shape, X_test.shape
# ((9947, 6056), (9947,), (2364, 6056))
简单模型
逻辑回归
logreg = LogisticRegression() # 调用模型
logreg.fit(X_train, Y_train) # 拟合训练集
Y_pred = logreg.predict(X_test) # 预测测试集
acc_log = round(logreg.score(X_train, Y_train) * 100, 2) # 计算准确率
acc_log # 81.19
预测出来的数值标签转换为真实的标签
Y_pred_name = lbl_enc.inverse_transform(Y_pred)
test_origin=test
test_origin['subject']=Y_pred_name # 添加新列
test_origin['sentiment_value']=0 # 简单处理后面的情感分析
test_origin['sentiment_word']='可以'
test_origin = test_origin.drop(['content'], axis=1) # 删除掉原始文本
test_origin.to_csv('data/output/logReg.csv',index=False) # 输出预测值到文本
各种模型
随机森林
# Random Forest
random_forest = RandomForestClassifier(n_estimators=100)
random_forest.fit(X_train, Y_train)
Y_pred_RF = random_forest.predict(X_test)
random_forest.score(X_train, Y_train)
acc_random_forest = round(random_forest.score(X_train, Y_train) * 100, 2)
acc_random_forest # 83.3
SGD
# Stochastic Gradient Descent
from sklearn.linear_model import SGDClassifier
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
Y_pred_SGD = sgd.predict(X_test)
acc_sgd = round(sgd.score(X_train, Y_train) * 100, 2)
acc_sgd # 80.41
Linear SVC
# Linear SVC
from sklearn.svm import SVC, LinearSVC
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
Y_pred = linear_svc.predict(X_test)
acc_linear_svc = round(linear_svc.score(X_train, Y_train) * 100, 2)
acc_linear_svc # 83.02
感知机
# Perceptron
from sklearn.linear_model import Perceptron
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
Y_pred = perceptron.predict(X_test)
acc_perceptron = round(perceptron.score(X_train, Y_train) * 100, 2)
acc_perceptron # 79.08
Naive Bayes
# Gaussian Naive Bayes
from sklearn.naive_bayes import GaussianNB
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
Y_pred = gaussian.predict(X_test)
acc_gaussian = round(gaussian.score(X_train, Y_train) * 100, 2)
acc_gaussian # 51.21
KNN
from sklearn.neighbors import KNeighborsClassifier
# KNN
knn = KNeighborsClassifier(n_neighbors = 3)
knn.fit(X_train, Y_train)
Y_pred = knn.predict(X_test)
acc_knn = round(knn.score(X_train, Y_train) * 100, 2)
acc_knn
总结
当前版本有基本的流程，包括文本的分词、向量化。以及基本的机器学习模型。
但是只是简单的单标签。也没有考虑情感词的处理
评价标准是准确率而不是F1score。需要进一步提升。