【发布时间】：2019-03-01 19:39:27
【问题描述】：
我正在尝试使用 20 个新闻组数据集来实现一个在线分类模型，以将帖子分类到相关组中。
预处理：我正在浏览所有帖子并用单词制作字典。然后我从 1 开始索引单词。然后我遍历所有帖子和每个单词在一篇文章中，我正在搜索词汇表并将相关的索引号放入一个数组中。然后我通过在末尾添加 0 来填充所有数组，使它们的大小都相同（6577）。
然后我正在创建嵌入层（嵌入大小=300）。并且每个输入在被馈送到 LSTM 层之前都会经过这个嵌入层（LSTM 输入形状= (1,6577,300)）。
在我的模型中，我有一个 LSTM 层（大小 = 200）和一个隐藏层（大小 = 25）。为此，我在 tensorflow 中使用 dynamic_rnn 单元格，并将序列长度参数设置为帖子的实际长度（没有填充 0 的长度）以避免分析填充的 0。然后从 LSTM 层的输出中，我只将相关输出提供给隐藏层。
从那里开始，它就像一个普通的 LSTM 实现。我已经尽我所能提高模型的准确性，但错误预测的数量非常多：
数据点数：18,846
错误：17876
错误率：0.9485301920832007
注意：在反向传播期间，我正在训练嵌入层和隐藏层。
问题：我想知道我在这里做错了什么，或者有什么想法可以改进模型。提前谢谢你。
我的完整代码如下所示：
from collections import Counter
import tensorflow as tf
from sklearn.datasets import fetch_20newsgroups
import matplotlib as mplt
mplt.use('agg') # Must be before importing matplotlib.pyplot or pylab!
import matplotlib.pyplot as plt
from string import punctuation
from sklearn.preprocessing import LabelBinarizer
import numpy as np
from nltk.corpus import stopwords
import nltk
nltk.download('stopwords')
def pre_process():
newsgroups_data = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))
words = []
temp_post_text = []
print(len(newsgroups_data.data))
for post in newsgroups_data.data:
all_text = ''.join([text for text in post if text not in punctuation])
all_text = all_text.split('\n')
all_text = ''.join(all_text)
temp_text = all_text.split(" ")
for word in temp_text:
if word.isalpha():
temp_text[temp_text.index(word)] = word.lower()
# temp_text = [word for word in temp_text if word not in stopwords.words('english')]
temp_text = list(filter(None, temp_text))
temp_text = ' '.join([i for i in temp_text if not i.isdigit()])
words += temp_text.split(" ")
temp_post_text.append(temp_text)
# temp_post_text = list(filter(None, temp_post_text))
dictionary = Counter(words)
# deleting spaces
# del dictionary[""]
sorted_split_words = sorted(dictionary, key=dictionary.get, reverse=True)
vocab_to_int = {c: i for i, c in enumerate(sorted_split_words,1)}
message_ints = []
for message in temp_post_text:
temp_message = message.split(" ")
message_ints.append([vocab_to_int[i] for i in temp_message])
# maximum message length = 6577
# message_lens = Counter([len(x) for x in message_ints])AAA
seq_length = 6577
num_messages = len(temp_post_text)
features = np.zeros([num_messages, seq_length], dtype=int)
for i, row in enumerate(message_ints):
# print(features[i, -len(row):])
# features[i, -len(row):] = np.array(row)[:seq_length]
features[i, :len(row)] = np.array(row)[:seq_length]
# print(features[i])
lb = LabelBinarizer()
lbl = newsgroups_data.target
labels = np.reshape(lbl, [-1])
labels = lb.fit_transform(labels)
sequence_lengths = [len(msg) for msg in message_ints]
return features, labels, len(sorted_split_words)+1, sequence_lengths
def get_batches(x, y, sql, batch_size=1):
for ii in range(0, len(y), batch_size):
yield x[ii:ii + batch_size], y[ii:ii + batch_size], sql[ii:ii+batch_size]
def plot(noOfWrongPred, dataPoints):
font_size = 14
fig = plt.figure(dpi=100,figsize=(10, 6))
mplt.rcParams.update({'font.size': font_size})
plt.title("Distribution of wrong predictions", fontsize=font_size)
plt.ylabel('Error rate', fontsize=font_size)
plt.xlabel('Number of data points', fontsize=font_size)
plt.plot(dataPoints, noOfWrongPred, label='Prediction', color='blue', linewidth=1.8)
# plt.legend(loc='upper right', fontsize=14)
plt.savefig('distribution of wrong predictions.png')
# plt.show()
def train_test():
features, labels, n_words, sequence_length = pre_process()
print(features.shape)
print(labels.shape)
# Defining Hyperparameters
lstm_layers = 1
batch_size = 1
lstm_size = 200
learning_rate = 0.01
# --------------placeholders-------------------------------------
# Create the graph object
graph = tf.Graph()
# Add nodes to the graph
with graph.as_default():
tf.set_random_seed(1)
inputs_ = tf.placeholder(tf.int32, [None, None], name="inputs")
# labels_ = tf.placeholder(dtype= tf.int32)
labels_ = tf.placeholder(tf.float32, [None, None], name="labels")
sql_in = tf.placeholder(tf.int32, [None], name= 'sql_in')
# output_keep_prob is the dropout added to the RNN's outputs, the dropout will have no effect on the calculation of the subsequent states.
keep_prob = tf.placeholder(tf.float32, name="keep_prob")
# Size of the embedding vectors (number of units in the embedding layer)
embed_size = 300
# generating random values from a uniform distribution (minval included and maxval excluded)
embedding = tf.Variable(tf.random_uniform((n_words, embed_size), -1, 1),trainable=True)
embed = tf.nn.embedding_lookup(embedding, inputs_)
print(embedding.shape)
print(embed.shape)
print(embed[0])
# Your basic LSTM cell
lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)
# Getting an initial state of all zeros
initial_state = lstm.zero_state(batch_size, tf.float32)
outputs, final_state = tf.nn.dynamic_rnn(lstm, embed, initial_state=initial_state, sequence_length=sql_in)
out_batch_size = tf.shape(outputs)[0]
out_max_length = tf.shape(outputs)[1]
out_size = int(outputs.get_shape()[2])
index = tf.range(0, out_batch_size) * out_max_length + (sql_in - 1)
flat = tf.reshape(outputs, [-1, out_size])
relevant = tf.gather(flat, index)
# hidden layer
hidden = tf.layers.dense(relevant, units=25, activation=tf.nn.relu,trainable=True)
print(hidden.shape)
logit = tf.contrib.layers.fully_connected(hidden, num_outputs=20, activation_fn=None)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logit, labels=labels_))
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
saver = tf.train.Saver()
# ----------------------------online training-----------------------------------------
with tf.Session(graph=graph) as sess:
tf.set_random_seed(1)
sess.run(tf.global_variables_initializer())
iteration = 1
state = sess.run(initial_state)
wrongPred = 0
noOfWrongPreds = []
dataPoints = []
for ii, (x, y, sql) in enumerate(get_batches(features, labels, sequence_length, batch_size), 1):
feed = {inputs_: x,
labels_: y,
sql_in : sql,
keep_prob: 0.5,
initial_state: state}
predictions = tf.nn.softmax(logit).eval(feed_dict=feed)
print("----------------------------------------------------------")
print("sez: ",sql)
print("Iteration: {}".format(iteration))
isequal = np.equal(np.argmax(predictions[0], 0), np.argmax(y[0], 0))
print(np.argmax(predictions[0], 0))
print(np.argmax(y[0], 0))
if not (isequal):
wrongPred += 1
print("nummber of wrong preds: ",wrongPred)
if iteration%50 == 0:
noOfWrongPreds.append(wrongPred/iteration)
dataPoints.append(iteration)
loss, states, _ = sess.run([cost, outputs, optimizer], feed_dict=feed)
print("Train loss: {:.3f}".format(loss))
iteration += 1
saver.save(sess, "checkpoints/sentiment.ckpt")
errorRate = wrongPred / len(labels)
print("ERRORS: ", wrongPred)
print("ERROR RATE: ", errorRate)
plot(noOfWrongPreds, dataPoints)
if __name__ == '__main__':
train_test()
编辑
【问题讨论】：
标签： python tensorflow machine-learning lstm text-classification