1 前言

?? 優(yōu)質(zhì)競賽項(xiàng)目系列，今天要分享的是

基于CNN實(shí)現(xiàn)謠言檢測

該項(xiàng)目較為新穎，適合作為競賽課題方向，學(xué)長非常推薦！

?? 更多資料, 項(xiàng)目分享：

https://gitee.com/dancheng-senior/postgraduate文章來源地址http://www.zghlxwxcb.cn/news/detail-662482.html

1.1 背景

社交媒體的發(fā)展在加速信息傳播的同時(shí)，也帶來了虛假謠言信息的泛濫，往往會(huì)引發(fā)諸多不安定因素，并對經(jīng)濟(jì)和社會(huì)產(chǎn)生巨大的影響。

2 數(shù)據(jù)集

本項(xiàng)目所使用的數(shù)據(jù)是從新浪微博不實(shí)信息舉報(bào)平臺抓取的中文謠言數(shù)據(jù)，數(shù)據(jù)集中共包含1538條謠言和1849條非謠言。

如下圖所示，每條數(shù)據(jù)均為json格式，其中text字段代表微博原文的文字內(nèi)容。

每個(gè)文件夾里又有很多新聞文本。

每個(gè)文本又是json格式，具體內(nèi)容如下：

3 實(shí)現(xiàn)過程

步驟入下：

*（1）解壓數(shù)據(jù)，讀取并解析數(shù)據(jù)，生成all_data.txt
*（2）生成數(shù)據(jù)字典，即dict.txt
*（3）生成數(shù)據(jù)列表，并進(jìn)行訓(xùn)練集與驗(yàn)證集的劃分，train_list.txt 、eval_list.txt
*（4）定義訓(xùn)練數(shù)據(jù)集提供器train_reader和驗(yàn)證數(shù)據(jù)集提供器eval_reader

import zipfile
import os
import io
import random
import json
import matplotlib.pyplot as plt
import numpy as np
import paddle
import paddle.fluid as fluid
from paddle.fluid.dygraph.nn import Conv2D, Linear, Embedding
from paddle.fluid.dygraph.base import to_variable

#解壓原始數(shù)據(jù)集，將Rumor_Dataset.zip解壓至data目錄下
src_path="/home/aistudio/data/data36807/Rumor_Dataset.zip" #這里填寫自己項(xiàng)目所在的數(shù)據(jù)集路徑
target_path="/home/aistudio/data/Chinese_Rumor_Dataset-master"
if(not os.path.isdir(target_path)):
    z = zipfile.ZipFile(src_path, 'r')
    z.extractall(path=target_path)
    z.close()

#分別為謠言數(shù)據(jù)、非謠言數(shù)據(jù)、全部數(shù)據(jù)的文件路徑
rumor_class_dirs = os.listdir(target_path+"非開源數(shù)據(jù)集") # 這里填寫自己項(xiàng)目所在的數(shù)據(jù)集路徑
non_rumor_class_dirs = os.listdir(target_path+"非開源數(shù)據(jù)集")
original_microblog = target_path+"非開源數(shù)據(jù)集"
#謠言標(biāo)簽為0，非謠言標(biāo)簽為1
rumor_label="0"
non_rumor_label="1"

#分別統(tǒng)計(jì)謠言數(shù)據(jù)與非謠言數(shù)據(jù)的總數(shù)
rumor_num = 0
non_rumor_num = 0
all_rumor_list = []
all_non_rumor_list = []

#解析謠言數(shù)據(jù)
for rumor_class_dir in rumor_class_dirs: 
    if(rumor_class_dir != '.DS_Store'):
        #遍歷謠言數(shù)據(jù)，并解析
        with open(original_microblog + rumor_class_dir, 'r') as f:
            rumor_content = f.read()
        rumor_dict = json.loads(rumor_content)
        all_rumor_list.append(rumor_label+"\t"+rumor_dict["text"]+"\n")
        rumor_num +=1
#解析非謠言數(shù)據(jù)
for non_rumor_class_dir in non_rumor_class_dirs: 
    if(non_rumor_class_dir != '.DS_Store'):
        with open(original_microblog + non_rumor_class_dir, 'r') as f2:
            non_rumor_content = f2.read()
        non_rumor_dict = json.loads(non_rumor_content)
        all_non_rumor_list.append(non_rumor_label+"\t"+non_rumor_dict["text"]+"\n")
        non_rumor_num +=1
        
print("謠言數(shù)據(jù)總量為："+str(rumor_num))
print("非謠言數(shù)據(jù)總量為："+str(non_rumor_num))

#全部數(shù)據(jù)進(jìn)行亂序后寫入all_data.txt
data_list_path="/home/aistudio/data/"
all_data_path=data_list_path + "all_data.txt"
all_data_list = all_rumor_list + all_non_rumor_list

random.shuffle(all_data_list)

#在生成all_data.txt之前，首先將其清空
with open(all_data_path, 'w') as f:
    f.seek(0)
    f.truncate() 
    
with open(all_data_path, 'a') as f:
    for data in all_data_list:
        f.write(data) 
print('all_data.txt已生成')

?

接下來就是生成數(shù)據(jù)字典。

?
# 生成數(shù)據(jù)字典
def create_dict(data_path, dict_path):
with open(dict_path, ‘w’) as f:
f.seek(0)
f.truncate()

    dict_set = set()
    # 讀取全部數(shù)據(jù)
    with open(data_path, 'r', encoding='utf-8') as f:
        lines = f.readlines()
    # 把數(shù)據(jù)生成一個(gè)元組
    for line in lines:
        content = line.split('\t')[-1].replace('\n', '')
        for s in content:
            dict_set.add(s)
    # 把元組轉(zhuǎn)換成字典，一個(gè)字對應(yīng)一個(gè)數(shù)字
    dict_list = []
    i = 0
    for s in dict_set:
        dict_list.append([s, i])
        i += 1
    # 添加未知字符
    dict_txt = dict(dict_list)
    end_dict = {"": i}
    dict_txt.update(end_dict)
    # 把這些字典保存到本地中
    with open(dict_path, 'w', encoding='utf-8') as f:
        f.write(str(dict_txt))
    print("數(shù)據(jù)字典生成完成！",'\t','字典長度為：',len(dict_list))

我們可以查看一下dict_txt的內(nèi)容

接下來就是數(shù)據(jù)列表的生成

?
# 創(chuàng)建序列化表示的數(shù)據(jù),并按照一定比例劃分訓(xùn)練數(shù)據(jù)與驗(yàn)證數(shù)據(jù)
def create_data_list(data_list_path):

    with open(os.path.join(data_list_path, 'dict.txt'), 'r', encoding='utf-8') as f_data:
        dict_txt = eval(f_data.readlines()[0])

    with open(os.path.join(data_list_path, 'all_data.txt'), 'r', encoding='utf-8') as f_data:
        lines = f_data.readlines()
    
    i = 0
    with open(os.path.join(data_list_path, 'eval_list.txt'), 'a', encoding='utf-8') as f_eval,\
    open(os.path.join(data_list_path, 'train_list.txt'), 'a', encoding='utf-8') as f_train:
        for line in lines:
            title = line.split('\t')[-1].replace('\n', '')
            lab = line.split('\t')[0]
            t_ids = ""
            if i % 8 == 0:
                for s in title:
                    temp = str(dict_txt[s])
                    t_ids = t_ids + temp + ','
                t_ids = t_ids[:-1] + '\t' + lab + '\n'
                f_eval.write(t_ids)
            else:
                for s in title:
                    temp = str(dict_txt[s])
                    t_ids = t_ids + temp + ','
                t_ids = t_ids[:-1] + '\t' + lab + '\n'
                f_train.write(t_ids)
            i += 1
        
    print("數(shù)據(jù)列表生成完成！")

定義數(shù)據(jù)讀取器

?
def data_reader(file_path, phrase, shuffle=False):
all_data = []
with io.open(file_path, “r”, encoding=‘utf8’) as fin:
for line in fin:
cols = line.strip().split(“\t”)
if len(cols) != 2:
continue
label = int(cols[1])

            wids = cols[0].split(",")
            all_data.append((wids, label))

    if shuffle:
        if phrase == "train":
            random.shuffle(all_data)

    def reader():
        for doc, label in all_data:
            yield doc, label
    return reader

class SentaProcessor(object):
    def __init__(self, data_dir,):
        self.data_dir = data_dir
        
    def get_train_data(self, data_dir, shuffle):
        return data_reader((self.data_dir + "train_list.txt"), 
                            "train", shuffle)

    def get_eval_data(self, data_dir, shuffle):
        return data_reader((self.data_dir + "eval_list.txt"), 
                            "eval", shuffle)

    def data_generator(self, batch_size, phase='train', shuffle=True):
        if phase == "train":
            return paddle.batch(
                self.get_train_data(self.data_dir, shuffle),
                batch_size,
                drop_last=True)
        elif phase == "eval":
            return paddle.batch(
                self.get_eval_data(self.data_dir, shuffle),
                batch_size,
                drop_last=True)
        else:
            raise ValueError(
                "Unknown phase, which should be in ['train', 'eval']")

總之在數(shù)據(jù)處理這一塊需要我們注意的是一共生成以下的幾個(gè)文件。

4 CNN網(wǎng)絡(luò)實(shí)現(xiàn)

接下來就是構(gòu)建以及配置卷積神經(jīng)網(wǎng)絡(luò)(Convolutional Neural Networks,
CNN)，開篇也說了，其實(shí)這里有很多模型的選擇，之所以選擇CNN是因?yàn)樽屛覀兪煜NN的相關(guān)實(shí)現(xiàn)。 輸入詞向量序列，產(chǎn)生一個(gè)特征圖（feature
map），對特征圖采用時(shí)間維度上的最大池化（max pooling over
time）操作得到此卷積核對應(yīng)的整句話的特征，最后，將所有卷積核得到的特征拼接起來即為文本的定長向量表示，對于文本分類問題，將其連接至softmax即構(gòu)建出完整的模型。在實(shí)際應(yīng)用中，我們會(huì)使用多個(gè)卷積核來處理句子，窗口大小相同的卷積核堆疊起來形成一個(gè)矩陣，這樣可以更高效的完成運(yùn)算。另外，我們也可使用窗口大小不同的卷積核來處理句子。具體的流程如下：

首先我們構(gòu)建單層CNN神經(jīng)網(wǎng)絡(luò)。

?

    #單層
    class SimpleConvPool(fluid.dygraph.Layer):
        def __init__(self,
                     num_channels, # 通道數(shù)
                     num_filters,  # 卷積核數(shù)量
                     filter_size,  # 卷積核大小
                     batch_size=None): # 16
            super(SimpleConvPool, self).__init__()
            self.batch_size = batch_size
            self._conv2d = Conv2D(num_channels = num_channels,
                num_filters = num_filters,
                filter_size = filter_size,
                act='tanh')
            self._pool2d = fluid.dygraph.Pool2D(
                pool_size = (150 - filter_size[0]+1,1),
                pool_type = 'max',
                pool_stride=1
            )
    
        def forward(self, inputs):
            # print('SimpleConvPool_inputs數(shù)據(jù)緯度',inputs.shape) # [16, 1, 148, 128]
            x = self._conv2d(inputs)
            x = self._pool2d(x)
            x = fluid.layers.reshape(x, shape=[self.batch_size, -1])
            return x



    class CNN(fluid.dygraph.Layer):
        def __init__(self):
            super(CNN, self).__init__()
            self.dict_dim = train_parameters["vocab_size"]
            self.emb_dim = 128   #emb緯度
            self.hid_dim = [32]  #卷積核數(shù)量
            self.fc_hid_dim = 96  #fc參數(shù)緯度
            self.class_dim = 2    #分類數(shù)
            self.channels = 1     #輸入通道數(shù)
            self.win_size = [[3, 128]]  # 卷積核尺寸
            self.batch_size = train_parameters["batch_size"] 
            self.seq_len = train_parameters["padding_size"]
            self.embedding = Embedding( 
                size=[self.dict_dim + 1, self.emb_dim],
                dtype='float32', 
                is_sparse=False)
            self._simple_conv_pool_1 = SimpleConvPool(
                self.channels,
                self.hid_dim[0],
                self.win_size[0],
                batch_size=self.batch_size)
            self._fc1 = Linear(input_dim = self.hid_dim[0],
                                output_dim = self.fc_hid_dim,
                                act="tanh")
            self._fc_prediction = Linear(input_dim = self.fc_hid_dim,
                                        output_dim = self.class_dim,
                                        act="softmax")
    
        def forward(self, inputs, label=None):
    
            emb = self.embedding(inputs) # [2400, 128]
            # print('CNN_emb',emb.shape)  
            emb = fluid.layers.reshape(   # [16, 1, 150, 128]
                emb, shape=[-1, self.channels , self.seq_len, self.emb_dim])
            # print('CNN_emb',emb.shape)
            conv_3 = self._simple_conv_pool_1(emb)
            fc_1 = self._fc1(conv_3)
            prediction = self._fc_prediction(fc_1)
            if label is not None:
                acc = fluid.layers.accuracy(prediction, label=label)
                return prediction, acc
            else:
                return prediction

接下來就是參數(shù)的配置，不過為了在模型訓(xùn)練過程中更直觀的查看我們訓(xùn)練的準(zhǔn)確率，我們首先利用python的matplotlib.pyplt函數(shù)實(shí)現(xiàn)一個(gè)可視化圖，具體的實(shí)現(xiàn)如下：

?
def draw_train_process(iters, train_loss, train_accs):
title=“training loss/training accs”
plt.title(title, fontsize=24)
plt.xlabel(“iter”, fontsize=14)
plt.ylabel(“l(fā)oss/acc”, fontsize=14)
plt.plot(iters, train_loss, color=‘red’, label=‘training loss’)
plt.plot(iters, train_accs, color=‘green’, label=‘training accs’)
plt.legend()
plt.grid()
plt.show()

5 模型訓(xùn)練部分

?
def train():
with fluid.dygraph.guard(place = fluid.CUDAPlace(0)): # 因?yàn)橐M(jìn)行很大規(guī)模的訓(xùn)練，因此我們用的是GPU，如果沒有安裝GPU的可以使用下面一句，把這句代碼注釋掉即可
# with fluid.dygraph.guard(place = fluid.CPUPlace()):

        processor = SentaProcessor( data_dir="data/")
    
        train_data_generator = processor.data_generator(
            batch_size=train_parameters["batch_size"],
            phase='train',
            shuffle=True)
            
        model = CNN()
        sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=train_parameters["adam"],parameter_list=model.parameters())
        steps = 0
        Iters,total_loss, total_acc = [], [], []
        for eop in range(train_parameters["epoch"]):
            for batch_id, data in enumerate(train_data_generator()):
                steps += 1
                #轉(zhuǎn)換為 variable 類型
                doc = to_variable(
                    np.array([
                        np.pad(x[0][0:train_parameters["padding_size"]],  #對句子進(jìn)行padding，全部填補(bǔ)為定長150
                              (0, train_parameters["padding_size"] - len(x[0][0:train_parameters["padding_size"]])),
                               'constant',
                              constant_values=(train_parameters["vocab_size"])) # 用  的id 進(jìn)行填補(bǔ)
                        for x in data
                    ]).astype('int64').reshape(-1))
                #轉(zhuǎn)換為 variable 類型
                label = to_variable(
                    np.array([x[1] for x in data]).astype('int64').reshape(
                        train_parameters["batch_size"], 1))

                model.train() #使用訓(xùn)練模式
                prediction, acc = model(doc, label)
                loss = fluid.layers.cross_entropy(prediction, label)
                avg_loss = fluid.layers.mean(loss)
                avg_loss.backward()
                sgd_optimizer.minimize(avg_loss)
                model.clear_gradients()
                
                if steps % train_parameters["skip_steps"] == 0:
                    Iters.append(steps)
                    total_loss.append(avg_loss.numpy()[0])
                    total_acc.append(acc.numpy()[0])
                    print("eop: %d, step: %d, ave loss: %f, ave acc: %f" %
                         (eop, steps,avg_loss.numpy(),acc.numpy()))
                if steps % train_parameters["save_steps"] == 0:
                    save_path = train_parameters["checkpoints"]+"/"+"save_dir_" + str(steps)
                    print('save model to: ' + save_path)
                    fluid.dygraph.save_dygraph(model.state_dict(),
                                                   save_path)
                # break
    draw_train_process(Iters, total_loss, total_acc)

訓(xùn)練的過程以及訓(xùn)練的結(jié)果如下：

6 模型評估

?
def to_eval():
with fluid.dygraph.guard(place = fluid.CUDAPlace(0)):
processor = SentaProcessor(data_dir=“data/”) #寫自己的路徑

        eval_data_generator = processor.data_generator(
                batch_size=train_parameters["batch_size"],
                phase='eval',
                shuffle=False)

        model_eval = CNN() #示例化模型
        model, _ = fluid.load_dygraph("data//save_dir_180.pdparams") #寫自己的路徑
        model_eval.load_dict(model)

        model_eval.eval() # 切換為eval模式
        total_eval_cost, total_eval_acc = [], []
        for eval_batch_id, eval_data in enumerate(eval_data_generator()):
            eval_np_doc = np.array([np.pad(x[0][0:train_parameters["padding_size"]],
                                    (0, train_parameters["padding_size"] -len(x[0][0:train_parameters["padding_size"]])),
                                    'constant',
                                    constant_values=(train_parameters["vocab_size"]))
                            for x in eval_data
                            ]).astype('int64').reshape(-1)
            eval_label = to_variable(
                                    np.array([x[1] for x in eval_data]).astype(
                                    'int64').reshape(train_parameters["batch_size"], 1))
            eval_doc = to_variable(eval_np_doc)
            eval_prediction, eval_acc = model_eval(eval_doc, eval_label)
            loss = fluid.layers.cross_entropy(eval_prediction, eval_label)
            avg_loss = fluid.layers.mean(loss)
            total_eval_cost.append(avg_loss.numpy()[0])
            total_eval_acc.append(eval_acc.numpy()[0])

    print("Final validation result: ave loss: %f, ave acc: %f" %
        (np.mean(total_eval_cost), np.mean(total_eval_acc) ))   

評估準(zhǔn)確率如下：

7 預(yù)測結(jié)果

?
# 獲取數(shù)據(jù)
def load_data(sentence):
# 讀取數(shù)據(jù)字典
with open(‘data/dict.txt’, ‘r’, encoding=‘utf-8’) as f_data:
dict_txt = eval(f_data.readlines()[0])
dict_txt = dict(dict_txt)
# 把字符串?dāng)?shù)據(jù)轉(zhuǎn)換成列表數(shù)據(jù)
keys = dict_txt.keys()
data = []
for s in sentence:
# 判斷是否存在未知字符
if not s in keys:
s = ‘’
data.append(int(dict_txt[s]))
return data

train_parameters["batch_size"] = 1
lab = [ '謠言', '非謠言']
 
with fluid.dygraph.guard(place = fluid.CUDAPlace(0)):
    
    data = load_data('興仁縣今天搶小孩沒搶走，把孩子母親捅了一刀，看見這車的注意了，真事，車牌號遼HFM055?。。。?！趕緊散播！ 都別帶孩子出去瞎轉(zhuǎn)悠了 尤其別讓老人自己帶孩子出去 太危險(xiǎn)了 注意了?。。。∵|HFM055北京現(xiàn)代朗動(dòng)，在各學(xué)校門口搶小孩?。。?10已經(jīng) 證實(shí)?。∪型ň儯。?)
    data_np = np.array(data)
    data_np = np.array(np.pad(data_np,(0,150-len(data_np)),"constant",constant_values =train_parameters["vocab_size"])).astype('int64').reshape(-1)

    infer_np_doc = to_variable(data_np)
   
    model_infer = CNN()
    model, _ = fluid.load_dygraph("data/save_dir_900.pdparams")
    model_infer.load_dict(model)
    model_infer.eval()
    result = model_infer(infer_np_doc)
    print('預(yù)測結(jié)果為：', lab[np.argmax(result.numpy())])

8 最后

?? 更多資料, 項(xiàng)目分享：

https://gitee.com/dancheng-senior/postgraduate

到了這里，關(guān)于計(jì)算機(jī)競賽 基于CNN實(shí)現(xiàn)謠言檢測 - python 深度學(xué)習(xí) 機(jī)器學(xué)習(xí)的文章就介紹完了。如果您還想了解更多內(nèi)容，請?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！