猿創(chuàng)征文｜信息抽取（2）——pytorch實(shí)現(xiàn)Bert-BiLSTM-CRF、Bert-CRF模型進(jìn)行實(shí)體抽取

這篇具有很好參考價(jià)值的文章主要介紹了猿創(chuàng)征文｜信息抽?。?）——pytorch實(shí)現(xiàn)Bert-BiLSTM-CRF、Bert-CRF模型進(jìn)行實(shí)體抽取。希望對大家有所幫助。如果存在錯(cuò)誤或未考慮完全的地方，請大家不吝賜教，您也可以點(diǎn)擊"舉報(bào)違法"按鈕提交疑問。

1 前言

猿創(chuàng)征文｜信息抽?。?）——pytorch實(shí)現(xiàn)Bert-BiLSTM-CRF、Bert-CRF模型進(jìn)行實(shí)體抽取

論文參考：
1 Neural Architectures for Named Entity Recognition
2 Attention is all you need
3 BERT: Pre-training of Deep Bidirectional Transformers for
Language Understanding
4 Bidirectional LSTM-CRF Models for Sequence Tagging
使用數(shù)據(jù)集：
https://www.datafountain.cn/competitions/529/ranking
Tips：文章可能存在一些漏洞，歡迎留言指出

2 數(shù)據(jù)準(zhǔn)備

使用了transformers和seqeval庫
安裝方法：
huggingface-transformers

conda install -c huggingface transformers

seqeval

pip install seqeval -i https://pypi.tuna.tsinghua.edu.cn/simple

代碼

import pandas as pd
import torch
from torch import optim
from torch.utils.data import DataLoader
from tqdm import tqdm
from bert_bilstm_crf import Bert_BiLSTM_CRF, NerDataset, NerDatasetTest
from bert_crf import Bert_CRF
from transformers import AutoTokenizer, BertTokenizer
from seqeval.metrics import f1_score

# 路徑
TRAIN_PATH = './dataset/train_data_public.csv'
TEST_PATH = './dataset/test_public.csv'
MODEL_PATH1 = './model/bert_bilstm_crf.pkl'
MODEL_PATH2 = '../model/bert_crf.pkl'

# 超參數(shù)
MAX_LEN = 64
BATCH_SIZE = 16
EPOCH = 5

# 預(yù)設(shè)
# 設(shè)備
DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu"
# tag2index
tag2index = {
    "O": 0,  # 其他
    "B-BANK": 1, "I-BANK": 2,  # 銀行實(shí)體
    "B-PRODUCT": 3, "I-PRODUCT": 4,  # 產(chǎn)品實(shí)體
    "B-COMMENTS_N": 5, "I-COMMENTS_N": 6,  # 用戶評(píng)論，名詞
    "B-COMMENTS_ADJ": 7, "I-COMMENTS_ADJ": 8  # 用戶評(píng)論，形容詞
}
index2tag = {v: k for k, v in tag2index.items()}

3 數(shù)據(jù)預(yù)處理

== 流程==

使用 $\textcolor{red}{series.apply(list)}$ 函數(shù)將str轉(zhuǎn)化為list格式
加載bert預(yù)訓(xùn)練tokenizer，使用 $\textcolor{red}{encode\_plus}$ 函數(shù)對每一個(gè)text進(jìn)行encode
如果是訓(xùn)練集，則執(zhí)行如下操作：首先按照空格將每一個(gè)tag分割，并轉(zhuǎn)化為索引列表，對每一個(gè)index_list，按照長度大于MAX_LEN裁剪，小于MAX_LEN填充的規(guī)則，合并為一個(gè)list，最后轉(zhuǎn)化為tensor格式

代碼

# 預(yù)處理
def data_preprocessing(dataset, is_train):
    # 數(shù)據(jù)str轉(zhuǎn)化為list
    dataset['text_split'] = dataset['text'].apply(list)
    # token
    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
    texts = dataset['text_split'].array.tolist()
    token_texts = []
    for text in tqdm(texts):
        tokenized = tokenizer.encode_plus(text=text,
                                          max_length=MAX_LEN,
                                          return_token_type_ids=True,
                                          return_attention_mask=True,
                                          return_tensors='pt',
                                          padding='max_length',
                                          truncation=True)
        token_texts.append(tokenized)

    # 訓(xùn)練集有tag，測試集沒有tag
    tags = None
    if is_train:
        dataset['tag'] = dataset['BIO_anno'].apply(lambda x: x.split(sep=' '))
        tags = []
        for tag in tqdm(dataset['tag'].array.tolist()):
            index_list = [0] + [tag2index[t] for t in tag] + [0]
            if len(index_list) < MAX_LEN:  # 填充
                pad_length = MAX_LEN - len(index_list)
                index_list += [tag2index['O']] * pad_length
            if len(index_list) > MAX_LEN:  # 裁剪
                index_list = index_list[:MAX_LEN-1] + [0]
            tags.append(index_list)
        tags = torch.LongTensor(tags)

    return token_texts, tags

4 Bert-BiLSTM-CRF模型

猿創(chuàng)征文｜信息抽?。?）——pytorch實(shí)現(xiàn)Bert-BiLSTM-CRF、Bert-CRF模型進(jìn)行實(shí)體抽取
相對于Bert-CRF，中間添加了雙向LSTM層。相對BiLSTM-CRF，相當(dāng)于前面的word_embedding層替換為了bert預(yù)訓(xùn)練模型。
代碼

import torch
from torch import nn
from torchcrf import CRF
from transformers import BertModel
from torch.utils.data import Dataset


class Bert_BiLSTM_CRF(nn.Module):
    def __init__(self, tag2index):
        super(Bert_BiLSTM_CRF, self).__init__()
        self.tagset_size = len(tag2index)

        # bert層
        self.bert = BertModel.from_pretrained('bert-base-chinese')
        # config = self.bert.config
        # lstm層
        self.lstm = nn.LSTM(input_size=768, hidden_size=128, num_layers=1, batch_first=True, bidirectional=True)
        # dropout層
        self.dropout = nn.Dropout(p=0.1)
        # Dense層
        self.dense = nn.Linear(in_features=256, out_features=self.tagset_size)
        # CRF層
        self.crf = CRF(num_tags=self.tagset_size)

        # 隱藏層
        self.hidden = None

    # 負(fù)對數(shù)似然損失函數(shù)
    def neg_log_likelihood(self, emissions, tags=None, mask=None, reduction=None):
        return -1 * self.crf(emissions=emissions, tags=tags, mask=mask, reduction=reduction)

    def forward(self, token_texts, tags):
        """
        token_texts:{"input_size": tensor,  [batch, 1, seq_len]->[batch, seq_len]
                    "token_type_ids": tensor,  [batch, 1, seq_len]->[batch, seq_len]
                     "attention_mask": tensor  [batch, 1, seq_len]->[batch, seq_len]->[seq_len, batch]
                     }
        tags:  [batch, seq_len]->[seq_len, batch]
        bert_out:  [batch, seq_len, hidden_size(768)]->[seq_len, batch, hidden_size]
        self.hidden:  [num_layers * num_directions, hidden_size(128)]
        out:  [seq_len, batch, hidden_size * 2(256)]
        lstm_feats:  [seq_len, batch, tagset_size]
        loss:  tensor
        predictions:  [batch, num]
        """
        texts, token_type_ids, masks = token_texts['input_ids'], token_texts['token_type_ids'], token_texts['attention_mask']
        texts = texts.squeeze(1)
        token_type_ids = token_type_ids.squeeze(1)
        masks = masks.squeeze(1)
        bert_out = self.bert(input_ids=texts, attention_mask=masks, token_type_ids=token_type_ids)[0]
        bert_out = bert_out.permute(1, 0, 2)
        # 檢測設(shè)備
        device = bert_out.device
        # 初始化隱藏層參數(shù)
        self.hidden = (torch.randn(2, bert_out.size(0), 128).to(device),
                       torch.randn(2, bert_out.size(0), 128).to(device))
        out, self.hidden = self.lstm(bert_out, self.hidden)
        lstm_feats = self.dense(out)

        # 格式轉(zhuǎn)換
        masks = masks.permute(1, 0)
        masks = masks.clone().detach().bool()
        # masks = torch.tensor(masks, dtype=torch.uint8)
        # 計(jì)算損失值和預(yù)測值
        if tags is not None:
            tags = tags.permute(1, 0)
            loss = self.neg_log_likelihood(lstm_feats, tags, masks, 'mean')
            predictions = self.crf.decode(emissions=lstm_feats, mask=masks)  # [batch, 任意數(shù)]
            return loss, predictions
        else:
            predictions = self.crf.decode(emissions=lstm_feats, mask=masks)
            return predictions

Dataset

class NerDataset(Dataset):
    def __init__(self, token_texts, tags):
        super(NerDataset, self).__init__()
        self.token_texts = token_texts
        self.tags = tags

    def __getitem__(self, index):
        return {
            "token_texts": self.token_texts[index],
            "tags": self.tags[index] if self.tags is not None else None,
        }

    def __len__(self):
        return len(self.token_texts)


class NerDatasetTest(Dataset):
    def __init__(self, token_texts):
        super(NerDatasetTest, self).__init__()
        self.token_texts = token_texts

    def __getitem__(self, index):
        return {
            "token_texts": self.token_texts[index],
            "tags": 0
        }

    def __len__(self):
        return len(self.token_texts)

前向傳播分析
token_texts:{
“input_size”: tensor, [batch, 1, seq_len]->[batch, seq_len]
“token_type_ids”: tensor, [batch, 1, seq_len]->[batch, seq_len]
“attention_mask”: tensor [batch, 1, seq_len]->[batch, seq_len]->[seq_len, batch]
}
tags: [batch, seq_len]->[seq_len, batch]
bert_out: [batch, seq_len, hidden_size(768)]->[seq_len, batch, hidden_size]
self.hidden: [num_layers * num_directions, hidden_size(128)]
out: [seq_len, batch, hidden_size * 2(256)]
lstm_feats: [seq_len, batch, tagset_size]
loss: tensor
predictions: [batch, num]

5 Bert-CRF模型

猿創(chuàng)征文｜信息抽取（2）——pytorch實(shí)現(xiàn)Bert-BiLSTM-CRF、Bert-CRF模型進(jìn)行實(shí)體抽取

from torch import nn
from torchcrf import CRF
from transformers import BertModel


class Bert_CRF(nn.Module):
    def __init__(self, tag2index):
        super(Bert_CRF, self).__init__()
        self.tagset_size = len(tag2index)

        # bert層
        self.bert = BertModel.from_pretrained('bert-base-chinese')
        # dense層
        self.dense = nn.Linear(in_features=768, out_features=self.tagset_size)
        # CRF層
        self.crf = CRF(num_tags=self.tagset_size)

        # 隱藏層
        self.hidden = None

    def neg_log_likelihood(self, emissions, tags=None, mask=None, reduction=None):
        return -1 * self.crf(emissions=emissions, tags=tags, mask=mask, reduction=reduction)

    def forward(self, token_texts, tags):
        """
        token_texts:{"input_size": tensor,  [batch, 1, seq_len]->[batch, seq_len]
                    "token_type_ids": tensor,  [batch, 1, seq_len]->[batch, seq_len]
                     "attention_mask": tensor  [batch, 1, seq_len]->[batch, seq_len]->[seq_len, batch]
                     }
        tags:  [batch, seq_len]->[seq_len, batch]
        bert_out:  [batch, seq_len, hidden_size(768)]->[seq_len, batch, hidden_size]
        feats:  [seq_len, batch, tagset_size]
        loss:  tensor
        predictions:  [batch, num]
        """
        texts, token_type_ids, masks = token_texts.values()
        texts = texts.squeeze(1)
        token_type_ids = token_type_ids.squeeze(1)
        masks = masks.squeeze(1)
        bert_out = self.bert(input_ids=texts, attention_mask=masks, token_type_ids=token_type_ids)[0]
        bert_out = bert_out.permute(1, 0, 2)
        feats = self.dense(bert_out)

        # 格式轉(zhuǎn)換
        masks = masks.permute(1, 0)
        masks = masks.clone().detach().bool()
        # 計(jì)算損失之和預(yù)測值
        if tags is not None:
            tags = tags.permute(1, 0)
            loss = self.neg_log_likelihood(feats, tags, masks, 'mean')
            predictions = self.crf.decode(emissions=feats, mask=masks)
            return loss, predictions
        else:
            predictions = self.crf.decode(emissions=feats, mask=masks)
            return predictions

前向傳播分析
token_texts:{
“input_size”: tensor, [batch, 1, seq_len]->[batch, seq_len]
“token_type_ids”: tensor, [batch, 1, seq_len]->[batch, seq_len]
“attention_mask”: tensor [batch, 1, seq_len]->[batch, seq_len]->[seq_len, batch]
}
tags: [batch, seq_len]->[seq_len, batch]
bert_out: [batch, seq_len, hidden_size(768)]->[seq_len, batch, hidden_size]
feats: [seq_len, batch, tagset_size]
loss: tensor
predictions: [batch, num]

6 模型訓(xùn)練

# 訓(xùn)練
def train(train_dataloader, model, optimizer, epoch):
    for i, batch_data in enumerate(train_dataloader):
        token_texts = batch_data['token_texts'].to(DEVICE)
        tags = batch_data['tags'].to(DEVICE)
        loss, predictions = model(token_texts, tags)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if i % 200 == 0:
            micro_f1 = get_f1_score(tags, predictions)
            print(f'Epoch:{epoch} | i:{i} | loss:{loss.item()} | Micro_F1:{micro_f1}')

7 結(jié)果評(píng)估

# 計(jì)算f1值
def get_f1_score(tags, predictions):
    tags = tags.to('cpu').data.numpy().tolist()
    temp_tags = []
    final_tags = []
    for index in range(BATCH_SIZE):
        # predictions先去掉頭，再去掉尾
        predictions[index].pop()
        length = len(predictions[index])
        temp_tags.append(tags[index][1:length])
        predictions[index].pop(0)
        # 格式轉(zhuǎn)化，轉(zhuǎn)化為List(str)
        temp_tags[index] = [index2tag[x] for x in temp_tags[index]]
        predictions[index] = [index2tag[x] for x in predictions[index]]
        final_tags.append(temp_tags[index])

    f1 = f1_score(final_tags, predictions, average='micro')
    return f1

Bert-BiLSTM-CRF

GPU_NAME:NVIDIA GeForce RTX 3060 Laptop GPU | Memory_Allocated:413399040
Epoch:0 | i:0 | loss:58.75139236450195 | Micro_F1:0.0
Epoch:0 | i:200 | loss:26.20857048034668 | Micro_F1:0.0
Epoch:0 | i:400 | loss:18.385879516601562 | Micro_F1:0.0
Epoch:1 | i:0 | loss:20.496620178222656 | Micro_F1:0.0
Epoch:1 | i:200 | loss:15.421577453613281 | Micro_F1:0.0
Epoch:1 | i:400 | loss:11.486358642578125 | Micro_F1:0.0
Epoch:2 | i:0 | loss:14.486601829528809 | Micro_F1:0.0
Epoch:2 | i:200 | loss:10.369649887084961 | Micro_F1:0.18867924528301888
Epoch:2 | i:400 | loss:8.056020736694336 | Micro_F1:0.5652173913043479
Epoch:3 | i:0 | loss:14.958343505859375 | Micro_F1:0.41025641025641024
Epoch:3 | i:200 | loss:9.968450546264648 | Micro_F1:0.380952380952381
Epoch:3 | i:400 | loss:8.947534561157227 | Micro_F1:0.5614035087719299
Epoch:4 | i:0 | loss:9.189300537109375 | Micro_F1:0.5454545454545454
Epoch:4 | i:200 | loss:8.673486709594727 | Micro_F1:0.43999999999999995
Epoch:4 | i:400 | loss:6.431578636169434 | Micro_F1:0.6250000000000001

Bert-CRF

GPU_NAME:NVIDIA GeForce RTX 3060 Laptop GPU | Memory_Allocated:409739264
Epoch:0 | i:0 | loss:57.06057357788086 | Micro_F1:0.0
Epoch:0 | i:200 | loss:12.05904483795166 | Micro_F1:0.0
Epoch:0 | i:400 | loss:13.805888175964355 | Micro_F1:0.39393939393939387
Epoch:1 | i:0 | loss:9.807424545288086 | Micro_F1:0.4905660377358491
Epoch:1 | i:200 | loss:8.098043441772461 | Micro_F1:0.509090909090909
Epoch:1 | i:400 | loss:7.059831619262695 | Micro_F1:0.611111111111111
Epoch:2 | i:0 | loss:6.629759788513184 | Micro_F1:0.6133333333333333
Epoch:2 | i:200 | loss:3.593130350112915 | Micro_F1:0.6896551724137931
Epoch:2 | i:400 | loss:6.8786163330078125 | Micro_F1:0.6666666666666666
Epoch:3 | i:0 | loss:5.009466648101807 | Micro_F1:0.6969696969696969
Epoch:3 | i:200 | loss:2.9549810886383057 | Micro_F1:0.8450704225352113
Epoch:3 | i:400 | loss:3.3801448345184326 | Micro_F1:0.868421052631579
Epoch:4 | i:0 | loss:5.864352226257324 | Micro_F1:0.626865671641791
Epoch:4 | i:200 | loss:3.308518409729004 | Micro_F1:0.7666666666666667
Epoch:4 | i:400 | loss:4.221902847290039 | Micro_F1:0.7000000000000001

分析
進(jìn)行了5個(gè)epoch的訓(xùn)練
數(shù)據(jù)集比較小，只有7000多條數(shù)據(jù)，因此兩個(gè)模型效果擬合效果相對BiLSTM+CRF模型提升不大。而添加了雙向LSTM層之后，模型效果反而有所下降。

8 訓(xùn)練集流水線

注意學(xué)習(xí)率要設(shè)置小一點(diǎn)（小于1e-5），否則預(yù)測結(jié)果均為0，不收斂。

def execute():
    # 加載訓(xùn)練集
    train_dataset = pd.read_csv(TRAIN_PATH, encoding='utf8')
    # 數(shù)據(jù)預(yù)處理
    token_texts, tags = data_preprocessing(train_dataset, is_train=True)
    # 數(shù)據(jù)集裝載
    train_dataset = NerDataset(token_texts, tags)
    train_dataloader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)
    # 構(gòu)建模型
    # model = Bert_BiLSTM_CRF(tag2index=tag2index).to(DEVICE)
    model = Bert_CRF(tag2index=tag2index).to(DEVICE)
    optimizer = optim.AdamW(model.parameters(), lr=1e-6)
    print(f"GPU_NAME:{torch.cuda.get_device_name()} | Memory_Allocated:{torch.cuda.memory_allocated()}")
    # 模型訓(xùn)練
    for i in range(EPOCH):
        train(train_dataloader, model, optimizer, i)

    # 保存模型
    torch.save(model.state_dict(), MODEL_PATH2)

9 測試集流水線

# 測試集預(yù)測實(shí)體標(biāo)簽
def test():
    # 加載數(shù)據(jù)集
    test_dataset = pd.read_csv(TEST_PATH, encoding='utf8')
    # 數(shù)據(jù)預(yù)處理
    token_texts, _ = data_preprocessing(test_dataset, is_train=False)
    # 裝載測試集
    dataset_test = NerDatasetTest(token_texts)
    test_dataloader = DataLoader(dataset=dataset_test, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)
    # 構(gòu)建模型
    # model = Bert_BiLSTM_CRF(tag2index).to(DEVICE)
    model = Bert_CRF(tag2index).to(DEVICE)
    model.load_state_dict(torch.load(MODEL_PATH2))
    # 模型預(yù)測
    model.eval()
    predictions_list = []
    with torch.no_grad():
        for i, batch_data in enumerate(test_dataloader):
            token_texts = batch_data['token_texts'].to(DEVICE)
            predictions = model(token_texts, None)
            predictions_list.extend(predictions)
    print(len(predictions_list))
    print(len(test_dataset['text']))

    # 將預(yù)測結(jié)果轉(zhuǎn)換為文本格式
    entity_tag_list = []
    index2tag = {v: k for k, v in tag2index.items()}  # 反轉(zhuǎn)字典
    for i, (text, predictions) in enumerate(zip(test_dataset['text'], predictions_list)):
        # 刪除首位和最后一位
        predictions.pop()
        predictions.pop(0)
        text_entity_tag = []
        for c, t in zip(text, predictions):
            if t != 0:
                text_entity_tag.append(c + index2tag[t])
        entity_tag_list.append(" ".join(text_entity_tag))  # 合并為str并加入列表中

    print(len(entity_tag_list))
    result_df = pd.DataFrame(data=entity_tag_list, columns=['result'])
    result_df.to_csv('./data/result_df3.csv')

猿創(chuàng)征文｜信息抽取（2）——pytorch實(shí)現(xiàn)Bert-BiLSTM-CRF、Bert-CRF模型進(jìn)行實(shí)體抽取
結(jié)果好像存在一些問題。。。

10 記錄遇到的一些坑

（1）模型預(yù)測結(jié)果全為O
原因：按照之前的模型，AdamW優(yōu)化器學(xué)習(xí)率設(shè)置0.001，學(xué)習(xí)率過高，導(dǎo)致梯度下降過程中落入了局部最低點(diǎn)。
解決方法：重新設(shè)置學(xué)習(xí)率為1e-6
（2）transformers的AdamW顯示過期
解決方法：直接使用torch.optim的AdamW即可
（3）transformers庫在ubuntu上無法使用
原因：缺少依賴
解決方法：

apt-get update
apt-get install libssl1.0.0 libssl-dev

使用此代碼在服務(wù)器終端上跑完后，仍會(huì)報(bào)錯(cuò)，原因未知，暫時(shí)用os.system()嵌入到代碼中，在windows系統(tǒng)中無此報(bào)錯(cuò)。
（4）筆記本（聯(lián)想R7000P2021）運(yùn)行代碼溫度過高（最高95度）
解決方法：先用均衡模式（CPU不睿頻）跑，溫度只有六七十度，然后開啟野獸模式跑一段時(shí)間，溫度高了再切換為均衡模式。文章來源地址http://www.zghlxwxcb.cn/news/detail-429090.html

11 完整代碼

import pandas as pd
import torch
from torch import optim
from torch.utils.data import DataLoader
from tqdm import tqdm
from bert_bilstm_crf import Bert_BiLSTM_CRF, NerDataset, NerDatasetTest
from bert_crf import Bert_CRF
from transformers import AutoTokenizer, BertTokenizer
from seqeval.metrics import f1_score

# 路徑
TRAIN_PATH = './dataset/train_data_public.csv'
TEST_PATH = './dataset/test_public.csv'
MODEL_PATH1 = './model/bert_bilstm_crf.pkl'
MODEL_PATH2 = '../model/bert_crf.pkl'

# 超參數(shù)
MAX_LEN = 64
BATCH_SIZE = 16
EPOCH = 5

# 預(yù)設(shè)
# 設(shè)備
DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu"
# tag2index
tag2index = {
    "O": 0,  # 其他
    "B-BANK": 1, "I-BANK": 2,  # 銀行實(shí)體
    "B-PRODUCT": 3, "I-PRODUCT": 4,  # 產(chǎn)品實(shí)體
    "B-COMMENTS_N": 5, "I-COMMENTS_N": 6,  # 用戶評(píng)論，名詞
    "B-COMMENTS_ADJ": 7, "I-COMMENTS_ADJ": 8  # 用戶評(píng)論，形容詞
}
index2tag = {v: k for k, v in tag2index.items()}


# 訓(xùn)練
def train(train_dataloader, model, optimizer, epoch):
    for i, batch_data in enumerate(train_dataloader):
        token_texts = batch_data['token_texts'].to(DEVICE)
        tags = batch_data['tags'].to(DEVICE)
        loss, predictions = model(token_texts, tags)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if i % 200 == 0:
            micro_f1 = get_f1_score(tags, predictions)
            print(f'Epoch:{epoch} | i:{i} | loss:{loss.item()} | Micro_F1:{micro_f1}')


# 計(jì)算f1值
def get_f1_score(tags, predictions):
    tags = tags.to('cpu').data.numpy().tolist()
    temp_tags = []
    final_tags = []
    for index in range(BATCH_SIZE):
        # predictions先去掉頭，再去掉尾
        predictions[index].pop()
        length = len(predictions[index])
        temp_tags.append(tags[index][1:length])
        predictions[index].pop(0)
        # 格式轉(zhuǎn)化，轉(zhuǎn)化為List(str)
        temp_tags[index] = [index2tag[x] for x in temp_tags[index]]
        predictions[index] = [index2tag[x] for x in predictions[index]]
        final_tags.append(temp_tags[index])

    f1 = f1_score(final_tags, predictions, average='micro')
    return f1


# 預(yù)處理
def data_preprocessing(dataset, is_train):
    # 數(shù)據(jù)str轉(zhuǎn)化為list
    dataset['text_split'] = dataset['text'].apply(list)
    # token
    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
    texts = dataset['text_split'].array.tolist()
    token_texts = []
    for text in tqdm(texts):
        tokenized = tokenizer.encode_plus(text=text,
                                          max_length=MAX_LEN,
                                          return_token_type_ids=True,
                                          return_attention_mask=True,
                                          return_tensors='pt',
                                          padding='max_length',
                                          truncation=True)
        token_texts.append(tokenized)

    # 訓(xùn)練集有tag，測試集沒有tag
    tags = None
    if is_train:
        dataset['tag'] = dataset['BIO_anno'].apply(lambda x: x.split(sep=' '))
        tags = []
        for tag in tqdm(dataset['tag'].array.tolist()):
            index_list = [0] + [tag2index[t] for t in tag] + [0]
            if len(index_list) < MAX_LEN:  # 填充
                pad_length = MAX_LEN - len(index_list)
                index_list += [tag2index['O']] * pad_length
            if len(index_list) > MAX_LEN:  # 裁剪
                index_list = index_list[:MAX_LEN-1] + [0]
            tags.append(index_list)
        tags = torch.LongTensor(tags)

    return token_texts, tags


# 執(zhí)行流水線
def execute():
    # 加載訓(xùn)練集
    train_dataset = pd.read_csv(TRAIN_PATH, encoding='utf8')
    # 數(shù)據(jù)預(yù)處理
    token_texts, tags = data_preprocessing(train_dataset, is_train=True)
    # 數(shù)據(jù)集裝載
    train_dataset = NerDataset(token_texts, tags)
    train_dataloader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)
    # 構(gòu)建模型
    # model = Bert_BiLSTM_CRF(tag2index=tag2index).to(DEVICE)
    model = Bert_CRF(tag2index=tag2index).to(DEVICE)
    optimizer = optim.AdamW(model.parameters(), lr=1e-6)
    print(f"GPU_NAME:{torch.cuda.get_device_name()} | Memory_Allocated:{torch.cuda.memory_allocated()}")
    # 模型訓(xùn)練
    for i in range(EPOCH):
        train(train_dataloader, model, optimizer, i)

    # 保存模型
    torch.save(model.state_dict(), MODEL_PATH2)


# 測試集預(yù)測實(shí)體標(biāo)簽
def test():
    # 加載數(shù)據(jù)集
    test_dataset = pd.read_csv(TEST_PATH, encoding='utf8')
    # 數(shù)據(jù)預(yù)處理
    token_texts, _ = data_preprocessing(test_dataset, is_train=False)
    # 裝載測試集
    dataset_test = NerDatasetTest(token_texts)
    test_dataloader = DataLoader(dataset=dataset_test, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)
    # 構(gòu)建模型
    model = Bert_BiLSTM_CRF(tag2index).to(DEVICE)
    model.load_state_dict(torch.load(MODEL_PATH2))
    # 模型預(yù)測
    model.eval()
    predictions_list = []
    with torch.no_grad():
        for i, batch_data in enumerate(test_dataloader):
            token_texts = batch_data['token_texts'].to(DEVICE)
            predictions = model(token_texts, None)
            predictions_list.extend(predictions)
    print(len(predictions_list))
    print(len(test_dataset['text']))

    # 將預(yù)測結(jié)果轉(zhuǎn)換為文本格式
    entity_tag_list = []
    index2tag = {v: k for k, v in tag2index.items()}  # 反轉(zhuǎn)字典
    for i, (text, predictions) in enumerate(zip(test_dataset['text'], predictions_list)):
        # 刪除首位和最后一位
        predictions.pop()
        predictions.pop(0)
        text_entity_tag = []
        for c, t in zip(text, predictions):
            if t != 0:
                text_entity_tag.append(c + index2tag[t])
        entity_tag_list.append(" ".join(text_entity_tag))  # 合并為str并加入列表中

    print(len(entity_tag_list))
    result_df = pd.DataFrame(data=entity_tag_list, columns=['result'])
    result_df.to_csv('./data/result_df3.csv')


if __name__ == '__main__':
    execute()
    test()

到了這里，關(guān)于猿創(chuàng)征文｜信息抽取（2）——pytorch實(shí)現(xiàn)Bert-BiLSTM-CRF、Bert-CRF模型進(jìn)行實(shí)體抽取的文章就介紹完了。如果您還想了解更多內(nèi)容，請?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！