數(shù)據(jù)集下載

data_dir = './DATA/rawdata'
dataset_name = 'ted2020'
urls = (
    "https://github.com/figisiwirf/ml2023-hw5-dataset/releases/download/v1.0.1/ml2023.hw5.data.tgz",
    "https://github.com/figisiwirf/ml2023-hw5-dataset/releases/download/v1.0.1/ml2023.hw5.test.tgz"
)
file_names = (
    'ted2020.tgz', # train & dev
    'test.tgz', # test
)
prefix = Path(data_dir).absolute() / dataset_name

prefix.mkdir(parents=True, exist_ok=True)
for u, f in zip(urls, file_names):
    path = prefix/f
    if not path.exists():
        !wget {u} -O {path}
    if path.suffix == ".tgz":
        !tar -xvf {path} -C {prefix}
    elif path.suffix == ".zip":
        !unzip -o {path} -d {prefix}
!mv {prefix/'raw.en'} {prefix/'train_dev.raw.en'}
!mv {prefix/'raw.zh'} {prefix/'train_dev.raw.zh'}
!mv {prefix/'test.en'} {prefix/'test.raw.en'}
!mv {prefix/'test.zh'} {prefix/'test.raw.zh'}

語(yǔ)言

src_lang = 'en'
tgt_lang = 'zh'

data_prefix = f'{prefix}/train_dev.raw'
test_prefix = f'{prefix}/test.raw'

!head {data_prefix+'.'+src_lang} -n 5
!head {data_prefix+'.'+tgt_lang} -n 5

預(yù)處理文件

• strQ2B(): 將全角字符轉(zhuǎn)變?yōu)榘虢亲址?/li>
• clean_s(): 清洗文本，將逗號(hào)/破折號(hào)/空格等字符刪除
• len_s(): 返回文本長(zhǎng)度
• clean_corpus: 使用上面的函數(shù)對(duì)指定的文本文件進(jìn)行清洗

import re

def strQ2B(ustring):
    """Full width -> half width"""
    # reference:https://ithelp.ithome.com.tw/articles/10233122
    ss = []
    for s in ustring:
        rstring = ""
        for uchar in s:
            inside_code = ord(uchar)
            if inside_code == 12288:  # Full width space: direct conversion
                inside_code = 32
            elif (inside_code >= 65281 and inside_code <= 65374):  # Full width chars (except space) conversion
                inside_code -= 65248
            rstring += chr(inside_code)
        ss.append(rstring)
    return ''.join(ss)

def clean_s(s, lang):
    if lang == 'en':
        s = re.sub(r"\([^()]*\)", "", s) # remove ([text])
        s = s.replace('-', '') # remove '-'
        s = re.sub('([.,;!?()\"])', r' \1 ', s) # keep punctuation
    elif lang == 'zh':
        s = strQ2B(s) # Q2B
        s = re.sub(r"\([^()]*\)", "", s) # remove ([text])
        s = s.replace(' ', '')
        s = s.replace('—', '')
        s = s.replace('“', '"')
        s = s.replace('”', '"')
        s = s.replace('_', '')
        s = re.sub('([。,;!?()\"~「」])', r' \1 ', s) # keep punctuation
    s = ' '.join(s.strip().split())
    return s

def len_s(s, lang):
    if lang == 'zh':
        return len(s)
    return len(s.split())

def clean_corpus(prefix, l1, l2, ratio=9, max_len=1000, min_len=1):
    if Path(f'{prefix}.clean.{l1}').exists() and Path(f'{prefix}.clean.{l2}').exists():
        print(f'{prefix}.clean.{l1} & {l2} exists. skipping clean.')
        return
    with open(f'{prefix}.{l1}', 'r') as l1_in_f:
        with open(f'{prefix}.{l2}', 'r') as l2_in_f:
            with open(f'{prefix}.clean.{l1}', 'w') as l1_out_f:
                with open(f'{prefix}.clean.{l2}', 'w') as l2_out_f:
                    for s1 in l1_in_f:
                        s1 = s1.strip()
                        s2 = l2_in_f.readline().strip()
                        s1 = clean_s(s1, l1)
                        s2 = clean_s(s2, l2)
                        s1_len = len_s(s1, l1)
                        s2_len = len_s(s2, l2)
                        if min_len > 0: # remove short sentence
                            if s1_len < min_len or s2_len < min_len:
                                continue
                        if max_len > 0: # remove long sentence
                            if s1_len > max_len or s2_len > max_len:
                                continue
                        if ratio > 0: # remove by ratio of length
                            if s1_len/s2_len > ratio or s2_len/s1_len > ratio:
                                continue
                        print(s1, file=l1_out_f)
                        print(s2, file=l2_out_f)

clean_corpus(data_prefix, src_lang, tgt_lang)
clean_corpus(test_prefix, src_lang, tgt_lang, ratio=-1, min_len=-1, max_len=-1)

!head {data_prefix+'.clean.'+src_lang} -n 5
!head {data_prefix+'.clean.'+tgt_lang} -n 5

劃分訓(xùn)練/驗(yàn)證集

valid_ratio = 0.01 # 3000～4000 就夠用了
train_ratio = 1 - valid_ratio

if (prefix/f'train.clean.{src_lang}').exists() \
and (prefix/f'train.clean.{tgt_lang}').exists() \
and (prefix/f'valid.clean.{src_lang}').exists() \
and (prefix/f'valid.clean.{tgt_lang}').exists():
    print(f'train/valid splits exists. skipping split.')
else:
    line_num = sum(1 for line in open(f'{data_prefix}.clean.{src_lang}'))
    labels = list(range(line_num))
    random.shuffle(labels)
    for lang in [src_lang, tgt_lang]:
        train_f = open(os.path.join(data_dir, dataset_name, f'train.clean.{lang}'), 'w')
        valid_f = open(os.path.join(data_dir, dataset_name, f'valid.clean.{lang}'), 'w')
        count = 0
        for line in open(f'{data_prefix}.clean.{lang}', 'r'):
            if labels[count]/line_num < train_ratio:
                train_f.write(line)
            else:
                valid_f.write(line)
            count += 1
        train_f.close()
        valid_f.close()

子詞單位

不在詞表中的單詞（OOV）是機(jī)器翻譯面臨的主要問(wèn)題。這個(gè)問(wèn)題可以通過(guò)使用子詞（subword）作為基本單位來(lái)緩解

• 我們將使用 sentencepiece 包
• 選擇 unigram 或者 byte-pair encoding (BPE) 算法

import sentencepiece as spm
vocab_size = 8000
if (prefix/f'spm{vocab_size}.model').exists():
    print(f'{prefix}/spm{vocab_size}.model exists. skipping spm_train.')
else:
    spm.SentencePieceTrainer.train(
        input=','.join([f'{prefix}/train.clean.{src_lang}',
                        f'{prefix}/valid.clean.{src_lang}',
                        f'{prefix}/train.clean.{tgt_lang}',
                        f'{prefix}/valid.clean.{tgt_lang}']),
        model_prefix=prefix/f'spm{vocab_size}',
        vocab_size=vocab_size,
        character_coverage=1,
        model_type='unigram', # 'bpe' works as well
        input_sentence_size=1e6,
        shuffle_input_sentence=True,
        normalization_rule_name='nmt_nfkc_cf',
    )

spm_model = spm.SentencePieceProcessor(model_file=str(prefix/f'spm{vocab_size}.model'))
in_tag = {
    'train': 'train.clean',
    'valid': 'valid.clean',
    'test': 'test.raw.clean',
}
for split in ['train', 'valid', 'test']:
    for lang in [src_lang, tgt_lang]:
        out_path = prefix/f'{split}.{lang}'
        if out_path.exists():
            print(f"{out_path} exists. skipping spm_encode.")
        else:
            with open(prefix/f'{split}.{lang}', 'w') as out_f:
                with open(prefix/f'{in_tag[split]}.{lang}', 'r') as in_f:
                    for line in in_f:
                        line = line.strip()
                        tok = spm_model.encode(line, out_type=str)
                        print(' '.join(tok), file=out_f)

!head {data_dir+'/'+dataset_name+'/train.'+src_lang} -n 5
!head {data_dir+'/'+dataset_name+'/train.'+tgt_lang} -n 5

數(shù)據(jù)二值化（使用 fairseq）

配對(duì)源語(yǔ)言和目標(biāo)語(yǔ)言的文件。

如果沒(méi)有對(duì)應(yīng)的文件，就生成偽配對(duì)來(lái)方便二值化。

binpath = Path('./DATA/data-bin', dataset_name)
if binpath.exists():
    print(binpath, "exists, will not overwrite!")
else:
    !python -m fairseq_cli.preprocess \
        --source-lang {src_lang}\
        --target-lang {tgt_lang}\
        --trainpref {prefix/'train'}\
        --validpref {prefix/'valid'}\
        --testpref {prefix/'test'}\
        --destdir {binpath}\
        --joined-dictionary\
        --workers 2

實(shí)驗(yàn)配置

config = Namespace(
    datadir = "./DATA/data-bin/ted2020",
    savedir = "./checkpoints/rnn",
    source_lang = src_lang,
    target_lang = tgt_lang,

    # 設(shè)置取數(shù)據(jù)和處理數(shù)據(jù)時(shí) cpu 的線程數(shù)
    num_workers=2,
    # batch size 按照 token 數(shù)量來(lái)計(jì)算。梯度累積可以增加有效的 batch size。
    max_tokens=8192,
    accum_steps=2,

    # 學(xué)習(xí)率通過(guò) Noam 調(diào)度器進(jìn)行計(jì)算。你可以修改lr_factor來(lái)調(diào)整最大的學(xué)習(xí)率。
    lr_factor=2.,
    lr_warmup=4000,

    # 梯度裁剪可以緩解梯度爆炸
    clip_norm=1.0,

    # 訓(xùn)練的最大輪數(shù)
    max_epoch=15,
    start_epoch=1,

    # 集束搜索中的 beam size
    beam=5,
    # 生成的序列的最大長(zhǎng)度為 ax + b，其中 x 是源長(zhǎng)度
    max_len_a=1.2,
    max_len_b=10,
    # 解碼時(shí)，通過(guò)去除 sentencepiece 符號(hào)和 jieba 分詞來(lái)后處理句子。
    post_process = "sentencepiece",

    # 檢查點(diǎn)
    keep_last_epochs=5,
    resume=None, # if resume 則根據(jù) checkpoint name 進(jìn)行恢復(fù)（文件保存在 config.savedir 下）

    # 日志記錄
    use_wandb=False,
)

日志

• logging 包用于記錄普通的信息
• wandb 記錄訓(xùn)練過(guò)程中的損失/bleu等

logging.basicConfig(
    format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
    level="INFO", # "DEBUG" "WARNING" "ERROR"
    stream=sys.stdout,
)
proj = "hw5.seq2seq"
logger = logging.getLogger(proj)
if config.use_wandb:
    import wandb
    wandb.init(project=proj, name=Path(config.savedir).stem, config=config)

CUDA 環(huán)境

cuda_env = utils.CudaEnvironment()
utils.CudaEnvironment.pretty_print_cuda_env_list([cuda_env])
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

數(shù)據(jù)導(dǎo)入

我們采用了 TranslationTask（來(lái)自 fairseq）

• 用于加載上面創(chuàng)建的二值化數(shù)據(jù)
• 實(shí)現(xiàn)數(shù)據(jù)迭代器（dataloader）
• 內(nèi)置的 task.source_dictionary 和 task.target_dictionary 也很有用
• 實(shí)現(xiàn)集束搜索解碼器

from fairseq.tasks.translation import TranslationConfig, TranslationTask

## setup task
task_cfg = TranslationConfig(
    data=config.datadir,
    source_lang=config.source_lang,
    target_lang=config.target_lang,
    train_subset="train",
    required_seq_len_multiple=8,
    dataset_impl="mmap",
    upsample_primary=1,
)
task = TranslationTask.setup_task(task_cfg)

logger.info("loading data for epoch 1")
task.load_dataset(split="train", epoch=1, combine=True) # combine if you have back-translation data.
task.load_dataset(split="valid", epoch=1)

sample = task.dataset("valid")[1]
pprint.pprint(sample)
pprint.pprint(
    "Source: " + \
    task.source_dictionary.string(
        sample['source'],
        config.post_process,
    )
)
pprint.pprint(
    "Target: " + \
    task.target_dictionary.string(
        sample['target'],
        config.post_process,
    )
)

數(shù)據(jù)集迭代器

• 控制每個(gè) batch 不超過(guò) N 個(gè) token，這樣可以優(yōu)化 GPU 內(nèi)存效率
• 在每個(gè) epoch 都對(duì)訓(xùn)練集進(jìn)行隨機(jī)打亂
• 忽略超過(guò)最大長(zhǎng)度的句子
• 將一個(gè) batch 中的所有句子填充到相同的長(zhǎng)度，這樣可以利用 GPU 進(jìn)行并行計(jì)算
• 添加 eos 并移動(dòng)一個(gè) token
  • teacher forcing 技術(shù): 為了訓(xùn)練模型根據(jù)前綴預(yù)測(cè)下一個(gè) token，我們將移動(dòng)后的目標(biāo)序列作為解碼器的輸入。
  • 一般來(lái)說(shuō)，在目標(biāo)前面加上 bos 就可以了（如下圖所示）
    
  • 但是在 fairseq 中，這是通過(guò)將 eos token 移動(dòng)到開(kāi)頭來(lái)實(shí)現(xiàn)的。在實(shí)驗(yàn)上，這個(gè)操作擁有相同的效果。例如:

  # 目標(biāo)輸出（target）和解碼器輸入（prev_output_tokens）:
                 eos = 2
              target = 419,  711,  238,  888,  792,   60,  968,    8,    2
  prev_output_tokens = 2,  419,  711,  238,  888,  792,   60,  968,    8
  

def load_data_iterator(task, split, epoch=1, max_tokens=4000, num_workers=1, cached=True):
    batch_iterator = task.get_batch_iterator(
        dataset=task.dataset(split),
        max_tokens=max_tokens,
        max_sentences=None,
        max_positions=utils.resolve_max_positions(
            task.max_positions(),
            max_tokens,
        ),
        ignore_invalid_inputs=True,
        seed=seed,
        num_workers=num_workers,
        epoch=epoch,
        disable_iterator_cache=not cached,
        # 如果設(shè)置為 False（cached=True），可以加快訓(xùn)練速度。
        # 但是，如果設(shè)置為 False，那么在第一次調(diào)用這個(gè)方法之后，再改變 max_tokens就沒(méi)有效果了。
    )
    return batch_iterator

demo_epoch_obj = load_data_iterator(task, "valid", epoch=1, max_tokens=20, num_workers=1, cached=False)
demo_iter = demo_epoch_obj.next_epoch_itr(shuffle=True)
sample = next(demo_iter)
sample

• each batch is a python dict, with string key and Tensor value. Contents are described below:

batch = {
    "id": id, # id for each example
    "nsentences": len(samples), # batch size (sentences)
    "ntokens": ntokens, # batch size (tokens)
    "net_input": {
        "src_tokens": src_tokens, # sequence in source language
        "src_lengths": src_lengths, # sequence length of each example before padding
        "prev_output_tokens": prev_output_tokens, # right shifted target, as mentioned above.
    },
    "target": target, # target sequence
}

模型架構(gòu)

• 我們?cè)俅卫^承 fairseq 的編碼器、解碼器和模型，以便在測(cè)試階段可以直接利用 fairseq 的集束搜索解碼器。

from fairseq.models import (
    FairseqEncoder,
    FairseqIncrementalDecoder,
    FairseqEncoderDecoderModel
)

編碼器

• 編碼器（Encoder）是一個(gè)循環(huán)神經(jīng)網(wǎng)絡(luò)（RNN）或者 Transformer 中的編碼器。下面的描述是針對(duì) RNN 的。對(duì)于每一個(gè)輸入的 token，編碼器會(huì)生成一個(gè)輸出向量和一個(gè)隱藏狀態(tài)向量，并且將隱藏狀態(tài)向量傳遞給下一步。換句話說(shuō)，編碼器順序地讀入輸入序列，并且在每一個(gè)時(shí)間步輸出一個(gè)單獨(dú)的向量，然后在最后一個(gè)時(shí)間步輸出最終的隱藏狀態(tài)，或者稱為內(nèi)容向量（content vector）。

• 參數(shù):

  • args
    • encoder_embed_dim: 嵌入的維度，將 one-hot 向量壓縮到固定的維度，實(shí)現(xiàn)降維的效果
    • encoder_ffn_embed_dim: 隱藏狀態(tài)和輸出向量的維度
    • encoder_layers: RNN 編碼器的層數(shù)
    • dropout 確定了一個(gè)神經(jīng)元的激活值被設(shè)為 0 的概率，用于防止過(guò)擬合。通常這個(gè)參數(shù)在訓(xùn)練時(shí)使用，在測(cè)試時(shí)移除
  • dictionary: fairseq 提供的字典。它用于獲取填充索引，進(jìn)而得到編碼器的填充掩碼（encoder padding mask）
  • embed_tokens: 一個(gè) token embedding 的實(shí)例（nn.Embedding）

• Inputs:

  • src_tokens: 一個(gè)表示英語(yǔ)的整數(shù)序列，例如: 1, 28, 29, 205, 2

• Outputs:

  • outputs: RNN 在每個(gè)時(shí)間步的輸出，可以由注意力機(jī)制（Attention）進(jìn)一步處理
  • final_hiddens: 每個(gè)時(shí)間步的隱藏狀態(tài)，會(huì)被傳遞給解碼器（decoder）進(jìn)行解碼
  • encoder_padding_mask: 這個(gè)參數(shù)告訴解碼器哪些位置要忽略

class RNNEncoder(FairseqEncoder):
    def __init__(self, args, dictionary, embed_tokens):
        super().__init__(dictionary)
        self.embed_tokens = embed_tokens

        self.embed_dim = args.encoder_embed_dim
        self.hidden_dim = args.encoder_ffn_embed_dim
        self.num_layers = args.encoder_layers

        self.dropout_in_module = nn.Dropout(args.dropout)
        self.rnn = nn.GRU(
            self.embed_dim,
            self.hidden_dim,
            self.num_layers,
            dropout=args.dropout,
            batch_first=False,
            bidirectional=True
        )
        self.dropout_out_module = nn.Dropout(args.dropout)

        self.padding_idx = dictionary.pad()

    def combine_bidir(self, outs, bsz: int):
        out = outs.view(self.num_layers, 2, bsz, -1).transpose(1, 2).contiguous()
        return out.view(self.num_layers, bsz, -1)

    def forward(self, src_tokens, **unused):
        bsz, seqlen = src_tokens.size()

        # 獲取 embeddings
        x = self.embed_tokens(src_tokens)
        x = self.dropout_in_module(x)

        # B x T x C -> T x B x C
        x = x.transpose(0, 1)

        # 經(jīng)過(guò)雙向的 RNN
        h0 = x.new_zeros(2 * self.num_layers, bsz, self.hidden_dim)
        x, final_hiddens = self.rnn(x, h0)
        outputs = self.dropout_out_module(x)
        # outputs = [sequence len, batch size, hid dim * directions]
        # hidden =  [num_layers * directions, batch size  , hid dim]

        # 由于編碼器是雙向的，我們需要將兩個(gè)方向的隱藏狀態(tài)連接起來(lái)
        final_hiddens = self.combine_bidir(final_hiddens, bsz)
        # hidden =  [num_layers x batch x num_directions*hidden]

        encoder_padding_mask = src_tokens.eq(self.padding_idx).t()
        return tuple(
            (
                outputs,  # seq_len x batch x hidden
                final_hiddens,  # num_layers x batch x num_directions*hidden
                encoder_padding_mask,  # seq_len x batch
            )
        )

    def reorder_encoder_out(self, encoder_out, new_order):
        # 這個(gè)被用于 fairseq 的集束搜索。它的具體細(xì)節(jié)和原因并不重要。
        return tuple(
            (
                encoder_out[0].index_select(1, new_order),
                encoder_out[1].index_select(1, new_order),
                encoder_out[2].index_select(1, new_order),
            )
        )

注意力

• 當(dāng)輸入序列很長(zhǎng)時(shí)，單獨(dú)的“內(nèi)容向量”就不能準(zhǔn)確地表示整個(gè)序列，注意力機(jī)制可以為解碼器提供更多信息。

• 根據(jù)當(dāng)前時(shí)間步的解碼器embeddings，將編碼器輸出與解碼器 embeddings 進(jìn)行匹配，確定相關(guān)性，然后將編碼器輸出按相關(guān)性加權(quán)求和作為解碼器 RNN 的輸入。

• 常見(jiàn)的注意力實(shí)現(xiàn)使用神經(jīng)網(wǎng)絡(luò)/點(diǎn)積作為 query（解碼器 embeddings）和 key（編碼器輸出）之間的相關(guān)性，然后用 softmax 得到一個(gè)分布，最后用該分布對(duì) value（編碼器輸出）進(jìn)行加權(quán)求和。

• 參數(shù):

  • input_embed_dim: key 的維度，應(yīng)該是解碼器中用于 attend 其他向量的向量的維度
  • source_embed_dim: query 的維度，應(yīng)該是被 attend 的向量（編碼器輸出）的維度
  • output_embed_dim: value 的維度，應(yīng)該是 after attention 的向量的維度，符合下一層的期望,

• Inputs:

  • inputs: key, 用于 attend 其他向量
  • encoder_outputs: query/value, 被 attend 的向量
  • encoder_padding_mask: 這個(gè)告訴解碼器應(yīng)該忽略那些位置

• Outputs:

  • output: attention 后的上下文向量
  • attention score: attention 的分?jǐn)?shù)

class AttentionLayer(nn.Module):
    def __init__(self, input_embed_dim, source_embed_dim, output_embed_dim, bias=False):
        super().__init__()

        self.input_proj = nn.Linear(input_embed_dim, source_embed_dim, bias=bias)
        self.output_proj = nn.Linear(
            input_embed_dim + source_embed_dim, output_embed_dim, bias=bias
        )

    def forward(self, inputs, encoder_outputs, encoder_padding_mask):
        # inputs: T, B, dim
        # encoder_outputs: S x B x dim
        # padding mask:  S x B

        # 將所有的輸入的維度改為 batch first
        inputs = inputs.transpose(1,0) # B, T, dim
        encoder_outputs = encoder_outputs.transpose(1,0) # B, S, dim
        encoder_padding_mask = encoder_padding_mask.transpose(1,0) # B, S

        # 投影到 encoder_outputs 的維度
        x = self.input_proj(inputs)

        # 計(jì)算 attention
        # (B, T, dim) x (B, dim, S) = (B, T, S)
        attn_scores = torch.bmm(x, encoder_outputs.transpose(1,2))

        # 取消與 padding 相對(duì)應(yīng)的位置的 attention
        if encoder_padding_mask is not None:
            # leveraging broadcast  B, S -> (B, 1, S)
            encoder_padding_mask = encoder_padding_mask.unsqueeze(1)
            attn_scores = (
                attn_scores.float()
                .masked_fill_(encoder_padding_mask, float("-inf"))
                .type_as(attn_scores)
            )  # FP16 support: cast to float and back

        # 在與源序列對(duì)應(yīng)的維度上進(jìn)行 softmax
        attn_scores = F.softmax(attn_scores, dim=-1)

        # shape (B, T, S) x (B, S, dim) = (B, T, dim) 加權(quán)求和
        x = torch.bmm(attn_scores, encoder_outputs)

        # (B, T, dim)
        x = torch.cat((x, inputs), dim=-1)
        x = torch.tanh(self.output_proj(x)) # concat + linear + tanh

        # restore shape (B, T, dim) -> (T, B, dim)
        return x.transpose(1,0), attn_scores

解碼器

• 解碼器的隱藏狀態(tài)將由編碼器的最終隱藏狀態(tài)（the content vector）初始化
• 同時(shí)，解碼器會(huì)根據(jù)當(dāng)前時(shí)間步的輸入（前一時(shí)間步的輸出）改變其隱藏狀態(tài)，并生成一個(gè)輸出
• 注意力機(jī)制可以提高性能
• seq2seq 的步驟是在解碼器中實(shí)現(xiàn)的，這樣以后 Seq2Seq 類(lèi)可以接受 RNN 和 Transformer，而不需要進(jìn)一步修改。

• 參數(shù):
  • args
    • decoder_embed_dim: 解碼器嵌入的維度，類(lèi)似于 encoder_embed_dim
    • decoder_ffn_embed_dim: 解碼器 RNN 隱藏狀態(tài)的維度，類(lèi)似于 encoder_ffn_embed_dim
    • decoder_layers: RNN 解碼器的層數(shù)
    • share_decoder_input_output_embed: 通常，解碼器的投影矩陣會(huì)與解碼器輸入 embeddings 共享權(quán)重
  • dictionary: fairseq 提供的字典
  • embed_tokens: 一個(gè) token embedding 的實(shí)例（nn.Embedding）
• 輸入:
  • prev_output_tokens: 表示右移目標(biāo)的整數(shù)序列，例如: 1, 28, 29, 205, 2
  • encoder_out: 編碼器的輸出
  • incremental_state: 為了加速測(cè)試時(shí)的解碼，我們會(huì)保存每個(gè)時(shí)間步的隱藏狀態(tài)。詳見(jiàn)forward()。
• 輸出:
  • outputs: 解碼器在每個(gè)時(shí)間步的輸出的對(duì)數(shù)（softmax之前）
  • extra: 未使用

class RNNDecoder(FairseqIncrementalDecoder):
    def __init__(self, args, dictionary, embed_tokens):
        super().__init__(dictionary)
        self.embed_tokens = embed_tokens

        assert args.decoder_layers == args.encoder_layers, f"""seq2seq rnn requires that encoder
        and decoder have same layers of rnn. got: {args.encoder_layers, args.decoder_layers}"""
        assert args.decoder_ffn_embed_dim == args.encoder_ffn_embed_dim*2, f"""seq2seq-rnn requires
        that decoder hidden to be 2*encoder hidden dim. got: {args.decoder_ffn_embed_dim, args.encoder_ffn_embed_dim*2}"""

        self.embed_dim = args.decoder_embed_dim
        self.hidden_dim = args.decoder_ffn_embed_dim
        self.num_layers = args.decoder_layers


        self.dropout_in_module = nn.Dropout(args.dropout)
        self.rnn = nn.GRU(
            self.embed_dim,
            self.hidden_dim,
            self.num_layers,
            dropout=args.dropout,
            batch_first=False,
            bidirectional=False
        )
        self.attention = AttentionLayer(
            self.embed_dim, self.hidden_dim, self.embed_dim, bias=False
        )
        # self.attention = None
        self.dropout_out_module = nn.Dropout(args.dropout)

        if self.hidden_dim != self.embed_dim:
            self.project_out_dim = nn.Linear(self.hidden_dim, self.embed_dim)
        else:
            self.project_out_dim = None

        if args.share_decoder_input_output_embed:
            self.output_projection = nn.Linear(
                self.embed_tokens.weight.shape[1],
                self.embed_tokens.weight.shape[0],
                bias=False,
            )
            self.output_projection.weight = self.embed_tokens.weight
        else:
            self.output_projection = nn.Linear(
                self.output_embed_dim, len(dictionary), bias=False
            )
            nn.init.normal_(
                self.output_projection.weight, mean=0, std=self.output_embed_dim ** -0.5
            )

    def forward(self, prev_output_tokens, encoder_out, incremental_state=None, **unused):
        # 從編碼器中提取輸出
        encoder_outputs, encoder_hiddens, encoder_padding_mask = encoder_out
        # outputs:          seq_len x batch x num_directions*hidden
        # encoder_hiddens:  num_layers x batch x num_directions*encoder_hidden
        # padding_mask:     seq_len x batch

        if incremental_state is not None and len(incremental_state) > 0:
            # 如果保留了上一個(gè)時(shí)間步的信息，可以從那里繼續(xù)，而不是從bos開(kāi)始
            prev_output_tokens = prev_output_tokens[:, -1:]
            cache_state = self.get_incremental_state(incremental_state, "cached_state")
            prev_hiddens = cache_state["prev_hiddens"]
        else:
            # 增量狀態(tài)不存在，要么是訓(xùn)練時(shí)，要么是測(cè)試時(shí)的第一個(gè)時(shí)間步
            # 為seq2seq做準(zhǔn)備：將編碼器的隱藏狀態(tài)傳遞給解碼器的隱藏狀態(tài)
            prev_hiddens = encoder_hiddens

        bsz, seqlen = prev_output_tokens.size()

        # embed tokens
        x = self.embed_tokens(prev_output_tokens)
        x = self.dropout_in_module(x)

        # B x T x C -> T x B x C
        x = x.transpose(0, 1)

        # decoder-to-encoder attention
        if self.attention is not None:
            x, attn = self.attention(x, encoder_outputs, encoder_padding_mask)

        # 經(jīng)過(guò)單向的 RNN
        x, final_hiddens = self.rnn(x, prev_hiddens)
        # outputs = [sequence len, batch size, hid dim]
        # hidden =  [num_layers * directions, batch size  , hid dim]
        x = self.dropout_out_module(x)

        # 投影到 embedding size（如果隱藏狀態(tài)與 embedding size 不同，并且 share_embedding 為T(mén)rue
        # 就需要做一個(gè)額外的投影）
        if self.project_out_dim != None:
            x = self.project_out_dim(x)

        # 投影到 vocab size
        x = self.output_projection(x)

        # T x B x C -> B x T x C
        x = x.transpose(1, 0)

        # if incremental, 則記錄當(dāng)前時(shí)間步的隱藏狀態(tài)，在下一個(gè)時(shí)間步恢復(fù)
        cache_state = {
            "prev_hiddens": final_hiddens,
        }
        self.set_incremental_state(incremental_state, "cached_state", cache_state)

        return x, None

    def reorder_incremental_state(self, incremental_state, new_order):
        # 這個(gè)被用于 fairseq 的集束搜索。它的具體細(xì)節(jié)和原因并不重要。
        cache_state = self.get_incremental_state(incremental_state, "cached_state")
        prev_hiddens = cache_state["prev_hiddens"]
        prev_hiddens = [p.index_select(0, new_order) for p in prev_hiddens]
        cache_state = {
            "prev_hiddens": torch.stack(prev_hiddens),
        }
        self.set_incremental_state(incremental_state, "cached_state", cache_state)
        return

Seq2Seq

• 由編碼器和解碼器組成
• 接收輸入并傳遞給編碼器
• 將編碼器的輸出傳遞給解碼器
• 解碼器會(huì)根據(jù)前一時(shí)間步的輸出以及編碼器的輸出進(jìn)行解碼
• 解碼完成后，返回解碼器的輸出

class Seq2Seq(FairseqEncoderDecoderModel):
    def __init__(self, args, encoder, decoder):
        super().__init__(encoder, decoder)
        self.args = args

    def forward(
        self,
        src_tokens,
        src_lengths,
        prev_output_tokens,
        return_all_hiddens: bool = True,
    ):
        """
        Run the forward pass for an encoder-decoder model.
        """
        encoder_out = self.encoder(
            src_tokens, src_lengths=src_lengths, return_all_hiddens=return_all_hiddens
        )
        logits, extra = self.decoder(
            prev_output_tokens,
            encoder_out=encoder_out,
            src_lengths=src_lengths,
            return_all_hiddens=return_all_hiddens,
        )
        return logits, extra

模型初始化

# # 提示: transformer 架構(gòu)
from fairseq.models.transformer import (
    TransformerEncoder,
    TransformerDecoder,
)

def build_model(args, task):
    """ build a model instance based on hyperparameters """
    src_dict, tgt_dict = task.source_dictionary, task.target_dictionary

    # token embeddings
    encoder_embed_tokens = nn.Embedding(len(src_dict), args.encoder_embed_dim, src_dict.pad())
    decoder_embed_tokens = nn.Embedding(len(tgt_dict), args.decoder_embed_dim, tgt_dict.pad())

    # encoder decoder
    # 提示: TODO: 轉(zhuǎn)變?yōu)?TransformerEncoder & TransformerDecoder
    encoder = RNNEncoder(args, src_dict, encoder_embed_tokens)
    decoder = RNNDecoder(args, tgt_dict, decoder_embed_tokens)
    # encoder = TransformerEncoder(args, src_dict, encoder_embed_tokens)
    # decoder = TransformerDecoder(args, tgt_dict, decoder_embed_tokens)

    # 序列到序列的模型
    model = Seq2Seq(args, encoder, decoder)

    # 初始化 seq2seq 模型很重要, 需要額外的處理
    def init_params(module):
        from fairseq.modules import MultiheadAttention
        if isinstance(module, nn.Linear):
            module.weight.data.normal_(mean=0.0, std=0.02)
            if module.bias is not None:
                module.bias.data.zero_()
        if isinstance(module, nn.Embedding):
            module.weight.data.normal_(mean=0.0, std=0.02)
            if module.padding_idx is not None:
                module.weight.data[module.padding_idx].zero_()
        if isinstance(module, MultiheadAttention):
            module.q_proj.weight.data.normal_(mean=0.0, std=0.02)
            module.k_proj.weight.data.normal_(mean=0.0, std=0.02)
            module.v_proj.weight.data.normal_(mean=0.0, std=0.02)
        if isinstance(module, nn.RNNBase):
            for name, param in module.named_parameters():
                if "weight" in name or "bias" in name:
                    param.data.uniform_(-0.1, 0.1)

    # 權(quán)重初始化
    model.apply(init_params)
    return model

架構(gòu)相關(guān)配置

為了達(dá)成 strong baseline，請(qǐng)參考 Attention is all you need 中表 3 中 transformer-base 的超參數(shù)

arch_args = Namespace(
    encoder_embed_dim=256,
    encoder_ffn_embed_dim=512,
    encoder_layers=1,
    decoder_embed_dim=256,
    decoder_ffn_embed_dim=1024,
    decoder_layers=1,
    share_decoder_input_output_embed=True,
    dropout=0.3,
)

# 提示: 這些是 Transformer 的參數(shù)補(bǔ)丁
def add_transformer_args(args):
    args.encoder_attention_heads=4
    args.encoder_normalize_before=True

    args.decoder_attention_heads=4
    args.decoder_normalize_before=True

    args.activation_fn="relu"
    args.max_source_positions=1024
    args.max_target_positions=1024

    # Transformer 默認(rèn)參數(shù)的補(bǔ)?。ㄎ丛谏厦嬖O(shè)置的參數(shù)）
    from fairseq.models.transformer import base_architecture
    base_architecture(arch_args)

# add_transformer_args(arch_args)

if config.use_wandb:
    wandb.config.update(vars(arch_args))

model = build_model(arch_args, task)
logger.info(model)

優(yōu)化

損失(Loss): Label Smoothing Regularization

• 讓模型學(xué)習(xí)生成更少集中的分布，防止過(guò)度自信
• 有時(shí)候正確答案可能不是唯一的。因此，在計(jì)算損失時(shí)，我們?yōu)殄e(cuò)誤標(biāo)簽保留一些概率。
• 避免過(guò)擬合

代碼 source

class LabelSmoothedCrossEntropyCriterion(nn.Module):
    def __init__(self, smoothing, ignore_index=None, reduce=True):
        super().__init__()
        self.smoothing = smoothing
        self.ignore_index = ignore_index
        self.reduce = reduce

    def forward(self, lprobs, target):
        if target.dim() == lprobs.dim() - 1:
            target = target.unsqueeze(-1)
        # nll: Negative log likelihood 負(fù)對(duì)數(shù)似然，當(dāng)目標(biāo)是 one-hot 時(shí)的交叉熵。下一行代碼等同于F.nll_loss
        nll_loss = -lprobs.gather(dim=-1, index=target)
        #  保留一些其他標(biāo)簽的概率，這樣在計(jì)算交叉熵的時(shí)候相當(dāng)于對(duì)所有標(biāo)簽的對(duì)數(shù)概率求和
        smooth_loss = -lprobs.sum(dim=-1, keepdim=True)
        if self.ignore_index is not None:
            pad_mask = target.eq(self.ignore_index)
            nll_loss.masked_fill_(pad_mask, 0.0)
            smooth_loss.masked_fill_(pad_mask, 0.0)
        else:
            nll_loss = nll_loss.squeeze(-1)
            smooth_loss = smooth_loss.squeeze(-1)
        if self.reduce:
            nll_loss = nll_loss.sum()
            smooth_loss = smooth_loss.sum()
        # 在計(jì)算交叉熵的時(shí)候，增加其他標(biāo)簽的損失
        eps_i = self.smoothing / lprobs.size(-1)
        loss = (1.0 - self.smoothing) * nll_loss + eps_i * smooth_loss
        return loss

# 通常來(lái)說(shuō)，0.1 已經(jīng)足夠好了
criterion = LabelSmoothedCrossEntropyCriterion(
    smoothing=0.1,
    ignore_index=task.target_dictionary.pad(),
)

優(yōu)化器: Adam + 學(xué)習(xí)率調(diào)度

在訓(xùn)練 Transformer 時(shí)，平方根倒數(shù)調(diào)度（Inverse square root scheduling）對(duì)于穩(wěn)定性非常重要，在后面也用于RNN。
根據(jù)以下公式更新學(xué)習(xí)率，第一階段線性增加，然后按時(shí)間步的平方根倒數(shù)成比例衰減。
$$lrate=d_{\text{model}}^{?0.5}?\min ({step\_num}^{?0.5},step\_num?{warmup\_steps}^{?1.5})$$

def get_rate(d_model, step_num, warmup_step):
    # TODO: 將 lr 從常數(shù)修改為上面顯示的公式
    lr = 0.001
    return lr

class NoamOpt:
    "Optim 包裝，用于實(shí)現(xiàn) rate"
    def __init__(self, model_size, factor, warmup, optimizer):
        self.optimizer = optimizer
        self._step = 0
        self.warmup = warmup
        self.factor = factor
        self.model_size = model_size
        self._rate = 0

    @property
    def param_groups(self):
        return self.optimizer.param_groups

    def multiply_grads(self, c):
        """將梯度乘以常數(shù)*c*."""
        for group in self.param_groups:
            for p in group['params']:
                if p.grad is not None:
                    p.grad.data.mul_(c)

    def step(self):
        "更新參數(shù)和 rate"
        self._step += 1
        rate = self.rate()
        for p in self.param_groups:
            p['lr'] = rate
        self._rate = rate
        self.optimizer.step()

    def rate(self, step = None):
        "實(shí)現(xiàn)上面的 `lrate`"
        if step is None:
            step = self._step
        return 0 if not step else self.factor * get_rate(self.model_size, step, self.warmup)

調(diào)度可視化

optimizer = NoamOpt(
    model_size=arch_args.encoder_embed_dim,
    factor=config.lr_factor,
    warmup=config.lr_warmup,
    optimizer=torch.optim.AdamW(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9, weight_decay=0.0001))
plt.plot(np.arange(1, 100000), [optimizer.rate(i) for i in range(1, 100000)])
plt.legend([f"{optimizer.model_size}:{optimizer.warmup}"])
None

訓(xùn)練過(guò)程

訓(xùn)練

from fairseq.data import iterators
from torch.cuda.amp import GradScaler, autocast

def train_one_epoch(epoch_itr, model, task, criterion, optimizer, accum_steps=1):
    itr = epoch_itr.next_epoch_itr(shuffle=True)
    itr = iterators.GroupedIterator(itr, accum_steps) # 梯度累積：每 accum_steps 個(gè)樣本更新一次

    stats = {"loss": []}
    scaler = GradScaler() # 自動(dòng)混合精度（amp）

    model.train()
    progress = tqdm.tqdm(itr, desc=f"train epoch {epoch_itr.epoch}", leave=False)
    for samples in progress:
        model.zero_grad()
        accum_loss = 0
        sample_size = 0
        # 梯度累積：每 accum_steps 個(gè)樣本更新一次
        for i, sample in enumerate(samples):
            if i == 1:
                # 在第一步之后清空 CUDA 緩存可以減少 OOM（out of memory）的機(jī)會(huì)
                torch.cuda.empty_cache()

            sample = utils.move_to_cuda(sample, device=device)
            target = sample["target"]
            sample_size_i = sample["ntokens"]
            sample_size += sample_size_i

            # 混合精度訓(xùn)練
            with autocast():
                net_output = model.forward(**sample["net_input"])
                lprobs = F.log_softmax(net_output[0], -1)
                loss = criterion(lprobs.view(-1, lprobs.size(-1)), target.view(-1))

                # 日志記錄
                accum_loss += loss.item()
                # 反向傳播
                scaler.scale(loss).backward()

        scaler.unscale_(optimizer)
        optimizer.multiply_grads(1 / (sample_size or 1.0)) # (sample_size or 1.0) 處理零梯度的情況
        gnorm = nn.utils.clip_grad_norm_(model.parameters(), config.clip_norm) # 梯度裁剪防止梯度爆炸

        scaler.step(optimizer)
        scaler.update()

        # 日志記錄
        loss_print = accum_loss/sample_size
        stats["loss"].append(loss_print)
        progress.set_postfix(loss=loss_print)
        if config.use_wandb:
            wandb.log({
                "train/loss": loss_print,
                "train/grad_norm": gnorm.item(),
                "train/lr": optimizer.rate(),
                "train/sample_size": sample_size,
            })

    loss_print = np.mean(stats["loss"])
    logger.info(f"training loss: {loss_print:.4f}")
    return stats

驗(yàn)證 & 推測(cè)

為了防止過(guò)擬合，每個(gè)訓(xùn)練周期都需要進(jìn)行驗(yàn)證，以驗(yàn)證模型在未見(jiàn)過(guò)的數(shù)據(jù)上的性能。

• 該過(guò)程與訓(xùn)練基本相同，只是多了一個(gè)推測(cè)步驟。
• 在驗(yàn)證后，我們可以保存模型的權(quán)重。

僅憑驗(yàn)證損失無(wú)法描述模型的實(shí)際性能

• 基于當(dāng)前模型直接生成翻譯假設(shè)，然后使用參考翻譯計(jì)算BLEU
• 我們也可以手動(dòng)檢查假設(shè)的質(zhì)量
• 我們使用 fairseq 的序列生成器進(jìn)行集束搜索以生成翻譯假設(shè)。

# fairseq 的集束搜索生成器
# 給定模型和輸入序列，通過(guò)集束搜索生成翻譯假設(shè)
sequence_generator = task.build_generator([model], config)

def decode(toks, dictionary):
    # 將 Tensor 轉(zhuǎn)換為人類(lèi)可讀的句子
    s = dictionary.string(
        toks.int().cpu(),
        config.post_process,
    )
    return s if s else "<unk>"

def inference_step(sample, model):
    gen_out = sequence_generator.generate([model], sample)
    srcs = []
    hyps = []
    refs = []
    for i in range(len(gen_out)):
        # 對(duì)于每個(gè)樣本，收集輸入、假設(shè)和參考，稍后用于計(jì)算 BLEU
        srcs.append(decode(
            utils.strip_pad(sample["net_input"]["src_tokens"][i], task.source_dictionary.pad()),
            task.source_dictionary,
        ))
        hyps.append(decode(
            gen_out[i][0]["tokens"], # 0 表示使用集束中的最佳假設(shè)
            task.target_dictionary,
        ))
        refs.append(decode(
            utils.strip_pad(sample["target"][i], task.target_dictionary.pad()),
            task.target_dictionary,
        ))
    return srcs, hyps, refs

import shutil
import sacrebleu

def validate(model, task, criterion, log_to_wandb=True):
    logger.info('begin validation')
    itr = load_data_iterator(task, "valid", 1, config.max_tokens, config.num_workers).next_epoch_itr(shuffle=False)

    stats = {"loss":[], "bleu": 0, "srcs":[], "hyps":[], "refs":[]}
    srcs = []
    hyps = []
    refs = []

    model.eval()
    progress = tqdm.tqdm(itr, desc=f"validation", leave=False)
    with torch.no_grad():
        for i, sample in enumerate(progress):
            # 驗(yàn)證損失
            sample = utils.move_to_cuda(sample, device=device)
            net_output = model.forward(**sample["net_input"])

            lprobs = F.log_softmax(net_output[0], -1)
            target = sample["target"]
            sample_size = sample["ntokens"]
            loss = criterion(lprobs.view(-1, lprobs.size(-1)), target.view(-1)) / sample_size
            progress.set_postfix(valid_loss=loss.item())
            stats["loss"].append(loss)

            # 做推測(cè)
            s, h, r = inference_step(sample, model)
            srcs.extend(s)
            hyps.extend(h)
            refs.extend(r)

    tok = 'zh' if task.cfg.target_lang == 'zh' else '13a'
    stats["loss"] = torch.stack(stats["loss"]).mean().item()
    stats["bleu"] = sacrebleu.corpus_bleu(hyps, [refs], tokenize=tok) # 計(jì)算BLEU score
    stats["srcs"] = srcs
    stats["hyps"] = hyps
    stats["refs"] = refs

    if config.use_wandb and log_to_wandb:
        wandb.log({
            "valid/loss": stats["loss"],
            "valid/bleu": stats["bleu"].score,
        }, commit=False)

    showid = np.random.randint(len(hyps))
    logger.info("example source: " + srcs[showid])
    logger.info("example hypothesis: " + hyps[showid])
    logger.info("example reference: " + refs[showid])

    # 顯示 bleu 結(jié)果
    logger.info(f"validation loss:\t{stats['loss']:.4f}")
    logger.info(stats["bleu"].format())
    return stats

保存和加載模型權(quán)重

def validate_and_save(model, task, criterion, optimizer, epoch, save=True):
    stats = validate(model, task, criterion)
    bleu = stats['bleu']
    loss = stats['loss']
    if save:
        # 保存 epoch checkpoints
        savedir = Path(config.savedir).absolute()
        savedir.mkdir(parents=True, exist_ok=True)

        check = {
            "model": model.state_dict(),
            "stats": {"bleu": bleu.score, "loss": loss},
            "optim": {"step": optimizer._step}
        }
        torch.save(check, savedir/f"checkpoint{epoch}.pt")
        shutil.copy(savedir/f"checkpoint{epoch}.pt", savedir/f"checkpoint_last.pt")
        logger.info(f"saved epoch checkpoint: {savedir}/checkpoint{epoch}.pt")

        # 保存 epoch 樣本
        with open(savedir/f"samples{epoch}.{config.source_lang}-{config.target_lang}.txt", "w") as f:
            for s, h in zip(stats["srcs"], stats["hyps"]):
                f.write(f"{s}\t{h}\n")

        # 獲取最佳的驗(yàn)證 bleu
        if getattr(validate_and_save, "best_bleu", 0) < bleu.score:
            validate_and_save.best_bleu = bleu.score
            torch.save(check, savedir/f"checkpoint_best.pt")

        del_file = savedir / f"checkpoint{epoch - config.keep_last_epochs}.pt"
        if del_file.exists():
            del_file.unlink()

    return stats

def try_load_checkpoint(model, optimizer=None, name=None):
    name = name if name else "checkpoint_last.pt"
    checkpath = Path(config.savedir)/name
    if checkpath.exists():
        check = torch.load(checkpath)
        model.load_state_dict(check["model"])
        stats = check["stats"]
        step = "unknown"
        if optimizer != None:
            optimizer._step = step = check["optim"]["step"]
        logger.info(f"loaded checkpoint {checkpath}: step={step} loss={stats['loss']} bleu={stats['bleu']}")
    else:
        logger.info(f"no checkpoints found at {checkpath}!")

Main

訓(xùn)練循環(huán)

model = model.to(device=device)
criterion = criterion.to(device=device)

logger.info("task: {}".format(task.__class__.__name__))
logger.info("encoder: {}".format(model.encoder.__class__.__name__))
logger.info("decoder: {}".format(model.decoder.__class__.__name__))
logger.info("criterion: {}".format(criterion.__class__.__name__))
logger.info("optimizer: {}".format(optimizer.__class__.__name__))
logger.info(
    "num. model params: {:,} (num. trained: {:,})".format(
        sum(p.numel() for p in model.parameters()),
        sum(p.numel() for p in model.parameters() if p.requires_grad),
    )
)
logger.info(f"max tokens per batch = {config.max_tokens}, accumulate steps = {config.accum_steps}")

epoch_itr = load_data_iterator(task, "train", config.start_epoch, config.max_tokens, config.num_workers)
try_load_checkpoint(model, optimizer, name=config.resume)
while epoch_itr.next_epoch_idx <= config.max_epoch:
    # 訓(xùn)練一個(gè) epoch
    train_one_epoch(epoch_itr, model, task, criterion, optimizer, config.accum_steps)
    stats = validate_and_save(model, task, criterion, optimizer, epoch=epoch_itr.epoch)
    logger.info("end of epoch {}".format(epoch_itr.epoch))
    epoch_itr = load_data_iterator(task, "train", epoch_itr.next_epoch_idx, config.max_tokens, config.num_workers)

Submission

# 對(duì)幾個(gè) checkpoints 進(jìn)行平均可以產(chǎn)生類(lèi)似于 ensemble 的效果
checkdir=config.savedir
!python ./fairseq/scripts/average_checkpoints.py \
--inputs {checkdir} \
--num-epoch-checkpoints 5 \
--output {checkdir}/avg_last_5_checkpoint.pt

確定用于生成 submission 的模型權(quán)重

# checkpoint_last.pt : 最新的 epoch
# checkpoint_best.pt : 最高的驗(yàn)證 BLEU
# avg_last_5_checkpoint.pt: 最近 5 次 epoch 的平均值
try_load_checkpoint(model, name="avg_last_5_checkpoint.pt")
validate(model, task, criterion, log_to_wandb=False)
None

生成預(yù)測(cè)

def generate_prediction(model, task, split="test", outfile="./prediction.txt"):
    task.load_dataset(split=split, epoch=1)
    itr = load_data_iterator(task, split, 1, config.max_tokens, config.num_workers).next_epoch_itr(shuffle=False)

    idxs = []
    hyps = []

    model.eval()
    progress = tqdm.tqdm(itr, desc=f"prediction")
    with torch.no_grad():
        for i, sample in enumerate(progress):
            # 驗(yàn)證損失
            sample = utils.move_to_cuda(sample, device=device)

            # 做推測(cè)
            s, h, r = inference_step(sample, model)

            hyps.extend(h)
            idxs.extend(list(sample['id']))

    # 根據(jù)預(yù)處理前的順序進(jìn)行排序
    hyps = [x for _,x in sorted(zip(idxs,hyps))]

    with open(outfile, "w") as f:
        for h in hyps:
            f.write(h+"\n")

generate_prediction(model, task)

raise

Back-translation

訓(xùn)練一個(gè) backward translation 模型

1. 將 config 中的 source_lang 和 target_lang 進(jìn)行切換
2. 更改 config 中的 savedir(例如: “./checkpoints/transformer-back”)
3. 訓(xùn)練模型

用后向模型生成人造數(shù)據(jù)

下載單語(yǔ)言數(shù)據(jù)

mono_dataset_name = 'mono'

mono_prefix = Path(data_dir).absolute() / mono_dataset_name
mono_prefix.mkdir(parents=True, exist_ok=True)

urls = (
    "https://github.com/figisiwirf/ml2023-hw5-dataset/releases/download/v1.0.1/ted_zh_corpus.deduped.gz",
)
file_names = (
    'ted_zh_corpus.deduped.gz',
)

for u, f in zip(urls, file_names):
    path = mono_prefix/f
    if not path.exists():
        !wget {u} -O {path}
    else:
        print(f'{f} is exist, skip downloading')
    if path.suffix == ".tgz":
        !tar -xvf {path} -C {prefix}
    elif path.suffix == ".zip":
        !unzip -o {path} -d {prefix}
    elif path.suffix == ".gz":
        !gzip -fkd {path}

TODO: 清洗語(yǔ)料

1. 移除太長(zhǎng)或者太短的句子
2. 統(tǒng)一標(biāo)點(diǎn)符號(hào)

提示: 你可以使用之前定義的 clean_s() 來(lái)執(zhí)行此操作

TODO: 子詞單位

使用后向模型的 spm 模型將數(shù)據(jù)標(biāo)記為子詞單位

提示: spm 模型位于 DATA/raw-data/[dataset]/spm[vocab_num].model

二值化

使用 fairseq 去二值化數(shù)據(jù)

binpath = Path('./DATA/data-bin', mono_dataset_name)
src_dict_file = './DATA/data-bin/ted2020/dict.en.txt'
tgt_dict_file = src_dict_file
monopref = str(mono_prefix/"mono.tok") # whatever filepath you get after applying subword tokenization
if binpath.exists():
    print(binpath, "exists, will not overwrite!")
else:
    !python -m fairseq_cli.preprocess\
        --source-lang 'zh'\
        --target-lang 'en'\
        --trainpref {monopref}\
        --destdir {binpath}\
        --srcdict {src_dict_file}\
        --tgtdict {tgt_dict_file}\
        --workers 2

TODO: 用后向模型生成人造數(shù)據(jù)

將二進(jìn)制化的單語(yǔ)言數(shù)據(jù)添加到原始數(shù)據(jù)目錄中，并將其命名為 “split_name”

例如: ./DATA/data-bin/ted2020/[split_name].zh-en.[“en”, “zh”].[“bin”, “idx”]

然后你可以使用 ‘generate_prediction(model, task, split=“split_name”)’ 來(lái)生成翻譯的預(yù)測(cè)

# 將二進(jìn)制化的單語(yǔ)言數(shù)據(jù)添加到原始數(shù)據(jù)目錄中，并將其命名為 "split_name"
# 例如: ./DATA/data-bin/ted2020/\[split_name\].zh-en.\["en", "zh"\].\["bin", "idx"\]
!cp ./DATA/data-bin/mono/train.zh-en.zh.bin ./DATA/data-bin/ted2020/mono.zh-en.zh.bin
!cp ./DATA/data-bin/mono/train.zh-en.zh.idx ./DATA/data-bin/ted2020/mono.zh-en.zh.idx
!cp ./DATA/data-bin/mono/train.zh-en.en.bin ./DATA/data-bin/ted2020/mono.zh-en.en.bin
!cp ./DATA/data-bin/mono/train.zh-en.en.idx ./DATA/data-bin/ted2020/mono.zh-en.en.idx

# hint: 在 split='mono' 上做預(yù)測(cè)來(lái)創(chuàng)建 prediction_file
# generate_prediction( ... ,split=... ,outfile=... )

TODO: 創(chuàng)建新的數(shù)據(jù)集

1. 將預(yù)測(cè)數(shù)據(jù)和單語(yǔ)數(shù)據(jù)結(jié)合
2. 使用原始的 spm 模型將數(shù)據(jù) tokenize 為子詞單位
3. 使用 fairseq 將數(shù)據(jù)二值化

# 將 prediction_file (.en) 和 mono.zh (.zh) 結(jié)合為新的數(shù)據(jù)集
#
# 提示: 用 spm 模型 tokenize prediction_file
# spm_model.encode(line, out_type=str)
# 輸出: ./DATA/rawdata/mono/mono.tok.en & mono.tok.zh
#
# 提示: 使用 fairseq 再次二值化這兩個(gè)文件
# binpath = Path('./DATA/data-bin/synthetic')
# src_dict_file = './DATA/data-bin/ted2020/dict.en.txt'
# tgt_dict_file = src_dict_file
# monopref = ./DATA/rawdata/mono/mono.tok # or whatever path after applying subword tokenization, w/o the suffix (.zh/.en)
# if binpath.exists():
#     print(binpath, "exists, will not overwrite!")
# else:
#     !python -m fairseq_cli.preprocess\
#         --source-lang 'zh'\
#         --target-lang 'en'\
#         --trainpref {monopref}\
#         --destdir {binpath}\
#         --srcdict {src_dict_file}\
#         --tgtdict {tgt_dict_file}\
#         --workers 2

# 根據(jù)上面準(zhǔn)備的所有文件創(chuàng)建一個(gè)新的數(shù)據(jù)集
!cp -r ./DATA/data-bin/ted2020/ ./DATA/data-bin/ted2020_with_mono/

!cp ./DATA/data-bin/synthetic/train.zh-en.zh.bin ./DATA/data-bin/ted2020_with_mono/train1.en-zh.zh.bin
!cp ./DATA/data-bin/synthetic/train.zh-en.zh.idx ./DATA/data-bin/ted2020_with_mono/train1.en-zh.zh.idx
!cp ./DATA/data-bin/synthetic/train.zh-en.en.bin ./DATA/data-bin/ted2020_with_mono/train1.en-zh.en.bin
!cp ./DATA/data-bin/synthetic/train.zh-en.en.idx ./DATA/data-bin/ted2020_with_mono/train1.en-zh.en.idx

創(chuàng)建新數(shù)據(jù)集 “ted2020_with_mono”文章來(lái)源地址http://www.zghlxwxcb.cn/news/detail-742687.html

1. 修改 config 中的 datadir (“./DATA/data-bin/ted2020_with_mono”)
2. 將 config 中的 source_lang 和 target_lang 進(jìn)行切換 (“en”, “zh”)
3. 更改 config 中的 savedir (例如: “./checkpoints/transformer-bt”)
4. 訓(xùn)練模型

References

1. Ott, M., Edunov, S., Baevski, A., Fan, A., Gross, S., Ng, N., … & Auli, M. (2019, June). fairseq: A Fast, Extensible Toolkit for Sequence Modeling. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics (Demonstrations) (pp. 48-53).
2. Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A. N., … & Polosukhin, I. (2017, December). Attention is all you need. In Proceedings of the 31st International Conference on Neural Information Processing Systems (pp. 6000-6010).
3. Reimers, N., & Gurevych, I. (2020, November). Making Monolingual Sentence Embeddings Multilingual Using Knowledge Distillation. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP) (pp. 4512-4525).
4. Tiedemann, J. (2012, May). Parallel Data, Tools and Interfaces in OPUS. In Lrec (Vol. 2012, pp. 2214-2218).
5. Kudo, T., & Richardson, J. (2018, November). SentencePiece: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing: System Demonstrations (pp. 66-71).
6. Sennrich, R., Haddow, B., & Birch, A. (2016, August). Improving Neural Machine Translation Models with Monolingual Data. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) (pp. 86-96).
7. Edunov, S., Ott, M., Auli, M., & Grangier, D. (2018). Understanding Back-Translation at Scale. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (pp. 489-500).
8. https://github.com/ajinkyakulkarni14/TED-Multilingual-Parallel-Corpus
9. https://ithelp.ithome.com.tw/articles/10233122
10. https://nlp.seas.harvard.edu/2018/04/03/attention.html
11. https://colab.research.google.com/github/ga642381/ML2021-Spring/blob/main/HW05/HW05.ipynb

到了這里，關(guān)于2023李宏毅機(jī)器學(xué)習(xí)HW05樣例代碼中文注釋版的文章就介紹完了。如果您還想了解更多內(nèi)容，請(qǐng)?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！