BiLSTM + CRF进行NER.ipynb【附源码】

BiLSTM + CRF进行NER

1. 数据预处理

 将数据处理成深度学习能够使用的方式。如将word、label转化为id等。 

2. 划分验证集

 将训练数据再划分出一部分作为验证集，可监控训练过程并使用early stoping方式防止过拟合。

import re
import pickle
import codecs
import numpy as np

1. 数据预处理

将所有训练集中的字符与标签映射成id

ner_train_path = '../dataset/ner/ner.train'
ner_test_path = '../dataset/ner/ner.test'

dl_dicts_path = './warehouse/dicts.pkl'

def load_dataset(path):
    sentences = []
    labels = []
    sentence = []
    label = []
    with codecs.open(path, 'r', encoding='utf-8') as f:
        for line in f.readlines():        
            if line == '\n':
                sentences.append(sentence)
                sentence = []
                labels.append(label)
                label = []
            else:
                try:
                    if line[0] == ' ':
                        word, tag = ' ', line[:-1].split(' ')[-1]
                    else:
                        word, _, tag = line[:-1].split(' ')
                    word = re.sub('[0-9]', '0',  word.lower())
                    sentence.append(word)
                    label.append(tag)
                except:
                    print(line)
                    break
    return np.array(sentences), np.array(labels)

train_sentences, train_labels = load_dataset(ner_train_path)
test_sentences, test_labels = load_dataset(ner_test_path)
len(train_labels), len(test_labels)

total_words = list(set([x for sentence in train_sentences for x in sentence]))
total_words.insert(0, 'unk')
total_words.insert(0, 'pad')

total_labels = list(set([x for label in train_labels for x in label]) - {'O'})
total_labels.insert(0, 'O')
len(total_words), len(total_labels)

total_words[:10]

print(total_labels)

word_2_id = {w: index for index, w in enumerate(total_words)}
id_2_word = {index: w for w, index in word_2_id.items()}
assert word_2_id['pad'] == 0
assert word_2_id['unk'] == 1

label_2_id = {label: index for index, label in enumerate(total_labels)}
id_2_label = {index: label for label, index in label_2_id.items()}
assert label_2_id['O'] == 0

with codecs.open(dl_dicts_path, "wb") as f:
    pickle.dump([word_2_id, id_2_word, label_2_id, id_2_label], f)

2.划分验证集

从训练集中划分1/10出来作为验证集，并将所有数据集转化为id序列

dl_train_path = './data/dl_ner.train'
dl_val_path = './data/dl_ner.val'
dl_test_path = './data/dl_ner.test'

len(train_labels), len(test_labels)

count = len(train_labels)
split_index = count//10

indexs = np.arange(count)
np.random.shuffle(indexs)
train_indexs = indexs[split_index:]
val_indexs = indexs[:split_index]
val_indexs[:10]

train_sentences_splited = train_sentences[train_indexs]
train_labels_splited = train_labels[train_indexs]
val_sentences_splited = train_sentences[val_indexs]
val_labels_splited = train_labels[val_indexs]

len(train_labels_splited), len(val_labels_splited), len(test_labels)

def build_dl_data(sentences, labels, word_2_id, label_2_id):
    data = []
    for index in range(len(sentences)):
        sentence, label = sentences[index], labels[index]
        sentence_id = [word_2_id.get(w, 1) for w in sentence]
        label_id = [label_2_id.get(l, 0) for l in label]
        data.append([sentence, sentence_id, label_id])
    return data

train_data = build_dl_data(train_sentences_splited, train_labels_splited, word_2_id, label_2_id)
val_data = build_dl_data(val_sentences_splited, val_labels_splited, word_2_id, label_2_id)
test_data = build_dl_data(test_sentences, test_labels, word_2_id, label_2_id)

len(train_data), len(val_data), len(test_data)

print(val_data[0])

with codecs.open(dl_train_path, "wb") as f:
    pickle.dump(train_data, f)
    
with codecs.open(dl_val_path, "wb") as f:
    pickle.dump(val_data, f)
    
with codecs.open(dl_test_path, "wb") as f:
    pickle.dump(test_data, f)

3. 模型训练

!ls -alh data

更多文章请关注《万象专栏》