1 自然语言推理任务

1.1 任务说明

确定在给定前提下，一个假设是必然的（真）、矛盾的（假）还是未定的（中性）

自然语言推理任务的样本数据集

1.2 代码

snli数据集文档: https://huggingface.co/datasets/snli
pipline文档: https://huggingface.co/docs/transformers/main_classes/pipelines
查看所有评估指标: evaluate.list_evaluation_modules("metric")

from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments
from transformers import DataCollatorWithPadding
from nlp import load_dataset
import evaluate
import numpy as np
from transformers import pipeline
from transformers.pipelines.pt_utils import KeyDataset, KeyPairDataset

# entailment: 包含, neutral: 中性, contradiction: 矛盾
id2label = {0: "entailment", 1: "neutral", 2: "contradiction"}
label2id = {"entailment": 0, "neutral": 1, "contradiction": 2}


def train():
    # snli数据集: https://huggingface.co/datasets/snli
    # snli数据集可视化：https://huggingface.co/datasets/viewer/?dataset=snli
    dataset = load_dataset('snli', split=['train[:10%]', 'validation[:100]'])
    print('dataset: {}'.format(dataset))

    train_set = dataset[0].filter(lambda x: x['label'] != -1)
    test_set = dataset[1].filter(lambda x: x['label'] != -1)
    print('train_set[0]: {}'.format(train_set[0]))
    print('test_set[0]: {}'.format(test_set[0]))
    print('train_set: {}'.format(train_set))
    print('test_set: {}'.format(test_set))

    model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=3, id2label=id2label, label2id=label2id)
    tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')

    def preprocess_function(examples):
        return tokenizer(examples['premise'], examples['hypothesis'], truncation=True)

    train_set = train_set.map(preprocess_function, batched=True)
    test_set = test_set.map(preprocess_function, batched=True)

    # 按一个batch的最大长度补齐，而非整个数据集的最大长度补齐
    data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

    # 查看所有评估指标: evaluate.list_evaluation_modules("metric")
    accuracy = evaluate.load('accuracy')

    def compute_metrics(eval_pred):
        predictions, labels = eval_pred
        predictions = np.argmax(predictions, axis=1)
        return accuracy.compute(predictions=predictions, references=labels)

    # 预训练参数
    training_args = TrainingArguments(
        output_dir='./results',
        logging_dir='./logs',
        optim='adamw_torch',
        learning_rate=2e-5,
        per_device_train_batch_size=8,
        per_device_eval_batch_size=8,
        num_train_epochs=10,
        weight_decay=0.01,
        warmup_steps=500,
        evaluation_strategy='epoch',
        save_strategy='epoch',
        load_best_model_at_end=True,
        metric_for_best_model='accuracy',
    )

    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_set,
        eval_dataset=test_set,
        tokenizer=tokenizer,
        data_collator=data_collator,
        compute_metrics=compute_metrics
    )

    trainer.train()
    trainer.evaluate()


def inference():
    # 推导，真实label=1/neutral，训练完这个case预测错了0v0
    premise = 'This church choir sings to the masses as they sing joyous songs from the book at a church.'
    hypothesis = 'The church has cracks in the ceiling.'
    # pipline类型: https://huggingface.co/docs/transformers/main_classes/pipelines
    classifier = pipeline('text-classification', model='./results/checkpoint-68700', tokenizer='bert-base-uncased')
    result = classifier({"text": premise, "text_pair": hypothesis}, top_k=None)
    print('premise: {}\nhypothesis: {}\nresult: {}\n'.format(premise, hypothesis, result))

    print('开始预测整个测试集')
    test_set = load_dataset('snli', split='test[:100]').filter(lambda x: x['label'] != -1)
    predictions = []
    for out in classifier(KeyPairDataset(test_set, 'premise', 'hypothesis')):
        predictions.append(label2id[out['label']])
    print('predictions: {}'.format(predictions))
    labels = []
    for label in KeyDataset(test_set, 'label'):
        labels.append(label)
    print('labels:      {}'.format(labels))
    # 有时候需要开代理, 不然会卡住
    accuracy = evaluate.load('accuracy')
    print(accuracy.compute(predictions=predictions, references=labels))
    # premise: This church choir sings to the masses as they sing joyous songs from the book at a church.
    # hypothesis: The church has cracks in the ceiling.
    # result: [{'label': 'contradiction', 'score': 0.9999877214431763}, {'label': 'neutral', 'score': 1.0622567970131058e-05}, {'label': 'entailment', 'score': 1.7016249103107839e-06}]
    # 开始预测整个测试集
    # predictions: [2, 0, 2, 1, 1, 1, 0, 1, 2, 1, 0, 2, 1, 0, 2, 2, 0, 1, 1, 2, 1, 2, 0, 1, 1, 2, 0, 0, 1, 2, 2, 1, 0, 0, 1, 2, 0, 0, 1, 0, 1, 2, 2, 1, 1, 1, 1, 0, 1, 2, 1, 0, 2, 1, 2, 0, 0, 2, 1, 0, 2, 1, 2, 0, 2, 0, 0, 2, 2, 1, 0, 2, 0, 1, 2, 0, 1, 0, 1, 0, 2, 2, 0, 0, 2, 1, 2, 1, 0, 1, 2, 0, 1, 0, 2, 2, 0, 1, 1]
    # labels:      [1, 0, 2, 1, 0, 2, 0, 1, 2, 1, 0, 2, 0, 0, 2, 2, 0, 1, 0, 2, 1, 2, 0, 1, 1, 2, 0, 0, 1, 2, 2, 0, 0, 0, 1, 2, 1, 0, 2, 0, 1, 2, 2, 0, 1, 1, 2, 0, 1, 2, 1, 0, 2, 1, 2, 0, 0, 2, 1, 0, 2, 1, 1, 0, 2, 0, 1, 1, 2, 1, 0, 2, 0, 1, 2, 0, 1, 0, 0, 2, 2, 2, 0, 0, 2, 1, 2, 1, 0, 1, 2, 0, 1, 0, 2, 2, 0, 1, 1]
    # {'accuracy': 0.8484848484848485}