2023全球人工智能技术创新大赛-影像学NLP赛题baseline

import pandas as pd
test = pd.read_csv('data/data201684/preliminary_a_test.csv',header=None,names=['yingxiang'])
train = pd.read_csv('data/data201684/train.csv',header=None,names=['yingxiang', 'zhenduan'])print('train:',train.shape)print('test:',test.shape)

train.head(5)

test.head(5)

test['len'] = test['yingxiang'].apply(lambda x:len(x.split()))
train['len'] = train['yingxiang'].apply(lambda x:len(x.split()))
train['len2'] = train['zhenduan'].apply(lambda x:len(x.split()))

train['len'].hist(),test['len'].hist()

train['len'].describe()

test['len'].describe()

train['len2'].hist()

train['len2'].describe()

from collections import Counter
l = train['yingxiang'].apply(lambda x:x.split()).tolist()
l = [i for j in l for i in j]

c = Counter(l)

df = pd.DataFrame({'key':list(c.keys()),'value':list(c.values())})
df['key'] = df['key'].astype('int')
df.sort_values('key')

from collections import Counter
l = test['yingxiang'].apply(lambda x:x.split()).tolist()
l = [i for j in l for i in j]

c = Counter(l)

df = pd.DataFrame({'key':list(c.keys()),'value':list(c.values())})
df['key'] = df['key'].astype('int')
df.sort_values('key')

from collections import Counter
l = train['zhenduan'].apply(lambda x:x.split()).tolist()
l = [i for j in l for i in j]

c = Counter(l)

df = pd.DataFrame({'key':list(c.keys()),'value':list(c.values())})
df['key'] = df['key'].astype('int')
df.sort_values('key')

import timeimport osimport numpy as npfrom tqdm import tqdmfrom functools import partialimport pandas as pd# 官方baseline中的评分标准CiderDfrom eval import CiderDfrom visualdl import LogWriterfrom datasets import load_datasetfrom paddlenlp.transformers import AutoTokenizer,AutoModelForConditionalGeneration,LinearDecayWithWarmupfrom paddlenlp.utils.log import loggerfrom paddlenlp.data import DataCollatorForSeq2Seqimport paddlefrom paddle.io import BatchSampler, DistributedBatchSampler, DataLoaderfrom paddle import nn# 开始字符bos_token_id = 1# 结束字符eos_token_id = 2# 补全字符pad_token_id = 0# 训练模型的保存路径save_dir = 'checkpoints2'# 最高分数的记录best_score = 0# 影像描述的最大长度max_source_length = 160# 诊断报告的最大长度max_target_length = 90# 诊断报告的最小长度min_target_length = 0# 训练轮次num_epochs = 5# 训练中，每个log_steps打印一次日志log_steps = 50# 训练中，每隔eval_steps进行一次模型评估eval_steps = 300# 设置batch_sizetrain_batch_size = 16 dev_batch_size = 64test_batch_size = 64

# 定义句子还原函数def array2str(arr):
    out = ''
    for i in range(len(arr)):        # 遇到结束标记就停止
        if arr[i]==eos_token_id or arr[i]==pad_token_id:            break
        # 遇到开始标记就继续
        if arr[i]==bos_token_id:            continue
        out += str(int(arr[i])) + ' '
    if len(out.strip())==0:
        out = '0'
    return out.strip()

array2str([1,10,11,12,13,2,0,0,0,0,0,0,0])

# 定义损失函数def CE(output, target):
    '''
    Output: (B,L,C)。未经过softmax的logits
    Target: (B,L)
    '''
    # print(target)
    # reshape 不同
    output = output.reshape((-1, output.shape[-1]))  # (*,C)
    target = target.reshape((-1,))#.long()  # (*)
    return nn.CrossEntropyLoss()(output, target) #默认size_average=True，会把B*L所有词loss平均

# 加载模型和tokenizer,batchify_fmodel_name = 'IDEA-CCNL/Randeng-Pegasus-238M-Summary-Chinese'tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForConditionalGeneration.from_pretrained(model_name, # 如果有预训练，换成预训练保存的文件夹名称 ‘pre’
                                                            vocab_size=1500, # 因为词表和原来不同，所以需要修改模型的token数量
                                                            bos_token_id = bos_token_id,# 重新定义
                                                            eos_token_id = eos_token_id,# 重新定义
                                                            decoder_start_token_id = eos_token_id,# 重新定义
                                                            forced_eos_token_id = eos_token_id,# 重新定义
                                                            pad_token_id = pad_token_id)
batchify_fn = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)

test = pd.read_csv('data/data201684/preliminary_a_test.csv',header=None)
test[2] = '0 0'test[[1,2]].to_csv('test.csv',header = None,index=False)

train = pd.read_csv('data/data201684/train.csv',header=None)
train[[1,2]].head(18000).to_csv('train.csv',header = None,index=False)
train[[1,2]].tail(2000).to_csv('valid.csv',header = None,index=False)

train_dataset = load_dataset("csv", data_files='train.csv',names=['yingxiang', 'zhenduan'], split="train")
dev_dataset = load_dataset("csv", data_files='valid.csv',names=['yingxiang', 'zhenduan'], split="train")
test_dataset = load_dataset("csv", data_files='test.csv',names=['yingxiang', 'zhenduan'], split="train")

# 处理数据的函数，因为本次直接是脱敏数据成数字，所以就不需要tokenizer再做处理了，直接拼接开始和结束符号作为输入def convert_example(example, text_column, summary_column):
    """
    构造模型的输入
    """
    inputs = example[text_column].split()
    inputs = [bos_token_id]+[int(i) for i in inputs]
    inputs.append(eos_token_id)
    
    targets = example[summary_column].split()
    targets = [bos_token_id]+[int(i) for i in targets]
    targets.append(eos_token_id)
    model_inputs = {}

    model_inputs["input_ids"] = inputs
    model_inputs["attention_mask"] = [1]*len(inputs)
    model_inputs["labels"] = targets    return model_inputs# 原始字段需要移除remove_columns = ['yingxiang', 'zhenduan']# 定义转换器trans_func = partial(convert_example,
                     text_column='yingxiang',
                     summary_column='zhenduan')                     
# train_dataset和dev_dataset分别转换train_dataset = train_dataset.map(trans_func,batched=False,# 对每条数据逐个处理
                                  load_from_cache_file=True,remove_columns=remove_columns)
dev_dataset = dev_dataset.map(trans_func,batched=False,
                              load_from_cache_file=True,remove_columns=remove_columns)
test_dataset = test_dataset.map(trans_func,batched=False,
                              load_from_cache_file=True,remove_columns=remove_columns)

# 分布式批采样器，用于多卡分布式训练train_batch_sampler = DistributedBatchSampler(train_dataset, batch_size=train_batch_size, shuffle=True)# 构造训练训练集Dataloadertrain_data_loader = DataLoader(dataset=train_dataset,batch_sampler=train_batch_sampler,
                               num_workers=0,collate_fn=batchify_fn,return_list=True)


dev_batch_sampler = BatchSampler(dev_dataset,batch_size=dev_batch_size,shuffle=False)
dev_data_loader = DataLoader(dataset=dev_dataset,batch_sampler=dev_batch_sampler,
                             num_workers=0,collate_fn=batchify_fn,return_list=True)

test_batch_sampler = BatchSampler(test_dataset,batch_size=test_batch_size,shuffle=False)
test_data_loader = DataLoader(dataset=test_dataset,batch_sampler=test_batch_sampler,
                            num_workers=0,collate_fn=batchify_fn,return_list=True)for idx, example in enumerate(dev_data_loader):    print(example)    break

# 学习率预热比例warmup_proportion = 0.02# 学习率learning_rate = 1e-4# 训练总步数num_training_steps = len(train_data_loader) * num_epochs# AdamW优化器参数epsilonadam_epsilon = 1e-6# AdamW优化器参数weight_decayweight_decay=0.01# 可视化log_writer = LogWriter('visualdl_log_dir')
lr_scheduler = LinearDecayWithWarmup(learning_rate, num_training_steps, warmup_proportion)# LayerNorm参数不参与weight_decaydecay_params = [
    p.name for n, p in model.named_parameters()    if not any(nd in n for nd in ["bias", "norm"])
]# 优化器AdamWoptimizer = paddle.optimizer.AdamW(
    learning_rate=lr_scheduler,
    beta1=0.9,
    beta2=0.999,
    epsilon=adam_epsilon,
    parameters=model.parameters(),
    weight_decay=weight_decay,
    apply_decay_param_fun=lambda x: x in decay_params)

# 模型评估函数@paddle.no_grad()def evaluate(model, data_loader, tokenizer, min_target_length,max_target_length):
    model.eval()
    model = model._layers if isinstance(model, paddle.DataParallel) else model

    res, gts = [], {}
    tot = 0
    for batch in tqdm(data_loader):
        targets = batch['labels']
        pred = model.generate(input_ids=batch['input_ids'],
                               attention_mask=batch['attention_mask'],
                               min_length=min_target_length,
                               max_length=max_target_length,
                               use_cache=True,
                               length_penalty=0.7,
                               decode_strategy='beam_search',
                               num_beams=5,
                               early_stopping=True)[0]
        pred = pred.cpu().numpy()        #print(pred.shape)
        for i in range(pred.shape[0]):
            res.append({'image_id':tot, 'caption': [array2str(pred[i])]})
            gts[tot] = [array2str(targets[i])]
            tot += 1

    CiderD_scorer = CiderD(df='corpus', sigma=15)
    cider_score, cider_scores = CiderD_scorer.compute_score(gts, res)    print('cid',cider_score)    return cider_score

# 定义FGM对抗训练

class FGM:
    def __init__(self, model, eps=1.):
        self.model = (model.module if hasattr(model, "module") else model)
        self.eps = eps
        self.backup = {}    # only attack embedding
    def attack(self, emb_name='embedding'):
        for name, param in self.model.named_parameters():            if param.stop_gradient and emb_name in name:
                self.backup[name] = param.data.clone()
                norm = paddle.norm(param.grad)                if norm and not paddle.isnan(norm):
                    r_at = self.eps * param.grad / norm
                    param.data.add_(r_at)    def restore(self, emb_name='embedding'):
        for name, para in self.model.named_parameters():            if para.stop_gradient and emb_name in name:                assert name in self.backup
                para.data = self.backup[name]

        self.backup = {}        
# 初始化fgm = FGM(model)

# 定义EMA，本次比赛尝试没有效果

# class ExponentialMovingAverage():#     def __init__(self, model, decay, thres_steps=True):#         self._model = model#         self._decay = decay#         self._thres_steps = thres_steps#         self._shadow = {}#         self._backup = {}#     def register(self):#         self._update_step = 0#         for name, param in self._model.named_parameters():#             if param.stop_gradient is False:   # 只记录可训练参数。bn层的均值、方差的stop_gradient默认是True，所以不会记录bn层的均值、方差。#                 self._shadow[name] = param.numpy().copy()#     def update(self):#         for name, param in self._model.named_parameters():#             if param.stop_gradient is False:#                 assert name in self._shadow#                 new_val = np.array(param.numpy().copy())#                 old_val = np.array(self._shadow[name])#                 decay = min(self._decay, (1 + self._update_step) / (10 + self._update_step)) if self._thres_steps else self._decay#                 new_average = decay * old_val + (1 - decay) * new_val#                 self._shadow[name] = new_average#         self._update_step += 1#         return decay#     def apply(self):#         for name, param in self._model.named_parameters():#             if param.stop_gradient is False:#                 assert name in self._shadow#                 self._backup[name] = np.array(param.numpy().copy())#                 param.set_value(np.array(self._shadow[name]))#     def restore(self):#         for name, param in self._model.named_parameters():#             if param.stop_gradient is False:#                 assert name in self._backup#                 param.set_value(self._backup[name])#         self._backup = {}# ema = ExponentialMovingAverage(model, 0.9998)# ema.register()

# 如果有训练好的模型参数，可以直接加载# state_dict = paddle.load('checkpoints2/model_state.pdparams')# model.set_dict(state_dict)

global_step = 0tic_train0 = time.time()
tic_train = time.time()for epoch in range(num_epochs):    for step, batch in enumerate(train_data_loader):
        global_step += 1
        # 模型前向训练，计算loss
        lm_logits, _, loss = model(**batch)
        loss.backward()

        fgm.attack() # 在embedding上添加对抗扰动
        lm_logits, _, loss = model(**batch)
        loss.backward()
        fgm.restore() # 恢复embedding参数

        optimizer.step()
        lr_scheduler.step()
        optimizer.clear_grad()        # ema.update()
        
        if global_step % log_steps == 0:
            logger.info('global step {}/{}, epoch: {}, loss: {}, lr: {}, speed: {} s/step, already: {}min, remain: {}min'.format(
                        global_step,
                        num_training_steps,
                        epoch,                        round(loss.item(),10), # 目前损失
                        round(optimizer.get_lr(),10), # 目前学习率
                        round((time.time() - tic_train)/log_steps,3), # 每步耗时
                        round((time.time() - tic_train0)/60,3), # 总耗时
                        round((num_training_steps-global_step)/log_steps*(time.time() - tic_train)/60),3)) # 预计耗时
            tic_train = time.time()        if global_step % eval_steps== 0 and global_step >= 0:
            tic_eval = time.time()
            score = evaluate(model, dev_data_loader, tokenizer,min_target_length, max_target_length)
            logger.info("eval done total : %s s" % (time.time() - tic_eval))            if best_score < score:
                best_score = score                if paddle.distributed.get_rank() == 0:                    if not os.path.exists(save_dir):
                        os.makedirs(save_dir)                    # Need better way to get inner model of DataParallel
                    model_to_save = model._layers if isinstance(
                        model, paddle.DataParallel) else model
                    model_to_save.save_pretrained(save_dir)
                    tokenizer.save_pretrained(save_dir)

# 模型评估函数@paddle.no_grad()def evaluate(model, data_loader, tokenizer, min_target_length,max_target_length):
    model.eval()
    model = model._layers if isinstance(model, paddle.DataParallel) else model

    res, gts = [], {}
    tot = 0
    for batch in tqdm(data_loader):
        targets = batch['labels']
        pred = model.generate(input_ids=batch['input_ids'],
                            attention_mask=batch['attention_mask'],
                            min_length=min_target_length,
                            max_length=max_target_length,
                            use_cache=True,
                            length_penalty=0.7,
                            decode_strategy='beam_search',
                            num_beams=5,
                            early_stopping=True)[0]
        pred = pred.cpu().numpy()        for i in range(pred.shape[0]):
            res.append({'image_id':tot, 'caption': [array2str(pred[i])]})
            gts[tot] = [array2str(targets[i])]
            tot += 1

    CiderD_scorer = CiderD(df='corpus', sigma=15)
    cider_score, cider_scores = CiderD_scorer.compute_score(gts, res)    print('cid',cider_score)    return cider_score

# ema.apply()evaluate(model, dev_data_loader, tokenizer,min_target_length, max_target_length)# state_dict = model.state_dict()# paddle.save(state_dict, "paddle_dy_ema.pdparams")

# state_dict = paddle.load('checkpoints/model_state.pdparams')# model.set_dict(state_dict)

# 模型评估函数@paddle.no_grad()def pre(model, data_loader, tokenizer, min_target_length,max_target_length):
    model.eval()
    all_preds = []
    model = model._layers if isinstance(model, paddle.DataParallel) else model    for batch in tqdm(data_loader, total=len(data_loader), desc="Eval step"):
        labels = batch.pop('labels').numpy()        # 模型生成
        preds = model.generate(input_ids=batch['input_ids'],
                               attention_mask=batch['attention_mask'],
                               min_length=min_target_length,
                               max_length=max_target_length,
                               use_cache=True,
                               length_penalty=0.7,
                               decode_strategy='beam_search',
                               num_beams=5)[0]        # tokenizer将id转为string
        for i in range(len(preds)):
            all_preds.append(array2str(preds[i]))        # break
    # print(all_preds, all_labels)
    # CiderD_scorer = CiderD(df='corpus', sigma=15)
    # cider_score = CiderD_scorer.compute_score(all_preds, all_labels)
    # model.train()
    return all_preds#rouge1, rouge2, rougel, bleu4

pred = pre(model, test_data_loader, tokenizer, min_target_length, max_target_length)

df = pd.DataFrame(pred)
df[1] = df.index
df.columns = ['prediction','report_ID']
df[['report_ID','prediction']].to_csv('pre.csv',index=False,header=None)# 输出预测文件，下载提交即可

<br/>