深度学习文本处理：XLNet编码TypeError及Tokenizer配置指南

train_input_ids,train_attention_masks = xlnet_encode(train[:50000],60)

def xlnet_encode(data,maximum_length) :
    input_ids = []
    attention_masks = []
    # 这里缺少了核心的编码逻辑和return语句

import pandas as pd
import torch
from transformers import XLNetTokenizer

# 初始化XLNet Tokenizer
# 'xlnet-base-cased' 是一个常用的预训练模型名称
tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')

def xlnet_encode(texts, tokenizer, maximum_length):
    """
    使用XLNet Tokenizer对文本数据进行编码。

    参数:
        texts (list or pd.Series): 待编码的文本列表或Pandas Series。
        tokenizer (XLNetTokenizer): 已初始化的XLNet Tokenizer实例。
        maximum_length (int): 序列的最大长度，用于填充和截断。

    返回:
        tuple: 包含input_ids和attention_masks的元组，均为PyTorch张量。
    """
    input_ids_list = []
    attention_masks_list = []

    for text in texts:
        # 使用tokenizer.encode_plus进行编码
        # add_special_tokens: 添加 [CLS], [SEP] 等特殊token
        # max_length: 序列最大长度
        # padding='max_length': 填充到max_length
        # truncation=True: 启用截断
        # return_attention_mask: 返回注意力掩码
        # return_tensors='pt': 返回PyTorch张量
        encoded_dict = tokenizer.encode_plus(
                            str(text), # 确保输入是字符串类型
                            add_special_tokens = True,
                            max_length = maximum_length,
                            padding = 'max_length',
                            truncation = True,
                            return_attention_mask = True,
                            return_tensors = 'pt',
                       )

        input_ids_list.append(encoded_dict['input_ids'])
        attention_masks_list.append(encoded_dict['attention_mask'])

    # 将列表中的PyTorch张量堆叠成一个大的张量
    input_ids = torch.cat(input_ids_list, dim=0)
    attention_masks = torch.cat(attention_masks_list, dim=0)

    return input_ids, attention_masks

import pandas as pd
import torch
from transformers import XLNetTokenizer

# 假设您的数据文件位于Kaggle环境中
# train = pd.read_csv('/kaggle/input/twitter2/train.csv', lineterminator='\n')
# test = pd.read_csv('/kaggle/input/twitter2/test.csv', lineterminator='\n')

# 为了示例可运行，我们创建模拟数据
train_data = {
    'tweet': [
        'i need this for when my wife and i live in our...',
        'why we never saw alfredhitchcock s bond and th...',
        'oh my gosh the excitement of coming back from ...',
        'because its monday and im missing him a little...',
        'so to cwnetwork for having the current episode...'
    ],
    'gender': [1, 0, 1, 1, 1]
}
test_data = {
    'tweet': [
        'the opposite of faith is not doubt its absolu...',
        'wen yu really value somethingyu stay commited ...',
        'today was such a bad day i wish i could text t...',
        'so i took a nap amp had the weirdest dream lit...',
        'savagejaspy i like the purple but you seem mor...'
    ],
    'gender': [1, 1, 1, 0, 1]
}
train = pd.DataFrame(train_data)
test = pd.DataFrame(test_data)


print("Train DataFrame Head:")
print(train.head())
print("\nTest DataFrame Head:")
print(test.head())

# 1. 初始化XLNet Tokenizer
print("\nInitializing XLNet Tokenizer...")
tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
print("Tokenizer initialized successfully.")

# 2. 定义编码函数
def xlnet_encode(texts, tokenizer, maximum_length):
    input_ids_list = []
    attention_masks_list = []

    for text in texts:
        encoded_dict = tokenizer.encode_plus(
                            str(text), # 确保输入是字符串
                            add_special_tokens = True,
                            max_length = maximum_length,
                            padding = 'max_length',
                            truncation = True,
                            return_attention_mask = True,
                            return_tensors = 'pt',
                       )
        input_ids_list.append(encoded_dict['input_ids'])
        attention_masks_list.append(encoded_dict['attention_mask'])

    input_ids = torch.cat(input_ids_list, dim=0)
    attention_masks = torch.cat(attention_masks_list, dim=0)

    return input_ids, attention_masks

# 3. 调用编码函数进行数据处理
# 从DataFrame中提取'tweet'列作为文本数据
train_texts = train['tweet'].values
test_texts = test['tweet'].values

# 设定最大长度
MAX_LEN = 60

print(f"\nEncoding training data (first {len(train_texts)} samples) with MAX_LEN={MAX_LEN}...")
train_input_ids, train_attention_masks = xlnet_encode(train_texts, tokenizer, MAX_LEN)

print(f"Encoding test data (first {len(test_texts)} samples) with MAX_LEN={MAX_LEN}...")
test_input_ids, test_attention_masks = xlnet_encode(test_texts, tokenizer, MAX_LEN)

print("\nEncoding complete. Check output shapes:")
print("Train Input IDs shape:", train_input_ids.shape)        # 预期输出: (样本数, MAX_LEN)
print("Train Attention Masks shape:", train_attention_masks.shape) # 预期输出: (样本数, MAX_LEN)
print("Test Input IDs shape:", test_input_ids.shape)
print("Test Attention Masks shape:", test_attention_masks.shape)

# 您现在可以使用这些 input_ids 和 attention_masks 来训练您的XLNet模型