Kaggle/Natural-Language-Processing-with-Disaster-Tweets/main.ipynb at master

yukun-hh/Kaggle

Fork 0

Files

T

yukun-hh fc30281b3e 非kaggle内容一些自学的模型

2026-08-18 23:21:35 +08:00

135 KiB

Raw Permalink Blame History

In [ ]:

import pandas as pd
import torch
import torch.nn as nn
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from torch.utils.data import Dataset, DataLoader

In [ ]:

train_df = pd.read_csv('/kaggle/input/competitions/nlp-getting-started/train.csv')
test_df = pd.read_csv('/kaggle/input/competitions/nlp-getting-started/test.csv')
from sklearn.model_selection import train_test_split
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=42)
print(f"Train: {len(train_df)}, Val: {len(val_df)}, Test: {len(test_df)}")

In [4]:

model_name = "xlm-roberta-large"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.

config.json:   0%|          | 0.00/616 [00:00<?, ?B/s]

tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]

sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]

tokenizer.json: 0.00B [00:00, ?B/s]

model.safetensors:   0%|          | 0.00/2.24G [00:00<?, ?B/s]

Loading weights:   0%|          | 0/389 [00:00<?, ?it/s]

XLMRobertaForSequenceClassification LOAD REPORT from: xlm-roberta-large
Key                         | Status     | 
----------------------------+------------+-
lm_head.dense.weight        | UNEXPECTED | 
lm_head.dense.bias          | UNEXPECTED | 
lm_head.bias                | UNEXPECTED | 
roberta.pooler.dense.weight | UNEXPECTED | 
roberta.pooler.dense.bias   | UNEXPECTED | 
lm_head.layer_norm.bias     | UNEXPECTED | 
lm_head.layer_norm.weight   | UNEXPECTED | 
classifier.dense.weight     | MISSING    | 
classifier.dense.bias       | MISSING    | 
classifier.out_proj.weight  | MISSING    | 
classifier.out_proj.bias    | MISSING    | 

Notes:
- UNEXPECTED	:can be ignored when loading from different task/architecture; not ok if you expect identical arch.
- MISSING	:those params were newly initialized because missing from the checkpoint. Consider training on your downstream task.

In [5]:

import numpy as np
class NliDataset(Dataset):
    def __init__(self, df, tokenizer, max_length=128):
        self.df = df.reset_index(drop=True)
        self.tokenizer = tokenizer
        self.max_length = max_length

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        row = self.df.iloc[idx]
        strs=''
        if row['keyword'] is not np.nan:
            strs=strs+'['+row['keyword']+']'
        if row['location'] is not np.nan:
            strs=strs+'['+row['location']+']'
        strs=strs+row['text']
        encoding = self.tokenizer(
            strs,
            truncation=True,
            padding='max_length',
            max_length=self.max_length,
            return_tensors='pt'  # 返回 PyTorch Tensor
        )
        # 去掉 batch 维度（因为只处理单条）
        item = {
            'input_ids': encoding['input_ids'].squeeze(0),
            'attention_mask': encoding['attention_mask'].squeeze(0)
        }
        if 'target' in row:
            item['labels'] = torch.tensor(row['target'], dtype=torch.long)
        return item

batch_size = 16  # 根据显存调整，推荐使用 16 或 32
max_length = 128

train_dataset = NliDataset(train_df, tokenizer, max_length)
val_dataset = NliDataset(val_df, tokenizer, max_length)

train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)

In [6]:

for name, param in model.named_parameters():
    #if param.requires_grad:
    print(f"{name}: requires_grad = {param.requires_grad}")

classifier.dense.weight: requires_grad = True
classifier.dense.bias: requires_grad = True
classifier.out_proj.weight: requires_grad = True
classifier.out_proj.bias: requires_grad = True
roberta.embeddings.word_embeddings.weight: requires_grad = True
roberta.embeddings.token_type_embeddings.weight: requires_grad = True
roberta.embeddings.LayerNorm.weight: requires_grad = True
roberta.embeddings.LayerNorm.bias: requires_grad = True
roberta.embeddings.position_embeddings.weight: requires_grad = True
roberta.encoder.layer.0.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.0.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.0.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.0.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.0.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.0.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.0.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.0.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.0.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.0.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.0.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.0.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.0.output.dense.weight: requires_grad = True
roberta.encoder.layer.0.output.dense.bias: requires_grad = True
roberta.encoder.layer.0.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.0.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.1.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.1.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.1.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.1.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.1.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.1.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.1.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.1.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.1.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.1.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.1.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.1.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.1.output.dense.weight: requires_grad = True
roberta.encoder.layer.1.output.dense.bias: requires_grad = True
roberta.encoder.layer.1.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.1.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.2.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.2.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.2.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.2.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.2.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.2.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.2.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.2.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.2.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.2.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.2.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.2.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.2.output.dense.weight: requires_grad = True
roberta.encoder.layer.2.output.dense.bias: requires_grad = True
roberta.encoder.layer.2.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.2.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.3.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.3.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.3.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.3.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.3.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.3.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.3.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.3.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.3.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.3.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.3.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.3.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.3.output.dense.weight: requires_grad = True
roberta.encoder.layer.3.output.dense.bias: requires_grad = True
roberta.encoder.layer.3.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.3.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.4.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.4.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.4.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.4.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.4.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.4.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.4.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.4.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.4.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.4.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.4.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.4.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.4.output.dense.weight: requires_grad = True
roberta.encoder.layer.4.output.dense.bias: requires_grad = True
roberta.encoder.layer.4.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.4.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.5.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.5.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.5.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.5.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.5.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.5.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.5.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.5.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.5.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.5.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.5.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.5.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.5.output.dense.weight: requires_grad = True
roberta.encoder.layer.5.output.dense.bias: requires_grad = True
roberta.encoder.layer.5.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.5.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.6.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.6.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.6.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.6.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.6.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.6.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.6.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.6.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.6.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.6.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.6.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.6.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.6.output.dense.weight: requires_grad = True
roberta.encoder.layer.6.output.dense.bias: requires_grad = True
roberta.encoder.layer.6.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.6.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.7.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.7.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.7.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.7.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.7.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.7.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.7.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.7.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.7.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.7.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.7.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.7.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.7.output.dense.weight: requires_grad = True
roberta.encoder.layer.7.output.dense.bias: requires_grad = True
roberta.encoder.layer.7.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.7.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.8.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.8.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.8.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.8.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.8.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.8.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.8.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.8.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.8.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.8.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.8.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.8.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.8.output.dense.weight: requires_grad = True
roberta.encoder.layer.8.output.dense.bias: requires_grad = True
roberta.encoder.layer.8.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.8.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.9.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.9.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.9.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.9.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.9.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.9.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.9.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.9.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.9.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.9.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.9.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.9.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.9.output.dense.weight: requires_grad = True
roberta.encoder.layer.9.output.dense.bias: requires_grad = True
roberta.encoder.layer.9.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.9.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.10.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.10.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.10.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.10.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.10.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.10.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.10.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.10.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.10.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.10.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.10.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.10.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.10.output.dense.weight: requires_grad = True
roberta.encoder.layer.10.output.dense.bias: requires_grad = True
roberta.encoder.layer.10.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.10.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.11.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.11.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.11.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.11.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.11.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.11.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.11.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.11.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.11.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.11.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.11.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.11.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.11.output.dense.weight: requires_grad = True
roberta.encoder.layer.11.output.dense.bias: requires_grad = True
roberta.encoder.layer.11.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.11.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.12.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.12.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.12.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.12.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.12.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.12.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.12.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.12.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.12.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.12.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.12.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.12.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.12.output.dense.weight: requires_grad = True
roberta.encoder.layer.12.output.dense.bias: requires_grad = True
roberta.encoder.layer.12.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.12.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.13.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.13.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.13.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.13.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.13.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.13.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.13.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.13.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.13.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.13.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.13.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.13.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.13.output.dense.weight: requires_grad = True
roberta.encoder.layer.13.output.dense.bias: requires_grad = True
roberta.encoder.layer.13.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.13.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.14.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.14.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.14.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.14.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.14.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.14.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.14.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.14.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.14.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.14.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.14.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.14.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.14.output.dense.weight: requires_grad = True
roberta.encoder.layer.14.output.dense.bias: requires_grad = True
roberta.encoder.layer.14.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.14.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.15.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.15.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.15.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.15.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.15.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.15.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.15.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.15.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.15.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.15.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.15.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.15.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.15.output.dense.weight: requires_grad = True
roberta.encoder.layer.15.output.dense.bias: requires_grad = True
roberta.encoder.layer.15.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.15.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.16.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.16.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.16.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.16.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.16.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.16.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.16.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.16.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.16.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.16.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.16.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.16.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.16.output.dense.weight: requires_grad = True
roberta.encoder.layer.16.output.dense.bias: requires_grad = True
roberta.encoder.layer.16.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.16.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.17.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.17.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.17.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.17.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.17.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.17.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.17.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.17.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.17.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.17.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.17.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.17.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.17.output.dense.weight: requires_grad = True
roberta.encoder.layer.17.output.dense.bias: requires_grad = True
roberta.encoder.layer.17.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.17.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.18.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.18.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.18.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.18.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.18.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.18.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.18.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.18.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.18.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.18.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.18.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.18.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.18.output.dense.weight: requires_grad = True
roberta.encoder.layer.18.output.dense.bias: requires_grad = True
roberta.encoder.layer.18.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.18.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.19.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.19.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.19.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.19.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.19.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.19.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.19.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.19.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.19.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.19.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.19.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.19.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.19.output.dense.weight: requires_grad = True
roberta.encoder.layer.19.output.dense.bias: requires_grad = True
roberta.encoder.layer.19.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.19.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.20.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.20.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.20.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.20.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.20.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.20.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.20.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.20.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.20.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.20.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.20.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.20.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.20.output.dense.weight: requires_grad = True
roberta.encoder.layer.20.output.dense.bias: requires_grad = True
roberta.encoder.layer.20.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.20.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.21.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.21.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.21.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.21.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.21.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.21.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.21.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.21.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.21.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.21.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.21.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.21.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.21.output.dense.weight: requires_grad = True
roberta.encoder.layer.21.output.dense.bias: requires_grad = True
roberta.encoder.layer.21.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.21.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.22.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.22.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.22.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.22.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.22.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.22.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.22.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.22.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.22.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.22.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.22.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.22.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.22.output.dense.weight: requires_grad = True
roberta.encoder.layer.22.output.dense.bias: requires_grad = True
roberta.encoder.layer.22.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.22.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.23.attention.self.query.weight: requires_grad = True
roberta.encoder.layer.23.attention.self.query.bias: requires_grad = True
roberta.encoder.layer.23.attention.self.key.weight: requires_grad = True
roberta.encoder.layer.23.attention.self.key.bias: requires_grad = True
roberta.encoder.layer.23.attention.self.value.weight: requires_grad = True
roberta.encoder.layer.23.attention.self.value.bias: requires_grad = True
roberta.encoder.layer.23.attention.output.dense.weight: requires_grad = True
roberta.encoder.layer.23.attention.output.dense.bias: requires_grad = True
roberta.encoder.layer.23.attention.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.23.attention.output.LayerNorm.bias: requires_grad = True
roberta.encoder.layer.23.intermediate.dense.weight: requires_grad = True
roberta.encoder.layer.23.intermediate.dense.bias: requires_grad = True
roberta.encoder.layer.23.output.dense.weight: requires_grad = True
roberta.encoder.layer.23.output.dense.bias: requires_grad = True
roberta.encoder.layer.23.output.LayerNorm.weight: requires_grad = True
roberta.encoder.layer.23.output.LayerNorm.bias: requires_grad = True

In [7]:

!pip install torchao==0.16.0
from peft import LoraConfig, get_peft_model
target_layers = [8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23]
target_modules=[]
for layer in target_layers:
    target_modules.append(f"roberta.encoder.layer.{layer}.attention.self.query")
    target_modules.append(f"roberta.encoder.layer.{layer}.attention.self.value")
# 配置 LoRA
config = LoraConfig(
    r=8,  # LoRA 的秩
    lora_alpha=16,  # LoRA 的缩放因子
    target_modules=target_modules,  # 目标模块
    lora_dropout=0.1,  # Dropout 概率
    bias="none",  # 是否更新偏置
    modules_to_save=["classifier"],  # 指定分类器需要被微调
)

# 封装为 LoRA 模型
model = get_peft_model(model, config)

# 验证分类器是否被微调
print("验证分类器参数是否被训练:")
for name, param in model.named_parameters():
    if param.requires_grad:
        print(f"{name}: requires_grad = {param.requires_grad}")

Collecting torchao==0.16.0
  Downloading torchao-0.16.0-cp310-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl.metadata (20 kB)
Downloading torchao-0.16.0-cp310-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (3.2 MB)
[2K   [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m3.2/3.2 MB[0m [31m47.8 MB/s[0m eta [36m0:00:00[0m
[?25hInstalling collected packages: torchao
  Attempting uninstall: torchao
    Found existing installation: torchao 0.10.0
    Uninstalling torchao-0.10.0:
      Successfully uninstalled torchao-0.10.0
Successfully installed torchao-0.16.0
验证分类器参数是否被训练:
base_model.model.classifier.modules_to_save.default.dense.weight: requires_grad = True
base_model.model.classifier.modules_to_save.default.dense.bias: requires_grad = True
base_model.model.classifier.modules_to_save.default.out_proj.weight: requires_grad = True
base_model.model.classifier.modules_to_save.default.out_proj.bias: requires_grad = True
base_model.model.roberta.encoder.layer.8.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.8.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.8.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.8.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.9.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.9.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.9.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.9.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.10.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.10.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.10.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.10.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.11.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.11.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.11.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.11.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.12.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.12.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.12.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.12.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.13.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.13.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.13.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.13.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.14.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.14.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.14.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.14.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.15.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.15.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.15.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.15.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.16.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.16.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.16.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.16.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.17.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.17.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.17.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.17.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.18.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.18.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.18.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.18.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.19.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.19.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.19.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.19.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.20.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.20.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.20.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.20.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.21.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.21.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.21.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.21.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.22.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.22.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.22.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.22.attention.self.value.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.23.attention.self.query.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.23.attention.self.query.lora_B.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.23.attention.self.value.lora_A.default.weight: requires_grad = True
base_model.model.roberta.encoder.layer.23.attention.self.value.lora_B.default.weight: requires_grad = True

In [8]:

@torch.no_grad()
def validate(model,loader):
    model.eval()
    acc=0
    total=0
    for batch in loader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        outputs = model(input_ids, attention_mask=attention_mask).logits
        pred=torch.argmax(outputs,dim=1)
        acc+=pred.eq(labels).sum()
        total+=labels.size(0)
    print(f"acc:{acc/total}")
    return acc/total
from tqdm import tqdm   
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model=model.to(device)
loss_func = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(), lr=2e-4)
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)
epochs = 30
best_acc=0.0
for epoch in range(epochs):
    model.train()
    training_loss = 0
    
    # 使用 tqdm 包装 dataloader，并设置描述信息
    progress_bar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{epochs}")
    lens=0
    for batch in progress_bar:
        optimizer.zero_grad()
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        outputs = model(input_ids, attention_mask=attention_mask).logits
        loss = loss_func(outputs, labels)
        loss.backward()
        optimizer.step()
        
        training_loss += loss.item()
        lens+=1
        # 更新进度条显示当前 batch 的损失
        progress_bar.set_postfix({
            'loss': f'{loss.item():.4f}',
            'avg_loss': f'{training_loss / (progress_bar.n+1):.4f}'  # progress_bar.n 是已处理 batch 数
        })
    
    scheduler.step()
    
    avg_train_loss = training_loss / lens
    print(f"Epoch {epoch+1} train_loss: {avg_train_loss:.4f}")
    
    # 验证（你也可以为验证添加进度条，见下方建议）
    current_acc=validate(model, val_loader)
    if current_acc > best_acc :
        torch.save(model.state_dict(), 'model.pth')
        print(f"best model save,acc:{current_acc}")
        best_acc=current_acc

Epoch 1/30: 100%|██████████| 429/429 [04:15<00:00,  1.68it/s, loss=0.5541, avg_loss=0.4941]

Epoch 1 train_loss: 0.4941
acc:0.8070865869522095
best model save,acc:0.8070865869522095

Epoch 2/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1806, avg_loss=0.4009]

Epoch 2 train_loss: 0.4009
acc:0.8280839920043945
best model save,acc:0.8280839920043945

Epoch 3/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0178, avg_loss=0.3768]

Epoch 3 train_loss: 0.3768
acc:0.8320209980010986
best model save,acc:0.8320209980010986

Epoch 4/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0461, avg_loss=0.3544]

Epoch 4 train_loss: 0.3544
acc:0.8359580039978027
best model save,acc:0.8359580039978027

Epoch 5/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0868, avg_loss=0.3336]

Epoch 5 train_loss: 0.3336
acc:0.8320209980010986

Epoch 6/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=1.1006, avg_loss=0.3171]

Epoch 6 train_loss: 0.3171
acc:0.847769021987915
best model save,acc:0.847769021987915

Epoch 7/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1015, avg_loss=0.2971]

Epoch 7 train_loss: 0.2971
acc:0.8372703194618225

Epoch 8/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1445, avg_loss=0.2843]

Epoch 8 train_loss: 0.2843
acc:0.8333333134651184

Epoch 9/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=1.0584, avg_loss=0.2712]

Epoch 9 train_loss: 0.2712
acc:0.8359580039978027

Epoch 10/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0560, avg_loss=0.2626]

Epoch 10 train_loss: 0.2626
acc:0.8372703194618225

Epoch 11/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.5303, avg_loss=0.2645]

Epoch 11 train_loss: 0.2645
acc:0.8372703194618225

Epoch 12/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.7138, avg_loss=0.2643]

Epoch 12 train_loss: 0.2643
acc:0.8359580039978027

Epoch 13/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0551, avg_loss=0.2616]

Epoch 13 train_loss: 0.2616
acc:0.8438320159912109

Epoch 14/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.8738, avg_loss=0.2686]

Epoch 14 train_loss: 0.2686
acc:0.8293963074684143

Epoch 15/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1841, avg_loss=0.2672]

Epoch 15 train_loss: 0.2672
acc:0.8202099800109863

Epoch 16/30: 100%|██████████| 429/429 [04:21<00:00,  1.64it/s, loss=0.0475, avg_loss=0.2644]

Epoch 16 train_loss: 0.2644
acc:0.8398950099945068

Epoch 17/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0248, avg_loss=0.2637]

Epoch 17 train_loss: 0.2637
acc:0.8228346705436707

Epoch 18/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1059, avg_loss=0.2565]

Epoch 18 train_loss: 0.2565
acc:0.8385826945304871

Epoch 19/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0538, avg_loss=0.2459]

Epoch 19 train_loss: 0.2459
acc:0.8372703194618225

Epoch 20/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.1534, avg_loss=0.2406]

Epoch 20 train_loss: 0.2406
acc:0.8070865869522095

Epoch 21/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0568, avg_loss=0.2275]

Epoch 21 train_loss: 0.2275
acc:0.8110235929489136

Epoch 22/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0785, avg_loss=0.2072]

Epoch 22 train_loss: 0.2072
acc:0.8280839920043945

Epoch 23/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.4414, avg_loss=0.2021]

Epoch 23 train_loss: 0.2021
acc:0.8241469860076904

Epoch 24/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0444, avg_loss=0.1876]

Epoch 24 train_loss: 0.1876
acc:0.8267716765403748

Epoch 25/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0273, avg_loss=0.1657]

Epoch 25 train_loss: 0.1657
acc:0.8215222954750061

Epoch 26/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0044, avg_loss=0.1466]

Epoch 26 train_loss: 0.1466
acc:0.8241469860076904

Epoch 27/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0474, avg_loss=0.1297]

Epoch 27 train_loss: 0.1297
acc:0.8188976049423218

Epoch 28/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0006, avg_loss=0.1282]

Epoch 28 train_loss: 0.1282
acc:0.8228346705436707

Epoch 29/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0009, avg_loss=0.1116]

Epoch 29 train_loss: 0.1116
acc:0.8228346705436707

Epoch 30/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0992, avg_loss=0.1074]

Epoch 30 train_loss: 0.1074
acc:0.8228346705436707

In [9]:

test_dataset = NliDataset(test_df, tokenizer, max_length)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
all_preds=[]
model.load_state_dict(torch.load('model.pth'))
model.eval()
with torch.no_grad():
    for batch in test_loader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        outputs = model(input_ids, attention_mask=attention_mask).logits
        preds = torch.argmax(outputs, dim=1)
        all_preds.extend(preds.cpu().numpy())

submission = pd.DataFrame({
    'id':test_df['id'],
    'target': all_preds
})

print(submission)
submission.to_csv('submission.csv', index=False)
print("Submission saved!")

         id  target
0         0       1
1         2       1
2         3       1
3         9       1
4        11       1
...     ...     ...
3258  10861       0
3259  10865       1
3260  10868       1
3261  10874       1
3262  10875       1

[3263 rows x 2 columns]
Submission saved!

In [ ]:

135 KiB Raw Permalink Blame History

135 KiB

Raw Permalink Blame History