{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "from torch.utils.data import Dataset, DataLoader"
   ],
   "id": "b90fab9688c875b3"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "train_df = pd.read_csv('/kaggle/input/competitions/nlp-getting-started/train.csv')\n",
    "test_df = pd.read_csv('/kaggle/input/competitions/nlp-getting-started/test.csv')\n",
    "from sklearn.model_selection import train_test_split\n",
    "train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=42)\n",
    "print(f\"Train: {len(train_df)}, Val: {len(val_df)}, Test: {len(test_df)}\")"
   ],
   "id": "d07abe4465837f46"
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2c6d716d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-22T02:00:25.795637Z",
     "iopub.status.busy": "2026-07-22T02:00:25.794874Z",
     "iopub.status.idle": "2026-07-22T02:00:43.802371Z",
     "shell.execute_reply": "2026-07-22T02:00:43.798282Z"
    },
    "papermill": {
     "duration": 18.250754,
     "end_time": "2026-07-22T02:00:44.042750+00:00",
     "exception": false,
     "start_time": "2026-07-22T02:00:25.791996+00:00",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad6193cf97eb4883bd1c6bd0b49cd4e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/616 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2dc334807a54ebcb8a2bcd042a29b7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4132e2c0c15c464abc7f0eb0a3b8557e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf852d7b51bf487eb170ef2b8c931fec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "afa1982d4c894d14afc9c364045b42be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/2.24G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef675f8da7c44aa8aa1b8829a4e8fd24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading weights:   0%|          | 0/389 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "XLMRobertaForSequenceClassification LOAD REPORT from: xlm-roberta-large\n",
      "Key                         | Status     | \n",
      "----------------------------+------------+-\n",
      "lm_head.dense.weight        | UNEXPECTED | \n",
      "lm_head.dense.bias          | UNEXPECTED | \n",
      "lm_head.bias                | UNEXPECTED | \n",
      "roberta.pooler.dense.weight | UNEXPECTED | \n",
      "roberta.pooler.dense.bias   | UNEXPECTED | \n",
      "lm_head.layer_norm.bias     | UNEXPECTED | \n",
      "lm_head.layer_norm.weight   | UNEXPECTED | \n",
      "classifier.dense.weight     | MISSING    | \n",
      "classifier.dense.bias       | MISSING    | \n",
      "classifier.out_proj.weight  | MISSING    | \n",
      "classifier.out_proj.bias    | MISSING    | \n",
      "\n",
      "Notes:\n",
      "- UNEXPECTED\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\n",
      "- MISSING\t:those params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
     ]
    }
   ],
   "source": [
    "model_name = \"xlm-roberta-large\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cac7fc28",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-22T02:00:44.049341Z",
     "iopub.status.busy": "2026-07-22T02:00:44.048999Z",
     "iopub.status.idle": "2026-07-22T02:00:44.057828Z",
     "shell.execute_reply": "2026-07-22T02:00:44.057304Z"
    },
    "papermill": {
     "duration": 0.013673,
     "end_time": "2026-07-22T02:00:44.059132+00:00",
     "exception": false,
     "start_time": "2026-07-22T02:00:44.045459+00:00",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "class NliDataset(Dataset):\n",
    "    def __init__(self, df, tokenizer, max_length=128):\n",
    "        self.df = df.reset_index(drop=True)\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_length = max_length\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        row = self.df.iloc[idx]\n",
    "        strs=''\n",
    "        if row['keyword'] is not np.nan:\n",
    "            strs=strs+'['+row['keyword']+']'\n",
    "        if row['location'] is not np.nan:\n",
    "            strs=strs+'['+row['location']+']'\n",
    "        strs=strs+row['text']\n",
    "        encoding = self.tokenizer(\n",
    "            strs,\n",
    "            truncation=True,\n",
    "            padding='max_length',\n",
    "            max_length=self.max_length,\n",
    "            return_tensors='pt'  # 返回 PyTorch Tensor\n",
    "        )\n",
    "        # 去掉 batch 维度（因为只处理单条）\n",
    "        item = {\n",
    "            'input_ids': encoding['input_ids'].squeeze(0),\n",
    "            'attention_mask': encoding['attention_mask'].squeeze(0)\n",
    "        }\n",
    "        if 'target' in row:\n",
    "            item['labels'] = torch.tensor(row['target'], dtype=torch.long)\n",
    "        return item\n",
    "\n",
    "batch_size = 16  # 根据显存调整，推荐使用 16 或 32\n",
    "max_length = 128\n",
    "\n",
    "train_dataset = NliDataset(train_df, tokenizer, max_length)\n",
    "val_dataset = NliDataset(val_df, tokenizer, max_length)\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "48f88e0c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-22T02:00:44.065561Z",
     "iopub.status.busy": "2026-07-22T02:00:44.065354Z",
     "iopub.status.idle": "2026-07-22T02:00:44.073182Z",
     "shell.execute_reply": "2026-07-22T02:00:44.072432Z"
    },
    "papermill": {
     "duration": 0.013598,
     "end_time": "2026-07-22T02:00:44.075169+00:00",
     "exception": false,
     "start_time": "2026-07-22T02:00:44.061571+00:00",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classifier.dense.weight: requires_grad = True\n",
      "classifier.dense.bias: requires_grad = True\n",
      "classifier.out_proj.weight: requires_grad = True\n",
      "classifier.out_proj.bias: requires_grad = True\n",
      "roberta.embeddings.word_embeddings.weight: requires_grad = True\n",
      "roberta.embeddings.token_type_embeddings.weight: requires_grad = True\n",
      "roberta.embeddings.LayerNorm.weight: requires_grad = True\n",
      "roberta.embeddings.LayerNorm.bias: requires_grad = True\n",
      "roberta.embeddings.position_embeddings.weight: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.0.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.0.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.0.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.0.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.0.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.0.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.0.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.1.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.1.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.1.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.1.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.1.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.1.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.1.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.2.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.2.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.2.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.2.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.2.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.2.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.2.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.3.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.3.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.3.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.3.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.3.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.3.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.3.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.4.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.4.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.4.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.4.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.4.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.4.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.4.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.5.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.5.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.5.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.5.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.5.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.5.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.5.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.6.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.6.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.6.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.6.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.6.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.6.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.6.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.7.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.7.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.7.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.7.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.7.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.7.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.7.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.8.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.8.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.8.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.8.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.8.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.8.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.8.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.9.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.9.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.9.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.9.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.9.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.9.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.9.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.10.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.10.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.10.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.10.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.10.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.10.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.10.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.11.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.11.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.11.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.11.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.11.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.11.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.11.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.12.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.12.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.12.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.12.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.12.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.12.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.12.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.13.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.13.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.13.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.13.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.13.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.13.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.13.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.14.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.14.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.14.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.14.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.14.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.14.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.14.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.15.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.15.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.15.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.15.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.15.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.15.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.15.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.16.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.16.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.16.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.16.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.16.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.16.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.16.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.17.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.17.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.17.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.17.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.17.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.17.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.17.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.18.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.18.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.18.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.18.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.18.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.18.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.18.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.19.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.19.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.19.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.19.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.19.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.19.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.19.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.20.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.20.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.20.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.20.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.20.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.20.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.20.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.21.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.21.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.21.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.21.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.21.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.21.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.21.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.22.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.22.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.22.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.22.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.22.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.22.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.22.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.self.query.weight: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.self.query.bias: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.self.key.weight: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.self.key.bias: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.self.value.weight: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.self.value.bias: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.23.attention.output.LayerNorm.bias: requires_grad = True\n",
      "roberta.encoder.layer.23.intermediate.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.23.intermediate.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.23.output.dense.weight: requires_grad = True\n",
      "roberta.encoder.layer.23.output.dense.bias: requires_grad = True\n",
      "roberta.encoder.layer.23.output.LayerNorm.weight: requires_grad = True\n",
      "roberta.encoder.layer.23.output.LayerNorm.bias: requires_grad = True\n"
     ]
    }
   ],
   "source": [
    "for name, param in model.named_parameters():\n",
    "    #if param.requires_grad:\n",
    "    print(f\"{name}: requires_grad = {param.requires_grad}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a24b83a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-22T02:00:44.082499Z",
     "iopub.status.busy": "2026-07-22T02:00:44.081954Z",
     "iopub.status.idle": "2026-07-22T02:00:59.414269Z",
     "shell.execute_reply": "2026-07-22T02:00:59.413336Z"
    },
    "papermill": {
     "duration": 15.338053,
     "end_time": "2026-07-22T02:00:59.416223+00:00",
     "exception": false,
     "start_time": "2026-07-22T02:00:44.078170+00:00",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting torchao==0.16.0\r\n",
      "  Downloading torchao-0.16.0-cp310-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl.metadata (20 kB)\r\n",
      "Downloading torchao-0.16.0-cp310-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (3.2 MB)\r\n",
      "\u001B[2K   \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m3.2/3.2 MB\u001B[0m \u001B[31m47.8 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
      "\u001B[?25hInstalling collected packages: torchao\r\n",
      "  Attempting uninstall: torchao\r\n",
      "    Found existing installation: torchao 0.10.0\r\n",
      "    Uninstalling torchao-0.10.0:\r\n",
      "      Successfully uninstalled torchao-0.10.0\r\n",
      "Successfully installed torchao-0.16.0\r\n",
      "验证分类器参数是否被训练:\n",
      "base_model.model.classifier.modules_to_save.default.dense.weight: requires_grad = True\n",
      "base_model.model.classifier.modules_to_save.default.dense.bias: requires_grad = True\n",
      "base_model.model.classifier.modules_to_save.default.out_proj.weight: requires_grad = True\n",
      "base_model.model.classifier.modules_to_save.default.out_proj.bias: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.8.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.8.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.8.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.8.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.9.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.9.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.9.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.9.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.10.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.10.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.10.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.10.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.11.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.11.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.11.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.11.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.12.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.12.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.12.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.12.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.13.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.13.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.13.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.13.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.14.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.14.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.14.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.14.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.15.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.15.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.15.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.15.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.16.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.16.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.16.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.16.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.17.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.17.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.17.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.17.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.18.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.18.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.18.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.18.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.19.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.19.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.19.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.19.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.20.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.20.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.20.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.20.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.21.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.21.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.21.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.21.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.22.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.22.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.22.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.22.attention.self.value.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.23.attention.self.query.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.23.attention.self.query.lora_B.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.23.attention.self.value.lora_A.default.weight: requires_grad = True\n",
      "base_model.model.roberta.encoder.layer.23.attention.self.value.lora_B.default.weight: requires_grad = True\n"
     ]
    }
   ],
   "source": [
    "!pip install torchao==0.16.0\n",
    "from peft import LoraConfig, get_peft_model\n",
    "target_layers = [8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23]\n",
    "target_modules=[]\n",
    "for layer in target_layers:\n",
    "    target_modules.append(f\"roberta.encoder.layer.{layer}.attention.self.query\")\n",
    "    target_modules.append(f\"roberta.encoder.layer.{layer}.attention.self.value\")\n",
    "# 配置 LoRA\n",
    "config = LoraConfig(\n",
    "    r=8,  # LoRA 的秩\n",
    "    lora_alpha=16,  # LoRA 的缩放因子\n",
    "    target_modules=target_modules,  # 目标模块\n",
    "    lora_dropout=0.1,  # Dropout 概率\n",
    "    bias=\"none\",  # 是否更新偏置\n",
    "    modules_to_save=[\"classifier\"],  # 指定分类器需要被微调\n",
    ")\n",
    "\n",
    "# 封装为 LoRA 模型\n",
    "model = get_peft_model(model, config)\n",
    "\n",
    "# 验证分类器是否被微调\n",
    "print(\"验证分类器参数是否被训练:\")\n",
    "for name, param in model.named_parameters():\n",
    "    if param.requires_grad:\n",
    "        print(f\"{name}: requires_grad = {param.requires_grad}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8082471f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-22T02:00:59.424781Z",
     "iopub.status.busy": "2026-07-22T02:00:59.424534Z",
     "iopub.status.idle": "2026-07-22T04:19:30.137552Z",
     "shell.execute_reply": "2026-07-22T04:19:30.136572Z"
    },
    "papermill": {
     "duration": 8310.719504,
     "end_time": "2026-07-22T04:19:30.139354+00:00",
     "exception": false,
     "start_time": "2026-07-22T02:00:59.419850+00:00",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1/30: 100%|██████████| 429/429 [04:15<00:00,  1.68it/s, loss=0.5541, avg_loss=0.4941]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 train_loss: 0.4941\n",
      "acc:0.8070865869522095\n",
      "best model save,acc:0.8070865869522095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1806, avg_loss=0.4009]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2 train_loss: 0.4009\n",
      "acc:0.8280839920043945\n",
      "best model save,acc:0.8280839920043945\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0178, avg_loss=0.3768]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3 train_loss: 0.3768\n",
      "acc:0.8320209980010986\n",
      "best model save,acc:0.8320209980010986\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0461, avg_loss=0.3544]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4 train_loss: 0.3544\n",
      "acc:0.8359580039978027\n",
      "best model save,acc:0.8359580039978027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0868, avg_loss=0.3336]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5 train_loss: 0.3336\n",
      "acc:0.8320209980010986\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=1.1006, avg_loss=0.3171]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6 train_loss: 0.3171\n",
      "acc:0.847769021987915\n",
      "best model save,acc:0.847769021987915\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 7/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1015, avg_loss=0.2971]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7 train_loss: 0.2971\n",
      "acc:0.8372703194618225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 8/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1445, avg_loss=0.2843]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8 train_loss: 0.2843\n",
      "acc:0.8333333134651184\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 9/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=1.0584, avg_loss=0.2712]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9 train_loss: 0.2712\n",
      "acc:0.8359580039978027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 10/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0560, avg_loss=0.2626]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10 train_loss: 0.2626\n",
      "acc:0.8372703194618225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 11/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.5303, avg_loss=0.2645]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11 train_loss: 0.2645\n",
      "acc:0.8372703194618225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 12/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.7138, avg_loss=0.2643]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12 train_loss: 0.2643\n",
      "acc:0.8359580039978027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 13/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0551, avg_loss=0.2616]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13 train_loss: 0.2616\n",
      "acc:0.8438320159912109\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 14/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.8738, avg_loss=0.2686]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14 train_loss: 0.2686\n",
      "acc:0.8293963074684143\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 15/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1841, avg_loss=0.2672]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15 train_loss: 0.2672\n",
      "acc:0.8202099800109863\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 16/30: 100%|██████████| 429/429 [04:21<00:00,  1.64it/s, loss=0.0475, avg_loss=0.2644]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16 train_loss: 0.2644\n",
      "acc:0.8398950099945068\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 17/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0248, avg_loss=0.2637]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17 train_loss: 0.2637\n",
      "acc:0.8228346705436707\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 18/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.1059, avg_loss=0.2565]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18 train_loss: 0.2565\n",
      "acc:0.8385826945304871\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 19/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0538, avg_loss=0.2459]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19 train_loss: 0.2459\n",
      "acc:0.8372703194618225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 20/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.1534, avg_loss=0.2406]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20 train_loss: 0.2406\n",
      "acc:0.8070865869522095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 21/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0568, avg_loss=0.2275]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21 train_loss: 0.2275\n",
      "acc:0.8110235929489136\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 22/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0785, avg_loss=0.2072]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22 train_loss: 0.2072\n",
      "acc:0.8280839920043945\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 23/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.4414, avg_loss=0.2021]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23 train_loss: 0.2021\n",
      "acc:0.8241469860076904\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 24/30: 100%|██████████| 429/429 [04:20<00:00,  1.65it/s, loss=0.0444, avg_loss=0.1876]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24 train_loss: 0.1876\n",
      "acc:0.8267716765403748\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 25/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0273, avg_loss=0.1657]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 25 train_loss: 0.1657\n",
      "acc:0.8215222954750061\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 26/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0044, avg_loss=0.1466]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 26 train_loss: 0.1466\n",
      "acc:0.8241469860076904\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 27/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0474, avg_loss=0.1297]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27 train_loss: 0.1297\n",
      "acc:0.8188976049423218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 28/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0006, avg_loss=0.1282]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28 train_loss: 0.1282\n",
      "acc:0.8228346705436707\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 29/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0009, avg_loss=0.1116]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29 train_loss: 0.1116\n",
      "acc:0.8228346705436707\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 30/30: 100%|██████████| 429/429 [04:19<00:00,  1.65it/s, loss=0.0992, avg_loss=0.1074]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30 train_loss: 0.1074\n",
      "acc:0.8228346705436707\n"
     ]
    }
   ],
   "source": [
    "@torch.no_grad()\n",
    "def validate(model,loader):\n",
    "    model.eval()\n",
    "    acc=0\n",
    "    total=0\n",
    "    for batch in loader:\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        outputs = model(input_ids, attention_mask=attention_mask).logits\n",
    "        pred=torch.argmax(outputs,dim=1)\n",
    "        acc+=pred.eq(labels).sum()\n",
    "        total+=labels.size(0)\n",
    "    print(f\"acc:{acc/total}\")\n",
    "    return acc/total\n",
    "from tqdm import tqdm   \n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model=model.to(device)\n",
    "loss_func = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr=2e-4)\n",
    "scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n",
    "epochs = 30\n",
    "best_acc=0.0\n",
    "for epoch in range(epochs):\n",
    "    model.train()\n",
    "    training_loss = 0\n",
    "    \n",
    "    # 使用 tqdm 包装 dataloader，并设置描述信息\n",
    "    progress_bar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{epochs}\")\n",
    "    lens=0\n",
    "    for batch in progress_bar:\n",
    "        optimizer.zero_grad()\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        outputs = model(input_ids, attention_mask=attention_mask).logits\n",
    "        loss = loss_func(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        training_loss += loss.item()\n",
    "        lens+=1\n",
    "        # 更新进度条显示当前 batch 的损失\n",
    "        progress_bar.set_postfix({\n",
    "            'loss': f'{loss.item():.4f}',\n",
    "            'avg_loss': f'{training_loss / (progress_bar.n+1):.4f}'  # progress_bar.n 是已处理 batch 数\n",
    "        })\n",
    "    \n",
    "    scheduler.step()\n",
    "    \n",
    "    avg_train_loss = training_loss / lens\n",
    "    print(f\"Epoch {epoch+1} train_loss: {avg_train_loss:.4f}\")\n",
    "    \n",
    "    # 验证（你也可以为验证添加进度条，见下方建议）\n",
    "    current_acc=validate(model, val_loader)\n",
    "    if current_acc > best_acc :\n",
    "        torch.save(model.state_dict(), 'model.pth')\n",
    "        print(f\"best model save,acc:{current_acc}\")\n",
    "        best_acc=current_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "61693fae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-22T04:19:32.122431Z",
     "iopub.status.busy": "2026-07-22T04:19:32.121674Z",
     "iopub.status.idle": "2026-07-22T04:20:44.644578Z",
     "shell.execute_reply": "2026-07-22T04:20:44.643783Z"
    },
    "papermill": {
     "duration": 74.494369,
     "end_time": "2026-07-22T04:20:45.673202+00:00",
     "exception": false,
     "start_time": "2026-07-22T04:19:31.178833+00:00",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         id  target\n",
      "0         0       1\n",
      "1         2       1\n",
      "2         3       1\n",
      "3         9       1\n",
      "4        11       1\n",
      "...     ...     ...\n",
      "3258  10861       0\n",
      "3259  10865       1\n",
      "3260  10868       1\n",
      "3261  10874       1\n",
      "3262  10875       1\n",
      "\n",
      "[3263 rows x 2 columns]\n",
      "Submission saved!\n"
     ]
    }
   ],
   "source": [
    "test_dataset = NliDataset(test_df, tokenizer, max_length)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "all_preds=[]\n",
    "model.load_state_dict(torch.load('model.pth'))\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    for batch in test_loader:\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        outputs = model(input_ids, attention_mask=attention_mask).logits\n",
    "        preds = torch.argmax(outputs, dim=1)\n",
    "        all_preds.extend(preds.cpu().numpy())\n",
    "\n",
    "submission = pd.DataFrame({\n",
    "    'id':test_df['id'],\n",
    "    'target': all_preds\n",
    "})\n",
    "\n",
    "print(submission)\n",
    "submission.to_csv('submission.csv', index=False)\n",
    "print(\"Submission saved!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "900e4ba5",
   "metadata": {
    "papermill": {
     "duration": 0.946718,
     "end_time": "2026-07-22T04:20:47.646925+00:00",
     "exception": false,
     "start_time": "2026-07-22T04:20:46.700207+00:00",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [],
   "dockerImageVersionId": 28755,
   "isGpuEnabled": false,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 8457.820464,
   "end_time": "2026-07-22T04:20:52.588673+00:00",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2026-07-22T01:59:54.768209+00:00",
   "version": "2.7.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "090c40f079f7433e9069e304e0dbb2b5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "0a94a2b0b5e5450eaac3b8771917ae84": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_b4c67109ed144306a156834426f85b77",
       "max": 389.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_432c82cbc9b84044b68645bb220bde16",
       "tabbable": null,
       "tooltip": null,
       "value": 389.0
      }
     },
     "0ce9c293dd84435b86512d4f378ad612": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1a69ec693f834e35a3ed83fcd7d98422": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_0ce9c293dd84435b86512d4f378ad612",
       "placeholder": "​",
       "style": "IPY_MODEL_8a8adda41d63458cb58a40e20b50ee78",
       "tabbable": null,
       "tooltip": null,
       "value": " 5.07M/5.07M [00:00&lt;00:00, 15.4MB/s]"
      }
     },
     "1e9adab8920b40ceaeb4b063f0a5a3db": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_8cdfe320f8f74e25ae31dd4ae75ccf46",
       "placeholder": "​",
       "style": "IPY_MODEL_dc07462ca4034c1a8b82821ce049d638",
       "tabbable": null,
       "tooltip": null,
       "value": "Loading weights: 100%"
      }
     },
     "26c66cc4cae74ab7b611c1425ffe333a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2957f62682394c2f95cf8876f243f236": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "2fd55f67774149cf932c9554929f40b9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3408d897d7c4403f99b6c576c569833c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "3b6a8c48ef4b4e76bab91161ecdc9a40": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_73678a7e38e546228273c9ce4d1dd750",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d1281ac7c62f44ca9234c6332a7b6cf5",
       "tabbable": null,
       "tooltip": null,
       "value": 1.0
      }
     },
     "3d8a4241d2624055878163b65577e003": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_b902fea558094e6c8b9fc2cd026bbd0e",
       "placeholder": "​",
       "style": "IPY_MODEL_5e0dc86420964900af197ee1de5e16e1",
       "tabbable": null,
       "tooltip": null,
       "value": " 25.0/25.0 [00:00&lt;00:00, 3.31kB/s]"
      }
     },
     "3f066f23751d42dcb32207f28b9bfd5f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_777e133b489e4a2fa5a08fe55312bb5a",
       "placeholder": "​",
       "style": "IPY_MODEL_3408d897d7c4403f99b6c576c569833c",
       "tabbable": null,
       "tooltip": null,
       "value": "tokenizer_config.json: 100%"
      }
     },
     "4132e2c0c15c464abc7f0eb0a3b8557e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_8d5c0298f9264814a668f540380036e1",
        "IPY_MODEL_e4cd93623daf4fe78e3bf47b89b1345f",
        "IPY_MODEL_1a69ec693f834e35a3ed83fcd7d98422"
       ],
       "layout": "IPY_MODEL_d133db57516f43e7847557e8625771df",
       "tabbable": null,
       "tooltip": null
      }
     },
     "432c82cbc9b84044b68645bb220bde16": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "482469099e2e4cb5a39447406e56cff0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "508e5b14f8804bcf905c2d37f9dbea31": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_e7eb78ea55134e44b53ad4ba13e847cd",
       "placeholder": "​",
       "style": "IPY_MODEL_84a66d7860de48d991754c61b3e65660",
       "tabbable": null,
       "tooltip": null,
       "value": "model.safetensors: 100%"
      }
     },
     "546d10a484304b2f943822b2eeec64ee": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "57ad1d5e68064863bd1a6de202ae0e9d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "5e0dc86420964900af197ee1de5e16e1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "60e1ba288fab4685b8e91ab575e7a50b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_26c66cc4cae74ab7b611c1425ffe333a",
       "placeholder": "​",
       "style": "IPY_MODEL_90d5d9ea21e34c1b9424531650fb3f80",
       "tabbable": null,
       "tooltip": null,
       "value": " 616/616 [00:00&lt;00:00, 55.8kB/s]"
      }
     },
     "645b2985951b4bff83b02085b9e36be8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "685fffe09d3340d3bdd8549273788f58": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_bda6a531e6d749e28407cfcbb6fbb98c",
       "max": 2.244817354E9,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_645b2985951b4bff83b02085b9e36be8",
       "tabbable": null,
       "tooltip": null,
       "value": 2.244817354E9
      }
     },
     "6972c432d03c41da9c2bbb0b25fcf832": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_dcef267d7dc44a279ae8ed5e6a1cabfe",
       "placeholder": "​",
       "style": "IPY_MODEL_2957f62682394c2f95cf8876f243f236",
       "tabbable": null,
       "tooltip": null,
       "value": "config.json: 100%"
      }
     },
     "73678a7e38e546228273c9ce4d1dd750": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": "20px"
      }
     },
     "73ce65eea94e41129c78fc60d6481c07": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "777e133b489e4a2fa5a08fe55312bb5a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "84a66d7860de48d991754c61b3e65660": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "87da9ab002614424a07e5fbcff86f375": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8920fc3433494e9d97c670223041845f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "8a8adda41d63458cb58a40e20b50ee78": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "8cdfe320f8f74e25ae31dd4ae75ccf46": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8d5c0298f9264814a668f540380036e1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_546d10a484304b2f943822b2eeec64ee",
       "placeholder": "​",
       "style": "IPY_MODEL_482469099e2e4cb5a39447406e56cff0",
       "tabbable": null,
       "tooltip": null,
       "value": "sentencepiece.bpe.model: 100%"
      }
     },
     "9098d933dd8d48f186c47d58a13546aa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "90d5d9ea21e34c1b9424531650fb3f80": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "97f5798b5f5e444dafa36fa55e951e84": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_fe886471368e49dea688303ec14f2dcf",
       "max": 25.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_57ad1d5e68064863bd1a6de202ae0e9d",
       "tabbable": null,
       "tooltip": null,
       "value": 25.0
      }
     },
     "a1f7705d0e594d97bf5393917f6113e9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_e6856331e9514cbb8e6c81940301de36",
       "placeholder": "​",
       "style": "IPY_MODEL_f27e863469d947bdadbaf7cc36aa4a07",
       "tabbable": null,
       "tooltip": null,
       "value": " 389/389 [00:00&lt;00:00, 921.59it/s, Materializing param=roberta.encoder.layer.23.output.dense.weight]"
      }
     },
     "a27cfe6dfd714cf5bea035f399d714f1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_090c40f079f7433e9069e304e0dbb2b5",
       "max": 616.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_8920fc3433494e9d97c670223041845f",
       "tabbable": null,
       "tooltip": null,
       "value": 616.0
      }
     },
     "ad6193cf97eb4883bd1c6bd0b49cd4e1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_6972c432d03c41da9c2bbb0b25fcf832",
        "IPY_MODEL_a27cfe6dfd714cf5bea035f399d714f1",
        "IPY_MODEL_60e1ba288fab4685b8e91ab575e7a50b"
       ],
       "layout": "IPY_MODEL_9098d933dd8d48f186c47d58a13546aa",
       "tabbable": null,
       "tooltip": null
      }
     },
     "afa1982d4c894d14afc9c364045b42be": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_508e5b14f8804bcf905c2d37f9dbea31",
        "IPY_MODEL_685fffe09d3340d3bdd8549273788f58",
        "IPY_MODEL_f9828122690848528a82610b7b75522d"
       ],
       "layout": "IPY_MODEL_bbcb8b483c114842974359143040d5be",
       "tabbable": null,
       "tooltip": null
      }
     },
     "b4c67109ed144306a156834426f85b77": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b902fea558094e6c8b9fc2cd026bbd0e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bbcb8b483c114842974359143040d5be": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bda6a531e6d749e28407cfcbb6fbb98c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c8ffc4357ebc470aa2b22549a8fb39d9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cf852d7b51bf487eb170ef2b8c931fec": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_e4101a49237e40f7abe37305cdd01fde",
        "IPY_MODEL_3b6a8c48ef4b4e76bab91161ecdc9a40",
        "IPY_MODEL_f0acf536573e44f6bfbdfd775631a1b1"
       ],
       "layout": "IPY_MODEL_ea86faa0f63f4d36b31c3db7aad7f88c",
       "tabbable": null,
       "tooltip": null
      }
     },
     "cf96198e48754787bfd396d20766af80": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d1281ac7c62f44ca9234c6332a7b6cf5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "d133db57516f43e7847557e8625771df": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d63c11434da54d1a8326a07e3a48c856": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "dc07462ca4034c1a8b82821ce049d638": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "dcef267d7dc44a279ae8ed5e6a1cabfe": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "de49874254e148d4ab45810d9350d20b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e072c50271264b1197d32377d851584e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "e4101a49237e40f7abe37305cdd01fde": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_2fd55f67774149cf932c9554929f40b9",
       "placeholder": "​",
       "style": "IPY_MODEL_ea773336a62d4f5096c9c9ceb01fce39",
       "tabbable": null,
       "tooltip": null,
       "value": "tokenizer.json: "
      }
     },
     "e4cd93623daf4fe78e3bf47b89b1345f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_de49874254e148d4ab45810d9350d20b",
       "max": 5069051.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_fd04cd65456646dabf5d22afd04b7e95",
       "tabbable": null,
       "tooltip": null,
       "value": 5069051.0
      }
     },
     "e6856331e9514cbb8e6c81940301de36": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e7eb78ea55134e44b53ad4ba13e847cd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ea773336a62d4f5096c9c9ceb01fce39": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "ea86faa0f63f4d36b31c3db7aad7f88c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ef675f8da7c44aa8aa1b8829a4e8fd24": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_1e9adab8920b40ceaeb4b063f0a5a3db",
        "IPY_MODEL_0a94a2b0b5e5450eaac3b8771917ae84",
        "IPY_MODEL_a1f7705d0e594d97bf5393917f6113e9"
       ],
       "layout": "IPY_MODEL_87da9ab002614424a07e5fbcff86f375",
       "tabbable": null,
       "tooltip": null
      }
     },
     "f0acf536573e44f6bfbdfd775631a1b1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_73ce65eea94e41129c78fc60d6481c07",
       "placeholder": "​",
       "style": "IPY_MODEL_e072c50271264b1197d32377d851584e",
       "tabbable": null,
       "tooltip": null,
       "value": " 9.10M/? [00:00&lt;00:00, 21.0MB/s]"
      }
     },
     "f27e863469d947bdadbaf7cc36aa4a07": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "f2dc334807a54ebcb8a2bcd042a29b7b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_3f066f23751d42dcb32207f28b9bfd5f",
        "IPY_MODEL_97f5798b5f5e444dafa36fa55e951e84",
        "IPY_MODEL_3d8a4241d2624055878163b65577e003"
       ],
       "layout": "IPY_MODEL_c8ffc4357ebc470aa2b22549a8fb39d9",
       "tabbable": null,
       "tooltip": null
      }
     },
     "f9828122690848528a82610b7b75522d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_cf96198e48754787bfd396d20766af80",
       "placeholder": "​",
       "style": "IPY_MODEL_d63c11434da54d1a8326a07e3a48c856",
       "tabbable": null,
       "tooltip": null,
       "value": " 2.24G/2.24G [00:10&lt;00:00, 602MB/s]"
      }
     },
     "fd04cd65456646dabf5d22afd04b7e95": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "fe886471368e49dea688303ec14f2dcf": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}