Kaggle/Petals-to-the-Metal-ViT/main.ipynb

In [1]:

import tensorflow as tf
import torch
import torchvision
from torch.utils.data import IterableDataset, DataLoader
from matplotlib import pyplot as plt
import numpy as np
import torch.nn as nn
from PIL import Image
def parse_tfrecord(example_proto):
    feature_description = {
        'image': tf.io.FixedLenFeature([], tf.string),
        'class': tf.io.FixedLenFeature([], tf.int64),
        'id' : tf.io.FixedLenFeature([], tf.string),
    }
    parsed = tf.io.parse_single_example(example_proto, feature_description)
    image = tf.image.decode_jpeg(parsed['image'], channels=3)
    image = tf.image.resize(image, [224, 224])
    #image = tf.image.convert_image_dtype(image, tf.float32)
    label = parsed['class']
    idd = parsed['id']
    return image, label,idd

def load_tfrecord_dataset(pattern):
    files = tf.io.gfile.glob(pattern)
    if not files:
        raise ValueError(f"No files found for pattern {pattern}")
    dataset = tf.data.TFRecordDataset(files)
    dataset = dataset.map(parse_tfrecord)
    # 可选：打乱、批处理等，但此处我们只返回样本级别的数据集
    return dataset

class TFRecordToPyTorch(IterableDataset):
    def __init__(self, tfrecord_pattern,transform=None):
        self.tfrecord_pattern = tfrecord_pattern
        self.transform=transform

    def __iter__(self):
        # 每次迭代创建新的数据集，保证可重复使用
        dataset = load_tfrecord_dataset(self.tfrecord_pattern)
        # 使用 as_numpy_iterator() 获取 NumPy 数组，便于转换为 PyTorch 张量
        for image_np, label_np,idd in dataset.as_numpy_iterator():
            # image_np shape: (224,224,3), dtype float32, label_np scalar int64
            # 转为 PyTorch 张量，并调整为 CxHxW
            image_pil = Image.fromarray((image_np).astype('uint8')) 
            if self.transform:
                image_tensor = self.transform(image_pil)
            else:
                # 如果不需要 transform，至少转为 tensor
                image_tensor = torch.from_numpy(image_np).permute(2,0,1)
            #image_torch = torch.from_numpy(image_np).permute(2, 0, 1)  # (3,224,224)
            label_torch = torch.tensor(label_np, dtype=torch.long)
            id_torch = idd
            yield image_tensor, label_torch,id_torch

In [2]:

from transformers import ViTForImageClassification, ViTImageProcessor

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 

model_name = "google/vit-base-patch16-224-in21k"  # 在ImageNet21k上预训练

model = ViTForImageClassification.from_pretrained(model_name, num_labels=104)
 
model.to(device)
 
feature_extractor = ViTImageProcessor.from_pretrained(model_name)
print(model,feature_extractor)

Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.

config.json:   0%|          | 0.00/502 [00:00<?, ?B/s]

model.safetensors:   0%|          | 0.00/346M [00:00<?, ?B/s]

Loading weights:   0%|          | 0/198 [00:00<?, ?it/s]

ViTForImageClassification LOAD REPORT from: google/vit-base-patch16-224-in21k
Key                 | Status     | 
--------------------+------------+-
pooler.dense.bias   | UNEXPECTED | 
pooler.dense.weight | UNEXPECTED | 
classifier.bias     | MISSING    | 
classifier.weight   | MISSING    | 

Notes:
- UNEXPECTED	:can be ignored when loading from different task/architecture; not ok if you expect identical arch.
- MISSING	:those params were newly initialized because missing from the checkpoint. Consider training on your downstream task.

preprocessor_config.json:   0%|          | 0.00/160 [00:00<?, ?B/s]

ViTForImageClassification(
  (vit): ViTModel(
    (embeddings): ViTEmbeddings(
      (patch_embeddings): ViTPatchEmbeddings(
        (projection): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))
      )
      (dropout): Dropout(p=0.0, inplace=False)
    )
    (encoder): ViTEncoder(
      (layer): ModuleList(
        (0-11): 12 x ViTLayer(
          (attention): ViTAttention(
            (attention): ViTSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
            )
            (output): ViTSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.0, inplace=False)
            )
          )
          (intermediate): ViTIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
            (intermediate_act_fn): GELUActivation()
          )
          (output): ViTOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (dropout): Dropout(p=0.0, inplace=False)
          )
          (layernorm_before): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (layernorm_after): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
        )
      )
    )
    (layernorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
  )
  (classifier): Linear(in_features=768, out_features=104, bias=True)
) ViTImageProcessor {
  "do_convert_rgb": null,
  "do_normalize": true,
  "do_rescale": true,
  "do_resize": true,
  "image_mean": [
    0.5,
    0.5,
    0.5
  ],
  "image_processor_type": "ViTImageProcessor",
  "image_std": [
    0.5,
    0.5,
    0.5
  ],
  "resample": 2,
  "rescale_factor": 0.00392156862745098,
  "size": {
    "height": 224,
    "width": 224
  }
}

In [3]:

transform = torchvision.transforms.Compose([
    torchvision.transforms.Resize((224, 224)),  # 调整尺寸为224x224
    torchvision.transforms.ToTensor(),  # 转换为张量
    # 使用特征提取器的参数进行标准化
    torchvision.transforms.Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
])
tfrecord_path = '/kaggle/input/competitions/tpu-getting-started/tfrecords-jpeg-224x224/train/*'
dataset = TFRecordToPyTorch(tfrecord_path,transform)
tfrecord_path = '/kaggle/input/competitions/tpu-getting-started/tfrecords-jpeg-224x224/val/*'
dataset2 = TFRecordToPyTorch(tfrecord_path,transform)
# 可以配合 DataLoader 使用
train_dataloader = DataLoader(dataset, batch_size=32, num_workers=0)  # num_workers 设为0，因为 TF 数据集内部已并行
val_dataloader = DataLoader(dataset2, batch_size=32, num_workers=0)
for batch in train_dataloader:
    plt.imshow(batch[0][1].permute(1,2,0).numpy())
    break
    plt.axis('off')
    plt.show()

WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1783570100.516263      58 gpu_device.cc:2020] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 13374 MB memory:  -> device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5
I0000 00:00:1783570100.518801      58 gpu_device.cc:2020] Created device /job:localhost/replica:0/task:0/device:GPU:1 with 13756 MB memory:  -> device: 1, name: Tesla T4, pci bus id: 0000:00:05.0, compute capability: 7.5
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Got range [-1.0..1.0].

In [5]:

@torch.no_grad()
def validate(model,loader):
    model.eval()
    acc=0
    total=0
    for batch in loader:
        X = batch[0]
        labels = batch[1]
        X = X.to(device)
        labels = labels.to(device)
        pred=torch.argmax(model(X).logits,dim=1)
        acc+=pred.eq(labels).sum()
        total+=labels.size(0)
    print(f"acc:{acc/total}")
    return acc/total
from tqdm import tqdm   
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model=model.to(device)
loss_func = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(), lr=2e-4)
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)
epochs = 10
best_acc=0.0
for epoch in range(epochs):
    model.train()
    training_loss = 0
    
    # 使用 tqdm 包装 dataloader，并设置描述信息
    progress_bar = tqdm(train_dataloader, desc=f"Epoch {epoch+1}/{epochs}")
    lens=0
    for batch in progress_bar:
        optimizer.zero_grad()
        X = batch[0].to(device)
        labels = batch[1].to(device)
        
        outputs = model(X).logits
        loss = loss_func(outputs, labels)
        loss.backward()
        optimizer.step()
        
        training_loss += loss.item()
        lens+=1
        # 更新进度条显示当前 batch 的损失
        progress_bar.set_postfix({
            'loss': f'{loss.item():.4f}',
            'avg_loss': f'{training_loss / (progress_bar.n+1):.4f}'  # progress_bar.n 是已处理 batch 数
        })
    
    scheduler.step()
    
    avg_train_loss = training_loss / lens
    print(f"Epoch {epoch+1} train_loss: {avg_train_loss:.4f}")
    
    # 验证（你也可以为验证添加进度条，见下方建议）
    current_acc=validate(model, val_dataloader)
    if current_acc > best_acc :
        torch.save(model.state_dict(), 'model.pth')
        print(f"best model save,acc:{current_acc}")
        best_acc=current_acc

Epoch 1/10: 0it [00:01, ?it/s, loss=4.1139, avg_loss=4.1139]

Epoch 1 train_loss: 4.1139
acc:0.125
best model save,acc:0.125

Epoch 2/10: 0it [00:01, ?it/s, loss=3.8877, avg_loss=3.8877]

Epoch 2 train_loss: 3.8877
acc:0.15625
best model save,acc:0.15625

Epoch 3/10: 0it [00:01, ?it/s, loss=3.5763, avg_loss=3.5763]

Epoch 3 train_loss: 3.5763
acc:0.125

Epoch 4/10: 0it [00:01, ?it/s, loss=3.3188, avg_loss=3.3188]

Epoch 4 train_loss: 3.3188
acc:0.125

Epoch 5/10: 0it [00:01, ?it/s, loss=3.1146, avg_loss=3.1146]

Epoch 5 train_loss: 3.1146
acc:0.125

Epoch 6/10: 0it [00:01, ?it/s, loss=2.9511, avg_loss=2.9511]

Epoch 6 train_loss: 2.9511
acc:0.125

Epoch 7/10: 0it [00:01, ?it/s, loss=2.8346, avg_loss=2.8346]

Epoch 7 train_loss: 2.8346
acc:0.125

Epoch 8/10: 0it [00:01, ?it/s, loss=2.7525, avg_loss=2.7525]

Epoch 8 train_loss: 2.7525
acc:0.125

Epoch 9/10: 0it [00:01, ?it/s, loss=2.7027, avg_loss=2.7027]

Epoch 9 train_loss: 2.7027
acc:0.125

Epoch 10/10: 0it [00:01, ?it/s, loss=2.6787, avg_loss=2.6787]

Epoch 10 train_loss: 2.6787

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
/tmp/ipykernel_58/4281004902.py in <cell line: 0>()
     55 
     56     # 验证（你也可以为验证添加进度条，见下方建议）
---> 57     current_acc=validate(model, val_dataloader)
     58     if current_acc > best_acc :
     59         torch.save(model.state_dict(), 'model.pth')

/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py in decorate_context(*args, **kwargs)
    122         # pyrefly: ignore [bad-context-manager]
    123         with ctx_factory():
--> 124             return func(*args, **kwargs)
    125 
    126     return decorate_context

/tmp/ipykernel_58/4281004902.py in validate(model, loader)
     13         total+=labels.size(0)
     14         break
---> 15     print(f"acc:{acc/total}")
     16     return acc/total
     17 from tqdm import tqdm

/usr/local/lib/python3.12/dist-packages/torch/_tensor.py in __format__(self, format_spec)
   1148             # Use detach() here to avoid the warning when converting a scalar Tensor that
   1149             # requires gradients to a python number. It is ok for formatting.
-> 1150             return self.detach().item().__format__(format_spec)
   1151         return object.__format__(self, format_spec)
   1152 

KeyboardInterrupt: 

In [6]:

import pandas as pd
def parse_tfrecord_test(example_proto):
    feature_description = {
        'image': tf.io.FixedLenFeature([], tf.string),
        'id' : tf.io.FixedLenFeature([], tf.string)
    }
    parsed = tf.io.parse_single_example(example_proto, feature_description)
    image = tf.image.decode_jpeg(parsed['image'], channels=3)
    image = tf.image.resize(image, [224, 224])
    #image = tf.image.convert_image_dtype(image, tf.float32)
    idd = parsed['id']
    return image,idd
def load_tfrecord_dataset_test(pattern):
    files = tf.io.gfile.glob(pattern)
    if not files:
        raise ValueError(f"No files found for pattern {pattern}")
    dataset = tf.data.TFRecordDataset(files)
    dataset = dataset.map(parse_tfrecord_test)
    # 可选：打乱、批处理等，但此处我们只返回样本级别的数据集
    return dataset
class TFRecordToPyTorchTest(IterableDataset):
    def __init__(self, tfrecord_pattern,transform=None):
        self.tfrecord_pattern = tfrecord_pattern
        self.transform=transform

    def __iter__(self):
        # 每次迭代创建新的数据集，保证可重复使用
        dataset = load_tfrecord_dataset_test(self.tfrecord_pattern)
        # 使用 as_numpy_iterator() 获取 NumPy 数组，便于转换为 PyTorch 张量
        for image_np,idd in dataset.as_numpy_iterator():
            # image_np shape: (224,224,3), dtype float32, label_np scalar int64
            # 转为 PyTorch 张量，并调整为 CxHxW
            image_pil = Image.fromarray((image_np).astype('uint8')) 
            if self.transform:
                image_tensor = self.transform(image_pil)
            else:
                # 如果不需要 transform，至少转为 tensor
                image_tensor = torch.from_numpy(image_np).permute(2,0,1)
            #image_torch = torch.from_numpy(image_np).permute(2, 0, 1)  # (3,224,224)
            #label_torch = torch.tensor(label_np, dtype=torch.long)
            id_torch = idd
            yield image_tensor,id_torch
tfrecord_path = '/kaggle/input/competitions/tpu-getting-started/tfrecords-jpeg-224x224/test/*'
dataset3 = TFRecordToPyTorchTest(tfrecord_path,transform)
test_dataloader = DataLoader(dataset3, batch_size=32, num_workers=0)
id_array=[]
all_preds=[]
model.load_state_dict(torch.load('model.pth'))
model.eval()
with torch.no_grad():
    for batch in test_dataloader:
        input_ids = batch[0].to(device)
        idd = batch[1]
        outputs = model(input_ids).logits
        preds = torch.argmax(outputs, dim=1)
        all_preds.extend(preds.cpu().numpy())
        id_array.extend(idd)
submission = pd.DataFrame({
    'id':id_array,
    'label': all_preds
})

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
/tmp/ipykernel_58/3358841615.py in <cell line: 0>()
     49 model.eval()
     50 with torch.no_grad():
---> 51     for batch in test_dataloader:
     52         input_ids = batch[0].to(device)
     53         idd = batch[1]

/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py in __next__(self)
    739                 # TODO(https://github.com/pytorch/pytorch/issues/76750)
    740                 self._reset()  # type: ignore[call-arg]
--> 741             data = self._next_data()
    742             self._num_yielded += 1
    743             if (

/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py in _next_data(self)
    799     def _next_data(self):
    800         index = self._next_index()  # may raise StopIteration
--> 801         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    802         if self._pin_memory:
    803             data = _utils.pin_memory.pin_memory(data, self._pin_memory_device)

/usr/local/lib/python3.12/dist-packages/torch/utils/data/_utils/fetch.py in fetch(self, possibly_batched_index)
     33             for _ in possibly_batched_index:
     34                 try:
---> 35                     data.append(next(self.dataset_iter))
     36                 except StopIteration:
     37                     self.ended = True

/tmp/ipykernel_58/3358841615.py in __iter__(self)
     33             image_pil = Image.fromarray((image_np).astype('uint8'))
     34             if self.transform:
---> 35                 image_tensor = self.transform(image_pil)
     36             else:
     37                 # 如果不需要 transform，至少转为 tensor

/usr/local/lib/python3.12/dist-packages/torchvision/transforms/transforms.py in __call__(self, img)
     93     def __call__(self, img):
     94         for t in self.transforms:
---> 95             img = t(img)
     96         return img
     97 

/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
   1774             return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1775         else:
-> 1776             return self._call_impl(*args, **kwargs)
   1777 
   1778     # torchrec tests the code consistency with the following code

/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
   1785                 or _global_backward_pre_hooks or _global_backward_hooks
   1786                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1787             return forward_call(*args, **kwargs)
   1788 
   1789         result = None

/usr/local/lib/python3.12/dist-packages/torchvision/transforms/transforms.py in forward(self, tensor)
    283             Tensor: Normalized Tensor image.
    284         """
--> 285         return F.normalize(tensor, self.mean, self.std, self.inplace)
    286 
    287     def __repr__(self) -> str:

/usr/local/lib/python3.12/dist-packages/torchvision/transforms/functional.py in normalize(tensor, mean, std, inplace)
    348         raise TypeError(f"img should be Tensor Image. Got {type(tensor)}")
    349 
--> 350     return F_t.normalize(tensor, mean=mean, std=std, inplace=inplace)
    351 
    352 

/usr/local/lib/python3.12/dist-packages/torchvision/transforms/_functional_tensor.py in normalize(tensor, mean, std, inplace)
    920     mean = torch.as_tensor(mean, dtype=dtype, device=tensor.device)
    921     std = torch.as_tensor(std, dtype=dtype, device=tensor.device)
--> 922     if (std == 0).any():
    923         raise ValueError(f"std evaluated to zero after conversion to {dtype}, leading to division by zero.")
    924     if mean.ndim == 1:

KeyboardInterrupt: 

In [ ]:

submission['id'] = submission['id'].apply(lambda x: x.decode('utf-8'))
print(submission)
submission.to_csv('submission.csv', index=False)
print("Submission saved!")

In [ ]: