PyTorch Forum Question: Frequencey of evaluation steps sffecting PyTorch model accuracy

example.py

import os
import random
import numpy as np
from tqdm import tqdm
from omegaconf import DictConfig

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler

from torchvision import datasets, transforms
from torchvision.models import vit_b_16, ViT_B_16_Weights


def seed_everything(seed):
    random.seed(seed)
    np.random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    cudnn.benchmark = True
    cudnn.deterministic = True


def build_dataset(cfg):
    train_trsf = transforms.Compose([
        transforms.RandomResizedCrop((cfg.img_size, cfg.img_size), scale=(0.05, 1.0)),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
    ])
    eval_trsf = transforms.Compose([
        transforms.Resize((cfg.img_size, cfg.img_size)),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
    ])

    trainset = datasets.CIFAR100(
        root=cfg.dataset_path, train=True,
        download=True, transform=train_trsf
    )
    evalset = datasets.CIFAR100(
        root=cfg.dataset_path, train=False,
        download=True, transform=eval_trsf
    )

    train_loader = DataLoader(
        trainset,
        sampler=RandomSampler(trainset),
        batch_size=cfg.batch_size,
        num_workers=cfg.num_workers,
        pin_memory=True
    )
    eval_loader = DataLoader(
        evalset,
        sampler=SequentialSampler(evalset),
        batch_size=cfg.batch_size,
        num_workers=cfg.num_workers,
        pin_memory=True
    )

    return train_loader, eval_loader


@torch.no_grad()
def evaluate(cfg, model, criterion, eval_loader, device):
    print("Evaluating ....")
    running_acc = 0.0
    running_loss = 0.0
    total_samples = 0
    model.eval()
    for batch_idx, (inputs, targets) in enumerate(eval_loader):
        with torch.no_grad():
            inputs, targets = inputs.to(device), targets.to(device)
            logits = model(inputs)
            # logits = logits.softmax(dim=-1)
            loss = criterion(logits, targets)
            predictions = torch.argmax(logits, dim=1)

        running_acc += (predictions == targets).sum().item()
        running_loss += loss.item()
        total_samples += inputs.shape[0]

    accuracy = running_acc / total_samples
    loss = running_loss / (batch_idx+1)

    return accuracy, loss


def train_one_epoch(cfg, model, criterion, optimizer, train_loader, device):
    print("Training ....")

    model.train()
    running_acc = 0.0
    running_loss = 0.0
    total_samples = 0
    for batch_idx, (inputs, targets) in enumerate(train_loader):
        inputs, targets = inputs.to(device), targets.to(device)

        optimizer.zero_grad()
        logits = model(inputs)
        # logits = logits.softmax(dim=-1)
        loss = criterion(logits, targets)
        predictions = torch.argmax(logits, dim=1)

        loss.backward()
        optimizer.step()

        running_acc += (predictions == targets).sum().item()
        running_loss += loss.item()
        total_samples += inputs.shape[0]

    accuracy = running_acc / total_samples
    loss = running_loss / (batch_idx+1)

    return accuracy, loss


def train_and_evaluate(cfg, model, optimizer, criterion, train_loader, eval_loader, device):

    for epoch in range(cfg.num_epochs):
        # uncomment below to see effect
        # evaluate(cfg, model, criterion, eval_loader, device)
        
        train_one_epoch(cfg, model, criterion, optimizer, train_loader, device)
        acc, loss = evaluate(cfg, model, criterion, eval_loader, device)

        print(acc, loss)


def main(cfg: DictConfig) -> None:

    seed_everything(cfg.seed)

    # dataset
    train_loader, eval_loader = build_dataset(cfg)
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    # model
    model = vit_b_16(weights=ViT_B_16_Weights.DEFAULT)
    model.heads.head = nn.Linear(model.heads.head.weight.shape[1], cfg.num_classes)
    if isinstance(model.heads.head, nn.Linear):
        nn.init.zeros_(model.heads.head.weight)
        nn.init.zeros_(model.heads.head.bias)

    model = model.to(device)
    
    # training modules
    optimizer = torch.optim.SGD(model.parameters(), lr=cfg.learning_rate)
    criterion = torch.nn.CrossEntropyLoss()

    train_and_evaluate(cfg, model, optimizer, criterion, train_loader, eval_loader, device)


if __name__ == "__main__":
    config = DictConfig(dict(
        img_size=224,
        dataset_path="../datasets/cifar100",
        batch_size=64,
        num_workers=4,
        seed=0,
        num_classes=100,
        learning_rate=0.01,
        num_epochs=1
    ))
    main(cfg=config)