Skip to content

Instantly share code, notes, and snippets.

Avatar

Konstantin Verner konverner

  • Pau, France
View GitHub Profile
@konverner
konverner / mnist_pytorch_pipeline.py
Last active May 28, 2023 22:15
Template for image classification with pytorch on mnist
View mnist_pytorch_pipeline.py
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor
from tqdm import tqdm
# Set device (GPU or CPU)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@konverner
konverner / pathlib.py
Created May 7, 2023 00:23
Defining paths with pathlib
View pathlib.py
from pathlib import Path
DIR = Path.cwd() # work directory
PATH_TEST_DIR = Path(DIR, 'data')
PATH_TEST_FILE = Path( DIR, 'file.csv')
@konverner
konverner / pil_image_from_url.py
Created April 27, 2023 22:31
Read image with PIL from url
View pil_image_from_url.py
from PIL import Image
import requests
from io import BytesIO
url = 'https://upload.wikimedia.org/wikipedia/en/7/7d/Lenna_%28test_image%29.png'
response = requests.get(url)
img = Image.open(BytesIO(response.content))
@konverner
konverner / iqr_anomaly_processing.py
Last active April 22, 2023 10:04
Apply IQR method for anomaly remove for the given columns in a dataframe
View iqr_anomaly_processing.py
N_before = len(df)
COLUMNS_TO_PROCESS = ['sqft_living', 'sqft_basement', 'bedrooms']
TRESHOLD_Q3 = 0.95
TRESHOLD_Q1 = 0.05
for column_name in COLUMNS_TO_PROCESS:
Q3 = df[column_name].quantile(TRESHOLD_Q3)
Q1 = df[column_name].quantile(TRESHOLD_Q1)
print(f"{column_name} [Q1, Q3] : [{Q1}, {Q3}]")
IQR = Q3 - Q1
@konverner
konverner / numb_nan_percent.py
Created April 22, 2023 09:15
show number of NaNs in percents
View numb_nan_percent.py
pd.options.display.float_format = '{:,.1f}%'.format
print((df.isna().sum()/df.count()).sort_values())
@konverner
konverner / clf_borders.py
Last active April 9, 2023 13:47
visualise classifier borders
View clf_borders.py
X_train = ... # (n,2) numpy array dataset
# Create a meshgrid
x_min, x_max = X_train[:, 0].min() - 1, X_train[:, 0].max() + 1
y_min, y_max = X_train[:, 1].min() - 1, X_train[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
np.arange(y_min, y_max, 0.1))
# Make predictions on the meshgrid
@konverner
konverner / fillna.py
Created April 8, 2023 21:35
fillna based on some conditions
View fillna.py
column = ...
cond1, cond2 = ..., ...
agg_func = 'mean'
if agg_func == 'mean':
new_value = df[(cond1) & (cond2)][column].mean()
else:
new_value = df[(cond1) & (cond2)][column].mode()[0]
df.loc[(cond1) & (cond2), column] = df.loc[(cond1) & (cond2)][column].fillna(new_value)
@konverner
konverner / mixup.py
Created February 12, 2023 21:58
pytorch implementation of mixup
View mixup.py
def mixup(X : torch.tensor,
Y: torch.tensor,
alpha: int=8,
device: str='cpu'):
"""
it performs mixup between the first half of the batch with the second one
params
---
X : images tensor [bs, c, h, w]
@konverner
konverner / grad_norm.py
Created February 5, 2023 11:58
compute gradient norm of pytorch model
View grad_norm.py
def grad_norm(model: nn.Module):
total_norm = 0
for p in model.parameters():
param_norm = p.grad.detach().data.norm(2)
total_norm += param_norm.item() ** 2
return total_norm ** 0.5
@konverner
konverner / resnet_encoder.py
Created January 26, 2023 23:56
get embeddings from resnet with pytorch
View resnet_encoder.py
import torch
from torchvision import models
modules=list(models.resnet18(weights='IMAGENET1K_V1').children())[:-1]
model = nn.Sequential(*modules)
x = torch.rand(size=[1, 3, 640, 480])
emb = model(x)