Skip to content

Instantly share code, notes, and snippets.

View gautierdag's full-sized avatar
🇺🇦

Gautier Dagan gautierdag

🇺🇦
View GitHub Profile
@gautierdag
gautierdag / plaid_models.py
Created May 31, 2020 17:35
Plaid Python Pydantic Models
import datetime
from typing import Optional, List
from pydantic import BaseModel
class PlaidBalance(BaseModel):
current: float
available: Optional[float]
iso_currency_code: Optional[str]
limit: Optional[float]
@gautierdag
gautierdag / generate_random_series.py
Last active July 15, 2020 08:26
Generating a random series
from random import randint
import pandas as pd
num_observations = 500
dates = pd.to_datetime([f"{randint(2018, 2020)}-{randint(1, 12)}-{randint(1, 27)}"
for _ in range(num_observations)]).unique()
df = pd.DataFrame({"date": dates,
"balance": [randint(0, 10000)
for _ in range(len(dates))]
})
@gautierdag
gautierdag / naive_compression.py
Created July 15, 2020 08:49
Naive Date Compression
# set first (oldest balance as index entry)
new_indexes = [0]
# choose columns that are not the date column
cols = [d for d in df.columns if d != "date"]
# iterate over rows
for i, row in df.iterrows():
if i == 0: #skip 0 index since it has no previous balance
continue
# if not all values of the previous row/date is not equal to current - then we have new observation
if not (df.iloc[i-1][cols] ==row[cols]).all():
@gautierdag
gautierdag / reduce_compress.py
Created July 15, 2020 08:51
Reduce Date Compression
from functools import reduce
values = df.values
new_values = []
cols = [df.columns.get_loc(d) for d in df.columns if d != "date"]
new_values.append(values[0])
def compress_helper(x, y):
if (x[cols] == y[cols]).all():
return x
@gautierdag
gautierdag / shift_compress.py
Created July 15, 2020 08:53
Shift Date Compression
cols = [d for d in df.columns if d != "date"]
compressed_df = df.loc[~(df[cols].shift() == df[cols]).all(axis=1)].reset_index(drop=True)
@gautierdag
gautierdag / numpy_compress.py
Last active July 15, 2020 10:47
Numpy Optimized Compress
import numpy as np
#cast as object to be able to set np.nan and handle different types
arr = df[cols].values.astype(object)
# build shifted array
shifted = np.roll(arr, 1)
shifted[0] = np.nan
# choose indexes based on shift comparaison
@gautierdag
gautierdag / print_macro.workflow
Last active September 9, 2020 09:22
MacOS macro for wrapping line in text with print("....")
# https://apple.stackexchange.com/questions/175215/how-do-i-assign-a-keyboard-shortcut-to-an-applescript-i-wrote
on run {input, parameters}
tell application "System Events"
key code 123 using command down
keystroke "print("
key code 124 using command down
keystroke ")"
end tell
@gautierdag
gautierdag / model_with_noam.py
Last active September 25, 2023 02:27
pytorch-lightning Transformer (noam) lr policy
import torch
import pytorch_lightning as pl
class MyTransformer(pl.LightningModule):
def __init__(
self,
learning_rate=0.001,
warmup=4000,
):
self.learning_rate = learning_rate
@gautierdag
gautierdag / NCE.py
Last active April 12, 2021 21:50
Pytorch NCE Loss
import torch
import torch.nn as nn
import pytorch_lightning as pl
class NCE(pl.LightningModule):
"""
This implementation is taken from https://github.com/Spijkervet/SimCLR/blob/master/simclr/modules/nt_xent.py
The mask_correlated_samples funtion has been modified to be much faster to compute
and therefore be able to be called at train time without a predifined batch size.
"""
@gautierdag
gautierdag / multiple_optims_pl.py
Created April 14, 2021 12:04
Example of how to use different optims for different layers or modules using pytorch lightning
import torch
import torch.nn as nn
import pytorch_lightning as pl
class BoringModel(pl.LightningModule):
def __init__(
self
):
super(BoringModel, self).__init__()
self.automatic_optimization = False