Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
from fastai import *
from fastai.tabular import *
from fastai.vision import *
PATH = os.path.abspath('..')
# distinguish categorical and continuous variables, and dependent variable
cat_names = ['cat1', 'cat2', 'cat3']
cont_names =['cont1', 'cont2']
dep_var = 'target'
# transformations for image augmentation
tfms = get_transforms(do_flip=True,
flip_vert=True,
max_rotate=15,
max_zoom=1.05,
max_warp=0,
max_lighting=0,
)
def get_dataframe():
"""Returns DataFrame containing tabular data, image names, and targets."""
# main data set
df = pd.read_csv(f'{PATH}/path/to/data.csv')
# isolate useful columns
df = df[cont_names + ['img_fn'] + [dep_var]]
return df
def get_val_idxs(n, seed=1234):
np.random.seed(seed)
return np.random.permutation(np.arange(n))[0:int(0.2 * n)]
class TabConvDataset(Dataset):
"""A Dataset of combined tabular data, image names, and targets."""
def __init__(self, x_tab, x_img, y):
self.x_tab, self.x_img, self.y = x_tab, x_img, y
def __len__(self): return len(self.y)
def __getitem__(self, i):
return (self.x_tab[i], self.x_img[i]), self.y[i]
class TabConvModel(nn.Module):
"""A combined neural network using the convnet and tabular model"""
def __init__(self, tab_model, img_model, layers, drops):
super().__init__()
self.tab_model = tab_model
self.img_model = img_model
lst_layers = []
activs = [nn.ReLU(inplace=True),] * (len(layers) - 2) + [None]
for n_in, n_out, p, actn in zip(layers[:-1], layers[1:], drops, activs):
lst_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn)
self.layers = nn.Sequential(*lst_layers)
def forward(self, *x):
x_tab = self.tab_model(*x[0])
x_img = self.img_model(x[1])
x = torch.cat([x_tab, x_img], dim=1)
return self.layers(x)
def initialize_combined_model(n_lin_tab=16, n_lin_conv=32, ps_final=0.25, bs=64, sz=502,
seed=1234, val_pct=0.2, img_tfms=tfms):
"""Initialize a combined model that can learn from both tabular and image data.
Params
------
n_lin_tab: int
number of linear nodes in the single hidden layer for the tabular model
(default is )
n_lin_conv: int
number of linear nodes in the final dense layer for the convolutional model
(default is 32)
ps_final: float
dropout fraction in the final linear layers (default is 0.25)
bs: int
batchsize for loading combined TabConvData (default is 64)
sz: int
image size
seed: int, optional
random seed passed to numpy (default is 1234)
val_pct: float, optional
fraction of data used for validation (default is 0.2)
img_tfms: Transforms, optional
set of transformations for image augmentation (global default set)
Returns
-------
learn: Learner
combined `Learner` object built on top of the TabConvModel and
supplied data
"""
df = get_dataframe()
val_idxs = get_val_idxs(len(df))
# preprocessing
procs = [Normalize]
# set up tabular data and learner
tab_data = (TabularList.from_df(df, path=PATH, cat_names=cat_names, cont_names=cont_names, procs=procs)
.split_by_idx(valid_idx=val_idxs)
.label_from_df(cols=dep_var, label_cls=FloatList)
.databunch(bs=bs))
tab_learn = tabular_learner(tab_data, layers=[n_lin_tab], loss_func=root_mean_squared_error)
# set up image data and learner
img_data = (ImageList.from_df(path=PATH,
df=df,
folder='data/images',
cols='img_fn',
suffix='.jpg')
.split_by_idx(valid_idx=val_idxs)
.label_from_df(cols='target', label_cls=FloatList)
.transform(img_tfms, size=sz)
.databunch(bs=bs)
)
img_learn = cnn_learner(img_data, models.resnet34,
pretrained=True,
loss_func=root_mean_squared_error,
)
# combined data
train_ds = TabConvDataset(tab_data.train_ds.x, img_data.train_ds.x, tab_data.train_ds.y)
valid_ds = TabConvDataset(tab_data.valid_ds.x, img_data.valid_ds.x, tab_data.valid_ds.y)
train_dl = DataLoader(train_ds, bs)
valid_dl = DataLoader(valid_ds, 2 * bs)
data = DataBunch(train_dl, valid_dl, path=PATH)
# chop off final layers from both models
tab_learn.model.layers = tab_learn.model.layers[:-2]
img_learn.model[-1] = nn.Sequential(*img_learn.model[-1][:-5], nn.Linear(1024, n_lin_conv, bias=True), nn.ReLU(inplace=True))
lin_layers = [n_lin_tab + n_lin_conv, 1]
ps = [ps_final]
# initialize model
model = TabConvModel(tab_learn.model, img_learn.model, lin_layers, ps)
layer_groups = [nn.Sequential(*flatten_model(img_learn.layer_groups[0])),
nn.Sequential(*flatten_model(img_learn.layer_groups[1])),
nn.Sequential(*(flatten_model(img_learn.layer_groups[2]) +
flatten_model(model.tab_model) +
flatten_model(model.layers)))
]
# combined learner
learn = Learner(data, model,
loss_func=root_mean_squared_error,
layer_groups=layer_groups,
)
return learn
if __name__ == '__main__':
learn = initialize_combined_model()
learn.freeze()
learn.fit_one_cycle(1, 1e-2)
learn.save('combined-init-train-1')
learn.freeze_to(-2)
learn.fit_one_cycle(1, 1e-2)
learn.save('combined-init-train-2')
learn.unfreeze()
learn.fit_one_cycle(20, 1e-3)
learn.save('combined-init-train-3')
@manchanda
Copy link

manchanda commented Feb 7, 2020

Can you also share a sample datafile. will make it a bit easy to work with the sample code. Thanks

@jwuphysics
Copy link
Author

jwuphysics commented Feb 7, 2020

Can you also share a sample datafile. will make it a bit easy to work with the sample code. Thanks

I didn't actually try this with an example data set, but imagine you would want to access a tabular data file in the location {PATH}/data/catalog.csv, which could contain data like the following:

cat1, cat2, cat3, cont1, cont2, img_fn, target
A, middle, True, 6.4, -3.1, object_13702, 0.45
B, lower, True, 1.1, 2.1, object_11204, 0.98
C, upper, False, 0.8, -3.3, object_11092, 0.2
...

Your directory structure would have to look something like:

{PATH}
└── data
    ├── catalog.csv
    └── images
        ├── object_13702.jpg 
        ├── object_11204.jpg
        └── ...

This is obviously a contrived example but hopefully it makes more sense this way. Note of course that the path to the CSV file in line 25 would need to be updated.

@noob9000
Copy link

noob9000 commented May 6, 2020

wanted to comment that I got a dtype error when using loss_func=root_mean_squared_error and had to switch to MSELossFlat instead

@ascientist
Copy link

ascientist commented Oct 8, 2020

I have trouble to get learn.predict() to work using TabConvDataset. How did you proceed outside training to get prediction ?

@jwuphysics
Copy link
Author

jwuphysics commented Oct 8, 2020

Sorry, this code is now considerably out of date. It uses fastai version 1.0 and Pytorch 1.0. There may be a better version for fastai v2.x on the forums.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment