from fastai import * | |
from fastai.tabular import * | |
from fastai.vision import * | |
PATH = os.path.abspath('..') | |
# distinguish categorical and continuous variables, and dependent variable | |
cat_names = ['cat1', 'cat2', 'cat3'] | |
cont_names =['cont1', 'cont2'] | |
dep_var = 'target' | |
# transformations for image augmentation | |
tfms = get_transforms(do_flip=True, | |
flip_vert=True, | |
max_rotate=15, | |
max_zoom=1.05, | |
max_warp=0, | |
max_lighting=0, | |
) | |
def get_dataframe(): | |
"""Returns DataFrame containing tabular data, image names, and targets.""" | |
# main data set | |
df = pd.read_csv(f'{PATH}/path/to/data.csv') | |
# isolate useful columns | |
df = df[cont_names + ['img_fn'] + [dep_var]] | |
return df | |
def get_val_idxs(n, seed=1234): | |
np.random.seed(seed) | |
return np.random.permutation(np.arange(n))[0:int(0.2 * n)] | |
class TabConvDataset(Dataset): | |
"""A Dataset of combined tabular data, image names, and targets.""" | |
def __init__(self, x_tab, x_img, y): | |
self.x_tab, self.x_img, self.y = x_tab, x_img, y | |
def __len__(self): return len(self.y) | |
def __getitem__(self, i): | |
return (self.x_tab[i], self.x_img[i]), self.y[i] | |
class TabConvModel(nn.Module): | |
"""A combined neural network using the convnet and tabular model""" | |
def __init__(self, tab_model, img_model, layers, drops): | |
super().__init__() | |
self.tab_model = tab_model | |
self.img_model = img_model | |
lst_layers = [] | |
activs = [nn.ReLU(inplace=True),] * (len(layers) - 2) + [None] | |
for n_in, n_out, p, actn in zip(layers[:-1], layers[1:], drops, activs): | |
lst_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn) | |
self.layers = nn.Sequential(*lst_layers) | |
def forward(self, *x): | |
x_tab = self.tab_model(*x[0]) | |
x_img = self.img_model(x[1]) | |
x = torch.cat([x_tab, x_img], dim=1) | |
return self.layers(x) | |
def initialize_combined_model(n_lin_tab=16, n_lin_conv=32, ps_final=0.25, bs=64, sz=502, | |
seed=1234, val_pct=0.2, img_tfms=tfms): | |
"""Initialize a combined model that can learn from both tabular and image data. | |
Params | |
------ | |
n_lin_tab: int | |
number of linear nodes in the single hidden layer for the tabular model | |
(default is ) | |
n_lin_conv: int | |
number of linear nodes in the final dense layer for the convolutional model | |
(default is 32) | |
ps_final: float | |
dropout fraction in the final linear layers (default is 0.25) | |
bs: int | |
batchsize for loading combined TabConvData (default is 64) | |
sz: int | |
image size | |
seed: int, optional | |
random seed passed to numpy (default is 1234) | |
val_pct: float, optional | |
fraction of data used for validation (default is 0.2) | |
img_tfms: Transforms, optional | |
set of transformations for image augmentation (global default set) | |
Returns | |
------- | |
learn: Learner | |
combined `Learner` object built on top of the TabConvModel and | |
supplied data | |
""" | |
df = get_dataframe() | |
val_idxs = get_val_idxs(len(df)) | |
# preprocessing | |
procs = [Normalize] | |
# set up tabular data and learner | |
tab_data = (TabularList.from_df(df, path=PATH, cat_names=cat_names, cont_names=cont_names, procs=procs) | |
.split_by_idx(valid_idx=val_idxs) | |
.label_from_df(cols=dep_var, label_cls=FloatList) | |
.databunch(bs=bs)) | |
tab_learn = tabular_learner(tab_data, layers=[n_lin_tab], loss_func=root_mean_squared_error) | |
# set up image data and learner | |
img_data = (ImageList.from_df(path=PATH, | |
df=df, | |
folder='data/images', | |
cols='img_fn', | |
suffix='.jpg') | |
.split_by_idx(valid_idx=val_idxs) | |
.label_from_df(cols='target', label_cls=FloatList) | |
.transform(img_tfms, size=sz) | |
.databunch(bs=bs) | |
) | |
img_learn = cnn_learner(img_data, models.resnet34, | |
pretrained=True, | |
loss_func=root_mean_squared_error, | |
) | |
# combined data | |
train_ds = TabConvDataset(tab_data.train_ds.x, img_data.train_ds.x, tab_data.train_ds.y) | |
valid_ds = TabConvDataset(tab_data.valid_ds.x, img_data.valid_ds.x, tab_data.valid_ds.y) | |
train_dl = DataLoader(train_ds, bs) | |
valid_dl = DataLoader(valid_ds, 2 * bs) | |
data = DataBunch(train_dl, valid_dl, path=PATH) | |
# chop off final layers from both models | |
tab_learn.model.layers = tab_learn.model.layers[:-2] | |
img_learn.model[-1] = nn.Sequential(*img_learn.model[-1][:-5], nn.Linear(1024, n_lin_conv, bias=True), nn.ReLU(inplace=True)) | |
lin_layers = [n_lin_tab + n_lin_conv, 1] | |
ps = [ps_final] | |
# initialize model | |
model = TabConvModel(tab_learn.model, img_learn.model, lin_layers, ps) | |
layer_groups = [nn.Sequential(*flatten_model(img_learn.layer_groups[0])), | |
nn.Sequential(*flatten_model(img_learn.layer_groups[1])), | |
nn.Sequential(*(flatten_model(img_learn.layer_groups[2]) + | |
flatten_model(model.tab_model) + | |
flatten_model(model.layers))) | |
] | |
# combined learner | |
learn = Learner(data, model, | |
loss_func=root_mean_squared_error, | |
layer_groups=layer_groups, | |
) | |
return learn | |
if __name__ == '__main__': | |
learn = initialize_combined_model() | |
learn.freeze() | |
learn.fit_one_cycle(1, 1e-2) | |
learn.save('combined-init-train-1') | |
learn.freeze_to(-2) | |
learn.fit_one_cycle(1, 1e-2) | |
learn.save('combined-init-train-2') | |
learn.unfreeze() | |
learn.fit_one_cycle(20, 1e-3) | |
learn.save('combined-init-train-3') |
Can you also share a sample datafile. will make it a bit easy to work with the sample code. Thanks
I didn't actually try this with an example data set, but imagine you would want to access a tabular data file in the location {PATH}/data/catalog.csv
, which could contain data like the following:
cat1, cat2, cat3, cont1, cont2, img_fn, target
A, middle, True, 6.4, -3.1, object_13702, 0.45
B, lower, True, 1.1, 2.1, object_11204, 0.98
C, upper, False, 0.8, -3.3, object_11092, 0.2
...
Your directory structure would have to look something like:
{PATH}
└── data
├── catalog.csv
└── images
├── object_13702.jpg
├── object_11204.jpg
└── ...
This is obviously a contrived example but hopefully it makes more sense this way. Note of course that the path to the CSV file in line 25 would need to be updated.
wanted to comment that I got a dtype error when using loss_func=root_mean_squared_error and had to switch to MSELossFlat instead
I have trouble to get learn.predict() to work using TabConvDataset. How did you proceed outside training to get prediction ?
Sorry, this code is now considerably out of date. It uses fastai version 1.0 and Pytorch 1.0. There may be a better version for fastai v2.x on the forums.
Can you also share a sample datafile. will make it a bit easy to work with the sample code. Thanks