Skip to content

Instantly share code, notes, and snippets.

@kirk86

kirk86/akmtdfgen.py

Forked from timehaven/akmtdfgen.py
Created August 26, 2017 06:24
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save kirk86/255cd74a6111e3b76299fd8ec2a499e5 to your computer and use it in GitHub Desktop.
Save kirk86/255cd74a6111e3b76299fd8ec2a499e5 to your computer and use it in GitHub Desktop.
Download ZIP
kmtdfgen: Keras multithreaded dataframe generator
Raw
akmtdfgen.py
"""akmtdfgen: A Keras multithreaded dataframe generator.
Works with Python 2.7 and Keras 2.x.
For Python 3.x, need to fiddle with the threadsafe generator code.
Test the generator_from_df() functions by running this file:
python akmtdfgen.py
Threadsafe generator code below taken from the answer of user
https://github.com/parag2489
on the Keras issue
https://github.com/fchollet/keras/issues/1638
which uses contributions from
http://anandology.com/blog/using-iterators-and-generators/
The rest of this file was written by
Ryan Woodard | AppNexus | Data Science | 2017
If you have bcolz errors like:
`start`+`nitems` out of boundsException RuntimeError:
RuntimeError('fatal error during Blosc
decompression: -1',) in
'bcolz.carray_ext.chunk._getitem' ignored
check that your versions are up to date. Here is what I am using:
In [1]: import bcolz
In [2]: bcolz.print_versions()
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
bcolz version: 1.1.2
NumPy version: 1.13.1
Blosc version: 1.11.2 ($Date:: 2017-01-27 #$)
Blosc compressors: ['blosclz', 'lz4', 'lz4hc', 'snappy', 'zlib', 'zstd']
Numexpr version: 2.6.2
Dask version: not available (version >= 0.9.0 not detected)
Python version: 2.7.13 |Continuum Analytics, Inc.| (default, Dec 20 2016, 23:09:15)
[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]
Platform: linux2-x86_64
Byte-ordering: little
Detected cores: 12
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
"""
from __future__ import print_function
from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
import numpy as np
import pandas as pd
import bcolz
import threading
import os
import sys
import glob
import shutil
bcolz_lock = threading.Lock()
# old_blosc_nthreads = bcolz.blosc_set_nthreads(1)
# assert bcolz.blosc_set_nthreads(1) == 1
def safe_bcolz_open(fname, idx=None, debug=False):
"""Threadsafe way to read bcolz arrays.
bcolz might have issues with multithreading and underlying blosc
compression code. Lots of discussion out there, here are some
starting points:
http://www.pytables.org/latest/cookbook/threading.html
https://github.com/dask/dask/issues/1033
Since our threads are read-only on the static bcolz array on disk,
we'll probably be ok, but no guarantees. Test, test, test! It is
so important that the auxiliary matrix rows stay properly aligned
with the images DataFrame rows.
"""
with bcolz_lock:
if idx is None:
X2 = bcolz.open(fname)
else:
X2 = bcolz.open(fname)[idx]
if debug:
df_debug = pd.DataFrame(X2, index=idx)
# print(len(idx))
assert X2.shape[0] == len(idx)
assert X2.shape == df_debug.shape
# Should see index matching int() of data values.
# print(df_debug.iloc[:5, :5])
# print(df_debug.iloc[-5:, -5:])
df_debug = df_debug.astype(int)
# print(df_debug.iloc[:5, :5])
# print(df_debug.iloc[-5:, -5:])
# Here is why we made the test data as we did. Make sure
# data cast to int (not rounded up!) matches index values.
test_idx = (df_debug.subtract(df_debug.index.values, axis=0) == 0).all(axis=1)
assert test_idx.all(), df_debug[~test_idx]
return X2
class threadsafe_iter(object):
"""Takes an iterator/generator and makes it thread-safe by
serializing call to the `next` method of given iterator/generator.
https://github.com/fchollet/keras/issues/1638
http://anandology.com/blog/using-iterators-and-generators/
"""
def __init__(self, it):
self.it = it
self.lock = threading.Lock()
assert self.lock is not bcolz_lock
def __iter__(self):
return self
def next(self):
with self.lock:
return self.it.next()
def threadsafe_generator(f):
"""A decorator that takes a generator function and makes it thread-safe.
https://github.com/fchollet/keras/issues/1638
http://anandology.com/blog/using-iterators-and-generators/
"""
def g(*a, **kw):
return threadsafe_iter(f(*a, **kw))
return g
@threadsafe_generator
def generator_from_df(df, batch_size, target_size, features=None,
debug_merged=False):
"""Generator that yields (X, Y).
If features is not None, assume it is the path to a bcolz array
that can be indexed by the same indexing of the input df.
Assume input DataFrame df has columns 'imgpath' and 'target', where
'imgpath' is full path to image file.
https://github.com/fchollet/keras/issues/1627
https://github.com/fchollet/keras/issues/1638
Be forewarned if/when you modify this function: some errors will
not be explicit, appearing only as a generic:
ValueError: output of generator should be a tuple `(x, y, sample_weight)` or `(x, y)`. Found: None
It usually means something in your infinite loop is not doing what
you think it is, so the loop crashes and returns None. Check your
DataFrame in this function with various print statements to see if
it is doing what you think it is doing.
Again, error messages will not be too helpful here--if in doubt,
print().
"""
if features is not None:
assert os.path.exists(features)
assert safe_bcolz_open(features).shape[0] == df.shape[0], "Features rows must match df!"
# Each epoch will only process an integral number of batch_size
# but with the shuffling of df at the top of each epoch, we will
# see all training samples eventually, but will skip an amount
# less than batch_size during each epoch.
nbatches, n_skipped_per_epoch = divmod(df.shape[0], batch_size)
# At the start of *each* epoch, this next print statement will
# appear once for *each* worker specified in the call to
# model.fit_generator(...,workers=nworkers,...)!
# print("""
# Initialize generator:
# batch_size = %d
# nbatches = %d
# df.shape = %s
# """ % (batch_size, nbatches, str(df.shape)))
count = 1
epoch = 0
# New epoch.
while 1:
# The advantage of the DataFrame holding the image file name
# and the labels is that the entire df fits into memory and
# can be easily shuffled at the start of each epoch.
#
# Shuffle each epoch using the tricky pandas .sample() way.
df = df.sample(frac=1) # frac=1 is same as shuffling df.
epoch += 1
i, j = 0, batch_size
# Mini-batches within epoch.
mini_batches_completed = 0
for _ in range(nbatches):
# Callbacks are more elegant but this print statement is
# included to be explicit.
# print("Top of generator for loop, epoch / count / i / j = "\
# "%d / %d / %d / %d" % (epoch, count, i, j))
sub = df.iloc[i:j]
try:
# preprocess_input()
# https://github.com/fchollet/keras/blob/master/keras/applications/inception_v3.py#L389
X = np.array([
(2 *
# Resizing on the fly is efficient enough for
# pre-caching when a GPU is training a
# mini-batch. Here is where some additional
# data augmentation could take place.
(img_to_array(load_img(f, target_size=target_size))
/ 255.0 - 0.5))
for f in sub.imgpath])
Y = sub.target.values
if features is None:
# Simple model, one input, one output.
mini_batches_completed += 1
yield X, Y
else:
# For merged model: two input, one output.
#
# HEY: You should probably test this very
# carefully!
# Make (slightly) more efficient by removing the
# debug_merged check.
X2 = safe_bcolz_open(features, sub.index.values, debug=debug_merged)
mini_batches_completed += 1
yield [X, X2], Y
# Or:
# yield [X, bcolz.open(features)[sub.index.values]], Y
except IOError as err:
# A type of lazy person's regularization: with
# millions of images, if there are a few bad ones, no
# need to find them, just skip their mini-batch if
# they throw an error and move on to the next
# mini-batch. With the shuffling of the df at the top
# of each epoch, the bad apples will be in a different
# mini-batch next time around. Yes, they will
# probably crash that mini-batch, too, but so what?
# This is easier than finding bad files each time.
# Let's decrement count in anticipation of the
# increment coming up--this one won't count, so to
# speak.
count -= 1
# Actually, we could make this a try...except...else
# with the count increment. Homework assignment left
# to the reader.
i = j
j += batch_size
count += 1
def file_path_from_db_id(db_id, pattern="blah_%d.png", top="/tmp/path/to/imgs"):
"""Return file path /top/yyy/xx/blah_zzzxxyyy.png for db_id zzzxxyyy.
The idea is to hash into 1k top level dirs, 000 - 999, then 100
second level dirs, 00-99, so that the following database ids
result in the associated file paths:
1234567 /tmp/path/to/imgs/567/34/blah_1234567.png
432 /tmp/path/to/imgs/432/00/blah_432.png
29847 /tmp/path/to/imgs/847/29/blah_29847.png
1432 /tmp/path/to/imgs/432/01/blah_1432.png
Notice that changing pattern to pattern="blah_%09d.png" and
top="" would result in:
1234567 567/34/blah_001234567.png
432 432/00/blah_000000432.png
29847 847/29/blah_000029847.png
1432 432/01/blah_000001432.png
In general, this will give a decent spread for up to 100 million images.
If you have more than 10 million images, or your database ids are
higher, then this function is easily modified.
"""
s = '%09d' % db_id
return os.path.join(top, s[-3:], s[-5:-3], pattern % db_id)
#
# Helper functions, just for blog post demo.
#
def new_tricks_from_old_dogs(stage, label):
"""Convert list of Kaggle data files into DataFrame generator format.
That is, go from:
cd /path/to/kaggle/data/
ls train/dogs| head
dog.1000.jpg
dog.1001.jpg
dog.1002.jpg
dog.1003.jpg
dog.1004.jpg
dog.1005.jpg
dog.1006.jpg
dog.1007.jpg
dog.1008.jpg
dog.1009.jpg
to this:
new orig label
760 /tmp/path/to/imgs/760/00/dog_760.jpg validation/dogs/dog.760.jpg dog
7724 /tmp/path/to/imgs/724/07/dog_7724.jpg validation/dogs/dog.7724.jpg dog
7685 /tmp/path/to/imgs/685/07/dog_7685.jpg validation/dogs/dog.7685.jpg dog
Only including 'cat' and 'dog' in 'new' file name because the
numbers in the cats/dogs directories are non-unique. This avoids
collisions.
"""
s = "data/%s/%ss/*.jpg" % (stage, label)
#print(s, os.abspath(os.curdir))
old_dogs = glob.glob(s)
print(len(old_dogs), stage, label)
index = list(map(int, [d.split('.')[-2] for d in old_dogs]))
new_tricks = [file_path_from_db_id(i, pattern='%s_%%d.jpg' % label) for i in index]
return pd.DataFrame({'orig': old_dogs, 'new': new_tricks, 'label': label}, index=index)
def mv_to_new_hierarchy(row, orig='orig', new='new'):
"""Copy file from orig to new."""
if os.path.exists(row[new]):
return
d, f = os.path.split(row[new])
os.path.exists(d) or os.makedirs(d) # , exist_ok=True)
#os.rename(row[orig], row[new]) # If you just want to move, not copy.
shutil.copy(row[orig], row[new])
def get_demo_data():
"""Create train and validation DataFrames for blog post demo.
Create something like this:
dftrain.sample(5)
imgpath target orig label
object_id
1797 /tmp/path/to/imgs/797/01/cat_1797.jpg 0 train/cats/cat.1797.jpg cat
1678 /tmp/path/to/imgs/678/01/cat_1678.jpg 0 train/cats/cat.1678.jpg cat
1348 /tmp/path/to/imgs/348/01/dog_1348.jpg 1 train/dogs/dog.1348.jpg dog
1430 /tmp/path/to/imgs/430/01/cat_1430.jpg 0 train/cats/cat.1430.jpg cat
1664 /tmp/path/to/imgs/664/01/cat_1664.jpg 0 train/cats/cat.1664.jpg cat
dfvalid.sample(5)
imgpath target orig label
object_id
7625 /tmp/path/to/imgs/625/07/cat_7625.jpg 0 validation/cats/cat.7625.jpg cat
7729 /tmp/path/to/imgs/729/07/cat_7729.jpg 0 validation/cats/cat.7729.jpg cat
760 /tmp/path/to/imgs/760/00/dog_760.jpg 1 validation/dogs/dog.760.jpg dog
7724 /tmp/path/to/imgs/724/07/dog_7724.jpg 1 validation/dogs/dog.7724.jpg dog
7685 /tmp/path/to/imgs/685/07/dog_7685.jpg 1 validation/dogs/dog.7685.jpg dog
"""
df_train = pd.concat([new_tricks_from_old_dogs('train', 'dog'),
new_tricks_from_old_dogs('train', 'cat')])
df_valid = pd.concat([new_tricks_from_old_dogs('validation', 'dog'),
new_tricks_from_old_dogs('validation', 'cat')])
# The only time we'll copy image files, just for directory hierarchy demo.
res = df_train.apply(mv_to_new_hierarchy, axis=1)
res = df_valid.apply(mv_to_new_hierarchy, axis=1)
# Belt and suspenders for demo purposes.
assert all([df['new'].apply(lambda n: os.path.exists(n)).all()
for df in (df_train, df_valid)])
# dog will be target 1, cat 0.
df_train['target'] = (df_train['label'] == 'dog').astype(int)
df_valid['target'] = (df_valid['label'] == 'dog').astype(int)
df_train.index.name = 'object_id'
df_valid.index.name = 'object_id'
cols = ['imgpath', 'target', 'orig', 'label'] # For ordering.
df_train = df_train.rename(columns={'new': 'imgpath'})[cols].reset_index()
df_valid = df_valid.rename(columns={'new': 'imgpath'})[cols].reset_index()
pd.options.display.width = 200
print("Some samples:", "", "df_train:", df_train.sample(5), sep='\n')
print("df_valid:", "", df_valid.sample(5), sep='\n')
return df_train, df_valid
def test_generator():
"""Simple function to test return behavior of generator code above.
This runs with and without merged model version.
df_train:
object_id imgpath target orig label
7 1518 /tmp/path/to/imgs/518/01/dog_1518.jpg 1 data/train/dogs/dog.1518.jpg dog
1113 1662 /tmp/path/to/imgs/662/01/cat_1662.jpg 0 data/train/cats/cat.1662.jpg cat
980 1409 /tmp/path/to/imgs/409/01/dog_1409.jpg 1 data/train/dogs/dog.1409.jpg dog
1615 1813 /tmp/path/to/imgs/813/01/cat_1813.jpg 0 data/train/cats/cat.1813.jpg cat
1029 1760 /tmp/path/to/imgs/760/01/cat_1760.jpg 0 data/train/cats/cat.1760.jpg cat
df_valid:
object_id imgpath target orig label
787 7747 /tmp/path/to/imgs/747/07/cat_7747.jpg 0 data/validation/cats/cat.7747.jpg cat
165 7563 /tmp/path/to/imgs/563/07/dog_7563.jpg 1 data/validation/dogs/dog.7563.jpg dog
749 7517 /tmp/path/to/imgs/517/07/cat_7517.jpg 0 data/validation/cats/cat.7517.jpg cat
458 7742 /tmp/path/to/imgs/742/07/cat_7742.jpg 0 data/validation/cats/cat.7742.jpg cat
225 7479 /tmp/path/to/imgs/479/07/dog_7479.jpg 1 data/validation/dogs/dog.7479.jpg dog
"""
pd.np.set_printoptions(linewidth=150)
df_train, df_valid = get_demo_data()
img_width, img_height = 150, 150
batch_size = 64
target_size = (img_width, img_height)
print("\nTest basic generator.\n")
for df in (df_train, df_valid):
i = 0
for X, Y in generator_from_df(df, batch_size, target_size, features=None):
print(X[:3, :3, 0])
print(Y[:3])
i += 1
if i > 1:
break
# Create random array for bcolz test.
#
# In the end, this test does not use bcolz.
# But, if it did, here are some hints to get you there.
print("\nTest merged generator.\n")
nfeatures = 74
# features_train = pd.np.random.randn(df_train.shape[0], nfeatures)
# features_valid = pd.np.random.randn(df_valid.shape[0], nfeatures)
# Make a 2D array, where each row is filled with the values of its
# index, which will be very convenient for testing the merged
# model generator.
# [[0, 0, 0, ...],
# [1, 1, 1, ...],
# [2, 2, 2, ...],
# ...
# ]
features_train = np.repeat(np.arange(df_train.shape[0], dtype=float)
.reshape((-1, 1)),
nfeatures, axis=1)
features_valid = np.repeat(np.arange(df_valid.shape[0], dtype=float)
.reshape((-1, 1)),
nfeatures, axis=1)
# Add a litle noise in [0, 1] just to pretend we have "real" data.
features_train += np.random.rand(*features_train.shape)
features_valid += np.random.rand(*features_valid.shape)
fname_train = "mm_features_train_bc"
if not os.path.exists(fname_train):
c = bcolz.carray(features_train, rootdir=fname_train, mode='w')
c.flush()
fname_valid = "mm_features_valid_bc"
if not os.path.exists(fname_valid):
c = bcolz.carray(features_valid, rootdir=fname_valid, mode='w')
c.flush()
# Big assumption here: each row of a features matrix corresponds
# exactly with the image represented by the row of the associated
# train or valid df. *YOU* will have to ensure this in your own
# code. This is only demo code!
for df, fname in ((df_train, fname_train),
(df_valid, fname_valid)):
nbatches = df.shape[0] / float(batch_size)
for i, ((X, features), Y) in enumerate(
generator_from_df(df, batch_size, target_size,
features=fname, debug_merged=True)):
if i == 0:
print(X[:3, :3, 0])
print(features[:3, :5])
print(Y[:3])
else:
if (i + 1) % 20 == 0:
print("%d / %d" % (i + i, nbatches), end=', ')
sys.stdout.flush()
# Keras automatically breaks out of the infinite "while 1"
# loop in the generator_from_df(). For this test, we need
# to break manually.
if i >= nbatches:
break
print("\nSuccessful (I think...) test of multithreaded read of bcolz!")
print("Note that for this test, all of the above X2 rows should"\
"have the same int() values within a row.")
if __name__ == '__main__':
test_generator()
Raw
classifier_from_little_data_script_dfgen.py
'''Script to go with AppNexus blog post.
Taken from and altered from:
classifier_from_little_data_script_1.py
https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d
which appears at
https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html
by Francois Chollet
It uses data that can be downloaded at:
https://www.kaggle.com/c/dogs-vs-cats/data
The rest of this file was written by
Ryan Woodard | AppNexus | Data Science | 2017
'''
#
# Original code from Francois Chollet, Keras
#
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import backend as K
# dimensions of our images.
img_width, img_height = 150, 150
train_data_dir = 'data/train'
validation_data_dir = 'data/validation'
nb_train_samples = 2000
nb_validation_samples = 800
epochs = 5 #50
batch_size = 16
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
#
# New code with generator using file path list in DataFrame.
#
from akmtdfgen import get_demo_data
from akmtdfgen import generator_from_df
df_train, df_valid = get_demo_data()
assert {2} == set([df_train.target.nunique(),
df_train.label.nunique(),
df_valid.target.nunique(),
df_valid.label.nunique()])
ntrain, nvalid = df_train.shape[0], df_valid.shape[0]
print("""
Training set: %d images, 2 classes.
Validation set: %d images, 2 classes.
""" % (ntrain, nvalid))
target_size = (img_width, img_height)
train_generator = generator_from_df(df_train, batch_size, target_size)
validation_generator = generator_from_df(df_valid, batch_size, target_size)
nbatches_train, mod = divmod(ntrain, batch_size)
nbatches_valid, mod = divmod(nvalid, batch_size)
nworkers = 10
# Latest Keras 2.0 API:
# fit_generator(self, generator, steps_per_epoch, epochs=1, verbose=1,
# callbacks=None, validation_data=None, validation_steps=None,
# class_weight=None, max_queue_size=10, workers=1,
# use_multiprocessing=False, initial_epoch=0)
model.fit_generator(
train_generator,
steps_per_epoch=nbatches_train,
epochs=epochs,
verbose=2,
validation_data=validation_generator,
validation_steps=nbatches_valid,
workers=nworkers)
model.save_weights('mt_df_gen.h5')
Raw
classifier_from_little_data_script_dfgen_merged.py
'''Script to go with AppNexus blog post.
Taken from and altered from:
classifier_from_little_data_script_1.py
https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d
which appears at
https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html
by Francois Chollet
It uses data that can be downloaded at:
https://www.kaggle.com/c/dogs-vs-cats/data
The rest of this file was written by
Ryan Woodard | AppNexus | Data Science | 2017
'''
#
# Original code from Francois Chollet, Keras
#
import keras
from keras import backend as K
from keras.layers import Input, Dense, Conv2D, MaxPooling2D, Flatten, Dropout
from keras.models import Model
# dimensions of our images.
img_width, img_height = 150, 150
train_data_dir = 'data/train'
validation_data_dir = 'data/validation'
nb_train_samples = 2000
nb_validation_samples = 800
epochs = 5 #50
batch_size = 16
#
# left branch of model (convnet)
#
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
# This returns a tensor
linput = Input(shape=input_shape)
x = Conv2D(32, (3, 3), padding='same', activation='relu')(linput)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(32, (3, 3), padding='same', activation='relu')(x)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(64, (3, 3), padding='same', activation='relu')(x)
x = MaxPooling2D((2, 2))(x)
loutput = Flatten()(x)
#
# right branch of model (simple feature data, design matrix)
#
nfeatures = 74 # From akmtdfgen.py test_generator()
rinput = Input(shape=(nfeatures,), name='rinput')
#
# Make the merged model.
#
x = keras.layers.concatenate([loutput, rinput])
x = Dense(64, activation='relu')(x)
x = Dense(64, activation='relu')(x)
x = Dropout(0.5)(x)
# And finally we add the main logistic regression layer
main_output = Dense(1, activation='sigmoid', name='main_output')(x)
model = Model(inputs=[linput, rinput], outputs=main_output)
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
from akmtdfgen import get_demo_data
from akmtdfgen import generator_from_df
df_train, df_valid = get_demo_data()
assert {2} == set([df_train.target.nunique(),
df_train.label.nunique(),
df_valid.target.nunique(),
df_valid.label.nunique()])
ntrain, nvalid = df_train.shape[0], df_valid.shape[0]
print("""
Training set: %d images, 2 classes.
Validation set: %d images, 2 classes.
""" % (ntrain, nvalid))
# lmodel.fit(data, labels) # starts training
target_size = (img_width, img_height)
train_generator = generator_from_df(df_train, batch_size, target_size, features="mm_features_train_bc")
validation_generator = generator_from_df(df_valid, batch_size, target_size, features="mm_features_valid_bc")
nbatches_train, mod = divmod(ntrain, batch_size)
nbatches_valid, mod = divmod(nvalid, batch_size)
nworkers = 10
# Latest Keras 2.0 API:
# fit_generator(self, generator, steps_per_epoch, epochs=1, verbose=1,
# callbacks=None, validation_data=None, validation_steps=None,
# class_weight=None, max_queue_size=10, workers=1,
# use_multiprocessing=False, initial_epoch=0)
model.fit_generator(
train_generator,
steps_per_epoch=nbatches_train,
epochs=epochs,
verbose=2,
validation_data=validation_generator,
validation_steps=nbatches_valid,
workers=nworkers)
#lmodel.save_weights('mm_mt_df_gen.h5')
Raw
make_fake_dirs.py
# Make fake dir structure for lots o' images.
import os
import numpy as np
from numpy.random import choice
from numpy.random import shuffle
import pandas as pd
def util1():
onek = np.arange(1000)
onec = np.arange(100)
nchildren_choices = np.arange(1, 5)
files = []
os.chdir('/tmp')
top = 'imgs'
if os.path.exists(top):
os.system('rm -fr %s' % top)
os.mkdir(top)
os.chdir(top)
shuffle(onek)
nchildren1 = 3
children1 = ['%03d' % c for c in onek[:nchildren1]]
for child1 in children1:
os.mkdir(child1)
os.chdir(child1)
shuffle(nchildren_choices)
shuffle(onec)
nchildren2 = nchildren_choices[0]
children2 = ['%02d' % c for c in onec[:nchildren2]]
for child2 in children2:
os.mkdir(child2)
os.chdir(child2)
shuffle(nchildren_choices)
shuffle(onek)
# No leading 0s for files (so '%d', not '%03d').
nfiles = nchildren_choices[0]
file_prefixes = ['%d' % c for c in onek[:nfiles]]
new_files = ['%s%s%s.png' % (f, child2, child1)
for f in file_prefixes]
res = [os.system('touch %s' % f) for f in new_files]
files += new_files
os.chdir('..')
os.chdir('..')
os.chdir('..')
print("Created %d fake files." % len(files))
binomial_choices = ['cat', 'dog']
multinomial_choices = ['black', 'grey', 'white']
object_ids = sorted([int(f.split('.')[0]) for f in files])
nobjects = len(files)
df = (pd.DataFrame({'object_id': object_ids,
'bi': choice(binomial_choices, nobjects),
'multi': choice(multinomial_choices, nobjects)})
[['object_id', 'bi', 'multi']])
print(df)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment