Managing experiments with one file
Presenting an experiment manager contained in one file for `tf.keras`
Let's say you want to test different hyperparameters on a given model:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import (
Conv2D, MaxPooling2D, Dropout, Flatten, Dense
)
def get_model(params):
model = Sequential()
model.add(Conv2D(params['conv_0'], kernel_size=params['kernel_0'],
activation='relu',
input_shape=params['input_shape']))
model.add(Conv2D(params['conv_1'], params['kernel_1'], activation='relu'))
model.add(MaxPooling2D(pool_size=params['pool_size']))
model.add(Dropout(params['dropout_0']))
model.add(Flatten())
model.add(Dense(params['dense'], activation='relu'))
model.add(Dropout(params['dropout_1']))
model.add(Dense(params['num_classes'], activation='softmax'))
return model
We then feed the get_model function with something like this:
'model': {
'input_shape': (28, 28, 1),
'conv_0': 32,
'conv_1': 64,
'kernel_0': (3,3),
'kernel_1': (3,3),
'pool_size': (2,2),
'dropout_0': 0.25,
'dense': 128,
'dropout_1': 0.5,
'num_classes': 10
}The experiment manager, is here to automatically keep a tidy model checkpoints, performance files for your hyperparameter search. It's doing so by maintaining a consistent folder hierarchy based on the hash of your hyperparameters.
Let's see a real example, by defining our hyperparameters. We choose here (but you can do what you want) to separate them into two sections:
-
trainingwhich contains the parameters such as the batch size, the selected optimizer, ... -
modelwhich contains the parameters that actually build your model
Note that you can create as many section as you want and use as much nested dictionary as necessary.
params = {
'debug': False,
'training': {
'batch_size': 128,
'epochs': 3
},
'model': {
'input_shape': (28, 28, 1),
'conv_0': 32,
'conv_1': 64,
'kernel_0': (3,3),
'kernel_1': (3,3),
'pool_size': (2,2),
'dropout_0': 0.25,
'dense': 128,
'dropout_1': 0.5,
'num_classes': 10
},
'comment': 'simple model from keras documentation',
'author': 'data-soup'
}
#collapse
!sudo apt-get install tree # usefull later to display the directories
from tensorflow.keras.datasets import mnist
from tensorflow.keras.utils import to_categorical
# the data, split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
# convert class vectors to binary class matrices
y_train = to_categorical(y_train, 10)
y_test = to_categorical(y_test, 10)
# input image dimensions
img_rows, img_cols = params['model']['input_shape'][:2]
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
#collapse
# Here lives the code for the experiement manager
!git clone https://github.com/maxpv/experiment_manager
We can now prepare the ExperimentManager for a first test run.
-
exp_base_diris the name of your experiment, can be the version ofget_modelor anything else -
monitored_param_keyswill manage your experiments based on those keys
from experiment_manager.experiment_manager import ExperimentManager
expm = ExperimentManager(exp_base_dir='experiments',
monitored_param_keys=['training', 'model'])
callbacks = expm.prepare(params)
model = get_model(params['model'])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(x_train[:5000], y_train[:5000],
**params['training'],
verbose=1,
callbacks = callbacks,
validation_data=(x_test, y_test))
The training above generated:
- a tree structure for each experiment under a specific identifier and the current date
- callbacks for tf.keras to ensure that training logs and model checkpoints are written in the same directory
!tree experiments
Now let's launch another run using the same parameters.
#collapse
callbacks = expm.prepare(params)
model = get_model(params['model'])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(x_train[:5000], y_train[:5000],
**params['training'],
verbose=1,
callbacks = callbacks,
validation_data=(x_test, y_test))
!tree experiments -d
Let's see what happens if we change the model parameters:
#collapse
params['model']['conv_1'] = 32
callbacks = expm.prepare(params)
model = get_model(params['model'])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(x_train[:5000], y_train[:5000],
**params['training'],
verbose=1,
callbacks = callbacks,
validation_data=(x_test, y_test))
!tree experiments -d