I'm using ScipyOptimizerInterface for training a tensorflow model. (tensorflow 1.13.1)
During the process of training, if the loss value is below a threshold, I want the training process to stop and save the model right before the threshold was crossed.
Below is the script that I tried. The idea is to raise an exception to exit optimizer.minimize, then save the model using tf.train.Saver.
However, this does not work. As you can see by comparing the initial loss value and the loss value computed by the saved model. The two values are the same which indicates it is the initial random model that is saved, not the desired model.
From @Patol75's answer, I understand the best model is not saved because the updated tf.Variables dies when the training session is interpreted.
How can the desired model be saved?
import numpy as np
import tensorflow as tf
from tensorflow.contrib.opt import ScipyOptimizerInterface
class test(Exception):
pass
def construct_graph():
graph = tf.Graph()
with graph.as_default():
x = tf.placeholder('float', shape = [None, 1])
w = tf.get_variable('w_0', shape = [1, 1], initializer = tf.contrib.layers.xavier_initializer())
b = tf.get_variable('b_0', shape = [1], initializer = tf.contrib.layers.xavier_initializer())
y_out = tf.matmul(x, w) + b
y = tf.placeholder('float', shape = [None, 1])
loss = tf.reduce_mean(tf.square(y - y_out))
return graph, x, y, loss
# create example datasets
x_train = np.linspace(1, 6, 100) + 0.1 * np.random.random(100)
x_train = x_train.reshape(100, 1)
y_train = np.sin(x_train)
x_val = np.linspace(6, 11, 100)
x_val = x_val.reshape(100, 1)
y_val = np.sin(x_val)
tf.reset_default_graph()
graph, x, y, loss = construct_graph()
feeddict_train = {x: x_train, y: y_train}
feeddict_val = {x: x_val, y: y_val}
with graph.as_default():
def step_callbackfun(x):
global iteration
train_part, val_part = valfunc_train(x), valfunc_val(x)
print('%10.5f %10.5f' % (*train_part, *val_part))
iteration += 1
if iteration == 5:
raise test()
sess = tf.Session()
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
optimizer = ScipyOptimizerInterface(loss, method='l-BFGS-b')
iteration = 0
valfunc_train = optimizer._make_eval_func(tensors=loss, session=sess, feed_dict=feeddict_train, fetches=[])
valfunc_val = optimizer._make_eval_func(tensors=loss, session=sess, feed_dict=feeddict_val, fetches=[])
print('The initial loss is %f' % sess.run(loss, feeddict_train))
try:
optimizer.minimize(sess, feeddict_train, step_callback=step_callbackfun)
except test:
saver.save(sess, 'model/model.ckpt')
graph2, x2, y2, loss2 = construct_graph()
with tf.Session(graph=graph2) as sess2:
feeddict_two = {x2: x_train, y2: y_train}
sess2.run(tf.global_variables_initializer())
saver2 = tf.train.Saver()
saver2.restore(sess2, 'model/model.ckpt')
loss_val2 = sess2.run(loss2, feeddict_two)
print('Outside', loss_val2)
