I want to implement the algorithm to reconstruct Restricted Boltzmann Machines via simulation. The M is number of visible variables and N is number of hidden variables. In order to parallel run on GPU, I want to write it on tensorflow using python first.
In my main function RBMIC(), I need to run M independent logistics regression with L1 penalty and update my weight and bias matrices:(w and b) and then use them later to impute the value of hidden variables. So I write an independent for loop. I wonder does tensorflow could identify the independent for loops and run it efficiently (each core with one iteration) on GPU?
The code is also very slow especially for running logistics regression since it need to run epochs=1000 times to minimize the loss function. But I find it is very fast if I use sklearn.linear_model.LogisticRegression. Why is it a huge difference? But in order to use GPU, I still want to use tensorflow to write logistics regression. Could anyone give me some advice on how to write it more efficiently?
When I writing logistics regression function: LogisticsReg(), I need to get the weight and bias and I also need them to keep as tensorflow variables for my further calculation. But according to my function:LogisticsReg(), it returns non-tensor variable after sess.run(). So I transform them again into tensor-variables. Does this part be reasonable? Or is there any efficient way to keep it in tensor variable and then can be used to update the weight and bias matrices (w and b)? Thank you for your suggestions!
I am very new to tensorflow and python. Sorry for interruption and Thank you for your time!!
import numpy as np
import tensorflow as tf
n = 200
M = 4
N = 2
mu, sigma = 0, 0.1
beta = 0.001
lr=0.05
epochs = 1000
Maxepochs = 10
iteration = 100
visible = np.array([[1,0,1,0],[0,1,1,0],[1,0,0,1],[0,1,0,1]])
vis = np.tile(visible,50).reshape(n,M)
vis = tf.cast(vis, tf.float32)
err_hat = np.zeros([iteration])
def Bionimal(x):
sample = tf.where(tf.random_uniform(shape=x.shape) - x < 0,
tf.ones(shape=x.shape), tf.zeros(shape=x.shape))
return sample
def LogisticsReg(X, Y, wj, bj, beta, lr, epochs):
logitj = tf.add(tf.matmul(X, wj), bj)
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=Y, logits=logitj))
l1_regularizer = tf.reduce_sum(tf.abs(wj))
loss = tf.reduce_mean(loss + beta * l1_regularizer)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=lr).minimize(loss, var_list=[wj, bj])
with tf.Session() as sess:
tf.global_variables_initializer().run()
for k in range(epochs): # train the model n_epochs times
_, bf, wf = sess.run([optimizer, bj, wj])
# bf = tf.Variable(bf,name="bf")
# wf = tf.Variable(wf,name="wf")
return [bf, wf]
def UpdateC(wi, hi, c_opt, Maxepochs):
ph = tf.sigmoid(tf.add(tf.matmul(vis, tf.transpose(wi)), c_opt))
lik = tf.add(tf.multiply(hi, tf.log(ph)), tf.multiply((1. - hi), tf.log(1. - ph)))
loss2 = -tf.reduce_sum(lik)
optimizer = tf.contrib.opt.ScipyOptimizerInterface(loss2, var_to_bounds={c_opt: (-1,1)},options={'maxiter': Maxepochs}, var_list=[c_opt])
with tf.Session() as sess:
tf.global_variables_initializer().run()
optimizer.minimize(sess)
return sess.run(c_opt)
# initial
w = tf.Variable(tf.random_normal(shape=(N, M), stddev=0.1), name="weights")
c = tf.Variable(tf.random_normal(shape=(1, N), stddev=0.1), name="hbias")
b = tf.Variable(tf.random_normal(shape=(1, M), stddev=0.1), name="vbias")
def RBMIC(w,c,vis):
# calculate hidden variables
logits = tf.add(tf.matmul(vis, tf.transpose(w)), tf.tile(c, [n, 1]))
prob = tf.sigmoid(logits)
hids = Bionimal(prob)
# estimate bias, weight by logistics regression with l1 penalty and also bias c for visible variables.
bs = np.zeros([1, M])
ws = np.zeros([N, M])
X = hids
for j in range(M):
Y = tf.reshape(vis[:, j], [n, 1])
wj = tf.Variable(tf.reshape(w[:, j], [N, 1]), name="wj")
bj = tf.Variable(tf.random_normal(shape=[1, 1], stddev=0.1), name="bj")
bf, wf = LogisticsReg(X, Y, wj, bj, beta, lr, epochs)
bs[0, j] = bf
ws[:, [j]] = wf
b = tf.cast(tf.Variable(bs, name="vbias"), tf.float32)
w = tf.cast(tf.Variable(ws, name="weights"), tf.float32)
cs = np.zeros([1, N])
for i in range(N):
wi = tf.reshape(w[i, :], [1, M])
hi = tf.reshape(hids[:, i], [n, 1])
c_opt = tf.Variable(c[0, i], name="c_opt")
cs[0, i] = UpdateC(wi, hi, c_opt, Maxepochs)
c = tf.cast(tf.Variable(cs, name="hbias"), tf.float32)
# evaluate performance
vis_pred = tf.sigmoid(tf.add(tf.matmul(hids, w), tf.tile(b, [n, 1])))
err = tf.reduce_sum((vis_pred - vis) ** 2)
return err
for step in range(iteration): # train the model iteration times
err = RBMIC(w,c,vis)
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
print 'reconstruct at step %d = \n' % (step)
print sess.run(err)
