Tensorflow - calculate euclidean distance for RBF Network

Question

I have problem with implementation of RBF Network in Tensorflow. I need to calculate Euclidean Distance between x and centroids (from definition of RBF newtork). I wroted this code:

    x_data = tf.placeholder(shape=[None, 3], dtype=tf.float32)
    y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

    # Create variables for NN layers
    A1 = tf.Variable(tf.random_normal(shape=[3, first_layer_nodes]))  # input -> first layer nodes
    A2 = tf.Variable(tf.random_normal(shape=[first_layer_nodes, 1]))  # first_layer nodes -> sum node
    c = tf.Variable(tf.random_normal(shape=[first_layer_nodes]))  # centroids

    # Declare NN
    inputs_with_weights = tf.matmul(x_data, A1)
    print(inputs_with_weights)
    # euclid_dist = tf.sqrt(tf.reduce_sum(tf.pow(tf.subtract(inputs_with_weights, c), 2)))
    euclid_dist = tf.norm(inputs_with_weights - c, ord='euclidean')
    print(euclid_dist)
    first_output = tf_gaussian_function(euclid_dist)
    print(first_output)

final_output = tf.matmul(first_output, A2)

But i is problem like this:

E:\#PROJEKTY\#PROGRAMOWANIE\AI-Project>python Iris.py
2018-04-27 00:49:37.800684: I C:\tf_jenkins\workspace\rel-win\M\windows\PY\36\tensorflow\core\platform\cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
<tf.Variable 'Variable_2:0' shape=(1, 1) dtype=float32_ref>
Tensor("MatMul:0", shape=(?, 1), dtype=float32)
Tensor("norm/Squeeze:0", shape=(), dtype=float32)
Tensor("gaussian_function:0", dtype=float32)
Traceback (most recent call last):
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1361, in _do_call
    return fn(*args)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1340, in _run_fn
    target_list, status, run_metadata)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\errors_impl.py", line 516, in __exit__
    c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: In[0] is not a matrix
         [[Node: MatMul_1 = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](gaussian_function, Variable_1/read)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "Iris.py", line 144, in <module>
    sess.run(train_step, feed_dict={x_data: x_d, y_target: y_d})
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\client\session.py", line 905, in run
    run_metadata_ptr)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1137, in _run
    feed_dict_tensor, options, run_metadata)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1355, in _do_run
    options, run_metadata)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1374, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: In[0] is not a matrix
         [[Node: MatMul_1 = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](gaussian_function, Variable_1/read)]]

Caused by op 'MatMul_1', defined at:
  File "Iris.py", line 124, in <module>
    final_output = tf.matmul(first_output, A2)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\math_ops.py", line 2064, in matmul
    a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 2790, in _mat_mul
    name=name)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 3271, in create_op
    op_def=op_def)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 1650, in __init__
    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access

InvalidArgumentError (see above for traceback): In[0] is not a matrix
         [[Node: MatMul_1 = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](gaussian_function, Variable_1/read)]]

If i try put any value to axis it stuck at:

E:\#PROJEKTY\#PROGRAMOWANIE\AI-Project>python Iris.py
2018-04-27 00:53:15.388129: I C:\tf_jenkins\workspace\rel-win\M\windows\PY\36\tensorflow\core\platform\cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
<tf.Variable 'Variable_2:0' shape=(1, 1) dtype=float32_ref>
Tensor("MatMul:0", shape=(?, 1), dtype=float32)
Traceback (most recent call last):
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\common_shapes.py", line 686, in _call_cpp_shape_fn_impl
    input_tensors_as_shapes, status)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\errors_impl.py", line 516, in __exit__
    c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: Invalid reduction dimension 2 for input with 2 dimensions. for 'norm/Sum' (op: 'Sum') with input shapes: [?,1], [1] and with computed input tensors: input[1] = <2>.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "Iris.py", line 120, in <module>
    euclid_dist = tf.norm(inputs_with_weights - c, axis = 2, ord='euclidean')
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\util\deprecation.py", line 432, in new_func
    return func(*args, **kwargs)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\linalg_ops.py", line 552, in norm
    tensor * math_ops.conj(tensor), axis, keepdims=True))
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\util\deprecation.py", line 432, in new_func
    return func(*args, **kwargs)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\math_ops.py", line 1373, in reduce_sum
    name=name))
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 5436, in _sum
    name=name)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 3273, in create_op
    compute_device=compute_device)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 3313, in _create_op_helper
    set_shapes_for_outputs(op)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 2501, in set_shapes_for_outputs
    return _set_shapes_for_outputs(op)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 2474, in _set_shapes_for_outputs
    shapes = shape_func(op)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 2404, in call_with_requiring
    return call_cpp_shape_fn(op, require_shape_fn=True)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\common_shapes.py", line 627, in call_cpp_shape_fn
    require_shape_fn)
  File "C:\Users\Szatku\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\common_shapes.py", line 691, in _call_cpp_shape_fn_impl
    raise ValueError(err.message)
ValueError: Invalid reduction dimension 2 for input with 2 dimensions. for 'norm/Sum' (op: 'Sum') with input shapes: [?,1], [1] and with computed input tensors: input[1] = <2>.

I have no idea how to correct code. Can someone helps me?

EDIT: Gaussian Implementation:

def gaussian_function(input_layer):
    initial = math.exp(-SC*math.pow(input_layer, 2))
    return initial


np_gaussian_function = np.vectorize(gaussian_function)


def d_gaussian_function(input_layer):
    initial = -2*SC*input_layer * math.exp(-SC * math.pow(input_layer, 2))
    return initial


np_d_gaussian_function = np.vectorize(d_gaussian_function)


def np_d_gaussian_function_32(input_layer):
    return np_d_gaussian_function(input_layer).astype(np.float32)


def tf_d_gaussian_function(input_layer, name=None):
    with ops.name_scope(name, "d_gaussian_function", [input_layer]) as name:
        y = tf.py_func(np_d_gaussian_function_32, [input_layer],[tf.float32], name=name, stateful=False)
    return y[0]


def py_func(func, inp, Tout, stateful=True, name=None, grad=None):
    rnd_name = 'PyFunGrad' + str(np.random.randint(0, 1E+8))

    tf.RegisterGradient(rnd_name)(grad)
    g = tf.get_default_graph()
    with g.gradient_override_map({"PyFunc": rnd_name}):
        return tf.py_func(func, inp, Tout, stateful=stateful, name=name)


def gaussian_function_grad(op, grad):
    input_variable = op.inputs[0]
    n_gr = tf_d_gaussian_function(input_variable)
    return grad * n_gr


def np_gaussian_function_32(input_layer):
    return np_gaussian_function(input_layer).astype(np.float32)


def tf_gaussian_function(input_layer, name=None):
    with ops.name_scope(name, "gaussian_function", [input_layer]) as name:
        y = py_func(np_gaussian_function_32, [input_layer], [tf.float32], name=name, grad=gaussian_function_grad)
    return y[0]
# end of defining activation function

Answer 1

The error says that the first argument of

final_output = tf.matmul(first_output, A2)

is not a matrix. So first_output is not okay. It comes from the previous line:

first_output = tf_gaussian_function(euclid_dist)

So we'd need to review that function tf_gaussian_function() but that's not in the posted code. I've checked if it's a typo and it should be tf.gaussian_function() but there is no such a thing.

So please post the definition of the function tf_gaussian_function().

---

So now that we have that, it turns out that you're trying to map the function math.exp( -SC * math.pow( input_layer, 2 ) ) (generally called rho or ρ) onto euclid_dist; I take that would be your radial basis function. Side note: looks like you're trying to do some serious heavy lifting by implementing your own gradient, and everything, going through np.vectorize() and all that. I'd suggest you implement something much simpler, like this code (untested):

first_output = tf.exp( -SC * euclid_dist * euclid_dist )

(I use x * x instead of the equivalent tf.pow( x, 2 ) because the first is more efficient.) This should create its own gradient and take care of all you're trying to achieve. But anyway, you sound confident that your implementation works, which might very well be the case as I found here; you can also double check if it yields the same results as this one-liner.

In terms of what euclid_dist is, tf.norm( x, ord = "euclidean", axis = None ) is returning the 2^nd norm (or Euclidean norm, otherwise also known as the square root of the sum of squares), so that is definitely a scalar. (It does exactly what the commented out line above says: euclid_dist = tf.sqrt(tf.reduce_sum(tf.pow(tf.subtract(inputs_with_weights, c), 2))).) Not sure what you're trying to achieve there? If you just want the squares of distances, then you should do it like this:

euclid_dist = ( inputs_with_weights - c ) * ( inputs_with_weights - c )

But as far as I understand, in a radial basis function network first you apply the weights on the input, then comes the radial basis function, and only as a third step comes the Euclidean distance as the cost function, which might very well be a scalar.

If that line is your attempt to implement normalization for your RBF network, please note that the normalization is generally done a bit differently. So your implementation could look more similar to this, if I catch your drift right:

from __future__ import print_function
import tensorflow as tf
from tensorflow.python.framework import ops
import numpy as np

x_data = tf.placeholder(shape=[1, 3], dtype=tf.float32)
y_target = tf.placeholder(shape=[1, 1], dtype=tf.float32)

# Create variables for NN layers
first_layer_nodes = 5
A1 = tf.Variable(tf.random_normal(shape=[3, first_layer_nodes]))  # input -> first layer nodes
A2 = tf.Variable(tf.random_normal(shape=[first_layer_nodes, 1]))  # first_layer nodes -> sum node
c = tf.Variable(tf.random_normal(shape=[first_layer_nodes]))  # centroids

# Declare NN
SC = 1
def rho( distances ): return tf.exp( -SC * distances * distances )
def norm( x ): return x / tf.reduce_sum( x, axis = -1 )
inputs_with_weights = tf.matmul( x_data, A1 )
print(inputs_with_weights)
distances = inputs_with_weights - c
print( distances )
first_output = norm( rho( distances ) ) # tf_gaussian_function(distances) # 
print(first_output)
final_output = tf.matmul(first_output, A2)

with tf.Session() as sess:
    sess.run( tf.global_variables_initializer() )
    r = sess.run( [ first_output, final_output ], feed_dict = {
                  x_data : np.array( [ [ 1.0, 2, 3 ] ] ) } )
    for v in r:
        print( v )

Outputs:

Tensor("MatMul_15:0", shape=(1, 5), dtype=float32)
Tensor("sub_8:0", shape=(1, 5), dtype=float32)
Tensor("div_5:0", shape=(1, 5), dtype=float32)
[[4.4366708e-03 6.8647589e-04 5.9621310e-01 7.5066246e-06 3.9865622e-01]]
[[0.31285414]]

---

To respond to the comment, where you posted this code fragment:

exp_list = []
for i in range(first_layer_nodes):
    euclid_distance = tf.reduce_sum(tf.square(tf.subtract(x_data, c[i, :])), 1)
    exp_list.append(tf.exp(-SC * euclid_distance))
phi = tf.transpose(tf.stack(exp_list))

this could be vectorized, taking advantage of implicit broadcasting on tf.subtract() with advanced indexing like this (untested):

ed = tf.reduce_sum( tf.square( tf.subtract( x_data[ None, ... ], c ) ), 2 )
phi = tf.transpose( tf.exp( -SC * ed ) )