1

I am trying to fuse features of two image inputs of shape (299, 299, 3), (224, 224, 3) and am getting shape errors.

Here is my code


    from tensorFlow.keras.applications.inception_v3 import InceptionV3
    from tensorflow.keras.applications.vgg16 import VGG16
    import tensorflow as tf
    from tensorflow.keras import layers, Input
    
    inp_pre_trained_model = InceptionV3( include_top=False)
    inp_pre_trained_model.trainable=False
    inp_input=tf.keras.Input(shape=(299,299,3),name="input_layer_inception_V3")
    inp_x=inp_pre_trained_model (inp_input)
    inp_x=layers.GlobalAveragePooling2D(name="global_average_pooling_layer_inception_v3")(inp_x)
    vgg_pre_trained_model = VGG16( include_top=False)
    vgg_pre_trained_model.trainable=False
    vgg_input=tf.keras.Input(shape=(224,224,3),name="input_layer_VGG_16")
    
    vgg_x=vgg_pre_trained_model(vgg_input)
    vgg_x=layers.GlobalAveragePooling2D(name="global_average_pooling_layer_vgg_16")(vgg_x)
    x=tf.keras.layers.concatenate([inp_x,vgg_x],axis=-1)
    x = tf.keras.layers.Flatten()(x)
    outputs=tf.keras.layers.Dense(5,activation="softmax", name= "output_layer") (x)
    model=tf.keras.Model(inputs=[inp_input,vgg_input],outputs=outputs)

    
    model.summary()

My model summary


    Model: "model_9"
    __________________________________________________________________________________________________
     Layer (type)                   Output Shape         Param #     Connected to                     
    ==================================================================================================
     input_layer_inception_V3 (Inpu  [(None, 224, 224, 3  0          []                               
     tLayer)                        )]                                                                
                                                                                                      
     input_layer_VGG_16 (InputLayer  [(None, 299, 299, 3  0          []                               
     )                              )]                                                                
                                                                                                      
     inception_v3 (Functional)      (None, None, None,   21802784    ['input_layer_inception_V3[0][0]'
                                    2048)                            ]                                
                                                                                                      
     vgg16 (Functional)             (None, None, None,   14714688    ['input_layer_VGG_16[0][0]']     
                                    512)                                                              
                                                                                                      
     global_average_pooling_incepti  (None, 2048)        0           ['inception_v3[0][0]']           
     on (GlobalAveragePooling2D)                                                                      
                                                                                                      
     global_average_pooling_vgg (Gl  (None, 512)         0           ['vgg16[0][0]']                  
     obalAveragePooling2D)                                                                            
                                                                                                      
     concatenate_71 (Concatenate)   (None, 2560)         0           ['global_average_pooling_inceptio
                                                                     n[0][0]',                        
                                                                      'global_average_pooling_vgg[0][0
                                                                     ]']                              
                                                                                                      
     output_layer (Dense)           (None, 5)            12805       ['concatenate_71[0][0]']         
                                                                                                      
    ==================================================================================================
    Total params: 36,530,277
    Trainable params: 12,805
    Non-trainable params: 36,517,472

compiler

model.compile(loss="sparse_categorical_crossentropy",optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),metrics=["accuracy"])

train = tf.data.Dataset.zip((cache_train_data, ceced_train_data))  
test = tf.data.Dataset.zip((cache_test_data, ceced_test_data))  
train_dataset = train.prefetch(tf.data.AUTOTUNE)  
test_dataset = test.prefetch(tf.data.AUTOTUNE)

train_dataset, test_dataset

--->(<PrefetchDataset element_spec=((TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name=None), TensorSpec(shape=(None, 5), dtype=tf.float32, name=None)), (TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name=None), TensorSpec(shape=(None, 5), dtype=tf.float32, name=None)))>,
<PrefetchDataset element_spec=((TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name=None), TensorSpec(shape=(None, 5), dtype=tf.float32, name=None)), (TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name=None), TensorSpec(shape=(None, 5), dtype=tf.float32, name=None)))>)

fit the model

model_history = model.fit(train_dataset, 
                              steps_per_epoch=len(train_dataset),
                              epochs=3,
                          validation_data=test_dataset,
                           validation_steps=len(test_dataset))

error ValueError: in user code:

File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1051, in train_function  *
    return step_function(self, iterator)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1040, in step_function  **
    outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1030, in run_step  **
    outputs = model.train_step(data)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 889, in train_step
    y_pred = self(x, training=True)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 67, in error_handler
    raise e.with_traceback(filtered_tb) from None
File "/usr/local/lib/python3.7/dist-packages/keras/engine/input_spec.py", line 264, in assert_input_compatibility
    raise ValueError(f'Input {input_index} of layer "{layer_name}" is '

ValueError: Input 1 of layer "model_9" is incompatible with the layer: expected shape=(None, 299, 299, 3), found shape=(None, 5)
Tamilselvi S
  • 11
  • 1
  • 1
  • 3
  • Also add the relevant code related to the train part, the code that is generating the error – ClaudiaR Aug 17 '22 at 08:58
  • It's clear from what you have posted that your model expects shape (None, 224, 224, 3) for its second input. The error indicates that the input data you are feeding into this input (in a .fit() call probably?) has shape (None, 5), but there's no indication of where the data is coming from or how it got to be that shape. So I doubt anyone will be able to help without further info on that. – David Harris Aug 17 '22 at 21:49
  • @DavidHarris I have included more information. – Tamilselvi S Aug 18 '22 at 04:04

1 Answers1

0

You have a problem in your fit function, related to the train data. See from the Keras documentation, fit can take the following arguments:

Model.fit(
    x=None,
    y=None,
    batch_size=None,
    epochs=1,
    verbose="auto",
    callbacks=None,
    validation_split=0.0,
    validation_data=None,
    shuffle=True,
    class_weight=None,
    sample_weight=None,
    initial_epoch=0,
    steps_per_epoch=None,
    validation_steps=None,
    validation_batch_size=None,
    validation_freq=1,
    max_queue_size=10,
    workers=1,
    use_multiprocessing=False,
)

But you are passing a train_dataset instead, that I think it’s a tf.data.Dataset that holds x_train and y_train together.

In order to fix the error you should separate x from y and pass those as arguments instead.

I think that something like this should do:

for images, labels in train_dataset.take(-1):
    X_train = images.numpy()
    y_train = labels.numpy()

# doing the same for validation
for images, labels in test_dataset.take(-1):
    X_test = images.numpy()
    y_test = labels.numpy()

You want to have something like this:

model_history = model.fit(x=X_train, y=y_train, steps_per_epoch=len(train_dataset), epochs=3, validation_data=(X_test, y_test), validation_steps=len(test_dataset))
ClaudiaR
  • 3,108
  • 2
  • 13
  • 27