0

I have an image classification problem for which I'd like to combine different models I've trained.

for example I have two models:

mobilenet_v2_100_96    = tf.keras.models.load_model("saved_model_mobilenet_v2_100_96")
mobilenet_v2_100_224    = tf.keras.models.load_model("saved_model_mobilenet_v2_100_224")

After which I lock the layers of the models and combine them into an array called "models" and create a new model to combine these like so:

ensemble_visible = [model.input for model in models]
ensemble_outputs = [model.output for model in models]

merge = tf.keras.layers.concatenate(ensemble_outputs)
merge = tf.keras.layers.Dense(200, activation='relu')(merge)
output = tf.keras.layers.Dense(200, activation='sigmoid')(merge)
model = tf.keras.models.Model(inputs=ensemble_visible, outputs=ensemble_outputs)

model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=0.005, momentum=0.9),
                loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1),
                metrics=['accuracy'])

As you can see I need two differently sized inputs in order to make it work.

I obtain my two datasets through the tf.keras.preprocessing.image_dataset_from_directory function.

Now I need a way to combine these two datasets. I made sure to set the seed value of both calls the same, so the images should be in the same order.

I've tried splitting the datasets via this tf_unzip (https://stackoverflow.com/a/68661507/20617581) function. And combining them like this:

model_inputs = tf.data.Dataset.zip(({"inputmobilenet_v2_100_96":train_x_96, "inputmobilenet_v2_100_224":train_x_224}, train_y_96))

When I run the model.fit function however the model is not able to learn above 1% which is way less than the original models. If I run the same model using only one input everything works as expected.

My complete code is following:

import tensorflow as tf
from utils.model import Model
from tqdm import tqdm
import numpy as np
import pandas as pd
import tensorflow_datasets as tfds

def tfdata_unzip(
    tfdata: tf.data.Dataset,
    *,
    recursive: bool=False,
    eager_numpy: bool=False,
    num_parallel_calls: int=tf.data.AUTOTUNE,
):
    """
    Unzip a zipped tf.data pipeline.

    Args:
        tfdata: the :py:class:`tf.data.Dataset`
            to unzip.

        recursive: Set to ``True`` to recursively unzip
            multiple layers of zipped pipelines.
            Defaults to ``False``.

        eager_numpy: Set this to ``True`` to return
            Python lists of primitive types or
            :py:class:`numpy.array` objects. Defaults
            to ``False``.

        num_parallel_calls: The level of parallelism to
            each time we ``map()`` over a
            :py:class:`tf.data.Dataset`.

    Returns:
        Returns a Python list of either
             :py:class:`tf.data.Dataset` or NumPy
             arrays.
    """
    if isinstance(tfdata.element_spec, tf.TensorSpec):
        if eager_numpy:
            return list(tfdata.as_numpy_iterator())
        return tfdata
        
    
    def tfdata_map(i: int) -> list:
        return tfdata.map(
            lambda *cols: cols[i],
            deterministic=True,
            num_parallel_calls=num_parallel_calls,
        )

    if isinstance(tfdata.element_spec, tuple):
        num_columns = len(tfdata.element_spec)
        if recursive:
            return [
                tfdata_unzip(
                    tfdata_map(i),
                    recursive=recursive,
                    eager_numpy=eager_numpy,
                    num_parallel_calls=num_parallel_calls,
                )
                for i in range(num_columns)
            ]
        else:
            return [
                tfdata_map(i)
                for i in range(num_columns)
            ]

    raise ValueError(
        "Unknown tf.data.Dataset element_spec: " +
        str(tfdata.element_spec)
    )

print("GPU is", "available" if tf.config.list_physical_devices('GPU') else "NOT AVAILABLE")

models = []

# mobilenet_v2_050_160    = tf.keras.models.load_model("saved_model_mobilenet_v2_050_160")
mobilenet_v2_100_96    = tf.keras.models.load_model("saved_model_mobilenet_v2_100_96")
mobilenet_v2_100_224    = tf.keras.models.load_model("saved_model_mobilenet_v2_100_224")

# models.append(mobilenet_v2_050_160)
models.append(mobilenet_v2_100_96)
models.append(mobilenet_v2_100_224)

for i, model in enumerate(models):
    for layer in model.layers:
        layer.trainable = False
        layer._name = layer.name + str(i)

    model.input._name = "input_" + str(i)
    model.input.type_spec._name = "input_" + str(i)
    # model.summary()


ensemble_visible = [model.input for model in models]
ensemble_outputs = [model.output for model in models]



merge = tf.keras.layers.concatenate(ensemble_outputs)
merge = tf.keras.layers.Dense(200, activation='relu')(merge)
output = tf.keras.layers.Dense(200, activation='sigmoid')(merge)
model = tf.keras.models.Model(inputs=ensemble_visible, outputs=output)

model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=0.005, momentum=0.9),
                loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1),
                metrics=['accuracy'])


def build_dataset(image_size):
    return tf.keras.preprocessing.image_dataset_from_directory(
        "img",
        validation_split=.20,
        subset="both",
        label_mode="categorical",
        # Seed needs to provided when using validation_split and shuffle = True.
        # A fixed seed is used so that the validation set is stable across runs.
        seed=123,
        image_size=image_size,
        batch_size=16
    )

batch_size = 16

def gen_datasets(image_size):
    train_ds, val_ds = build_dataset(image_size)
    class_names = tuple(train_ds.class_names)
    train_size = train_ds.cardinality().numpy()
    train_ds = train_ds.unbatch().batch(batch_size)
    train_ds = train_ds.repeat()

    normalization_layer = tf.keras.layers.Rescaling(1. / 255)
    preprocessing_model = tf.keras.Sequential([normalization_layer])
    do_data_augmentation = False  # @param {type:"boolean"}
    if do_data_augmentation:
        preprocessing_model.add(tf.keras.layers.RandomRotation(40))
        preprocessing_model.add(tf.keras.layers.RandomTranslation(0, 0.2))
        preprocessing_model.add(tf.keras.layers.RandomTranslation(0.2, 0))
    preprocessing_model.add(tf.keras.layers.RandomZoom(0.2, 0.2))
    preprocessing_model.add(tf.keras.layers.RandomFlip(mode="horizontal"))
    train_ds = train_ds.map(lambda images, labels: (preprocessing_model(images), labels))

    valid_size = val_ds.cardinality().numpy()
    val_ds = val_ds.unbatch().batch(batch_size) #self.batch_size)
    val_ds = val_ds.map(lambda images, labels:
                        (normalization_layer(images), labels))

    return train_ds, val_ds, train_size, valid_size, class_names


train_ds_96, val_ds_96, train_size, valid_size, class_names = gen_datasets([96,96])
train_ds_224, val_ds_224, train_size, valid_size, class_names = gen_datasets([224,224])

print("aquired data")

train_x_96, train_y_96 = tfdata_unzip(train_ds_96)
train_x_224, train_y_224 = tfdata_unzip(train_ds_224)

val_x_96, val_y_96 = tfdata_unzip(val_ds_96)
val_x_224, val_y_224 = tfdata_unzip(val_ds_224)

model.summary()

model_inputs = tf.data.Dataset.zip(({"inputmobilenet_v2_100_96":train_x_96, "inputmobilenet_v2_100_224":train_x_224}, train_y_96))
model_vals = tf.data.Dataset.zip(({"inputmobilenet_v2_100_96":val_x_96, "inputmobilenet_v2_100_224":val_x_224}, val_y_96))

steps_per_epoch = train_size // batch_size
validation_steps = valid_size // batch_size
hist = model.fit(
    model_inputs,
    epochs=1,
    steps_per_epoch=steps_per_epoch,
    validation_data=model_vals,
    validation_steps=validation_steps).history

model.save("joined_model")

What i've tried so far

I've tried:

splitting the dataset with:

def fit_generator(dataset,len):
  df = tfds.as_numpy(dataset)
  X_ret = np.array([])
  Y_ret = np.array([])

  for a,b in tqdm(df, total=len):
    np.append(X_ret,a)
    np.append(Y_ret,b)

  return X_ret, Y_ret

which was to slow to handle the amount of data.

I've tried using a generator, but it was not accepted by the model.fit function like this:

def fit_generator(dataset,len):
  df = tfds.as_numpy(dataset)
  X_ret = np.array(e[0] for e in df)
  Y_ret = np.array(e[1] for e in df)

  return X_ret, Y_ret

Lastly i've tried the answer provided in the problem description tf_unzip (https://stackoverflow.com/a/68661507/20617581)

But the model does not learn in any significant way.

0x7477
  • 3
  • 4
  • Are we sure using merge = ...followed by merge = ...not causing an issue? And also for ensemble...shouldnt your output be output and not ensemble_outputs? I.e. you might have a typo in your model creation. ALso it should be output and not mobilenet_v2_100_224.output... See second code window. – Jason Chia Feb 07 '23 at 15:29
  • If I train with only one input the network works well, therefore I think that the model creatin is not the problem. Thank you for pointing out the error in the outputs, I fixed it. The second code I posted was already a debbuging iteration – 0x7477 Feb 07 '23 at 15:56
  • If you create your output as per the debug...theres is no training being done on your model......because whatever you do to your 'model' has no effect on its output ensemble_outputs....because (a) everything in the loaded models is set to false...the only trainable parts are stuff in merge etc. So if you train...merge doesnt do anything since your evaluating your model on whatever was before.... – Jason Chia Feb 07 '23 at 16:00
  • The loaded model is already trained and classifies images into 200 classes with ~80% accuracy. Now I want to put several of these together and train only some small parameters for weighting purposes like which model classifies which object best etc – 0x7477 Feb 07 '23 at 16:08
  • Maybe we are misunderstanding each other. The typo i pointed out will mean that your model does not train. I.e even the small parameter layers as the 'fitness function' is wrong. Anyhow...did you verify that your inputs are as expected? See: https://www.tensorflow.org/api_docs/python/tf/random/set_seed – Jason Chia Feb 07 '23 at 16:44

1 Answers1

0

I used the ImageDataGenerator.flow_from_directory() and used a custom Generator to supply the model.fit function:

class JoinedGen(tf.keras.utils.Sequence):
    def __init__(self, input_gens):
        self.gens = input_gens

    def __len__(self):
        return len(self.gens[0])

    def __getitem__(self, i):
        x = [gen[i][0] for gen in self.gens]
        y = self.gens[0][i][1]

        return x, y

    def on_epoch_end(self):
        for gen in self.gens:
            gen.on_epoch_end()

def _build_generators(self):
        train_datagen = ImageDataGenerator(
            rescale=1 / 255.0,
            rotation_range=20,
            zoom_range=0.05,
            width_shift_range=0.05,
            height_shift_range=0.05,
            shear_range=0.05,
            horizontal_flip=True,
            fill_mode="nearest",
            validation_split=self.validation_split)

        sizes = [(96, 96), (224, 224), (299, 299), (224, 224), (160, 160)]
        train_gens = []
        val_gens = []

        for size in sizes:
            train_gens.append(
                train_datagen.flow_from_directory(
                    directory="img_enhanced",
                    target_size=size,
                    color_mode="rgb",
                    batch_size=self.batch_size,
                    class_mode="categorical",
                    subset='training',
                    shuffle=True,
                    seed=42
                )
            )
            val_gens.append(
                train_datagen.flow_from_directory(
                    directory="img_enhanced",
                    target_size=size,
                    color_mode="rgb",
                    batch_size=self.batch_size,
                    class_mode="categorical",
                    subset='validation',
                    shuffle=True,
                    seed=42
                )
            )
        
        return JoinedGen(train_gens), JoinedGen(val_gens)

        self.train_gens, self.val_gens = self._build_generators()


        model.fit(
            x=self.train_gens,
            epochs=1,
            steps_per_epoch=steps_per_epoch,
            validation_data=self.train_gens,
            validation_steps=validation_steps,
            batch_size=self.batch_size
        )
0x7477
  • 3
  • 4