Merge two sequential models on Keras for hybrid model-CodePudding

I want to combine two sequential models for a hybrid model (with Keras 2.6.0). The first model is a succession of dense layer of a set of 4 parameters, and the second is a succession of 2D convolution of an image ((32,32)). The goal is to predict a curve of 128 points.

Here is an illustration of my model hybrid idea.

My actual model:

def get_model_v2(params_shape, img_shape):
    params_model = models.Sequential()
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
    params_model.add(layers.Dense(256, name='Output'))
    
    img_model = models.Sequential()
    img_model.add(layers.Input(img_shape, name='InputLayer2'))
    img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Flatten())
    
    concat = tf.keras.layers.concatenate([params_model, img_model])

    model = models.Sequential()
    model.add(layers.Input(concat, name='InputLayer3'))
    model.add(layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
    model.add(layers.Dense(128, name='Output'))
    
    model.compile(optimizer = 'adam', 
                  loss      = 'mse', 
                  metrics   = ['mae', 'mse'])
    return model


model = get_model_v2 ( (4,),  (32, 32, 1) )

My problem is when I have to combine the two models, I don't know what to use, with this "concatenate" example I have an error like: TypeError: 'NoneType' object is not subscriptable. I understand the problem, but I can't find an other solution...

CodePudding user response：

Few issues here,

You are not using params_shape for your params_model (which comes out with an undefined shape).
As you understood, you can't concatenate models with a concatenation layer
The final model needs to through the Functional API
You got a bunch of layers with same name - you cannot have the same name for two layers in the same model

import tensorflow.keras.layers as layers
import tensorflow.keras.models as models
import tensorflow.keras.regularizers as regularizers
import tensorflow as tf

def get_model_v2(params_shape, img_shape):
    params_model = models.Sequential()
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1', input_shape=params_shape))
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
    params_model.add(layers.Dense(256, name='Output'))
    
    img_model = models.Sequential()
    img_model.add(layers.Input(img_shape, name='InputLayer2'))
    img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Flatten())
    
    param_out = params_model.outputs[0]
    img_out = img_model.outputs[0]

    concat_out = tf.keras.layers.concatenate([param_out, img_out])

    full_dense_out = layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n3')(concat_out)
    final_out = layers.Dense(128, name='Output_final')(full_dense_out)
    
    model = models.Model(inputs=[params_model.inputs, img_model.inputs], outputs=final_out)

    model.summary()

    model.compile(optimizer = 'adam', 
                  loss      = 'mse', 
                  metrics   = ['mae', 'mse'])
    return model


model = get_model_v2 ( (4,),  (32, 32, 1) )