below is code, when delete regularizers running good, if add regularizers raise error,how to fix it

import pandas as pd
from tensorflow.keras import layers, Model,Input,Sequential
from tensorflow.keras.optimizers import Adam,RMSprop

def tower_tasks_model():
    input_layer_tst=Input(shape=(2,))
    #defined three towner network
    A_1_1 = layers.Dense(units=2, activation='relu', name='A_1_1', kernel_initializer='VarianceScaling',)(input_layer_tst)
    A_2_1 = layers.Dense(units=2, activation='relu', name='A_2_1', kernel_initializer='VarianceScaling',)(input_layer_tst)
    A_3_1 = layers.Dense(units=2, activation='relu', name='A_3_1',kernel_initializer='VarianceScaling',kernel_regularizer=tf.keras.regularizers.l2(1e-3), activity_regularizer=tf.keras.regularizers.l1(1e-3),)(input_layer_tst)

    A_1_1_concat = layers.Concatenate(name='A_1_1_concat')([A_1_1, input_layer_tst])
    A_2_1_concat = layers.Concatenate(name='A_2_1_concat')([A_2_1, input_layer_tst])
    A_3_1_concat = layers.Concatenate(name='A_3_1_concat')([A_3_1, input_layer_tst])

    A_1_result = layers.Dense(units=1, name='A_1', activation='sigmoid', kernel_initializer='VarianceScaling')(A_1_1_concat)
    A_2_result = layers.Dense(units=1, name='A_2', activation='sigmoid', kernel_initializer='VarianceScaling')(A_2_1_concat)
    A_3_result = layers.Dense(units=1, name='A_3', activation='sigmoid',kernel_initializer='VarianceScaling')(A_3_1_concat)

    model = Model(inputs=[input_layer_tst], outputs=[A_1_result, A_2_result, A_3_result],name='tower_result_mode')
    return model

class CustomMultiLossLayer(tf.keras.layers.Layer):
    def __init__(self, nb_outputs=3, **kwargs):
        self.nb_outputs = nb_outputs
        super(CustomMultiLossLayer, self).__init__(**kwargs)

    def focal_loss(self, y_true, y_pred, gamma, alpha):
        idx = tf.where(y_true >= 0)
        y_true = tf.gather_nd(y_true, idx)
        y_pred = tf.gather_nd(y_pred, idx)

        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))

        pt_1 = tf.keras.backend.clip(pt_1, 1e-3, .999)
        pt_0 = tf.keras.backend.clip(pt_0, 1e-3, .999)

        return -tf.keras.backend.sum(
            alpha * tf.keras.backend.pow(1. - pt_1, gamma) * tf.keras.backend.log(pt_1)) - tf.keras.backend.sum(
            (1 - alpha) * tf.keras.backend.pow(pt_0, gamma) * tf.keras.backend.log(1. - pt_0))

    def build(self, input_shape=None):
        self.log_vars = []
        for i in range(self.nb_outputs):
            self.log_vars  = [self.add_weight(name='log_var'   str(i), shape=(1,),initializer=tf.keras.initializers.Constant(1.), trainable=True)]
        super(CustomMultiLossLayer, self).build(input_shape)

    def multi_loss(self, ys_true, ys_pred):
        assert len(ys_true) == self.nb_outputs and len(ys_pred) == self.nb_outputs
        loss = 0
        for index, (y_true, y_pred, log_var) in enumerate(zip(ys_true, ys_pred, self.log_vars)):
            precision = tf.keras.backend.exp(-log_var)
            if index == 0:
                single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.30)
            elif index == 1:
                single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.25)
            else:
                single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.25)
            loss  = precision * single_task_loss   log_var
        return loss

    def call(self, inputs):
        ys_true = inputs[:self.nb_outputs]
        ys_pred = inputs[self.nb_outputs:]
        loss = self.multi_loss(ys_true, ys_pred)
        self.add_loss(loss, inputs=inputs)
        return tf.keras.backend.concatenate(inputs, -1)


def weigh_losses_mode(prediction_model):
    input_layer_tst=Input(shape=(2,))

    A_1_predit, A_2_predit, A_3_predit = prediction_model([input_layer_tst])
    A_1_true = Input(shape=(1,), name='A_1_true')
    A_2_true = Input(shape=(1,), name='A_2_true')
    A_3_true = Input(shape=(1,), name='A_3_true')
    out = CustomMultiLossLayer(nb_outputs=3, name='multi_loss_layer')(
        [A_1_true, A_2_true, A_3_true, A_1_predit, A_2_predit, A_3_predit])
    return Model([input_layer_tst, A_1_true, A_2_true, A_3_true], out)

tower_result_predict_model = tower_tasks_model()


train_model = weigh_losses_mode(tower_result_predict_model)
adam_optimizer = Adam(lr=0.0005)
train_model.compile(optimizer=adam_optimizer, loss=None)
train_model.summary()
a=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
b=[2,2,2,2,2,1,1,0,1,1,1,1,1,1,1,1,1]
c=[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0]
d=[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0]
e=[1,1,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0]

df=pd.DataFrame({'A':a,'B':b,'C':c,'D':d,'E':e})
hist = train_model.fit(x=[df[['A','B']],  df['C'], df['D'],df['E']],batch_size=10,epochs=10,verbose=2)

error is: 2021-10-20 16:24:42.901252: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) Epoch 1/10 Traceback (most recent call last): File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 3427, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "", line 1, in runfile('/Users/zhang_james/Documents/study/my_py_env/regular_t.py', wdir='/Users/zhang_james/Documents/study/my_py_env') File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_bundle/pydev_umd.py", line 197, in runfile pydev_imports.execfile(filename, global_vars, local_vars) # execute the script File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile exec(compile(contents "\n", file, 'exec'), glob, loc) File "/Users/zhang_james/Documents/study/my_py_env/regular_t.py", line 121, in hist = train_model.fit(x=[df[['A','B']], df['C'], df['D'],df['E']],batch_size=10,epochs=10,verbose=2) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py", line 1100, in fit tmp_logs = self.train_function(iterator) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 828, in call result = self._call(*args, **kwds) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 871, in _call self._initialize(args, kwds, add_initializers_to=initializers) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 725, in _initialize self._stateful_fn._get_concrete_function_internal_garbage_collected( # pylint: disable=protected-access File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py", line 2969, in _get_concrete_function_internal_garbage_collected graph_function, _ = self._maybe_define_function(args, kwargs) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py", line 3361, in _maybe_define_function graph_function = self._create_graph_function(args, kwargs) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py", line 3196, in _create_graph_function func_graph_module.func_graph_from_py_func( File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py", line 990, in func_graph_from_py_func func_outputs = python_func(*func_args, **func_kwargs) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 634, in wrapped_fn out = weak_wrapped_fn().wrapped(*args, **kwds) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py", line 977, in wrapper raise e.ag_error_metadata.to_exception(e) ValueError: in user code: /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:805 train_function * return step_function(self, iterator) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:795 step_function ** outputs = model.distribute_strategy.run(run_step, args=(data,)) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:1259 run return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:2730 call_for_each_replica return self._call_for_each_replica(fn, args, kwargs) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:3417 _call_for_each_replica return fn(*args, **kwargs) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:788 run_step ** outputs = model.train_step(data) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:755 train_step loss = self.compiled_loss( /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/compile_utils.py:229 call reg_loss = math_ops.add_n(regularization_losses) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py:201 wrapper return target(*args, **kwargs) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/ops/math_ops.py:3572 add_n return gen_math_ops.add_n(inputs, name=name) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/ops/gen_math_ops.py:418 add_n _, _, _op, _outputs = _op_def_library._apply_op_helper( /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/op_def_library.py:748 _apply_op_helper op = g._create_op_internal(op_type_name, inputs, dtypes=None, /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py:590 _create_op_internal return super(FuncGraph, self)._create_op_internal( # pylint: disable=protected-access /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:3528 _create_op_internal ret = Operation( /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:2015 init self._c_op = _create_c_op(self._graph, node_def, inputs, /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:1856 _create_c_op raise ValueError(str(e)) ValueError: Shapes must be equal rank, but are 0 and 1 From merging shape 1 with other shapes. for '{{node AddN}} = AddN[N=3, T=DT_FLOAT](model/tower_result_mode/A_3_1/ActivityRegularizer/truediv, A_3_1/kernel/Regularizer/mul, model/multi_loss_layer/add_5)' with input shapes: [], [], [1].

CodePudding user response：

This error usually occurs because the calculated loss is not a scalar, but an n-dimensional tensor. Just use tf.keras.backend.sum(*) or tf.keras.backend.mean(*) to reduce your loss to a scalar and then it should work with the regularizers:

def multi_loss(self, ys_true, ys_pred):
    assert len(ys_true) == self.nb_outputs and len(ys_pred) == self.nb_outputs
    loss = 0
    for index, (y_true, y_pred, log_var) in enumerate(zip(ys_true, ys_pred, self.log_vars)):
        precision = tf.keras.backend.exp(-log_var)
        if index == 0:
            single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.30)
        elif index == 1:
            single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.25)
        else:
            single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.25)
        loss  = precision * single_task_loss   log_var

    return tf.keras.backend.sum(loss)