Below is my code. When I delete the regularizers the code runs fine. If I add the regularizers, an error is raised as seen below.
import pandas as pd
from tensorflow.keras import layers, Model,Input,Sequential
from tensorflow.keras.optimizers import Adam,RMSprop
def tower_tasks_model():
input_layer_tst=Input(shape=(2,))
#defined three towner network
A_1_1 = layers.Dense(units=2, activation='relu', name='A_1_1', kernel_initializer='VarianceScaling',)(input_layer_tst)
A_2_1 = layers.Dense(units=2, activation='relu', name='A_2_1', kernel_initializer='VarianceScaling',)(input_layer_tst)
A_3_1 = layers.Dense(units=2, activation='relu', name='A_3_1',kernel_initializer='VarianceScaling',kernel_regularizer=tf.keras.regularizers.l2(1e-3), activity_regularizer=tf.keras.regularizers.l1(1e-3),)(input_layer_tst)
A_1_1_concat = layers.Concatenate(name='A_1_1_concat')([A_1_1, input_layer_tst])
A_2_1_concat = layers.Concatenate(name='A_2_1_concat')([A_2_1, input_layer_tst])
A_3_1_concat = layers.Concatenate(name='A_3_1_concat')([A_3_1, input_layer_tst])
A_1_result = layers.Dense(units=1, name='A_1', activation='sigmoid', kernel_initializer='VarianceScaling')(A_1_1_concat)
A_2_result = layers.Dense(units=1, name='A_2', activation='sigmoid', kernel_initializer='VarianceScaling')(A_2_1_concat)
A_3_result = layers.Dense(units=1, name='A_3', activation='sigmoid',kernel_initializer='VarianceScaling')(A_3_1_concat)
model = Model(inputs=[input_layer_tst], outputs=[A_1_result, A_2_result, A_3_result],name='tower_result_mode')
return model
class CustomMultiLossLayer(tf.keras.layers.Layer):
def __init__(self, nb_outputs=3, **kwargs):
self.nb_outputs = nb_outputs
super(CustomMultiLossLayer, self).__init__(**kwargs)
def focal_loss(self, y_true, y_pred, gamma, alpha):
idx = tf.where(y_true >= 0)
y_true = tf.gather_nd(y_true, idx)
y_pred = tf.gather_nd(y_pred, idx)
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
pt_1 = tf.keras.backend.clip(pt_1, 1e-3, .999)
pt_0 = tf.keras.backend.clip(pt_0, 1e-3, .999)
return -tf.keras.backend.sum(
alpha * tf.keras.backend.pow(1. - pt_1, gamma) * tf.keras.backend.log(pt_1)) - tf.keras.backend.sum(
(1 - alpha) * tf.keras.backend.pow(pt_0, gamma) * tf.keras.backend.log(1. - pt_0))
def build(self, input_shape=None):
self.log_vars = []
for i in range(self.nb_outputs):
self.log_vars = [self.add_weight(name='log_var' str(i), shape=(1,),initializer=tf.keras.initializers.Constant(1.), trainable=True)]
super(CustomMultiLossLayer, self).build(input_shape)
def multi_loss(self, ys_true, ys_pred):
assert len(ys_true) == self.nb_outputs and len(ys_pred) == self.nb_outputs
loss = 0
for index, (y_true, y_pred, log_var) in enumerate(zip(ys_true, ys_pred, self.log_vars)):
precision = tf.keras.backend.exp(-log_var)
if index == 0:
single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.30)
elif index == 1:
single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.25)
else:
single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.25)
loss = precision * single_task_loss log_var
return loss
def call(self, inputs):
ys_true = inputs[:self.nb_outputs]
ys_pred = inputs[self.nb_outputs:]
loss = self.multi_loss(ys_true, ys_pred)
self.add_loss(loss, inputs=inputs)
return tf.keras.backend.concatenate(inputs, -1)
def weigh_losses_mode(prediction_model):
input_layer_tst=Input(shape=(2,))
A_1_predit, A_2_predit, A_3_predit = prediction_model([input_layer_tst])
A_1_true = Input(shape=(1,), name='A_1_true')
A_2_true = Input(shape=(1,), name='A_2_true')
A_3_true = Input(shape=(1,), name='A_3_true')
out = CustomMultiLossLayer(nb_outputs=3, name='multi_loss_layer')(
[A_1_true, A_2_true, A_3_true, A_1_predit, A_2_predit, A_3_predit])
return Model([input_layer_tst, A_1_true, A_2_true, A_3_true], out)
tower_result_predict_model = tower_tasks_model()
train_model = weigh_losses_mode(tower_result_predict_model)
adam_optimizer = Adam(lr=0.0005)
train_model.compile(optimizer=adam_optimizer, loss=None)
train_model.summary()
a=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
b=[2,2,2,2,2,1,1,0,1,1,1,1,1,1,1,1,1]
c=[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0]
d=[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0]
e=[1,1,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0]
df=pd.DataFrame({'A':a,'B':b,'C':c,'D':d,'E':e})
hist = train_model.fit(x=[df[['A','B']], df['C'], df['D'],df['E']],batch_size=10,epochs=10,verbose=2)
The error is:
2021-10-20 16:24:42.901252: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) Epoch 1/10 Traceback (most recent call last): File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 3427, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "", line 1, in runfile('/Users/zhang_james/Documents/study/my_py_env/regular_t.py', wdir='/Users/zhang_james/Documents/study/my_py_env') File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_bundle/pydev_umd.py", line 197, in runfile pydev_imports.execfile(filename, global_vars, local_vars) # execute the script File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile exec(compile(contents "\n", file, 'exec'), glob, loc) File "/Users/zhang_james/Documents/study/my_py_env/regular_t.py", line 121, in hist = train_model.fit(x=[df[['A','B']], df['C'], df['D'],df['E']],batch_size=10,epochs=10,verbose=2) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py", line 1100, in fit tmp_logs = self.train_function(iterator) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 828, in call result = self._call(*args, **kwds) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 871, in _call self._initialize(args, kwds, add_initializers_to=initializers) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 725, in _initialize self._stateful_fn._get_concrete_function_internal_garbage_collected( # pylint: disable=protected-access File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py", line 2969, in _get_concrete_function_internal_garbage_collected graph_function, _ = self._maybe_define_function(args, kwargs) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py", line 3361, in _maybe_define_function graph_function = self._create_graph_function(args, kwargs) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/function.py", line 3196, in _create_graph_function func_graph_module.func_graph_from_py_func( File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py", line 990, in func_graph_from_py_func func_outputs = python_func(*func_args, **func_kwargs) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 634, in wrapped_fn out = weak_wrapped_fn().wrapped(*args, **kwds) File "/Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py", line 977, in wrapper raise e.ag_error_metadata.to_exception(e) ValueError: in user code: /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:805 train_function * return step_function(self, iterator) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:795 step_function ** outputs = model.distribute_strategy.run(run_step, args=(data,)) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:1259 run return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:2730 call_for_each_replica return self._call_for_each_replica(fn, args, kwargs) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:3417 _call_for_each_replica return fn(*args, **kwargs) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:788 run_step ** outputs = model.train_step(data) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:755 train_step loss = self.compiled_loss( /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/compile_utils.py:229 call reg_loss = math_ops.add_n(regularization_losses) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py:201 wrapper return target(*args, **kwargs) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/ops/math_ops.py:3572 add_n return gen_math_ops.add_n(inputs, name=name) /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/ops/gen_math_ops.py:418 add_n _, _, _op, _outputs = _op_def_library._apply_op_helper( /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/op_def_library.py:748 _apply_op_helper op = g._create_op_internal(op_type_name, inputs, dtypes=None, /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py:590 _create_op_internal return super(FuncGraph, self)._create_op_internal( # pylint: disable=protected-access /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:3528 _create_op_internal ret = Operation( /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:2015 init self._c_op = _create_c_op(self._graph, node_def, inputs, /Users/zhang_james/software/anaconda3/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:1856 _create_c_op raise ValueError(str(e)) ValueError: Shapes must be equal rank, but are 0 and 1 From merging shape 1 with other shapes. for '{{node AddN}} = AddN[N=3, T=DT_FLOAT](model/tower_result_mode/A_3_1/ActivityRegularizer/truediv, A_3_1/kernel/Regularizer/mul, model/multi_loss_layer/add_5)' with input shapes: [], [], [1].
CodePudding user response：
This error usually occurs because the calculated loss is not a scalar, but an n-dimensional tensor. Just use tf.keras.backend.sum(*) or tf.keras.backend.mean(*) to reduce your loss to a scalar and then it should work with the regularizers:
def multi_loss(self, ys_true, ys_pred):
assert len(ys_true) == self.nb_outputs and len(ys_pred) == self.nb_outputs
loss = 0
for index, (y_true, y_pred, log_var) in enumerate(zip(ys_true, ys_pred, self.log_vars)):
precision = tf.keras.backend.exp(-log_var)
if index == 0:
single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.30)
elif index == 1:
single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.25)
else:
single_task_loss = self.focal_loss(y_true, y_pred, gamma=4, alpha=0.25)
loss = precision * single_task_loss log_var
return tf.keras.backend.sum(loss)