I am trying to get the initial weights for a given network.
This thread suggests that one needs to specify the input dimension: How to view initialized weights (i.e. before training)?
This thread suggests that after compilation weights should be available: Reset weights in Keras layer
Save the initial weights right after compiling the model but before training it.
While I reproduced the results in the first post I am unable to apply it to my case:
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
import tensorflow as tf
# Reproducing https://stackoverflow.com/questions/46798708/how-to-view-initialized-weights-i-e-before-training
# First model without input_dim prints an empty list
model = Sequential()
model.add(Dense(5, weights=[np.ones((3, 5)), np.zeros(5)],
activation='relu'))
print(model.get_weights())
# Second model with input_dim prints the assigned weights
model1 = Sequential()
model1.add(Dense(5, weights=[np.ones((3, 5)), np.zeros(5)],
input_dim=3,
activation='relu'))
model1.add(Dense(1, activation='sigmoid'))
print(model1.get_weights())
class Test(tf.keras.Model):
def __init__(self, n_inputs: int, neurons=10):
super(Test, self).__init__(name="Test")
self.neurons = neurons
# Initilializers
mean, std = 0., 0.0005
bias_normalization = None
kernel_initializer = tf.keras.initializers.RandomNormal(mean=mean,
stddev=std)
self.h1 = Dense(n_inputs, activation="linear", name="h1",
kernel_initializer=kernel_initializer,
bias_initializer=bias_normalization,
input_dim=n_inputs)
def call(self, inputs):
x = self.h1(inputs)
return x
# Model Test
test = Test(n_inputs=1, neurons=100)
test.get_weights() # empty, expected
test.compile()
test.get_weights() # empty, unexpected
CodePudding user response：
Not sure why you choose the burden to define a specific class here, but your Test class does not define a model, only a single Dense layer; hence, it is not strange that you do not end up with any weights whatsoever. Changing self.h1 to:
self.h1 = Sequential(Dense(n_inputs, activation="linear", name="h1",
kernel_initializer=kernel_initializer,
bias_initializer=bias_normalization,
input_dim=n_inputs))
will do the trick in both cases:
test = Test(n_inputs=1, neurons=100)
test.get_weights()
# [array([[-0.00030265]], dtype=float32), array([-1.6327941], dtype=float32)]
test.compile()
test.get_weights() # same weights as above
# [array([[-0.00030265]], dtype=float32), array([-1.6327941], dtype=float32)]
Notice that there seem to be changes in different versions, as the frameworks evolve, so some details from old threads might be different today. For example, in the Keras version currently used in Google Colab (2.6.0), model.get_weights() will throw an error if input_dim is not defined - it will not return an empty list:
from keras.models import Sequential
from keras.layers import Dense
import numpy as np
import keras
keras.__version__
# '2.6.0'
model = Sequential()
model.add(Dense(5, weights=[np.ones((3, 5)), np.zeros(5)],
activation='relu'))
print(model.get_weights())
This gives:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-4-1f22e9c3ce78> in <module>()
2 model.add(Dense(5, weights=[np.ones((3, 5)), np.zeros(5)],
3 activation='relu'))
----> 4 print(model.get_weights())
5 frames
/usr/local/lib/python3.7/dist-packages/keras/engine/training.py in get_weights(self)
2090 """
2091 with self.distribute_strategy.scope():
-> 2092 return super(Model, self).get_weights()
2093
2094 def save(self,
/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py in get_weights(self)
1844 Weights values as a list of NumPy arrays.
1845 """
-> 1846 weights = self.weights
1847 output_weights = []
1848 for weight in weights:
/usr/local/lib/python3.7/dist-packages/keras/engine/training.py in weights(self)
2488 A list of variables.
2489 """
-> 2490 return self._dedup_weights(self._undeduplicated_weights)
2491
2492 @property
/usr/local/lib/python3.7/dist-packages/keras/engine/training.py in _undeduplicated_weights(self)
2493 def _undeduplicated_weights(self):
2494 """Returns the undeduplicated list of all layer variables/weights."""
-> 2495 self._assert_weights_created()
2496 weights = []
2497 for layer in self._self_tracked_trackables:
/usr/local/lib/python3.7/dist-packages/keras/engine/sequential.py in _assert_weights_created(self)
465 # When the graph has not been initialized, use the Model's implementation to
466 # to check if the weights has been created.
--> 467 super(functional.Functional, self)._assert_weights_created() # pylint: disable=bad-super-call
468
469
/usr/local/lib/python3.7/dist-packages/keras/engine/training.py in _assert_weights_created(self)
2672 'Weights are created when the Model is first called on '
2673 'inputs or `build()` is called with an `input_shape`.' %
-> 2674 self.name)
2675
2676 def _check_call_args(self, method_name):
ValueError: Weights for model sequential_1 have not yet been created. Weights are created when the Model is first called on inputs or `build()` is called with an `input_shape`.
Finally, as a general remark, you should be careful not to mix functionality from keras and tf.keras; these are different libraries, so choose one and stick to it consistently throughout your code.
CodePudding user response：
In your case, I think it all depends on when the call method of tf.keras.Model is actually called. Also, Keras sequential models and subclassing models behave differently.
For example if you try the following you will get an output of some weights:
test_model = Test(n_inputs=1, neurons=100)
test_model(np.random.random((32, 1)))
print(test_model.get_weights())
# [array([[0.00057544]]), array([0.3752869])]
or
test_model.build(input_shape=(32, 1))
print(test_model.get_weights())
# [array([[8.942684e-05]], dtype=float32), array([-1.6799461], dtype=float32)]
Basically, the call method internally calls the __call__ method. If you check out the official Tensorflow website you can read about this behavior:
To call a model on an input, always use the call method, i.e. model(inputs), which relies on the underlying call method.
You could by the way also define your Test class as follows:
class Test(tf.keras.Model):
def __init__(self, n_inputs: int, neurons=10):
super(Test, self).__init__(name="Test")
self.neurons = neurons
# Initilializers
mean, std = 0., 0.0005
bias_normalization = None
kernel_initializer = tf.keras.initializers.RandomNormal(mean=mean,
stddev=std)
model_input = tf.keras.layers.Input(shape=(n_inputs,))
x= tf.keras.layers.Dense(n_inputs, activation="linear", name="h1",
kernel_initializer=kernel_initializer,
bias_initializer=bias_normalization)(model_input)
self.model = tf.keras.Model(model_input, x)
test_model = Test(n_inputs=1, neurons=100)
print(test_model.get_weights())
# [array([[0.00045629]], dtype=float32), array([0.9945322], dtype=float32)]