Apply weighting to generator fed model-CodePudding

Due to the size of my data, I have my training, validation and test data fed via a generator as below:

def sequence_generator(data_type):
        data_type = data_type.decode('ascii')
        sequence_folder = f"{model_attributes['sequences_path']}/{data_type}"
        for numpy_file in random.sample(os.listdir(sequence_folder),len(os.listdir(sequence_folder))):
            if numpy_file.endswith(".npz"):
                with np.load(f"{sequence_folder}/{numpy_file}") as numpy_data:
                    sequences = numpy_data['x']
                    labels = numpy_data['y']
                    labels = tf.one_hot(labels, 3)
                    for X,y in random.sample(list(zip(sequences,labels)),len(sequences)):
                        yield X,y

train_dataset = tf.data.Dataset.from_generator(generator=sequence_generator,args= ['train'], output_types = (tf.float32, tf.float32))# output_shapes = ((train_shape), (3,)))

train_dataset = train_dataset.cache().batch(BATCH).prefetch(tf.data.AUTOTUNE)

history = model.fit(train_dataset, epochs=EPOCHS, validation_data=validation_dataset, callbacks=[tensorboard,checkpoint,earlystop,lr_callback])

From my previous questions accepted answer, I want to be able to apply weighting to 2 of the 3 classes. The accepted answers code is:

sample_weight = np.ones(shape=(len(train_y),))
sample_weight[(train_y[:, 1] == 1) | (train_y[:, 2] == 1)] = 1.5

model = get_model()
model.fit(
    train_x,
    train_y,
    sample_weight=sample_weight,
    ....,
    ...,
    ..,
)

But given that my training data is behind the generator code I am using, I am scratching my head on how best to apply the weighting and with minimal overhead

EDIT

I added the following code as recommended

class_weight = {0: 1.,
                1: 2.,
                2: 2.}

history = model.fit(train_dataset, epochs=EPOCHS, validation_data=validation_dataset,class_weight=class_weight, callbacks=[tensorboard,checkpoint,earlystop,lr_callback])

but I get this error

  File "Train.py", line 1993, in build_and_train_model
    history = model.fit(train_dataset, epochs=EPOCHS, validation_data=validation_dataset,class_weight=class_weight, callbacks=[tensorboard,checkpoint,earlystop,lr_callback])
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1147, in fit
    steps_per_execution=self._steps_per_execution)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1364, in get_data_handler
    return DataHandler(*args, **kwargs)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1175, in __init__
    class_weight, distribute)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1186, in _configure_dataset_and_inferred_steps
    dataset = dataset.map(_make_class_weight_map_fn(class_weight))
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 1925, in map
    return MapDataset(self, map_func, preserve_cardinality=True)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 4487, in __init__
    use_legacy_function=use_legacy_function)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 3712, in __init__
    self._function = fn_factory()
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\eager\function.py", line 3135, in get_concrete_function
    *args, **kwargs)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\eager\function.py", line 3100, in _get_concrete_function_garbage_collected
    graph_function, _ = self._maybe_define_function(args, kwargs)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\eager\function.py", line 3444, in _maybe_define_function
    graph_function = self._create_graph_function(args, kwargs)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\eager\function.py", line 3289, in _create_graph_function
    capture_by_value=self._capture_by_value),
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\func_graph.py", line 999, in func_graph_from_py_func
    func_outputs = python_func(*func_args, **func_kwargs)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 3687, in wrapped_fn
    ret = wrapper_helper(*args)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 3617, in wrapper_helper
    ret = autograph.tf_convert(self._func, ag_ctx)(*nested_args)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 692, in wrapper
    return converted_call(f, args, kwargs, options=options)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 382, in converted_call
    return _call_unconverted(f, args, kwargs, options)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 463, in _call_unconverted
    return f(*args, **kwargs)
  File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1400, in _class_weights_map_fn
    if y.shape.rank > 2:
TypeError: '>' not supported between instances of 'NoneType' and 'int

Not sure what is being referred to as NoneType, from the research I read I get the inclination that it has something to do with one_hotting maybe?

CodePudding user response：

This looks a great moment to use class_weight instead of sample_weight:

Optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function (during training only). This can be useful to tell the model to "pay more attention" to samples from an under-represented class.

CodePudding user response：

I ended up doing this by modifying the generator:

train_dataset  = tf.data.Dataset.from_generator(generator=sequence_generator,args= ['train'], output_types = (tf.float32, tf.float32, tf.float32))# output_shapes = ((train_shape), (3,)))

def sequence_generator(data_type):
    data_type = data_type.decode('ascii')
    sequence_folder = f"{model_attributes['sequences_path']}/{data_type}"
    for numpy_file in random.sample(os.listdir(sequence_folder),len(os.listdir(sequence_folder))):
        if numpy_file.endswith(".npz"):
            with np.load(f"{sequence_folder}/{numpy_file}") as numpy_data:
                sequences = numpy_data['x']
                labels = numpy_data['y']
                class_weight = {0: 1, 1: 2 ,2: 2}
                labels = tf.one_hot(labels, 3)
                sample_weights = np.ones(shape=(len(labels),))
                sample_weights[(labels[:, 1] == 1) | (labels[:, 2] == 1) ] = 2

                for X,y,sample_weight in random.sample(list(zip(sequences,labels,sample_weights)),len(sequences)):
                    yield X,y,sample_weight

The class_weights method did not seem to work. This article seemed to shed some light on it