So I wrote a code for a CNN model. It was running alright when I was testing it with a smaller dataset of about 200-250 images total(training, testing, validation included). I then proceeded to move the rest of my data (over 8000 photos total) to the directory I had set up for the model and after hours of organizing the folders, now it stops at the point where it should run the first epoch. I don't know why this is happening. I haven't changed the code. I need to submit it before the end of the weekend and I am completely clueless right now the code and the output from the debug window are :
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from keras.layers import Dense, Conv2D, Flatten, MaxPool2D
from tensorflow.keras.preprocessing import image
from tensorflow.keras import layers
from keras.models import Sequential
from tensorflow.keras import Model
import matplotlib.pyplot as plt
import tensorflow.keras.models
from tensorflow import keras
from keras import optimizers
import tensorflow as tf
import numpy as np
import cv2 as cv
import os
img = image.load_img('C:/Users/Asus/Pictures/thesisdata/training/pepsi_500ml_ful/WIN_20210605_02_05_35_Pro.jpg')
plt.imshow(img)
shape = cv.imread('C:/Users/Asus/Pictures/thesisdata/training/pepsi_500ml_ful/WIN_20210605_02_05_35_Pro.jpg').shape
print(shape)
train = ImageDataGenerator(rescale = 1/255,shear_range=0.2, zoom_range=0.2,rotation_range=0.2)
validation = ImageDataGenerator(rescale= 1/255,shear_range=0.2,zoom_range=0.2,validation_split=0.999999999)
train_dataset = train.flow_from_directory('C:/Users/Asus/Pictures/thesisdata/training',target_size=(200,200), batch_size= 10, class_mode= 'categorical', subset='training')
validation_dataset = validation.flow_from_directory('C:/Users/Asus/Pictures/thesisdata/validation',target_size=(200,200), batch_size= 10, class_mode='categorical', subset='validation')
print(train_dataset.class_indices)
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(filters=16,padding='same',strides=2,kernel_size=3,activation='relu',input_shape=(200,200,3)))
model.add(tf.keras.layers.MaxPool2D(pool_size=2,strides=2))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(4,activation='softmax'))
opt = keras.optimizers.RMSprop(learning_rate=0.013)
model.compile(loss='categorical_crossentropy',optimizer = opt ,metrics = ['accuracy'])
model_fit = model.fit(train_dataset, steps_per_epoch = 3, epochs = 20, validation_data= validation_dataset)
Here's the debug message:
C:\Users\Asus\PycharmProjects\helloworld2\venv\Scripts\python.exe C:/Users/Asus/PycharmProjects/helloworld2/venv/hello.py
2021-10-23 18:11:23.847265: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cudart64_110.dll'; dlerror: cudart64_110.dll not found
2021-10-23 18:11:23.847388: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
(720, 1280, 3)
Found 4964 images belonging to 61 classes.
Found 1553 images belonging to 61 classes.
{'7up_1500ml_emty': 0, '7up_1500ml_ful': 1, '7up_300ml_emty': 2, '7up_300ml_ful': 3, '7up_diet_1500ml_emty': 4, '7up_diet_1500ml_ful': 5, '7up_diet_500ml_emty': 6, '7up_diet_500ml_ful': 7, 'aquafina_500ml_emty': 8, 'aquafina_500ml_ful': 9, 'cocacola_2250ml_emty': 10, 'cocacola_2250ml_ful': 11, 'cocacola_500ml_emty': 12, 'cocacola_500ml_ful': 13, 'fanta_1500ml_emty': 14, 'fanta_1500ml_ful': 15, 'fanta_2250ml_emty': 16, 'fanta_2250ml_ful': 17, 'fanta_500ml_ful': 18, 'gatorade_blue_500ml_ful': 19, 'gatorade_red_500ml_ful': 20, 'gatorade_white_500ml_emty': 21, 'gatorade_white_500ml_ful': 22, 'mirinda_300ml_ful': 23, 'mirinda_500ml_emty': 24, 'mirinda_500ml_ful': 25, 'mtdew_1500ml_emty': 26, 'mtdew_1500ml_ful': 27, 'mtdew_2250ml_emty': 28, 'mtdew_2250ml_ful': 29, 'mtdew_300ml_emty': 30, 'mtdew_300ml_ful': 31, 'mtdew_500ml_emty': 32, 'mtdew_500ml_ful': 33, 'pakola_green_1500ml_emty': 34, 'pakola_green_1500ml_ful': 35, 'pakola_green_500ml_emty': 36, 'pakola_green_500ml_ful': 37, 'pakola_lemon_300ml_ful': 38, 'pakola_lemon_500ml_emty': 39, 'pakola_lemon_500ml_ful': 40, 'pakola_lychee_1500ml_emty': 41, 'pakola_lychee_1500ml_ful': 42, 'pakola_lychee_300ml_emty': 43, 'pakola_lychee_500ml_emty': 44, 'pakola_lychee_500ml_ful': 45, 'pepsi_1500ml_ful': 46, 'pepsi_300ml_ful': 47, 'pepsi_500ml_ful': 48, 'slice_1000ml_emty': 49, 'slice_1000ml_ful': 50, 'slice_355ml_ful': 51, 'sprite_1500ml_emty': 52, 'sprite_1500ml_ful': 53, 'sprite_2250ml_emty': 54, 'sprite_2250ml_ful': 55, 'sprite_500ml_ful': 56, 'sting_red_500ml_ful': 57, 'sting_yellow_500ml_emty': 58, 'sting_yellow_500ml_ful': 59, 'sunlay_300ml_emty': 60}
2021-10-23 18:11:27.764834: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cudart64_110.dll'; dlerror: cudart64_110.dll not found
2021-10-23 18:11:27.765399: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cublas64_11.dll'; dlerror: cublas64_11.dll not found
2021-10-23 18:11:27.765975: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cublasLt64_11.dll'; dlerror: cublasLt64_11.dll not found
2021-10-23 18:11:27.766536: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cufft64_10.dll'; dlerror: cufft64_10.dll not found
2021-10-23 18:11:27.767063: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'curand64_10.dll'; dlerror: curand64_10.dll not found
2021-10-23 18:11:27.767598: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cusolver64_11.dll'; dlerror: cusolver64_11.dll not found
2021-10-23 18:11:27.768131: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cusparse64_11.dll'; dlerror: cusparse64_11.dll not found
2021-10-23 18:11:27.768681: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cudnn64_8.dll'; dlerror: cudnn64_8.dll not found
2021-10-23 18:11:27.768777: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1835] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
2021-10-23 18:11:27.769190: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX AVX2
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-10-23 18:11:27.937756: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)
Epoch 1/20
Traceback (most recent call last):
File "C:\Users\Asus\PycharmProjects\helloworld2\venv\hello.py", line 35, in <module>
model_fit = model.fit(train_dataset, steps_per_epoch = 3, epochs = 20, validation_data= validation_dataset)
File "C:\Users\Asus\PycharmProjects\helloworld2\venv\lib\site-packages\keras\engine\training.py", line 1184, in fit
tmp_logs = self.train_function(iterator)
File "C:\Users\Asus\PycharmProjects\helloworld2\venv\lib\site-packages\tensorflow\python\eager\def_function.py", line 885, in __call__
result = self._call(*args, **kwds)
File "C:\Users\Asus\PycharmProjects\helloworld2\venv\lib\site-packages\tensorflow\python\eager\def_function.py", line 950, in _call
return self._stateless_fn(*args, **kwds)
File "C:\Users\Asus\PycharmProjects\helloworld2\venv\lib\site-packages\tensorflow\python\eager\function.py", line 3039, in __call__
return graph_function._call_flat(
File "C:\Users\Asus\PycharmProjects\helloworld2\venv\lib\site-packages\tensorflow\python\eager\function.py", line 1963, in _call_flat
return self._build_call_outputs(self._inference_function.call(
File "C:\Users\Asus\PycharmProjects\helloworld2\venv\lib\site-packages\tensorflow\python\eager\function.py", line 591, in call
outputs = execute.execute(
File "C:\Users\Asus\PycharmProjects\helloworld2\venv\lib\site-packages\tensorflow\python\eager\execute.py", line 59, in quick_execute
tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
tensorflow.python.framework.errors_impl.InvalidArgumentError: logits and labels must be broadcastable: logits_size=[10,4] labels_size=[10,61]
[[node categorical_crossentropy/softmax_cross_entropy_with_logits (defined at \hello.py:35) ]] [Op:__inference_train_function_626]
Function call stack:
train_function
CodePudding user response：
model.add(tf.keras.layers.Dense(4,activation='softmax'))
As your output layer is 4 and your class is:
Found 4964 images belonging to 61 classes.
Found 1553 images belonging to 61 classes.
Most likely, you have to change 4 --> 61 in
model.add(tf.keras.layers.Dense(4,activation='softmax'))