The command I am using to start the training process for my model is:
python Tensorflow/models/research/object_detection/model_main_tf2.py --model_dir=Tensorflow/workspace/models/my_ssd_mobnet --pipeline_config_path=Tensorflow/workspace/models/my_ssd_mobnet/pipeline.config --num_train_steps=2000
And I am getting this error...
Traceback (most recent call last):
File "Tensorflow/models/research/object_detection/model_main_tf2.py", line 114, in <module>
tf.compat.v1.app.run()
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/platform/app.py", line 36, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/usr/local/lib/python3.7/dist-packages/absl/app.py", line 308, in run
_run_main(main, args)
File "/usr/local/lib/python3.7/dist-packages/absl/app.py", line 254, in _run_main
sys.exit(main(argv))
File "Tensorflow/models/research/object_detection/model_main_tf2.py", line 111, in main
record_summaries=FLAGS.record_summaries)
File "/usr/local/lib/python3.7/dist-packages/object_detection/model_lib_v2.py", line 609, in train_loop
train_input, unpad_groundtruth_tensors)
File "/usr/local/lib/python3.7/dist-packages/object_detection/model_lib_v2.py", line 401, in load_fine_tune_checkpoint
_ensure_model_is_built(model, input_dataset, unpad_groundtruth_tensors)
File "/usr/local/lib/python3.7/dist-packages/object_detection/model_lib_v2.py", line 179, in _ensure_model_is_built
labels,
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py", line 1312, in run
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py", line 2888, in call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/mirrored_strategy.py", line 677, in _call_for_each_replica
self._container_strategy(), fn, args, kwargs)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/mirrored_run.py", line 83, in call_for_each_replica
return wrapped(args, kwargs)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py", line 153, in error_handler
raise e.with_traceback(filtered_tb) from None
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py", line 55, in quick_execute
inputs, attrs, num_outputs)
tensorflow.python.framework.errors_impl.UnimplementedError: Graph execution error:
Detected at node 'ssd_mobile_net_v2_fpn_keras_feature_extractor/model/Conv1/Conv2D' defined at (most recent call last):
File "/usr/lib/python3.7/threading.py", line 890, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.7/threading.py", line 926, in _bootstrap_inner
self.run()
File "/usr/local/lib/python3.7/dist-packages/object_detection/model_lib_v2.py", line 171, in _dummy_computation_fn
return _compute_losses_and_predictions_dicts(model, features, labels,
File "/usr/local/lib/python3.7/dist-packages/object_detection/model_lib_v2.py", line 124, in _compute_losses_and_predictions_dicts
prediction_dict = model.predict(
File "/usr/local/lib/python3.7/dist-packages/object_detection/meta_architectures/ssd_meta_arch.py", line 569, in predict
if self._feature_extractor.is_keras_model:
File "/usr/local/lib/python3.7/dist-packages/object_detection/meta_architectures/ssd_meta_arch.py", line 570, in predict
feature_maps = self._feature_extractor(preprocessed_inputs)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 490, in __call__
return super().__call__(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py", line 1014, in __call__
outputs = call_fn(inputs, *args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/object_detection/meta_architectures/ssd_meta_arch.py", line 251, in call
return self._extract_features(inputs)
File "/usr/local/lib/python3.7/dist-packages/object_detection/models/ssd_mobilenet_v2_fpn_keras_feature_extractor.py", line 219, in _extract_features
image_features = self.classification_backbone(
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 490, in __call__
return super().__call__(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py", line 1014, in __call__
outputs = call_fn(inputs, *args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/functional.py", line 459, in call
inputs, training=training, mask=mask)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/functional.py", line 596, in _run_internal_graph
outputs = node.layer(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py", line 1014, in __call__
outputs = call_fn(inputs, *args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 92, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/layers/convolutional/base_conv.py", line 250, in call
outputs = self.convolution_op(inputs, self.kernel)
File "/usr/local/lib/python3.7/dist-packages/keras/layers/convolutional/base_conv.py", line 232, in convolution_op
name=self.__class__.__name__)
Node: 'ssd_mobile_net_v2_fpn_keras_feature_extractor/model/Conv1/Conv2D'
DNN library is not found.
[[{{node ssd_mobile_net_v2_fpn_keras_feature_extractor/model/Conv1/Conv2D}}]] [Op:__inference__dummy_computation_fn_15090]
I have double checked that my runtime/notebook settings that I am using the GPU as my hardware accelerator. I have also confirmed that the GPU has been found. Does anybody know how to fix this?
my current pip list is:
Package Version
----------------------------- ------------------------------
absl-py 1.2.0
alabaster 0.7.12
albumentations 0.1.12
altair 4.2.0
apache-beam 2.40.0
appdirs 1.4.4
argon2-cffi 21.3.0
argon2-cffi-bindings 21.2.0
arviz 0.12.1
astor 0.8.1
astropy 4.3.1
astunparse 1.6.3
atari-py 0.2.9
atomicwrites 1.4.1
attrs 21.4.0
audioread 2.1.9
autograd 1.4
avro-python3 1.10.2
Babel 2.10.3
backcall 0.2.0
beautifulsoup4 4.6.3
bleach 5.0.1
blis 0.7.8
bokeh 2.3.3
branca 0.5.0
bs4 0.0.1
CacheControl 0.12.11
cached-property 1.5.2
cachetools 4.2.4
catalogue 2.0.7
certifi 2022.6.15
cffi 1.15.1
cftime 1.6.1
chardet 3.0.4
charset-normalizer 2.1.0
click 7.1.2
cloudpickle 2.1.0
cmake 3.22.5
cmdstanpy 0.9.5
colorama 0.4.5
colorcet 3.0.0
colorlover 0.3.0
community 1.0.0b1
contextlib2 0.5.5
convertdate 2.4.0
coverage 3.7.1
coveralls 0.5
crcmod 1.7
cufflinks 0.17.3
cupy-cuda111 9.4.0
cvxopt 1.2.7
cvxpy 1.0.31
cycler 0.11.0
cymem 2.0.6
Cython 0.29.30
daft 0.0.4
dask 2.12.0
datascience 0.10.6
debugpy 1.0.0
decorator 4.4.2
defusedxml 0.7.1
deprecat 2.1.1
descartes 1.1.0
dill 0.3.1.1
distributed 1.25.3
dlib 19.18.0 zzzcolab20220513001918
dm-tree 0.1.7
docopt 0.6.2
docutils 0.17.1
dopamine-rl 1.0.5
earthengine-api 0.1.317
easydict 1.9
ecos 2.0.10
editdistance 0.5.3
en-core-web-sm 3.3.0
entrypoints 0.4
ephem 4.1.3
et-xmlfile 1.1.0
etils 0.6.0
fa2 0.3.5
fastai 2.7.7
fastavro 1.5.3
fastcore 1.5.6
fastdownload 0.0.7
fastdtw 0.3.4
fastjsonschema 2.16.1
fastprogress 1.0.2
fastrlock 0.8
fbprophet 0.7.1
feather-format 0.4.1
filelock 3.7.1
firebase-admin 4.4.0
fix-yahoo-finance 0.0.22
Flask 1.1.4
flatbuffers 1.12
folium 0.8.3
future 0.16.0
gast 0.4.0
GDAL 2.2.2
gdown 4.4.0
gensim 3.6.0
geographiclib 1.52
geopy 1.17.0
gin-config 0.5.0
glob2 0.7
google 2.0.3
google-api-core 1.31.6
google-api-python-client 1.12.11
google-auth 1.35.0
google-auth-httplib2 0.0.4
google-auth-oauthlib 0.4.6
google-cloud-bigquery 1.21.0
google-cloud-bigquery-storage 1.1.2
google-cloud-core 1.0.3
google-cloud-datastore 1.8.0
google-cloud-firestore 1.7.0
google-cloud-language 1.2.0
google-cloud-storage 1.18.1
google-cloud-translate 1.5.0
google-colab 1.0.0
google-pasta 0.2.0
google-resumable-media 0.4.1
googleapis-common-protos 1.56.4
googledrivedownloader 0.4
graphviz 0.10.1
greenlet 1.1.2
grpcio 1.47.0
gspread 3.4.2
gspread-dataframe 3.0.8
gym 0.17.3
h5py 3.1.0
hdfs 2.7.0
HeapDict 1.0.1
hijri-converter 2.2.4
holidays 0.10.5.2
holoviews 1.14.9
html5lib 1.0.1
httpimport 0.5.18
httplib2 0.17.4
httplib2shim 0.0.3
humanize 0.5.1
hyperopt 0.1.2
ideep4py 2.0.0.post3
idna 2.10
imageio 2.4.1
imagesize 1.4.1
imbalanced-learn 0.8.1
imblearn 0.0
imgaug 0.2.9
importlib-metadata 4.12.0
importlib-resources 5.8.0
imutils 0.5.4
inflect 2.1.0
iniconfig 1.1.1
intel-openmp 2022.1.0
intervaltree 2.1.0
ipykernel 4.10.1
ipython 5.5.0
ipython-genutils 0.2.0
ipython-sql 0.3.9
ipywidgets 7.7.1
itsdangerous 1.1.0
jax 0.3.14
jaxlib 0.3.14 cuda11.cudnn805
jedi 0.18.1
jieba 0.42.1
Jinja2 2.11.3
joblib 1.1.0
jpeg4py 0.1.4
jsonschema 4.3.3
jupyter 1.0.0
jupyter-client 5.3.5
jupyter-console 5.2.0
jupyter-core 4.11.1
jupyterlab-pygments 0.2.2
jupyterlab-widgets 1.1.1
kaggle 1.5.12
kapre 0.3.7
keras 2.9.0
Keras-Preprocessing 1.1.2
keras-vis 0.4.1
kiwisolver 1.4.4
korean-lunar-calendar 0.2.1
langcodes 3.3.0
libclang 14.0.1
librosa 0.8.1
lightgbm 2.2.3
llvmlite 0.34.0
lmdb 0.99
LunarCalendar 0.0.9
lvis 0.5.3
lxml 4.2.6
Markdown 3.4.1
MarkupSafe 2.0.1
matplotlib 3.2.2
matplotlib-inline 0.1.3
matplotlib-venn 0.11.7
missingno 0.5.1
mistune 0.8.4
mizani 0.6.0
mkl 2019.0
mlxtend 0.14.0
more-itertools 8.13.0
moviepy 0.2.3.5
mpmath 1.2.1
msgpack 1.0.4
multiprocess 0.70.13
multitasking 0.0.11
murmurhash 1.0.7
music21 5.5.0
natsort 5.5.0
nbclient 0.6.6
nbconvert 5.6.1
nbformat 5.4.0
nest-asyncio 1.5.5
netCDF4 1.6.0
networkx 2.6.3
nibabel 3.0.2
nltk 3.7
notebook 5.3.1
numba 0.51.2
numexpr 2.8.3
numpy 1.21.6
oauth2client 4.1.3
oauthlib 3.2.0
object-detection 0.1
okgrade 0.4.3
opencv-contrib-python 4.6.0.66
opencv-python 4.6.0.66
opencv-python-headless 4.6.0.66
openpyxl 3.0.10
opt-einsum 3.3.0
orjson 3.7.8
osqp 0.6.2.post0
packaging 21.3
palettable 3.3.0
pandas 1.3.5
pandas-datareader 0.9.0
pandas-gbq 0.13.3
pandas-profiling 1.4.1
pandocfilters 1.5.0
panel 0.12.1
param 1.12.2
parso 0.8.3
pathlib 1.0.1
pathy 0.6.2
patsy 0.5.2
pep517 0.12.0
pexpect 4.8.0
pickleshare 0.7.5
Pillow 7.1.2
pip 21.1.3
pip-tools 6.2.0
plotly 5.5.0
plotnine 0.6.0
pluggy 0.7.1
pooch 1.6.0
portalocker 2.5.1
portpicker 1.3.9
prefetch-generator 1.0.1
preshed 3.0.6
prettytable 3.3.0
progressbar2 3.38.0
prometheus-client 0.14.1
promise 2.3
prompt-toolkit 1.0.18
proto-plus 1.20.6
protobuf 3.19.4
psutil 5.4.8
psycopg2 2.7.6.1
ptyprocess 0.7.0
py 1.11.0
py-cpuinfo 8.0.0
pyarrow 6.0.1
pyasn1 0.4.8
pyasn1-modules 0.2.8
pycocotools 2.0.4
pycparser 2.21
pyct 0.4.8
pydantic 1.8.2
pydata-google-auth 1.4.0
pydot 1.3.0
pydot-ng 2.0.0
pydotplus 2.0.2
PyDrive 1.3.1
pyemd 0.5.1
pyerfa 2.0.0.1
pyglet 1.5.0
Pygments 2.6.1
pygobject 3.26.1
pymc3 3.11.5
PyMeeus 0.5.11
pymongo 3.12.3
pymystem3 0.2.0
PyOpenGL 3.1.6
pyparsing 2.4.7
pyrsistent 0.18.1
pysndfile 1.3.8
PySocks 1.7.1
pystan 2.19.1.1
pytest 3.6.4
python-apt 0.0.0
python-chess 0.23.11
python-dateutil 2.8.2
python-louvain 0.16
python-slugify 6.1.2
python-utils 3.3.3
pytz 2022.1
pyviz-comms 2.2.0
PyWavelets 1.3.0
PyYAML 5.4.1
pyzmq 23.2.0
qdldl 0.1.5.post2
qtconsole 5.3.1
QtPy 2.1.0
regex 2022.6.2
requests 2.28.1
requests-oauthlib 1.3.1
resampy 0.3.1
rpy2 3.4.5
rsa 4.8
sacrebleu 2.1.0
scikit-image 0.18.3
scikit-learn 1.0.2
scipy 1.7.3
screen-resolution-extra 0.0.0
scs 3.2.0
seaborn 0.11.2
semver 2.13.0
Send2Trash 1.8.0
sentencepiece 0.1.96
seqeval 1.2.2
setuptools 57.4.0
setuptools-git 1.2
Shapely 1.8.2
simplegeneric 0.8.1
six 1.15.0
sklearn 0.0
sklearn-pandas 1.8.0
smart-open 5.2.1
snowballstemmer 2.2.0
sortedcontainers 2.4.0
SoundFile 0.10.3.post1
soupsieve 2.3.2.post1
spacy 3.3.1
spacy-legacy 3.0.9
spacy-loggers 1.0.3
Sphinx 1.8.6
sphinxcontrib-serializinghtml 1.1.5
sphinxcontrib-websupport 1.2.4
SQLAlchemy 1.4.39
sqlparse 0.4.2
srsly 2.4.3
statsmodels 0.10.2
sympy 1.7.1
tables 3.7.0
tabulate 0.8.10
tblib 1.7.0
tenacity 8.0.1
tensorboard 2.9.1
tensorboard-data-server 0.6.1
tensorboard-plugin-wit 1.8.1
tensorflow 2.9.1
tensorflow-addons 0.17.1
tensorflow-datasets 4.0.1
tensorflow-estimator 2.9.0
tensorflow-gcs-config 2.8.0
tensorflow-gpu 2.9.1
tensorflow-hub 0.12.0
tensorflow-io 0.26.0
tensorflow-io-gcs-filesystem 0.26.0
tensorflow-metadata 1.9.0
tensorflow-model-optimization 0.7.3
tensorflow-probability 0.16.0
tensorflow-text 2.9.0
termcolor 1.1.0
terminado 0.13.3
testpath 0.6.0
text-unidecode 1.3
textblob 0.15.3
tf-models-official 2.9.2
tf-slim 1.1.0
Theano-PyMC 1.1.2
thinc 8.0.17
threadpoolctl 3.1.0
tifffile 2021.11.2
tinycss2 1.1.1
tomli 2.0.1
toolz 0.12.0
torch 1.12.0 cu113
torchaudio 0.12.0 cu113
torchsummary 1.5.1
torchtext 0.13.0
torchvision 0.13.0 cu113
tornado 5.1.1
tqdm 4.64.0
traitlets 5.1.1
tweepy 3.10.0
typeguard 2.7.1
typer 0.4.2
typing-extensions 4.1.1
tzlocal 1.5.1
uritemplate 3.0.1
urllib3 1.24.3
vega-datasets 0.9.0
wasabi 0.9.1
wcwidth 0.2.5
webencodings 0.5.1
Werkzeug 1.0.1
wheel 0.37.1
widgetsnbextension 3.6.1
wordcloud 1.5.0
wrapt 1.14.1
xarray 0.20.2
xarray-einstats 0.2.2
xgboost 0.90
xkit 0.0.0
xlrd 1.1.0
xlwt 1.3.0
yellowbrick 1.4
zict 2.2.0
zipp 3.8.1
CodePudding user response：
I figure it out. I added
!pip install tensorflow==2.8
!apt install --allow-change-held-packages libcudnn8=8.1.0.77-1 cuda11.2
before running the training command and it resolved the issue.