Can MediaPipe specify which parts of the face mesh are the lips or nose or eyes?-CodePudding

MediaPipe is capable of providing the x,y,z points of multiple points on the face, enabling it to generate a face mesh. However, the output is just in x,y,z points. Is there any way to know which of those points are those of the lips?

import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_face_mesh = mp.solutions.face_mesh

# For static images:
IMAGE_FILES = []
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
with mp_face_mesh.FaceMesh(
    static_image_mode=True,
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5) as face_mesh:
  for idx, file in enumerate(IMAGE_FILES):
    image = cv2.imread(file)
    # Convert the BGR image to RGB before processing.
    results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))

    # Print and draw face mesh landmarks on the image.
    if not results.multi_face_landmarks:
      continue
    annotated_image = image.copy()
    for face_landmarks in results.multi_face_landmarks:
      print('face_landmarks:', face_landmarks)
      mp_drawing.draw_landmarks(
          image=annotated_image,
          landmark_list=face_landmarks,
          connections=mp_face_mesh.FACEMESH_TESSELATION,
          landmark_drawing_spec=None,
          connection_drawing_spec=mp_drawing_styles
          .get_default_face_mesh_tesselation_style())
      mp_drawing.draw_landmarks(
          image=annotated_image,
          landmark_list=face_landmarks,
          connections=mp_face_mesh.FACEMESH_CONTOURS,
          landmark_drawing_spec=None,
          connection_drawing_spec=mp_drawing_styles
          .get_default_face_mesh_contours_style())
      mp_drawing.draw_landmarks(
          image=annotated_image,
          landmark_list=face_landmarks,
          connections=mp_face_mesh.FACEMESH_IRISES,
          landmark_drawing_spec=None,
          connection_drawing_spec=mp_drawing_styles
          .get_default_face_mesh_iris_connections_style())
    cv2.imwrite('/tmp/annotated_image'   str(idx)   '.png', annotated_image)

Output:

landmark {
  x: 0.5328186750411987
  y: 0.3934963345527649
  z: -0.008206618018448353
}
landmark {
  x: 0.5807108879089355
  y: 0.3586674928665161
  z: 0.017649170011281967
}
landmark {
  x: 0.5844370126724243
  y: 0.3515523076057434
  z: 0.01841720938682556
}
...and more such points

CodePudding user response：

Make sure you import matplotlib import matplotlib.pyplot as plt.

xVal = []; yVal = []; zVal = []
xS = []; yS = []
for land in results.multi_face_landmarks:
    print(type(land.landmark))
    count = 0
    for i in land.landmark:
      xVal.append(i.x)
      yVal.append(i.y)
      zVal.append(i.z)
      if count == 0 or (count >= 11 and count <= 17): #some points on the lip
        xS.append(i.x); yS.append(i.y)
      count = count   1
    print("Total = ", count)
plt.scatter(xVal, yVal, color = 'blue')
plt.scatter(xS, yS, color = 'red')
plt.show()

The points on the face will match the points shown in this diagram. You'll have to download the image and zoom in to see the numbers.

CodePudding user response：

Nav's answer is good and creative. but you can also use the official medaipipe framework canonical_face_model_uv_visualization. download this image from mediapipe repository and zoom image to explore landmarks.