在Blender中使用代码控制人物模型的头部姿态 - 代码实践Dlib版本
flyfish
在文章的最后，贴上完整的代码
需要一个普通的摄像头，可以像虚拟UP主一样做头部动作
环境：
Ubuntu18.04
Blender 版本2.82
Dlib：用于人脸关键点检测
OpenCV版本3.4.16
solvePnP：从3D-2D点对应关系中找到一个目标的姿态，在这里用于通过人脸关键点计算出头部姿态
目标：输入二维的人脸关键点，输出三维的头部姿态
这里使用的dlib模型检测出68个关键点，在计算头部姿态时使用其中6个
关键点的索引分别是：代码中索引从0开始计算
下巴：8
鼻尖：30
左眼角：36
右眼角：45
左嘴角：48
右嘴角：54
3D头部模型（3D Head Model）
构建一个关键点的3D头部模型
所以会看到这样的代码
image_points = np.array([shape[30],# Nose tip - 31shape[8],# Chin - 9shape[36],# Left eye left corner - 37shape[45],# Right eye right corne - 46shape[48],# Left Mouth corner - 49shape[54]# Right mouth corner - 55], dtype = np.float32)# 3D model points. model_points = np.array([(0.0, 0.0, 0.0), # Nose tip(0.0, -330.0, -65.0), # Chin(-225.0, 170.0, -135.0),# Left eye left corner(225.0, 170.0, -135.0),# Right eye right corne(-150.0, -150.0, -125.0), # Left Mouth corner(150.0, -150.0, -125.0)# Right mouth corner], dtype = np.float32)
例如下巴：(0.0，-330.0，-65.0)这个叫世界坐标（World Coordinates），在OpenCV中叫模型坐标（Model Coordinates）
Blender中的目标存在一个世界中，摄像头拍摄的目标存在另一个世界中
在代码中，需要转换Blender和OpenCV和之间的坐标
通过对比两者的关系
X轴相同
Blender的Y轴是OpenCV的负Z轴
Blender的Z轴是OpenCV的Y轴
总结就是
Blender's red X axis = OpenCV 's X axisBlender's green Y axis = OpenCV's -Z axisBlender's blue Z axis = OpenCV's Y axis
所以会看到这样的代码
bones["head_fk"].rotation_euler[0] = xbones["head_fk"].rotation_euler[1] = -zbones["head_fk"].rotation_euler[2] = y
solvePnP函数说明
输入
objectPoints - 世界坐标系下的关键点的坐标，即代码中的model_points
imagePoints - 在图像坐标系下对应的关键点的坐标。即代码中的image_points，与model_points的关键点顺序一致
cameraMatrix - 相机的内参矩阵
distCoeffs - 相机的畸变系数
输出
旋转向量(roatation vector)也就是上图中R
平移向量(translation vector)上图中的t
w：目标在世界坐标系下的坐标
c：目标在相机坐标系下的坐标
solvePnP求出w和c之间的映射关系
solvePnP函数返回结果包括旋转向量(roatation vector)和平移向量(translation vector)。这里只用了rotation_vector，赋值给bones[“head_fk”].rotation_euler
函数说明参考
/4.x/dc/d2c/tutorial_real_time_pose.html
solvePnP() 对异常点敏感,当相机拍摄目标时如果出现一些异常点，导致位姿估计的不准，可以尝试使用solvePnPRansac()
其他解决方案参考
/4.x/d9/d0c/group__calib3d.html#ga357634492a94efe8858d0ce1509da869
完整代码
import bpyfrom imutils import face_utilsimport dlibimport cv2import timeimport numpy as npclass DlibAnimOperator(bpy.types.Operator):"""Operator which runs its self from a timer"""bl_idname = "wm.dlib_operator"bl_label = "Dlib Animation Operator"p = "/media/ubuntu/data/tool/blender-2.82-linux64/face/shape_predictor_68_face_landmarks.dat" detector = dlib.get_frontal_face_detector()predictor = dlib.shape_predictor(p)# rig_name - take it from your scene collection treerig_name = "RIG-Vincent"_timer = None_cap = Nonewidth = 800height = 600stop :bpy.props.BoolProperty()# 3D model points. model_points = np.array([(0.0, 0.0, 0.0), # Nose tip(0.0, -330.0, -65.0), # Chin(-225.0, 170.0, -135.0),# Left eye left corner(225.0, 170.0, -135.0),# Right eye right corne(-150.0, -150.0, -125.0), # Left Mouth corner(150.0, -150.0, -125.0)# Right mouth corner], dtype = np.float32)# Camera internalscamera_matrix = np.array([[height, 0.0, width/2],[0.0, height, height/2],[0.0, 0.0, 1.0]], dtype = np.float32)# Keeps a moving average of given lengthdef smooth_value(self, name, length, value):if not hasattr(self, 'smooth'):self.smooth = {}if not name in self.smooth:self.smooth[name] = np.array([value])else:self.smooth[name] = np.insert(arr=self.smooth[name], obj=0, values=value)if self.smooth[name].size > length:self.smooth[name] = np.delete(self.smooth[name], self.smooth[name].size-1, 0)sum = 0for val in self.smooth[name]:sum += valreturn sum / self.smooth[name].size# Keeps min and max values, then returns the value in a ranve 0 - 1def get_range(self, name, value):if not hasattr(self, 'range'):self.range = {}if not name in self.range:self.range[name] = np.array([value, value])else:self.range[name] = np.array([min(value, self.range[name][0]), max(value, self.range[name][1])] )val_range = self.range[name][1] - self.range[name][0]if val_range != 0:return (value - self.range[name][0]) / val_rangeelse:return 0def modal(self, context, event):if (event.type in {'RIGHTMOUSE', 'ESC'}) or self.stop == True:self.cancel(context)return {'CANCELLED'}if event.type == 'TIMER':self.init_camera()_, image = self._cap.read()gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)rects = self.detector(gray, 0)# bpy.context.scene.frame_set(frame_num)# For each detected face, find the landmark.for (i, rect) in enumerate(rects):shape = self.predictor(gray, rect)shape = face_utils.shape_to_np(shape)#2D image points. If you change the image, you need to change vectorimage_points = np.array([shape[30],# Nose tip - 31shape[8],# Chin - 9shape[36],# Left eye left corner - 37shape[45],# Right eye right corne - 46shape[48],# Left Mouth corner - 49shape[54]# Right mouth corner - 55], dtype = np.float32)dist_coeffs = np.zeros((4,1)) # Assuming no lens distortionif hasattr(self, 'rotation_vector'):(success, self.rotation_vector, self.translation_vector) = cv2.solvePnP(self.model_points, image_points, self.camera_matrix, dist_coeffs, flags=cv2.SOLVEPNP_ITERATIVE, rvec=self.rotation_vector, tvec=self.translation_vector, useExtrinsicGuess=True)else:(success, self.rotation_vector, self.translation_vector) = cv2.solvePnP(self.model_points, image_points, self.camera_matrix, dist_coeffs, flags=cv2.SOLVEPNP_ITERATIVE, useExtrinsicGuess=False)if not hasattr(self, 'first_angle'):self.first_angle = np.copy(self.rotation_vector)bones = bpy.data.objects[self.rig_name].pose.bonesx=self.smooth_value("h_x", 3, (self.rotation_vector[0] - self.first_angle[0])) y=self.smooth_value("h_y", 3, (self.rotation_vector[1] - self.first_angle[1])) z=self.smooth_value("h_z", 3, (self.rotation_vector[2] - self.first_angle[2])) bones["head_fk"].rotation_euler[0] = xbones["head_fk"].rotation_euler[1] = -zbones["head_fk"].rotation_euler[2] = ybones["head_fk"].keyframe_insert(data_path="rotation_euler", index=-1)for (x, y) in shape:cv2.circle(image, (x, y), 2, (0, 255, 255), -1)cv2.imshow("Output", image)cv2.waitKey(1)return {'PASS_THROUGH'}def init_camera(self):if self._cap == None:self._cap = cv2.VideoCapture(0)#self._cap = cv2.VideoCapture("/media/ubuntu/data/sign_videos/HabenSieSchmerzen0.mp4")self._cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.width)self._cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height)self._cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)time.sleep(0.5)def stop_playback(self, scene):print(format(scene.frame_current) + " / " + format(scene.frame_end))if scene.frame_current == scene.frame_end:bpy.ops.screen.animation_cancel(restore_frame=False)def execute(self, context):bpy.app.handlers.frame_change_pre.append(self.stop_playback)wm = context.window_managerself._timer = wm.event_timer_add(0.02, window=context.window)wm.modal_handler_add(self)return {'RUNNING_MODAL'}def cancel(self, context):wm = context.window_managerwm.event_timer_remove(self._timer)cv2.destroyAllWindows()self._cap.release()self._cap = Nonedef register():bpy.utils.register_class(DlibAnimOperator)def unregister():bpy.utils.unregister_class(DlibAnimOperator)if __name__ == "__main__":register()# test callbpy.ops.wm.dlib_operator()
恢复原始位置执行的代码
import bpyimport mathimport mathutilsdef Original():ob = bpy.data.objects['RIG-Vincent'] head=ob.pose.bones['head_fk'] head.rotation_mode = 'XYZ'head.rotation_euler = mathutils.Euler((0.0, 0.0, 0.0), 'XYZ')head.rotation_mode = 'QUATERNION'head.keyframe_insert(data_path="rotation_euler" ,frame=-1)Original()
