使用Python和MediaPipe实现手势虚拟鼠标控制

概述

使用Python实现虚拟鼠标控制，利用手势识别来替代传统鼠标操作。这一实现依赖于计算机视觉库OpenCV、手势识别库MediaPipe以及其他辅助库如PyAutoGUI和Pynput。

环境配置

在开始之前，请确保已安装以下Python库：

pip install opencv-python mediapipe pynput pyautogui numpy pillow

模块介绍

1. utils.py

utils.py包含一个Utils类，主要提供在图像上添加中文文本的功能。这对于在实时视频流中显示信息非常有用。

代码解析

import cv2
import numpy as np
from PIL import Image, ImageDraw, ImageFontclass Utils:def __init__(self):passdef cv2AddChineseText(self, img, text, position, textColor=(0, 255, 0), textSize=30):if isinstance(img, np.ndarray):  # 判断是否OpenCV图片类型img = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))draw = ImageDraw.Draw(img)fontStyle = ImageFont.truetype("./fonts/simsun.ttc", textSize, encoding="utf-8")draw.text(position, text, textColor, font=fontStyle)return cv2.cvtColor(np.asarray(img), cv2.COLOR_RGB2BGR)

2. handProcess.py

handProcess.py负责手势的识别和处理。该模块使用MediaPipe库来检测和跟踪手部的关键点，并根据手势的不同动作触发相应的鼠标操作。

代码解析

import cv2
import mediapipe as mp
import time
import math
import numpy as np
from utils import Utilsclass HandProcess:def __init__(self, static_image_mode=False, max_num_hands=2):self.mp_drawing = mp.solutions.drawing_utilsself.mp_hands = mp.solutions.handsself.hands = self.mp_hands.Hands(static_image_mode=static_image_mode,min_detection_confidence=0.7,min_tracking_confidence=0.5,max_num_hands=max_num_hands)self.landmark_list = []self.action_labels = {'none': '无','move': '鼠标移动','click_single_active': '触发单击','click_single_ready': '单击准备','click_right_active': '触发右击','click_right_ready': '右击准备','scroll_up': '向上滑页','scroll_down': '向下滑页','drag': '鼠标拖拽'}self.action_deteted = ''def checkHandsIndex(self, handedness):if len(handedness) == 1:handedness_list = [handedness[0].classification[0].label]else:handedness_list = [handedness[0].classification[0].label, handedness[1].classification[0].label]return handedness_listdef getDistance(self, pointA, pointB):return math.hypot((pointA[0] - pointB[0]), (pointA[1] - pointB[1]))def getFingerXY(self, index):return (self.landmark_list[index][1], self.landmark_list[index][2])def drawInfo(self, img, action):thumbXY, indexXY, middleXY = map(self.getFingerXY, [4, 8, 12])if action == 'move':img = cv2.circle(img, indexXY, 20, (255, 0, 255), -1)elif action == 'click_single_active':middle_point = int((indexXY[0] + thumbXY[0]) / 2), int((indexXY[1] + thumbXY[1]) / 2)img = cv2.circle(img, middle_point, 30, (0, 255, 0), -1)elif action == 'click_single_ready':img = cv2.circle(img, indexXY, 20, (255, 0, 255), -1)img = cv2.circle(img, thumbXY, 20, (255, 0, 255), -1)img = cv2.line(img, indexXY, thumbXY, (255, 0, 255), 2)elif action == 'click_right_active':middle_point = int((indexXY[0] + middleXY[0]) / 2), int((indexXY[1] + middleXY[1]) / 2)img = cv2.circle(img, middle_point, 30, (0, 255, 0), -1)elif action == 'click_right_ready':img = cv2.circle(img, indexXY, 20, (255, 0, 255), -1)img = cv2.circle(img, middleXY, 20, (255, 0, 255), -1)img = cv2.line(img, indexXY, middleXY, (255, 0, 255), 2)return imgdef checkHandAction(self, img, drawKeyFinger=True):upList = self.checkFingersUp()action = 'none'if len(upList) == 0:return img, action, Nonedete_dist = 100key_point = self.getFingerXY(8)if upList == [0, 1, 0, 0, 0]:action = 'move'if upList == [1, 1, 0, 0, 0]:l1 = self.getDistance(self.getFingerXY(4), self.getFingerXY(8))action = 'click_single_active' if l1 < dete_dist else 'click_single_ready'if upList == [0, 1, 1, 0, 0]:l1 = self.getDistance(self.getFingerXY(8), self.getFingerXY(12))action = 'click_right_active' if l1 < dete_dist else 'click_right_ready'if upList == [1, 1, 1, 1, 1]:action = 'scroll_up'if upList == [0, 1, 1, 1, 1]:action = 'scroll_down'if upList == [0, 0, 1, 1, 1]:key_point = self.getFingerXY(12)action = 'drag'img = self.drawInfo(img, action) if drawKeyFinger else imgself.action_deteted = self.action_labels[action]return img, action, key_pointdef checkFingersUp(self):fingerTipIndexs = [4, 8, 12, 16, 20]upList = []if len(self.landmark_list) == 0:return upListif self.landmark_list[fingerTipIndexs[0]][1] < self.landmark_list[fingerTipIndexs[0] - 1][1]:upList.append(1)else:upList.append(0)for i in range(1, 5):if self.landmark_list[fingerTipIndexs[i]][2] < self.landmark_list[fingerTipIndexs[i] - 2][2]:upList.append(1)else:upList.append(0)return upListdef processOneHand(self, img, drawBox=True, drawLandmarks=True):utils = Utils()results = self.hands.process(img)self.landmark_list = []if results.multi_hand_landmarks:for hand_index, hand_landmarks in enumerate(results.multi_hand_landmarks):if drawLandmarks:self.mp_drawing.draw_landmarks(img, hand_landmarks, self.mp_hands.HAND_CONNECTIONS,self.mp_drawing_styles.get_default_hand_landmarks_style(),self.mp_drawing_styles.get_default_hand_connections_style())for landmark_id, finger_axis in enumerate(hand_landmarks.landmark):h, w, c = img.shapep_x, p_y = math.ceil(finger_axis.x * w), math.ceil(finger_axis.y * h)self.landmark_list.append([landmark_id, p_x, p_y, finger_axis.z])if drawBox:x_min, x_max = min(self.landmark_list, key=lambda i: i[1])[1], max(self.landmark_list, key=lambda i: i[1])[1]y_min, y_max = min(self.landmark_list, key=lambda i: i[2])[2], max(self.landmark_list, key=lambda i: i[2])[2]img = cv2.rectangle(img, (x_min - 30, y_min - 30), (x_max + 30, y_max + 30), (0, 255, 0), 2)img = utils.cv2AddChineseText(img, self.action_deteted, (x_min - 20, y_min - 120), textColor=(255, 0, 255), textSize=60)return img

3. virtual_mouse.py

virtual_mouse.py是主程序模块，整合了手势识别和鼠标控制功能，实现了通过手势控制鼠标移动、点击和滚动的功能。

代码解析

import cv2
import handProcess
import time
import numpy as np
import pyautogui
from utils import Utils
from pynput.mouse import Button, Controllerclass VirtualMouse:def __init__(self):self.image = Noneself.mouse = Controller()def recognize(self):handprocess = handProcess.HandProcess(False, 1)utils = Utils()fpsTime = time.time()cap = cv2.VideoCapture(0)resize_w = 960resize_h = 720frameMargin = 100screenWidth, screenHeight = pyautogui.size()stepX, stepY = 0, 0finalX, finalY = 0, 0smoothening = 7action_trigger_time = {'single_click': 0,'double_click': 0,'right_click': 0}mouseDown = Falsewhile cap.isOpened():action_zh = ''success, self.image = cap.read()self.image = cv2.resize(self.image, (resize_w, resize_h))if not success:print("空帧")continueself.image.flags.writeable = Falseself.image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)self.image = cv2.flip(self.image, 1)self.image = handprocess.processOneHand(self.image)cv2.rectangle(self.image, (frameMargin, frameMargin), (resize_w - frameMargin, resize_h - frameMargin), (255, 0, 255), 2)self.image, action, key_point = handprocess.checkHandAction(self.image, drawKeyFinger=True)action_zh = handprocess.action_labels[action]if key_point:x3 = np.interp(key_point[0], (frameMargin, resize_w - frameMargin), (0, screenWidth))y3 = np.interp(key_point[1], (frameMargin, resize_h - frameMargin), (0, screenHeight))finalX = stepX + (x3 - stepX) / smootheningfinalY = stepY + (y3 - stepY) / smootheningnow = time.time()if action_zh == '鼠标拖拽':if not mouseDown:self.mouse.press(Button.left)mouseDown = Trueself.mouse.position = (finalX, finalY)else:if mouseDown:self.mouse.release(Button.left)mouseDown = Falseif action_zh == '鼠标移动':self.mouse.position = (finalX, finalY)elif action_zh == '单击准备':passelif action_zh == '触发单击' and (now - action_trigger_time['single_click'] > 0.3):self.mouse.click(Button.left, 1)action_trigger_time['single_click'] = nowelif action_zh == '右击准备':passelif action_zh == '触发右击' and (now - action_trigger_time['right_click'] > 2):self.mouse.click(Button.right, 1)action_trigger_time['right_click'] = nowelif action_zh == '向上滑页':pyautogui.scroll(30)elif action_zh == '向下滑页':pyautogui.scroll(-30)stepX, stepY = finalX, finalYself.image.flags.writeable = Trueself.image = cv2.cvtColor(self.image, cv2.COLOR_RGB2BGR)cTime = time.time()fps_text = 1 / (cTime - fpsTime)fpsTime = cTimeself.image = utils.cv2AddChineseText(self.image, "帧率: " + str(int(fps_text)), (10, 30), textColor=(255, 0, 255), textSize=50)self.image = cv2.resize(self.image, (resize_w // 2, resize_h // 2))cv2.imshow('virtual mouse', self.image)if cv2.waitKey(5) & 0xFF == 27:breakcap.release()control = VirtualMouse()
control.recognize()

4. 功能列表

在这个虚拟鼠标控制项目中，通过识别不同的手势来触发相应的鼠标操作。以下是该项目中实现的主要功能及其对应的手势：

1. 鼠标移动

   • 手势：食指竖起（其他手指收回）。
   • 描述：食指指尖的移动映射到屏幕上的鼠标光标移动。

2. 单击准备

   • 手势：拇指和食指都竖起且未接触。
   • 描述：准备触发单击。

3. 触发单击

   • 手势：拇指和食指接触（捏合）。
   • 描述：触发一次鼠标左键单击。

4. 右击准备

   • 手势：食指和中指都竖起且未接触。
   • 描述：准备触发右击。

5. 触发右击

   • 手势：食指和中指接触（捏合）。
   • 描述：触发一次鼠标右键单击。

6. 鼠标拖拽

   • 手势：中指、无名指和小指竖起（拇指和食指收回）。
   • 描述：模拟鼠标左键按住并拖动。

7. 向上滚动

   • 手势：五指全部竖起。
   • 描述：触发页面向上滚动。

8. 向下滚动

   • 手势：除了拇指外，其他四指竖起。
   • 描述：触发页面向下滚动。

5.测试