face_controller.py

import cv2
import numpy as np
import pyautogui as gui
import time
import os
import sys
import json
from pathlib import Path

# Set keypress delay to 0.
gui.PAUSE = 0

# Config file for customizable settings
CONFIG_FILE = Path('./config.json')
DEFAULT_CONFIG = {
    'confidence_threshold': 0.5,
    'control_box_size': (220, 240),  # width, height
    'use_keyboard_alternative': True,
    'keyboard_controls': {
        'up': 'w',
        'down': 's',
        'left': 'a',
        'right': 'd'
    },
    'click_position': (500, 500),
    'feathering': {
        'enabled': True,
        'feather_duration': 0.1,  # seconds to hold key down for feathered presses
        'feather_threshold': 50   # pixels from edge to trigger feathering
    }
}

# Global variables
config = {}
last_mov = ''
keyboard_mode = False
diagonal_movement = False
current_keys_pressed = []  # Track which keys are currently being pressed

def load_config():
    """Load user configuration from file or create with defaults if not exists."""
    global config
    try:
        if CONFIG_FILE.exists():
            with open(CONFIG_FILE, 'r') as f:
                config = json.load(f)
            print("Configuration loaded successfully.")
        else:
            config = DEFAULT_CONFIG
            with open(CONFIG_FILE, 'w') as f:
                json.dump(config, f, indent=4)
            print("Default configuration created.")
    except Exception as e:
        print(f"Error loading configuration: {e}")
        config = DEFAULT_CONFIG
        print("Using default configuration.")

# Model paths with error handling
def get_model_paths():
    """Get paths to the model files with error handling."""
    model_dir = Path('./model')
    model_path = model_dir / 'res10_300x300_ssd_iter_140000.caffemodel'
    prototxt_path = model_dir / 'deploy.prototxt'
    
    # Check if model directory and files exist
    if not model_dir.exists():
        print(f"Error: Model directory not found at {model_dir}")
        print("Creating model directory...")
        os.makedirs(model_dir, exist_ok=True)
        print("\nPlease download the model files:")
        print("1. res10_300x300_ssd_iter_140000.caffemodel")
        print("2. deploy.prototxt")
        print(f"And place them in: {model_dir}\n")
        return None, None
        
    if not model_path.exists() or not prototxt_path.exists():
        print(f"Error: Model files not found in {model_dir}")
        print("\nPlease download the missing model files:")
        if not model_path.exists():
            print(f"- res10_300x300_ssd_iter_140000.caffemodel")
        if not prototxt_path.exists():
            print(f"- deploy.prototxt")
        print(f"And place them in: {model_dir}\n")
        return None, None
        
    return str(model_path), str(prototxt_path)

def detect(net, frame):
    '''
    Detect the faces in the frame.

    returns: list of faces in the frame
                here each face is a dictionary of format-
                {'start': (startX,startY), 'end': (endX,endY), 'confidence': confidence}
    '''
    detected_faces = []
    (h, w) = frame.shape[:2]
    blob = cv2.dnn.blobFromImage(
        cv2.resize(frame, (300, 300)),
        1.0,
        (300, 300),
        (104.0, 177.0, 123.0))
    net.setInput(blob)
    detections = net.forward()
    for i in range(0, detections.shape[2]):
        confidence = detections[0, 0, i, 2]
        if confidence > config['confidence_threshold']:
            box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
            (startX, startY, endX, endY) = box.astype("int")
            detected_faces.append({
                'start': (startX, startY),
                'end': (endX, endY),
                'confidence': confidence})
    return detected_faces


def drawFace(frame, detected_faces):
    '''
    Draw rectangular box over detected faces.

    returns: frame with rectangular boxes over detected faces.
    '''
    for face in detected_faces:
        cv2.rectangle(frame, face['start'], face['end'], (0, 255, 0), 10)
        # Add confidence text
        confidence = face['confidence']
        y = face['start'][1] - 10 if face['start'][1] - 10 > 10 else face['start'][1] + 10
        cv2.putText(frame, f"{confidence:.2f}", face['start'],
                    cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2)
    return frame


def checkRect(detected_faces, bbox):
    '''
    Check for a detected face inside the bounding box at the center of the frame.

    returns: True or False.
    '''
    for face in detected_faces:
        x1, y1 = face['start']
        x2, y2 = face['end']
        if x1 > bbox[0] and x2 < bbox[1]:
            if y1 > bbox[3] and y2 < bbox[2]:
                return True
    return False


def move(detected_faces, bbox):
    '''
    Press or hold correct button depending on the position of detected face and bbox.
    
    Instead of just pressing keys once, this now continuously holds keys down
    until the face position changes. Implements feathering for left/right controls.
    '''
    global last_mov, diagonal_movement, current_keys_pressed
    movement_detected = False
    new_keys_to_press = []
    
    for face in detected_faces:
        movement_detected = True
        x1, y1 = face['start']
        x2, y2 = face['end']

        # Center
        if checkRect(detected_faces, bbox):
            if last_mov != 'center':
                # Release any keys that were being pressed
                for key in current_keys_pressed:
                    gui.keyUp(key)
                current_keys_pressed = []
                last_mov = 'center'
                diagonal_movement = False
            return

        # Track horizontal and vertical movements separately to detect diagonals
        horizontal_move = None
        vertical_move = None
        feathered_move = False
        
        # Check horizontal movement with feathering
        if config.get('feathering', {}).get('enabled', False):
            feather_threshold = config.get('feathering', {}).get('feather_threshold', 50)
            
            # Left movement with feathering
            if x1 < bbox[0]:
                if bbox[0] - x1 <= feather_threshold:
                    horizontal_move = 'left'
                    feathered_move = True
                else:
                    horizontal_move = 'left'
            # Right movement with feathering
            elif x2 > bbox[1]:
                if x2 - bbox[1] <= feather_threshold:
                    horizontal_move = 'right'
                    feathered_move = True
                else:
                    horizontal_move = 'right'
        else:
            # Original horizontal movement logic without feathering
            if x1 < bbox[0]:
                horizontal_move = 'left'
            elif x2 > bbox[1]:
                horizontal_move = 'right'
            
        # Check vertical movement
        if y2 > bbox[2]:
            vertical_move = 'down'
        elif y1 < bbox[3]:
            vertical_move = 'up'
        
        # Determine which keys need to be pressed
        if horizontal_move:
            new_keys_to_press.append(horizontal_move)
        if vertical_move:
            new_keys_to_press.append(vertical_move)
        
        # Handle movement changes
        if new_keys_to_press != current_keys_pressed:
            # Release keys that are no longer needed
            for key in current_keys_pressed:
                if key not in new_keys_to_press:
                    gui.keyUp(key)
            
            # Press new keys
            for key in new_keys_to_press:
                if key not in current_keys_pressed:
                    # For feathered moves, tap the key briefly instead of holding
                    if feathered_move and (key == 'left' or key == 'right'):
                        feather_duration = config.get('feathering', {}).get('feather_duration', 0.1)
                        gui.keyDown(key)
                        time.sleep(feather_duration)
                        gui.keyUp(key)
                    else:
                        gui.keyDown(key)
            
            # Update tracking variables - only track non-feathered keys as pressed
            if feathered_move:
                # Remove feathered keys from tracking since they're not being held
                current_keys_pressed = [k for k in new_keys_to_press if not (feathered_move and (k == 'left' or k == 'right'))]
            else:
                current_keys_pressed = new_keys_to_press.copy()
            
            # Update movement display and logging
            if len(new_keys_to_press) > 1:
                # Diagonal movement
                diagonal_movement = True
                if 'up' in new_keys_to_press and 'left' in new_keys_to_press:
                    last_mov = 'up_left'
                elif 'up' in new_keys_to_press and 'right' in new_keys_to_press:
                    last_mov = 'up_right'
                elif 'down' in new_keys_to_press and 'left' in new_keys_to_press:
                    last_mov = 'down_left'
                elif 'down' in new_keys_to_press and 'right' in new_keys_to_press:
                    last_mov = 'down_right'
                print(f"Diagonal: {last_mov}")
            elif len(new_keys_to_press) == 1:
                # Single direction
                diagonal_movement = False
                last_mov = new_keys_to_press[0]
                if feathered_move:
                    print(f"Feathered: {last_mov}")
                else:
                    print(last_mov)
            else:
                # No movement (should not happen in normal flow)
                diagonal_movement = False
                last_mov = 'center'
    
    # If no face detected for a while, reset to center and release all keys
    if not movement_detected and current_keys_pressed:
        for key in current_keys_pressed:
            gui.keyUp(key)
        current_keys_pressed = []
        last_mov = 'center'
        diagonal_movement = False


def handle_keyboard_input(key):
    """Handle keyboard input as an alternative control method."""
    global keyboard_mode, diagonal_movement, current_keys_pressed
    
    # Toggle between face and keyboard control
    if key == ord('m'):
        keyboard_mode = not keyboard_mode
        # Release any keys that were being pressed when switching modes
        if keyboard_mode and current_keys_pressed:
            for key in current_keys_pressed:
                gui.keyUp(key)
            current_keys_pressed = []
        return True
    
    # Handle movement keys when in keyboard mode
    if keyboard_mode:
        # Check for diagonal key combinations
        keys_pressed = []
        
        # Check individual keys
        if key == ord(config['keyboard_controls']['up']):
            keys_pressed.append('up')
        elif key == ord(config['keyboard_controls']['down']):
            keys_pressed.append('down')
        elif key == ord(config['keyboard_controls']['left']):
            keys_pressed.append('left')
        elif key == ord(config['keyboard_controls']['right']):
            keys_pressed.append('right')
            
        # If a key is pressed, simulate the movement
        if keys_pressed:
            for direction in keys_pressed:
                gui.press(direction)
            return True
    
    # Exit on escape key with confirmation
    if key == 27:  # ESC key
        return False
        
    return True


def draw_help_overlay(frame):
    """Draw helpful information overlay on the frame."""
    h, w = frame.shape[:2]
    overlay = frame.copy()

    # Add semi-transparent overlay at the bottom, narrower window
    overlay_height = 130
    overlay_width = w // 2
    overlay_x = (w - overlay_width) // 2
    cv2.rectangle(overlay, (overlay_x, h - overlay_height), (overlay_x + overlay_width, h), (0, 0, 0), -1)
    alpha = 0.7
    frame = cv2.addWeighted(overlay, alpha, frame, 1 - alpha, 0)

    # Add control instructions at the bottom
    mode_text = "KEYBOARD MODE" if keyboard_mode else "FACE DETECTION MODE"
    cv2.putText(frame, mode_text, (overlay_x + 10, h - overlay_height + 25), 
                cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 255), 2)

    cv2.putText(frame, "Press 'M' to toggle control mode", (overlay_x + 10, h - overlay_height + 55), 
                cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)
    cv2.putText(frame, "Press 'ESC' to exit", (overlay_x + 10, h - overlay_height + 80), 
                cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)

    return frame


def play():
    '''
    Run the main loop until cancelled.
    '''
    global last_mov, keyboard_mode, diagonal_movement, current_keys_pressed
    
    # Load configuration
    load_config()
    
    # Get model paths with error handling
    model_path, prototxt_path = get_model_paths()
    if model_path is None or prototxt_path is None:
        print("Exiting due to missing model files.")
        return
    
    # Used to record the time when we processed last frame.
    prev_frame_time = 0
    # Used to record the time at which we processed current frame.
    new_frame_time = 0

    # Initialize face detection network
    try:
        net = cv2.dnn.readNetFromCaffe(prototxt_path, model_path)
        print("Face detection model loaded successfully.")
    except Exception as e:
        print(f"Error loading face detection model: {e}")
        print("Please make sure the model files are valid.")
        return

    # Initialize camera with error handling
    cap = cv2.VideoCapture(0)
    if not cap.isOpened():
        print("Error: Unable to access the camera.")
        print("Please check your camera connection and permissions.")
        
        # Try other camera indices
        for i in range(1, 3):
            print(f"Trying camera index {i}...")
            cap = cv2.VideoCapture(i)
            if cap.isOpened():
                print(f"Camera found at index {i}")
                break
        
        if not cap.isOpened():
            print("No camera found. Exiting.")
            return

    # Counter for skipping frame.
    count = 0

    # Used to initialize the game.
    init = 0

    # Getting the Frame width and height.
    frame_width, frame_height = int(cap.get(3)), int(cap.get(4))

    # Co-ordinates of the bounding box on frame
    box_width, box_height = config['control_box_size']
    left_x = frame_width // 2 - box_width // 2
    right_x = frame_width // 2 + box_width // 2
    top_y = frame_height // 2 - box_height // 2
    bottom_y = frame_height // 2 + box_height // 2
    bbox = [left_x, right_x, bottom_y, top_y]

    print("\n=== Face-controlled Game ===")
    print("Put your face inside the red rectangle to initialize.")
    print("Move your face to control the game.")
    print("Press 'M' to toggle between face detection and keyboard controls.")
    print("Press 'ESC' to exit the game.")
    print("==============================\n")

    # Main game loop
    while True:
        fps = 0
        ret, frame = cap.read()

        if not ret:
            print("Error: Failed to capture frame from camera.")
            print("Attempting to reconnect...")
            cap.release()
            cap = cv2.VideoCapture(0)
            if not cap.isOpened():
                print("Could not reconnect to camera. Exiting.")
                return

        frame = cv2.flip(frame, 1)
        
        # Only process face detection if not in keyboard mode
        if not keyboard_mode:
            # Detect the face.
            detected_faces = detect(net, frame)
            # Draw bounding box around detected faces.
            frame = drawFace(frame, detected_faces)
        else:
            detected_faces = []
        
        # Drawing the control rectangle in the center of the frame.
        rect_color = (0, 255, 255) if keyboard_mode else (0, 0, 255)
        frame = cv2.rectangle(
            frame, (left_x, top_y), (right_x, bottom_y), rect_color, 5)
        
        # Draw movement prompt above bounding box
        if last_mov != 'center' and last_mov != '':
            movement_text = last_mov.upper().replace('_', '+')
            feathered_text = " (FEATHERED)" if 'feathered' in last_mov.lower() else ''
            cv2.putText(frame, f"Moving: {movement_text}{feathered_text}", (left_x, top_y - 10),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2)

        # Skipping every alternate frame.
        if count % 2 == 0:
            # For first pass.
            if init == 0:
                # If face is inside the control rectangle or keyboard mode is active.
                if keyboard_mode or checkRect(detected_faces, bbox):
                    init = 1
                    # Move game running text to bottom of frame
                    h, w = frame.shape[:2]
                    cv2.putText(
                        frame, 'Game is running', (w//2 - 100, h - 150),
                        cv2.FONT_HERSHEY_SIMPLEX, 1.3, (0, 255, 0), 3)
                    cv2.waitKey(10)
                    last_mov = 'center'
                    # Click to start the game.
                    click_pos = config['click_position']
                    gui.click(x=click_pos[0], y=click_pos[1])
                    print(f"Game started with click at position {click_pos}")
            else:
                if not keyboard_mode:
                    move(detected_faces, bbox)
                cv2.waitKey(50)
                
        # Calculating the FPS.
        new_frame_time = time.time()
        fps = int(1 / (new_frame_time - prev_frame_time))
        prev_frame_time = new_frame_time

        # Add FPS counter to bottom right instead of top right
        h, w = frame.shape[:2]
        frame = cv2.putText(
            frame, str(fps) + ' FPS', (w-150, h - 30),
            cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 255), 3)
            
        # Add help overlay
        frame = draw_help_overlay(frame)
        
        # Resize the frame to increase webcam window size
        frame = cv2.resize(frame, (800, 600))
        
        cv2.imshow('Face-Controlled Game', frame)
        count += 1

        # Handle keyboard input
        k = cv2.waitKey(5)
        if not handle_keyboard_input(k):
            print("Exiting game. Thank you for playing!")
            break

    # Clean up
    cap.release()
    
    # Make sure to release any keys that might still be pressed
    for key in current_keys_pressed:
        gui.keyUp(key)
    current_keys_pressed = []
    
    cv2.destroyAllWindows()


if __name__ == "__main__":
    last_mov = ''
    keyboard_mode = False
    diagonal_movement = False
    current_keys_pressed = []
    
    try:
        play()
    except Exception as e:
        print(f"An error occurred: {e}")
        print("Game terminated unexpectedly.")
        
        # Make sure to release any keys that might still be pressed
        for key in current_keys_pressed:
            gui.keyUp(key)
        
        cv2.destroyAllWindows()