Playable
This commit is contained in:
cubernetes
parent
a39279ee7f
commit
5737d511a4
Binary file not shown.
Binary file not shown.
268
track_hand.py
268
track_hand.py
|
|
@ -1,108 +1,202 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
# import time
|
import random
|
||||||
from typing import NoReturn
|
from enum import Enum
|
||||||
|
from typing import NoReturn, Generator
|
||||||
|
from types import ModuleType
|
||||||
|
from subprocess import Popen
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import mediapipe as mp
|
import mediapipe as mp
|
||||||
|
import cv2
|
||||||
|
from cv2 import VideoCapture
|
||||||
|
|
||||||
def main() -> NoReturn:
|
class FingerType(Enum):
|
||||||
cap = cv2.VideoCapture(0)
|
BASE = 0
|
||||||
mpHands = mp.solutions.hands
|
BASE_RIGHT = 1
|
||||||
hands = mpHands.Hands(
|
THUMB_BASE = 2
|
||||||
static_image_mode=False,
|
THUMB_KNUCKLE_1 = 3
|
||||||
max_num_hands=1,
|
THUMB_TIP = 4
|
||||||
min_detection_confidence=.5,
|
INDEX_BASE = 5
|
||||||
min_tracking_confidence=.5,
|
INDEX_KNUCKLE_1 = 6
|
||||||
)
|
INDEX_KNUCKLE_2 = 7
|
||||||
mpDraw = mp.solutions.drawing_utils
|
INDEX_TIP = 8
|
||||||
# pTime = 0
|
MIDDLE_BASE = 9
|
||||||
# cTime = 0
|
MIDDLE_KNUCKLE_1 = 10
|
||||||
|
MIDDLE_KNUCKLE_2 = 11
|
||||||
|
MIDDLE_TIP = 12
|
||||||
|
RING_BASE = 13
|
||||||
|
RING_KNUCKLE_1 = 14
|
||||||
|
RING_KNUCKLE_2 = 15
|
||||||
|
RING_TIP = 16
|
||||||
|
PINKY_BASE = 17
|
||||||
|
PINKY_KNUCKLE_1 = 18
|
||||||
|
PINKY_KNUCKLE_2 = 19
|
||||||
|
PINKY_TIP = 20
|
||||||
|
|
||||||
img = cv2.imread("42.png", 0)
|
def get_42_img(
|
||||||
if len(img.shape) == 2 or img.shape[2] == 1:
|
img_path: str,
|
||||||
|
margin_top: int,
|
||||||
|
margin_bottom: int,
|
||||||
|
margin_left: int,
|
||||||
|
margin_right: int,
|
||||||
|
) -> np.ndarray:
|
||||||
|
global img42_side_len
|
||||||
|
|
||||||
|
img: np.ndarray = cv2.imread(img_path, 0)
|
||||||
|
|
||||||
|
if len(img.shape) in [1, 2]:
|
||||||
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
|
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
|
||||||
|
|
||||||
img = cv2.flip(img, 1)
|
img = cv2.flip(img, 1)
|
||||||
|
|
||||||
height, width, _ = img.shape
|
img_height, img_width = img.shape[:2]
|
||||||
margin = 100
|
img = img[
|
||||||
top_margin = margin + 20
|
margin_top:img_height-margin_bottom,
|
||||||
bottom_margin = margin + 20
|
margin_left:img_width-margin_right,
|
||||||
left_margin = margin
|
]
|
||||||
right_margin = margin
|
|
||||||
img = img[top_margin:height-bottom_margin, left_margin:width-right_margin]
|
|
||||||
|
|
||||||
side_length = min(640, 480) // 3
|
b_top, b_bottom, b_left, b_right = [10]*4
|
||||||
overlay_resized = cv2.resize(img, (side_length, side_length))
|
img = cv2.copyMakeBorder(img, b_top, b_bottom, b_left, b_right, cv2.BORDER_CONSTANT, value=(0, 0, 0))
|
||||||
print(repr(overlay_resized))
|
|
||||||
input()
|
|
||||||
sys.exit()
|
|
||||||
|
|
||||||
|
img = cv2.resize(img, (img42_side_len, img42_side_len))
|
||||||
|
|
||||||
|
return img
|
||||||
|
|
||||||
|
mp_hands = mp.solutions.hands
|
||||||
|
mp_draw: ModuleType = mp.solutions.drawing_utils
|
||||||
|
|
||||||
|
img42_side_len = 70
|
||||||
|
img42: np.ndarray = get_42_img(
|
||||||
|
"./42.png",
|
||||||
|
margin_top = 100 + 20,
|
||||||
|
margin_bottom = 100 + 20,
|
||||||
|
margin_left = 100,
|
||||||
|
margin_right = 100,
|
||||||
|
)
|
||||||
|
|
||||||
|
def touches_42(x: int, y: int, img42_x: int, img42_y: int) -> bool:
|
||||||
|
global collected_42
|
||||||
|
|
||||||
|
return (
|
||||||
|
img42_x <= x <= img42_x + img42_side_len
|
||||||
|
and img42_y <= y <= img42_y + img42_side_len
|
||||||
|
)
|
||||||
|
|
||||||
|
def add_directional_triangle(
|
||||||
|
frame: np.ndarray,
|
||||||
|
x1: int,
|
||||||
|
y1: int,
|
||||||
|
x2: int,
|
||||||
|
y2: int,
|
||||||
|
rgb: tuple[int, int, int],
|
||||||
|
side_len: int,
|
||||||
|
stretch: float,
|
||||||
|
) -> tuple[int, int]:
|
||||||
|
dir_vector = np.array([
|
||||||
|
x1 - x2, y1 - y2
|
||||||
|
]).astype(np.float64)
|
||||||
|
|
||||||
|
# normalize
|
||||||
|
dir_vector /= np.linalg.norm(dir_vector)
|
||||||
|
|
||||||
|
triangle_height = side_len * (3**0.5) / 2
|
||||||
|
half_base = side_len / 2
|
||||||
|
|
||||||
|
perp_vector = np.array([-dir_vector[1], dir_vector[0]])
|
||||||
|
|
||||||
|
apex_vertex = (int(x1 + dir_vector[0] * triangle_height * 2/3 * stretch), int(y1 + dir_vector[1] * triangle_height * 2/3 * stretch))
|
||||||
|
left_vertex = (int(x1 - perp_vector[0] * half_base - dir_vector[0] * triangle_height/3),
|
||||||
|
int(y1 - perp_vector[1] * half_base - dir_vector[1] * triangle_height/3))
|
||||||
|
right_vertex = (int(x1 + perp_vector[0] * half_base - dir_vector[0] * triangle_height/3),
|
||||||
|
int(y1 + perp_vector[1] * half_base - dir_vector[1] * triangle_height/3))
|
||||||
|
|
||||||
|
triangle = np.array([apex_vertex, left_vertex, right_vertex])
|
||||||
|
cv2.drawContours(frame, [triangle], 0, rgb, -1)
|
||||||
|
|
||||||
|
return apex_vertex
|
||||||
|
|
||||||
|
def get_finger_positions(
|
||||||
|
frame: np.ndarray,
|
||||||
|
hands: mp.solutions.hands.Hands,
|
||||||
|
add_landmarks: bool,
|
||||||
|
) -> Generator[list[tuple[int, int, int]], None, None]:
|
||||||
|
height, width = frame.shape[:2]
|
||||||
|
|
||||||
|
img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||||
|
results = hands.process(img_rgb)
|
||||||
|
|
||||||
|
if results.multi_hand_landmarks:
|
||||||
|
for hand_landmarks in results.multi_hand_landmarks:
|
||||||
|
positions = []
|
||||||
|
for id, lm in enumerate(hand_landmarks.landmark):
|
||||||
|
x = int(lm.x * width)
|
||||||
|
y = int(lm.y * height)
|
||||||
|
positions.append((FingerType(id), x, y))
|
||||||
|
yield positions
|
||||||
|
if add_landmarks:
|
||||||
|
mp_draw.draw_landmarks(frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
||||||
|
|
||||||
|
def show_frame(frame: np.ndarray, to_stdout: bool=False) -> None:
|
||||||
|
if to_stdout:
|
||||||
|
sys.stdout.buffer.write(frame.tobytes())
|
||||||
|
else:
|
||||||
|
cv2.imshow("Image", frame)
|
||||||
|
cv2.waitKey(1)
|
||||||
|
|
||||||
|
def collect_sfx() -> None:
|
||||||
|
Popen(['paplay', './sfx/collect.mp3'])
|
||||||
|
|
||||||
|
def main() -> NoReturn:
|
||||||
|
Popen(['paplay', './sfx/start.mp3'])
|
||||||
|
|
||||||
|
capture: VideoCapture = cv2.VideoCapture(0)
|
||||||
|
hands = mp_hands.Hands(max_num_hands=2)
|
||||||
|
collected_42 = True
|
||||||
|
img42_x = -img42_side_len - 1
|
||||||
|
img42_y = -img42_side_len - 1
|
||||||
|
|
||||||
|
i = 0
|
||||||
while True:
|
while True:
|
||||||
success, frame = cap.read()
|
success: bool
|
||||||
|
frame: np.ndarray
|
||||||
|
success, frame = capture.read()
|
||||||
if not success:
|
if not success:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
imgRGB = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
if i > 30:
|
||||||
results = hands.process(imgRGB)
|
if collected_42:
|
||||||
|
collected_42 = False
|
||||||
h, w, c = frame.shape
|
frame_height, frame_width = frame.shape[:2]
|
||||||
|
img42_x = random.randint(0, frame_width - img42_side_len - 1)
|
||||||
img_x, img_y = 10, 10
|
img42_y = random.randint(0, frame_height - img42_side_len - 1)
|
||||||
frame[img_y:img_y+side_length, img_x:img_x+side_length] = overlay_resized
|
frame[
|
||||||
|
img42_y : img42_y+img42_side_len,
|
||||||
if results.multi_hand_landmarks:
|
img42_x : img42_x+img42_side_len,
|
||||||
for handLms in results.multi_hand_landmarks:
|
] = img42
|
||||||
visited_7 = False
|
|
||||||
for id, lm in enumerate(handLms.landmark):
|
|
||||||
cx = int(lm.x * w)
|
|
||||||
cy = int(lm.y * h)
|
|
||||||
# 7 one-before-index, 8 index, 4 thumb, 0 base
|
|
||||||
if id == 7:
|
|
||||||
one_before_index_x = cx
|
|
||||||
one_before_index_y = cy
|
|
||||||
visited_7 = True
|
|
||||||
if id == 8:
|
|
||||||
pass
|
|
||||||
# open('/dev/pts/1', 'w').write(
|
|
||||||
# f'{id}, {cx}, {cy}\n'
|
|
||||||
# )
|
|
||||||
|
|
||||||
if visited_7:
|
|
||||||
dir_vector = np.array([cx - one_before_index_x, cy - one_before_index_y])
|
|
||||||
dir_vector = dir_vector.astype(np.float64)
|
|
||||||
dir_vector /= np.linalg.norm(dir_vector)
|
|
||||||
|
|
||||||
s = 100
|
|
||||||
h = s * (3**0.5) / 2
|
|
||||||
half_base = s / 2
|
|
||||||
|
|
||||||
perp_vector = np.array([-dir_vector[1], dir_vector[0]])
|
|
||||||
|
|
||||||
pt1 = (int(cx + dir_vector[0] * h * 2/3), int(cy + dir_vector[1] * h * 2/3))
|
|
||||||
pt2 = (int(cx - perp_vector[0] * half_base - dir_vector[0] * h/3),
|
|
||||||
int(cy - perp_vector[1] * half_base - dir_vector[1] * h/3))
|
|
||||||
pt3 = (int(cx + perp_vector[0] * half_base - dir_vector[0] * h/3),
|
|
||||||
int(cy + perp_vector[1] * half_base - dir_vector[1] * h/3))
|
|
||||||
|
|
||||||
triangle_cnt = np.array([pt1, pt2, pt3])
|
|
||||||
cv2.drawContours(frame, [triangle_cnt], 0, (0,0,0), -1)
|
|
||||||
else:
|
|
||||||
cv2.circle(frame, (cx, cy), 40, (0, 0, 0), cv2.FILLED)
|
|
||||||
mpDraw.draw_landmarks(frame, handLms, mpHands.HAND_CONNECTIONS)
|
|
||||||
|
|
||||||
# cTime = time.time()
|
|
||||||
# fps = 1 / (cTime - pTime)
|
|
||||||
# pTime = cTime
|
|
||||||
|
|
||||||
# cv2.putText(frame, str(int(fps)), (10,70), cv2.FONT_HERSHEY_PLAIN, 3, (255, 0, 255), 3)
|
|
||||||
|
|
||||||
sys.stdout.buffer.write(frame.tobytes())
|
|
||||||
# cv2.imshow("Image", frame)
|
|
||||||
# cv2.waitKey(1)
|
|
||||||
|
|
||||||
|
for positions in get_finger_positions(frame, hands, add_landmarks=True):
|
||||||
|
index_knuckle_1_pos: tuple[int, int] = (-1, -1)
|
||||||
|
for finger_id, finger_x, finger_y in positions:
|
||||||
|
if finger_id == FingerType.INDEX_KNUCKLE_2:
|
||||||
|
index_knuckle_1_pos = (finger_x, finger_y)
|
||||||
|
elif finger_id == FingerType.INDEX_TIP and index_knuckle_1_pos != (-1, -1):
|
||||||
|
apex_x, apex_y = add_directional_triangle(
|
||||||
|
frame,
|
||||||
|
finger_x,
|
||||||
|
finger_y,
|
||||||
|
*index_knuckle_1_pos,
|
||||||
|
(0, 0, 0,),
|
||||||
|
side_len=70,
|
||||||
|
stretch=2.0,
|
||||||
|
)
|
||||||
|
if not collected_42 and touches_42(apex_x, apex_y, img42_x, img42_y):
|
||||||
|
collected_42 = True
|
||||||
|
i = 0
|
||||||
|
collect_sfx()
|
||||||
|
show_frame(frame, to_stdout=True)
|
||||||
|
i += 1
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,36 @@
|
||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# xterm -bg black -fg white -s -fullscreen -fa 'SauceCodePro Nerd Font Mono' -fs 8 -e '{ ./track_hand.py | 2>/dev/null ffmpeg -y -f rawvideo -s 640x480 -pix_fmt bgr24 -i - -map 0:V:0 -filter:v "format=gray,hflip" -c:v libx264 -preset ultrafast -tune zerolatency -crf 30 -f nut - | TERM=xterm-mono CACA_DRIVER=ncurses DISPLAY= mpv --really-quiet --no-cache --no-config --vo=caca --untimed --profile=low-latency - || { echo Error 1>&2; read X; }; }'
|
|
||||||
# st -f 'SauceCodePro Nerd Font Mono:size=10' -e sh -c '{ ./track_hand.py | 2>/dev/null ffmpeg -y -f rawvideo -s 640x480 -pix_fmt bgr24 -i - -map 0:V:0 -filter:v "format=gray,hflip" -c:v libx265 -preset ultrafast -tune zerolatency -crf 30 -f nut - | TERM=xterm-mono CACA_DRIVER=ncurses DISPLAY= mpv --really-quiet --no-cache --no-config --vo=caca --untimed --profile=low-latency - || { echo Error 1>&2; read X; }; } | ./game.py'
|
# st -f 'SauceCodePro Nerd Font Mono:size=10' -e sh -c '{ ./track_hand.py | 2>/dev/null ffmpeg -y -f rawvideo -s 640x480 -pix_fmt bgr24 -i - -map 0:V:0 -filter:v "format=gray,hflip" -c:v libx265 -preset ultrafast -tune zerolatency -crf 30 -f nut - | TERM=xterm-mono CACA_DRIVER=ncurses DISPLAY= mpv --really-quiet --no-cache --no-config --vo=caca --untimed --profile=low-latency - || { echo Error 1>&2; read X; }; } | ./game.py'
|
||||||
st -f 'SauceCodePro Nerd Font Mono:size=10' -e sh -c '{ ./track_hand.py | 2>/dev/null ffmpeg -y -f rawvideo -s 640x480 -pix_fmt bgr24 -i - -map 0:V:0 -filter:v "format=gray,hflip" -c:v libx265 -preset ultrafast -tune zerolatency -crf 30 -f nut - | TERM=xterm-mono CACA_DRIVER=ncurses DISPLAY= mpv --really-quiet --no-cache --no-config --vo=caca --untimed --profile=low-latency - || { echo Error 1>&2; read X; }; }'
|
# st -f 'SauceCodePro Nerd Font Mono:size=10' -e sh -c '{ ./track_hand.py | 2>/dev/null ffmpeg -y -f rawvideo -s 640x480 -pix_fmt bgr24 -i - -map 0:V:0 -filter:v "format=gray,hflip" -c:v libx265 -preset ultrafast -tune zerolatency -crf 30 -f nut - | TERM=xterm-mono CACA_DRIVER=ncurses DISPLAY= mpv --really-quiet --no-cache --no-config --vo=caca --untimed --profile=low-latency - || { echo Error 1>&2; read X; }; }'
|
||||||
# st -f 'SauceCodePro Nerd Font Mono:size=10' -e sh -c '{ ./track_hand.py | 2>/dev/null ffmpeg -y -f rawvideo -s 640x480 -pix_fmt bgr24 -i - -map 0:V:0 -filter:v "format=gray,hflip" -c:v libx264 -preset ultrafast -tune zerolatency -crf 30 -f nut - | TERM=xterm-mono CACA_DRIVER=ncurses DISPLAY= mpv --really-quiet --no-cache --no-config --vo=tct --untimed --profile=low-latency - || { echo Error 1>&2; read X; }; }'
|
# st -f 'SauceCodePro Nerd Font Mono:size=10' -e sh -c '{ ./track_hand.py | 2>/dev/null ffmpeg -y -f rawvideo -s 640x480 -pix_fmt bgr24 -i - -map 0:V:0 -filter:v "format=gray,hflip" -c:v libx264 -preset ultrafast -tune zerolatency -crf 30 -f nut - | TERM=xterm-mono CACA_DRIVER=ncurses DISPLAY= mpv --really-quiet --no-cache --no-config --vo=tct --untimed --profile=low-latency - || { echo Error 1>&2; read X; }; }'
|
||||||
|
|
||||||
|
xterm \
|
||||||
|
-bg black \
|
||||||
|
-fg white \
|
||||||
|
-s -fullscreen \
|
||||||
|
-fa 'SauceCodePro Nerd Font Mono' \
|
||||||
|
-fs 8 \
|
||||||
|
-e '{
|
||||||
|
./track_hand.py |
|
||||||
|
2>/dev/null ffmpeg -y \
|
||||||
|
-f rawvideo \
|
||||||
|
-s 640x480 \
|
||||||
|
-pix_fmt bgr24 \
|
||||||
|
-i - \
|
||||||
|
-map 0:V:0 \
|
||||||
|
-filter:v "format=gray,hflip" \
|
||||||
|
-c:v libx265 \
|
||||||
|
-preset ultrafast \
|
||||||
|
-tune zerolatency \
|
||||||
|
-crf 30 \
|
||||||
|
-f nut \
|
||||||
|
- |
|
||||||
|
TERM=xterm-mono CACA_DRIVER=ncurses DISPLAY= mpv \
|
||||||
|
--really-quiet \
|
||||||
|
--no-cache \
|
||||||
|
--no-config \
|
||||||
|
--vo=caca \
|
||||||
|
--untimed \
|
||||||
|
--profile=low-latency \
|
||||||
|
- \
|
||||||
|
|| { echo Error 1>&2; read X; };
|
||||||
|
}'
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue