init: animation pipeline hub — Mac↔PC bridge for converting and implementing animations
Skills (.claude/skills/): animation-creation, iclone-video-mocap, pose-estimation (MediaPipe models via LFS), retarget-animations (deprecated). Tools: cc/mixamo/kevin/mocap retargeters + composite baker. Plans: animation-gen-pipeline + skills-adoption. exchange/: incoming-fbx, converted-glb, reference-video (LFS for fbx/glb/mp4). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract human pose landmarks from a video (or single image) with MediaPipe Pose
|
||||
(Tasks API, mediapipe >= 0.10.30 — the legacy `solutions` API no longer exists).
|
||||
|
||||
Outputs one JSON document with per-sample 2D landmarks (normalized image coords)
|
||||
and 3D world landmarks (meters, hip-centered) — the world landmarks are what you
|
||||
want for pose comparison because they are camera-scale invariant.
|
||||
|
||||
Sampling: --fps N samples the video at N Hz (default 10). --times "0.0,0.5,1.0"
|
||||
samples at explicit timestamps instead (e.g. a musical beat grid). --overlay
|
||||
writes the sampled frames with the skeleton drawn to an .mp4 for eyeball checks.
|
||||
|
||||
Works on real footage and (usually) on stylized humanoid game characters, but
|
||||
detection confidence drops on non-photoreal rigs — lower --min-conf and try
|
||||
--model heavy if frames come back undetected.
|
||||
"""
|
||||
import argparse, json, os, sys
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import mediapipe as mp
|
||||
from mediapipe.tasks import python as mp_tasks
|
||||
from mediapipe.tasks.python import vision
|
||||
|
||||
SKILL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
# The 33 pose landmarks, in model output order (fixed contract of the pose model).
|
||||
POSE_LANDMARKS = [
|
||||
"nose", "left_eye_inner", "left_eye", "left_eye_outer", "right_eye_inner",
|
||||
"right_eye", "right_eye_outer", "left_ear", "right_ear", "mouth_left",
|
||||
"mouth_right", "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
||||
"left_wrist", "right_wrist", "left_pinky", "right_pinky", "left_index",
|
||||
"right_index", "left_thumb", "right_thumb", "left_hip", "right_hip",
|
||||
"left_knee", "right_knee", "left_ankle", "right_ankle", "left_heel",
|
||||
"right_heel", "left_foot_index", "right_foot_index",
|
||||
]
|
||||
|
||||
# Skeleton edges for overlay drawing (standard MediaPipe pose connections, torso+limbs).
|
||||
CONNECTIONS = [
|
||||
(11, 12), (11, 13), (13, 15), (12, 14), (14, 16), # shoulders + arms
|
||||
(11, 23), (12, 24), (23, 24), # torso
|
||||
(23, 25), (25, 27), (27, 29), (27, 31), # left leg
|
||||
(24, 26), (26, 28), (28, 30), (28, 32), # right leg
|
||||
(15, 17), (15, 19), (15, 21), (16, 18), (16, 20), (16, 22), # hands
|
||||
]
|
||||
|
||||
|
||||
def lms_to_list(lms):
|
||||
return [
|
||||
{"name": POSE_LANDMARKS[i],
|
||||
"x": round(p.x, 5), "y": round(p.y, 5), "z": round(p.z, 5),
|
||||
"visibility": round(p.visibility or 0.0, 3)}
|
||||
for i, p in enumerate(lms)
|
||||
]
|
||||
|
||||
|
||||
def draw_skeleton(frame_bgr, lms, t, detected):
|
||||
h, w = frame_bgr.shape[:2]
|
||||
if detected:
|
||||
pts = [(int(p.x * w), int(p.y * h)) for p in lms]
|
||||
for a, b in CONNECTIONS:
|
||||
cv2.line(frame_bgr, pts[a], pts[b], (0, 220, 255), 2)
|
||||
for x, y in pts:
|
||||
cv2.circle(frame_bgr, (x, y), 3, (0, 255, 0), -1)
|
||||
cv2.putText(frame_bgr, f"t={t:.2f}s {'OK' if detected else 'NO POSE'}",
|
||||
(10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.8,
|
||||
(0, 255, 0) if detected else (0, 0, 255), 2)
|
||||
return frame_bgr
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("input", help="video file (.mp4/.mov/...) or single image")
|
||||
ap.add_argument("--out", required=True, help="output JSON path")
|
||||
ap.add_argument("--fps", type=float, default=10.0, help="sample rate in Hz (default 10)")
|
||||
ap.add_argument("--times", default=None, help="comma-separated timestamps in seconds (overrides --fps)")
|
||||
ap.add_argument("--overlay", default=None, help="optional .mp4 path: sampled frames with skeleton drawn")
|
||||
ap.add_argument("--model", default="full", choices=["lite", "full", "heavy"],
|
||||
help="pose model variant (heavy = most accurate; auto-downloads)")
|
||||
ap.add_argument("--min-conf", type=float, default=0.5,
|
||||
help="min detection/tracking confidence (lower for stylized characters)")
|
||||
args = ap.parse_args()
|
||||
|
||||
model_path = os.path.join(SKILL_DIR, "models", f"pose_landmarker_{args.model}.task")
|
||||
if not os.path.exists(model_path):
|
||||
url = (f"https://storage.googleapis.com/mediapipe-models/pose_landmarker/"
|
||||
f"pose_landmarker_{args.model}/float16/latest/pose_landmarker_{args.model}.task")
|
||||
os.makedirs(os.path.dirname(model_path), exist_ok=True)
|
||||
print(f"downloading model {args.model}...", file=sys.stderr)
|
||||
import subprocess
|
||||
subprocess.run(["curl", "-sL", "-o", model_path, url], check=True)
|
||||
|
||||
is_image = os.path.splitext(args.input)[1].lower() in (".png", ".jpg", ".jpeg", ".webp", ".bmp")
|
||||
|
||||
options = vision.PoseLandmarkerOptions(
|
||||
base_options=mp_tasks.BaseOptions(model_asset_path=model_path),
|
||||
running_mode=vision.RunningMode.IMAGE if is_image else vision.RunningMode.VIDEO,
|
||||
min_pose_detection_confidence=args.min_conf,
|
||||
min_pose_presence_confidence=args.min_conf,
|
||||
min_tracking_confidence=args.min_conf,
|
||||
)
|
||||
landmarker = vision.PoseLandmarker.create_from_options(options)
|
||||
|
||||
frames_out, overlay_frames = [], []
|
||||
|
||||
def process(frame_bgr, t, idx):
|
||||
mp_img = mp.Image(image_format=mp.ImageFormat.SRGB,
|
||||
data=cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB))
|
||||
if is_image:
|
||||
res = landmarker.detect(mp_img)
|
||||
else:
|
||||
res = landmarker.detect_for_video(mp_img, int(t * 1000))
|
||||
detected = bool(res.pose_landmarks)
|
||||
entry = {"t": round(t, 4), "index": idx, "detected": detected}
|
||||
if detected:
|
||||
entry["landmarks"] = lms_to_list(res.pose_landmarks[0])
|
||||
if res.pose_world_landmarks:
|
||||
entry["world_landmarks"] = lms_to_list(res.pose_world_landmarks[0])
|
||||
frames_out.append(entry)
|
||||
if args.overlay is not None:
|
||||
overlay_frames.append(draw_skeleton(
|
||||
frame_bgr.copy(), res.pose_landmarks[0] if detected else None, t, detected))
|
||||
|
||||
if is_image:
|
||||
img = cv2.imread(args.input)
|
||||
if img is None:
|
||||
sys.exit(f"ERROR: cannot read image {args.input}")
|
||||
process(img, 0.0, 0)
|
||||
src_fps, duration = 0.0, 0.0
|
||||
else:
|
||||
cap = cv2.VideoCapture(args.input)
|
||||
if not cap.isOpened():
|
||||
sys.exit(f"ERROR: cannot open video {args.input}")
|
||||
src_fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
||||
duration = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) / src_fps if src_fps else 0.0
|
||||
|
||||
if args.times:
|
||||
sample_ts = sorted(float(s) for s in args.times.split(",") if s.strip())
|
||||
else:
|
||||
sample_ts = list(np.arange(0.0, max(duration - 1e-6, 0.0), 1.0 / args.fps))
|
||||
|
||||
for i, t in enumerate(sample_ts): # ascending — VIDEO mode needs monotonic timestamps
|
||||
cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000.0)
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
frames_out.append({"t": round(t, 4), "index": i, "detected": False, "error": "read failed"})
|
||||
continue
|
||||
process(frame, t, i)
|
||||
cap.release()
|
||||
|
||||
landmarker.close()
|
||||
|
||||
if args.overlay and overlay_frames:
|
||||
h, w = overlay_frames[0].shape[:2]
|
||||
vw = cv2.VideoWriter(args.overlay, cv2.VideoWriter_fourcc(*"mp4v"),
|
||||
args.fps if not args.times else 4.0, (w, h))
|
||||
for f in overlay_frames:
|
||||
vw.write(f)
|
||||
vw.release()
|
||||
|
||||
detected = sum(1 for f in frames_out if f.get("detected"))
|
||||
doc = {
|
||||
"source": os.path.abspath(args.input),
|
||||
"source_fps": round(src_fps, 3),
|
||||
"duration_s": round(duration, 3),
|
||||
"samples": len(frames_out),
|
||||
"detected": detected,
|
||||
"landmark_names": POSE_LANDMARKS,
|
||||
"frames": frames_out,
|
||||
}
|
||||
os.makedirs(os.path.dirname(os.path.abspath(args.out)) or ".", exist_ok=True)
|
||||
with open(args.out, "w") as f:
|
||||
json.dump(doc, f)
|
||||
print(f"OK: {detected}/{len(frames_out)} samples had a detected pose -> {args.out}"
|
||||
+ (f" (overlay: {args.overlay})" if args.overlay else ""))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user