From 31ba2911dfda7e3fa7971cb99aa560a71d780506 Mon Sep 17 00:00:00 2001 From: Jeremy Kashkett Date: Wed, 15 Jul 2026 10:29:17 -0700 Subject: [PATCH] =?UTF-8?q?init:=20animation=20pipeline=20hub=20=E2=80=94?= =?UTF-8?q?=20Mac=E2=86=94PC=20bridge=20for=20converting=20and=20implement?= =?UTF-8?q?ing=20animations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skills (.claude/skills/): animation-creation, iclone-video-mocap, pose-estimation (MediaPipe models via LFS), retarget-animations (deprecated). Tools: cc/mixamo/kevin/mocap retargeters + composite baker. Plans: animation-gen-pipeline + skills-adoption. exchange/: incoming-fbx, converted-glb, reference-video (LFS for fbx/glb/mp4). Co-Authored-By: Claude Fable 5 --- .claude/skills/animation-creation/SKILL.md | 124 ++++++++ .../references/blender-mcp-recipes.md | 159 ++++++++++ .../references/gltf-transform-recipes.md | 101 +++++++ .claude/skills/iclone-video-mocap/SKILL.md | 42 +++ .../references/ariki-integration.md | 75 +++++ .../export-pipeline-to-blender-godot.md | 108 +++++++ .../references/iclone8-core-animation.md | 170 +++++++++++ .../references/video-mocap-plugin.md | 134 +++++++++ .claude/skills/pose-estimation/SKILL.md | 79 +++++ .../models/pose_landmarker_full.task | 3 + .../models/pose_landmarker_heavy.task | 3 + .../pose-estimation/scripts/compare_poses.py | 122 ++++++++ .../pose-estimation/scripts/extract_pose.py | 179 +++++++++++ .../skills/pose-estimation/scripts/setup.sh | 33 +++ .claude/skills/retarget-animations/SKILL.md | 131 ++++++++ .gitattributes | 5 + .gitignore | 4 + README.md | 60 ++++ exchange/converted-glb/.gitkeep | 0 exchange/incoming-fbx/.gitkeep | 0 exchange/reference-video/.gitkeep | 0 plans/animation-gen-pipeline-2026-07-13.md | 114 +++++++ plans/animation-skills-adoption-2026-07-13.md | 112 +++++++ tools/bake_run_punch.mjs | 280 ++++++++++++++++++ tools/cc_retarget.py | 268 +++++++++++++++++ tools/kevin_retarget.py | 254 ++++++++++++++++ tools/mixamo_retarget.py | 257 ++++++++++++++++ tools/mocap_retarget.py | 274 +++++++++++++++++ 28 files changed, 3091 insertions(+) create mode 100644 .claude/skills/animation-creation/SKILL.md create mode 100644 .claude/skills/animation-creation/references/blender-mcp-recipes.md create mode 100644 .claude/skills/animation-creation/references/gltf-transform-recipes.md create mode 100644 .claude/skills/iclone-video-mocap/SKILL.md create mode 100644 .claude/skills/iclone-video-mocap/references/ariki-integration.md create mode 100644 .claude/skills/iclone-video-mocap/references/export-pipeline-to-blender-godot.md create mode 100644 .claude/skills/iclone-video-mocap/references/iclone8-core-animation.md create mode 100644 .claude/skills/iclone-video-mocap/references/video-mocap-plugin.md create mode 100644 .claude/skills/pose-estimation/SKILL.md create mode 100644 .claude/skills/pose-estimation/models/pose_landmarker_full.task create mode 100644 .claude/skills/pose-estimation/models/pose_landmarker_heavy.task create mode 100644 .claude/skills/pose-estimation/scripts/compare_poses.py create mode 100644 .claude/skills/pose-estimation/scripts/extract_pose.py create mode 100644 .claude/skills/pose-estimation/scripts/setup.sh create mode 100644 .claude/skills/retarget-animations/SKILL.md create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 README.md create mode 100644 exchange/converted-glb/.gitkeep create mode 100644 exchange/incoming-fbx/.gitkeep create mode 100644 exchange/reference-video/.gitkeep create mode 100644 plans/animation-gen-pipeline-2026-07-13.md create mode 100644 plans/animation-skills-adoption-2026-07-13.md create mode 100644 tools/bake_run_punch.mjs create mode 100644 tools/cc_retarget.py create mode 100644 tools/kevin_retarget.py create mode 100644 tools/mixamo_retarget.py create mode 100644 tools/mocap_retarget.py diff --git a/.claude/skills/animation-creation/SKILL.md b/.claude/skills/animation-creation/SKILL.md new file mode 100644 index 0000000..359a18b --- /dev/null +++ b/.claude/skills/animation-creation/SKILL.md @@ -0,0 +1,124 @@ +--- +name: animation-creation +description: Create new skeletal animations for ariki-game on the shared Quaternius skeleton — retarget Mixamo/Kevin-Iglesias FBX to game-ready GLB packs, bake composite clips (upper body of one clip over lower body of another), and author dance sequences as hot-reload JSON. Use when the user wants a new animation, dance, or move that doesn't exist in the shipped packs. +--- + +# Animation Creation (ariki-game) + +Every animation in the game runs on the **one shared Quaternius skeleton** (65 bones, +UE names: `pelvis`, `spine_01..03`, `Head` capital-H, `thigh_l`, ...). Creating an +animation means getting motion onto that skeleton. Four proven paths, cheapest first. + +## Path 1 — It probably already exists (~810 clips) + +Check before creating. Packs: `assets/quaternius/kevin/*.glb` (12 packs, male+female × +combat/idles/misc/movement/social/work — social has `Dance01`–`Dance18`, +`DancePose01_Loop`–`07` with `_Begin`/`_Stop`, claps, waves, cheers), UAL1/UAL2 +standard libraries, `assets/quaternius/mixamo/*.glb`. List a pack's clips: + +```bash +# clip names live in the GLB's JSON chunk +node -e "const b=require('fs').readFileSync('assets/quaternius/kevin/kevin_male_social.glb');const len=b.readUInt32LE(12);const j=JSON.parse(b.slice(20,20+len));console.log(j.animations.map(a=>a.name).join('\n'))" +``` + +## Path 2 — Retarget a Mixamo FBX (new motion from Mixamo's huge library) + +Download FBX(s) from mixamo.com (any character, "Without Skin" is fine), then: + +```bash +/Applications/Blender.app/Contents/MacOS/Blender --background \ + --python tools/mixamo_retarget.py -- \ + --src "[,...]" --out assets/quaternius/mixamo/.glb +``` + +- World-rotation-delta method; handles `mixamorig:` prefix variants automatically. +- Proven end-to-end: `northern_soul.glb`, `run_to_dive.glb`. +- Output GLBs in `assets/quaternius/mixamo/` are auto-discovered by ClipCatalog / + AnimationShowcase — referenceable immediately as `/`. +- Kevin Iglesias FBX sources use the sibling `tools/kevin_retarget.py` (different rig + root handling — Kevin has a demo-scene offset bone). + +## Path 3 — Composite clips (new move from two existing clips, no Blender) + +The dance system bakes "upper body of clip A over lower body of clip B" at load time. +Author it as a move in `assets/dances/*.json` — hot-reloads in ~0.5s while the dance +test bed runs: + +```json +{ "layers": [ + { "mask": "upper", "clip": "kevin_male_social/HandClap01" }, + { "mask": "lower", "clip": "kevin_male_movement/Crouch01_Walk_Forward" } ], + "duration": 4.0 } +``` + +- Masks: `lower` = `root|pelvis|thigh*|calf*|foot*|ball*` (owns hip position AND + rotation — always); `upper` = everything else. Authoritative predicate: + `tools/bake_run_punch.mjs`. +- For a composite needed **outside** the dance system, pre-bake a merged pack with a + `bake_run_punch.mjs`-style node script instead (same mask logic, writes a GLB). + +## Path 4 — Dance sequences (choreography = ordered moves + BPM) + +A full "animation" at the choreography level is a JSON file in `assets/dances/`: +`{ "name", "bpm", "moves": [ {"clip": "pack/Clip", "loops": 2}, {composite...} ] }`. +Move transitions quantize to the beat clock. View in the test bed: +`SCENE=dance_test_bed bash tools/game.sh spawn` (ASK Jeremy before launching the game). +See `.agents/plans/dance-test-bed-2026-07-13.md` for full format + class specs. + +## Path 5 — Author in live Blender (blender-mcp) + +For motion that exists nowhere and can't be composited from existing clips — hand-keyed +poses, IK-assisted tweaks, NLA layering of retargeted clips. Prerequisites: Blender 5.1.2 +open with the BlenderMCP addon connected (sidebar → BlenderMCP → Connect) and the `blender` +MCP server registered in `.mcp.json`. Workflow: import an existing game GLB to pull in the +real Quaternius armature → author (keyframes, IK, NLA) → export GLB into +`assets/quaternius/mixamo/` so ClipCatalog auto-discovers it as `/`. +Recipes: `references/blender-mcp-recipes.md`. + +## GLB surgery without Blender (gltf-transform) + +Inspect clip names, transplant or merge clips between same-skeleton GLBs, and +resample/prune/dedup to shrink packs — no Blender required. Runs via +`npx @gltf-transform/cli`, or as a JS SDK from a node script like the existing +`tools/bake_run_punch.mjs`. Complements Path 3's node-script compositing when the job is a +clip move or pack merge, not new motion. Recipes: `references/gltf-transform-recipes.md`. + +## Hard rules (violations = invisible or broken animation) + +- **Never use `*RM` root-motion clip variants** in dances/composites — they translate the rig. +- **Skeleton path remap is mandatory** when playing raw GLB clips on a game rig: GLB + tracks target `Armature/Skeleton3D:bone`; derive the rig's real prefix from an + already-remapped clip (`ap.GetAnimation("idle").TrackGetPath(0)`), remap once, share. +- Clip name fallback: exact → `+"_Loop"` → `-"_Loop"` (Godot import sometimes strips it). +- Skip `RESET` clips; set `LoopMode.Linear` on extracted clips. +- `.uid` files auto-generate — never hand-author. + +## Verify what you created + +- Single clips / packs: `scenes/animation_showcase.tscn` pages through every shipped clip. +- Dances / composites: dance test bed; `VideoRecorder` (in-scene) records the viewport + to `~/Downloads/*.mp4` for review — feed that to the `pose-estimation` skill to score + a recreation against source footage (crop-zoom one dancer first; wide shots don't detect). + +## Path 6 — Generate from video (pose-driven mocap, BUILT 2026-07-13) + +`tools/mocap_retarget.py` turns `pose-estimation` skill output (extract_pose.py JSON, +world landmarks) into a GLB clip on the Quaternius skeleton: + +```bash +/Applications/Blender.app/Contents/MacOS/Blender --background \ + --python tools/mocap_retarget.py -- \ + --pose --out assets/quaternius/mixamo/.glb --name \ + --range 166:192 --smooth 5 --render-check /tmp/check # renders frames to eyeball +``` + +- Limb bones: shortest-arc aim from rest direction (preserves rest roll); pelvis/spine: + full-basis deltas conjugated through char-frame alignment (same convention as + mixamo_retarget.py — landmark space is subject-left=+X, up=+Z, facing=-Y, NOT identity). +- MediaPipe world landmarks are hip-centered → no root motion; pelvis height is + reconstructed from pelvis-above-ankle extent vs standing (carries squats). +- Known v1 limits: no twist control, no fingers, no horizontal root motion. +- Proven: `haka_mocap.glb` / `HakaFull` (26s, 208 keys) generated from a YouTube haka + tutorial via `~/.claude/skills/pose-estimation`. + +Full pipeline context: `.agents/plans/animation-gen-pipeline-2026-07-13.md`. diff --git a/.claude/skills/animation-creation/references/blender-mcp-recipes.md b/.claude/skills/animation-creation/references/blender-mcp-recipes.md new file mode 100644 index 0000000..0de945c --- /dev/null +++ b/.claude/skills/animation-creation/references/blender-mcp-recipes.md @@ -0,0 +1,159 @@ +# blender-mcp recipes + +Practical `bpy` snippets the agent pastes through blender-mcp's `execute-code` tool to +author animations on the shared Quaternius skeleton and export them as game-ready GLB +packs. These mirror the proven conventions in `tools/mixamo_retarget.py` — keep them in +lockstep. + +## Setup assumptions + +- Blender **5.1.2** at `/Applications/Blender.app` (call headless as + `/Applications/Blender.app/Contents/MacOS/Blender`). +- The BlenderMCP addon is installed and **connected** (3D View → sidebar → BlenderMCP → + Connect). If the `blender` MCP tool isn't available, the addon isn't connected. +- **One game GLB imported** so the scene holds the real Quaternius armature. The game + ships these as animations-only GLBs; import one to reuse its exact rig: + + ```python + bpy.ops.import_scene.gltf(filepath="/Users/jeremykashkett/Tinqs/local.repo/ariki-game/assets/quaternius/kevin/kevin_male_social.glb") + arm = next(o for o in bpy.data.objects if o.type == "ARMATURE") + arm.name = "Armature" # match the convention mixamo_retarget.py enforces + bpy.context.scene.render.fps = 30 + ``` + +## Skeleton orientation + +65 bones, UE-style names: `pelvis`, `spine_01`, `spine_02`, `spine_03`, `neck_01`, +`Head` (**capital H**), `clavicle_l`, `upperarm_l`, `lowerarm_l`, `hand_l`, `thigh_l`, +`calf_l`, `foot_l`, `ball_l`, and the `_r` twins, plus 3-segment fingers +(`index_0{l,r}` … `pinky_03`, `thumb_0{l,r}` … `thumb_03`). The armature object is named +`Armature`; work in pose mode via `bpy.context.object.pose.bones[""]`. + +## Keyframing recipe + +Build an action from an explicit list of poses (name, frame, per-bone quaternions). +Quaternius bones are authored in **quaternion** rotation mode, so set every pose bone's +`rotation_mode = "QUATERNION"` first (this is what `mixamo_retarget.py` does): + +```python +import bpy +from mathutils import Quaternion + +arm = bpy.data.objects["Armature"] +for pb in arm.pose.bones: + pb.rotation_mode = "QUATERNION" + +act = bpy.data.actions.new("Wave01") +arm.animation_data_create() +arm.animation_data.action = act +bpy.context.scene.frame_start, bpy.context.scene.frame_end = 1, 30 + +# poses: (frame, {bone: (w,x,y,z)}) +poses = [ + (1, {"hand_l": (1,0,0,0), "lowerarm_l": (1,0,0,0)}), + (15, {"hand_l": (0.7,0,0,0.7), "lowerarm_l": (0.9,0.3,0,0)}), # raise hand + (30, {"hand_l": (1,0,0,0), "lowerarm_l": (1,0,0,0)}), +] +for frame, bones in poses: + bpy.context.scene.frame_set(frame) + for name, q in bones.items(): + pb = arm.pose.bones[name] + pb.rotation_quaternion = Quaternion(q) + pb.keyframe_insert(data_path="rotation_quaternion", frame=frame) +``` + +## IK setup recipe + +Drive `calf_l` (or `lowerarm_l`) toward a target empty with a 2-bone chain, then bake +the solved motion to real keyframes and drop the constraint — the game never needs IK: + +```python +import bpy + +arm = bpy.data.objects["Armature"] +target = bpy.data.objects.new("IK_target_l", None) # empty +bpy.context.collection.objects.link(target) +target.location = arm.matrix_world @ arm.pose.bones["foot_l"].head + (0.0, 0.0, 0.2) + +pb = arm.pose.bones["calf_l"] +ik = pb.constraints.new("IK") +ik.target = target +ik.chain_count = 2 # thigh_l + calf_l +ik.use_tail = True + +# Bake the visual (constraint-solved) result to plain keyframes, then remove the constraint. +bpy.ops.object.select_all(action="DESELECT") +arm.select_set(True) +bpy.context.view_layer.objects.active = arm +bpy.ops.nla.bake(frame_start=1, frame_end=30, visual_keying=True, + bake_types={"POSE"}, clean_curves=True) +for c in list(pb.constraints): + pb.constraints.remove(c) +``` + +## NLA layering recipe + +The glTF exporter emits each **NLA track** as a separate animation clip. To ship multiple +clips in one GLB (or layer retargeted clips), push each action onto its own NLA track — +this is the exact pattern `tools/mixamo_retarget.py` uses to export a pack: + +```python +import bpy +arm = bpy.data.objects["Armature"] +if arm.animation_data is None: + arm.animation_data_create() +for act in bpy.data.actions: # every action you want to ship + tr = arm.animation_data.nla_tracks.new() + tr.name = act.name # clip name = track name + tr.strips.new(act.name, 1, act) +arm.animation_data.action = None # avoid a stray duplicate clip +``` + +To **blend** two clips on the stack (e.g. a base loop under a gesture), set the upper +strip's `blend_type` (`"ADD"`, `"REPLACE"`) and `blend_in`/`blend_out` frames, then bake +the combined result down to a single action (visual keying) and re-export that. + +## Export recipe + +The canonical export block lives in `tools/mixamo_retarget.py` and **must be mirrored +exactly** — the game relies on these flags (`NLA_TRACKS` mode = one clip per track; +`export_force_sampling` for clean per-bone channels; size optimization off to preserve +retargeted curves). Select only the armature and drop meshes first so the GLB is +animations-only (the game binds clips to its own body): + +```python +import bpy +out = "/Users/jeremykashkett/Tinqs/local.repo/ariki-game/assets/quaternius/mixamo/wave_pack.glb" + +# Drop meshes — animations-only GLB (mirrors mixamo_retarget.py). +for o in [o for o in bpy.data.objects if o.type == "MESH"]: + bpy.data.objects.remove(o, do_unlink=True) + +bpy.ops.object.select_all(action="DESELECT") +arm = bpy.data.objects["Armature"] +arm.select_set(True) +bpy.context.view_layer.objects.active = arm + +bpy.ops.export_scene.gltf( + filepath=out, use_selection=True, + export_animations=True, export_animation_mode="NLA_TRACKS", + export_force_sampling=True, export_optimize_animation_size=False) +print("[blender-mcp] EXPORTED →", out) +``` + +Output GLBs in `assets/quaternius/mixamo/` are auto-discovered by ClipCatalog / +AnimationShowcase — referenceable immediately as `/` where +`` is the NLA track (Blender action) name. + +## Gotchas + +- **Never export `*RM` root-motion variants** for dances/composites — they translate the + rig and break compositing. Author clips in place. +- **Skip the `RESET`/rest action** — don't push it to an NLA track or it ships as a clip. +- **Clip name = Blender action/NLA-track name.** The game refers to clips as + `/` (e.g. `wave_pack/Wave01`). +- **One armature per exported GLB.** If you imported a GLB that brought its own body + meshes, delete them before export (see the export block). +- **Match `fps=30`** so frame counts line up with shipped packs and the dance beat clock. +- **Quaternion mode** for all pose bones before keyframing (matches the retargeter). +- `.uid` files auto-generate in Godot — never hand-author them. diff --git a/.claude/skills/animation-creation/references/gltf-transform-recipes.md b/.claude/skills/animation-creation/references/gltf-transform-recipes.md new file mode 100644 index 0000000..6fd37cc --- /dev/null +++ b/.claude/skills/animation-creation/references/gltf-transform-recipes.md @@ -0,0 +1,101 @@ +# gltf-transform recipes + +Headless GLB surgery with gltf-transform (donmccurdy, gltf-transform.dev) — inspect, +merge, transplant, and shrink animation packs with no Blender required. Same Quaternius +skeleton across all game GLBs, so node names line up 1:1 and clip moves are mechanical. + +## Install / run + +CLI (no global install — `npx` pulls it on demand): + +```bash +npx @gltf-transform/cli --help +npx @gltf-transform/cli inspect assets/quaternius/kevin/kevin_male_social.glb +``` + +Or as a JS SDK from a node script (mirror the layout of the existing +`tools/bake_run_punch.mjs`): + +```js +import { NodeIO } from "@gltf-transform/core"; +import { prune, dedup, resample } from "@gltf-transform/functions"; +const io = new NodeIO(); +const doc = io.read("path/to/file.glb"); +``` + +Install once into the repo's node tooling: `npm i -D @gltf-transform/core @gltf-transform/functions` +(the orchestrator installs — you author the script only). + +## Inspect clips + +```bash +# Full report: meshes, animations, channels, duration. +npx @gltf-transform/cli inspect assets/quaternius/kevin/kevin_male_social.glb +``` + +For a quick clip-name list without the SDK, the pure-node JSON-chunk one-liner already in +`SKILL.md` (Path 1) reads the GLB's JSON chunk and prints `animations[].name` — use it to +confirm a clip exists before importing/transplanting. + +## Clip transplant between same-skeleton GLBs + +Both files share the Quaternius skeleton, so every channel's target node matches by name. +Sketch: read source + target, copy the source `Animation` into the target document, then +re-point each channel's `targetNode` to the target-doc node with the same name. + +```js +// verify against gltf-transform v4 API before first use +import { NodeIO } from "@gltf-transform/core"; +import { copyToDocument } from "@gltf-transform/functions"; + +const io = new NodeIO(); +const target = io.read("assets/quaternius/mixamo/northern_soul.glb"); // pack to extend +const source = io.read("downloads/clip_to_add.glb"); // same skeleton + +const srcAnim = source.getRoot().listAnimations() + .find(a => a.getName() === "NewClip"); +const targetNodes = new Map(target.getRoot().listNodes().map(n => [n.getName(), n])); + +copyToDocument(target, source, [srcAnim]); // copy the Animation property +const dstAnim = target.getRoot().listAnimations().at(-1); +for (const ch of dstAnim.listChannels()) { + const srcNode = ch.getTargetNode(); + if (srcNode) ch.setTargetNode(targetNodes.get(srcNode.getName())); // re-point by name +} +io.write("assets/quaternius/mixamo/northern_soul.glb", target); // overwrites — back up first +``` + +Common variations: copy **all** source animations (iterate `listAnimations()`), rename on +the way in (`dstAnim.setName("Wave01")`), or strip a clip (remove its channels then +`prune`). If `copyToDocument` isn't the right entry point in your installed version, fall +back to walking `srcAnim.listChannels()` and rebuilding each (sampler interpolation + +input/output accessors + targetNode/path) on a fresh `target.createAnimation()`. + +## Shrink / clean + +Standard transforms, applied in this order for pack size: + +```bash +npx @gltf-transform/cli optimize # resample + prune + dedup combined +# or individually: +npx @gltf-transform/cli resample # drop redundant keyframes +npx @gltf-transform/cli prune # remove unreferenced data +npx @gltf-transform/cli dedup # merge identical accessors/meshes +``` + +In the SDK: `await doc.transform(prune(), dedup(), resample())`. `resample` takes an +optional `tolerance`/`ratio` to trade fidelity for size — retargeted clips have clean +curves and tolerate aggressive resampling; mocap-sourced clips do not. + +## When to prefer this over Blender + +Prefer gltf-transform when there's **no new motion** to author: + +- Same-skeleton clip moves (one clip from a downloaded GLB into a shipped pack). +- Batch renames (fix a pack of `*_Loop` suffixes, normalize clip names). +- Pack merges / splits (combine two `assets/quaternius/mixamo/` packs). +- Headless size passes before shipping (resample/prune/dedup). + +Reach for **Blender (Path 2 retarget / Path 5 authoring)** when the source uses a +*different* skeleton (needs the world-rotation-delta retarget), or when you must author, +IK-solve, or layer new motion that exists nowhere. diff --git a/.claude/skills/iclone-video-mocap/SKILL.md b/.claude/skills/iclone-video-mocap/SKILL.md new file mode 100644 index 0000000..beb4b3e --- /dev/null +++ b/.claude/skills/iclone-video-mocap/SKILL.md @@ -0,0 +1,42 @@ +--- +name: iclone-video-mocap +description: Advise on Reallusion iClone 8 and the Video Mocap plugin (AI body mocap from video) — filming rules, generation settings, mocap cleanup (Motion Correction, Curve Editor), face layering with AccuFACE, FBX export, and integrating the result into ariki-game via tools/cc_retarget.py. Use when the user asks how to do anything in iClone/Character Creator, wants to record video for mocap, or wants an iClone export playing in the game. +--- + +# iClone 8 + Video Mocap (advisory + ariki-game integration) + +Jeremy uses **iClone 8** (Windows) with the **iClone Video Mocap plugin** — Reallusion's cloud AI body-mocap from ordinary video (QuickMagic-powered, launched Oct 2025, requires iClone 8.63+). This skill answers "how do I do X in iClone" and "how do I get it into the game". + +**Reference files (read the one that matches the question):** +- `references/video-mocap-plugin.md` — Video Mocap product facts, pricing (~$2.50–3/generation, points), full 5-stage workflow, filming rules, known bugs, AccuFACE. +- `references/iclone8-core-animation.md` — motion clips/layer keys, Edit Motion Layer (IK/pins), Reach Target, timeline ops, Curve Editor filters, Motion Correction, face animation, characterization, FBX dialog options, hotkeys. +- `references/export-pipeline-to-blender-godot.md` — iClone/CC4 FBX+BVH export details, soupday Blender Tools, retargeting to simple rigs, FBX→GLB→Godot, pitfalls checklist. +- `references/ariki-integration.md` — **the game-side path**: `tools/cc_retarget.py` (CC_Base_→Quaternius), ClipCatalog discovery, dance JSON, end-to-end recipe. + +## The 60-second mental model + +- iClone animation = **Motion Clip + Layer Keys** (layer keys live inside the clip and can't leave it). Pose edits go through **Edit Motion Layer** (hotkey N, HumanIK IK/FK, pinning). Foot/hand locking via **Reach Target**; foot-slide fix via right-click clip > **Motion Correction** (flatten between iterations). Jitter fix: right-click clip > **Sample Motion Clip to New Layer**, then Curve Editor (F12) **Smooth / Butterworth / Optimize** filters. +- Video Mocap = separate plugin panel: import video → trim to ≤60 s region → **Full Body / Upper Body** (+ Includes Fingers) → cloud generate (5–8 min, costs points) → **Apply Motion** to selected character (check "Apply reference video" for rotoscope cleanup) → clip lands in Content Manager > Motion > VideoMocap as rlMotion. **Body only — no face**; layer AccuFACE/LIVE FACE for talking. +- System: iClone is **60 fps internal, cm units, Z-up**; CC3+ characters bind in **A-pose**; FBX axis conversion comes from the **Target Tool Preset**. Constraint keys (Reach/Link/Look At/Path) need **Animation > Flatten All Motion with Constraint** before export. + +## Filming rules that make or break Video Mocap (advise BEFORE recording) + +Eye-level static camera, single continuous shot; whole actor in frame the entire time; one person; textured clothing that contrasts the background; avoid flips, jumps, fast spins, hand-occluding poses. Money-saver: **stack several motions into one ≤60 s take with ~2 s gaps** — one generation fee, split into clips in iClone. + +## Getting it into ariki-game (short version) + +```bash +/Applications/Blender.app/Contents/MacOS/Blender --background \ + --python tools/cc_retarget.py -- \ + --src "" --out assets/quaternius/dancegen/.glb --name +``` + +Export from iClone with preset **Blender**, range **All**, **Preserve Bone Names (CC Base)** ON. The tool maps `CC_Base_*` → the shared 65-bone Quaternius skeleton, strips the mesh, and the GLB is auto-discovered by ClipCatalog as `/` (dirs: kevin/, mixamo/, dancegen/). Details, dance-JSON format, and the **unvalidated A-pose caveat** (first export needs an arm-orientation QA; workaround: FBX Advanced > Use T-Pose As Bind Pose): `references/ariki-integration.md`. + +## Cross-cutting gotchas + +- iClone 8 has **no animation-only FBX export** (CC4 does: FBX Options > Motion) and **no BVH export** (CC4 4.4+ does). No native glTF anywhere — route through Blender or Godot's ufbx. +- Marketplace content needs a per-pack **Export License** to leave via FBX/USD; your own recordings/characters don't. +- Video Mocap: keep plugin ≥1.02 (pre-1.02 double-charged points on re-open); Loop-enabled clips silently discard motion-layer edits — Flatten first. +- MixMoves is gone in iClone 8 (Motion Director replaced it); AccuLips only works on CC-Standard/Gamebase characters. +- ariki side: ASK Jeremy before launching the game; never use `*RM`/root-motion clip variants in dances. diff --git a/.claude/skills/iclone-video-mocap/references/ariki-integration.md b/.claude/skills/iclone-video-mocap/references/ariki-integration.md new file mode 100644 index 0000000..4d1753b --- /dev/null +++ b/.claude/skills/iclone-video-mocap/references/ariki-integration.md @@ -0,0 +1,75 @@ +# ariki-game — Integrating iClone Animation Exports + +How motion authored in iClone 8 (incl. Video Mocap output) lands on ariki-game's shared Quaternius skeleton. Mapped 2026-07-15 from the repo. + +**Headline: the retargeter already exists — `tools/cc_retarget.py` (created 2026-07-14) handles `CC_Base_*` (Reallusion CC/iClone) FBX → Quaternius GLB.** The pipeline is: iClone FBX export → `cc_retarget.py` → mesh-stripped GLB in an auto-discovered pack dir → referenceable as `/` in dances/showcase. + +## 1. The shared Quaternius skeleton (retarget target) + +- **Canonical rig:** `assets/quaternius/base-characters/Universal Base Characters[Standard]/Base Characters/Godot - UE/Superhero_Male_FullBody.gltf` — all retargeters default to it (`--target` overrides). `kevin_retarget.py:26` is the exception (hardcoded absolute path, non-portable). +- **65 bones, UE-style names, T-pose rest.** Spine: `root, pelvis, spine_01..03, neck_01, Head` (capital H). Per side: `clavicle_, upperarm_, lowerarm_, hand_, thigh_, calf_, foot_, ball_, ball_leaf_` + fingers `thumb/index/middle/ring/pinky_01..03_` with `_04_leaf_` tips. + +## 2. ClipCatalog discovery (`src/Testing/Dance/ClipCatalog.cs`) + +- Hardcoded packs (lines 20-25): UAL1, UAL2, Run_Punch. +- Auto-discovered dirs (lines 37-45): `res://assets/quaternius/kevin`, `res://assets/quaternius/mixamo`, `res://assets/quaternius/dancegen` — drop a GLB in one and it's live; **pack name = GLB basename**. +- Clip ref format: `"/"` (e.g. `dancegen/HakaFull`). +- Lazy extraction (lines 80-118): first `AnimationPlayer` in GLB scene, skips `RESET`, forces `LoopMode.Linear`, caches. Name fallback: `exact → +"_Loop" → -"_Loop"`. +- Gotcha baked into the code (lines 63-64): `DirAccess.GetNext()` returns `""` at end, not null — null check pegs CPU. + +## 3. `tools/cc_retarget.py` — the iClone/CC ingestion tool + +Blender 5 headless, same world-rotation-delta method as `mixamo_retarget.py`: + +```bash +/Applications/Blender.app/Contents/MacOS/Blender --background \ + --python tools/cc_retarget.py -- \ + --src "[,...]" --out assets/quaternius/dancegen/.glb \ + [--name ] [--target ] +``` + +- **BONE_MAP (lines 25-48):** `CC_Base_Hip→pelvis`, `CC_Base_Waist→spine_01`, `CC_Base_Spine01→spine_02`, `CC_Base_Spine02→spine_03`, `CC_Base_NeckTwist01→neck_01`, `CC_Base_Head→Head`; limbs `CC_Base_L_Clavicle/Upperarm/Forearm/Hand/Thigh/Calf/Foot/ToeBase → clavicle_l/upperarm_l/lowerarm_l/hand_l/thigh_l/calf_l/foot_l/ball_l`. **Thumbs 3-seg, other fingers 1-seg** (`CC_Base_L_Index1→index_01_l` etc. — game-optimized CC rigs have one segment per finger). Twist helpers (`UpperarmTwist01…`) unmapped → ride parent rigidly. +- **Char-frame conjugation:** `q_delta = C @ (q_src·q_src_rest⁻¹) @ C⁻¹` with `C = char_frame(target) @ char_frame(source)⁻¹`; source frame from `CC_Base_Hip`/`CC_Base_Head`/`CC_Base_L_Upperarm` (lines 128-129). +- **Units/scale:** `unit = src_arm.scale.x` captured BEFORE `transform_apply` (importer's cm→m 0.01); pelvis = world-space delta `× tscale × unit` (lines 176-177). Handles iClone's cm natively. +- **fps 60** (only tool at 60 — "iClone exports at 60; glTF export resamples fine"). +- **Motion-presence guard** (`pick_action`, lines 89-105): picks the longest action whose fcurves reference `CC_Base_` bones; **rejects `TempMotion` <5-frame placeholders** with error "FBX exported WITHOUT motion — Re-export with Include Motion". +- Export tail (shared with all retargeters): drop MESH objects, one NLA track per action (track name = clip name = glTF animation name), `export_scene.gltf(export_animation_mode="NLA_TRACKS", export_force_sampling=True, export_optimize_animation_size=False)`. + +### ⚠ Open validation item: A-pose +The tool's comments claim "CC rigs rest in T-pose, verified on ActorBuild exports", but iClone/CC3+ characters bind in **A-pose**. The char-frame delta math measures off actual rest and should self-cancel — but the left-arm char-frame axis (`CC_Base_L_Upperarm` bone-Y) points down-and-out in A-pose, not straight +X. **QA the first A-pose export**: if arms bake rotated, fix the char-frame or add a per-arm rest-alignment quaternion. This is the one genuine divergence vs the proven Mixamo (T-pose) path. Alternatively force T-pose on the iClone side: FBX Advanced Settings > **Use T-Pose As Bind Pose**. + +## 4. Recommended iClone export settings for this pipeline + +- iClone: `File > Export > Export Fbx...`, Target Tool Preset **Blender** (or Unity 3D), Export Range **All**/**Range**, FPS `Project(60)` fine (tool handles it). Advanced: **Preserve Bone Names (CC Base)** ON (default — the bone map depends on it), consider **Use T-Pose As Bind Pose** (see §3 caveat), **Reset Bone Scale** ON. +- Run **Menu > Animation > Flatten All Motion with Constraint** first if the take uses Reach/Link/Look At/Path — those keys don't survive FBX. +- iClone 8 always exports mesh+animation (no motion-only mode); `cc_retarget.py` strips the mesh anyway, so don't fight it — but uncheck Embed Textures / set Max Texture Size low to keep the FBX small. +- Animation-only alternative: export via CC4 (`FBX Options: Motion`) — the retargeter still needs a rest pose, and Blender only gets bind pose from a weighted mesh, so prefer the iClone mesh+motion export. +- Store-content license wall: Reallusion marketplace motions/characters need a per-pack **Export License** to leave via FBX; your own Video Mocap recordings and characters are unrestricted. + +## 5. Runtime consumers + +- **AnimationShowcase** (`src/Testing/AnimationShowcase.cs`, `scenes/animation_showcase.tscn`): pages all clips. `SCENE=animation_showcase bash tools/game.sh spawn` (game.sh:926). ASK Jeremy before launching the game. +- **Dance test bed** (`scenes/dance_test_bed.tscn`, `src/Testing/Dance/DanceTestBed.cs:118-124`): hot-reload dances via `DanceLibrary.Poll`; in-scene `VideoRecorder` → `~/Downloads/*.mp4` at 30fps. `SCENE=dance_test_bed bash tools/game.sh spawn` (game.sh:927). +- **Dance JSON** (`assets/dances/*.json`): `{ "name", "bpm", "moves": [...] }`; move = `{ "clip": "pack/Clip", "loops": N }` or composite `{ "layers": [{"mask":"upper","clip":...},{"mask":"lower","clip":...}], "duration": s }`. Transitions quantize to the BPM beat clock. Examples: `haka_mocap.json` (uses `haka_mocap/HakaFull`), `clap_stomp.json` (110), `war_challenge.json` (120). +- **CompositeClipBaker.cs**: lower mask = `root|pelvis|thigh*|calf*|foot*|ball*` (always owns hip pos+rot — prevents hip-fight/foot-slide); upper = rest; first-writer-wins per track. Cache keyed by definition → JSON edits need no invalidation. +- **Hard rules:** skeleton path remap mandatory for raw GLB clips (GLB tracks target `Armature/Skeleton3D:bone`; derive real prefix from an already-remapped clip — `PlayerController.LoadUalLibrary` is canonical). **Never use `*RM`/`*RootMotion` clip variants** — everything is in-place. + +## 6. Godot import conventions + +- `.import` sidecars: `importer="scene"`, **`animation/fps=30`** (even 60fps cc output resamples to 30 at import), `animation/remove_immutable_tracks=true`, `animation/import_rest_as_RESET=false`, `meshes/generate_lods=false`. `.uid` files auto-generate — never hand-author. +- Commit GLB + let Godot import → immediately referenceable as `/`. + +## 7. Precedents + +- **Kevin Iglesias**: 199 Unity FBX (`B-*` rig) → 12 packs in `assets/quaternius/kevin/` via `kevin_retarget.py` batch — the batch-directory template. +- **Video-to-mocap (free, local)**: `extract_pose.py` (MediaPipe) → `mocap_retarget.py` (rotation-solving, no source armature) → `dancegen/haka_mocap.glb` / `HakaFull`. iClone Video Mocap is the paid, higher-quality alternative feeding the same `cc_retarget.py` path. +- Four bone-map dicts exist: `kevin_retarget.py:29`, `mixamo_retarget.py:26`, `cc_retarget.py:25`, `mocap_retarget.py` AIM_BONES. + +## 8. End-to-end recipe (iClone Video Mocap → in-game dance) + +1. Film per the Video Mocap guidelines (eye-level static camera, full body, single actor, textured clothing; stack takes with 2s gaps into one ≤60s region). +2. Generate in the plugin (Full Body [+ Includes Fingers]), apply to a CC character, clean up in iClone (Motion Correction for feet, Curve Editor smooth/Butterworth for jitter — see the other references). +3. Export FBX per §4. +4. `cc_retarget.py --src --out assets/quaternius/dancegen/.glb --name `. +5. Verify: open in animation_showcase or reference `/` from a dance JSON in `assets/dances/` (hot-reloads in the dance test bed). +6. First A-pose export: eyeball arm orientation (§3 caveat) before batching. diff --git a/.claude/skills/iclone-video-mocap/references/export-pipeline-to-blender-godot.md b/.claude/skills/iclone-video-mocap/references/export-pipeline-to-blender-godot.md new file mode 100644 index 0000000..d8a24bd --- /dev/null +++ b/.claude/skills/iclone-video-mocap/references/export-pipeline-to-blender-godot.md @@ -0,0 +1,108 @@ +# iClone 8 / CC4 → Blender → Godot Animation Export Pipeline Reference + +Compiled 2026-07-15 from Reallusion manuals, soupday docs, Godot docs, Reallusion forum. Claims not confirmed against a primary source are marked [UNVERIFIED]. + +## 1. iClone 8 FBX Export + +**Menu path:** `Menu > File > Export > Export Fbx...` (select one or more avatars/props/accessories/cameras first). iClone 8 ships as a single edition — no separate "Pipeline" SKU and no 3DXchange needed; FBX/OBJ export is built in. + +**Export dialog options (exact names, iClone 8 manual):** +- **Target Tool Preset** — dropdown, auto-configures Advanced Settings per DCC. Supported-tools list is rendered as an image in the manual page (not text-extractable); known presets include 3ds Max, Blender, Cinema 4D, Maya, MotionBuilder, Unity, Unreal [UNVERIFIED exact/full list]. +- **Export Range**: `T-Pose` | `Current Frame` | `All` | `Range` — this is the mesh-vs-animation control. `T-Pose`/`Current Frame` = static export; `All`/`Range` = baked animation. +- **Export Animation FPS** — free field / dropdown to resample export frame rate. +- **Embed Textures** — embed in FBX vs. export as separate files. +- **Max Image Size** — cap texture resolution (default: original size). +- **Convert Image Format to** — force texture format (default JPG). +- **Delete Hidden Face** — strips occluded mesh. +- **Use Smooth Mesh** — export subdivided/smoothed surface. +- **Merge Face Hair to One Object**, **Tear Ducts UV Adjustment** — face-mesh consolidation options. +- **Embed Timeline Timecode**. +- **Convert Skinned Expressions to Morphs** — bakes skinned (bone-driven) facial expressions into morph targets so faces survive in non-Reallusion tools. Important for facial anim in Godot. +- **Delete Unused Morphs** — drops unkeyed blendshapes (smaller files; disable if you need the full ARKit/viseme set at runtime). +- **Advanced Settings** panel — per-tool fixups; for the Blender template this includes **Rename Duplicated Nodes** and **Reset Bone Offset** (needed so bones behave in Blender Object/Pose/Edit modes). + +**Animation-only export:** iClone 8 has **no** "export animation only" FBX mode — mesh/materials always come along (confirmed by forum threads and Feedback Tracker "IClone 8 has limited FBX export options compared to CC4"). Workarounds: export via CC4 (which has `FBX Options: Motion`), or import the full FBX in Blender/soupday tools and discard meshes. For constraint-based animation (Link/Look At/Attach/Path/Reach), run `Menu > Animation > Flatten All Motion with Constraint` before export or those keys are lost. + +**Not exportable as FBX:** iLight, iEffect, iParticle, iTree, iGrass, heightmap iTerrain, iWater, iSky, aml, iTalk, iPath. + +**Export License gating:** The *software* does not gate FBX export; the *content* does. Reallusion marketplace/Content Store items require a purchased **Export License** per pack to leave the ecosystem; without it you get "You must first purchase the content licenses in order to export FBX and USD files." Your own characters and content flagged export-ready export freely. The Content EULA restricts exporting iContent to FBX/OBJ etc. without that license. + +Sources: https://manual.reallusion.com/iClone-8/Content/ENU/8.0/83-FBX/FBX_UI_Intro.htm , https://manual.reallusion.com/iClone-8/Content/ENU/8.0/83-FBX/Exporting_FBX_from_iClone.htm , https://forum.reallusion.com/516723/export-animation-only-to-fbx-with-iclone-8/Forum445.aspx , https://www.reallusion.com/license/content.html , https://forum.reallusion.com/556427/ + +## 2. CC4 FBX Export (the better animation exit door) + +**Menu path:** `File > Export > FBX (Clothed Character)`. + +**Dialog options:** +- **Target Tool Preset** dropdown (auto-sets Advanced Settings): Blender, Unity, Unreal, Maya, 3ds Max, etc. +- **FBX Options**: `Mesh` | `Motion` | `Mesh and Motion` — `Motion` is the true animation-only export (armature + animation, no character mesh). Separate checkbox **Export Mesh and Motion Individually** writes mesh FBX + one FBX per motion. +- **Include Motion**: `Calibration` (T-pose calibration motion) | `Current Pose` | `Custom` (queue .iMotion/.iMotionPlus clips; MotionPlus carries facial + body). Disabled when Export Type = Mesh only. +- **Frame rate**: dedicated page "Setting Frame Rate for Exporting FBX" — resample dropdown/manual entry. +- **First Frame in Bind-Pose** — inserts bind pose (T-pose for CC1/CC3 Base, **A-pose for CC3+ characters**) one frame ahead of the motion. Critical for retargeting tools that read frame 0 as rest pose. +- Texture size/format, embed toggle; **Delete Hidden Mesh**; **Merge Beard and Brows into one object**; **Bake diffuse and specular maps from Digital Human Hair Shader**; **Bake diffuse maps from skin color**; **Use Subdivided Mesh**; InstaLOD/Remesher/Merge Material for game LODs. + +Sources: https://manual.reallusion.com/Character-Creator-4/Content/ENU/4.1/17-Export/Export-FBX.htm , https://manual.reallusion.com/Character-Creator-4/Content/ENU/4.0/17_Export/Exporting_Mesh_and_Motion_Individually.htm , https://manual.reallusion.com/Character-Creator-4/Content/ENU/4.0/17_Export/Setting_Frame_Rate_for_FBX.htm , https://manual.reallusion.com/Character-Creator-4/Content/ENU/4.0/17_Export/Using-First-Fram-in-Bind-Pose-Feature.htm + +## 3. BVH Export + +- **CC4 4.4+ has native BVH export**: `File > Export > BVH`. Options: Prefix/Suffix; bone selection with designated **motion root** (map to Hips); **Target Tool Preset**; **Rename Duplicated Nodes** + **Reset Bone Offset** (Blender compat); **Frame Rate** (sampling); Include Motion: `Calibration` | `Current Pose` | `Current Animation` | `Custom` (batch via Load Perform/Open File). Supports .iMotion/.iMotionPlus/.rlMotion/.rlPose. Known failures exporting motions retargeted onto non-standard (e.g. Daz G8) skeletons. +- **iClone 8:** no BVH exporter documented in the iClone 8 manual [UNVERIFIED — historical `File > Export to Other 3D Format > Export BVH` docs are 3DXchange5-era]. Practical path: send motion to CC4 and BVH-export there, or export FBX and convert in Blender. +- Both tools **import** BVH (Convert External Motions → iMotion). + +Sources: https://manual.reallusion.com/Character-Creator-4/Content/ENU/4.4/17-Export/Exporting-Motions-to-BVH.htm , https://discussions.reallusion.com/t/bvh-export-fails-from-cc4/13056 , https://manual.reallusion.com/iClone-8/Content/ENU/8.4/50-Animation/Import-External-Motions/Convert-External-Motions.htm + +## 4. USD / glTF + +- **USD:** via the free **Omniverse Connector plugin** (iClone must be 8.3+). After install, `Plugins > Omniverse` exposes **Export USD** plus Live Sync (one-click whole-project or per-item two-way transfer). CC4 has an equivalent USD/Omniverse export [UNVERIFIED exact CC4 menu]. USD export of store content is also Export-License-gated. +- **glTF/GLB:** **no native glTF export** in iClone 8 or CC4 [UNVERIFIED as absolute, but no manual page exists]. Standard route: FBX → Blender → glTF exporter, or FBX → FBX2glTF/ufbx inside Godot. Community tutorials (Freedom Arts "CC4 Export to GLB and GLTF with Blender") all route through Blender. + +Sources: https://manual.reallusion.com/Omniverse-Plug-in/Content/ENU/iC-8.3/01-Installation/For-IC-Users.htm , https://docs.omniverse.nvidia.com/connect/iclone.html , https://www.classcentral.com/course/youtube-character-creator-4-export-to-glb-gltf-with-blender-cc4-pipeline-tutorial-blender-310648 + +## 5. soupday CC/iC Blender Tools (cc_blender_tools) + +Add-on (a.k.a. "CC/iC Pipeline") for importing/auto-setup of CC3/CC4 + iClone 7/8 exports into Blender. Docs explicitly warn: **"Do Not use the standard blender FBX importer"** for CC/iC characters. + +**Recommended export settings** — CC4: preset `Blender`, `FBX (Clothed Character)`, FBX Options `Mesh and Motion`, Include Motion `Current Pose`, check `Delete Hidden Faces`, textures NOT embedded. iClone: `File > Export > Export FBX`, target `Blender`, range `All`, check `Embed Textures` + `Delete Hidden Faces`. + +**Features:** +- **Import Character** button: auto-builds full material setup (from the sidecar `.json`), optional toggles for Lighting, Physics, Wrinkles, Rigify; `Import Animation` checkbox brings motions in as Blender Actions. +- **Import Animations** button (Retargeting section): imports armature + actions **including shape-key actions** from extra FBX files, discarding meshes/materials. +- **Rigify** button (Rigging and Animation panel): one-click Rigify control rig, optional **Full Face Rig**; original CC skeleton is kept (hidden) as deform rig. +- **Retargeting**: sources supported — Character Creator, ActorCore, GameBase, iClone, **Mixamo (X/Y Bots)**, "General Humans". Workflow: pick Source Armature + action → **Preview Retarget** (live tweak arm spread, leg spacing, heel/toe angle, root height) → **Bake Retarget** (new action named `RigifiedChar | | ClipName`) → **Retarget Shape Keys** copies facial/viseme shape-key animation onto every mesh with matching shape keys. iClone facial performances arrive as "iCTM" temp-motion actions. NLA editor recommended for stacking body + shape-key actions. +- **Round-trip:** `Export to CC3/4` button requires the **FbxKey** from the original CC export; writes back JSON material params/textures for lossless re-import into CC. **Export to Unity** (.fbx or .blend); after Rigify, three export modes: mesh-only, armature+animation-only, or both. + +Sources: https://github.com/soupday/cc_blender_tools , https://soupday.github.io/cc_blender_tools/usage.html , https://soupday.github.io/cc_blender_tools/animation.html , https://soupday.github.io/cc_blender_tools/adv-animation.html , https://soupday.github.io/cc_blender_tools/pipeline.html + +## 6. CC Skeleton & Retargeting to Other Rigs + +- **Naming convention:** every bone prefixed `CC_Base_` — e.g. `CC_Base_BoneRoot`, `CC_Base_Hip`, `CC_Base_Waist`, `CC_Base_Spine01/02`, `CC_Base_NeckTwist01/02`, `CC_Base_Head`, `CC_Base_L_Clavicle/L_Upperarm/L_Forearm/L_Hand`, `CC_Base_L_Thigh/L_Calf/L_Foot/L_ToeBase`, plus twist chains (`CC_Base_L_UpperarmTwist01/02`, `ForearmTwist`, `ThighTwist`, `CalfTwist`) and full finger chains [names verified via CC4 Bone List/Bone Manager docs and community presets; exact full hierarchy: see CC4 Bone List F3 panel]. GameBase conversion produces a simplified skeleton with different names (breaks soupday Rigify — see issue #60). +- **Not Mixamo-named** (`mixamorig:` ≠ `CC_Base_`); retargeting is by humanoid mapping, not name matching. soupday retarget handles Mixamo→CC-Rigify directly. +- **Blender retarget options:** (a) soupday Rigify retarget (best for CC targets); (b) **Auto-Rig Pro** Remap — community bone-list preset "Character Creator 4 to Auto-Rig Pro IK" exists (gist by muratagawa); ARP is the commonly recommended robust option; (c) **Rokoko Studio Live** addon Retargeting tab — select source/target armature → **Build Bone List** → fix mappings → retarget; a ready-made CC3 bone-list JSON is sold/shared on Gumroad; addon has Blender-version lag issues. +- **To a Quaternius-style simple rig:** map only the ~20 core humanoid bones (Hips←CC_Base_Hip, Spine←Spine01/02 collapsed, Head, UpperArm/LowerArm/Hand, UpperLeg/LowerLeg/Foot); ignore twist bones and finger chains, or bake twist rotation into parent limb bones before transfer. Rest-pose difference (CC A-pose vs Quaternius T-pose) must be corrected with a rest-pose alignment step (ARP "Redefine Rest Pose" / Rokoko auto rest-pose matching) or shoulders will bake ~45° off. + +Sources: https://manual.reallusion.com/Character-Creator-4/Content/ENU/4.0/04_Introducing_the_User_Interface/Bone-List.htm , https://gist.github.com/muratagawa/ed07632bdd07bdec442fa194821b92b9 , https://github.com/Rokoko/rokoko-studio-live-blender , https://support.rokoko.com/hc/en-us/articles/4410463481489 , https://github.com/soupday/cc_blender_tools/issues/60 + +## 7. FBX → GLB → Godot + +**Blender leg:** +- CC FBX is authored in **centimeters**; Blender's FBX importer compensates with **armature object scale 0.01** (and Edit Mode shows 0.01 scaling). Fix: use soupday importer, or apply transforms (select armature+meshes → `Object > Apply > All Transforms`) before doing anything else; the classic manual fix is scale ×100 → Apply Scale → ×0.01 dance when round-tripping to engines. +- Standard importer bone options: `Automatic Bone Orientation` + `Force Connect Children` make bones viewable but **alter bone rolls/joint rotations** — avoid if you will export animation back out; CC bones have zero-length/leaf-bone issues that iClone's "Reset Bone Offset" advanced option mitigates. +- **GLB export (File > Export > glTF 2.0):** glTF is Godot's recommended format. Key settings: `Data > Armature > Export Deformation Bones Only` = ON (required when meshes have shape keys, prevents shading errors; also strips Rigify control bones), Include `Shape Keys` + `Shape Key Animations` [UNVERIFIED exact checkbox names per Blender version], Animation mode `Actions` or push actions to **NLA tracks** so every clip exports as a named animation; `+Y Up` handled automatically; enable Backface Culling on materials. If exporting a Rigify character, bake action to the deform (CC_Base_) bones first or export with "Export Deformation Bones Only". + +**Godot leg:** +- **Godot 4.3+ imports .fbx natively via ufbx** — no external converter; handles unit conversion (cm→m applied to mesh data, not node scale), no extraneous RootNode. Pre-4.3 / legacy files use **FBX2glTF** (requires downloaded binary; per-file importer switch in Import dock; switching importers changes node hierarchy, rest poses, surface order). Direct iClone-FBX → Godot-ufbx works but skips Blender-side cleanup (material rebuild, morph naming) [UNVERIFIED quality for CC characters specifically]. +- GLB path is still the most predictable: animations arrive in an `AnimationPlayer`, blendshapes as mesh blend-shape tracks. Import dock: set `Animation > FPS` (default 30 — raise to 60 for iClone-sourced clips to avoid resampling loss [Godot 4 samples at import]), loop suffixes (`-loop`/`-cycle` name suffix or import options), and use **Root Motion** by assigning the hip/root bone as `root_motion_track` in AnimationTree. + +Sources: https://godotengine.org/article/introducing-the-improved-ufbx-importer-in-godot-4-3/ , https://docs.godotengine.org/en/stable/tutorials/assets_pipeline/importing_3d_scenes/available_formats.html , https://docs.godotengine.org/en/stable/classes/class_editorsceneformatimporterfbx2gltf.html , https://blenderartists.org/t/bringing-armature-into-edit-mode-scales-it-to-0-01-no-matter-what-i-do/1485346 + +## 8. Common Pitfalls Checklist + +1. **60 fps:** iClone's realtime engine/timeline runs at 60 fps [UNVERIFIED whether iClone 8 allows changing project fps]; set **Export Animation FPS** deliberately and match Godot's import bake FPS, or clips get resampled twice. +2. **cm units / 0.01 scale:** CC/iClone FBX = centimeters. Blender import → 0.01-scaled armature; apply transforms before retarget/export or child meshes, physics, and GLB scale drift. Godot ufbx converts units into mesh data automatically. +3. **A-pose vs T-pose:** CC3+ (CC4 default) characters' **bind pose is A-pose**; CC1/CC3 Base is T-pose. The Export dialog's T-pose option only *prepends a T-pose frame* — the skin bind pose stays A-pose (Feedback Tracker: "Exported FBX character stays in A pose when I select T pose"). Use **First Frame in Bind-Pose** so retargeters can read frame 0; align rest poses before baking onto T-pose target rigs. +4. **Bind pose only appears with animation:** importing CC FBX into Blender *without* animation shows bind pose, not the current pose you saw in iClone. Conversely, Blender's FBX exporter writes bind poses only when a weighted mesh is exported — armature-only motion FBX loses bind pose. +5. **Twist bones:** CC twist chains (`...Twist01/02`) carry real rotation; dropping them without baking their rotation into parent bones causes candy-wrapper forearms/thighs on simple rigs. +6. **Facial blendshapes:** enable **Convert Skinned Expressions to Morphs** (iClone) and don't over-prune with **Delete Unused Morphs**; in Blender keep shape keys + use `Export Deformation Bones Only` in glTF export; shape-key actions need **Retarget Shape Keys** (soupday) to follow a retarget; verify blend-shape tracks in Godot's imported AnimationPlayer. +7. **Root motion:** iClone motions move `CC_Base_BoneRoot`/Hip in world space; decide in-place vs root-motion before export (flatten constraints first), then configure Godot AnimationTree root-motion track; Rigify retarget offers root-height adjustment at preview time. +8. **Store-content license wall:** FBX/USD export silently blocked per-asset without Export License — audit content packs before building a pipeline around them. +9. **GameBase/non-standard skeletons:** break both soupday Rigify and CC4 BVH export; keep characters on the standard CC_Base_ skeleton until final export. diff --git a/.claude/skills/iclone-video-mocap/references/iclone8-core-animation.md b/.claude/skills/iclone-video-mocap/references/iclone8-core-animation.md new file mode 100644 index 0000000..c7e65c8 --- /dev/null +++ b/.claude/skills/iclone-video-mocap/references/iclone8-core-animation.md @@ -0,0 +1,170 @@ +# iClone 8 Core Animation System — Technical Reference + +Compiled 2026-07-15 from manual.reallusion.com (iClone 8 Online Manual, sections 8.0–8.5), Reallusion KB, and Reallusion forums. Excludes the Video Mocap plugin (see video-mocap-plugin.md). `[UNVERIFIED]` marks low-confidence items. + +## 1. Character Animation Architecture + +### 1.1 Core model: Motion Clips + Layer Keys +- "A **Motion** or **Animation** is the combination of the **Motion Clip** and the **Layer Keys**." Since iClone 1.x-era redesign, **layer keys are encased inside motion/animation clips**: dragging a clip drags its layer keys; a clip's layer key **cannot be moved outside the range of the clip**. +- Motion track has sub-tracks: **Motion Layer** keys (per body part: e.g. `R Arm`, `LFingers`, `RFingers`) and **gesture layer clips** for both hands (LHand/RHand). Gesture data can be **Sampled** (extracted into LFingers/RFingers Motion Layer tracks) or **Flattened** (merged back into the motion clip). +- **Relative Motion Layer Keys** blend an offset into the underlying motion. Old "absolute" motion layer keys were removed — the equivalent is removing the motion and posing via Edit Motion Layer. +- A motion layer key's effect **persists to end of clip until another key is set**; transitions between consecutive layer keys are auto-generated. + +### 1.2 Edit Motion Layer panel (Modify > Animation tab > Edit Motion Layer; menu Animation > Edit Motion Layer; hotkey **N**) +Engine is **HumanIK** ("automatic floor contact while the intuitive real-time IK motion control keeps the body balanced"). Panel sections: +1. **Set as Default / Reset to Default** (base pose retrieval; Default button adds a layer key standing the character straight in default pose). +2. Mode tabs: **IK / Mask / Detail**. +3. **Body pictogram** with effector-point states: Selected / Free / Move-locked / Rotate-locked / fully Locked / blend-weight-activated joints; quick lock toggle. +4. **Pin presets**: lock joint translation, joint rotation, Both Feet, Both Hands, Hands+Feet, Both Toes, Unlock All. +5. **Mirror controls**: edit bilateral bones simultaneously, copy bone status to opposite side, mirror entire pose. +6. Selection modes: **Full Body** (torso may drag along when an effector moves; Ctrl+Click multi-select; keys inserted on ALL Motion Layer sub-tracks except fingers) vs **Body Part** (limbs/head only; key lands only in the relevant sub-track, e.g. R Arm). +7. **Set Key** and **Reset Pose** buttons. +8. **Viewport Controller** (colored on-screen bone pickers), **Bone Display Mode** (bones/bounding boxes), Bone Settings (Size, Affect all Bones, Color, Opacity). +9. Palm pictogram for **hand gestures**. +10. **Effector Blend Weight** activate/deactivate. +11. Three **Mirror Pose** buttons (incl. Pose Mirror Right to Left, Full Body Pose Mirror). +12. **Vertical Pose Alignment** toggle — includes Z-axis values (normally only X and Y are aligned → confirms Z = vertical axis in iClone). +13. **Properties** dropdown for numeric translate/rotate entry. +14. **Copy Pose / Paste Pose**. +- IK posing: pick an effector (e.g. hand), use Move tool (W) and drag in viewport — 2D mouse motion drives the limb; manual says "make sure your target body parts face the camera" for precise 2D→3D control. Rotate tool (E) = FK-style joint angle edits. **Foot Contact / Hand Contact** checkboxes prevent ground penetration. Recommended: deactivate sub-joint blend weights and pin (lock) bones you don't want dragged before editing. +- **Pose Mixer / Mask Mode** (inside Edit Motion Layer): pick body parts on pictogram (Select All / Select None / Invert Selection), then apply `*.rlPose` files from Content Manager — only masked parts take the pose; repeat with other pose files to compose partial poses; masked mirroring via Pose Mirror buttons. Custom poses saved via Add/save to library (`.rlPose`; iClone 8.4 added save-custom-poses page). +- **Edit Pose vs motion layer keys**: "Edit Pose" panel is the Character Creator 4 equivalent (CC manual: "Using the Edit Pose Panel", same Body Key Editor); inside iClone all posing goes through Edit Motion Layer and always lands as **Motion Layer keys** relative to the current clip (or on an empty motion as pose-to-pose keyframing). +- Layering workflow onto existing motion: apply clip → Edit Motion Layer → go to frame → adjust effectors → key auto-added to Motion Layer track and blended into the Motion-track pose. + +### 1.3 Animation Layer panel (F11; iClone 8.x, distinct from the single Motion Layer track) +- Stores pose keys in stackable named layers to "set their weight together". Toolbar: **Add New Layer, Duplicate Layer, Delete Layer, Rename Layer, Merge** (merge as **optimized keys** or **sampled key per frame**), reorder up/down, **Lock**, layer **Color**, **activation toggle** (mute), **Weight slider** with **weight keyframing** over time. Supports per-body-part layering (manual: "Animation Layers for Body Parts"). + +### 1.4 Reach Target (Modify > Animation tab > Reach Target; menu Animation > Reach Target) +Panel: (1) **Bone Edit Mode** checkbox (show bones); (2) **Bone Settings** (Bone Size / Affect to all bones / Color / Opacity); (3) **Pictogram** effector states: Selected / Free / **Reaching** (contacts target) / Motion-compelling (keeps original motion); (4) **Reach Target** section incl. **Select Target and Keep Current Pose** button (reach-key effect without snapping to the target); (5) **Reach Mode** — **Position** checkbox (hand snaps/aligns to target pivot) and **Rotation** checkbox (also follows target orientation); (6) **Reach Offset** (position and angle offsets from the target); (7) **Lock to Origin** (pin effector to original position); (8) **Release** (sets a release key on the timeline). +- **Lock to Original workflow** (protecting existing pose while one part reaches): lock affected effectors **one at a time** (multi-selection not allowed for locking); secondary drift may require additional locks; verify by playback. +- Reach keys/constraints are **not** exported directly to FBX — use **Menu > Animation > Flatten All Motion with Constraint** to bake Link / Look At / Attach / Path / Reach keys into motion clips first. +- Effector **dummy props** can be used as reach targets to fix motions ("Fixing Motions with Effector" page); Look At / Reach / Path control also via dummy props (Set > Prop > Using Dummy Props). + +Sources: manual pages `50-Animation/Motion-Layer/Introducing_the_Edit_Motion_Layer_Panel.htm`, `Using_Body_Key_Editor.htm`, `How_to_Use_IK.htm`, `Layering_Motion_Layer_Keys_to_Exising_Motions.htm`, `Using-Mask-Mode.htm`, `50-Animation/Animation-Layer/Introducing-the-Animation-Layer-Panel.htm`, `50-Animation/Reach_Target/*.htm`, `51-Animation-Timeline-Editing/Clips_and_Keys.htm`. + +## 2. Timeline (F3; Shift+F3 focuses it) + +### 2.1 Tracks +Per-object track stack includes: **Transform**, **Motion** (with Motion Layer + gesture sub-tracks), **Facial layer tracks**, **Viseme** (expand triangle → **Voice** and **Lips** sub-tracks), Expression tracks (see §5), **Visible**, Effect/HDR/IBL property tracks, **Audio/Sound + Lips** sync tracks, **Part tracks** (bone-skinned objects) and **Bone tracks** (individual bones + IK effectors), **Collect Clip** track, and the **Project track** (flags/markers). Track show/hide hotkeys: O, Q, T, M, R, Z, E, L, V, S, D, I. + +### 2.2 Clip operations (toolbar / right-click / hotkeys) +- **Add Key** `A`; **Break** `Ctrl+B` (non-destructive split, hidden data preserved); **Break Flatten** (destructive split, discards data beyond boundaries); **Auto Extend** (expands clip when keys fall outside; also Menu > Animation > **Clip Auto Extend**); **Loop** `1` (drag right clip edge); **Speed** `2` (drag edge; shown as % on clip title bar); **Resize Clip** `3`; **Motion Direction Control** `4`; **Motion Modifier** `5`; Cut/Copy/Paste `Ctrl+X/C/V`; duplicate = `Ctrl+LMB drag`; Delete key. Prev/Next key: `Ctrl+Left / Ctrl+Right` (toolbar also lists Tab/Shift+Tab). Playback range on/off: `'` / `;`. Zoom `+`/`-`, Fit `Ctrl+*`, **Actual Size** `Ctrl+/` (1 cell = 1 frame at 60fps). Insert/Delete frames for project or single object; Mark In/Out; flags on Project track. +- **Reversing Clips**: right-click clip > Reverse (page `Reversing_Clips.htm`). +- **Time Warp** (right-click motion clip > Time Warp): default = **Linear**; presets **Ease In / Ease In & Out / Ease Out / Ease Out & In**; **Custom** panel = curve-type dropdown + variation slider (left ≈ linear, right = more dramatic). Changes speed *within* the clip. +- **Transitions between clips**: transition range sits ahead of the latter clip; each clip has 4 draggable triangles: **Fade In Start** (weight 0) / **Fade In End** (100) / **Fade Out Start** (100) / **Fade Out End** (0); blend progression between lower triangles is always **linear**. Weights do nothing if only one clip is on the track. +- **Transition Curves between two keys** (Transform and other key tracks): set 2 keys, right-click the **later** key > **Transition Curve** > pick from **16 curves + Default** (Ease In, Ease Out, Ease In & Out, Pause and Move, Gain momentum before Move, Damping, Gain momentum and Damp, Stutter Start, End in a Bounce, Stuttering Start and End, Elastic Start and Sudden End, Sudden Start with Elastic end, Elastic Start and End, Step, Linear, Default). 4 basic presets have a **Strength** slider; **Default** curve comes from **Preference panel > Default Key Tangent**. Live viewport preview when hovering choices. +- **Hollow vs solid keys**: hollow = partially keyed frame (not all axes keyed — common after Sampling due to optimization); solid = all axes keyed. Editing hollow keys in detail requires the **Curve Editor**. + +### 2.3 Sampling & Flattening (right-click on motion/animation clip) +- **Sample Motion Clip to New Layer** >> **Sample All Parts - Optimized** | **Sample All Parts - Per Frame** | **Sample Selected Parts** — exposes clip contents as **Layer Keys** (standard CC characters). Props/accessories: **Sample Animation Clip**. +- **Flatten Motion Clip** >> **Flatten All Layers** | **Flatten And Keep All Keys** — merges Layer Keys into the clip. For clips with Speed/Time Warp applied, keys are **recalculated and optimized** rather than merged, and cells re-match timeline units (standard characters only). Menu names differ for standard vs non-standard characters vs props. +- **Menu > Animation > Flatten All Motion with Constraint** bakes constraint keys (Link, Look At, Attach, Path, Reach) — required before FBX export of constrained animation. + +### 2.4 Collect Clip track +- Open Collect Clip track → click-drag a range covering the desired clips/keys → right-click: **Add Motion to Library** (actors; merges/compacts Transform + Motion + Hands + Motion Layer tracks into one motion, save to Custom library), **Add Gesture to Library** (hand motions), **Add to Perform List** (iProps: Transform + Animation tracks), **Add to Perform** (props: Transform track). Re-import via right-click Motion track > **Import**, or the Perform menu. This is the standard way to consolidate a mocap-plus-layer-edit result into a reusable `.iMotion`/MotionPlus asset. + +### 2.5 Motion Direction Control (hotkey 4) +Fixes the "transform keys fight motion clips" problem: select clip > Motion Direction Control button → leading line(s) show clip travel direction; **Rotate** fields (exact degrees), **Translate** fields (initialize clip start position), or use Object Rotate tool visually. With multiple clips, adjust the second clip while keeping adjacent motions aligned/connected. Related: **Menu > Animation > Auto Motion Alignment** >> **No Alignment / Align Position Only / Align Position and Direction**. + +Sources: `51-Animation-Timeline-Editing/Timeline-Operation-Basic.htm`, `Clips_and_Keys.htm`, `Sampling_and_Flattening.htm`, `Transition_Methods_in_Tracks.htm`, `Creating-Transition-between-Clips.htm`, `Time_Warp_for_Motion_Clips.htm`, `Collect_Clip_Track.htm`, `Using-Motion-Direction-Control.htm`, `90_Keyboard_Shortcuts/Timeline_Shortcuts.htm`, `02-Main-Menu/Menu_Animation.htm`. + +## 3. Curve Editor (built-in plugin; F12, Shift+F12 focuses; separate manual section 58-Curve-Editor) + +- Purpose: classic-animation curve control over iClone keys ("time and space adjustment", ease in/out, arcs, smooth/linear camera moves). Select motion keys in Timeline, edit their curves. +- UI: **Menu bar** (Key, Tangent, Navigation, Zoom, Optimize/Filter, Curve menus); **Toolbar** areas: Outliner Display Options, **Tangent Tools**, **Handle Tools**, **Key Editing Tools**, Navigation Tools; **Search bar**; **Outliner** (structure nodes + curve nodes of selected object); **Edit mode** dropdown: **Layer Keys** | **Transformation**; **Time mode**: **Project Time** | **Clip Time**; Current Frame field; **Graph view**; **FK Mode** button (launches motion-layer editing for bone selection); Status bar with Frame/Value numeric fields. +- Key Editing Tools: **Move Key** (Shift+Drag = single-axis constraint), **Offset Key** (shifts entire sequence before/after to preserve surrounding animation), **Add Keyframe** (directly on curve; toggle "Add Keyframes for All Curves"), **Remove Keys**, **Scale Keys** (bounding-box: horizontal = speed, vertical = magnitude). +- Filter menu: **Optimize** (Threshold 0–1 curve-fitting; higher = smoother/fewer keys; options: Optimize Selected Keys only / Selected Curves only / Selected Keys on Selected Curves; Preview shows before/after key counts — for densely keyed mocap curves), **Smooth** (Smooth Factor strength, **Falloff** to avoid cliffs at range edges, **Keep Custom Tangent**, Preview), **Butterworth** smoothing (**Cut-off (Hz)**, **Sample Rate (fps)**, Falloff, **Align to frames** = float keys → integer frames, Preview). These are the primary **mocap jitter cleanup** tools — sample the imported mocap clip to layer keys first, then filter. +- Tangent menu/tools include standard tangent types (Auto/Linear/Step/Ease etc.) `[UNVERIFIED — exact list not confirmed]`. Dedicated Curve Editor keyboard shortcut page exists (`58-Curve-Editor/Keyboard_Shortcuts.htm`). +- Gotcha: to curve-edit inside a motion clip you must expose keys (Sample) — clip contents themselves aren't directly curve-editable; hollow keys are adjusted here. + +Sources: `58-Curve-Editor/Welcome_to_Curve_Editor.htm`, `UI_Intro_for_Curve_Editor.htm`, `UI_Intro_for_Curve_Editor_Key_Editing_Tools.htm`, `UI_Intro_for_Curve_Editor_Menu_Optimize.htm`. + +## 4. Motion Correction / Cleanup Tools + +### 4.1 Motion Correction (foot/hand sliding fix) +- Right-click motion clip > **Motion Correction**. Derived from Reach: calculates ground-contact times, auto-creates **hand/foot prints**, sets Reach keys to feet/hands so they stick to prints. +- Panel: **Presets** (pre-optimized body part sets), **Body Parts** (manual per-part activation), **Parts Settings**: **Threshold** (higher = severe sliding, lower = minor sliding — the main dial), **Transition** (frames for Reach/Release key transition), **Reach Offset** / **Release Offset** (shift Reach/Release keys left/right), **Reach Mode** checkboxes (touch prints / align to print directions), **Sync Threshold with all Parts** checkbox. Click **Correct**. +- Iteration gotcha: after each Correct, run **Flatten Motion Clip > Flatten All Layers**, deactivate the already-fixed parts, then correct remaining parts; repeat. +- This is the primary post-import cleanup pass for external/video mocap clips. + +### 4.2 Motion Modifier (Timeline button, hotkey 5) +- Slider-based whole-clip body-part adjustment ("fix posture issues or alter overall look"). **Base Pose** options: **Default Pose** (motions that don't change much), **Auto Generate** (averages pose for highly varying clips), **Load** (use first pose of a loaded iMotion). **Part Modify** section with **Stand Mode** / **Move Mode**. Converts entire clip into an altered one. Not recommended for dramatic motions (big turns/heavy transforms). + +### 4.3 Motion Director (iClone 8 flagship; replaces MixMoves conceptually) +- Behavior files (**iMD**) drive characters: each character can have multiple **Behaviors**; each behavior has **Modes** (currently **Casual** and **Athletic**); modes contain speeds ('Stroll', 'Walk', 'Jog'); each speed has perform-motions and **Mix-motions** ('Main Mixer' + sub-mixers). +- Control: keyboard/gamepad (WASD-style; **Ctrl+R** start/stop playback-record; Shift = accelerate a level, Ctrl = decelerate; customizable via **Hotkey Manager**, Edit > Hotkey Manager, Ctrl+`); waypoints via **Alt+left-click** (queue) and **Alt+right-click** (absolute, cancels others). Cameras: **MD Camera** (system), **Free Camera**, **Follow Camera** (8 presets + 5 customizable) on MD Triggers panel. **Snap to Surface** for terrain-following characters (8.1). **Menu > Animation > Create iMD Cache** pre-retargets actors for MD performance. MD Props with **Reach Effect** settings exist (8.5). +- Recording writes normal motion clips to the timeline, editable with everything in §1–§3. + +### 4.4 MixMoves — REMOVED in iClone 8 +- The iClone 7 MixMoves system (automatic motion-graph transitions) is **not in iClone 8**; legacy MixMoves animations still usable as regular motions (Reallusion forum confirmation). Motion Director is the successor. + +### 4.5 Motion Puppet / Direct Puppet / Prop Puppet / Morph Animation (Menu > Animation) +- **Motion Puppet**: record motions from puppet templates onto the selected character (mask body parts, mouse-driven). **Direct Puppet**: simultaneously move+record body to generate motions. **Prop Puppet**: puppet-record prop animation. **Morph Animation**: Morph Animator panel — keyed morphing sliders (also how morph-based prop/character shape animation is keyed). **Edit Animation Layer**: layer keys on existing prop animations. + +### 4.6 Smoothing +- Body: Curve Editor **Smooth / Butterworth** filters (§3), Optimize for key reduction. +- HumanIK foot floor contact: automatic in Edit Motion Layer (Foot/Hand Contact checkboxes). Character-level contact geometry comes from CC's **Floor Contact Planes** (soles/palms; toes/fingers bend to avoid penetration) — set in Character Creator, honored by iClone terrain snapping. + +Sources: `51-Animation-Timeline-Editing/Motion-Correction-for-Sliding-Issues.htm`, `Modifying_and_Fixing_Motions.htm`, `50-Animation/Motion-Director/Basic-Concepts.htm`, `Setting-Hotkeys-with-Hotkey-Manager.htm`, `Snap-to-Surface.htm` (8.1), `02-Main-Menu/Menu_Animation.htm`, forum.reallusion.com/519260 (MixMoves), CC manual `06_Body/Setting_the_Floor_Contact_Planes.htm`. + +## 5. Face Animation + +### 5.1 Track structure +- **Viseme track** → sub-tracks **Voice** (audio clip) and **Lips** (auto-generated viseme keys). **Expression** timeline = 4 tracks: **Muscle** (morph data), **Eye Blink**, **Eyeball**, **Head** (orient/tilt). **Facial Layer** tracks hold Face Key output. Face clips support Auto-Extend, Loop, Speed; layer keys copy/paste only within their originating clip. **Sample/Flatten** available to convert expression clips ↔ three-track layer keys. **Menu > Animation > Expression Clip Auto Crop** trims overlapping expression clips. + +### 5.2 AccuLips +- Open: **Modify > Animation tab > Motion section > Facial group > Create Script**, or **Menu > Animation > Create Script > AccuLips** (other Create Script options: Record Voice, Text-To-Speech, Wave File). +- Works only for **CC G1–G3+ Standard and CC Gamebase** characters. Audio: wav, mp3, m4a, aac, wma. +- Workflow: Open Audio File / Record Voice → **Generate Text** (speech-to-text; fix red-highlighted words) → **Align** (word-by-word sync; or **Update Selected** for a range) → adjust word durations → **Apply** → visemes generated on Lips track. Optional: load a TXT/SRT script with identical filename to skip transcription; custom viseme-word pairs saveable to **dictionary**. TTS input auto-aligns. Old iTalk scripts convertible. + +### 5.3 Viseme editing & strength (3 levels) +1. **Project/character level**: Modify > Animation tab > **Facial Animation Settings > Viseme Strength** slider. 2. **Clip level**: right-click voice clip on Viseme track > **Talking Style Editor** (hotkey 6) — per-clip viseme strength sliders. 3. **Individual key**: double-click viseme key (or empty cell to add) → **Lips Editor** (hotkey 7) — phoneme choice (e.g. "Phoneme: Oh") + **Expressiveness** slider (result visible after slider release). +- **Smoothing mouth**: right-click Lips/Lip Options track range > **Lip Sync Options** → radio **Full Mouth** (+Smooth checkbox + smooth value) or **Parts** (independent lips / tongue / jaw sliders). + +### 5.4 Face Key panel (Menu > Animation > Face Key, or Modify > Animation tab) +- Tabs: **Expression** (preset expression style sets via dropdown), **Modify** (detail morph sliders grouped: **Brow, Eye, Nose, Cheek, Mouth, Custom**), **Morph** (**Expressiveness** slider + **Revert**). **Muscle** editing: click highlighted face regions and drag in viewport to sculpt an expression; **Muscle Symmetrical Selection** for mirrored edits; **Reset to Zero** button. Keys land on **Facial Layer** tracks; blend sequential keys by placing the playhead between them. Muscle vs Detail distinction: Muscle = region-drag posing, Detail = per-morph sliders. + +### 5.5 Face Puppet (Menu > Animation > Face Puppet) +- Real-time mouse-driven expression recording. Profiles: **Male, Female, Stylized** ("00_"-prefixed enhanced versions), **Universal** (mouth shut + six mouth expressions). **Facial Profile Strength** slider exaggerates/smooths. **Solo Feature Profile** states: None / Active / Selected. During Preview/Record: **left-click = blink**, hotkeys **Q W E R T Y** trigger Full Face Control Profile commands. **Profile Details** (Edit Property) table: 60 standard + 24 custom morphs. Preference > Control > "Confine Face Puppet area to 3D view-port". Attribute tab **Auto-Blink** toggle (disable during puppeteering). + +Sources: `50-Animation/Facial-Animation/Creating-Voice-Script-with-Accurate-Lipsync.htm`, `Three-Levels-of-Setting-Strength-for-Visemes.htm`, `50-Animation/Pro_Facial_Animation/Face_Puppet.htm`, `Face_Key_Editing.htm`, `Facial_Timeline.htm`, `51-Animation-Timeline-Editing/Adding_or_Modifying_Lip_Synching_Keys.htm`, `Smoothing_Mouth_Movements.htm`. + +## 6. Characters & Characterization + +- **CC4 Standard characters** (CC3+/CC Gamebase etc.): full feature support — Edit Motion Layer/HumanIK, facial system (AccuLips requires CC G1–G3+ Standard/Gamebase), Motion Director, sampling with "All Parts - Optimized". Created/edited in Character Creator 4; CC4's **Characterization panel** data (bone-mapping, bone-rotation, floor contact, **HIK property data**) is what iClone uses for retargeting. +- **ActorCore characters/motions**: Reallusion's store rigs (AccuRig-rigged); load directly in iClone and accept iClone/ActorCore motions; AccuRig converts arbitrary humanoid meshes → compatible rigs. +- **Non-standard/Humanoid characters**: custom FBX rigs are converted via CC4's **Convert to Non-Standard / Humanoid** (Characterization mode: map custom bones to standard iClone bones, adjust to **T-pose** — all `.iMotion` are T-pose-based; skipping T-pose causes body-part offsets when motions apply). 3DXchange's old conversion role now lives in CC4/AccuRig. Feature availability in iClone differs (e.g. sampling menu items differ for non-standard characters; AccuLips unsupported). +- **Imported mocap interaction (core side)**: **File > Import > Import External Motion** (iClone 8.4+) accepts **FBX or BVH** motion for standard/humanoid characters. Options: **Motion Profile** (from CC Characterization data), optional **T-pose** file from the same source, **Sample Per Second** (higher = more detail), **Root Bone** dropdown (choose source root node), **Auto-generate to Perform List**; **Convert All** → clips stored under custom **Motion > External Motion** library. Once converted, mocap clips are ordinary motion clips: trim/loop/speed/time-warp, Motion Correction, sample-to-layer + Curve Editor filtering, Edit Motion Layer layering, Reach Target locking. +- **Props/Accessories**: animate via **Transform track** keys, **Prop Puppet**, **Edit Animation Layer**, Morph Animator; clip form = Animation clips ("Sample/Flatten Animation Clip"); collect via Collect Clip > Add to Perform (List). Constraint animation via Link/Attach/Path/Look At dummy workflows, flattened before export. + +Sources: `8.4/50-Animation/Import-External-Motions/Import-External-Motions.htm`, CC4 manual `09-Import-Humanoid/*`, 3DXchange 6 legacy `Converting_Models_to_Non_Standard_Characters.htm`, actorcore.reallusion.com learn-and-support. + +## 7. Import / Export + +- **Menu paths**: `File > Export > Export FBX...` | `Export Alembic...` | `USD (Omniverse)`. `File > Import > Import External Motion` (mocap in). 3DXchange is **discontinued**; all conversion/export features merged into iClone 8 / CC4 (kb.reallusion.com/Product/53036) — iClone 8 ships pipeline-capable (FBX/OBJ export built in, no separate Pipeline SKU). +- **Export FBX dialog** (verbatim, from manual screenshots): **Target Tool Preset**: Maya, 3ds Max, Unity 3D, Unreal (UE4 Skeleton), Unreal (UE5 Skeleton), Motion Builder, Blender, Cinema 4D, **Maya (Motion Pipeline)**, **Motion Builder (Motion Pipeline)**, Marmoset Toolbag. Preset changes downstream settings incl. axis conversion (no manual Y-up/Z-up radio in iClone 8; legacy 3DXchange had explicit Y-Up/Z-Up choice). **FPS** dropdown (default `Project(60)`). **Export Range**: T-Pose (no animation, avatar moved to origin (0,0,0)) / Current Frame / All / Range (From/To). Checkboxes: **Embed Textures**, **Embed Timeline Timecode**, **Max Texture Size** (default on, 2048), **Convert Image Format to** (default Bmp shown; JPG if unset), **Delete Hidden Face**, **Use Smooth Mesh**, **Convert Skinned Expressions to Morphs**, **Merge Face Hair to One Object**, **Delete Unused Morphs**, **Tear Ducts UV Adjustment** (character can't round-trip to CC if on). +- **Export FBX Advanced Settings** (gear icon): **Merge Opacity to Diffuse Texture**, **Preserve Bone Names (CC Base)** (default on), **Mouth Open as Morph**, **Epic Skeleton** (UE4 Skeleton / UE5 Skeleton radio), **Epic Skeleton Hand and Foot IK**, **Reset Motion Root**, **Export Weight-map to Vertex Color** (default on), **Use T-Pose As Bind Pose**, **Reset Bone Scale** (default on), **Add Reference Pose** (default on) with radio: T-Pose for Unity / A-Pose for UE4 Skeleton / A-Pose for UE5 Skeleton / A-Pose for UEFN. Also **Export JSON for Auto Material Setup** (default on, for Auto Setup plugins) `[per Auto Setup manual]`. +- **Root motion**: **Reset Motion Root** — since 7.6, for Unity/Unreal game-engine presets the root node **follows the hip** instead of staying at origin; needed for root-motion scripting/blending in engines. Constraint animation must be flattened first (**Animation > Flatten All Motion with Constraint**). Motion-only export: use the **(Motion Pipeline)** presets. +- **Batch export**: select multiple objects; with Target Tool Preset = **Unreal** or **Unity 3D** each object exports to a **separate FBX** named after the iClone object; any non-game-engine preset merges all picked objects into **one** FBX. +- **Non-exportable types**: iLight, iEffect, iParticle, iTree, iGrass, iTerrain, iWater, iSky, aml, iTalk, iPath. +- **USD**: via free **Omniverse connector plugin** — Menu > Plugin > Omniverse > Export USD, File > Export, or toolbar. Options: Export All/Selected; Material Render Mode (RTX Real-Time = OV PBR shader; RTX Interactive Path Tracing = SSS MDL for digital humans); Export Range > **Motion FPS: 60/30/24**; Send to Server (Nucleus). Outputs main USD + light/material/motion dependency folders. +- **BVH**: **import** supported (Import External Motion); **BVH export is NOT available from iClone 8** — legacy BVH export was 3DXchange Pipeline; CC4 4.4+ has BVH export (see export-pipeline-to-blender-godot.md §3). +- **Blender/Godot**: Blender via Target Tool Preset "Blender" + free **CC/iC Blender Tools** (soupday) importer; Godot has **no preset** — use Unity 3D or Blender preset FBX, or round-trip through Blender→glTF `[workflow inference, not Reallusion-documented]`. +- **Licensing**: Reallusion content items require an **Export License** to leave the Reallusion ecosystem in FBX/OBJ/USD ("You must first purchase the content licenses in order to export FBX and USD files" error otherwise); iClone-only content (legacy "maroon" items) is blocked without it. Your own imported/created assets are unrestricted. FBX-key/`.fbxkey` applies to CC character mesh round-trips. Export License purchase deducts already-owned iContent cost. +- **Alembic** export exists for vertex-cache workflows (`Export Alembic...`). + +Sources: `83-FBX/FBX_UI_Intro.htm` (+ dialog screenshots), `Exporting_FBX_from_iClone.htm`, `Batch_Exporting_FBX_Files.htm`, `Resetting_Motion_Root.htm`, kb.reallusion.com/Product/53036, reallusion.com/ContentStore/Export_License.pdf, Omniverse plugin manual (iC-8.3), forum.reallusion.com/556427. + +## 8. System Fundamentals + +- **Frame rate**: Project Settings > **Time Unit** section — FPS dropdown per project; **default project length 1800 frames, maximum 54000**; timeline "Actual Size" = 1 cell : 1 frame **at 60fps** (iClone's native/internal sampling is 60fps; FBX FPS default `Project(60)`; render/export offers 12/24/25/30/60). Time Mode: Realtime vs **By Frame** (physics). Time Unit display: time-based or frame-based. +- **Units**: **1 iClone unit = 1 cm**; default grid 100×100 units = 1×1 m. CC characters ≈175 units tall. Matches Unreal cm scale directly. +- **Axes**: iClone is **Z-up** (Reallusion: "iClone and 3DXchange are Z-Up based 3D tools"); corroborated by Edit Motion Layer "Vertical Pose Alignment includes Z-axis values". FBX export axis conversion is handled by the Target Tool Preset (Y-up conversion for Maya/Unity etc.). Character bones do **not** share a common up-vector (spine Z-up, legs point down, feet back) — a known pain point when retargeting exported skeletons. A literal `Z_UpAxis` node/track name: not found in iClone 8 docs `[UNVERIFIED — likely a 3DXchange-era conversion dummy]`. +- **Key global shortcuts**: Space play/pause; ,/. start/end frame; ←/→ frame step; **F3 Timeline**, **F11 Animation Layer panel**, **F12 Curve Editor** (Shift+ variants focus); **N = Edit Motion Layer**; Q/W/E/R = Select/Move/Rotate/Scale; Ctrl+Z/Ctrl+Y undo/redo; X/Z/C/V camera Pan/Zoom/Orbit/Roll; Home = 45° perspective; F = front view of selection. Timeline-specific keys in §2.2. All remappable: **Edit > Hotkey Manager (Ctrl+`)** — Device: Keyboard, expand item, press new key. +- **Rotation order**: editable per object via **Menu > Animation > Rotation Order** (Euler order — matters for curve editing and export). **Remove Object Animation** / **Remove Scene Animation** clear animation (scene removal offers frame-position options). + +Sources: `03-Introducing-the-User-Interface/Project_Settings_Time_Unit.htm`, `90_Keyboard_Shortcuts/Global_Shortcuts.htm`, `Timeline_Shortcuts.htm`, `Hotkey-Manager.htm`, 3DXchange5 `Y_Up_or_Z_Up.htm`, forum.reallusion.com/109018 & /326528 (cm units), forum.reallusion.com/525257 (bone axes). + +--- + +Key gotchas worth surfacing: layer keys can't leave their clip; flatten-between-iterations for Motion Correction; Flatten All Motion with Constraint before FBX export; MixMoves removed in iC8; AccuLips is CC-Standard-only; game-engine presets (Unreal/Unity) are the only batch/per-file + Reset-Motion-Root paths; 60fps internal / cm / Z-up when planning Godot import. diff --git a/.claude/skills/iclone-video-mocap/references/video-mocap-plugin.md b/.claude/skills/iclone-video-mocap/references/video-mocap-plugin.md new file mode 100644 index 0000000..62f4de8 --- /dev/null +++ b/.claude/skills/iclone-video-mocap/references/video-mocap-plugin.md @@ -0,0 +1,134 @@ +# Reallusion AI Video Mocap for iClone 8 — Reference (researched 2026-07-15) + +## 1. Product Disambiguation (Reallusion mocap ecosystem, mid-2026) + +| Product | What it is | Body/Face | Input | Pricing model | +|---|---|---|---|---| +| **iClone Video Mocap** (plug-in) | AI body mocap from ordinary video, powered by **QuickMagic** cloud AI. THE "video mocap plugin". Launched **Oct 29–31, 2025** | Body + fingers; **NO face** | Recorded video file | Free plugin; pay-per-use points per generation (cloud) | +| **AccuFACE** (Motion LIVE gear) | AI **facial** tracker from webcam (live) or video file (offline). Local NVIDIA RTX inference | Face only | Webcam or video file | Perpetual plugin, $499 MSRP (launch promo $250, Nov 2023) | +| **Motion LIVE** | Hub plugin aggregating real-time mocap hardware/gear (Xsens, Rokoko, Perception Neuron, Leap Motion, iPhone LIVE FACE, Faceware, AccuFACE) into one unified real-time recording panel | Body+face+hands (via gear) | Live hardware streams | Hub + per-gear profile purchases | +| **Motion Director** | Gameplay-style interactive character locomotion/control system inside iClone (WASD-style driving, not video mocap) | Body locomotion | Keyboard/gamepad | Built into iClone 8 | + +Video Mocap deliberately does NOT do faces — official workflow layers AccuFACE (or iPhone LIVE FACE) on top for talking animations. +Sources: https://www.cgchannel.com/2025/10/reallusion-launches-ai-motion-capture-service-video-mocap/ , https://www.reallusion.com/iclone/motion-capture/default.html + +## 2. Video Mocap — Identity, Versions, Requirements + +- Exact name: **"iClone Video Mocap"** / **"Video Mocap Plug-in for iClone"**. Tagline: "Precise AI Mocap from Any Video". +- Technology partner: **QuickMagic** (AI mocap provider); processing is **cloud/online** (upload video → AI conversion on QuickMagic cloud → download motion). Not local; no GPU requirement for generation itself. +- Requirements: **iClone 8.63 or above**, **Windows 10 & 11 only**. Plugin itself is a **free download**, installed/updated via **Reallusion Hub**. +- Version history: launch ~v1.0 (Oct 2025, alongside iClone 8.63); **v1.02 (announced 2026-02-26)** — urgent fix for task-status-refresh bug that caused re-submission and **redundant point charges** after closing the app; also added diagnostic logs and auto-recovery of task statuses from local video/motion files. Users on iClone 8.71 + plugin 1.02 reported in forum (Mar 2026+). Manual page shows revision date June 8, 2026. +- Official manual root: https://manual.reallusion.com/iclone-video-mocap-plug-in/content/welcome.htm (sections: Workflow, Full Body, Upper Body, Import, Edit, Generate (purchase.htm), Apply, Save, Adding Facial Expression (talking-animation.htm), Video Guidelines (rule.htm)). + +Sources: https://www.reallusion.com/iclone/video-mocap/download.html , https://discussions.reallusion.com/t/urgent-update-video-mocap-1-02-fixes-task-status-refresh-bug-to-prevent-redundant-charges/16659 + +## 3. Pricing + +- Current metering (product page + manual, mid-2026): **5 points per second of processed video** (AI / Bonus / DA Points), **100 points = US$1.00** → **$0.05/sec, max $3.00 for a 60 s task**. Deduction order: **AI Points → Bonus Points → DA Points**. +- At launch (Oct 2025) press reported a flat **250 DA Points (US$2.50) per task** up to 60 s; the Apr 2026 beginner article still says "$2.50 per motion". Flat-250 vs 5/sec discrepancy: pricing appears to have moved to per-second metering; treat exact figure as ~$2.50–3.00/generation [UNVERIFIED which applies at any given moment]. +- Points sources: **DA Points** ($10 / $50 / $100 packs, min $10 spend) or **Reallusion AI Cloud subscription** AI Points (FREE $0 = 500 pts; STARTER $11/mo = 1,100; STANDARD $35/mo = 3,500; PRO $95/mo = 9,500; annual −20%). AI Points are universal across AI Studio, Video Mocap, Headshot image gen. DA:AI = 1:1 value. +- Fully licensed iClone users get complimentary Video Mocap generation coupon(s); one coupon = one task regardless of duration. (Launch promo advertised "100 free AI mocap generations" limited-time; one forum user mentions losing "5 complimentary generations" — exact freebie count varies by promo [UNVERIFIED].) +- Failed tasks: Reallusion support has refunded points for failed tasks case-by-case (forum evidence). Verify tasks/transactions: https://kb.reallusion.com/Print53250.aspx + +Sources: https://manual.reallusion.com/iclone-video-mocap-plug-in/content/purchase.htm , https://ai.reallusion.com/plans.html , https://www.cgchannel.com/2025/10/reallusion-launches-ai-motion-capture-service-video-mocap/ + +## 4. Workflow (official 5 stages: Import Video | Edit Video | Generate Motion | Apply Motion | Save Motion) + +### 4.1 Import Video +- Open the plugin from iClone's Plugins menu after Hub install [menu path UNVERIFIED, standard for RL plugins]. +- Import via **"+ Add Video"** button or drag-and-drop into the panel. +- **13 supported formats**: `.avi .flv .m4v .mkv .mov .mp4 .mpeg .ogv .webm .wmv .3gp .asf .dv .mxf`. +- Source video duration: **1 s – 15 min** accepted; **processing region must be 1–60 s** (longer videos auto-trimmed or must be trimmed). Max file size **400 MB** (else reduce bitrate/rescale/compress/trim). Resolution: **≥720p recommended; 4K accepted but may hurt performance**. FPS not specified. +- Task queue: up to **100 videos**; batch generation: up to **10 tasks at once**. Statuses: Unprocessed → Trimmed → (processing) → Applicable / Error. + +### 4.2 Edit Video panel +- Drag playhead to preview; drag **Start Time** / **End Time** handles to set the processing region (timecode MM:SS:FF); drag the whole region to shift it with fixed duration; **Reset Range** restores full span; **Apply Settings** confirms → status "Trimmed". +- **Detect Actors** button: when multiple people are in frame, click it and pick the single target actor in a frame where they are clearly visible. **One actor per task**; for group scenes, duplicate the task per performer and select a different detection each time. +- Trim out segments where no person appears (hurts AI quality). + +### 4.3 Generate Motion +- Choose tracking mode: **Full Body** (default) or **Upper Body**; optional **"Includes Fingers"** checkbox in both modes for finger capture. +- Full Body: whole body must be clearly visible throughout. Upper Body: designed for framing cut off below the pelvis (sign language, instrument playing, speech); everything below the hip is ignored/not generated. +- Submit → upload → cloud processing, typically **5–8 minutes per task**; progress monitored in the task list. Insufficient points → "Add Funds" / "Cancel" dialog. + +### 4.4 Apply Motion +- Select completed task → **Apply Motion** button (bottom-right), or double-click the task, or right-click → **Apply Motion**. +- Motion is saved as **rlMotion** to a preset folder and applied to the **currently selected character**; if none selected, **a dummy character is auto-loaded**. Auto-retargeting adapts the motion to any iClone/CC4/AccuRIG standard character body type. +- **"Apply reference video"** checkbox: loads the source video as a 2D background object in the viewport behind the character for frame-by-frame comparison during cleanup. + +### 4.5 Save Motion +- Auto-saved to Content Manager **Animation > Motion > VideoMocap**; default disk path `C:\Users\Public\Documents\Reallusion\Reallusion Custom\Animation\Motion\VideoMocap`. Folder button changes path; Reset restores default. Right-click task → **Find Motion File**. Thumbnail from Edit Video frame (Quick Thumbnail on) or posed character (off). + +Sources: https://manual.reallusion.com/iclone-video-mocap-plug-in/content/video-mocap-workflow.htm , .../import.htm , .../edit.htm , .../purchase.htm , .../video-mocap-full-body.htm , .../video-mocap-upper-body.htm , .../apply.htm , .../save.htm + +## 5. Official Video Guidelines (rule.htm) — filming rules + +- **Camera angle**: flat/eye-level. Overhead or elevated angles make the result lean forward/backward. +- **Camera motion**: static camera, single continuous shot, no cuts/switched shots; only minor shake tolerated ("Counteracts camera shakes" is marketing — keep it steady). +- **Framing**: entire actor (or pelvis-up for Upper Body) visible for the whole clip; "any body part that moves out of the frame will present a problem". Actor not too small in frame; size consistent (no zooms/dollies). +- **Clothing**: avoid solid colors and colors matching the background; patterns/texture are fine. +- **Background/lighting**: clean and bright preferred, though the engine tolerates dim light, busy backdrops, patterned clothing. +- **Motion restrictions**: avoid very fast motion, violent shaking, acrobatic flips, vertical actions (pull-ups, jumps), and ungrounded-feet activity (swimming). +- **Occlusion**: single person only; avoid limbs intersecting other actors; avoid hand-occluding poses ("running fingers through hair"). + +Source: https://manual.reallusion.com/iclone-video-mocap-plug-in/content/rule.htm + +## 6. Cleanup / Post-Processing in iClone (official recommended tools) + +Artifact → tool mapping (from Reallusion Magazine + 80.lv + tutorials): +- **Foot sliding** → **Motion Correction** tool (Motion Correction for Sliding Issues, iClone 8 manual) + **Reach Target** to lock/release foot positions; also Foot Contact tool + iClone footstep visualization. Official tutorial: "Mocap Motion Fix – Eliminating Foot Sliding | iClone 8 Tutorial" (https://www.youtube.com/watch?v=t8yeA-DJ0ds). +- **Sudden elevation changes / abrupt vertical pops** → dedicated fix tutorial "Fixing Sudden Elevation Change"; edit hip height in Curve Editor. +- **Depth misalignment / motion offset** → **Auto Motion Alignment** / Root Alignment. +- **Jitter/noise** → **Curve Editor** smoothing. +- **Sudden body twists / rotating limbs** → Motion Layer editing with HumanIK controls. +- **Hand/gesture errors** → iClone gesture tools / hand pose library. +- **Missing or garbage frames** → rotoscope against the 2D reference video and bridge with transition curves. +- The auto-loaded reference-video background is the key rotoscoping aid — compare character to footage in real time. +- **Gotcha**: if the applied clip has **Loop** enabled, motion-layer edits "spring back" (looped frames reference the source); **Flatten** the motion clip (or disable Loop) before keyframe editing (forum-confirmed staff answer). +- Tutorial course hub: https://courses.reallusion.com/home/iclone/pipelines/video-mocap-plug-in + +Sources: https://magazine.reallusion.com/2025/10/29/iclone-video-mocap-converts-videos-into-editable-3d-motion/ , https://80.lv/articles/getting-ai-mocap-ready-for-production-with-reallusion-s-iclone-8 , https://discussions.reallusion.com/t/cant-adjust-motions-after-applying-from-video-mocap/16711 , https://manual.reallusion.com/iClone-8/Content/ENU/8.0/51-Animation-Timeline-Editing/Motion-Correction-for-Sliding-Issues.htm + +## 7. Adding Face (talking animation) — official 7-step layering with AccuFACE + +1. Apply Video Mocap body motion + reference video. 2. In **AccuFACE VIDEO mode**, load a speech video framed on the actor's head (RL strongly recommends **separate recordings for body and face**). 3. Connect AccuFACE in **Motion LIVE**, enable **"Audio for Viseme Track"** (auto voice clip). 4. Preview + record facial clips via Motion LIVE. 5. Align viseme/expression clips to the motion clip on the timeline using the reference video. 6. Verify sync in playback. 7. Save to Content Manager custom library. +Source: https://manual.reallusion.com/iclone-video-mocap-plug-in/content/talking-animation.htm + +## 8. AccuFACE (face-from-video) — key facts + +- Motion LIVE gear plugin; **iClone 8.6+**, Windows 10/11 64-bit, 8 GB RAM, **NVIDIA RTX only** (20/30/40/50 series + RTX Pro; driver 520.46+ or 572+ for RTX 50) — local GPU inference, no cloud, no AMD/Intel support. $499 MSRP (frequent discounts). +- Input: live webcam (min 1280×720@30fps; 720p/30 optimal) or video files (**AVI, WMV, MP4, MKV, MOV**). Captures up to 60 fps facial data; single-pass audio + face capture; **AccuLIPS** refines lip/tongue; output framerate-independent (24/30/60 fps). +- Calibration: guided process (live mode) or pick calibration poses across video frames (video mode). Refinement controls: **Strength Balance** (per-region), **Smooth Filter**, **Denoise**, **Anti-Interference** (cross-region triggering). +Source: https://www.reallusion.com/iclone/motion-capture/accuface.html + +## 9. Known Limitations & Community Issues + +- Hard limits: 60 s/task, 1 actor/task, no face capture, cloud-only (needs internet + points), 400 MB file cap, 10-task batch cap. +- AI-typical artifacts acknowledged by Reallusion itself: foot sliding, depth misalignment, unnatural twists, jitter — cleanup is an expected part of the workflow, not an edge case. +- Failure modes: subject leaving frame, background-matching clothing, fast/occluded motion, spins with limb occlusion, airborne/ungrounded feet. +- Forum-reported bugs: (a) pre-1.02 task-status refresh bug → duplicate charges (fixed in 1.02); (b) failed tasks bricking the queue — Error tasks with retry/delete greyed out and Generate Motion disabled for all new videos; support refunded points but the user had to hand-edit plugin task files to recover (https://discussions.reallusion.com/t/video-mocap-problem/17173); (c) Loop-enabled clips blocking motion-layer edits (see §6). +- Community consensus: results are solid for grounded, single-person, moderate-speed motion; twitchy feet and torso rotation remain the main manual-cleanup costs, consistent with other AI mocap (DeepMotion, Plask). + +## 10. Best-Practice Tips (Reallusion Magazine "Video Mocap for Beginners", Apr 2026) + +- A mid-range phone (Samsung A54) or 1080p webcam (Logitech C922) on a tripod is fully sufficient; ordinary home lighting worked. +- Rehearse, then **overact** mundane motions; record multiple takes. +- **Stack multiple motions in one 60 s clip with ~2 s gaps** between them — one generation fee yields several clips to split in iClone. +- Pre-trim in an external editor; use real props to anchor hand positions and prevent hand drift. +- Use Upper Body mode + Includes Fingers for seated/talking/instrument performances. +Source: https://magazine.reallusion.com/2026/04/13/video-mocap-for-beginners/ + +## 11. Brief Comparison + +- **vs Motion LIVE hardware**: Motion LIVE = real-time, suit/camera hardware ($$$–$$$$), no per-use fee, best fidelity and long takes; Video Mocap = offline, zero hardware, ~$2.50–3/clip, 60 s cap, cleanup required. Reallusion positions them as hybrid-complementary (ActorCore packs + hardware for hero shots, Video Mocap for bulk/reference-driven motion). +- **vs free/DIY (Rokoko Video free tier, DeepMotion, Plask, open-source VIBE/4DHumans pipelines)**: those export FBX/BVH requiring manual retarget; Video Mocap's differentiator is native rlMotion output, auto-retargeting to CC4/AccuRIG characters, the in-viewport reference-video rotoscope, and direct access to iClone's correction toolset — then FBX/USD export from iClone to Unreal, Unity, Blender, Maya, MotionBuilder. + +## 12. Primary Sources + +- Product: https://www.reallusion.com/iclone/video-mocap/ | Download: https://www.reallusion.com/iclone/video-mocap/download.html +- Manual: https://manual.reallusion.com/iclone-video-mocap-plug-in/content/welcome.htm (+ import/edit/purchase/apply/save/rule/talking-animation/video-mocap-full-body/video-mocap-upper-body .htm) +- Magazine: https://magazine.reallusion.com/2025/10/29/iclone-video-mocap-converts-videos-into-editable-3d-motion/ , https://magazine.reallusion.com/2026/04/13/video-mocap-for-beginners/ +- Press: https://www.cgchannel.com/2025/10/reallusion-launches-ai-motion-capture-service-video-mocap/ , https://beforesandafters.com/2025/11/08/iclone-video-mocap-converts-videos-into-editable-3d-motions/ +- AI plans: https://ai.reallusion.com/plans.html | AccuFACE: https://www.reallusion.com/iclone/motion-capture/accuface.html +- Forum: https://discussions.reallusion.com/t/urgent-update-video-mocap-1-02-fixes-task-status-refresh-bug-to-prevent-redundant-charges/16659 , /t/video-mocap-problem/17173 , /t/cant-adjust-motions-after-applying-from-video-mocap/16711 +- Courses: https://courses.reallusion.com/home/iclone/pipelines/video-mocap-plug-in | 80.lv: https://80.lv/articles/getting-ai-mocap-ready-for-production-with-reallusion-s-iclone-8 diff --git a/.claude/skills/pose-estimation/SKILL.md b/.claude/skills/pose-estimation/SKILL.md new file mode 100644 index 0000000..7ed412b --- /dev/null +++ b/.claude/skills/pose-estimation/SKILL.md @@ -0,0 +1,79 @@ +--- +name: pose-estimation +description: Extract human pose landmarks from video or images (MediaPipe Tasks API) and compare pose sequences with DTW joint-angle scoring. Use for analyzing dance/movement videos, scoring a recreated animation against source footage, or any "what is the body doing at time t" question. Works on real footage AND stylized game characters when a single subject fills the frame. +--- + +# Pose Estimation + +Two tested scripts, one venv. Extract 33-landmark pose sequences from video, then +score two sequences against each other (e.g. source dance video vs game recreation). + +## Setup (idempotent, run once per machine) + +```bash +bash ~/.claude/skills/pose-estimation/scripts/setup.sh +``` + +Creates `.venv` (pins python3.12 — MediaPipe does not support 3.13+) and downloads the +`full` pose model. `lite`/`heavy` model variants auto-download on first use via curl +(NOT urllib — macOS python lacks SSL certs; keep the curl download path). + +Always run scripts with the venv python: `~/.claude/skills/pose-estimation/.venv/bin/python`. + +## Extract poses + +```bash +VENV=~/.claude/skills/pose-estimation/.venv/bin/python +SKILL=~/.claude/skills/pose-estimation/scripts + +$VENV $SKILL/extract_pose.py input.mp4 --out poses.json --fps 8 \ + --overlay check.mp4 --min-conf 0.3 --model heavy +``` + +- `--fps N` sample rate; `--times "0.0,0.5,1.0"` instead samples an explicit beat grid + (pass a musical beat grid for dance work — samples land on choreographically meaningful frames). +- `--overlay out.mp4` draws the skeleton on sampled frames — ALWAYS eyeball this before + trusting the data. +- Output JSON has both `landmarks` (normalized image coords) and `world_landmarks` + (meters, hip-centered). Use world landmarks for any comparison — they are camera- and + scale-invariant. +- The `OK: N/M samples had a detected pose` line is the health signal. Low N/M → see + framing rules below. + +## Compare two pose sequences + +```bash +$VENV $SKILL/compare_poses.py source_poses.json recreation_poses.json \ + --a-range 2.0:6.0 --b-range 0:4.0 +``` + +Reduces each frame to 8 joint angles (elbows, shoulders, hips, knees), aligns the two +sequences with DTW (tolerates tempo drift), prints JSON: `score_0_100`, +`mean_angle_error_deg`, and `per_joint_error_deg` sorted worst-first (tells you WHERE +the recreation diverges — e.g. arms right, legs wrong). Verified: identical input → 100.0; +different dance segments → ~70. + +## Framing rules (the thing that actually determines success) + +MediaPipe pose is **single-person** and needs the subject to fill a large fraction of +the frame. Verified findings: + +- Wide shot with many small figures (e.g. a game formation capture at 1600x900 with + ~80px characters): **0% detection**. This is framing, not the model. +- Same video, one character crop-zoomed to fill frame (`ffmpeg -vf "crop=W:H:X:Y,scale=4x"`): + **94% detection** — even on stylized low-poly game characters (Quaternius rigs). +- Real photos/footage: works out of the box. + +So: for multi-person or wide footage, crop to one subject first: + +```bash +ffmpeg -i wide.mp4 -vf "crop=140:180:350:420,scale=560:720:flags=lanczos" -an solo.mp4 +``` + +Knobs when detection is weak: `--min-conf 0.15`, `--model heavy`, bigger crop upscale. + +## Typical workflow (video → analysis) + +1. Extract a frame, look at it, choose the crop for the subject. +2. Extract with `--overlay`, check the overlay video, check detection %. +3. Compare / analyze from `world_landmarks` or via compare_poses.py. diff --git a/.claude/skills/pose-estimation/models/pose_landmarker_full.task b/.claude/skills/pose-estimation/models/pose_landmarker_full.task new file mode 100644 index 0000000..f24cf38 --- /dev/null +++ b/.claude/skills/pose-estimation/models/pose_landmarker_full.task @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eaa5eb7a98365221087693fcc286334cf0858e2eb6e15b506aa4a7ecdcec4ad +size 9398198 diff --git a/.claude/skills/pose-estimation/models/pose_landmarker_heavy.task b/.claude/skills/pose-estimation/models/pose_landmarker_heavy.task new file mode 100644 index 0000000..5f2c1e2 --- /dev/null +++ b/.claude/skills/pose-estimation/models/pose_landmarker_heavy.task @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64437af838a65d18e5ba7a0d39b465540069bc8aae8308de3e318aad31fcbc7b +size 30664242 diff --git a/.claude/skills/pose-estimation/scripts/compare_poses.py b/.claude/skills/pose-estimation/scripts/compare_poses.py new file mode 100644 index 0000000..ff24d24 --- /dev/null +++ b/.claude/skills/pose-estimation/scripts/compare_poses.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Compare two pose sequences (extract_pose.py JSON) and score their similarity. + +Method: each detected frame is reduced to a joint-angle signature (8 angles: +elbows, shoulders, hips, knees) computed from 3D world landmarks — scale- and +translation-invariant, so a phone video and a game capture compare fairly. +The two angle sequences are aligned with dynamic time warping (handles tempo +drift / offset), then scored: + + mean_angle_error_deg average per-joint angular error along the DTW path + per_joint_error_deg which joints diverge most (arms vs legs etc.) + score_0_100 100 * max(0, 1 - mean_error/90) (rough but comparable) + +Typical use: score a recreated game dance against the source video, per move +segment (--a-range/--b-range trim by time before aligning). +""" +import argparse, json, sys + +import numpy as np + +# (name, a, b, c) -> angle at b between vectors (a-b) and (c-b), landmark indices +ANGLE_DEFS = [ + ("l_elbow", 11, 13, 15), ("r_elbow", 12, 14, 16), + ("l_shoulder", 13, 11, 23), ("r_shoulder", 14, 12, 24), + ("l_hip", 11, 23, 25), ("r_hip", 12, 24, 26), + ("l_knee", 23, 25, 27), ("r_knee", 24, 26, 28), +] + + +def load_angles(path, t_min, t_max): + with open(path) as f: + doc = json.load(f) + ts, sigs = [], [] + for fr in doc["frames"]: + if not fr.get("detected"): + continue + t = fr["t"] + if t < t_min or (t_max is not None and t > t_max): + continue + lms = fr.get("world_landmarks") or fr.get("landmarks") + pts = np.array([[p["x"], p["y"], p["z"]] for p in lms]) + sig = [] + for _, a, b, c in ANGLE_DEFS: + v1, v2 = pts[a] - pts[b], pts[c] - pts[b] + n1, n2 = np.linalg.norm(v1), np.linalg.norm(v2) + if n1 < 1e-8 or n2 < 1e-8: + sig.append(0.0) + continue + cosang = np.clip(np.dot(v1, v2) / (n1 * n2), -1.0, 1.0) + sig.append(float(np.degrees(np.arccos(cosang)))) + ts.append(t) + sigs.append(sig) + if not sigs: + sys.exit(f"ERROR: no detected frames in range in {path}") + return np.array(ts), np.array(sigs) + + +def dtw_path(A, B): + """O(n*m) DTW on mean-abs-angle-diff cost. Returns (path, cost_matrix).""" + n, m = len(A), len(B) + cost = np.mean(np.abs(A[:, None, :] - B[None, :, :]), axis=2) # n x m, degrees + acc = np.full((n + 1, m + 1), np.inf) + acc[0, 0] = 0.0 + for i in range(1, n + 1): + for j in range(1, m + 1): + acc[i, j] = cost[i - 1, j - 1] + min(acc[i - 1, j], acc[i, j - 1], acc[i - 1, j - 1]) + path = [] + i, j = n, m + while i > 0 and j > 0: + path.append((i - 1, j - 1)) + step = np.argmin([acc[i - 1, j - 1], acc[i - 1, j], acc[i, j - 1]]) + if step == 0: i, j = i - 1, j - 1 + elif step == 1: i -= 1 + else: j -= 1 + path.reverse() + return path, cost + + +def parse_range(s): + if not s: + return 0.0, None + lo, _, hi = s.partition(":") + return float(lo or 0.0), (float(hi) if hi else None) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("pose_a", help="pose JSON (e.g. source video)") + ap.add_argument("pose_b", help="pose JSON (e.g. recreation)") + ap.add_argument("--a-range", default=None, help="time window in A, 'start:end' seconds") + ap.add_argument("--b-range", default=None, help="time window in B, 'start:end' seconds") + ap.add_argument("--out", default=None, help="optional JSON report path") + args = ap.parse_args() + + a_lo, a_hi = parse_range(args.a_range) + b_lo, b_hi = parse_range(args.b_range) + ts_a, A = load_angles(args.pose_a, a_lo, a_hi) + ts_b, B = load_angles(args.pose_b, b_lo, b_hi) + + path, cost = dtw_path(A, B) + path_errs = np.array([np.abs(A[i] - B[j]) for i, j in path]) # steps x joints + mean_err = float(path_errs.mean()) + per_joint = {ANGLE_DEFS[k][0]: round(float(path_errs[:, k].mean()), 1) + for k in range(len(ANGLE_DEFS))} + score = round(100.0 * max(0.0, 1.0 - mean_err / 90.0), 1) + + report = { + "a": {"file": args.pose_a, "frames": len(A), "span_s": [float(ts_a[0]), float(ts_a[-1])]}, + "b": {"file": args.pose_b, "frames": len(B), "span_s": [float(ts_b[0]), float(ts_b[-1])]}, + "mean_angle_error_deg": round(mean_err, 2), + "per_joint_error_deg": dict(sorted(per_joint.items(), key=lambda kv: -kv[1])), + "score_0_100": score, + "dtw_path_len": len(path), + } + print(json.dumps(report, indent=2)) + if args.out: + with open(args.out, "w") as f: + json.dump(report, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/pose-estimation/scripts/extract_pose.py b/.claude/skills/pose-estimation/scripts/extract_pose.py new file mode 100644 index 0000000..bf8bdad --- /dev/null +++ b/.claude/skills/pose-estimation/scripts/extract_pose.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Extract human pose landmarks from a video (or single image) with MediaPipe Pose +(Tasks API, mediapipe >= 0.10.30 — the legacy `solutions` API no longer exists). + +Outputs one JSON document with per-sample 2D landmarks (normalized image coords) +and 3D world landmarks (meters, hip-centered) — the world landmarks are what you +want for pose comparison because they are camera-scale invariant. + +Sampling: --fps N samples the video at N Hz (default 10). --times "0.0,0.5,1.0" +samples at explicit timestamps instead (e.g. a musical beat grid). --overlay +writes the sampled frames with the skeleton drawn to an .mp4 for eyeball checks. + +Works on real footage and (usually) on stylized humanoid game characters, but +detection confidence drops on non-photoreal rigs — lower --min-conf and try +--model heavy if frames come back undetected. +""" +import argparse, json, os, sys + +import cv2 +import numpy as np +import mediapipe as mp +from mediapipe.tasks import python as mp_tasks +from mediapipe.tasks.python import vision + +SKILL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# The 33 pose landmarks, in model output order (fixed contract of the pose model). +POSE_LANDMARKS = [ + "nose", "left_eye_inner", "left_eye", "left_eye_outer", "right_eye_inner", + "right_eye", "right_eye_outer", "left_ear", "right_ear", "mouth_left", + "mouth_right", "left_shoulder", "right_shoulder", "left_elbow", "right_elbow", + "left_wrist", "right_wrist", "left_pinky", "right_pinky", "left_index", + "right_index", "left_thumb", "right_thumb", "left_hip", "right_hip", + "left_knee", "right_knee", "left_ankle", "right_ankle", "left_heel", + "right_heel", "left_foot_index", "right_foot_index", +] + +# Skeleton edges for overlay drawing (standard MediaPipe pose connections, torso+limbs). +CONNECTIONS = [ + (11, 12), (11, 13), (13, 15), (12, 14), (14, 16), # shoulders + arms + (11, 23), (12, 24), (23, 24), # torso + (23, 25), (25, 27), (27, 29), (27, 31), # left leg + (24, 26), (26, 28), (28, 30), (28, 32), # right leg + (15, 17), (15, 19), (15, 21), (16, 18), (16, 20), (16, 22), # hands +] + + +def lms_to_list(lms): + return [ + {"name": POSE_LANDMARKS[i], + "x": round(p.x, 5), "y": round(p.y, 5), "z": round(p.z, 5), + "visibility": round(p.visibility or 0.0, 3)} + for i, p in enumerate(lms) + ] + + +def draw_skeleton(frame_bgr, lms, t, detected): + h, w = frame_bgr.shape[:2] + if detected: + pts = [(int(p.x * w), int(p.y * h)) for p in lms] + for a, b in CONNECTIONS: + cv2.line(frame_bgr, pts[a], pts[b], (0, 220, 255), 2) + for x, y in pts: + cv2.circle(frame_bgr, (x, y), 3, (0, 255, 0), -1) + cv2.putText(frame_bgr, f"t={t:.2f}s {'OK' if detected else 'NO POSE'}", + (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.8, + (0, 255, 0) if detected else (0, 0, 255), 2) + return frame_bgr + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("input", help="video file (.mp4/.mov/...) or single image") + ap.add_argument("--out", required=True, help="output JSON path") + ap.add_argument("--fps", type=float, default=10.0, help="sample rate in Hz (default 10)") + ap.add_argument("--times", default=None, help="comma-separated timestamps in seconds (overrides --fps)") + ap.add_argument("--overlay", default=None, help="optional .mp4 path: sampled frames with skeleton drawn") + ap.add_argument("--model", default="full", choices=["lite", "full", "heavy"], + help="pose model variant (heavy = most accurate; auto-downloads)") + ap.add_argument("--min-conf", type=float, default=0.5, + help="min detection/tracking confidence (lower for stylized characters)") + args = ap.parse_args() + + model_path = os.path.join(SKILL_DIR, "models", f"pose_landmarker_{args.model}.task") + if not os.path.exists(model_path): + url = (f"https://storage.googleapis.com/mediapipe-models/pose_landmarker/" + f"pose_landmarker_{args.model}/float16/latest/pose_landmarker_{args.model}.task") + os.makedirs(os.path.dirname(model_path), exist_ok=True) + print(f"downloading model {args.model}...", file=sys.stderr) + import subprocess + subprocess.run(["curl", "-sL", "-o", model_path, url], check=True) + + is_image = os.path.splitext(args.input)[1].lower() in (".png", ".jpg", ".jpeg", ".webp", ".bmp") + + options = vision.PoseLandmarkerOptions( + base_options=mp_tasks.BaseOptions(model_asset_path=model_path), + running_mode=vision.RunningMode.IMAGE if is_image else vision.RunningMode.VIDEO, + min_pose_detection_confidence=args.min_conf, + min_pose_presence_confidence=args.min_conf, + min_tracking_confidence=args.min_conf, + ) + landmarker = vision.PoseLandmarker.create_from_options(options) + + frames_out, overlay_frames = [], [] + + def process(frame_bgr, t, idx): + mp_img = mp.Image(image_format=mp.ImageFormat.SRGB, + data=cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)) + if is_image: + res = landmarker.detect(mp_img) + else: + res = landmarker.detect_for_video(mp_img, int(t * 1000)) + detected = bool(res.pose_landmarks) + entry = {"t": round(t, 4), "index": idx, "detected": detected} + if detected: + entry["landmarks"] = lms_to_list(res.pose_landmarks[0]) + if res.pose_world_landmarks: + entry["world_landmarks"] = lms_to_list(res.pose_world_landmarks[0]) + frames_out.append(entry) + if args.overlay is not None: + overlay_frames.append(draw_skeleton( + frame_bgr.copy(), res.pose_landmarks[0] if detected else None, t, detected)) + + if is_image: + img = cv2.imread(args.input) + if img is None: + sys.exit(f"ERROR: cannot read image {args.input}") + process(img, 0.0, 0) + src_fps, duration = 0.0, 0.0 + else: + cap = cv2.VideoCapture(args.input) + if not cap.isOpened(): + sys.exit(f"ERROR: cannot open video {args.input}") + src_fps = cap.get(cv2.CAP_PROP_FPS) or 30.0 + duration = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) / src_fps if src_fps else 0.0 + + if args.times: + sample_ts = sorted(float(s) for s in args.times.split(",") if s.strip()) + else: + sample_ts = list(np.arange(0.0, max(duration - 1e-6, 0.0), 1.0 / args.fps)) + + for i, t in enumerate(sample_ts): # ascending — VIDEO mode needs monotonic timestamps + cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000.0) + ok, frame = cap.read() + if not ok: + frames_out.append({"t": round(t, 4), "index": i, "detected": False, "error": "read failed"}) + continue + process(frame, t, i) + cap.release() + + landmarker.close() + + if args.overlay and overlay_frames: + h, w = overlay_frames[0].shape[:2] + vw = cv2.VideoWriter(args.overlay, cv2.VideoWriter_fourcc(*"mp4v"), + args.fps if not args.times else 4.0, (w, h)) + for f in overlay_frames: + vw.write(f) + vw.release() + + detected = sum(1 for f in frames_out if f.get("detected")) + doc = { + "source": os.path.abspath(args.input), + "source_fps": round(src_fps, 3), + "duration_s": round(duration, 3), + "samples": len(frames_out), + "detected": detected, + "landmark_names": POSE_LANDMARKS, + "frames": frames_out, + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)) or ".", exist_ok=True) + with open(args.out, "w") as f: + json.dump(doc, f) + print(f"OK: {detected}/{len(frames_out)} samples had a detected pose -> {args.out}" + + (f" (overlay: {args.overlay})" if args.overlay else "")) + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/pose-estimation/scripts/setup.sh b/.claude/skills/pose-estimation/scripts/setup.sh new file mode 100644 index 0000000..a4014eb --- /dev/null +++ b/.claude/skills/pose-estimation/scripts/setup.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Bootstrap the pose-estimation venv + model. Idempotent — safe to run every time. +# MediaPipe supports Python <= 3.12, so we pin the interpreter explicitly. +set -euo pipefail + +SKILL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV="$SKILL_DIR/.venv" +MODEL="$SKILL_DIR/models/pose_landmarker_full.task" +MODEL_URL="https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_full/float16/latest/pose_landmarker_full.task" + +PY="" +for cand in python3.12 python3.11 python3.10; do + if command -v "$cand" >/dev/null 2>&1; then PY="$cand"; break; fi +done +if [ -z "$PY" ]; then + echo "ERROR: need python 3.10-3.12 for mediapipe (found none). brew install python@3.12" >&2 + exit 1 +fi + +if [ ! -x "$VENV/bin/python" ]; then + "$PY" -m venv "$VENV" +fi + +"$VENV/bin/pip" install --quiet --upgrade pip +"$VENV/bin/pip" install --quiet "mediapipe>=0.10.9" "opencv-python>=4.8" "numpy" + +if [ ! -f "$MODEL" ]; then + mkdir -p "$SKILL_DIR/models" + curl -sL -o "$MODEL" "$MODEL_URL" +fi + +echo "OK: venv at $VENV, model at $MODEL" +echo "Run: $VENV/bin/python $SKILL_DIR/scripts/extract_pose.py --help" diff --git a/.claude/skills/retarget-animations/SKILL.md b/.claude/skills/retarget-animations/SKILL.md new file mode 100644 index 0000000..60f5c6a --- /dev/null +++ b/.claude/skills/retarget-animations/SKILL.md @@ -0,0 +1,131 @@ +# Skill: Animation Retargeting (Kevin → BoZo) — DEPRECATED + +**DEPRECATED May 19, 2026.** BoZo and Kevin characters retired. Quaternius character system replaces them — no retargeting needed. This skill is kept for engine reference (Godot fork rest fixer fix). + +```bash +# 1. Clean import (creates default .import files) +rm -f assets/animations/kevin/*.import assets/models/characters/Body/Body_BasicBody.fbx.import +rm -f .godot/imported/Archer*.scn .godot/imported/Body_BasicBody*.scn .godot/imported/Villager*.scn .godot/imported/Throwing*.scn +godot --headless --path . --import + +# 2. Patch .import files with BoneMap + retarget config +godot --headless --path . -s setup_retarget.gd + +# 3. Delete cache + reimport with retarget config +rm -f .godot/imported/Archer*.scn .godot/imported/Body_BasicBody*.scn .godot/imported/Villager*.scn .godot/imported/Throwing*.scn +godot --headless --path . --import +``` + +IMPORTANT: Must delete `.godot/imported/*.scn` cache between steps 2→3, otherwise Godot skips reimport. + +## What the pipeline does + +1. **Bone renamer:** Renames Kevin bones (`B-hips` → `Hips`) and BoZo bones (`pelvis` → `Hips`) to Humanoid standard +2. **Overwrite Axis (method 1):** Changes skeleton rest poses to match SkeletonProfileHumanoid reference +3. **Animation conversion:** Transforms keyframe values from old rest coordinate system to new rest coordinate system using formula from `post_import_plugin_skeleton_rest_fixer.cpp:777`: + ``` + new_q = new_pg_q.inv * old_pg_q * qt * old_rest_q.inv * old_pg_q.inv * new_pg_q * new_rest_q + ``` + +## Known Bug (patched in tinqs/godot fork) + +**Commit:** `4e16c3f5bd` on `main` + +The `%GeneralSkeleton` unique node resolution via `get_node()` fails in headless import mode. Original code at line 733: +```cpp +Node *node = (ap->get_node(ap->get_root_node()))->get_node(NodePath(track_path)); +ERR_CONTINUE(!node); // Silently skips ALL animation tracks! +``` + +**Fix:** Falls back to `find_child()` when unique name resolution fails. Without this fix, rest poses change but animation keyframe values stay in the old coordinate system. + +## Current Status (2026-05-18) + +**Working:** +- Bone renaming (Kevin → Humanoid, BoZo → Humanoid) ✓ +- Rest pose overwrite to Humanoid reference ✓ +- Skeleton node resolution fix (find_child fallback) ✓ +- Animation conversion runs for spine/chest bones ✓ + +**NOT working:** +- Animation conversion appears to be a NO-OP for limb bones (arms, legs) +- Spine keys → near identity (correct). Arm/leg keys → near rest (unconverted) +- Root cause: unknown. Possibly old_rest captured AFTER partial overwrite, or formula output matches input for bones with specific rest orientations + +**Visual symptoms:** +- Arms cross behind back during animation +- Legs may be double-rotated (rest * rest) +- Spine animates correctly + +## Bone Mapping + +### Kevin → Humanoid +| Humanoid | Kevin | +|----------|-------| +| Hips | B-hips | +| Spine | B-spine | +| Chest | B-chest | +| UpperChest | B-upperChest | +| Neck | B-neck | +| Head | B-head | +| LeftShoulder | B-shoulder.L | +| LeftUpperArm | B-upper_arm.L | +| LeftLowerArm | B-forearm.L | +| LeftHand | B-hand.L | +| Right* | B-*.R (mirror) | +| LeftUpperLeg | B-thigh.L | +| LeftLowerLeg | B-shin.L | +| LeftFoot | B-foot.L | +| LeftToes | B-toe.L | +| Right* | B-*.R (mirror) | + +### BoZo → Humanoid +| Humanoid | BoZo | +|----------|------| +| Hips | pelvis | +| Spine | spine_01 | +| Chest | spine_03 | +| UpperChest | spine_05 | +| Neck | neck_01 | +| Head | head | +| LeftShoulder | clavicle_l | +| LeftUpperArm | upperarm_l | +| LeftLowerArm | lowerarm_l | +| LeftHand | hand_l | +| Right* | *_r (mirror) | +| LeftUpperLeg | thigh_l | +| LeftLowerLeg | calf_l | +| LeftFoot | foot_l | +| LeftToes | ball_l | +| Right* | *_r (mirror) | + +BoZo intermediate bones (spine_02, spine_04, neck_02) have no Kevin equivalent and keep global rest. + +## Key Files + +| File | Purpose | +|------|---------| +| `setup_retarget.gd` | Headless script: patches .import files with BoneMap + retarget config | +| `run_retarget.gd` | EditorScript: same but runs inside Godot editor GUI | +| `addons/retarget_setup/plugin.gd` | EditorPlugin: auto-runs on editor start | +| `assets/retarget/kevin_bonemap.tres` | Kevin → Humanoid BoneMap resource | +| `assets/retarget/bozo_bonemap.tres` | BoZo → Humanoid BoneMap resource | +| `inspect_fbx.gd` | Debug: prints bone names, rest poses, animation values | +| `src/Viewer/PlayerController.cs` | Character: loads animation, remaps %GeneralSkeleton paths | + +## Godot Engine Source (for debugging) + +| File | What | +|------|------| +| `editor/import/3d/post_import_plugin_skeleton_rest_fixer.cpp` | Overwrite Axis + animation conversion | +| `editor/import/3d/post_import_plugin_skeleton_renamer.cpp` | Bone renaming | +| `scene/3d/retarget_modifier_3d.cpp` | Runtime retargeting (RetargetModifier3D) | +| `scene/resources/bone_map.cpp` | BoneMap resource | +| `scene/resources/skeleton_profile.cpp` | SkeletonProfile + Humanoid preset | + +## What NOT to do + +1. **Manual quaternion retargeting in C#** — tried 5 different formulas, all failed. The formula requires parent GLOBAL rest transforms for the entire bone chain. Only Godot's engine code handles this correctly. +2. **Rename BoZo bones at runtime** — breaks outfit/clothing FBX loading (bone name mismatch). +3. **BoneAttachment3D for head without inverse rest compensation** — double-transforms the head. +4. **Headless import without deleting .scn cache** — Godot skips reimport if source FBX unchanged. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..da72b90 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.task filter=lfs diff=lfs merge=lfs -text +*.fbx filter=lfs diff=lfs merge=lfs -text +*.FBX filter=lfs diff=lfs merge=lfs -text +*.glb filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b63d8e7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.venv/ +__pycache__/ +*.pyc +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..f87665a --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +# animation + +Jeremy's animation-pipeline hub AND the **Mac↔PC bridge** for converting files +and implementing animations. iClone 8 + Video Mocap live on the PC; Blender 5, +the retarget tools, and the ariki-game repo live on both. This repo carries the +skills, tools, and the files in flight between machines. + +## Bridge workflow (Mac↔PC) + +1. **PC**: export FBX from iClone (Blender preset, Range=All, Preserve Bone + Names ON) → drop into `exchange/incoming-fbx/` → commit + `tinqs push`. +2. **Either machine**: `tinqs pull`, run the retargeter → + + ``` + blender --background --python tools/cc_retarget.py -- \ + --src exchange/incoming-fbx/.fbx \ + --out exchange/converted-glb/.glb --name \ + --target + ``` + + → commit the GLB + `tinqs push`. +3. **Implementing machine**: copy the GLB into ariki-game + `assets/quaternius/dancegen/` (auto-discovered by ClipCatalog as + `/`), add a dance JSON, verify in the dance test bed. +4. Optional: reference footage for QA goes in `exchange/reference-video/`. + +Large binaries (`.fbx`, `.glb`, `.mp4`, `.task`) ride Git LFS — see +`.gitattributes`. Clean out `exchange/` folders once a take has shipped +into the game repo. + +## Layout + +- `.claude/skills/` — Claude Code skills (auto-discovered when running Claude here) + - `animation-creation/` — creating clips on the shared Quaternius skeleton: existing packs, Mixamo/Kevin retargeting, composite baking, dance JSON (paths reference the ariki-game repo) + - `iclone-video-mocap/` — iClone 8 + Video Mocap plugin advisory: filming rules, workflow, cleanup, FBX export, game integration + - `pose-estimation/` — MediaPipe video→pose-landmark extraction (`scripts/extract_pose.py`, `compare_poses.py`; models in `models/`, venv rebuilt via `scripts/setup.sh`) + - `retarget-animations/` — deprecated early retarget notes (kept for history) +- `tools/` — Blender-headless retargeters (all bake onto the Quaternius rig and export mesh-stripped GLB, one NLA track per clip) + - `cc_retarget.py` — Reallusion CC/iClone `CC_Base_*` FBX (60fps; TempMotion guard) + - `mixamo_retarget.py` — Mixamo FBX (prefix auto-detect) + - `kevin_retarget.py` — Kevin Iglesias Unity FBX (`B-*` rig; hardcoded target path — non-portable) + - `mocap_retarget.py` — MediaPipe pose JSON → GLB (rotation solving, no source armature) + - `bake_run_punch.mjs` — composite upper/lower clip baking reference +- `plans/` — pipeline design docs (animation generation pipeline, skills adoption) + +## Canonical target rig + +All retargeters default to ariki-game's +`assets/quaternius/base-characters/.../Godot - UE/Superhero_Male_FullBody.gltf` +via a repo-relative path — **when running from this repo, pass `--target` with an +explicit path to that GLTF** (or a local copy). Output GLBs are consumed by +ariki-game's ClipCatalog from `assets/quaternius/{kevin,mixamo,dancegen}/` as +`/`. + +## Provenance + +Copied (not yet removed) from `ariki-game` `tools/` + `.claude/skills/` + +`.agents/plans/`, and `~/.claude/skills/pose-estimation`. The ariki-game +originals are still in place; the Windows-side iClone FBX conversion handoff +(`ariki-game/.agents/handoff/iclone-fbx-conversion.md`) depends on them. diff --git a/exchange/converted-glb/.gitkeep b/exchange/converted-glb/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/exchange/incoming-fbx/.gitkeep b/exchange/incoming-fbx/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/exchange/reference-video/.gitkeep b/exchange/reference-video/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/plans/animation-gen-pipeline-2026-07-13.md b/plans/animation-gen-pipeline-2026-07-13.md new file mode 100644 index 0000000..47912f5 --- /dev/null +++ b/plans/animation-gen-pipeline-2026-07-13.md @@ -0,0 +1,114 @@ +# Animation Gen Pipeline — Video → Recreated Dance + +**Goal:** give an agent a video of people dancing; the agent breaks the choreography +down and recreates it in-game as a dance JSON, using Kevin Iglesias (KI) clips, UAL +clips, composites, and Mixamo retargets — then verifies its own recreation against the +source and iterates. + +**Honest framing:** this produces a *cover version*, not motion capture. Continuous +human motion gets quantized into a discrete vocabulary of clips + composites. For game +choreography (kapa haka teams, dance floor content) that is the desired output. + +## Verified infrastructure (all exists, all tested 2026-07-13) + +| Piece | Where | Status | +|---|---|---| +| Dance runtime + hot-reload JSON | `src/Testing/Dance/`, `assets/dances/*.json` | SHIPPED — test bed plays dances, reloads in ~0.5s | +| Composite baking (upper/lower masks) | `CompositeClipBaker.cs` | SHIPPED | +| BPM beat clock + quantized transitions | `BeatClock.cs`, `bpm` in dance JSON | SHIPPED | +| Viewport → mp4 recording | `VideoRecorder.cs` → `~/Downloads/dance_*.mp4` | SHIPPED | +| Mixamo FBX → Quaternius GLB | `tools/mixamo_retarget.py` (Blender 5.1.2 headless) | PROVEN (`northern_soul.glb`) | +| Clip vocabulary | 12 KI packs + UAL1/2 + mixamo dir, ~810 clips; KI social = Dance01–18, DancePoses, claps, cheers | SHIPPED | +| Pose extraction from video | `pose-estimation` skill (user-level, MediaPipe Tasks) | BUILT + TESTED today | +| Pose sequence comparison (DTW, 8 joint angles) | `compare_poses.py` in same skill | BUILT + TESTED (identity=100.0, cross-segment≈71) | +| Animation creation recipes | `.claude/skills/animation-creation/SKILL.md` | BUILT today | + +**Key experimental finding (2026-07-13):** MediaPipe tracks the stylized Quaternius +characters at **94% detection** when ONE character is crop-zoomed to fill the frame +(`crop=140:180:350:420,scale=560:720:flags=lanczos` on a test bed recording). Wide +formation shots detect **0%**. Every capture step below therefore frames a single dancer. + +## Pipeline stages + +### Stage 1 — Ingest the source video +1. `ffprobe` duration/fps; extract audio (`ffmpeg -vn audio.wav`). +2. **BPM + beat grid**: `librosa.beat.beat_track` (add librosa to the pose-estimation + venv when first needed). Beat times become the dance's `bpm` and the sampling grid. +3. Extract pose: `extract_pose.py video.mp4 --times "" --overlay check.mp4`. + If multi-person/wide: crop-zoom the clearest dancer first. Eyeball the overlay. +4. Extract frames on the beat grid for the vision pass (`ffmpeg -vf select=...`). + +### Stage 2 — Choreography breakdown +- **Programmatic move segmentation**: the 8-angle signature (elbows/shoulders/hips/knees) + from the pose JSON gives a per-beat pose vector; a spike in angle-space distance + between consecutive beats = likely move boundary. Cheap, deterministic. +- **Vision pass**: agent reads beat-grid frames and writes a per-segment description + ("beats 1–8: crouched stomp, arms slapping thighs..."). Combines with the programmatic + boundaries — vision names the moves, angles locate them. +- Output: `breakdown.json` — segments with beat ranges, text description, pose signature. + +### Stage 3 — Clip card catalog (the one missing build item) +One-time batch job; the enabler for matching. For each dance-relevant clip +(KI Dance/DancePose/claps/cheers + UAL emotes + mixamo pack): +1. Test bed plays the clip on ONE dancer, camera framed close (solo framing per the + 94% finding). VideoRecorder captures ~2 loops. +2. `extract_pose.py` on the capture → **pose signature** stored per clip. +3. Delegated vision (pi `nova_describe` / cheap model) writes a **text card** per clip. +4. Output: `assets/dances/catalog/clip_cards.json` — `{clipRef, text, bpm-ish tempo, + energy, pose_signature_file}`. Rebuild only when packs change. + +Build item: a `CatalogMode` in the dance test bed (iterate clips → frame solo dancer → +record → next). Everything else is scripting. + +### Stage 4 — Matching (segment → clip or composite) +1. **Numeric**: DTW each video segment's angle sequence against every catalog clip + signature (`compare_poses.py` logic). Rank by score. +2. **Split-mask matching for composites**: score arms (4 arm angles) and legs (4 leg + angles) *independently*. If no whole clip scores well but clip A wins arms and clip + B wins legs → propose `{layers:[{mask:"upper",clip:A},{mask:"lower",clip:B}]}`. +3. **Vision tiebreak**: agent compares top-3 candidates' text cards against the + segment description; picks; records rationale. + +### Stage 5 — Gap-fill via Mixamo +Segment with no acceptable match → agent emits a **shopping list** (search terms for +mixamo.com; downloads are behind Adobe login, so Jeremy grabs FBXs manually — ~5 min) +→ `mixamo_retarget.py` → new pack in `assets/quaternius/mixamo/` → add to catalog +(Stage 3 for just that pack) → re-match. + +### Stage 6 — Emit the dance +Write `assets/dances/.json`: `bpm` from Stage 1, one move per segment, `loops` +from beat counts. Hot-reload shows it immediately if the test bed is running. + +### Stage 7 — Verify loop (closes the circle) +1. Test bed plays the dance; camera solo-framed on one dancer; VideoRecorder captures. +2. `extract_pose.py` on the capture → `compare_poses.py` vs source, per segment + (`--a-range/--b-range` from the beat grid). +3. `score_0_100` per segment + `per_joint_error_deg` says WHERE it diverges (arms vs + legs → swap just that layer of a composite). Iterate moves until segments clear a + threshold (start ~65; calibrate). +4. Final deliverable: side-by-side mp4 (`ffmpeg hstack`) + per-segment score table. + +## Deliverables ledger + +| Item | Status | +|---|---| +| `pose-estimation` skill (extract + compare, venv, models) | DONE 2026-07-13 | +| `animation-creation` skill (project-level recipes) | DONE 2026-07-13 | +| This pipeline doc | DONE 2026-07-13 | +| Test bed `CatalogMode` (solo-frame + record each clip) | TODO — first build item | +| Catalog batch script + `clip_cards.json` | TODO | +| librosa beat grid helper | TODO (trivial, add when needed) | +| Segment/match/emit agent workflow (`/dance-from-video`) | TODO — after catalog | + +## Risks / open questions +- **Camera angle mismatch**: source video vs game capture viewpoint differ; world + landmarks + joint angles are view-invariant in principle, but MediaPipe's 3D lift is + weaker at oblique angles. Mitigation: match game camera height/angle roughly to source. +- **Multi-dancer sources**: pick ONE dancer (ideally the leader) and track them; group + formation is authored separately (formations are already a test bed parameter). +- **Catalog scale**: ~810 clips × record+describe is hours of batch time. Start with + the ~40 obviously dance-relevant clips; expand on demand. +- **Score threshold semantics**: 71 = "different segments of the same dance", so + same-move recreations should land higher; calibrate on known pairs before trusting. +- KI clips are polished loops; Mixamo retargets can have foot slide — acceptable for + dance (no locomotion), watch for it. diff --git a/plans/animation-skills-adoption-2026-07-13.md b/plans/animation-skills-adoption-2026-07-13.md new file mode 100644 index 0000000..cf4f6cb --- /dev/null +++ b/plans/animation-skills-adoption-2026-07-13.md @@ -0,0 +1,112 @@ +# Animation skills adoption — 2026-07-13 + +Goal: let agents **create** animations and rig them onto GLBs for ariki-game, not just +retarget existing packs. All animation runs on the one shared Quaternius skeleton (65 +bones, UE names), so the bottleneck is tooling that gets new motion onto that rig. This +report records the research tiering, what this pass integrated (file authoring only), and +the orchestrator runbook for the install/launch steps an agent must not do. + +## Research findings + +**Tier 1 — adopt now (integrated or queued for trial this pass):** + +- **blender-mcp** (github.com/ahujasid/blender-mcp, MIT, 16k+ stars) — MCP server bridging + the agent to a live Blender via a socket addon; the agent sends arbitrary `bpy` to build + armatures, keyframe, set up IK/NLA, and export GLB. *Why Tier 1:* the one capability the + project was missing — direct programmatic Blender authoring from the agent loop. +- **Blender Toolkit skill** (majiayu000) — Mixamo→custom-rig retarget with fuzzy bone + matching over a WebSocket bridge. *Why Tier 1:* directly comparable to the project's own + `tools/mixamo_retarget.py`; trial it head-to-head and keep the winner. +- **Blender Animation & Rigging Expert skill** — programmatic armatures, IK/FK, drivers, + NLA via MCP. *Why Tier 1:* the deep-blender companion to blender-mcp for non-trivial rigs. +- **Extend our own project skill** (this pass) — the existing `.claude/skills/animation-creation/` + is the cheapest surface; adding Path 5 + gltf-transform keeps everything in one place. + +**Tier 2 — trial before committing:** + +- **gltf-transform** (gltf-transform.dev, donmccurdy) — headless GLB surgery: inspect, + merge, resample, prune, dedup; same-skeleton clip transplants need no Blender. *Why Tier + 2:* documented here and usable via `npx`, but the project's clip-editing volume is + currently low; adopt the recipes, verify the v4 API on first real use. +- **PoseCap** (github.com/CorridorTech/PoseCap) — markerless mocap, video → PEAR model → + SMPL-X poses → Blender keyframes; works on pre-recorded clips; custom-rig retarget is + roadmap-only; UI extension, not headless. *Why Tier 2:* promising video-to-motion path, + but SMPL-X→Quaternius retarget is unbuilt — eval before investing. +- **OpenClaw / freshtechbro Blender pipeline skills** — engine export-settings knowledge. + *Why Tier 2:* useful reference for engine-specific gotchas, redundant once our export + block is canonical. + +**Tier 3 — skip for now:** + +- **godot-mcp servers** — redundant: the game already has an agent API on port 4329 plus an + in-scene `VideoRecorder`; a Godot MCP adds nothing. +- **DavinciDreams 3D Design Team plugin and other knowledge-only skill packs** — broad and + shallow; the targeted recipes above cover the same ground more concretely. + +## Integrated in this pass + +File authoring only (no installs, no commits): + +- **A — `.mcp.json`**: registered the `blender` MCP server (`uvx blender-mcp`). +- **B — `.claude/skills/animation-creation/SKILL.md`**: added **Path 5 — Author in live + Blender (blender-mcp)** and a **GLB surgery without Blender (gltf-transform)** section, + both pointing at the new reference docs. +- **C — `references/blender-mcp-recipes.md`** (new): `bpy` recipes for setup, keyframing, + IK setup + bake, NLA layering, and an **export block mirrored exactly from + `tools/mixamo_retarget.py`** (`NLA_TRACKS` mode, `export_force_sampling`, size-opt off), + plus gotchas (`*RM` variants, `RESET` clip, one armature/GLB, fps=30, quaternion mode). +- **D — `references/gltf-transform-recipes.md`** (new): CLI/SDK inspect, same-skeleton clip + transplant sketch (flagged `verify against gltf-transform v4 API before first use`), + resample/prune/dedup shrink pass, and when-to-prefer-over-Blender guidance. + +## Orchestrator runbook + +**Not run by the authoring agent.** The orchestrator (or Jeremy) executes these: + +1. **blender-mcp smoke test.** From a shell: `uvx blender-mcp` — confirm the server starts + and binds the socket (Ctrl-C to stop; this only checks installability). +2. **Install the Blender addon.** Download `addon.py` from + github.com/ahujasid/blender-mcp and in Blender 5.1.2 + (`/Applications/Blender.app`): Edit → Preferences → Add-ons → Install → select + `addon.py` → enable. Then 3D View sidebar → BlenderMCP → **Connect**. +3. **Reload MCP config.** Restart Claude Code so the `blender` server in `.mcp.json` is + picked up; confirm the `blender` MCP tools (`execute-code`, etc.) are now available. +4. **blender-mcp agent smoke test.** Through the MCP tools: import a game GLB + (`assets/quaternius/kevin/kevin_male_social.glb`), keyframe a wave on `hand_l` + (see `blender-mcp-recipes.md` keyframing recipe), export GLB into + `assets/quaternius/mixamo/`, and confirm the new clip appears in + `scenes/animation_showcase.tscn`. +5. **gltf-transform smoke test.** `npx @gltf-transform/cli inspect + assets/quaternius/kevin/kevin_male_social.glb` — confirm it lists the animations and + their durations. (No install step; `npx` fetches the CLI on demand.) +6. **Skill trials (user-level install at `~/.claude/skills/`).** Install **Blender Toolkit** + (majiayu000) and **Blender Animation & Rigging Expert**. Run one Mixamo clip → Quaternius + retarget through Blender Toolkit and compare the output against + `tools/mixamo_retarget.py` in the dance test bed — keep whichever loses as + reference-only. +7. **PoseCap eval.** Install the PoseCap Blender extension; run one short dance clip → + SMPL-X keyframes; assess quality vs the existing "match video to shipped clips via the + pose-estimation skill" approach **before** investing in SMPL-X→Quaternius retarget. + +> **Always ASK Jeremy before any game launch.** Run `tinqs pull` before launching the game +> so the working tree is current. + +## Verification + +- **Retarget round-trip:** a clip authored via blender-mcp (or retargeted through Blender + Toolkit) plays correctly in the dance test bed (`SCENE=dance_test_bed + bash tools/game.sh spawn` — ask Jeremy first), recorded with the in-scene + `VideoRecorder` to `~/Downloads/*.mp4` and scored against source footage with the + `pose-estimation` skill. +- **gltf-transform clip transplant:** a clip moved between two same-skeleton GLBs via the + SDK recipe plays in `scenes/animation_showcase.tscn` at the expected `/`, + confirming the channel-target re-pointing worked. + +## Open items + +- Confirm the gltf-transform v4 SDK call for cross-document animation copy on first real + use (the `copyToDocument` sketch in `gltf-transform-recipes.md` is flagged for this). +- Decide Blender Toolkit vs `tools/mixamo_retarget.py` after the head-to-head retarget + trial (runbook step 6). +- Decide whether to invest in SMPL-X→Quaternius retarget after the PoseCap quality eval + (runbook step 7). diff --git a/tools/bake_run_punch.mjs b/tools/bake_run_punch.mjs new file mode 100644 index 0000000..e1765f8 --- /dev/null +++ b/tools/bake_run_punch.mjs @@ -0,0 +1,280 @@ +#!/usr/bin/env node +// Bake a combined "Run_Punch" animation: legs from Jog_Fwd_Loop + upper body from Punch_Jab. +// +// No Blender required — edits the glTF JSON chunk only. The new animation reuses the source +// GLB's accessors/buffer verbatim (same skeleton); arm-lift and wind-up append fresh keyframes +// to the binary chunk. +// +// --mode arms-only : spine+neck+head+legs from RUN (torso upright, gaze forward), arms from PUNCH +// --arm-lift DEG : pitch both upperarms up (Y axis, mirrored per arm) → fist at eye level +// --windup DEG : synthesize a wind-up on the PUNCHING (left) upperarm: neutral → cocked +// (pulled back DEG on Z axis) → extended → recovered. Adds anticipation. +// --windup-dur S : wind-up (pull-back) duration in seconds (default 0.18) +// +// Usage: node tools/bake_run_punch.mjs --mode arms-only --arm-lift 35 --windup 45 + +import fs from "node:fs"; +import path from "node:path"; + +const argv = process.argv.slice(2); +const getOpt = (name, fallback) => { + const i = argv.indexOf(name); + return i >= 0 ? argv[i + 1] : fallback; +}; +const mode = getOpt("--mode", "arms-only"); +const armLiftDeg = parseFloat(getOpt("--arm-lift", "0")); +const windupDeg = parseFloat(getOpt("--windup", "0")); +const windupDur = parseFloat(getOpt("--windup-dur", "0.18")); + +const SRC = "assets/quaternius/anim-lib-1/Universal Animation Library[Standard]/Unreal-Godot/UAL1_Standard.glb"; +const OUT = "assets/quaternius/anim-combined/Run_Punch.glb"; +const RUN_NAME = "Jog_Fwd_Loop"; +const PUNCH_NAME = "Punch_Jab"; +const OUT_CLIP = "Run_Punch"; +const THROW_DUR = 0.15; // cocked → extended (the fast thrust) + +function parseGlb(buf) { + if (buf.toString("ascii", 0, 4) !== "glTF") throw new Error("not a GLB"); + const length = buf.readUInt32LE(8); + let o = 12, json = null, bin = null; + while (o < length) { + const clen = buf.readUInt32LE(o); + const ctype = buf.toString("ascii", o + 4, o + 8); + const data = buf.subarray(o + 8, o + 8 + clen); + if (ctype === "JSON") json = JSON.parse(data.toString("utf8")); + else if (ctype.startsWith("BIN")) bin = Buffer.from(data); + o += 8 + clen; + } + return { json, bin }; +} + +function writeGlb(jsonObj, binBuf, outPath) { + const jsonBytes = Buffer.from(JSON.stringify(jsonObj), "utf8"); + const jsonPad = (4 - (jsonBytes.length % 4)) % 4; + const jsonPadded = Buffer.concat([jsonBytes, Buffer.alloc(jsonPad, 0x20)]); + const binPad = (4 - (binBuf.length % 4)) % 4; + const binPadded = Buffer.concat([binBuf, Buffer.alloc(binPad, 0x00)]); + const total = 12 + 8 + jsonPadded.length + 8 + binPadded.length; + const out = Buffer.alloc(total); + let p = 0; + out.write("glTF", p, "ascii"); p += 4; + out.writeUInt32LE(2, p); p += 4; + out.writeUInt32LE(total, p); p += 4; + out.writeUInt32LE(jsonPadded.length, p); p += 4; out.write("JSON", p, "ascii"); p += 4; + jsonPadded.copy(out, p); p += jsonPadded.length; + out.writeUInt32LE(binPadded.length, p); p += 4; out.write("BIN\0", p, "ascii"); p += 4; + binPadded.copy(out, p); p += binPadded.length; + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, out); + return total; +} + +// ── Quaternion + vector math (xyzw) ── +function qMul(a, b) { + const [ax, ay, az, aw] = a, [bx, by, bz, bw] = b; + return [aw * bx + ax * bw + ay * bz - az * by, + aw * by - ax * bz + ay * bw + az * bx, + aw * bz + ax * by - ay * bx + az * bw, + aw * bw - ax * bx - ay * by - az * bz]; +} +function qAxis(axis, deg) { + const sign = deg < 0 ? -1 : 1; + const r = Math.abs(deg) * Math.PI / 180, h = Math.sin(r / 2) * sign; + const v = [0, 0, 0]; v[axis] = h; + return [v[0], v[1], v[2], Math.cos(r / 2)]; +} +function qSlerp(a, b, t) { + let [ax, ay, az, aw] = a; + let [bx, by, bz, bw] = b; + let dot = ax * bx + ay * by + az * bz + aw * bw; + if (dot < 0) { bx = -bx; by = -by; bz = -bz; bw = -bw; dot = -dot; } + if (dot > 0.9995) { + return [ax + t * (bx - ax), ay + t * (by - ay), az + t * (bz - az), aw + t * (bw - aw)].map(v => v); + } + const theta = Math.acos(dot), sinT = Math.sin(theta); + const s0 = Math.sin((1 - t) * theta) / sinT, s1 = Math.sin(t * theta) / sinT; + return [s0 * ax + s1 * bx, s0 * ay + s1 * by, s0 * az + s1 * bz, s0 * aw + s1 * bw]; +} +// Lift quaternion per arm side (Y axis, mirrored: right +, left −). Raises the fist. +const qLiftFor = (bone, deg) => bone.endsWith("_l") ? qAxis(1, -deg) : qAxis(1, deg); +// Retract (wind-up) quaternion per arm side (Z axis, mirrored). Pulls the fist backward. +const qRetractFor = (bone, deg) => bone.endsWith("_l") ? qAxis(2, -deg) : qAxis(2, deg); + +const isLowerBody = b => !b ? false : b === "root" || b === "pelvis" || /thigh|calf|foot|ball/i.test(b); +const isArm = b => !b ? false : /clavicle|upperarm|lowerarm|^hand_|hand_l|hand_r|index_|middle_|pinky_|ring_|thumb_/i.test(b); +const fromRun = mode === "arms-only" ? (b => !isArm(b)) + : mode === "no-head" ? (b => isLowerBody(b) || b === "neck_01" || b === "Head") + : isLowerBody; + +const { json, bin } = parseGlb(fs.readFileSync(SRC)); +const nodes = json.nodes; +const anims = json.animations; +const accessors = json.accessors; +const bufferViews = json.bufferViews; +const run = anims.find(a => (a.name || "") === RUN_NAME); +const punch = anims.find(a => (a.name || "") === PUNCH_NAME); +if (!run || !punch) throw new Error(`source clips not found`); + +// parent map (glTF uses children arrays) +const parentOf = new Array(nodes.length).fill(-1); +for (let i = 0; i < nodes.length; i++) if (nodes[i].children) for (const c of nodes[i].children) parentOf[c] = i; +const ni = nm => nodes.findIndex(n => (n.name || "") === nm); + +const newSamplers = []; +const newChannels = []; +const runBones = new Set(), punchBones = new Set(); + +function addChannels(srcAnim, include, set) { + for (const ch of srcAnim.channels) { + const bone = nodes[ch.target.node]?.name; + if (!include(bone)) continue; + set.add(bone); + const ss = srcAnim.samplers[ch.sampler]; + newSamplers.push({ input: ss.input, output: ss.output, interpolation: ss.interpolation }); + newChannels.push({ sampler: newSamplers.length - 1, target: { node: ch.target.node, path: ch.target.path } }); + } +} +addChannels(run, fromRun, runBones); +addChannels(punch, b => !fromRun(b), punchBones); + +// ── Binary-append helpers (append float data + create bufferView + accessor) ── +let outBin = bin; +function appendFloats(floats, type, count) { + const comps = type === "SCALAR" ? 1 : type === "VEC3" ? 3 : 4; + const byteOffset = outBin.length; + const buf = Buffer.alloc(count * comps * 4); + for (let i = 0; i < floats.length; i++) buf.writeFloatLE(floats[i], i * 4); + outBin = Buffer.concat([outBin, buf]); + const bvIdx = bufferViews.length; + bufferViews.push({ buffer: 0, byteOffset, byteLength: count * comps * 4 }); + const accIdx = accessors.length; + const min = new Array(comps).fill(Infinity), max = new Array(comps).fill(-Infinity); + for (let i = 0; i < count; i++) for (let c = 0; c < comps; c++) { + const v = floats[i * comps + c]; + if (v < min[c]) min[c] = v; if (v > max[c]) max[c] = v; + } + accessors.push({ bufferView: bvIdx, componentType: 5126, count, type, min, max }); + return accIdx; +} + +// ── --arm-lift: pitch both upperarms up (mirrored Y) ── +if (armLiftDeg !== 0) { + for (const ch of newChannels) { + const bone = nodes[ch.target.node]?.name; + if ((bone !== "upperarm_l" && bone !== "upperarm_r") || ch.target.path !== "rotation") continue; + const sampler = newSamplers[ch.sampler]; + const outAcc = accessors[sampler.output]; + const bv = bufferViews[outAcc.bufferView]; + const off = (bv.byteOffset || 0) + (outAcc.byteOffset || 0); + const count = outAcc.count; + const lift = qLiftFor(bone, armLiftDeg); + const floats = []; + for (let i = 0; i < count; i++) { + const q = [bin.readFloatLE(off + (i * 4) * 4), bin.readFloatLE(off + (i * 4 + 1) * 4), + bin.readFloatLE(off + (i * 4 + 2) * 4), bin.readFloatLE(off + (i * 4 + 3) * 4)]; + const r = qMul(lift, q); + floats.push(...r); + } + sampler.output = appendFloats(floats, "VEC4", count); + } +} + +// ── --windup: synthesize neutral → cocked → extended → recovered on the punching (left) upperarm ── +let windupApplied = false; +if (windupDeg !== 0) { + const bone = "upperarm_l"; + const boneIdx = ni(bone); + const ch = newChannels.find(c => c.target.node === boneIdx && c.target.path === "rotation"); + if (ch) { + const sampler = newSamplers[ch.sampler]; + const outAcc = accessors[sampler.output]; + const inAcc = accessors[sampler.input]; + const bvO = bufferViews[outAcc.bufferView], bvI = bufferViews[inAcc.bufferView]; + const offO = (bvO.byteOffset || 0) + (outAcc.byteOffset || 0); + const offI = (bvI.byteOffset || 0) + (inAcc.byteOffset || 0); + const count = outAcc.count; + const rots = [], times = []; + for (let i = 0; i < count; i++) { + rots.push([outBin.readFloatLE(offO + (i * 4) * 4), outBin.readFloatLE(offO + (i * 4 + 1) * 4), + outBin.readFloatLE(offO + (i * 4 + 2) * 4), outBin.readFloatLE(offO + (i * 4 + 3) * 4)]); + times.push(outBin.readFloatLE(offI + i * 4)); + } + // Note: rots/times read from outBin (post-lift) so the lift is preserved in neutral/extended/recover. + const L = times[times.length - 1]; // clip length (~0.93s, matches run) + const neutral = rots[0]; + const recovered = rots[rots.length - 1]; + // find the extension key = max forward (Z) hand via FK on the left arm chain + const chain = []; + { let i = boneIdx; while (i >= 0) { chain.unshift(i); i = parentOf[i]; } } + function handZ(upperarmRot) { + let M = [[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]]; + for (const idx of chain) { + const nm = nodes[idx].name; + let rot = nodes[idx].rotation ? [...nodes[idx].rotation] : [0,0,0,1]; + if (nm === bone && upperarmRot) rot = upperarmRot; + else { + const pch = punch.channels.find(c => c.target.node === idx && c.target.path === "rotation"); + if (pch) rot = rots0(pch); // falls back below + } + const r = rot, t = nodes[idx].translation || [0,0,0]; + const q2m = q => { const[x,y,z,w]=q,x2=x*x,y2=y*y,z2=z*z,xy=x*y,xz=x*z,yz=y*z,wx=w*x,wy=w*y,wz=w*z; + return[[1-2*(y2+z2),2*(xy-wz),2*(xz+wy)],[2*(xy+wz),1-2*(x2+z2),2*(yz-wx)],[2*(xz-wy),2*(yz+wx),1-2*(x2+y2)]]; }; + const m = q2m(r); + M = mm(M, [[m[0][0],m[0][1],m[0][2],t[0]],[m[1][0],m[1][1],m[1][2],t[1]],[m[2][0],m[2][1],m[2][2],t[2]],[0,0,0,1]]); + } + return M[2][3]; // hand world Z (forward) + } + function mm(a,b){const r=[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]];for(let i=0;i<4;i++)for(let j=0;j<4;j++){let s=0;for(let k=0;k<4;k++)s+=a[i][k]*b[k][j];r[i][j]=s;}return r;} + function rots0(pch){return null;} // placeholder — replaced by per-key lookup below + // Proper FK: find extension key by scanning punch upperarm keys + let extK = 0, extZ = -Infinity; + for (let k = 0; k < count; k++) { + // build chain with this key's upperarm rotation + let M = [[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]]; + for (const idx of chain) { + const nm = nodes[idx].name; + let rot = nodes[idx].rotation ? [...nodes[idx].rotation] : [0,0,0,1]; + if (nm === bone) rot = rots[k]; + else { + const pch = punch.channels.find(c => c.target.node === idx && c.target.path === "rotation"); + if (pch) { + const ps = punch.samplers[pch.sampler]; + const pa = accessors[ps.output]; const pbv = bufferViews[pa.bufferView]; + const poff = (pbv.byteOffset||0)+(pa.byteOffset||0); + rot = [bin.readFloatLE(poff+(k*4)*4),bin.readFloatLE(poff+(k*4+1)*4),bin.readFloatLE(poff+(k*4+2)*4),bin.readFloatLE(poff+(k*4+3)*4)]; + } + } + const q2m = q => { const[x,y,z,w]=q,x2=x*x,y2=y*y,z2=z*z,xy=x*y,xz=x*z,yz=y*z,wx=w*x,wy=w*y,wz=w*z; + return[[1-2*(y2+z2),2*(xy-wz),2*(xz+wy)],[2*(xy+wz),1-2*(x2+z2),2*(yz-wx)],[2*(xz-wy),2*(yz+wx),1-2*(x2+y2)]]; }; + const m = q2m(rot), t = nodes[idx].translation||[0,0,0]; + M = mm(M, [[m[0][0],m[0][1],m[0][2],t[0]],[m[1][0],m[1][1],m[1][2],t[1]],[m[2][0],m[2][1],m[2][2],t[2]],[0,0,0,1]]); + } + const z = M[2][3]; + if (z > extZ) { extZ = z; extK = k; } + } + const extended = rots[extK]; + // cocked = neutral pulled back (retract in bone-local space, pre-multiply) + const retract = qRetractFor(bone, windupDeg); + const cocked = qMul(retract, neutral); + // 4-key synthesized curve: neutral@0 → cocked@windUpDur → extended@(+throw) → recovered@L + const extT = windupDur + THROW_DUR; + const sTimes = [0, windupDur, extT, L]; + const sRots = [neutral, cocked, extended, recovered]; + const floats = []; + for (const q of sRots) floats.push(...q); + sampler.input = appendFloats(sTimes, "SCALAR", 4); + sampler.output = appendFloats(floats, "VEC4", 4); + sampler.interpolation = "LINEAR"; + windupApplied = true; + console.log(` wind-up: punching arm '${bone}' → neutral@0 → cocked@${windupDur}s → extended@${extT}s (key ${extK}) → recovered@${L.toFixed(3)}s (retract Z ${windupDeg}°)`); + } +} + +json.buffers[0].byteLength = outBin.length; +const runPunch = { name: OUT_CLIP, channels: newChannels, samplers: newSamplers }; +const outJson = { ...json, animations: [runPunch] }; +const bytes = writeGlb(outJson, outBin, OUT); + +const combinedLen = Math.max(...newSamplers.map(s => accessors[s.input]?.max?.[0] ?? 0)); +console.log(`baked ${OUT} [mode=${mode}, arm-lift=${armLiftDeg}°, windup=${windupDeg}°]`); +console.log(` from RUN(${runBones.size}) + PUNCH(${punchBones.size}) = ${runBones.size + punchBones.size} bones, len=${combinedLen.toFixed(3)}s, ${(bytes/1024/1024).toFixed(2)}MB`); diff --git a/tools/cc_retarget.py b/tools/cc_retarget.py new file mode 100644 index 0000000..fcd354d --- /dev/null +++ b/tools/cc_retarget.py @@ -0,0 +1,268 @@ +# Reallusion Character Creator / iClone FBX → Quaternius-skeleton GLB retargeter. +# +# Sibling of tools/mixamo_retarget.py — identical world-rotation-delta method (CC rigs +# rest in T-pose, verified on ActorBuild exports), adapted for the CC_Base_* skeleton: +# - fixed "CC_Base_" naming, hips bone = CC_Base_Hip (parent of Pelvis+Waist) +# - twist helpers (UpperarmTwist01...) are unmapped → ride their parent at rest offset +# - game-optimized CC exports have 1-segment fingers (Index1, no Index2/3) — mapped +# where present, absent segments ride the parent +# - iClone FBX takes named "*TempMotion" (2 frames) mean the export had NO motion — +# the take is skipped with a loud warning (re-export with Include Motion). +# +# Usage: +# blender --background --python tools/cc_retarget.py -- \ +# --src "[,…]" --out [--name ] [--target ] + +import bpy, sys, os, re +from mathutils import Matrix, Quaternion, Vector + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DEFAULT_TARGET_GLTF = os.path.join( + REPO_ROOT, "assets", "quaternius", "base-characters", + "Universal Base Characters[Standard]", "Base Characters", "Godot - UE", + "Superhero_Male_FullBody.gltf") + +BONE_MAP = { + "CC_Base_Hip": "pelvis", + "CC_Base_Waist": "spine_01", + "CC_Base_Spine01": "spine_02", + "CC_Base_Spine02": "spine_03", + "CC_Base_NeckTwist01": "neck_01", + "CC_Base_Head": "Head", +} +for S, T in (("L", "l"), ("R", "r")): + BONE_MAP[f"CC_Base_{S}_Clavicle"] = f"clavicle_{T}" + BONE_MAP[f"CC_Base_{S}_Upperarm"] = f"upperarm_{T}" + BONE_MAP[f"CC_Base_{S}_Forearm"] = f"lowerarm_{T}" + BONE_MAP[f"CC_Base_{S}_Hand"] = f"hand_{T}" + BONE_MAP[f"CC_Base_{S}_Thigh"] = f"thigh_{T}" + BONE_MAP[f"CC_Base_{S}_Calf"] = f"calf_{T}" + BONE_MAP[f"CC_Base_{S}_Foot"] = f"foot_{T}" + BONE_MAP[f"CC_Base_{S}_ToeBase"] = f"ball_{T}" + for i in (1, 2, 3): + BONE_MAP[f"CC_Base_{S}_Thumb{i}"] = f"thumb_0{i}_{T}" + # game-optimized CC rigs: one segment per finger + BONE_MAP[f"CC_Base_{S}_Index1"] = f"index_01_{T}" + BONE_MAP[f"CC_Base_{S}_Mid1"] = f"middle_01_{T}" + BONE_MAP[f"CC_Base_{S}_Ring1"] = f"ring_01_{T}" + BONE_MAP[f"CC_Base_{S}_Pinky1"] = f"pinky_01_{T}" + + +def apply_object_transform(obj): + bpy.ops.object.select_all(action="DESELECT") + obj.select_set(True) + bpy.context.view_layer.objects.active = obj + bpy.ops.object.transform_apply(location=True, rotation=True, scale=True) + bpy.context.view_layer.update() + + +def clip_name(path): + base = os.path.splitext(os.path.basename(path))[0] + base = base.replace(" - ", "_").replace(" ", "") + return re.sub(r"[^A-Za-z0-9_\-]", "", base) + + +def hierarchy_order(arm): + out, stack = [], [b for b in arm.data.bones if b.parent is None] + while stack: + b = stack.pop(0) + out.append(b) + stack.extend(b.children) + return out + + +def char_frame(rest, hips, head, upper_arm_l): + u = (rest[head].translation - rest[hips].translation).normalized() + l = (rest[upper_arm_l].to_quaternion() @ Vector((0, 1, 0))).normalized() + f = l.cross(u).normalized() + l = u.cross(f).normalized() + return Matrix((l, u, f)).transposed().to_quaternion().normalized() + + +def action_fcurves(a): + """All fcurves across Blender 5 slotted-action layers/strips (legacy fallback).""" + if hasattr(a, "fcurves") and a.fcurves is not None: + return list(a.fcurves) + fcs = [] + for layer in getattr(a, "layers", []): + for strip in layer.strips: + for bag in strip.channelbags: + fcs.extend(bag.fcurves) + return fcs + + +def pick_action(src_arm, acts): + """The armature's motion take: fcurves must reference CC_Base_ bones; longest wins. + TempMotion 2-frame placeholders (export without motion) are rejected.""" + best, best_len = None, 0.0 + for a in acts: + if not any("CC_Base_" in fc.data_path for fc in action_fcurves(a)): + continue + length = a.frame_range[1] - a.frame_range[0] + if length > best_len: + best, best_len = a, length + if best is not None and best_len < 5 and "TempMotion" in best.name: + raise RuntimeError( + f"'{best.name}' is a {int(best_len)+1}-frame iClone placeholder — the FBX was " + "exported WITHOUT motion. Re-export with Include Motion / Export Motion checked.") + return best + + +def retarget_one(src_arm, tgt_arm, action, name, scn, unit): + f0, f1 = int(action.frame_range[0]), int(action.frame_range[1]) + if src_arm.animation_data is None: + src_arm.animation_data_create() + src_arm.animation_data.action = action + if getattr(action, "slots", None) and src_arm.animation_data.action_slot is None: + src_arm.animation_data.action_slot = action.slots[0] + + inv_map = {v: k for k, v in BONE_MAP.items()} + + tgt_rest = {b.name: (tgt_arm.matrix_world @ b.matrix_local) for b in tgt_arm.data.bones} + src_rest = {b.name: (src_arm.matrix_world @ b.matrix_local) for b in src_arm.data.bones} + src_rest_q = {n: m.to_quaternion().normalized() for n, m in src_rest.items()} + tgt_rest_q = {n: m.to_quaternion().normalized() for n, m in tgt_rest.items()} + + C = (char_frame(tgt_rest, "pelvis", "Head", "upperarm_l") + @ char_frame(src_rest, "CC_Base_Hip", "CC_Base_Head", "CC_Base_L_Upperarm").inverted()).normalized() + Cinv = C.inverted() + + h_src = (src_rest["CC_Base_Head"].translation - src_rest["CC_Base_Hip"].translation).length + h_tgt = (tgt_rest["Head"].translation - tgt_rest["pelvis"].translation).length + tscale = h_tgt / h_src if h_src > 1e-5 else 1.0 + + new_act = bpy.data.actions.new(name) + if tgt_arm.animation_data is None: + tgt_arm.animation_data_create() + tgt_arm.animation_data.action = new_act + + order = hierarchy_order(tgt_arm) + tw_inv = tgt_arm.matrix_world.inverted() + for pb in tgt_arm.pose.bones: + pb.rotation_mode = "QUATERNION" + + src_hips_rest_t = src_rest["CC_Base_Hip"].translation + + for f in range(f0, f1 + 1): + scn.frame_set(f) + dg = bpy.context.evaluated_depsgraph_get() + se = src_arm.evaluated_get(dg) + spose = {pb.name: (src_arm.matrix_world @ pb.matrix) for pb in se.pose.bones} + + tgt_world = {} + for tb in order: + nt = tb.name + rest_t = tgt_rest[nt] + ns = inv_map.get(nt) + if ns is None or ns not in spose: + if tb.parent is None: + tgt_world[nt] = rest_t + else: + rel = tgt_rest[tb.parent.name].inverted() @ rest_t + tgt_world[nt] = tgt_world[tb.parent.name] @ rel + continue + q_delta = (spose[ns].to_quaternion().normalized() @ src_rest_q[ns].inverted()).normalized() + q_delta = (C @ q_delta @ Cinv).normalized() + q_world = (q_delta @ tgt_rest_q[nt]).normalized() + if nt == "pelvis": + d = C @ (spose[ns].translation - src_hips_rest_t) * tscale * unit + t_world = rest_t.translation + d + else: + rel = tgt_rest[tb.parent.name].inverted() @ rest_t + t_world = (tgt_world[tb.parent.name] @ rel).translation + tgt_world[nt] = Matrix.LocRotScale(t_world, q_world, Vector((1, 1, 1))) + + for tb in order: + pb = tgt_arm.pose.bones[tb.name] + m_arm = tw_inv @ tgt_world[tb.name] + if tb.parent: + m_parent_arm = tw_inv @ tgt_world[tb.parent.name] + rel_rest = tb.parent.matrix_local.inverted() @ tb.matrix_local + basis = rel_rest.inverted() @ (m_parent_arm.inverted() @ m_arm) + else: + basis = tb.matrix_local.inverted() @ m_arm + pb.matrix_basis = basis + pb.keyframe_insert("rotation_quaternion", frame=f) + if tb.name == "pelvis" or tb.parent is None: + pb.keyframe_insert("location", frame=f) + + tgt_arm.animation_data.action = None + return new_act + + +def main(): + argv = sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else [] + args = dict(zip(argv[::2], argv[1::2])) + srcs, out = args["--src"].split(","), args["--out"] + target = args.get("--target", DEFAULT_TARGET_GLTF) + forced_name = args.get("--name") + + fbx_files = [] + for s in srcs: + s = s.strip() + if os.path.isdir(s): + for root, _, files in os.walk(s): + fbx_files += [os.path.join(root, x) for x in sorted(files) if x.lower().endswith(".fbx")] + else: + fbx_files.append(s) + print(f"[cc] {len(fbx_files)} clips → {out}") + + bpy.ops.object.select_all(action="SELECT") + bpy.ops.object.delete() + scn = bpy.context.scene + scn.render.fps = 60 # iClone exports at 60; glTF export resamples fine + + bpy.ops.import_scene.gltf(filepath=target) + tgt_arm = next(o for o in bpy.data.objects if o.type == "ARMATURE") + tgt_arm.name = "Armature" + for o in [o for o in bpy.data.objects if o.type == "MESH"]: + bpy.data.objects.remove(o, do_unlink=True) + bpy.context.view_layer.update() + apply_object_transform(tgt_arm) + + done = [] + for i, f in enumerate(fbx_files): + name = forced_name if (forced_name and len(fbx_files) == 1) else clip_name(f) + pre = set(bpy.data.objects) + pre_actions = set(bpy.data.actions) + bpy.ops.import_scene.fbx(filepath=f) + new_objs = [o for o in bpy.data.objects if o not in pre] + src_arm = next((o for o in new_objs + if o.type == "ARMATURE" and any("CC_Base_Hip" == b.name for b in o.data.bones)), None) + new_acts = [a for a in bpy.data.actions if a not in pre_actions] + if src_arm is None or not new_acts: + print(f"[cc] SKIP (no CC armature/action): {f}") + else: + bpy.context.view_layer.update() + unit = src_arm.scale.x # capture BEFORE transform_apply (fcurves stay in cm) + apply_object_transform(src_arm) + act = pick_action(src_arm, new_acts) + if act is None: + print(f"[cc] SKIP (no armature take): {f}") + else: + baked = retarget_one(src_arm, tgt_arm, act, name, scn, unit) + done.append(baked) + print(f"[cc] {i + 1}/{len(fbx_files)} {name} " + f"({int(baked.frame_range[1])}f @60, from take '{act.name}')") + for o in new_objs: + bpy.data.objects.remove(o, do_unlink=True) + for a in new_acts: + bpy.data.actions.remove(a) + + if not done: + raise RuntimeError("nothing retargeted — see messages above") + for act in done: + tr = tgt_arm.animation_data.nla_tracks.new() + tr.name = act.name + tr.strips.new(act.name, 1, act) + bpy.ops.object.select_all(action="DESELECT") + tgt_arm.select_set(True) + bpy.context.view_layer.objects.active = tgt_arm + bpy.ops.export_scene.gltf( + filepath=out, use_selection=True, + export_animations=True, export_animation_mode="NLA_TRACKS", + export_force_sampling=True, export_optimize_animation_size=False) + print(f"[cc] EXPORTED {len(done)} clips → {out}") + + +main() diff --git a/tools/kevin_retarget.py b/tools/kevin_retarget.py new file mode 100644 index 0000000..3a22b1b --- /dev/null +++ b/tools/kevin_retarget.py @@ -0,0 +1,254 @@ +# Kevin Iglesias FBX → Quaternius-skeleton GLB batch retargeter (Blender 5.x, headless). +# +# THE one-time migration path (Ozan 2026-07-07 "I want to do this migration once"): +# originals live in ariki-assets/animations/kevin-iglesias (Unity FBX, T-pose "B-*" rig); +# this script bakes them onto the game's Quaternius UE-named skeleton so the runtime loads +# them exactly like the UAL GLBs (PlayerController.LoadUalLibrary) — no runtime retarget, +# no deprecated Godot BoneMap import path. +# +# Method: world-rotation-delta retarget. Both rigs rest in a T-POSE (verified 2026-07-07: +# Quaternius upperarm_l dir=(1,0,0), Kevin B-upperArm.L dir=(1,0,0)), so for every mapped +# bone the world rotation delta from rest transfers 1:1: +# q_target_world(f) = (q_src_world(f) · q_src_rest⁻¹) · q_target_rest +# Pelvis additionally carries scaled root translation. Unmapped target bones (spine_03, +# leaf/ball bones, missing fingers) ride their parent rigidly at rest offset. +# +# Usage: +# blender --background --python tools/kevin_retarget.py -- \ +# --src "[,…]" --out [--limit N] +# +# Each FBX becomes one glTF animation named from the file: "HumanM@Eat01_R - Loop" → +# "Eat01_R_Loop". Output GLB = armature + all animations (no mesh). + +import bpy, sys, os, re +from mathutils import Matrix, Quaternion, Vector + +TARGET_GLTF = "/Users/behcetozanbozkurt/Documents/tinqs-ltd/ariki-game/assets/quaternius/base-characters/Universal Base Characters[Standard]/Base Characters/Godot - UE/Superhero_Male_FullBody.gltf" + +# Kevin "B-*" → Quaternius UE names. Fingers map 1:1 (both rigs have 3-segment fingers). +BONE_MAP = { + "B-hips": "pelvis", + "B-spine": "spine_01", + "B-chest": "spine_02", + "B-neck": "neck_01", + "B-head": "Head", +} +for S, T in (("L", "l"), ("R", "r")): + BONE_MAP[f"B-shoulder.{S}"] = f"clavicle_{T}" + BONE_MAP[f"B-upperArm.{S}"] = f"upperarm_{T}" + BONE_MAP[f"B-forearm.{S}"] = f"lowerarm_{T}" + BONE_MAP[f"B-hand.{S}"] = f"hand_{T}" + BONE_MAP[f"B-thigh.{S}"] = f"thigh_{T}" + BONE_MAP[f"B-shin.{S}"] = f"calf_{T}" + BONE_MAP[f"B-foot.{S}"] = f"foot_{T}" + BONE_MAP[f"B-toe.{S}"] = f"ball_{T}" + for i in (1, 2, 3): + BONE_MAP[f"B-indexFinger0{i}.{S}"] = f"index_0{i}_{T}" + BONE_MAP[f"B-middleFinger0{i}.{S}"] = f"middle_0{i}_{T}" + BONE_MAP[f"B-ringFinger0{i}.{S}"] = f"ring_0{i}_{T}" + BONE_MAP[f"B-pinky0{i}.{S}"] = f"pinky_0{i}_{T}" + BONE_MAP[f"B-thumb0{i}.{S}"] = f"thumb_0{i}_{T}" +INV_MAP = {v: k for k, v in BONE_MAP.items()} + + +def apply_object_transform(obj): + """Bake the importer's object-level rotation/scale into the armature data. Makes Blender + world == data space (meters, Z-up) for BOTH rigs, so exported glTF matches what Godot's + importer produced for the original Quaternius file — same units, same axes, same rests.""" + bpy.ops.object.select_all(action="DESELECT") + obj.select_set(True) + bpy.context.view_layer.objects.active = obj + bpy.ops.object.transform_apply(location=True, rotation=True, scale=True) + bpy.context.view_layer.update() + + +def clip_name(path): + base = os.path.splitext(os.path.basename(path))[0] + base = base.split("@", 1)[-1] # drop HumanM@ / HumanF@ / Villager@ … + base = base.replace(" - ", "_").replace(" ", "") + return re.sub(r"[^A-Za-z0-9_\-]", "", base) + + +def hierarchy_order(arm): + out, stack = [], [b for b in arm.data.bones if b.parent is None] + while stack: + b = stack.pop(0) + out.append(b) + stack.extend(b.children) + return out + + +def char_frame(rest, hips, head, upper_arm_l): + """Orthonormal character frame from the rest pose: columns = (left-arm axis, hips→head up, + their cross). Lets us align two rigs whatever world orientation their importer left them in + — the old pipeline's limb bug was exactly an unhandled world-frame mismatch.""" + u = (rest[head].translation - rest[hips].translation).normalized() + l = (rest[upper_arm_l].to_quaternion() @ Vector((0, 1, 0))).normalized() # bone Y = along bone + f = l.cross(u).normalized() + l = u.cross(f).normalized() + m = Matrix((l, u, f)).transposed() + return m.to_quaternion().normalized() + + +def retarget_one(src_arm, tgt_arm, action, name, scn, unit): + f0, f1 = int(action.frame_range[0]), int(action.frame_range[1]) + if src_arm.animation_data is None: + src_arm.animation_data_create() + src_arm.animation_data.action = action + # Blender 5 slotted actions: bind the first slot so the action actually evaluates. + if getattr(action, "slots", None) and src_arm.animation_data.action_slot is None: + src_arm.animation_data.action_slot = action.slots[0] + + tgt_rest = {b.name: (tgt_arm.matrix_world @ b.matrix_local) for b in tgt_arm.data.bones} + src_rest = {b.name: (src_arm.matrix_world @ b.matrix_local) for b in src_arm.data.bones} + src_rest_q = {n: m.to_quaternion().normalized() for n, m in src_rest.items()} + tgt_rest_q = {n: m.to_quaternion().normalized() for n, m in tgt_rest.items()} + + # World-frame alignment C: rotates source-world axes onto target-world axes (both rigs are + # T-pose, so the character frames coincide semantically). All rotation deltas and root + # translation deltas are conjugated/rotated through C before applying to the target. + C = (char_frame(tgt_rest, "pelvis", "Head", "upperarm_l") + @ char_frame(src_rest, "B-hips", "B-head", "B-upperArm.L").inverted()).normalized() + Cinv = C.inverted() + + # Uniform size ratio from hips→head world distance (unit/scale agnostic). + h_src = (src_rest["B-head"].translation - src_rest["B-hips"].translation).length + h_tgt = (tgt_rest["Head"].translation - tgt_rest["pelvis"].translation).length + tscale = h_tgt / h_src if h_src > 1e-5 else 1.0 + + new_act = bpy.data.actions.new(name) + if tgt_arm.animation_data is None: + tgt_arm.animation_data_create() + tgt_arm.animation_data.action = new_act + + order = hierarchy_order(tgt_arm) + tw_inv = tgt_arm.matrix_world.inverted() + for pb in tgt_arm.pose.bones: + pb.rotation_mode = "QUATERNION" + + # ── Pelvis translation: the raw hips fcurve channel ── + # Calibrated on Eat (static ≈0) + SleepGround Down (y 0→−86cm): Kevin's hips LOCAL location + # is placement-free (the demo-scene offset lives on the unmapped B-root), cm-scaled, and + # standing = 0 with −Y = down, X/Z = horizontal. Maps straight onto the target's Z-up world: + # (x, z, y)·unit·tscale added to the pelvis rest. No world-frame algebra, no rest quirks. + + for f in range(f0, f1 + 1): + scn.frame_set(f) + dg = bpy.context.evaluated_depsgraph_get() + se = src_arm.evaluated_get(dg) + spose = {pb.name: (src_arm.matrix_world @ pb.matrix) for pb in se.pose.bones} + + tgt_world = {} + for tb in order: + nt = tb.name + rest_t = tgt_rest[nt] + ns = INV_MAP.get(nt) + if ns is None or ns not in spose: + if tb.parent is None: + tgt_world[nt] = rest_t + else: + rel = tgt_rest[tb.parent.name].inverted() @ rest_t + tgt_world[nt] = tgt_world[tb.parent.name] @ rel + continue + q_delta = (spose[ns].to_quaternion().normalized() @ src_rest_q[ns].inverted()).normalized() + q_delta = (C @ q_delta @ Cinv).normalized() # express the delta in TARGET world axes + q_world = (q_delta @ tgt_rest_q[nt]).normalized() + if nt == "pelvis": + loc = se.pose.bones["B-hips"].location + t_world = rest_t.translation + Vector((loc.x, loc.z, loc.y)) * unit * tscale + else: + rel = tgt_rest[tb.parent.name].inverted() @ rest_t + t_world = (tgt_world[tb.parent.name] @ rel).translation + tgt_world[nt] = Matrix.LocRotScale(t_world, q_world, Vector((1, 1, 1))) + + for tb in order: + pb = tgt_arm.pose.bones[tb.name] + m_arm = tw_inv @ tgt_world[tb.name] + if tb.parent: + m_parent_arm = tw_inv @ tgt_world[tb.parent.name] + rel_rest = tb.parent.matrix_local.inverted() @ tb.matrix_local + basis = rel_rest.inverted() @ (m_parent_arm.inverted() @ m_arm) + else: + basis = tb.matrix_local.inverted() @ m_arm + pb.matrix_basis = basis + pb.keyframe_insert("rotation_quaternion", frame=f) + if tb.name == "pelvis" or tb.parent is None: + pb.keyframe_insert("location", frame=f) + + tgt_arm.animation_data.action = None + return new_act + + +def main(): + argv = sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else [] + args = dict(zip(argv[::2], argv[1::2])) + srcs, out, limit = args["--src"].split(","), args["--out"], int(args.get("--limit", 0)) + + fbx_files = [] + for s in srcs: + s = s.strip() + if os.path.isdir(s): + for root, _, files in os.walk(s): + fbx_files += [os.path.join(root, x) for x in sorted(files) if x.lower().endswith(".fbx")] + else: + fbx_files.append(s) + if limit: + fbx_files = fbx_files[:limit] + print(f"[kevin] {len(fbx_files)} clips → {out}") + + bpy.ops.object.select_all(action="SELECT") + bpy.ops.object.delete() + scn = bpy.context.scene + scn.render.fps = 30 + + bpy.ops.import_scene.gltf(filepath=TARGET_GLTF) + tgt_arm = next(o for o in bpy.data.objects if o.type == "ARMATURE") + tgt_arm.name = "Armature" + # Drop meshes — animations-only GLB (small, loads fast; the game binds clips to its own body). + for o in [o for o in bpy.data.objects if o.type == "MESH"]: + bpy.data.objects.remove(o, do_unlink=True) + bpy.context.view_layer.update() + apply_object_transform(tgt_arm) + + done = [] + for i, f in enumerate(fbx_files): + name = clip_name(f) + pre = set(bpy.data.objects) | set() + pre_actions = set(bpy.data.actions) + bpy.ops.import_scene.fbx(filepath=f) + new_objs = [o for o in bpy.data.objects if o not in pre] + src_arm = next((o for o in new_objs if o.type == "ARMATURE"), None) + new_acts = [a for a in bpy.data.actions if a not in pre_actions] + if src_arm is None or not new_acts: + print(f"[kevin] SKIP (no armature/action): {f}") + else: + bpy.context.view_layer.update() + # transform_apply keeps the ROTATION math in one consistent world frame (engine- + # validated); the pelvis TRANSLATION reads the raw hips fcurve channel instead, so + # capture the importer's unit scale (cm→m 0.01) before apply wipes it. + unit = src_arm.scale.x + apply_object_transform(src_arm) + act = retarget_one(src_arm, tgt_arm, new_acts[0], name, scn, unit) + done.append(act) + print(f"[kevin] {i + 1}/{len(fbx_files)} {name} ({int(act.frame_range[1])}f)") + for o in new_objs: + bpy.data.objects.remove(o, do_unlink=True) + for a in new_acts: + bpy.data.actions.remove(a) + + # Stash every baked action on an NLA track so the glTF exporter emits it as an animation. + for act in done: + tr = tgt_arm.animation_data.nla_tracks.new() + tr.name = act.name + tr.strips.new(act.name, 1, act) + bpy.ops.object.select_all(action="DESELECT") + tgt_arm.select_set(True) + bpy.context.view_layer.objects.active = tgt_arm + bpy.ops.export_scene.gltf( + filepath=out, use_selection=True, + export_animations=True, export_animation_mode="NLA_TRACKS", + export_force_sampling=True, export_optimize_animation_size=False) + print(f"[kevin] EXPORTED {len(done)} clips → {out}") + + +main() diff --git a/tools/mixamo_retarget.py b/tools/mixamo_retarget.py new file mode 100644 index 0000000..a7fb7cf --- /dev/null +++ b/tools/mixamo_retarget.py @@ -0,0 +1,257 @@ +# Mixamo FBX → Quaternius-skeleton GLB batch retargeter (Blender headless). +# +# Sibling of tools/kevin_retarget.py — same world-rotation-delta method, adapted for the +# Mixamo rig ("mixamorig:*" bones, prefix auto-detected). Both rigs rest in a T-pose, so +# for every mapped bone the world rotation delta from rest transfers 1:1: +# q_target_world(f) = (q_src_world(f) · q_src_rest⁻¹) · q_target_rest +# Pelvis translation transfers as a WORLD-space delta from rest (rotated through the +# frame-alignment quaternion C, scaled by the hips→head size ratio) — unlike Kevin's +# raw-fcurve read, because Mixamo roots the armature at the hips with no demo-scene +# offset bone, and world deltas survive whatever local axes the importer picked. +# Unmapped target bones (leaf/ball bones etc.) ride their parent rigidly at rest offset. +# +# Usage: +# blender --background --python tools/mixamo_retarget.py -- \ +# --src "[,…]" --out [--limit N] [--target ] + +import bpy, sys, os, re +from mathutils import Matrix, Quaternion, Vector + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DEFAULT_TARGET_GLTF = os.path.join( + REPO_ROOT, "assets", "quaternius", "base-characters", + "Universal Base Characters[Standard]", "Base Characters", "Godot - UE", + "Superhero_Male_FullBody.gltf") + +# Mixamo (prefix-stripped) → Quaternius UE names. Fingers map 1:1 (both rigs 3-segment). +BONE_MAP = { + "Hips": "pelvis", + "Spine": "spine_01", + "Spine1": "spine_02", + "Spine2": "spine_03", + "Neck": "neck_01", + "Head": "Head", +} +for S, T in (("Left", "l"), ("Right", "r")): + BONE_MAP[f"{S}Shoulder"] = f"clavicle_{T}" + BONE_MAP[f"{S}Arm"] = f"upperarm_{T}" + BONE_MAP[f"{S}ForeArm"] = f"lowerarm_{T}" + BONE_MAP[f"{S}Hand"] = f"hand_{T}" + BONE_MAP[f"{S}UpLeg"] = f"thigh_{T}" + BONE_MAP[f"{S}Leg"] = f"calf_{T}" + BONE_MAP[f"{S}Foot"] = f"foot_{T}" + BONE_MAP[f"{S}ToeBase"] = f"ball_{T}" + for i in (1, 2, 3): + BONE_MAP[f"{S}HandIndex{i}"] = f"index_0{i}_{T}" + BONE_MAP[f"{S}HandMiddle{i}"] = f"middle_0{i}_{T}" + BONE_MAP[f"{S}HandRing{i}"] = f"ring_0{i}_{T}" + BONE_MAP[f"{S}HandPinky{i}"] = f"pinky_0{i}_{T}" + BONE_MAP[f"{S}HandThumb{i}"] = f"thumb_0{i}_{T}" + + +def detect_prefix(arm): + """Mixamo bones are usually 'mixamorig:Hips' but exports vary ('mixamorig1:', none).""" + for b in arm.data.bones: + if b.name.endswith("Hips"): + return b.name[: -len("Hips")] + raise RuntimeError("no *Hips bone found — not a Mixamo rig?") + + +def apply_object_transform(obj): + """Bake importer object-level rotation/scale into the armature data (meters, Z-up).""" + bpy.ops.object.select_all(action="DESELECT") + obj.select_set(True) + bpy.context.view_layer.objects.active = obj + bpy.ops.object.transform_apply(location=True, rotation=True, scale=True) + bpy.context.view_layer.update() + + +def clip_name(path): + base = os.path.splitext(os.path.basename(path))[0] + base = base.replace(" - ", "_").replace(" ", "") + return re.sub(r"[^A-Za-z0-9_\-]", "", base) + + +def hierarchy_order(arm): + out, stack = [], [b for b in arm.data.bones if b.parent is None] + while stack: + b = stack.pop(0) + out.append(b) + stack.extend(b.children) + return out + + +def char_frame(rest, hips, head, upper_arm_l): + """Orthonormal character frame from the rest pose: columns = (left-arm axis, hips→head up, + their cross) — aligns the two rigs whatever world orientation the importer left them in.""" + u = (rest[head].translation - rest[hips].translation).normalized() + l = (rest[upper_arm_l].to_quaternion() @ Vector((0, 1, 0))).normalized() # bone Y = along bone + f = l.cross(u).normalized() + l = u.cross(f).normalized() + m = Matrix((l, u, f)).transposed() + return m.to_quaternion().normalized() + + +def retarget_one(src_arm, tgt_arm, action, name, scn, prefix, unit): + f0, f1 = int(action.frame_range[0]), int(action.frame_range[1]) + if src_arm.animation_data is None: + src_arm.animation_data_create() + src_arm.animation_data.action = action + # Blender 5 slotted actions: bind the first slot so the action actually evaluates. + if getattr(action, "slots", None) and src_arm.animation_data.action_slot is None: + src_arm.animation_data.action_slot = action.slots[0] + + inv_map = {v: prefix + k for k, v in BONE_MAP.items()} + + tgt_rest = {b.name: (tgt_arm.matrix_world @ b.matrix_local) for b in tgt_arm.data.bones} + src_rest = {b.name: (src_arm.matrix_world @ b.matrix_local) for b in src_arm.data.bones} + src_rest_q = {n: m.to_quaternion().normalized() for n, m in src_rest.items()} + tgt_rest_q = {n: m.to_quaternion().normalized() for n, m in tgt_rest.items()} + + C = (char_frame(tgt_rest, "pelvis", "Head", "upperarm_l") + @ char_frame(src_rest, prefix + "Hips", prefix + "Head", prefix + "LeftArm").inverted()).normalized() + Cinv = C.inverted() + + # Uniform size ratio from hips→head world distance (unit/scale agnostic). + h_src = (src_rest[prefix + "Head"].translation - src_rest[prefix + "Hips"].translation).length + h_tgt = (tgt_rest["Head"].translation - tgt_rest["pelvis"].translation).length + tscale = h_tgt / h_src if h_src > 1e-5 else 1.0 + + new_act = bpy.data.actions.new(name) + if tgt_arm.animation_data is None: + tgt_arm.animation_data_create() + tgt_arm.animation_data.action = new_act + + order = hierarchy_order(tgt_arm) + tw_inv = tgt_arm.matrix_world.inverted() + for pb in tgt_arm.pose.bones: + pb.rotation_mode = "QUATERNION" + + src_hips_rest_t = src_rest[prefix + "Hips"].translation + + for f in range(f0, f1 + 1): + scn.frame_set(f) + dg = bpy.context.evaluated_depsgraph_get() + se = src_arm.evaluated_get(dg) + spose = {pb.name: (src_arm.matrix_world @ pb.matrix) for pb in se.pose.bones} + + tgt_world = {} + for tb in order: + nt = tb.name + rest_t = tgt_rest[nt] + ns = inv_map.get(nt) + if ns is None or ns not in spose: + if tb.parent is None: + tgt_world[nt] = rest_t + else: + rel = tgt_rest[tb.parent.name].inverted() @ rest_t + tgt_world[nt] = tgt_world[tb.parent.name] @ rel + continue + q_delta = (spose[ns].to_quaternion().normalized() @ src_rest_q[ns].inverted()).normalized() + q_delta = (C @ q_delta @ Cinv).normalized() # express the delta in TARGET world axes + q_world = (q_delta @ tgt_rest_q[nt]).normalized() + if nt == "pelvis": + # World-space hips delta from rest → target world, size-scaled. Carries the + # dance's floor work (crouches, drops) without caring about local axes. + # × unit: transform_apply bakes the importer's cm→m scale into bone REST data, + # but the hips location FCURVES stay in cm — the raw world delta is 100× real. + d = C @ (spose[ns].translation - src_hips_rest_t) * tscale * unit + t_world = rest_t.translation + d + else: + rel = tgt_rest[tb.parent.name].inverted() @ rest_t + t_world = (tgt_world[tb.parent.name] @ rel).translation + tgt_world[nt] = Matrix.LocRotScale(t_world, q_world, Vector((1, 1, 1))) + + for tb in order: + pb = tgt_arm.pose.bones[tb.name] + m_arm = tw_inv @ tgt_world[tb.name] + if tb.parent: + m_parent_arm = tw_inv @ tgt_world[tb.parent.name] + rel_rest = tb.parent.matrix_local.inverted() @ tb.matrix_local + basis = rel_rest.inverted() @ (m_parent_arm.inverted() @ m_arm) + else: + basis = tb.matrix_local.inverted() @ m_arm + pb.matrix_basis = basis + pb.keyframe_insert("rotation_quaternion", frame=f) + if tb.name == "pelvis" or tb.parent is None: + pb.keyframe_insert("location", frame=f) + + tgt_arm.animation_data.action = None + return new_act + + +def main(): + argv = sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else [] + args = dict(zip(argv[::2], argv[1::2])) + srcs, out = args["--src"].split(","), args["--out"] + limit = int(args.get("--limit", 0)) + target = args.get("--target", DEFAULT_TARGET_GLTF) + + fbx_files = [] + for s in srcs: + s = s.strip() + if os.path.isdir(s): + for root, _, files in os.walk(s): + fbx_files += [os.path.join(root, x) for x in sorted(files) if x.lower().endswith(".fbx")] + else: + fbx_files.append(s) + if limit: + fbx_files = fbx_files[:limit] + print(f"[mixamo] {len(fbx_files)} clips → {out}") + + bpy.ops.object.select_all(action="SELECT") + bpy.ops.object.delete() + scn = bpy.context.scene + scn.render.fps = 30 + + bpy.ops.import_scene.gltf(filepath=target) + tgt_arm = next(o for o in bpy.data.objects if o.type == "ARMATURE") + tgt_arm.name = "Armature" + # Drop meshes — animations-only GLB (the game binds clips to its own body). + for o in [o for o in bpy.data.objects if o.type == "MESH"]: + bpy.data.objects.remove(o, do_unlink=True) + bpy.context.view_layer.update() + apply_object_transform(tgt_arm) + + done = [] + for i, f in enumerate(fbx_files): + name = clip_name(f) + pre = set(bpy.data.objects) + pre_actions = set(bpy.data.actions) + bpy.ops.import_scene.fbx(filepath=f) + new_objs = [o for o in bpy.data.objects if o not in pre] + src_arm = next((o for o in new_objs if o.type == "ARMATURE"), None) + new_acts = [a for a in bpy.data.actions if a not in pre_actions] + if src_arm is None or not new_acts: + print(f"[mixamo] SKIP (no armature/action): {f}") + else: + bpy.context.view_layer.update() + prefix = detect_prefix(src_arm) + # Importer unit scale (cm→m 0.01) — capture BEFORE transform_apply wipes it; + # the hips translation math needs it (fcurves stay in source units). + unit = src_arm.scale.x + apply_object_transform(src_arm) + act = retarget_one(src_arm, tgt_arm, new_acts[0], name, scn, prefix, unit) + done.append(act) + print(f"[mixamo] {i + 1}/{len(fbx_files)} {name} ({int(act.frame_range[1])}f, prefix='{prefix}')") + for o in new_objs: + bpy.data.objects.remove(o, do_unlink=True) + for a in new_acts: + bpy.data.actions.remove(a) + + # Stash every baked action on an NLA track so the glTF exporter emits it as an animation. + for act in done: + tr = tgt_arm.animation_data.nla_tracks.new() + tr.name = act.name + tr.strips.new(act.name, 1, act) + bpy.ops.object.select_all(action="DESELECT") + tgt_arm.select_set(True) + bpy.context.view_layer.objects.active = tgt_arm + bpy.ops.export_scene.gltf( + filepath=out, use_selection=True, + export_animations=True, export_animation_mode="NLA_TRACKS", + export_force_sampling=True, export_optimize_animation_size=False) + print(f"[mixamo] EXPORTED {len(done)} clips → {out}") + + +main() diff --git a/tools/mocap_retarget.py b/tools/mocap_retarget.py new file mode 100644 index 0000000..5fc1e60 --- /dev/null +++ b/tools/mocap_retarget.py @@ -0,0 +1,274 @@ +# MediaPipe pose JSON → Quaternius-skeleton GLB animation (Blender headless). +# +# Sibling of tools/mixamo_retarget.py — same target rig, same NLA/export tail, but the +# source is not an armature: it's per-frame 3D world landmarks from the pose-estimation +# skill (extract_pose.py). Rotations are SOLVED, not transferred: +# - limb bones (arms/legs/hands/feet): shortest-arc delta from the bone's REST world +# direction to the landmark-derived direction, applied on top of the rest rotation — +# preserves the rig's rest roll, so skinning stays sane (no twist control in v1). +# - pelvis + spine chain: full-basis deltas built from the hip line and the hips→ +# shoulders up vector; spine_01..03 slerp from pelvis delta toward chest delta. +# - pelvis HEIGHT: MediaPipe world landmarks are hip-centered (origin = mid-hips every +# frame), so global translation is lost. We reconstruct the vertical: per-frame +# pelvis-above-ankle extent vs the standing extent (95th pct) scales the rest height +# — carries squats/crouches, which is most of a haka. +# Landmarks are box-smoothed over time before solving (mocap jitter). +# +# MediaPipe world axes: x=subject-image right, y=down, z=depth (smaller=nearer camera). +# Converted here to Blender Z-up with the subject facing -Y: b = (x, z, -y). +# +# Usage: +# blender --background --python tools/mocap_retarget.py -- \ +# --pose --out --name \ +# [--range lo:hi] [--smooth 5] [--target ] [--render-check ] + +import bpy, sys, os, json +import numpy as np +from mathutils import Matrix, Quaternion, Vector + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DEFAULT_TARGET_GLTF = os.path.join( + REPO_ROOT, "assets", "quaternius", "base-characters", + "Universal Base Characters[Standard]", "Base Characters", "Godot - UE", + "Superhero_Male_FullBody.gltf") + +# MediaPipe pose landmark indices +NOSE, L_SHO, R_SHO, L_ELB, R_ELB, L_WRI, R_WRI = 0, 11, 12, 13, 14, 15, 16 +L_PINK, R_PINK, L_IDX, R_IDX = 17, 18, 19, 20 +L_HIP, R_HIP, L_KNE, R_KNE, L_ANK, R_ANK, L_FT, R_FT = 23, 24, 25, 26, 27, 28, 31, 32 + +# target bone -> (landmark_from, landmark_to): desired bone direction (bone +Y points from→to) +AIM_BONES = { + "upperarm_l": (L_SHO, L_ELB), "lowerarm_l": (L_ELB, L_WRI), + "upperarm_r": (R_SHO, R_ELB), "lowerarm_r": (R_ELB, R_WRI), + "thigh_l": (L_HIP, L_KNE), "calf_l": (L_KNE, L_ANK), "foot_l": (L_ANK, L_FT), + "thigh_r": (R_HIP, R_KNE), "calf_r": (R_KNE, R_ANK), "foot_r": (R_ANK, R_FT), +} +HAND_BONES = {"hand_l": (L_WRI, L_IDX, L_PINK), "hand_r": (R_WRI, R_IDX, R_PINK)} +SPINE_WEIGHTS = {"spine_01": 0.35, "spine_02": 0.7, "spine_03": 1.0, + "neck_01": 1.0, "Head": 1.0} + + +def apply_object_transform(obj): + bpy.ops.object.select_all(action="DESELECT") + obj.select_set(True) + bpy.context.view_layer.objects.active = obj + bpy.ops.object.transform_apply(location=True, rotation=True, scale=True) + bpy.context.view_layer.update() + + +def hierarchy_order(arm): + out, stack = [], [b for b in arm.data.bones if b.parent is None] + while stack: + b = stack.pop(0) + out.append(b) + stack.extend(b.children) + return out + + +def torso_quat(pts): + """Orthonormal torso basis from landmarks (Blender axes): columns = (subject-left, + up, forward). Subject-left = l_hip - r_hip; up = mid-shoulders - mid-hips.""" + left = Vector(pts[L_HIP] - pts[R_HIP]).normalized() + up = Vector((pts[L_SHO] + pts[R_SHO]) / 2 - (pts[L_HIP] + pts[R_HIP]) / 2).normalized() + fwd = left.cross(up).normalized() + left = up.cross(fwd).normalized() + return Matrix((left, up, fwd)).transposed().to_quaternion().normalized() + + +def chest_quat(pts): + left = Vector(pts[L_SHO] - pts[R_SHO]).normalized() + up = Vector((pts[L_SHO] + pts[R_SHO]) / 2 - (pts[L_HIP] + pts[R_HIP]) / 2).normalized() + fwd = left.cross(up).normalized() + left = up.cross(fwd).normalized() + return Matrix((left, up, fwd)).transposed().to_quaternion().normalized() + + +def main(): + argv = sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else [] + args = dict(zip(argv[::2], argv[1::2])) + pose_path, out, clip = args["--pose"], args["--out"], args.get("--name", "Mocap01") + target = args.get("--target", DEFAULT_TARGET_GLTF) + smooth = int(args.get("--smooth", 5)) + render_dir = args.get("--render-check") + lo, hi = 0.0, 1e9 + if "--range" in args: + a, _, b = args["--range"].partition(":") + lo, hi = float(a or 0), float(b) if b else 1e9 + + doc = json.load(open(pose_path)) + frames = [f for f in doc["frames"] + if f.get("detected") and "world_landmarks" in f and lo <= f["t"] <= hi] + if len(frames) < 4: + raise RuntimeError(f"only {len(frames)} usable frames in range {lo}:{hi}") + ts = np.array([f["t"] for f in frames]); ts -= ts[0] + # MediaPipe (x right, y down, z depth) → Blender (x right, y=z_mp, z=-y_mp) + raw = np.array([[[p["x"], p["z"], -p["y"]] for p in f["world_landmarks"]] for f in frames]) + if smooth > 1: # box smooth over time, per landmark/axis + k = np.ones(smooth) / smooth + pad = smooth // 2 + padded = np.concatenate([raw[:1].repeat(pad, 0), raw, raw[-1:].repeat(pad, 0)]) + raw = np.apply_along_axis(lambda v: np.convolve(v, k, mode="valid"), 0, padded)[:len(ts)] + print(f"[mocap] {len(frames)} frames, {ts[-1]:.1f}s, smooth={smooth}") + + # ── target rig ── + bpy.ops.object.select_all(action="SELECT") + bpy.ops.object.delete() + scn = bpy.context.scene + scn.render.fps = 30 + bpy.ops.import_scene.gltf(filepath=target) + tgt_arm = next(o for o in bpy.data.objects if o.type == "ARMATURE") + tgt_arm.name = "Armature" + bpy.context.view_layer.update() + apply_object_transform(tgt_arm) + + tgt_rest = {b.name: (tgt_arm.matrix_world @ b.matrix_local) for b in tgt_arm.data.bones} + tgt_rest_q = {n: m.to_quaternion().normalized() for n, m in tgt_rest.items()} + + # Char-frame alignment, mixamo_retarget style: A maps landmark-space vectors and + # rotation deltas into target world. Landmark space after conversion: subject-left=+X, + # up=+Z, facing=-Y (NOT identity — it carries a 90° X rotation vs standard basis). + up_t = (tgt_rest["Head"].translation - tgt_rest["pelvis"].translation).normalized() + left_t = (tgt_rest_q["upperarm_l"] @ Vector((0, 1, 0))).normalized() # bone Y = along bone + fwd_t = left_t.cross(up_t).normalized() + left_t = up_t.cross(fwd_t).normalized() + Ct = Matrix((left_t, up_t, fwd_t)).transposed().to_quaternion().normalized() + Cs = Matrix((Vector((1, 0, 0)), Vector((0, 0, 1)), Vector((0, -1, 0)))).transposed() \ + .to_quaternion().normalized() + A = (Ct @ Cs.inverted()).normalized() + Ainv = A.inverted() + Cs_inv = Cs.inverted() + + # Size + pelvis-height model + h_tgt = (tgt_rest["Head"].translation - tgt_rest["pelvis"].translation).length + pelvis_up = raw[:, [L_ANK, R_ANK], 2].min(axis=1) * -1.0 # pelvis height above lowest ankle + standing_h = float(np.percentile(pelvis_up, 95)) + rest_pelvis = tgt_rest["pelvis"].translation.copy() + + act = bpy.data.actions.new(clip) + if tgt_arm.animation_data is None: + tgt_arm.animation_data_create() + tgt_arm.animation_data.action = act + order = hierarchy_order(tgt_arm) + tw_inv = tgt_arm.matrix_world.inverted() + for pb in tgt_arm.pose.bones: + pb.rotation_mode = "QUATERNION" + + check = [] + for i in range(len(ts)): + pts = raw[i] + frame = int(round(ts[i] * 30)) + 1 + + # delta from landmark rest basis (Cs), conjugated into target world axes + q_pelvis_d = (A @ (torso_quat(pts) @ Cs_inv) @ Ainv).normalized() + q_chest_d = (A @ (chest_quat(pts) @ Cs_inv) @ Ainv).normalized() + + tgt_world = {} + for tb in order: + nt = tb.name + rest_t = tgt_rest[nt] + q_world = None + if nt == "pelvis": + q_world = (q_pelvis_d @ tgt_rest_q[nt]).normalized() + elif nt in SPINE_WEIGHTS: + qd = Quaternion(q_pelvis_d).slerp(q_chest_d, SPINE_WEIGHTS[nt]) + q_world = (qd @ tgt_rest_q[nt]).normalized() + elif nt in AIM_BONES: + a, b = AIM_BONES[nt] + d_lm = Vector(pts[b] - pts[a]) + if d_lm.length > 1e-6: + d_tgt = A @ d_lm + rest_dir = tgt_rest_q[nt] @ Vector((0, 1, 0)) + q_delta = rest_dir.rotation_difference(d_tgt.normalized()) + q_world = (q_delta @ tgt_rest_q[nt]).normalized() + elif nt in HAND_BONES: + w, ix, pk = HAND_BONES[nt] + d_lm = Vector((pts[ix] + pts[pk]) / 2 - pts[w]) + if d_lm.length > 1e-6: + d_tgt = A @ d_lm + rest_dir = tgt_rest_q[nt] @ Vector((0, 1, 0)) + q_delta = rest_dir.rotation_difference(d_tgt.normalized()) + q_world = (q_delta @ tgt_rest_q[nt]).normalized() + + if q_world is None: # unmapped: ride parent rigidly at rest offset + if tb.parent is None: + tgt_world[nt] = rest_t + else: + rel = tgt_rest[tb.parent.name].inverted() @ rest_t + tgt_world[nt] = tgt_world[tb.parent.name] @ rel + continue + + if nt == "pelvis": + ratio = float(np.clip(pelvis_up[i] / max(standing_h, 1e-5), 0.35, 1.15)) + t_world = Vector((rest_pelvis.x, rest_pelvis.y, rest_pelvis.z * ratio)) + else: + rel = tgt_rest[tb.parent.name].inverted() @ rest_t + t_world = (tgt_world[tb.parent.name] @ rel).translation + tgt_world[nt] = Matrix.LocRotScale(t_world, q_world, Vector((1, 1, 1))) + + for tb in order: + pb = tgt_arm.pose.bones[tb.name] + m_arm = tw_inv @ tgt_world[tb.name] + if tb.parent: + m_parent_arm = tw_inv @ tgt_world[tb.parent.name] + rel_rest = tb.parent.matrix_local.inverted() @ tb.matrix_local + basis = rel_rest.inverted() @ (m_parent_arm.inverted() @ m_arm) + else: + basis = tb.matrix_local.inverted() @ m_arm + pb.matrix_basis = basis + pb.keyframe_insert("rotation_quaternion", frame=frame) + if tb.name == "pelvis" or tb.parent is None: + pb.keyframe_insert("location", frame=frame) + + # numeric sanity: solved elbow angle vs landmark elbow angle (should match closely) + if i % max(1, len(ts) // 6) == 0: + def ang(u, v): + u, v = u.normalized(), v.normalized() + return np.degrees(np.arccos(np.clip(u.dot(v), -1, 1))) + lm = ang(Vector(pts[L_SHO] - pts[L_ELB]), Vector(pts[L_WRI] - pts[L_ELB])) + ua = (tgt_world["upperarm_l"].to_quaternion() @ Vector((0, 1, 0))) + la = (tgt_world["lowerarm_l"].to_quaternion() @ Vector((0, 1, 0))) + rig = ang(-ua, la) + check.append((ts[i], lm, rig)) + + print("[mocap] elbow-angle sanity (t, landmark°, rig°):") + for t, a, b in check: + print(f" t={t:5.1f} lm={a:6.1f} rig={b:6.1f} d={abs(a-b):5.1f}") + + # optional visual check renders (with mesh, before it's dropped) + if render_dir: + os.makedirs(render_dir, exist_ok=True) + cam_data = bpy.data.cameras.new("ChkCam") + cam = bpy.data.objects.new("ChkCam", cam_data) + scn.collection.objects.link(cam) + cam.location = (0, -3.2, 1.0) + cam.rotation_euler = (np.radians(87), 0, 0) + scn.camera = cam + sun = bpy.data.objects.new("Sun", bpy.data.lights.new("Sun", "SUN")) + scn.collection.objects.link(sun) + sun.rotation_euler = (np.radians(45), 0, np.radians(30)) + scn.render.resolution_x, scn.render.resolution_y = 480, 640 + for i in range(0, len(ts), max(1, len(ts) // 6)): + scn.frame_set(int(round(ts[i] * 30)) + 1) + scn.render.filepath = os.path.join(render_dir, f"chk_{ts[i]:05.1f}s.png") + bpy.ops.render.render(write_still=True) + print(f"[mocap] check renders → {render_dir}") + + # animations-only GLB (game binds clips to its own body) + for o in [o for o in bpy.data.objects if o.type == "MESH"]: + bpy.data.objects.remove(o, do_unlink=True) + tgt_arm.animation_data.action = None + tr = tgt_arm.animation_data.nla_tracks.new() + tr.name = act.name + tr.strips.new(act.name, 1, act) + bpy.ops.object.select_all(action="DESELECT") + tgt_arm.select_set(True) + bpy.context.view_layer.objects.active = tgt_arm + bpy.ops.export_scene.gltf( + filepath=out, use_selection=True, + export_animations=True, export_animation_mode="NLA_TRACKS", + export_force_sampling=True, export_optimize_animation_size=False) + print(f"[mocap] EXPORTED '{clip}' ({len(ts)} keyed frames, {ts[-1]:.1f}s) → {out}") + + +main()