#!/usr/bin/env python3 """ warp_poc.py — Malin FC vtuber-rig PROOF OF CONCEPT. Parameter-driven mesh warp of her ACTUAL anchor portrait (her pixels, identity locked — nothing generated). Demonstrates the rig principle: (1) parametric expression transitions (not frame-by-frame baked art) (2) independent composable params — mouth/lip-sync runs WHILE an expression holds Usage: python3 warp_poc.py still # QC one frame -> _still.png python3 warp_poc.py smile_clip # smile in/out -> smile.mp4 python3 warp_poc.py talk_hold_clip # brow-hold + mouth talking -> talk_hold.mp4 expr in: smile | brow | open | smile_open """ import sys, os, math, subprocess import numpy as np import cv2 SRC = "/Users/feral/dev/feral-cc-bots/calypso/specs/malin-look/malin_anchor.jpg" OUTDIR = "/Users/feral/dev/feral-cc-bots/calypso/specs/malin-harness/fc_warp_poc" os.makedirs(OUTDIR, exist_ok=True) img = cv2.imread(SRC) H, W = img.shape[:2] ys, xs = np.mgrid[0:H, 0:W].astype(np.float32) # --- Control points (x, y) on the 832x1216 anchor (hand-placed, tunable) --- P = { "mouth_l": (428, 690), "mouth_r": (520, 688), "mouth_top": (477, 668), "mouth_bot": (478, 716), "cheek_l": (372, 560), "cheek_r": (580, 555), "lid_l": (372, 452), "lid_r": (580, 448), "lower_l": (375, 470), "lower_r": (578, 466), "brow_l_in": (410, 372), "brow_l_out": (322, 362), "brow_r_in": (542, 366), "brow_r_out": (632, 356), } # Expression = list of (point, dx, dy, radius) at full intensity. # +dy is DOWN in image coords, so "up" deltas are negative. SMILE = [ ("mouth_l", -20, -13, 50), ("mouth_r", 20, -13, 50), # corners pull up + OUT from each side ("cheek_l", 0, -6, 90), ("cheek_r", 0, -6, 90), ("lid_l", 0, -3, 55), ("lid_r", 0, -3, 55), ] BROW = [ ("brow_l_in", 0, -14, 85), ("brow_l_out", 0, -11, 85), ("brow_r_in", 0, -14, 85), ("brow_r_out", 0, -11, 85), ] OPEN = [ ("mouth_bot", 0, 15, 70), ("mouth_top", 0, -5, 60), ("mouth_l", -2, 2, 60), ("mouth_r", 2, 2, 60), ] EXPRS = {"smile": SMILE, "brow": BROW, "open": OPEN, "smile_open": SMILE + OPEN} def build_maps(active): """active = list of (point, dx, dy, radius) already scaled by intensity.""" fx = np.zeros((H, W), np.float32) fy = np.zeros((H, W), np.float32) for pname, dx, dy, rad in active: px, py = P[pname] w = np.exp(-((xs - px) ** 2 + (ys - py) ** 2) / (2.0 * rad * rad)) fx += dx * w fy += dy * w # backward map: output sampled from (x - f) so the feature appears shifted by +f return (xs - fx), (ys - fy) def render(active): mx, my = build_maps(active) return cv2.remap(img, mx, my, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT) def render_on(src, active): mx, my = build_maps(active) return cv2.remap(src, mx, my, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT) def scale(deltas, k): return [(p, dx * k, dy * k, r) for (p, dx, dy, r) in deltas] def ease(t): # ease-in-out cubic return 4 * t * t * t if t < 0.5 else 1 - pow(-2 * t + 2, 3) / 2.0 def write_clip(frames, name, fps=30): fdir = os.path.join(OUTDIR, "frames") os.makedirs(fdir, exist_ok=True) for f in os.listdir(fdir): os.remove(os.path.join(fdir, f)) for i, fr in enumerate(frames): cv2.imwrite(os.path.join(fdir, f"f{i:04d}.png"), fr) out = os.path.join(OUTDIR, name) subprocess.run(["ffmpeg", "-y", "-framerate", str(fps), "-i", os.path.join(fdir, "f%04d.png"), "-c:v", "libx264", "-pix_fmt", "yuv420p", "-crf", "18", out], check=True, capture_output=True) print("wrote", out, f"({len(frames)} frames)") def main(): mode = sys.argv[1] if len(sys.argv) > 1 else "still" if mode == "still": expr = sys.argv[2] if len(sys.argv) > 2 else "smile" k = float(sys.argv[3]) if len(sys.argv) > 3 else 1.0 out = os.path.join(OUTDIR, f"_still_{expr}_{k}.png") cv2.imwrite(out, render(scale(EXPRS[expr], k))) print("wrote", out) elif mode == "smile_clip": # neutral -> smile in -> hold -> smile out -> neutral frames = [] PK = 1.8 for _ in range(8): frames.append(img.copy()) # hold neutral first for i in range(20): # entry ~0.67s frames.append(render(scale(SMILE, PK * ease(i / 19)))) for _ in range(18): # hold ~0.6s frames.append(render(scale(SMILE, PK))) for i in range(16): # exit, faster frames.append(render(scale(SMILE, PK * (1.0 - ease(i / 15))))) for _ in range(8): frames.append(img.copy()) write_clip(frames, "smile.mp4") elif mode == "talk_hold_clip": # brow-raise holds the whole time; mouth talks independently on top frames = [] BPK = 1.5 for _ in range(6): frames.append(img.copy()) for i in range(12): # brow eases up frames.append(render(scale(BROW, BPK * ease(i / 11)))) for c in range(5): # 5 mouth "syllables" while brow stays up for i in range(8): m = math.sin(math.pi * i / 7) ** 2 # open-close frames.append(render(scale(BROW, BPK) + scale(OPEN, 1.6 * m))) for _ in range(10): # hold brow, mouth closed frames.append(render(scale(BROW, BPK))) for i in range(12): # brow eases down frames.append(render(scale(BROW, BPK * (1.0 - ease(i / 11))))) write_clip(frames, "talk_hold.mp4") elif mode == "hybrid_smile_clip": # STAGED HYBRID (Jun's frame-by-frame + muscle-onset note): # Phase 0: zygomaticus pulls the CORNERS out then up on the CLOSED mouth # (lip center still, no teeth). Phases 1-3: lips open progressively # through inpaint keys A->B->C (teeth reveal in stages). Eye-squint # (orbicularis oculi) builds LAST = the Duchenne tell. keyA = cv2.imread(os.path.join(OUTDIR, "key_a.png")) keyB = cv2.imread(os.path.join(OUTDIR, "key_b.png")) keyC = cv2.imread(os.path.join(OUTDIR, "smile_key_aligned.png")) PK = 1.6 def clamp(x): return max(0.0, min(1.0, x)) CORNER_OUT = [("mouth_l", -22, 0, 30), ("mouth_r", 22, 0, 30)] CORNER_UP = [("mouth_l", 0, -16, 16), ("mouth_r", 0, -16, 16)] SQUINT = [("lower_l", 0, -7, 36), ("lower_r", 0, -7, 36)] base = scale(CORNER_OUT, PK) + scale(CORNER_UP, PK) frames = [] for _ in range(8): frames.append(img.copy()) # PHASE 0 - corners reshape on the CLOSED mouth, center still, no teeth for i in range(22): p = i / 21.0 t_out = ease(clamp(p / 0.45)) t_up = ease(clamp((p - 0.25) / 0.6)) sq = 0.35 * ease(clamp((p - 0.3) / 0.7)) frames.append(render(scale(CORNER_OUT, PK * t_out) + scale(CORNER_UP, PK * t_up) + scale(SQUINT, sq))) # PHASES 1-3 - open the mouth in stages through the inpaint keys; squint builds stages = [(img, keyA, 0.35, 0.60, True), (keyA, keyB, 0.60, 0.80, False), (keyB, keyC, 0.80, 1.00, False)] for src, dst, sq0, sq1, warp_src in stages: for i in range(15): p = i / 14.0 sq = sq0 + (sq1 - sq0) * ease(p) a = render_on(src, (base + scale(SQUINT, sq)) if warp_src else scale(SQUINT, sq)) b = render_on(dst, scale(SQUINT, sq)) frames.append(cv2.addWeighted(a, 1 - ease(p), b, ease(p), 0)) # hold the full smile + squint peak = render_on(keyC, scale(SQUINT, 1.0)) for _ in range(18): frames.append(peak.copy()) # EXIT - lips close back through the keys (C->B->A), then release the warp for src, dst in [(keyC, keyB), (keyB, keyA), (keyA, img)]: for i in range(13): p = i / 12.0 a = render_on(src, scale(SQUINT, 1.0)) b = render_on(dst, (base + scale(SQUINT, 1.0)) if dst is img else scale(SQUINT, 1.0)) frames.append(cv2.addWeighted(a, 1 - ease(p), b, ease(p), 0)) for i in range(16): p = i / 15.0 k = 1.0 - ease(p) frames.append(render(scale(CORNER_OUT, PK * k) + scale(CORNER_UP, PK * k) + scale(SQUINT, k))) for _ in range(8): frames.append(img.copy()) write_clip(frames, "hybrid_smile.mp4") else: print("unknown mode", mode) if __name__ == "__main__": main()