pireel 浏览器内 Agent 视频工作台

85/** Tools answered directly on the server (body only on server / pure catalog / direct cloud-state ops): no bridge. */
86export const MCP_SERVER_TOOL_IDS = new Set(['read_editing_guide', 'read_frame', 'list_frames', 'get_icons', 'import_media', 'create_browser_handoff', 'create_project', 'list_projects', 'switch_project', 'rename_project']);
87
88/** MCP-only bridge tools (not in STUDIO_TOOLS, invisible to internal chat):
89 *  get_state=state snapshot; apply_block/submit_plan=the validate-and-place surface for BYO generation output;
90 *  capture_frame=visual verification (returns a captured frame as image content so the agent can "see" its own edits).
91 *  compose_block_brief/plan_brief are "bridge-fetch context + server-assemble" composite tools, dispatched separately. */
92export const MCP_BRIDGE_EXTRA_TOOL_IDS = new Set(['get_state', 'apply_block', 'submit_plan', 'capture_frame', 'visual_brief', 'submit_visual']);
93
94/** Brief composite tools → bridge context-operation names (implemented browser-side in runExternalTool). */
95export const MCP_BRIEF_TOOLS: Record<string, string> = {
96  compose_block_brief: 'compose_context',
97  plan_brief: 'plan_context',
98};
99
100/** Bridge timeout for slow tools (generation/analysis in the browser, minutes-scale); instant ops get 60s. */
101const CARD_TIMEOUT_MS = 600_000;
102const BADGE_TIMEOUT_MS = 60_000;
103
104export function bridgeTimeoutMs(toolId: string): number {
105  if (toolId === 'visual_brief') return CARD_TIMEOUT_MS; // runs the free geometry pass inside (minutes-scale); don't cap it at extra's 60s
106  return STUDIO_TOOL_MAP[toolId]?.kind === 'card' ? CARD_TIMEOUT_MS : BADGE_TIMEOUT_MS;
107}

154    /* ---------- BYO-brain generation surface: brief → you generate → apply validates and places (no Pireel credits burned) ---------- */
155    {
156      name: 'compose_block_brief',
157      description:
158        "Get the FULL generation contract {system, prompt} for one overlay block, assembled from the live composition (theme tokens, frame design language, box size, on-screen beats, neighbor roster). YOU then generate the response with your own model, following the contract exactly (one short note, then ```html fence, then ```js timeline fence), and submit the raw text via apply_block. Targets: `blockId` of a pending placeholder (its design spec becomes the instruction — omit `instruction`); `blockId` of an existing block + `instruction` = rewrite; no blockId + `instruction` = new element at `atSec` (defaults to playhead). This is the default way to create/edit block content — it does NOT charge Pireel credits.",
159      inputSchema: {
160        type: 'object',
161        additionalProperties: false,
162        properties: {
163          blockId: { type: 'string', description: 'Target block id (placeholder to fill, or existing block to rewrite). Omit for a new element.' },
164          atSec: { type: 'number', description: 'New element only: timeline start seconds (defaults to playhead).' },
165          instruction: { type: 'string', description: 'What to build/change. Required unless targeting a placeholder.' },
166        },
167      },
168    },
169    {
170      name: 'apply_block',
171      description:
172        "Validate and place a block you generated from compose_block_brief. Pass the SAME blockId/atSec you gave the brief, and `raw` = your full generated text (note + ```html + ```js fences; fences are parsed out). The block is linted (scoped CSS, no scripts, deterministic animation); on lint failure you get the issues back — fix ONLY those and re-apply. Placeholder blockId → fills it; existing blockId → overwrites its content; neither → inserts a new element (optional durationSec, default 3s; optional label).",
173      inputSchema: {
174        type: 'object',
175        additionalProperties: false,
176        properties: {
177          blockId: { type: 'string' },
178          atSec: { type: 'number' },
179          durationSec: { type: 'number', description: 'New element only: seconds on screen (default 3).' },
180          label: { type: 'string', description: 'New element only: short timeline label.' },
181          raw: { type: 'string', description: 'Your full generated text: note, then ```html fence, then ```js fence.' },
182        },
183        required: ['raw'],
184      },

31  useEffect(() => {
32    let ws: WebSocket | null = null;
33    let alive = true;
34    let retries = 0;
35    let everConnected = false;
36    let queue: Promise<void> = Promise.resolve();
37
38    const connect = () => {
39      if (!alive) return;
40      const url = new URL('/api/studio/bridge', location.href);
41      url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
42      let sock: WebSocket;
43      try {
44        sock = new WebSocket(url);
45      } catch {
46        return; // environment unsupported, give up
47      }
48      ws = sock;
49      sock.onopen = () => {
50        everConnected = true;
51        retries = 0;
52      };
53      sock.onmessage = (ev) => {
54        if (typeof ev.data !== 'string' || ev.data === 'pong') return;
55        let m: { id?: string; tool?: string; input?: Record<string, unknown> };
56        try {
57          m = JSON.parse(ev.data) as typeof m;
58        } catch {
59          return;
60        }
61        if (!m.id || !m.tool) return;
62        const { id, tool, input } = m as { id: string; tool: string; input?: Record<string, unknown> };
63        queue = queue.then(async () => {
64          let out: StudioToolResult & { state?: string };
65          try {
66            out = tool === 'get_state' ? { ok: true, state: optsRef.current.getState() } : await optsRef.current.runTool(tool, input ?? {});
67          } catch (e) {
68            out = { ok: false, error: e instanceof Error ? e.message : String(e) };
69          }
70          try {
71            sock.send(JSON.stringify({ id, ...out }));
72          } catch {
73            /* socket already dead, DO side will time out / fail */
74          }
75          optsRef.current.onExternalCall?.(tool, out);
76        });
77      };
78      sock.onclose = (ev) => {
79        if (ws === sock) ws = null;
80        if (!alive) return;
81        if (ev.code === 4000) return; // kicked by a new tab: this page exits the bridge, don't fight for it
82        if (!everConnected && retries >= 6) return; // never connected (not logged in, etc.): don't spin
83        setTimeout(connect, Math.min(30_000, 1_000 * 2 ** retries++));
84      };

356/** Tool result → MCP content (text JSON; isError tells the agent to correct course rather than parrot it).
357 *  Captured frames become image content (the agent "sees" directly); snapshots are given as raw body (not wrapped in JSON). */
358function toolResponse(id: JsonRpcRequest['id'], r: McpBridgeResult): JsonRpcResponse {
359  if (r.ok && Array.isArray(r.images) && r.images.length) {
360    // multiple images (visual_brief sample frames): text (index/timestamp/labeling contract) first, frames follow in index order
361    const imgs = (r.images as { data: string; mimeType?: string }[]).filter((i) => typeof i?.data === 'string');
362    const { images: _drop, ...rest } = r;
363    return rpcResult(id, {
364      content: [
365        { type: 'text', text: JSON.stringify(rest) },
366        ...imgs.map((i) => ({ type: 'image' as const, data: i.data, mimeType: i.mimeType ?? 'image/jpeg' })),
367      ],
368      isError: false,
369    });
370  }
371  if (r.ok && r.image && typeof (r.image as { data?: unknown }).data === 'string') {
372    const img = r.image as { data: string; mimeType?: string };
373    return rpcResult(id, {
374      content: [
375        { type: 'image', data: img.data, mimeType: img.mimeType ?? 'image/jpeg' },
376        { type: 'text', text: r.summary ?? 'frame captured' },
377      ],
378      isError: false,
379    });
380  }
381  const text = r.ok && typeof r.state === 'string' ? r.state : JSON.stringify(r);
382  return rpcResult(id, { content: [{ type: 'text', text }], isError: !r.ok });
383}

23/** Server-operable editing context (fuel for the offline MCP executor): client autosave mirrors
24 *  it up alongside comp. plan stored loosely (shape is normalized by parsePlan on the use side);
25 *  video bytes still never persisted. */
26export interface StudioProjectContext {
27  asr?: TranscriptSegment[];
28  clipAsr?: Record<string, TranscriptSegment[]>;
29  plan?: unknown;
30  /** Index into the cloud byte rendezvous (R2): main video / inserted sources sig→key (the bytes
31   *  live in R2, content-addressed). Cross-device retrieval and future offline ASR / cloud render look up here. */
32  media?: {
33    video?: { sig: string; key: string };
34    clips?: Record<string, { key: string }>;
35  };
36}
37
38/** Full project payload between client and server. */
39export interface StudioProjectDto {
40  id: string;
41  title: string;
42  comp: Composition;
43  chat: unknown[];
44  context: StudioProjectContext;
45  videoSig: string | null;
46  videoDurationSec: number | null;
47  coverThumb: string | null;
48  version: number;
49  updatedAt: number; // epoch ms
50}
51
52/** Lightweight meta for lists (without the big comp/chat fields). */
53export interface StudioProjectMeta {
54  id: string;
55  title: string;
56  videoDurationSec: number | null;
57  blocks: number;
58  shots: number;
59  coverThumb: string | null;
60  version: number;
61  updatedAt: number;
62}
63
64/** Save payload from client to server (ProjectStore.save's arg; shared by cloud sync and the provider contract). */
65export interface ProjectSavePayload {
66  title?: string;
67  comp: Composition;
68  chat: unknown[];
69  /** Editing context (asr/clipAsr/plan/media): needed by the offline executor and cross-device retrieval. */
70  context?: StudioProjectContext;
71  videoSig: string | null;
72  videoDurationSec: number | null;
73  coverThumb: string | null;
74}
75
76/** Save payload cap (comp's LLM-generated HTML + chat history can be sizable, but keep it bounded). */
77export const MAX_PROJECT_BYTES = 8 * 1024 * 1024;

267export function buildSaveWire(
268  p: ProjectSavePayload,
269  baseVersion: number | null,
270  acked: AckedSections | null,
271): { wire: ProjectSaveWire; acked: AckedSections } | null {
272  const compCanon = canonicalJson({ ...p.comp, video: null });
273  const chatCanon = canonicalJson(p.chat ?? []);
274  const ctxCanon = canonicalJson(p.context ?? {});
275  const hashes: SectionHashes = {
276    comp: hashSection(compCanon),
277    chat: hashSection(chatCanon),
278    context: hashSection(ctxCanon),
279    coverThumb: hashSection(p.coverThumb ?? ''),
280    meta: metaHashOf(p.title, p.videoSig, p.videoDurationSec),
281  };
282  // JSON-clean current values (parsed from the canonical string: incidentally drops undefined, so diff is structurally comparable to the baseline)
283  const values: AckedSections['values'] = {
284    comp: JSON.parse(compCanon) as Composition,
285    chat: JSON.parse(chatCanon) as unknown[],
286    context: JSON.parse(ctxCanon) as StudioProjectContext,
287  };
288  const wire: ProjectSaveWire = { baseVersion };
289  const w = wire as unknown as Record<string, unknown>;
290  let changed = false;
291
292  /** Big-section tri-state: unchanged → absent / has baseline and patch is smaller → send patch / else whole section. */
293  const emitBig = (key: 'comp' | 'chat' | 'context', canon: string, withHash: boolean) => {
294    if (acked && acked.hashes[key] === hashes[key]) return;
295    changed = true;
296    if (acked) {
297      try {
298        const ops = diffOps(acked.values[key], values[key]);
299        if (JSON.stringify(ops).length < canon.length * PATCH_WORTH_RATIO) {
300          w[`${key}Patch`] = ops;
301          if (withHash) w[`${key}Hash`] = hashes[key];
302          return;
303        }
304      } catch {
305        /* diff failed → fall back to whole section */
306      }
307    }
308    w[key] = values[key];
309  };
310  emitBig('comp', compCanon, true);
311  emitBig('chat', chatCanon, true);
312  emitBig('context', ctxCanon, false);
313
314  if (!acked || acked.hashes.coverThumb !== hashes.coverThumb) {
315    wire.coverThumb = p.coverThumb;
316    changed = true;
317  }
318  if (!acked || acked.hashes.meta !== hashes.meta) {
319    if (p.title !== undefined) wire.title = p.title;
320    wire.videoSig = p.videoSig;
321    wire.videoDurationSec = p.videoDurationSec;
322    changed = true;
323  }
324  return changed ? { wire, acked: { values, hashes } } : null;

31/** Probe video metadata locally (MediaBunny dynamically loaded, no upload). */
32export async function probeVideoFile(file: File): Promise<ProbedFile> {
33  const { ALL_FORMATS, BlobSource, Input } = await import('mediabunny');
34  const input = new Input({ source: new BlobSource(file), formats: ALL_FORMATS });
35  try {
36    const v = await input.getPrimaryVideoTrack();
37    const a = await input.getPrimaryAudioTrack().catch(() => null);
38    // computeDuration measures the "max end timestamp": an mp4 with a non-zero first packet inflates the
39    // duration by that offset. Subtract getFirstTimestamp to convert to <video> playback frame (currentTime 0 = earliest sample).
40    const duration = (await input.computeDuration()) - Math.max(0, await input.getFirstTimestamp());
41    return {
42      durationSec: Number.isFinite(duration) && duration > 0 ? duration : 0,
43      width: v?.displayWidth ?? 0,
44      height: v?.displayHeight ?? 0,
45      hasAudio: !!a,
46    };
47  } finally {
48    await input.dispose();
49  }
50}
51
52/** Extract audio (upload audio only) → ASR → sentence-level shots. Cached by fileSig (same clip transcribed once). */
53export async function transcribeFile(file: File): Promise<AsrSegment[]> {
54  const sig = fileSig(file);
55  const cached = getCachedAsr(sig);
56  if (cached) return cached;
57
58  const audio = await extractAudio(file);
59  const { url } = await studioProviders().uploads.upload(audio, { contentType: audio.type || 'audio/mp4', filename: 'studio-audio.m4a' });
60  const r = await fetch('/api/auto-edit/asr', {
61    method: 'POST',
62    headers: { 'content-type': 'application/json' },
63    body: JSON.stringify({ audio_url: url }),
64  });
65  // Server failures must throw a clear error, not silently become an empty array — callers need to tell "ASR failed" apart from "the video truly has no speech"
66  if (!r.ok) throw new Error(t('ASR 请求失败(HTTP {status})', { status: r.status }));
67  const j = (await r.json()) as { segments?: Array<{ start: number; end: number; text: string }> };
68  // ASR times are in the "audio track's own zero" frame (audio extraction subtracts the audio track's first
69  // packet), while playback is in "earliest sample across all tracks". Files whose two tracks' first packets
70  // are out of sync need this delta added back, or captions/cut points shift wholesale.
71  const off = await audioPlaybackOffset(file);
72  const segs: AsrSegment[] = (Array.isArray(j.segments) ? j.segments : [])
73    .filter((s) => s.text?.trim())
74    .map((s) => ({ start: Math.max(0, s.start + off), end: Math.max(s.start + off + 0.1, s.end + off), text: s.text.trim() }));
75  if (segs.length) setCachedAsr(sig, segs);
76  return segs;
77}

58/** Source-time-anchored filmstrip window (like CapCut): tile k always covers source time
59 *  [k,k+1)*tileDur; take tiles that intersect the window [srcStart,srcEnd). First tile's left
60 *  can be negative (clipped by the card's overflow-hidden) — a split at 2.5 tiles gives 2.5
61 *  tiles before and the rest continuing from the middle of tile 2.5, so trailing tiles never resample. */
62function stripTiles(strip: FilmstripFrame[], srcStart: number, srcEnd: number, tileDur: number, pps: number): { left: number; url: string }[] {
63  if (!strip.length || tileDur <= 0 || srcEnd <= srcStart) return [];
64  const tiles: { left: number; url: string }[] = [];
65  for (let k = Math.floor(srcStart / tileDur); k * tileDur < srcEnd; k++) {
66    const srcT = (k + 0.5) * tileDur;
67    let url = strip[0]!.url;
68    let bd = Infinity;
69    for (const f of strip) {
70      const d = Math.abs(f.t - srcT);
71      if (d < bd) {
72        bd = d;
73        url = f.url;
74      }
75    }
76    tiles.push({ left: (k * tileDur - srcStart) * pps, url });
77  }
78  return tiles;
79}

60  /** Full comp JSON + source fingerprint + options: if anything affecting the result changes, the key changes.
61   *  (Insert clips appear as blob URLs in comp.shots, stable within a session, so they're naturally in the key.) */
62  const exportKey = (c: Composition, opts: ExportRenderOpts): string =>
63    `${videoFileRef.current ? fileSig(videoFileRef.current) : (c.video?.url ?? '')}|${JSON.stringify(opts)}|${JSON.stringify(c)}`;
64
65  /** Whether client compositing is possible locally (WebCodecs + a source File). If not = honest error, no longer handed to the deprecated server render. */
66  const canClientExport = () => typeof window !== 'undefined' && 'VideoEncoder' in window && !!videoFileRef.current;
67  const noExportReason = () =>
68    !videoFileRef.current ? t('本地视频丢失，重新上传原片后再导出') : t('这个浏览器不支持本地导出，换 Chrome/Edge 打开再试');
69
70  /** Get the finished-video blob for the current content: use the cache on hit, otherwise composite once client-side (reports progress, cancelable). */
71  const renderBlob = async (c: Composition, key: string, opts: ExportRenderOpts): Promise<Blob> => {
72    const cached = lastExportRef.current;
73    if (cached && cached.key === key && cached.blob) return cached.blob;
74    const blob = await clientExportVideo({
75      comp: c,
76      videoFile: videoFileRef.current!,
77      clipFiles: clipFilesRef?.current ?? new Map(),
78      render: opts,
79      onProgress: (done, total) => setExportPct(Math.round((done / total) * 100)),
80      shouldCancel: () => exportCancelRef.current,
81    });
82    lastExportRef.current = { key, blob, opts };
83    return blob;
84  };
85
86  /** Export = client compositing + auto download. Unsupported → honest error (no longer silently routed to server render that produces a black clip).
87   *  Returns a result for the agent export tools (export_video/track_export): on success includes the saved filename. */
88  async function exportVideo(opts: ExportRenderOpts): Promise<{ ok: boolean; filename?: string; error?: string }> {
89    const c = compRef.current;
90    if (!c.video?.url) {
91      toast.error(t('先上传口播视频再导出'));
92      return { ok: false, error: t('先上传口播视频再导出') };
93    }
94    if (exporting || publishing) return { ok: false, error: t('已有导出在进行中') };
95    if (!canClientExport()) {
96      toast.error(noExportReason());
97      return { ok: false, error: noExportReason() };
98    }
99    const key = exportKey(c, opts);
100    const name = filenameFor(opts);
101    if (lastExportRef.current?.key === key && lastExportRef.current.blob) {
102      toast.success(t('内容没有改动，已下载上次的成片'));
103      downloadBlob(lastExportRef.current.blob, name);
104      return { ok: true, filename: name };
105    }
106    setExporting(true);
107    setExportPct(0);
108    exportCancelRef.current = false;
109    try {
110      const blob = await renderBlob(c, key, opts);
111      downloadBlob(blob, name);
112      toast.success(t('导出完成，已开始下载'));
113      return { ok: true, filename: name };
114    } catch (e) {
115      if (e instanceof ExportCanceled) {
116        toast.info(t('已取消导出'));
117        return { ok: false, error: t('导出被取消') };
118      }
119      console.warn('[studio] client export failed', e);
120      toast.error(t('导出失败，稍后重试'));
121      return { ok: false, error: e instanceof Error ? e.message : t('导出失败') };
122    } finally {
123      setExporting(false);
124    }

331export async function clientExportVideo(opts: ClientExportOpts): Promise<Blob> {
332  const { comp, videoFile, clipFiles } = opts;
333  const render = opts.render ?? DEFAULT_RENDER_OPTS;
334  const FPS = render.fps;
335  const W = comp.width;
336  const H = comp.height;
337  // Output size: res=short side, downscale only, never upscale; layout/framing all still computed
338  // in comp coordinates, one setTransform scales at the canvas end.
339  // Width and height each rounded to even (encoder requires it) → Sx/Sy computed independently, no black bars from sub-pixel diff
340  const even = (x: number) => Math.max(2, Math.round(x / 2) * 2);
341  // res=output short-side pixels. Upscaling to 4K relies on the vector layer re-rasterizing at target
342  // resolution (svgOpen's viewBox) + the source video drawImage scaling straight from native frames to
343  // output (no 1080 intermediate). Cap at 4× as a safety net against canvas/memory blowup (dropdown max 2160=2×).
344  const k = Math.min(4, render.res / Math.min(W, H));
345  const outW = even(W * k);
346  const outH = even(H * k);
347  const Sx = outW / W;
348  const Sy = outH / H;
349  const shots = comp.shots?.length ? comp.shots : [{ id: 'all', srcStart: 0, srcEnd: comp.video?.durationSec ?? 0, treatment: 'full' as const }];
350  const durationSec = Math.max(0.5, totalDuration(comp));
351
352  // Segment table (edited order) + each source's File
353  const segs: ExpSeg[] = [];
354  const files = new Map<string, File>([['main', videoFile]]);
355  for (const sp of clipSpans(shots)) {
356    const s = sp.clip as (typeof shots)[number] & { src?: string; filter?: ShotFilter };
357    const filterCss = shotFilterCss(s.filter);
358    const filter = filterCss === 'none' ? {} : { filter: filterCss };
359    if (!s.src) {
360      segs.push({ srcStart: s.srcStart, srcEnd: s.srcEnd, key: 'main', ...filter });
361      continue;
362    }
363    const key = `clip_${s.id}`;
364    segs.push({ srcStart: s.srcStart, srcEnd: s.srcEnd, key, ...filter });
365    if (!files.has(key)) {
366      const local = clipFiles.get(s.src);
367      if (local) files.set(key, local);
368      else {
369        const r = await fetch(`/api/media/fetch?url=${encodeURIComponent(s.src)}`);
370        if (!r.ok) throw new Error(t('插入片段拉取失败'));
371        files.set(key, new File([await r.blob()], 'clip.mp4', { type: 'video/mp4' }));
372      }
373    }
374  }
375
376  // Per-source sequential sample stream: start = earliest time this source is used, end = latest
377  const rigs = new Map<string, SourceRig>();
378  for (const [key, file] of files) {
379    const mine = segs.filter((s) => s.key === key);
380    if (!mine.length) continue;
381    const from = Math.min(...mine.map((s) => s.srcStart));
382    const to = Math.max(...mine.map((s) => s.srcEnd));
383    rigs.set(key, await openSource(file, from, to, W, H));
384  }
385
386  // Overlay document + asset inlining
387  const overlay = await createOverlay(injectPreviewRuntime(assembleHtml(comp)), W, H);
388  try {
389    await inlineImages(overlay.root);
390    const fontCss = await buildInlineFontCss(overlay.root.textContent ?? '');
391    const css = `${fontCss}\n${overlay.headCss}\n#root{background:transparent !important;}`;
392    // Layout coordinate system is always comp's W×H (font-size calibration unchanged); device size =
393    // output outW×outH → vectors rasterize crisply at 4K
394    const preEnc = 'data:image/svg+xml;charset=utf-8,' + encodeURIComponent(svgOpen(W, H, outW, outH, css));
395    const postEnc = encodeURIComponent(SVG_CLOSE);
396    // Theme background bitmap (once): empty root goes through the same rasterization pipeline (also at output resolution)
397    const bg = await rasterize(
398      'data:image/svg+xml;charset=utf-8,' +
399        encodeURIComponent(svgOpen(W, H, outW, outH, overlay.headCss) + `<div xmlns="http://www.w3.org/1999/xhtml" id="root"></div>` + SVG_CLOSE),
400    );
401
402    const canvas = document.createElement('canvas');
403    canvas.width = outW;
404    canvas.height = outH;
405    const ctx = canvas.getContext('2d')!;
406
407    const outFormat = render.format === 'webm' ? new WebMOutputFormat() : render.format === 'mov' ? new MovOutputFormat() : new Mp4OutputFormat();
408    const output = new Output({ format: outFormat, target: new BufferTarget() });
409    const videoSource = new CanvasSource(canvas, { codec: render.format === 'webm' ? 'vp9' : 'avc', bitrate: QUALITY_HIGH });
410    output.addVideoTrack(videoSource, { frameRate: FPS });
411    const anyAudio = [...rigs.values()].some((r) => r.audio);
412    // webm container can't hold aac, so audio switches to opus for that format
413    const audioSource = anyAudio ? new AudioSampleSource({ codec: render.format === 'webm' ? 'opus' : 'aac', bitrate: QUALITY_MEDIUM }) : null;
414    if (audioSource) output.addAudioTrack(audioSource);
415    await output.start();

497    // Overlay serialization only takes blocks "visible at this moment": seekTimelines already marks
498    // off-window blocks visibility:hidden, and skipping them leaves the hot-frame SVG doc with just a
499    // few blocks — parse/layout cost no longer scales with the project's total block count (serializing
500    // the whole root, an 11-minute project re-lays-out all several-hundred blocks on every hot frame).
501    const rootShell = XS.serializeToString(overlay.root.cloneNode(false));
502    const rootOpen = rootShell.endsWith('/>') ? `${rootShell.slice(0, -2)}>` : rootShell.slice(0, rootShell.lastIndexOf('</'));
503    const serializeVisible = (): string => {
504      let s = rootOpen;
505      for (const el of overlay.root.children) {
506        if ((el as HTMLElement).style?.visibility === 'hidden') continue;
507        s += XS.serializeToString(el);
508      }
509      return `${s}</div>`;
510    };
511
512    const total = Math.max(1, Math.round(durationSec * FPS));
513    // Overlay frame cache: during tween gaps / static caption periods the serialized string is identical
514    // frame to frame → reuse the previous frame's bitmap, skipping parse and rasterization of the whole
515    // data URI (with inlined fonts, MB-scale — the bulk of export time).
516    let lastSerial = '';
517    let lastOverlayImg: HTMLImageElement | null = null;
518    // Overlap encoding with next-frame prep: CanvasSource.add grabs the frame synchronously on call, and
519    // the returned Promise is only encoder backpressure — the canvas is free to change once grabbed, and
520    // we only await the backpressure right before the next frame's add.
521    let pendingAdd: Promise<void> | null = null;
522    // Per-phase timing (parallel phases each track their own wall clock, so the sum can exceed total wall time): data shows where the bottleneck is
523    const tm = { prep: 0, raster: 0, video: 0, draw: 0, enc: 0, rasterN: 0 };
524    const expT0 = performance.now();
525    for (let i = 0; i < total; i++) {
526      if (opts.shouldCancel?.()) throw new ExportCanceled();
527      const t = i / FPS;
528      const tPrep = performance.now();
529      overlay.win.__hfPreview!.seekTimelines(t);
530      const { seg, srcT } = segAt(t);
531      const rig = rigs.get(seg.key);
532      // Read the framing transform uniformly from #vidEl: in canvas mode every segment's (including
533      // insert clips') framing keyframes land on this one canvas (videoFrameTimelineBody only does
534      // tl.to('#vidEl')); looking up an element by clip_<id> is a leftover from the old "insert clip =
535      // separate <video>" era — the element doesn't exist → identity matrix → export loses insert-clip framing
536      const el = overlay.doc.getElementById('vidEl');
537      const vs = readTransform(overlay.win, el);
538      const serial = serializeVisible();
539      tm.prep += performance.now() - tPrep;
540      const hot = serial !== lastSerial || !lastOverlayImg;
541      if (hot) tm.rasterN++;
542      const overlayP: Promise<HTMLImageElement> = hot
543        ? (() => {
544            const s0 = performance.now();
545            return rasterize(preEnc + encodeURIComponent(serial) + postEnc).then((img) => ((tm.raster += performance.now() - s0), img));
546          })()
547        : Promise.resolve(lastOverlayImg!);
548
549      const tr = trsX.find((x) => t >= x.cut - x.half && t <= x.cut + x.half) ?? null;
550      // Compute shadow sampling params first; the three decodes (overlay/current frame/shadow frame) are independent → parallel
551      const pre = tr ? t < tr.cut : false;
552      const gseg = tr ? (pre ? tr.segB : tr.segA) : null;
553      const gRig = tr ? rigs.get(pre ? tr.preKey : tr.postKey) : undefined;
554      const gSrcT = tr && gseg ? (pre ? Math.max(0, gseg.srcStart - (tr.cut - t)) : gseg.srcEnd + (t - tr.cut)) : 0;
555      const v0 = performance.now();
556      const [overlayImg, sample, gSample] = await Promise.all([
557        overlayP,
558        rig ? sampleAt(rig, srcT).then((s) => ((tm.video += performance.now() - v0), s)) : null,
559        gRig ? sampleAt(gRig, gSrcT) : null,
560      ]);
561      lastSerial = serial;
562      lastOverlayImg = overlayImg;
563      // bg / overlay are already rasterized at output resolution (outW×outH) → draw 1:1 with identity
564      // transform (crisp text/graphics at 4K). The source video is still drawn in comp coordinates;
565      // setTransform(Sx) maps the target rect to output size, drawImage scales straight from native frames.

126describe('BYO-brain 契约(brief → 外部生成 → apply,客户端无关)', () => {
127  it('compose_block_brief:桥取上下文(compose_context)→ 服务端组装,agent 的 instruction 透传', async () => {
128    const d = deps({ callBridge: vi.fn(async () => ({ ok: true, data: { block: { id: 'b1' }, theme: 'general' } })) });
129    const r = await handleMcpRequest({ id: 11, method: 'tools/call', params: { name: 'compose_block_brief', arguments: { blockId: 'b1', instruction: '做张对比卡' } } }, d);
130    expect(d.callBridge).toHaveBeenCalledWith('compose_context', { blockId: 'b1', instruction: '做张对比卡' }, 60_000);
131    expect(d.assembleComposeBrief).toHaveBeenCalledWith({ block: { id: 'b1' }, theme: 'general' }, '做张对比卡');
132    expect((r!.result as { isError: boolean }).isError).toBe(false);
133  });
134  it('compose_block_brief:占位自带设计规格,无 instruction 也能出简报;两者皆无则打回', async () => {
135    const d = deps({ callBridge: vi.fn(async () => ({ ok: true, data: { block: { id: 'p1' }, suggested_instruction: 'component: kpi …' } })) });
136    await handleMcpRequest({ id: 12, method: 'tools/call', params: { name: 'compose_block_brief', arguments: { blockId: 'p1' } } }, d);
137    expect(d.assembleComposeBrief).toHaveBeenCalledWith(expect.anything(), 'component: kpi …');
138    const d2 = deps({ callBridge: vi.fn(async () => ({ ok: true, data: { block: { id: 'b2' } } })) });
139    const r2 = await handleMcpRequest({ id: 13, method: 'tools/call', params: { name: 'compose_block_brief', arguments: {} } }, d2);
140    expect((r2!.result as { isError: boolean }).isError).toBe(true);
141    expect(d2.assembleComposeBrief).not.toHaveBeenCalled();
142  });
143  it('plan_brief:桥取 plan_context → 服务端组装;apply_block/submit_plan 过桥', async () => {
144    const d = deps({ callBridge: vi.fn(async () => ({ ok: true, data: { sentences: [] } })) });
145    await handleMcpRequest({ id: 14, method: 'tools/call', params: { name: 'plan_brief' } }, d);
146    expect(d.callBridge).toHaveBeenCalledWith('plan_context', {}, 60_000);
147    expect(d.assemblePlanBrief).toHaveBeenCalled();
148    await handleMcpRequest({ id: 15, method: 'tools/call', params: { name: 'apply_block', arguments: { blockId: 'b1', raw: 'x' } } }, d);
149    expect(d.callBridge).toHaveBeenCalledWith('apply_block', { blockId: 'b1', raw: 'x' }, 60_000);
150    await handleMcpRequest({ id: 16, method: 'tools/call', params: { name: 'submit_plan', arguments: { plan: '{}' } } }, d);
151    expect(d.callBridge).toHaveBeenCalledWith('submit_plan', { plan: '{}' }, 60_000);
152  });
153  it('get_icons 服务端直答(BLOCK_SYSTEM 引用的同名工具在 MCP 面可用);桥上下文失败原样透传', async () => {
154    const d = deps();
155    await handleMcpRequest({ id: 17, method: 'tools/call', params: { name: 'get_icons', arguments: { names: ['trending-up'] } } }, d);
156    expect(d.lookupIcons).toHaveBeenCalledWith(['trending-up'], undefined);
157    expect(d.callBridge).not.toHaveBeenCalled();
158    const d2 = deps({ callBridge: vi.fn(async () => ({ ok: false, error: 'studio_not_open' })) });
159    const r = await handleMcpRequest({ id: 18, method: 'tools/call', params: { name: 'plan_brief' } }, d2);
160    expect((r!.result as { isError: boolean }).isError).toBe(true);
161    expect(d2.assemblePlanBrief).not.toHaveBeenCalled();
162  });
163  it('capture_frame:截帧回 image content(agent 的眼睛)', async () => {
164    const d = deps({ callBridge: vi.fn(async () => ({ ok: true, summary: '已截取 3s 处画面', image: { data: 'AAAA', mimeType: 'image/jpeg' } })) });
165    const r = await handleMcpRequest({ id: 20, method: 'tools/call', params: { name: 'capture_frame', arguments: { atSec: 3 } } }, d);
166    const result = r!.result as { content: { type: string; data?: string }[]; isError: boolean };
167    expect(result.isError).toBe(false);
168    expect(result.content[0]!.type).toBe('image');
169    expect(result.content[0]!.data).toBe('AAAA');
170    expect(new Set(buildMcpTools().map((t) => t.name)).has('capture_frame')).toBe(true);
171  });
172  it('MCP instructions 教 BYO 主路径', async () => {
173    const r = await handleMcpRequest({ id: 19, method: 'initialize' }, deps());
174    const ins = (r!.result as { instructions: string }).instructions;
175    expect(ins).toContain('YOU ARE THE MODEL');
176    expect(ins).toContain('compose_block_brief');
177    expect(ins).toContain('submit_plan');
178  });