媒体、时间线与客户端导出 深度解析

31/** Probe video metadata locally (MediaBunny dynamically loaded, no upload). */
32export async function probeVideoFile(file: File): Promise<ProbedFile> {
33  const { ALL_FORMATS, BlobSource, Input } = await import('mediabunny');
34  const input = new Input({ source: new BlobSource(file), formats: ALL_FORMATS });
35  try {
36    const v = await input.getPrimaryVideoTrack();
37    const a = await input.getPrimaryAudioTrack().catch(() => null);
38    // computeDuration measures the "max end timestamp": an mp4 with a non-zero first packet inflates the
39    // duration by that offset. Subtract getFirstTimestamp to convert to <video> playback frame (currentTime 0 = earliest sample).
40    const duration = (await input.computeDuration()) - Math.max(0, await input.getFirstTimestamp());
41    return {
42      durationSec: Number.isFinite(duration) && duration > 0 ? duration : 0,
43      width: v?.displayWidth ?? 0,
44      height: v?.displayHeight ?? 0,
45      hasAudio: !!a,
46    };
47  } finally {
48    await input.dispose();
49  }
50}
51
52/** Extract audio (upload audio only) → ASR → sentence-level shots. Cached by fileSig (same clip transcribed once). */
53export async function transcribeFile(file: File): Promise<AsrSegment[]> {
54  const sig = fileSig(file);
55  const cached = getCachedAsr(sig);
56  if (cached) return cached;
57
58  const audio = await extractAudio(file);
59  const { url } = await studioProviders().uploads.upload(audio, { contentType: audio.type || 'audio/mp4', filename: 'studio-audio.m4a' });
60  const r = await fetch('/api/auto-edit/asr', {
61    method: 'POST',
62    headers: { 'content-type': 'application/json' },
63    body: JSON.stringify({ audio_url: url }),
64  });
65  // Server failures must throw a clear error, not silently become an empty array — callers need to tell "ASR failed" apart from "the video truly has no speech"
66  if (!r.ok) throw new Error(t('ASR 请求失败(HTTP {status})', { status: r.status }));
67  const j = (await r.json()) as { segments?: Array<{ start: number; end: number; text: string }> };
68  // ASR times are in the "audio track's own zero" frame (audio extraction subtracts the audio track's first
69  // packet), while playback is in "earliest sample across all tracks". Files whose two tracks' first packets
70  // are out of sync need this delta added back, or captions/cut points shift wholesale.
71  const off = await audioPlaybackOffset(file);
72  const segs: AsrSegment[] = (Array.isArray(j.segments) ? j.segments : [])
73    .filter((s) => s.text?.trim())
74    .map((s) => ({ start: Math.max(0, s.start + off), end: Math.max(s.start + off + 0.1, s.end + off), text: s.text.trim() }));
75  if (segs.length) setCachedAsr(sig, segs);
76  return segs;
77}

331export async function clientExportVideo(opts: ClientExportOpts): Promise<Blob> {
332  const { comp, videoFile, clipFiles } = opts;
333  const render = opts.render ?? DEFAULT_RENDER_OPTS;
334  const FPS = render.fps;
335  const W = comp.width;
336  const H = comp.height;
337  // Output size: res=short side, downscale only, never upscale; layout/framing all still computed
338  // in comp coordinates, one setTransform scales at the canvas end.
339  // Width and height each rounded to even (encoder requires it) → Sx/Sy computed independently, no black bars from sub-pixel diff
340  const even = (x: number) => Math.max(2, Math.round(x / 2) * 2);
341  // res=output short-side pixels. Upscaling to 4K relies on the vector layer re-rasterizing at target
342  // resolution (svgOpen's viewBox) + the source video drawImage scaling straight from native frames to
343  // output (no 1080 intermediate). Cap at 4× as a safety net against canvas/memory blowup (dropdown max 2160=2×).
344  const k = Math.min(4, render.res / Math.min(W, H));
345  const outW = even(W * k);
346  const outH = even(H * k);
347  const Sx = outW / W;
348  const Sy = outH / H;
349  const shots = comp.shots?.length ? comp.shots : [{ id: 'all', srcStart: 0, srcEnd: comp.video?.durationSec ?? 0, treatment: 'full' as const }];
350  const durationSec = Math.max(0.5, totalDuration(comp));
351
352  // Segment table (edited order) + each source's File
353  const segs: ExpSeg[] = [];
354  const files = new Map<string, File>([['main', videoFile]]);
355  for (const sp of clipSpans(shots)) {
356    const s = sp.clip as (typeof shots)[number] & { src?: string; filter?: ShotFilter };
357    const filterCss = shotFilterCss(s.filter);
358    const filter = filterCss === 'none' ? {} : { filter: filterCss };
359    if (!s.src) {
360      segs.push({ srcStart: s.srcStart, srcEnd: s.srcEnd, key: 'main', ...filter });
361      continue;
362    }
363    const key = `clip_${s.id}`;
364    segs.push({ srcStart: s.srcStart, srcEnd: s.srcEnd, key, ...filter });
365    if (!files.has(key)) {
366      const local = clipFiles.get(s.src);
367      if (local) files.set(key, local);
368      else {
369        const r = await fetch(`/api/media/fetch?url=${encodeURIComponent(s.src)}`);
370        if (!r.ok) throw new Error(t('插入片段拉取失败'));
371        files.set(key, new File([await r.blob()], 'clip.mp4', { type: 'video/mp4' }));
372      }
373    }
374  }
375
376  // Per-source sequential sample stream: start = earliest time this source is used, end = latest
377  const rigs = new Map<string, SourceRig>();
378  for (const [key, file] of files) {
379    const mine = segs.filter((s) => s.key === key);
380    if (!mine.length) continue;
381    const from = Math.min(...mine.map((s) => s.srcStart));
382    const to = Math.max(...mine.map((s) => s.srcEnd));
383    rigs.set(key, await openSource(file, from, to, W, H));
384  }
385
386  // Overlay document + asset inlining
387  const overlay = await createOverlay(injectPreviewRuntime(assembleHtml(comp)), W, H);
388  try {
389    await inlineImages(overlay.root);
390    const fontCss = await buildInlineFontCss(overlay.root.textContent ?? '');
391    const css = `${fontCss}\n${overlay.headCss}\n#root{background:transparent !important;}`;
392    // Layout coordinate system is always comp's W×H (font-size calibration unchanged); device size =
393    // output outW×outH → vectors rasterize crisply at 4K
394    const preEnc = 'data:image/svg+xml;charset=utf-8,' + encodeURIComponent(svgOpen(W, H, outW, outH, css));
395    const postEnc = encodeURIComponent(SVG_CLOSE);
396    // Theme background bitmap (once): empty root goes through the same rasterization pipeline (also at output resolution)
397    const bg = await rasterize(
398      'data:image/svg+xml;charset=utf-8,' +
399        encodeURIComponent(svgOpen(W, H, outW, outH, overlay.headCss) + `<div xmlns="http://www.w3.org/1999/xhtml" id="root"></div>` + SVG_CLOSE),
400    );
401
402    const canvas = document.createElement('canvas');
403    canvas.width = outW;
404    canvas.height = outH;
405    const ctx = canvas.getContext('2d')!;
406
407    const outFormat = render.format === 'webm' ? new WebMOutputFormat() : render.format === 'mov' ? new MovOutputFormat() : new Mp4OutputFormat();
408    const output = new Output({ format: outFormat, target: new BufferTarget() });
409    const videoSource = new CanvasSource(canvas, { codec: render.format === 'webm' ? 'vp9' : 'avc', bitrate: QUALITY_HIGH });
410    output.addVideoTrack(videoSource, { frameRate: FPS });
411    const anyAudio = [...rigs.values()].some((r) => r.audio);
412    // webm container can't hold aac, so audio switches to opus for that format
413    const audioSource = anyAudio ? new AudioSampleSource({ codec: render.format === 'webm' ? 'opus' : 'aac', bitrate: QUALITY_MEDIUM }) : null;
414    if (audioSource) output.addAudioTrack(audioSource);
415    await output.start();

497    // Overlay serialization only takes blocks "visible at this moment": seekTimelines already marks
498    // off-window blocks visibility:hidden, and skipping them leaves the hot-frame SVG doc with just a
499    // few blocks — parse/layout cost no longer scales with the project's total block count (serializing
500    // the whole root, an 11-minute project re-lays-out all several-hundred blocks on every hot frame).
501    const rootShell = XS.serializeToString(overlay.root.cloneNode(false));
502    const rootOpen = rootShell.endsWith('/>') ? `${rootShell.slice(0, -2)}>` : rootShell.slice(0, rootShell.lastIndexOf('</'));
503    const serializeVisible = (): string => {
504      let s = rootOpen;
505      for (const el of overlay.root.children) {
506        if ((el as HTMLElement).style?.visibility === 'hidden') continue;
507        s += XS.serializeToString(el);
508      }
509      return `${s}</div>`;
510    };
511
512    const total = Math.max(1, Math.round(durationSec * FPS));
513    // Overlay frame cache: during tween gaps / static caption periods the serialized string is identical
514    // frame to frame → reuse the previous frame's bitmap, skipping parse and rasterization of the whole
515    // data URI (with inlined fonts, MB-scale — the bulk of export time).
516    let lastSerial = '';
517    let lastOverlayImg: HTMLImageElement | null = null;
518    // Overlap encoding with next-frame prep: CanvasSource.add grabs the frame synchronously on call, and
519    // the returned Promise is only encoder backpressure — the canvas is free to change once grabbed, and
520    // we only await the backpressure right before the next frame's add.
521    let pendingAdd: Promise<void> | null = null;
522    // Per-phase timing (parallel phases each track their own wall clock, so the sum can exceed total wall time): data shows where the bottleneck is
523    const tm = { prep: 0, raster: 0, video: 0, draw: 0, enc: 0, rasterN: 0 };
524    const expT0 = performance.now();
525    for (let i = 0; i < total; i++) {
526      if (opts.shouldCancel?.()) throw new ExportCanceled();
527      const t = i / FPS;
528      const tPrep = performance.now();
529      overlay.win.__hfPreview!.seekTimelines(t);
530      const { seg, srcT } = segAt(t);
531      const rig = rigs.get(seg.key);
532      // Read the framing transform uniformly from #vidEl: in canvas mode every segment's (including
533      // insert clips') framing keyframes land on this one canvas (videoFrameTimelineBody only does
534      // tl.to('#vidEl')); looking up an element by clip_<id> is a leftover from the old "insert clip =
535      // separate <video>" era — the element doesn't exist → identity matrix → export loses insert-clip framing
536      const el = overlay.doc.getElementById('vidEl');
537      const vs = readTransform(overlay.win, el);
538      const serial = serializeVisible();
539      tm.prep += performance.now() - tPrep;
540      const hot = serial !== lastSerial || !lastOverlayImg;
541      if (hot) tm.rasterN++;
542      const overlayP: Promise<HTMLImageElement> = hot
543        ? (() => {
544            const s0 = performance.now();
545            return rasterize(preEnc + encodeURIComponent(serial) + postEnc).then((img) => ((tm.raster += performance.now() - s0), img));
546          })()
547        : Promise.resolve(lastOverlayImg!);
548
549      const tr = trsX.find((x) => t >= x.cut - x.half && t <= x.cut + x.half) ?? null;
550      // Compute shadow sampling params first; the three decodes (overlay/current frame/shadow frame) are independent → parallel
551      const pre = tr ? t < tr.cut : false;
552      const gseg = tr ? (pre ? tr.segB : tr.segA) : null;
553      const gRig = tr ? rigs.get(pre ? tr.preKey : tr.postKey) : undefined;
554      const gSrcT = tr && gseg ? (pre ? Math.max(0, gseg.srcStart - (tr.cut - t)) : gseg.srcEnd + (t - tr.cut)) : 0;
555      const v0 = performance.now();
556      const [overlayImg, sample, gSample] = await Promise.all([
557        overlayP,
558        rig ? sampleAt(rig, srcT).then((s) => ((tm.video += performance.now() - v0), s)) : null,
559        gRig ? sampleAt(gRig, gSrcT) : null,
560      ]);
561      lastSerial = serial;
562      lastOverlayImg = overlayImg;
563      // bg / overlay are already rasterized at output resolution (outW×outH) → draw 1:1 with identity
564      // transform (crisp text/graphics at 4K). The source video is still drawn in comp coordinates;
565      // setTransform(Sx) maps the target rect to output size, drawImage scales straight from native frames.