feat: AI frame selection via a vision model (GLM-4.6V-Flash / Z.ai)

When a frame-selector API key is set, hook/keyframe send the extension's
deduped candidate frames and a target count; the plugin asks the vision
model to keep the most effect-heavy frames (skip talking-head shots) and
stores only those. Robust index parsing + heuristic fallback (even-sample)
when unconfigured or on any failure. New "AI 挑帧" settings (Z.ai base URL
+ GLM-4.6V-Flash prefilled; user supplies the key).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/clip-router.ts

@@ -1,6 +1,7 @@
 import { ClipPayload, HookPayload, KeyframePayload, LegacyClipPayload, ScreenshotPayload, ThumbnailPayload } from './server';
-import { AIProvider, ClipRule, isMultiFrameProvider, MultiFrameRequest, PluginSettings, ScreenshotClipRule, ThumbnailClipRule, WatchRule } from './types';
+import { AIProvider, ClipRule, isMultiFrameProvider, MultiFrameRequest, PluginSettings, ScreenshotClipRule, ThumbnailClipRule, WatchRule, FrameSelectorSettings } from './types';
 import { postProcessMarkdown, sanitize, buildVideoEmbed, extractVideoId, detectPlatform, videoKey } from './util';
+import { selectFrames } from './frame-select';
 import { buildAnchor, mergeSection, coverSection, hookSection, keyframeSection, screenshotSection, VideoNoteMeta, NewSection } from './video-note';
 
 export interface VaultOps {
@@ -21,6 +22,7 @@ export async function routeClip(
   clipRules: PluginSettings['clipRules'],
   watchRules: WatchRule[],
   vaultOps: VaultOps,
+  frameSelector?: FrameSelectorSettings,
 ): Promise<{ notePath?: string; notice?: string }> {
   if (isLegacy(payload)) {
     await handleLegacyScreenshot(payload, watchRules, vaultOps);
@@ -31,8 +33,8 @@ export async function routeClip(
     const normalized = normalizeScreenshot(payload);
     return handleScreenshot(normalized, providers, clipRules.screenshot, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder);
   }
-  if (payload.mode === 'hook') return handleMultiFrame(payload, providers, clipRules.hook, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder);
-  if (payload.mode === 'keyframe') return handleMultiFrame(payload, providers, clipRules.keyframe, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder);
+  if (payload.mode === 'hook') return handleMultiFrame(payload, providers, clipRules.hook, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder, frameSelector);
+  if (payload.mode === 'keyframe') return handleMultiFrame(payload, providers, clipRules.keyframe, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder, frameSelector);
   throw new Error('Unknown clip mode');
 }
 
@@ -279,10 +281,21 @@ async function handleMultiFrame(
   vaultOps: VaultOps,
   searchFolder: string,
   assetFolder: string,
+  frameSelector?: FrameSelectorSettings,
 ): Promise<{ notePath: string; notice?: string }> {
-  // ── Save frames (both modes need them for manual; auto uses them for AI) ──────
-  const max = rule.maxFrames ?? 5;
-  const sampled = sampleFrames(payload.frames, max);
+  // ── Pick which frames to keep from the candidates the extension sent ──────────
+  const count = (payload.frames_select && payload.frames_select > 0) ? payload.frames_select : (rule.maxFrames ?? 5);
+  let sampled: string[];
+  try {
+    if (frameSelector?.apiKey) {
+      const idx = await selectFrames(payload.frames.map((f) => Buffer.from(f, 'base64')), count, payload.mode, frameSelector);
+      sampled = idx.map((i) => payload.frames[i]);
+    } else {
+      sampled = sampleFrames(payload.frames, count);
+    }
+  } catch (_) {
+    sampled = sampleFrames(payload.frames, count); // heuristic fallback on any AI failure
+  }
   const stem = `${payload.mode}-${sanitize(payload.video_title)}-${Date.now()}`;
 
   const platform = detectPlatform(payload.url);

src/frame-select.ts

@@ -0,0 +1,49 @@
+import OpenAI from 'openai';
+import { FrameSelectorSettings } from './types';
+
+// Robustly turn the model's reply into exactly `count` valid, distinct indices.
+// Extract numbers, keep in-range/distinct ones, then pad from the unused indices
+// (in order) so we always return a usable selection even if the model misbehaves.
+export function parseSelection(text: string, count: number, total: number): number[] {
+  const seen = new Set<number>();
+  const keep: number[] = [];
+  for (const m of text.match(/\d+/g) ?? []) {
+    const n = Number(m);
+    if (n >= 0 && n < total && !seen.has(n)) {
+      seen.add(n); keep.push(n);
+      if (keep.length >= count) break;
+    }
+  }
+  for (let i = 0; i < total && keep.length < count; i++) {
+    if (!seen.has(i)) { seen.add(i); keep.push(i); }
+  }
+  return keep;
+}
+
+const PROMPTS: Record<string, string> = {
+  hook: '这些是一个视频开头的候选帧。请挑出最抓眼球、最有视觉冲击或动效的画面，避开纯黑场和静止的口播人脸定格。',
+  keyframe: '这些是一段视频的候选帧。请挑出动效、转场、运镜或动作最明显的画面，避开纯人脸定格和几乎静止的画面。',
+};
+
+// Ask the vision model which `count` frames to keep. Throws on any failure so the
+// caller can fall back to a heuristic selection.
+export async function selectFrames(
+  frames: Buffer[],
+  count: number,
+  mode: string,
+  cfg: FrameSelectorSettings,
+): Promise<number[]> {
+  const client = new OpenAI({ apiKey: cfg.apiKey, baseURL: cfg.baseUrl, dangerouslyAllowBrowser: true });
+  const labelled = frames.flatMap((f, i) => [
+    { type: 'text' as const, text: `[${i}]` },
+    { type: 'image_url' as const, image_url: { url: `data:image/jpeg;base64,${f.toString('base64')}` } },
+  ]);
+  const prompt = `${PROMPTS[mode] ?? PROMPTS.keyframe}\n每张图前面都有编号 [n]。从中选出最好的 ${count} 张，只返回 JSON：{"keep":[编号,...]}，不要任何其它文字。`;
+  const resp = await client.chat.completions.create({
+    model: cfg.model,
+    messages: [{ role: 'user', content: [...labelled, { type: 'text' as const, text: prompt }] }],
+    max_tokens: 200,
+  });
+  const text = resp.choices[0]?.message?.content ?? '';
+  return parseSelection(text, count, frames.length);
+}

src/main.ts

@@ -50,6 +50,7 @@ export default class VaultAutopilotPlugin extends Plugin {
       },
       rules: loaded?.rules ?? DEFAULT_SETTINGS.rules,
       providers: loaded?.providers ?? DEFAULT_SETTINGS.providers,
+      frameSelector: { ...DEFAULT_SETTINGS.frameSelector, ...(loaded?.frameSelector ?? {}) },
     };
   }
 
@@ -175,7 +176,7 @@ export default class VaultAutopilotPlugin extends Plugin {
     this.server = createServer(
       port,
       async (payload) => {
-        const { notePath, notice } = await routeClip(payload, this.providers, this.settings.clipRules, this.settings.rules, vaultOps);
+        const { notePath, notice } = await routeClip(payload, this.providers, this.settings.clipRules, this.settings.rules, vaultOps, this.settings.frameSelector);
         const obsidianUrl = notePath
           ? `obsidian://open?vault=${encodeURIComponent(this.app.vault.getName())}&file=${encodeURIComponent(notePath)}`
           : undefined;

src/server.ts

@@ -25,6 +25,7 @@ export type HookPayload = {
   captured_at: string;
   time_range?: { start: number; end: number };
   cover_url?: string;
+  frames_select?: number;
 };
 
 export type KeyframePayload = {
@@ -35,6 +36,7 @@ export type KeyframePayload = {
   time_range: { start: number; end: number };
   captured_at: string;
   cover_url?: string;
+  frames_select?: number;
 };
 
 export type ThumbnailPayload = {

src/settings.ts

@@ -16,6 +16,7 @@ export const DEFAULT_SETTINGS: PluginSettings = {
     hook: { sopPath: '', outputFolder: '', providerId: '', processingMode: 'manual', maxFrames: 5, framesFolder: 'Assets/images' },
     keyframe: { sopPath: '', outputFolder: '', providerId: '', processingMode: 'manual', maxFrames: 5, framesFolder: 'Assets/images' },
   },
+  frameSelector: { baseUrl: 'https://api.z.ai/api/paas/v4/', model: 'GLM-4.6V-Flash', apiKey: '' },
 };
 
 export class VaultAutopilotSettingTab extends PluginSettingTab {
@@ -47,6 +48,29 @@ export class VaultAutopilotSettingTab extends PluginSettingTab {
         if (n > 1024 && n < 65536) { this.plugin.settings.httpServer.port = n; await this.plugin.saveSettings(); }
       }));
 
+    // ── AI Frame Selection ───────────────────────────────────────────────────────
+    new Setting(containerEl).setName('AI 挑帧（可选）').setHeading();
+    new Setting(containerEl)
+      .setDesc('填了 API Key 后，Hook/关键帧会让视觉模型从候选帧里挑最有动效的几张、跳过口播脸；留空则用启发式（去黑+均匀取）。');
+    new Setting(containerEl)
+      .setName('API Key')
+      .setDesc('Z.ai / 智谱 等 OpenAI 兼容平台的 key。')
+      .addText(t => { t.inputEl.type = 'password'; t.setValue(this.plugin.settings.frameSelector.apiKey).onChange(async v => {
+        this.plugin.settings.frameSelector.apiKey = v.trim(); await this.plugin.saveSettings();
+      }); });
+    new Setting(containerEl)
+      .setName('Base URL')
+      .setDesc('OpenAI 兼容端点。Z.ai 默认：https://api.z.ai/api/paas/v4/')
+      .addText(t => t.setValue(this.plugin.settings.frameSelector.baseUrl).onChange(async v => {
+        this.plugin.settings.frameSelector.baseUrl = v.trim(); await this.plugin.saveSettings();
+      }));
+    new Setting(containerEl)
+      .setName('Model')
+      .setDesc('视觉模型名，默认 GLM-4.6V-Flash。')
+      .addText(t => t.setValue(this.plugin.settings.frameSelector.model).onChange(async v => {
+        this.plugin.settings.frameSelector.model = v.trim(); await this.plugin.saveSettings();
+      }));
+
     // ── Providers ──────────────────────────────────────────────────────────────
     new Setting(containerEl).setName('AI Providers').setHeading();
     new Setting(containerEl)

src/types.ts

@@ -122,6 +122,12 @@ export interface HttpServerSettings {
   port: number;
 }
 
+export interface FrameSelectorSettings {
+  baseUrl: string;
+  model: string;
+  apiKey: string;
+}
+
 export interface PluginSettings {
   rules: WatchRule[];
   providers: ProviderConfig[];
@@ -132,4 +138,5 @@ export interface PluginSettings {
     hook: ClipRule;
     keyframe: ClipRule;
   };
+  frameSelector: FrameSelectorSettings;
 }

tests/frame-select.test.ts

@@ -0,0 +1,21 @@
+import { parseSelection } from '../src/frame-select';
+
+test('parses a clean keep list', () => {
+  expect(parseSelection('{"keep":[2,5,7]}', 3, 10)).toEqual([2, 5, 7]);
+});
+
+test('drops out-of-range and duplicate indices, then pads', () => {
+  expect(parseSelection('{"keep":[2,2,99,5]}', 3, 10)).toEqual([2, 5, 0]);
+});
+
+test('pads with unused indices when the model returns too few', () => {
+  expect(parseSelection('keep frame 3', 3, 10)).toEqual([3, 0, 1]);
+});
+
+test('falls back to the first N on a non-numeric reply', () => {
+  expect(parseSelection('sorry, I cannot', 2, 5)).toEqual([0, 1]);
+});
+
+test('caps to count when the model returns too many', () => {
+  expect(parseSelection('[1,2,3,4,5,6]', 3, 10)).toEqual([1, 2, 3]);
+});