diff --git a/src/clip-router.ts b/src/clip-router.ts index f95cfca..679e1fc 100644 --- a/src/clip-router.ts +++ b/src/clip-router.ts @@ -1,6 +1,7 @@ import { ClipPayload, HookPayload, KeyframePayload, LegacyClipPayload, ScreenshotPayload, ThumbnailPayload } from './server'; -import { AIProvider, ClipRule, isMultiFrameProvider, MultiFrameRequest, PluginSettings, ScreenshotClipRule, ThumbnailClipRule, WatchRule } from './types'; +import { AIProvider, ClipRule, isMultiFrameProvider, MultiFrameRequest, PluginSettings, ScreenshotClipRule, ThumbnailClipRule, WatchRule, FrameSelectorSettings } from './types'; import { postProcessMarkdown, sanitize, buildVideoEmbed, extractVideoId, detectPlatform, videoKey } from './util'; +import { selectFrames } from './frame-select'; import { buildAnchor, mergeSection, coverSection, hookSection, keyframeSection, screenshotSection, VideoNoteMeta, NewSection } from './video-note'; export interface VaultOps { @@ -21,6 +22,7 @@ export async function routeClip( clipRules: PluginSettings['clipRules'], watchRules: WatchRule[], vaultOps: VaultOps, + frameSelector?: FrameSelectorSettings, ): Promise<{ notePath?: string; notice?: string }> { if (isLegacy(payload)) { await handleLegacyScreenshot(payload, watchRules, vaultOps); @@ -31,8 +33,8 @@ export async function routeClip( const normalized = normalizeScreenshot(payload); return handleScreenshot(normalized, providers, clipRules.screenshot, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder); } - if (payload.mode === 'hook') return handleMultiFrame(payload, providers, clipRules.hook, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder); - if (payload.mode === 'keyframe') return handleMultiFrame(payload, providers, clipRules.keyframe, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder); + if (payload.mode === 'hook') return handleMultiFrame(payload, providers, clipRules.hook, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder, frameSelector); + if (payload.mode === 'keyframe') return handleMultiFrame(payload, providers, clipRules.keyframe, vaultOps, clipRules.thumbnail.outputFolder, clipRules.thumbnail.thumbnailFolder, frameSelector); throw new Error('Unknown clip mode'); } @@ -279,10 +281,21 @@ async function handleMultiFrame( vaultOps: VaultOps, searchFolder: string, assetFolder: string, + frameSelector?: FrameSelectorSettings, ): Promise<{ notePath: string; notice?: string }> { - // ── Save frames (both modes need them for manual; auto uses them for AI) ────── - const max = rule.maxFrames ?? 5; - const sampled = sampleFrames(payload.frames, max); + // ── Pick which frames to keep from the candidates the extension sent ────────── + const count = (payload.frames_select && payload.frames_select > 0) ? payload.frames_select : (rule.maxFrames ?? 5); + let sampled: string[]; + try { + if (frameSelector?.apiKey) { + const idx = await selectFrames(payload.frames.map((f) => Buffer.from(f, 'base64')), count, payload.mode, frameSelector); + sampled = idx.map((i) => payload.frames[i]); + } else { + sampled = sampleFrames(payload.frames, count); + } + } catch (_) { + sampled = sampleFrames(payload.frames, count); // heuristic fallback on any AI failure + } const stem = `${payload.mode}-${sanitize(payload.video_title)}-${Date.now()}`; const platform = detectPlatform(payload.url); diff --git a/src/frame-select.ts b/src/frame-select.ts new file mode 100644 index 0000000..0359906 --- /dev/null +++ b/src/frame-select.ts @@ -0,0 +1,49 @@ +import OpenAI from 'openai'; +import { FrameSelectorSettings } from './types'; + +// Robustly turn the model's reply into exactly `count` valid, distinct indices. +// Extract numbers, keep in-range/distinct ones, then pad from the unused indices +// (in order) so we always return a usable selection even if the model misbehaves. +export function parseSelection(text: string, count: number, total: number): number[] { + const seen = new Set(); + const keep: number[] = []; + for (const m of text.match(/\d+/g) ?? []) { + const n = Number(m); + if (n >= 0 && n < total && !seen.has(n)) { + seen.add(n); keep.push(n); + if (keep.length >= count) break; + } + } + for (let i = 0; i < total && keep.length < count; i++) { + if (!seen.has(i)) { seen.add(i); keep.push(i); } + } + return keep; +} + +const PROMPTS: Record = { + hook: '这些是一个视频开头的候选帧。请挑出最抓眼球、最有视觉冲击或动效的画面,避开纯黑场和静止的口播人脸定格。', + keyframe: '这些是一段视频的候选帧。请挑出动效、转场、运镜或动作最明显的画面,避开纯人脸定格和几乎静止的画面。', +}; + +// Ask the vision model which `count` frames to keep. Throws on any failure so the +// caller can fall back to a heuristic selection. +export async function selectFrames( + frames: Buffer[], + count: number, + mode: string, + cfg: FrameSelectorSettings, +): Promise { + const client = new OpenAI({ apiKey: cfg.apiKey, baseURL: cfg.baseUrl, dangerouslyAllowBrowser: true }); + const labelled = frames.flatMap((f, i) => [ + { type: 'text' as const, text: `[${i}]` }, + { type: 'image_url' as const, image_url: { url: `data:image/jpeg;base64,${f.toString('base64')}` } }, + ]); + const prompt = `${PROMPTS[mode] ?? PROMPTS.keyframe}\n每张图前面都有编号 [n]。从中选出最好的 ${count} 张,只返回 JSON:{"keep":[编号,...]},不要任何其它文字。`; + const resp = await client.chat.completions.create({ + model: cfg.model, + messages: [{ role: 'user', content: [...labelled, { type: 'text' as const, text: prompt }] }], + max_tokens: 200, + }); + const text = resp.choices[0]?.message?.content ?? ''; + return parseSelection(text, count, frames.length); +} diff --git a/src/main.ts b/src/main.ts index d71ff67..859eab2 100644 --- a/src/main.ts +++ b/src/main.ts @@ -50,6 +50,7 @@ export default class VaultAutopilotPlugin extends Plugin { }, rules: loaded?.rules ?? DEFAULT_SETTINGS.rules, providers: loaded?.providers ?? DEFAULT_SETTINGS.providers, + frameSelector: { ...DEFAULT_SETTINGS.frameSelector, ...(loaded?.frameSelector ?? {}) }, }; } @@ -175,7 +176,7 @@ export default class VaultAutopilotPlugin extends Plugin { this.server = createServer( port, async (payload) => { - const { notePath, notice } = await routeClip(payload, this.providers, this.settings.clipRules, this.settings.rules, vaultOps); + const { notePath, notice } = await routeClip(payload, this.providers, this.settings.clipRules, this.settings.rules, vaultOps, this.settings.frameSelector); const obsidianUrl = notePath ? `obsidian://open?vault=${encodeURIComponent(this.app.vault.getName())}&file=${encodeURIComponent(notePath)}` : undefined; diff --git a/src/server.ts b/src/server.ts index 50f6db2..4f6a8a4 100644 --- a/src/server.ts +++ b/src/server.ts @@ -25,6 +25,7 @@ export type HookPayload = { captured_at: string; time_range?: { start: number; end: number }; cover_url?: string; + frames_select?: number; }; export type KeyframePayload = { @@ -35,6 +36,7 @@ export type KeyframePayload = { time_range: { start: number; end: number }; captured_at: string; cover_url?: string; + frames_select?: number; }; export type ThumbnailPayload = { diff --git a/src/settings.ts b/src/settings.ts index 2616559..131f453 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -16,6 +16,7 @@ export const DEFAULT_SETTINGS: PluginSettings = { hook: { sopPath: '', outputFolder: '', providerId: '', processingMode: 'manual', maxFrames: 5, framesFolder: 'Assets/images' }, keyframe: { sopPath: '', outputFolder: '', providerId: '', processingMode: 'manual', maxFrames: 5, framesFolder: 'Assets/images' }, }, + frameSelector: { baseUrl: 'https://api.z.ai/api/paas/v4/', model: 'GLM-4.6V-Flash', apiKey: '' }, }; export class VaultAutopilotSettingTab extends PluginSettingTab { @@ -47,6 +48,29 @@ export class VaultAutopilotSettingTab extends PluginSettingTab { if (n > 1024 && n < 65536) { this.plugin.settings.httpServer.port = n; await this.plugin.saveSettings(); } })); + // ── AI Frame Selection ─────────────────────────────────────────────────────── + new Setting(containerEl).setName('AI 挑帧(可选)').setHeading(); + new Setting(containerEl) + .setDesc('填了 API Key 后,Hook/关键帧会让视觉模型从候选帧里挑最有动效的几张、跳过口播脸;留空则用启发式(去黑+均匀取)。'); + new Setting(containerEl) + .setName('API Key') + .setDesc('Z.ai / 智谱 等 OpenAI 兼容平台的 key。') + .addText(t => { t.inputEl.type = 'password'; t.setValue(this.plugin.settings.frameSelector.apiKey).onChange(async v => { + this.plugin.settings.frameSelector.apiKey = v.trim(); await this.plugin.saveSettings(); + }); }); + new Setting(containerEl) + .setName('Base URL') + .setDesc('OpenAI 兼容端点。Z.ai 默认:https://api.z.ai/api/paas/v4/') + .addText(t => t.setValue(this.plugin.settings.frameSelector.baseUrl).onChange(async v => { + this.plugin.settings.frameSelector.baseUrl = v.trim(); await this.plugin.saveSettings(); + })); + new Setting(containerEl) + .setName('Model') + .setDesc('视觉模型名,默认 GLM-4.6V-Flash。') + .addText(t => t.setValue(this.plugin.settings.frameSelector.model).onChange(async v => { + this.plugin.settings.frameSelector.model = v.trim(); await this.plugin.saveSettings(); + })); + // ── Providers ────────────────────────────────────────────────────────────── new Setting(containerEl).setName('AI Providers').setHeading(); new Setting(containerEl) diff --git a/src/types.ts b/src/types.ts index dc5dc44..758f666 100644 --- a/src/types.ts +++ b/src/types.ts @@ -122,6 +122,12 @@ export interface HttpServerSettings { port: number; } +export interface FrameSelectorSettings { + baseUrl: string; + model: string; + apiKey: string; +} + export interface PluginSettings { rules: WatchRule[]; providers: ProviderConfig[]; @@ -132,4 +138,5 @@ export interface PluginSettings { hook: ClipRule; keyframe: ClipRule; }; + frameSelector: FrameSelectorSettings; } diff --git a/tests/frame-select.test.ts b/tests/frame-select.test.ts new file mode 100644 index 0000000..9a4d0f4 --- /dev/null +++ b/tests/frame-select.test.ts @@ -0,0 +1,21 @@ +import { parseSelection } from '../src/frame-select'; + +test('parses a clean keep list', () => { + expect(parseSelection('{"keep":[2,5,7]}', 3, 10)).toEqual([2, 5, 7]); +}); + +test('drops out-of-range and duplicate indices, then pads', () => { + expect(parseSelection('{"keep":[2,2,99,5]}', 3, 10)).toEqual([2, 5, 0]); +}); + +test('pads with unused indices when the model returns too few', () => { + expect(parseSelection('keep frame 3', 3, 10)).toEqual([3, 0, 1]); +}); + +test('falls back to the first N on a non-numeric reply', () => { + expect(parseSelection('sorry, I cannot', 2, 5)).toEqual([0, 1]); +}); + +test('caps to count when the model returns too many', () => { + expect(parseSelection('[1,2,3,4,5,6]', 3, 10)).toEqual([1, 2, 3]); +});