From 8eb95a25ce81ea988cc95c67973a980a0d65ae26 Mon Sep 17 00:00:00 2001 From: Richard McCorkle Date: Mon, 18 May 2026 14:28:30 +0200 Subject: [PATCH] refactor: remove broken watch-page/ANDROID/MWEB/WEB transcript methods Delete the four obsolete local transcript methods, their six helper methods, the YouTubeConfig interface, and the now-unused statics. The iOS player method added earlier is the sole local fallback. Co-Authored-By: Claude Opus 4.7 --- src/youtube-transcript.ts | 767 -------------------------------------- 1 file changed, 767 deletions(-) diff --git a/src/youtube-transcript.ts b/src/youtube-transcript.ts index d76d89e..29665ef 100644 --- a/src/youtube-transcript.ts +++ b/src/youtube-transcript.ts @@ -63,28 +63,11 @@ export interface TranscriptResult { } -/** - * YouTube configuration extracted from watch page - */ -interface YouTubeConfig { - apiKey: string; - clientVersion: string; - visitorData: string | null; - captionTracks: CaptionTrack[]; - metadata: TranscriptMetadata; -} - export class YouTubeTranscriptExtractor { private static readonly USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; - private static cookieStore: string = ''; - // Cache YouTube config after first extraction to avoid repeated HTML fetches within the same video request - private static cachedConfig: YouTubeConfig | null = null; - private static cachedVideoId: string | null = null; // Cache the iOS player API response per video to avoid a redundant POST within one request private static cachedPlayerData: UnknownRecord | null = null; private static cachedPlayerVideoId: string | null = null; - // Fallback client version if extraction fails (updated to current version) - private static readonly FALLBACK_CLIENT_VERSION = '2.20260128.05.00'; /** @@ -344,209 +327,6 @@ export class YouTubeTranscriptExtractor { return isValid; } - /** - * Extract a JSON object assigned to a JavaScript variable in HTML. - * Uses brace-balanced counting (respecting string escapes) instead of fragile regex. - * @param html The HTML source to search - * @param variableName The JS variable name (e.g. "ytInitialPlayerResponse") - * @returns Parsed JSON object or null if not found - */ - private static extractJsonFromHtml(html: string, variableName: string): unknown { - const searchPattern = `${variableName}\\s*=\\s*\\{`; - const match = html.match(new RegExp(searchPattern)); - if (!match || match.index === undefined) { - return null; - } - - // Find the opening brace position - const startIdx = html.indexOf('{', match.index); - if (startIdx === -1) return null; - - let depth = 0; - let inString = false; - let stringChar = ''; - let escaped = false; - - for (let i = startIdx; i < html.length; i++) { - const ch = html[i]; - - if (escaped) { - escaped = false; - continue; - } - - if (ch === '\\' && inString) { - escaped = true; - continue; - } - - if (inString) { - if (ch === stringChar) { - inString = false; - } - continue; - } - - if (ch === '"' || ch === "'") { - inString = true; - stringChar = ch; - continue; - } - - if (ch === '{') { - depth++; - } else if (ch === '}') { - depth--; - if (depth === 0) { - const jsonStr = html.substring(startIdx, i + 1); - try { - return JSON.parse(jsonStr); - } catch { - transcriptLogger.debug(`extractJsonFromHtml: failed to parse ${variableName} JSON (${jsonStr.length} chars)`); - return null; - } - } - } - } - - return null; - } - - /** - * Fetch YouTube configuration (API key, client version) from the watch page HTML. - * Results are cached to avoid repeated HTML fetches. - * Reference: https://scrapecreators.com/blog/how-to-scrape-youtube-transcripts-with-node-js-in-2025 - */ - private static async getYouTubeConfig(videoId: string): Promise { - if (this.cachedConfig) { - transcriptLogger.debug(`Using cached YouTube config (clientVersion: ${this.cachedConfig.clientVersion})`); - return this.cachedConfig; - } - - const watchUrl = `https://www.youtube.com/watch?v=${videoId}`; - try { - let html = await this.fetchWatchPageHtml(watchUrl); - - // Consent page detection: if we got a consent page instead of the real watch page, - // retry with CONSENT cookie to bypass - const hasPlayerResponse = html.includes('ytInitialPlayerResponse'); - const isConsentPage = html.includes('consent.youtube.com') || html.includes('CONSENT'); - if (!hasPlayerResponse && isConsentPage) { - transcriptLogger.debug('Detected consent page, retrying with CONSENT cookie'); - html = await this.fetchWatchPageHtml(watchUrl, 'CONSENT=YES+1'); - } - - // Extract INNERTUBE_API_KEY - const keyMatch = html.match(/"INNERTUBE_API_KEY":"([^"]+)"/); - if (!keyMatch || !keyMatch[1]) { - throw new Error('INNERTUBE_API_KEY not found in watch page'); - } - const apiKey = keyMatch[1]; - - // Extract INNERTUBE_CLIENT_VERSION (dynamically get current YouTube version) - const versionMatch = html.match(/"INNERTUBE_CLIENT_VERSION":"([^"]+)"/); - const clientVersion = versionMatch && versionMatch[1] - ? versionMatch[1] - : this.FALLBACK_CLIENT_VERSION; - - // Extract VISITOR_DATA (required for session binding in API calls) - const visitorMatch = html.match(/"VISITOR_DATA":"([^"]+)"/); - const visitorData = visitorMatch && visitorMatch[1] ? visitorMatch[1] : null; - - // Extract captionTracks and metadata from ytInitialPlayerResponse - let captionTracks: CaptionTrack[] = []; - let metadata: TranscriptMetadata = {}; - - const playerResponse = this.extractJsonFromHtml(html, 'ytInitialPlayerResponse'); - if (isRecord(playerResponse)) { - // Extract caption tracks - const captions = (playerResponse as { - captions?: { - playerCaptionsTracklistRenderer?: { - captionTracks?: Array<{ - baseUrl?: string; - languageCode?: string; - kind?: string; - vssId?: string; - isTranslatable?: boolean; - }>; - }; - }; - }).captions?.playerCaptionsTracklistRenderer?.captionTracks; - - if (captions && Array.isArray(captions)) { - captionTracks = captions.map(track => ({ - languageCode: track.languageCode || '', - kind: track.kind, - baseUrl: track.baseUrl, - vssId: track.vssId, - isTranslatable: track.isTranslatable - })); - transcriptLogger.debug(`Extracted ${captionTracks.length} caption tracks from watch page`); - } - - // Extract metadata from videoDetails - const videoDetails = (playerResponse as { - videoDetails?: { title?: string; author?: string }; - }).videoDetails; - if (videoDetails) { - metadata = { - title: typeof videoDetails.title === 'string' ? videoDetails.title : undefined, - author: typeof videoDetails.author === 'string' ? videoDetails.author : undefined - }; - transcriptLogger.debug(`Extracted metadata from watch page: title="${metadata.title}", author="${metadata.author}"`); - } - } else { - transcriptLogger.debug('ytInitialPlayerResponse not found or failed to parse from watch page'); - } - - this.cachedConfig = { - apiKey, - clientVersion, - visitorData, - captionTracks, - metadata - }; - - transcriptLogger.debug(`Extracted YouTube config - clientVersion: ${clientVersion}, visitorData: ${visitorData ? 'present' : 'not found'}, captionTracks: ${captionTracks.length}`); - return this.cachedConfig; - - } catch (err) { - const errorMessage = getSafeErrorMessage(err); - transcriptLogger.warn(`Failed to extract YouTube config dynamically: ${errorMessage}`); - - // Return fallback config without API key - caller should handle this - throw new Error(`Unable to extract YouTube configuration from watch page: ${errorMessage}`); - } - } - - /** - * Fetch the watch page HTML with standard headers. - */ - private static async fetchWatchPageHtml(watchUrl: string, extraCookie?: string): Promise { - const cookies = [ - YouTubeTranscriptExtractor.cookieStore, - extraCookie - ].filter(Boolean).join('; '); - - const response = await obsidianFetch(watchUrl, { - method: 'GET', - headers: { - 'User-Agent': YouTubeTranscriptExtractor.USER_AGENT, - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.9', - 'DNT': '1', - ...(cookies && { 'Cookie': cookies }) - } - }); - - if (!response.ok) { - throw new Error(`HTTP ${response.status}`); - } - - return response.text(); - } - /** * Select the best caption track for a requested language. * Priority: manual+exact → manual+base → auto+exact → auto+base @@ -593,552 +373,6 @@ export class YouTubeTranscriptExtractor { return { track: tracks[0], useTlang: false }; } - /** - * Primary method: Extract transcript from watch-page captionTracks. - * Reuses the cached HTML from getYouTubeConfig — zero additional HTTP cost for config. - */ - private static async fetchViaWatchPage(videoId: string, options: TranscriptOptions): Promise { - const lang = options.lang || 'en'; - - const pageData = await this.getYouTubeConfig(videoId); - - if (!pageData.captionTracks || pageData.captionTracks.length === 0) { - throw new Error('No caption tracks found in watch page'); - } - - transcriptLogger.debug(`Watch-page: ${pageData.captionTracks.length} caption tracks available`); - - const selection = this.pickBestTrack(pageData.captionTracks, lang); - if (!selection || !selection.track.baseUrl) { - throw new Error('No suitable caption track with baseUrl found'); - } - - const { track, useTlang } = selection; - const trackBaseUrl = track.baseUrl as string; // Guaranteed non-null by check above - transcriptLogger.debug(`Watch-page: selected track lang=${track.languageCode}, kind=${track.kind || 'manual'}, useTlang=${useTlang}`); - - const tlang = useTlang ? lang : undefined; - const segments = await this.fetchCaptionTrack(trackBaseUrl, tlang); - - transcriptLogger.debug(`Watch-page method succeeded with ${segments.length} segments`); - return { segments, metadata: pageData.metadata }; - } - - /** - * WEB ScrapeCreators method: Two-step approach using /next + /get_transcript internal APIs. - * Reference: https://scrapecreators.com/blog/how-to-scrape-youtube-transcripts-with-node-js-in-2025 - */ - private static async fetchViaWebScrapeCreators(videoId: string, _options: TranscriptOptions): Promise { - const watchUrl = `https://www.youtube.com/watch?v=${videoId}`; - const ytConfig = await this.getYouTubeConfig(videoId); - transcriptLogger.debug(`WEB ScrapeCreators: using clientVersion ${ytConfig.clientVersion}`); - - // Step 1: Get transcript parameters from YouTube's internal /next API - const nextApiUrl = `https://www.youtube.com/youtubei/v1/next?prettyPrint=false`; - const nextRequestBody = { - context: { - client: { - clientName: "WEB", - clientVersion: ytConfig.clientVersion, - ...(ytConfig.visitorData && { visitorData: ytConfig.visitorData }) - } - }, - videoId: videoId - }; - - transcriptLogger.debug(`WEB ScrapeCreators Step 1: Requesting transcript parameters from ${nextApiUrl}`); - - const nextResponse = await obsidianFetch(nextApiUrl, { - method: 'POST', - headers: { - 'User-Agent': YouTubeTranscriptExtractor.USER_AGENT, - 'Accept': 'application/json', - 'Accept-Language': 'en-US,en;q=0.9', - 'Content-Type': 'application/json', - 'Referer': watchUrl, - 'Origin': 'https://www.youtube.com', - 'x-youtube-client-name': '1', - 'x-youtube-client-version': ytConfig.clientVersion, - ...(ytConfig.visitorData && { 'x-goog-visitor-id': ytConfig.visitorData }), - 'DNT': '1', - ...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore }) - }, - body: JSON.stringify(nextRequestBody) - }); - - if (!nextResponse.ok) { - throw new Error(`Failed to fetch next API data: HTTP ${nextResponse.status}`); - } - - const nextData = await nextResponse.json() as unknown; - transcriptLogger.debug(`WEB ScrapeCreators Step 1 completed: received ${JSON.stringify(nextData).length} characters`); - - // Extract metadata from nextData response - const metadata = this.extractMetadataFromNextData(nextData); - transcriptLogger.debug(`WEB ScrapeCreators metadata: title="${metadata.title || 'N/A'}", author="${metadata.author || 'N/A'}"`); - - // Find transcript endpoint parameters - const transcriptParams = this.findTranscriptEndpoint(nextData); - if (!transcriptParams) { - if (isRecord(nextData)) { - transcriptLogger.debug(`Next API response keys: ${Object.keys(nextData).join(', ')}`); - } - throw new Error(`No transcript parameters found in YouTube API response for video ${videoId}`); - } - - transcriptLogger.debug(`WEB ScrapeCreators: Found transcript parameters: ${transcriptParams.substring(0, 100)}...`); - - // Step 2: Request the actual transcript using the parameters - const getTranscriptUrl = `https://www.youtube.com/youtubei/v1/get_transcript?prettyPrint=false`; - const transcriptRequestBody = { - context: { - client: { - clientName: "WEB", - clientVersion: ytConfig.clientVersion, - ...(ytConfig.visitorData && { visitorData: ytConfig.visitorData }) - } - }, - params: transcriptParams - }; - - transcriptLogger.debug(`WEB ScrapeCreators Step 2: Requesting transcript from ${getTranscriptUrl}`); - - const transcriptResponse = await obsidianFetch(getTranscriptUrl, { - method: 'POST', - headers: { - 'User-Agent': YouTubeTranscriptExtractor.USER_AGENT, - 'Accept': 'application/json', - 'Accept-Language': 'en-US,en;q=0.9', - 'Content-Type': 'application/json', - 'Referer': watchUrl, - 'Origin': 'https://www.youtube.com', - 'x-youtube-client-name': '1', - 'x-youtube-client-version': ytConfig.clientVersion, - ...(ytConfig.visitorData && { 'x-goog-visitor-id': ytConfig.visitorData }), - 'DNT': '1', - ...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore }) - }, - body: JSON.stringify(transcriptRequestBody) - }); - - if (!transcriptResponse.ok) { - let errorDetail = ''; - try { errorDetail = (await transcriptResponse.text()).substring(0, 500); } catch { /* ignore */ } - transcriptLogger.error(`get_transcript failed with HTTP ${transcriptResponse.status}: ${errorDetail}`); - throw new Error(`Failed to fetch transcript: HTTP ${transcriptResponse.status}`); - } - - const transcriptData = await transcriptResponse.json() as UnknownRecord; - transcriptLogger.debug(`WEB ScrapeCreators Step 2 completed: received ${JSON.stringify(transcriptData).length} characters`); - - const segments = this.parseScrapeCreatorsTranscript(transcriptData); - transcriptLogger.debug(`WEB ScrapeCreators method succeeded with ${segments.length} segments`); - return { segments, metadata }; - } - - /** - * Extract metadata from a /next API response. - */ - private static extractMetadataFromNextData(nextData: unknown): TranscriptMetadata { - const metadata: TranscriptMetadata = {}; - if (!isRecord(nextData)) return metadata; - - const typedNextData = nextData as { - playerOverlays?: { - playerOverlayRenderer?: { - videoDetails?: { - playerOverlayVideoDetailsRenderer?: { - title?: { simpleText?: string }; - subtitle?: { runs?: Array<{ text?: string }> }; - }; - }; - }; - }; - contents?: { - twoColumnWatchNextResults?: { - results?: { - results?: { - contents?: Array<{ - videoPrimaryInfoRenderer?: { title?: { runs?: Array<{ text?: string }> } }; - videoSecondaryInfoRenderer?: { owner?: { videoOwnerRenderer?: { title?: { runs?: Array<{ text?: string }> } } } }; - }>; - }; - }; - }; - }; - microformat?: { - playerMicroformatRenderer?: { - title?: { simpleText?: string }; - ownerChannelName?: string; - }; - }; - }; - - try { - // Method 1: From playerOverlays (most reliable) - const playerOverlayDetails = typedNextData.playerOverlays?.playerOverlayRenderer?.videoDetails?.playerOverlayVideoDetailsRenderer; - if (playerOverlayDetails) { - metadata.title = playerOverlayDetails.title?.simpleText; - metadata.author = playerOverlayDetails.subtitle?.runs?.[0]?.text; - } - - // Method 2: From contents (fallback) - if (!metadata.title || !metadata.author) { - const contents = typedNextData.contents?.twoColumnWatchNextResults?.results?.results?.contents; - if (Array.isArray(contents)) { - const primaryInfo = contents.find((c) => c.videoPrimaryInfoRenderer); - if (primaryInfo && !metadata.title) { - metadata.title = primaryInfo.videoPrimaryInfoRenderer?.title?.runs?.[0]?.text; - } - const secondaryInfo = contents.find((c) => c.videoSecondaryInfoRenderer); - if (secondaryInfo && !metadata.author) { - metadata.author = secondaryInfo.videoSecondaryInfoRenderer?.owner?.videoOwnerRenderer?.title?.runs?.[0]?.text; - } - } - } - - // Method 3: From microformat (additional fallback) - if (!metadata.title || !metadata.author) { - const microformat = typedNextData.microformat?.playerMicroformatRenderer; - if (microformat) { - if (!metadata.title && microformat.title?.simpleText) metadata.title = microformat.title.simpleText; - if (!metadata.author && microformat.ownerChannelName) metadata.author = microformat.ownerChannelName; - } - } - } catch (error) { - transcriptLogger.debug(`Error extracting metadata from nextData: ${getSafeErrorMessage(error)}`); - } - - return metadata; - } - - /** - * Recursively search for getTranscriptEndpoint.params in a /next API response. - */ - private static findTranscriptEndpoint(obj: unknown): string | null { - if (!isRecord(obj)) return null; - - // Check direct getTranscriptEndpoint - const endpoint = obj.getTranscriptEndpoint as { params?: unknown } | undefined; - if (endpoint && isString(endpoint.params)) return endpoint.params; - - // Check continuationEndpoint.getTranscriptEndpoint - const contEndpoint = (obj.continuationEndpoint as { - getTranscriptEndpoint?: { params?: unknown } - } | undefined)?.getTranscriptEndpoint; - if (contEndpoint && isString(contEndpoint.params)) return contEndpoint.params; - - for (const value of Object.values(obj)) { - const result = this.findTranscriptEndpoint(value); - if (result) return result; - } - return null; - } - - /** - * Parse transcript data from /get_transcript API response into segments. - */ - private static parseScrapeCreatorsTranscript(transcriptData: UnknownRecord): TranscriptSegment[] { - // Look for transcript text in the response with enhanced search - const findTranscriptText = (obj: unknown, depth = 0): unknown => { - if (depth > 10) return null; - - if (isRecord(obj)) { - // Check for the main transcript structure - const transcriptBody = (obj['transcriptBody'] as { transcriptBodyRenderer?: { cueGroups?: unknown } } | undefined)?.transcriptBodyRenderer?.cueGroups; - if (transcriptBody) return transcriptBody; - - // Check for alternative transcript structures - const directCueGroups = obj['cueGroups']; - if (directCueGroups && Array.isArray(directCueGroups)) return directCueGroups; - - // Check for updateEngagementPanelAction structure - const initialSegments = (obj['updateEngagementPanelAction'] as { - content?: { - transcriptRenderer?: { - content?: { - transcriptSearchPanelRenderer?: { - body?: { - transcriptSegmentListRenderer?: { - initialSegments?: unknown; - }; - }; - }; - }; - }; - }; - } | undefined)?.content?.transcriptRenderer?.content?.transcriptSearchPanelRenderer?.body?.transcriptSegmentListRenderer?.initialSegments; - if (initialSegments) return initialSegments; - - // Search through actions array - const actions = obj['actions']; - if (actions && Array.isArray(actions)) { - for (let i = 0; i < actions.length; i++) { - const result = findTranscriptText(actions[i], depth + 1); - if (result) return result; - } - } - - // Search through all object properties - for (const [, value] of Object.entries(obj)) { - if (typeof value === 'object') { - const result = findTranscriptText(value, depth + 1); - if (result) return result; - } - } - } - return null; - }; - - const cueGroups = findTranscriptText(transcriptData); - - if (!cueGroups || !Array.isArray(cueGroups)) { - transcriptLogger.error(`No cueGroups found in transcript response`); - transcriptLogger.debug(`Transcript response keys: ${Object.keys(transcriptData).join(', ')}`); - - const transcriptActions = transcriptData.actions; - if (Array.isArray(transcriptActions)) { - transcriptLogger.debug(`Found ${transcriptActions.length} actions, examining structure...`); - transcriptActions.forEach((action, i: number) => { - if (isRecord(action)) { - transcriptLogger.debug(`Action ${i} keys: ${Object.keys(action).join(', ')}`); - } - }); - } - - throw new Error(`No transcript cueGroups found in YouTube API response`); - } - - transcriptLogger.debug(`Found ${cueGroups.length} cue groups in transcript`); - - // Convert cue groups to segments - handle multiple formats - const segments = cueGroups.map((cueGroup: unknown, index: number) => { - // Traditional cueGroup format - const cue = (cueGroup as { - transcriptCueGroupRenderer?: { - cues?: Array<{ - transcriptCueRenderer?: { - cue?: { simpleText?: string }; - startOffsetMs?: string; - durationMs?: string; - }; - }>; - }; - }).transcriptCueGroupRenderer?.cues?.[0]?.transcriptCueRenderer; - if (cue) { - return { - text: (cue.cue?.simpleText || '').trim(), - start: parseInt(cue.startOffsetMs || '0') / 1000, - duration: parseInt(cue.durationMs || '0') / 1000 - }; - } - - // Alternative format: transcriptSegmentRenderer (from initialSegments) - const segmentRenderer = (cueGroup as { - transcriptSegmentRenderer?: { - snippet?: { runs?: Array<{ text?: string }> }; - startMs?: string; - endMs?: string; - }; - }).transcriptSegmentRenderer; - if (segmentRenderer) { - const startMs = parseInt(segmentRenderer.startMs || '0'); - const endMs = parseInt(segmentRenderer.endMs || '0'); - return { - text: (segmentRenderer.snippet?.runs?.[0]?.text || '').trim(), - start: startMs / 1000, - duration: (endMs - startMs) / 1000 - }; - } - - if (index < 3) { - const info = isRecord(cueGroup) ? Object.keys(cueGroup).join(', ') : 'non-object'; - transcriptLogger.debug(`Unknown cueGroup format at index ${index}: ${info}`); - } - return null; - }).filter((segment): segment is TranscriptSegment => segment !== null && !!segment.text); - - if (segments.length === 0) { - throw new Error(`No valid transcript segments found in YouTube API response`); - } - - return segments; - } - - /** - * MWEB client fallback: Uses YouTube's mobile-web client, which appears to have less strict - * POT (Proof-of-Origin Token) enforcement than the WEB client for caption fetches. - * clientName=MWEB, clientId=2 - */ - private static async fetchViaPlayerApiMWEB(videoId: string, options: TranscriptOptions): Promise { - const lang = options.lang || 'en'; - const country = options.country || 'US'; - - const ytConfig = await this.getYouTubeConfig(videoId); - transcriptLogger.debug('Player API (MWEB): attempting with MWEB client'); - - const playerUrl = `https://www.youtube.com/youtubei/v1/player?key=${ytConfig.apiKey}`; - const playerBody = { - context: { - client: { - clientName: 'MWEB', - clientVersion: '2.20260408.02.00', - hl: lang, - gl: country, - ...(ytConfig.visitorData && { visitorData: ytConfig.visitorData }) - } - }, - videoId - }; - - transcriptLogger.debug(`Player API (MWEB): requesting caption tracks for ${videoId}`); - - const playerResponse = await obsidianFetch(playerUrl, { - method: 'POST', - headers: { - 'User-Agent': 'Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36', - 'Accept': 'application/json', - 'Accept-Language': 'en-US,en;q=0.9', - 'Content-Type': 'application/json', - 'Origin': 'https://m.youtube.com', - 'Referer': `https://m.youtube.com/watch?v=${videoId}`, - 'X-Youtube-Client-Name': '2', // 2 = MWEB - 'X-Youtube-Client-Version': '2.20260408.02.00', - ...(ytConfig.visitorData && { 'x-goog-visitor-id': ytConfig.visitorData }), - ...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore }) - }, - body: JSON.stringify(playerBody) - }); - - if (!playerResponse.ok) { - throw new Error(`Player API (MWEB) error: HTTP ${playerResponse.status}`); - } - - const playerData = await playerResponse.json() as UnknownRecord; - - const videoDetails = (playerData as { - videoDetails?: { title?: string; author?: string }; - }).videoDetails; - - const metadata: TranscriptMetadata = { - title: videoDetails?.title, - author: videoDetails?.author - }; - - const captions = (playerData as { - captions?: { - playerCaptionsTracklistRenderer?: { - captionTracks?: Array<{ baseUrl?: string; languageCode?: string }>; - }; - }; - }).captions?.playerCaptionsTracklistRenderer?.captionTracks; - - if (!captions || !Array.isArray(captions) || captions.length === 0) { - throw new Error('No caption tracks available from Player API (MWEB)'); - } - - transcriptLogger.debug(`Player API (MWEB): found ${captions.length} caption tracks`); - - const preferredTrack = captions.find(track => track.languageCode?.startsWith(lang)) || captions[0]; - if (!preferredTrack?.baseUrl) { - throw new Error('Caption track missing baseUrl'); - } - - // Use mobile Chrome UA consistently for the caption fetch - const mwebUA = 'Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36'; - const segments = await this.fetchCaptionTrack(preferredTrack.baseUrl, undefined, mwebUA); - transcriptLogger.debug(`Player API (MWEB): successfully extracted ${segments.length} segments`); - return { segments, metadata }; - } - - /** - * ANDROID client fallback: Uses ANDROID client which bypasses WEB restrictions. - * Reference: https://github.com/LuanRT/YouTube.js and ScrapeCreators research - */ - private static async fetchViaPlayerApiAndroid(videoId: string, options: TranscriptOptions): Promise { - const lang = options.lang || 'en'; - const country = options.country || 'US'; - - // Get API key from config - const ytConfig = await this.getYouTubeConfig(videoId); - transcriptLogger.debug('Player API (ANDROID): attempting with ANDROID client'); - - // ANDROID client configuration - bypasses WEB restrictions - // Version 20.10.38 from youtube-transcript-api (actively maintained Python library) - const playerUrl = `https://www.youtube.com/youtubei/v1/player?key=${ytConfig.apiKey}`; - const playerBody = { - context: { - client: { - clientName: 'ANDROID', - clientVersion: '20.10.38', - androidSdkVersion: 30, - hl: lang, - gl: country - } - }, - videoId - }; - - transcriptLogger.debug(`Player API (ANDROID): requesting caption tracks for ${videoId}`); - - const playerResponse = await obsidianFetch(playerUrl, { - method: 'POST', - headers: { - 'User-Agent': 'com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip', - 'Accept': 'application/json', - 'Accept-Language': 'en-US,en;q=0.9', - 'Content-Type': 'application/json', - 'Origin': 'https://www.youtube.com', - 'X-Youtube-Client-Name': '3', // 3 = ANDROID - 'X-Youtube-Client-Version': '20.10.38', - ...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore }) - }, - body: JSON.stringify(playerBody) - }); - - if (!playerResponse.ok) { - throw new Error(`Player API (ANDROID) error: HTTP ${playerResponse.status}`); - } - - const playerData = await playerResponse.json() as UnknownRecord; - - const videoDetails = (playerData as { - videoDetails?: { title?: string; author?: string }; - }).videoDetails; - - const metadata: TranscriptMetadata = { - title: videoDetails?.title, - author: videoDetails?.author - }; - - const captions = (playerData as { - captions?: { - playerCaptionsTracklistRenderer?: { - captionTracks?: Array<{ baseUrl?: string; languageCode?: string }>; - }; - }; - }).captions?.playerCaptionsTracklistRenderer?.captionTracks; - - if (!captions || !Array.isArray(captions) || captions.length === 0) { - throw new Error('No caption tracks available from Player API (ANDROID)'); - } - - transcriptLogger.debug(`Player API (ANDROID): found ${captions.length} caption tracks`); - - // Prefer matching language, otherwise first track - const preferredTrack = captions.find(track => track.languageCode?.startsWith(lang)) || captions[0]; - if (!preferredTrack?.baseUrl) { - throw new Error('Caption track missing baseUrl'); - } - - // Pass the Android UA for the caption fetch — keeps a consistent client identity - // end-to-end, which may help bypass POT requirements on the timedtext endpoint - const androidUA = 'com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip'; - const segments = await this.fetchCaptionTrack(preferredTrack.baseUrl, undefined, androidUA); - transcriptLogger.debug(`Player API (ANDROID): successfully extracted ${segments.length} segments`); - return { segments, metadata }; - } - /** * Supadata API fallback: Uses Supadata's transcript API as a third fallback. * Requires an API key from supadata.ai. @@ -1457,7 +691,6 @@ export class YouTubeTranscriptExtractor { 'Origin': 'https://www.youtube.com', 'Referer': 'https://www.youtube.com/', 'DNT': '1', - ...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore }) } });