mirror of
https://github.com/rmccorkl/TubeSage.git
synced 2026-07-22 06:45:31 +00:00
refactor: remove broken watch-page/ANDROID/MWEB/WEB transcript methods
Delete the four obsolete local transcript methods, their six helper methods, the YouTubeConfig interface, and the now-unused statics. The iOS player method added earlier is the sole local fallback. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Richard McCorkle
parent
314a737437
commit
8eb95a25ce
1 changed files with 0 additions and 767 deletions
|
|
@ -63,28 +63,11 @@ export interface TranscriptResult {
|
|||
}
|
||||
|
||||
|
||||
/**
|
||||
* YouTube configuration extracted from watch page
|
||||
*/
|
||||
interface YouTubeConfig {
|
||||
apiKey: string;
|
||||
clientVersion: string;
|
||||
visitorData: string | null;
|
||||
captionTracks: CaptionTrack[];
|
||||
metadata: TranscriptMetadata;
|
||||
}
|
||||
|
||||
export class YouTubeTranscriptExtractor {
|
||||
private static readonly USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
private static cookieStore: string = '';
|
||||
// Cache YouTube config after first extraction to avoid repeated HTML fetches within the same video request
|
||||
private static cachedConfig: YouTubeConfig | null = null;
|
||||
private static cachedVideoId: string | null = null;
|
||||
// Cache the iOS player API response per video to avoid a redundant POST within one request
|
||||
private static cachedPlayerData: UnknownRecord | null = null;
|
||||
private static cachedPlayerVideoId: string | null = null;
|
||||
// Fallback client version if extraction fails (updated to current version)
|
||||
private static readonly FALLBACK_CLIENT_VERSION = '2.20260128.05.00';
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -344,209 +327,6 @@ export class YouTubeTranscriptExtractor {
|
|||
return isValid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a JSON object assigned to a JavaScript variable in HTML.
|
||||
* Uses brace-balanced counting (respecting string escapes) instead of fragile regex.
|
||||
* @param html The HTML source to search
|
||||
* @param variableName The JS variable name (e.g. "ytInitialPlayerResponse")
|
||||
* @returns Parsed JSON object or null if not found
|
||||
*/
|
||||
private static extractJsonFromHtml(html: string, variableName: string): unknown {
|
||||
const searchPattern = `${variableName}\\s*=\\s*\\{`;
|
||||
const match = html.match(new RegExp(searchPattern));
|
||||
if (!match || match.index === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Find the opening brace position
|
||||
const startIdx = html.indexOf('{', match.index);
|
||||
if (startIdx === -1) return null;
|
||||
|
||||
let depth = 0;
|
||||
let inString = false;
|
||||
let stringChar = '';
|
||||
let escaped = false;
|
||||
|
||||
for (let i = startIdx; i < html.length; i++) {
|
||||
const ch = html[i];
|
||||
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === '\\' && inString) {
|
||||
escaped = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (inString) {
|
||||
if (ch === stringChar) {
|
||||
inString = false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === '"' || ch === "'") {
|
||||
inString = true;
|
||||
stringChar = ch;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === '{') {
|
||||
depth++;
|
||||
} else if (ch === '}') {
|
||||
depth--;
|
||||
if (depth === 0) {
|
||||
const jsonStr = html.substring(startIdx, i + 1);
|
||||
try {
|
||||
return JSON.parse(jsonStr);
|
||||
} catch {
|
||||
transcriptLogger.debug(`extractJsonFromHtml: failed to parse ${variableName} JSON (${jsonStr.length} chars)`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch YouTube configuration (API key, client version) from the watch page HTML.
|
||||
* Results are cached to avoid repeated HTML fetches.
|
||||
* Reference: https://scrapecreators.com/blog/how-to-scrape-youtube-transcripts-with-node-js-in-2025
|
||||
*/
|
||||
private static async getYouTubeConfig(videoId: string): Promise<YouTubeConfig> {
|
||||
if (this.cachedConfig) {
|
||||
transcriptLogger.debug(`Using cached YouTube config (clientVersion: ${this.cachedConfig.clientVersion})`);
|
||||
return this.cachedConfig;
|
||||
}
|
||||
|
||||
const watchUrl = `https://www.youtube.com/watch?v=${videoId}`;
|
||||
try {
|
||||
let html = await this.fetchWatchPageHtml(watchUrl);
|
||||
|
||||
// Consent page detection: if we got a consent page instead of the real watch page,
|
||||
// retry with CONSENT cookie to bypass
|
||||
const hasPlayerResponse = html.includes('ytInitialPlayerResponse');
|
||||
const isConsentPage = html.includes('consent.youtube.com') || html.includes('CONSENT');
|
||||
if (!hasPlayerResponse && isConsentPage) {
|
||||
transcriptLogger.debug('Detected consent page, retrying with CONSENT cookie');
|
||||
html = await this.fetchWatchPageHtml(watchUrl, 'CONSENT=YES+1');
|
||||
}
|
||||
|
||||
// Extract INNERTUBE_API_KEY
|
||||
const keyMatch = html.match(/"INNERTUBE_API_KEY":"([^"]+)"/);
|
||||
if (!keyMatch || !keyMatch[1]) {
|
||||
throw new Error('INNERTUBE_API_KEY not found in watch page');
|
||||
}
|
||||
const apiKey = keyMatch[1];
|
||||
|
||||
// Extract INNERTUBE_CLIENT_VERSION (dynamically get current YouTube version)
|
||||
const versionMatch = html.match(/"INNERTUBE_CLIENT_VERSION":"([^"]+)"/);
|
||||
const clientVersion = versionMatch && versionMatch[1]
|
||||
? versionMatch[1]
|
||||
: this.FALLBACK_CLIENT_VERSION;
|
||||
|
||||
// Extract VISITOR_DATA (required for session binding in API calls)
|
||||
const visitorMatch = html.match(/"VISITOR_DATA":"([^"]+)"/);
|
||||
const visitorData = visitorMatch && visitorMatch[1] ? visitorMatch[1] : null;
|
||||
|
||||
// Extract captionTracks and metadata from ytInitialPlayerResponse
|
||||
let captionTracks: CaptionTrack[] = [];
|
||||
let metadata: TranscriptMetadata = {};
|
||||
|
||||
const playerResponse = this.extractJsonFromHtml(html, 'ytInitialPlayerResponse');
|
||||
if (isRecord(playerResponse)) {
|
||||
// Extract caption tracks
|
||||
const captions = (playerResponse as {
|
||||
captions?: {
|
||||
playerCaptionsTracklistRenderer?: {
|
||||
captionTracks?: Array<{
|
||||
baseUrl?: string;
|
||||
languageCode?: string;
|
||||
kind?: string;
|
||||
vssId?: string;
|
||||
isTranslatable?: boolean;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
}).captions?.playerCaptionsTracklistRenderer?.captionTracks;
|
||||
|
||||
if (captions && Array.isArray(captions)) {
|
||||
captionTracks = captions.map(track => ({
|
||||
languageCode: track.languageCode || '',
|
||||
kind: track.kind,
|
||||
baseUrl: track.baseUrl,
|
||||
vssId: track.vssId,
|
||||
isTranslatable: track.isTranslatable
|
||||
}));
|
||||
transcriptLogger.debug(`Extracted ${captionTracks.length} caption tracks from watch page`);
|
||||
}
|
||||
|
||||
// Extract metadata from videoDetails
|
||||
const videoDetails = (playerResponse as {
|
||||
videoDetails?: { title?: string; author?: string };
|
||||
}).videoDetails;
|
||||
if (videoDetails) {
|
||||
metadata = {
|
||||
title: typeof videoDetails.title === 'string' ? videoDetails.title : undefined,
|
||||
author: typeof videoDetails.author === 'string' ? videoDetails.author : undefined
|
||||
};
|
||||
transcriptLogger.debug(`Extracted metadata from watch page: title="${metadata.title}", author="${metadata.author}"`);
|
||||
}
|
||||
} else {
|
||||
transcriptLogger.debug('ytInitialPlayerResponse not found or failed to parse from watch page');
|
||||
}
|
||||
|
||||
this.cachedConfig = {
|
||||
apiKey,
|
||||
clientVersion,
|
||||
visitorData,
|
||||
captionTracks,
|
||||
metadata
|
||||
};
|
||||
|
||||
transcriptLogger.debug(`Extracted YouTube config - clientVersion: ${clientVersion}, visitorData: ${visitorData ? 'present' : 'not found'}, captionTracks: ${captionTracks.length}`);
|
||||
return this.cachedConfig;
|
||||
|
||||
} catch (err) {
|
||||
const errorMessage = getSafeErrorMessage(err);
|
||||
transcriptLogger.warn(`Failed to extract YouTube config dynamically: ${errorMessage}`);
|
||||
|
||||
// Return fallback config without API key - caller should handle this
|
||||
throw new Error(`Unable to extract YouTube configuration from watch page: ${errorMessage}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the watch page HTML with standard headers.
|
||||
*/
|
||||
private static async fetchWatchPageHtml(watchUrl: string, extraCookie?: string): Promise<string> {
|
||||
const cookies = [
|
||||
YouTubeTranscriptExtractor.cookieStore,
|
||||
extraCookie
|
||||
].filter(Boolean).join('; ');
|
||||
|
||||
const response = await obsidianFetch(watchUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'User-Agent': YouTubeTranscriptExtractor.USER_AGENT,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'DNT': '1',
|
||||
...(cookies && { 'Cookie': cookies })
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
return response.text();
|
||||
}
|
||||
|
||||
/**
|
||||
* Select the best caption track for a requested language.
|
||||
* Priority: manual+exact → manual+base → auto+exact → auto+base
|
||||
|
|
@ -593,552 +373,6 @@ export class YouTubeTranscriptExtractor {
|
|||
return { track: tracks[0], useTlang: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Primary method: Extract transcript from watch-page captionTracks.
|
||||
* Reuses the cached HTML from getYouTubeConfig — zero additional HTTP cost for config.
|
||||
*/
|
||||
private static async fetchViaWatchPage(videoId: string, options: TranscriptOptions): Promise<TranscriptResult> {
|
||||
const lang = options.lang || 'en';
|
||||
|
||||
const pageData = await this.getYouTubeConfig(videoId);
|
||||
|
||||
if (!pageData.captionTracks || pageData.captionTracks.length === 0) {
|
||||
throw new Error('No caption tracks found in watch page');
|
||||
}
|
||||
|
||||
transcriptLogger.debug(`Watch-page: ${pageData.captionTracks.length} caption tracks available`);
|
||||
|
||||
const selection = this.pickBestTrack(pageData.captionTracks, lang);
|
||||
if (!selection || !selection.track.baseUrl) {
|
||||
throw new Error('No suitable caption track with baseUrl found');
|
||||
}
|
||||
|
||||
const { track, useTlang } = selection;
|
||||
const trackBaseUrl = track.baseUrl as string; // Guaranteed non-null by check above
|
||||
transcriptLogger.debug(`Watch-page: selected track lang=${track.languageCode}, kind=${track.kind || 'manual'}, useTlang=${useTlang}`);
|
||||
|
||||
const tlang = useTlang ? lang : undefined;
|
||||
const segments = await this.fetchCaptionTrack(trackBaseUrl, tlang);
|
||||
|
||||
transcriptLogger.debug(`Watch-page method succeeded with ${segments.length} segments`);
|
||||
return { segments, metadata: pageData.metadata };
|
||||
}
|
||||
|
||||
/**
|
||||
* WEB ScrapeCreators method: Two-step approach using /next + /get_transcript internal APIs.
|
||||
* Reference: https://scrapecreators.com/blog/how-to-scrape-youtube-transcripts-with-node-js-in-2025
|
||||
*/
|
||||
private static async fetchViaWebScrapeCreators(videoId: string, _options: TranscriptOptions): Promise<TranscriptResult> {
|
||||
const watchUrl = `https://www.youtube.com/watch?v=${videoId}`;
|
||||
const ytConfig = await this.getYouTubeConfig(videoId);
|
||||
transcriptLogger.debug(`WEB ScrapeCreators: using clientVersion ${ytConfig.clientVersion}`);
|
||||
|
||||
// Step 1: Get transcript parameters from YouTube's internal /next API
|
||||
const nextApiUrl = `https://www.youtube.com/youtubei/v1/next?prettyPrint=false`;
|
||||
const nextRequestBody = {
|
||||
context: {
|
||||
client: {
|
||||
clientName: "WEB",
|
||||
clientVersion: ytConfig.clientVersion,
|
||||
...(ytConfig.visitorData && { visitorData: ytConfig.visitorData })
|
||||
}
|
||||
},
|
||||
videoId: videoId
|
||||
};
|
||||
|
||||
transcriptLogger.debug(`WEB ScrapeCreators Step 1: Requesting transcript parameters from ${nextApiUrl}`);
|
||||
|
||||
const nextResponse = await obsidianFetch(nextApiUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'User-Agent': YouTubeTranscriptExtractor.USER_AGENT,
|
||||
'Accept': 'application/json',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Content-Type': 'application/json',
|
||||
'Referer': watchUrl,
|
||||
'Origin': 'https://www.youtube.com',
|
||||
'x-youtube-client-name': '1',
|
||||
'x-youtube-client-version': ytConfig.clientVersion,
|
||||
...(ytConfig.visitorData && { 'x-goog-visitor-id': ytConfig.visitorData }),
|
||||
'DNT': '1',
|
||||
...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore })
|
||||
},
|
||||
body: JSON.stringify(nextRequestBody)
|
||||
});
|
||||
|
||||
if (!nextResponse.ok) {
|
||||
throw new Error(`Failed to fetch next API data: HTTP ${nextResponse.status}`);
|
||||
}
|
||||
|
||||
const nextData = await nextResponse.json() as unknown;
|
||||
transcriptLogger.debug(`WEB ScrapeCreators Step 1 completed: received ${JSON.stringify(nextData).length} characters`);
|
||||
|
||||
// Extract metadata from nextData response
|
||||
const metadata = this.extractMetadataFromNextData(nextData);
|
||||
transcriptLogger.debug(`WEB ScrapeCreators metadata: title="${metadata.title || 'N/A'}", author="${metadata.author || 'N/A'}"`);
|
||||
|
||||
// Find transcript endpoint parameters
|
||||
const transcriptParams = this.findTranscriptEndpoint(nextData);
|
||||
if (!transcriptParams) {
|
||||
if (isRecord(nextData)) {
|
||||
transcriptLogger.debug(`Next API response keys: ${Object.keys(nextData).join(', ')}`);
|
||||
}
|
||||
throw new Error(`No transcript parameters found in YouTube API response for video ${videoId}`);
|
||||
}
|
||||
|
||||
transcriptLogger.debug(`WEB ScrapeCreators: Found transcript parameters: ${transcriptParams.substring(0, 100)}...`);
|
||||
|
||||
// Step 2: Request the actual transcript using the parameters
|
||||
const getTranscriptUrl = `https://www.youtube.com/youtubei/v1/get_transcript?prettyPrint=false`;
|
||||
const transcriptRequestBody = {
|
||||
context: {
|
||||
client: {
|
||||
clientName: "WEB",
|
||||
clientVersion: ytConfig.clientVersion,
|
||||
...(ytConfig.visitorData && { visitorData: ytConfig.visitorData })
|
||||
}
|
||||
},
|
||||
params: transcriptParams
|
||||
};
|
||||
|
||||
transcriptLogger.debug(`WEB ScrapeCreators Step 2: Requesting transcript from ${getTranscriptUrl}`);
|
||||
|
||||
const transcriptResponse = await obsidianFetch(getTranscriptUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'User-Agent': YouTubeTranscriptExtractor.USER_AGENT,
|
||||
'Accept': 'application/json',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Content-Type': 'application/json',
|
||||
'Referer': watchUrl,
|
||||
'Origin': 'https://www.youtube.com',
|
||||
'x-youtube-client-name': '1',
|
||||
'x-youtube-client-version': ytConfig.clientVersion,
|
||||
...(ytConfig.visitorData && { 'x-goog-visitor-id': ytConfig.visitorData }),
|
||||
'DNT': '1',
|
||||
...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore })
|
||||
},
|
||||
body: JSON.stringify(transcriptRequestBody)
|
||||
});
|
||||
|
||||
if (!transcriptResponse.ok) {
|
||||
let errorDetail = '';
|
||||
try { errorDetail = (await transcriptResponse.text()).substring(0, 500); } catch { /* ignore */ }
|
||||
transcriptLogger.error(`get_transcript failed with HTTP ${transcriptResponse.status}: ${errorDetail}`);
|
||||
throw new Error(`Failed to fetch transcript: HTTP ${transcriptResponse.status}`);
|
||||
}
|
||||
|
||||
const transcriptData = await transcriptResponse.json() as UnknownRecord;
|
||||
transcriptLogger.debug(`WEB ScrapeCreators Step 2 completed: received ${JSON.stringify(transcriptData).length} characters`);
|
||||
|
||||
const segments = this.parseScrapeCreatorsTranscript(transcriptData);
|
||||
transcriptLogger.debug(`WEB ScrapeCreators method succeeded with ${segments.length} segments`);
|
||||
return { segments, metadata };
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract metadata from a /next API response.
|
||||
*/
|
||||
private static extractMetadataFromNextData(nextData: unknown): TranscriptMetadata {
|
||||
const metadata: TranscriptMetadata = {};
|
||||
if (!isRecord(nextData)) return metadata;
|
||||
|
||||
const typedNextData = nextData as {
|
||||
playerOverlays?: {
|
||||
playerOverlayRenderer?: {
|
||||
videoDetails?: {
|
||||
playerOverlayVideoDetailsRenderer?: {
|
||||
title?: { simpleText?: string };
|
||||
subtitle?: { runs?: Array<{ text?: string }> };
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
contents?: {
|
||||
twoColumnWatchNextResults?: {
|
||||
results?: {
|
||||
results?: {
|
||||
contents?: Array<{
|
||||
videoPrimaryInfoRenderer?: { title?: { runs?: Array<{ text?: string }> } };
|
||||
videoSecondaryInfoRenderer?: { owner?: { videoOwnerRenderer?: { title?: { runs?: Array<{ text?: string }> } } } };
|
||||
}>;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
microformat?: {
|
||||
playerMicroformatRenderer?: {
|
||||
title?: { simpleText?: string };
|
||||
ownerChannelName?: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
try {
|
||||
// Method 1: From playerOverlays (most reliable)
|
||||
const playerOverlayDetails = typedNextData.playerOverlays?.playerOverlayRenderer?.videoDetails?.playerOverlayVideoDetailsRenderer;
|
||||
if (playerOverlayDetails) {
|
||||
metadata.title = playerOverlayDetails.title?.simpleText;
|
||||
metadata.author = playerOverlayDetails.subtitle?.runs?.[0]?.text;
|
||||
}
|
||||
|
||||
// Method 2: From contents (fallback)
|
||||
if (!metadata.title || !metadata.author) {
|
||||
const contents = typedNextData.contents?.twoColumnWatchNextResults?.results?.results?.contents;
|
||||
if (Array.isArray(contents)) {
|
||||
const primaryInfo = contents.find((c) => c.videoPrimaryInfoRenderer);
|
||||
if (primaryInfo && !metadata.title) {
|
||||
metadata.title = primaryInfo.videoPrimaryInfoRenderer?.title?.runs?.[0]?.text;
|
||||
}
|
||||
const secondaryInfo = contents.find((c) => c.videoSecondaryInfoRenderer);
|
||||
if (secondaryInfo && !metadata.author) {
|
||||
metadata.author = secondaryInfo.videoSecondaryInfoRenderer?.owner?.videoOwnerRenderer?.title?.runs?.[0]?.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Method 3: From microformat (additional fallback)
|
||||
if (!metadata.title || !metadata.author) {
|
||||
const microformat = typedNextData.microformat?.playerMicroformatRenderer;
|
||||
if (microformat) {
|
||||
if (!metadata.title && microformat.title?.simpleText) metadata.title = microformat.title.simpleText;
|
||||
if (!metadata.author && microformat.ownerChannelName) metadata.author = microformat.ownerChannelName;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
transcriptLogger.debug(`Error extracting metadata from nextData: ${getSafeErrorMessage(error)}`);
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively search for getTranscriptEndpoint.params in a /next API response.
|
||||
*/
|
||||
private static findTranscriptEndpoint(obj: unknown): string | null {
|
||||
if (!isRecord(obj)) return null;
|
||||
|
||||
// Check direct getTranscriptEndpoint
|
||||
const endpoint = obj.getTranscriptEndpoint as { params?: unknown } | undefined;
|
||||
if (endpoint && isString(endpoint.params)) return endpoint.params;
|
||||
|
||||
// Check continuationEndpoint.getTranscriptEndpoint
|
||||
const contEndpoint = (obj.continuationEndpoint as {
|
||||
getTranscriptEndpoint?: { params?: unknown }
|
||||
} | undefined)?.getTranscriptEndpoint;
|
||||
if (contEndpoint && isString(contEndpoint.params)) return contEndpoint.params;
|
||||
|
||||
for (const value of Object.values(obj)) {
|
||||
const result = this.findTranscriptEndpoint(value);
|
||||
if (result) return result;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse transcript data from /get_transcript API response into segments.
|
||||
*/
|
||||
private static parseScrapeCreatorsTranscript(transcriptData: UnknownRecord): TranscriptSegment[] {
|
||||
// Look for transcript text in the response with enhanced search
|
||||
const findTranscriptText = (obj: unknown, depth = 0): unknown => {
|
||||
if (depth > 10) return null;
|
||||
|
||||
if (isRecord(obj)) {
|
||||
// Check for the main transcript structure
|
||||
const transcriptBody = (obj['transcriptBody'] as { transcriptBodyRenderer?: { cueGroups?: unknown } } | undefined)?.transcriptBodyRenderer?.cueGroups;
|
||||
if (transcriptBody) return transcriptBody;
|
||||
|
||||
// Check for alternative transcript structures
|
||||
const directCueGroups = obj['cueGroups'];
|
||||
if (directCueGroups && Array.isArray(directCueGroups)) return directCueGroups;
|
||||
|
||||
// Check for updateEngagementPanelAction structure
|
||||
const initialSegments = (obj['updateEngagementPanelAction'] as {
|
||||
content?: {
|
||||
transcriptRenderer?: {
|
||||
content?: {
|
||||
transcriptSearchPanelRenderer?: {
|
||||
body?: {
|
||||
transcriptSegmentListRenderer?: {
|
||||
initialSegments?: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
} | undefined)?.content?.transcriptRenderer?.content?.transcriptSearchPanelRenderer?.body?.transcriptSegmentListRenderer?.initialSegments;
|
||||
if (initialSegments) return initialSegments;
|
||||
|
||||
// Search through actions array
|
||||
const actions = obj['actions'];
|
||||
if (actions && Array.isArray(actions)) {
|
||||
for (let i = 0; i < actions.length; i++) {
|
||||
const result = findTranscriptText(actions[i], depth + 1);
|
||||
if (result) return result;
|
||||
}
|
||||
}
|
||||
|
||||
// Search through all object properties
|
||||
for (const [, value] of Object.entries(obj)) {
|
||||
if (typeof value === 'object') {
|
||||
const result = findTranscriptText(value, depth + 1);
|
||||
if (result) return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const cueGroups = findTranscriptText(transcriptData);
|
||||
|
||||
if (!cueGroups || !Array.isArray(cueGroups)) {
|
||||
transcriptLogger.error(`No cueGroups found in transcript response`);
|
||||
transcriptLogger.debug(`Transcript response keys: ${Object.keys(transcriptData).join(', ')}`);
|
||||
|
||||
const transcriptActions = transcriptData.actions;
|
||||
if (Array.isArray(transcriptActions)) {
|
||||
transcriptLogger.debug(`Found ${transcriptActions.length} actions, examining structure...`);
|
||||
transcriptActions.forEach((action, i: number) => {
|
||||
if (isRecord(action)) {
|
||||
transcriptLogger.debug(`Action ${i} keys: ${Object.keys(action).join(', ')}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error(`No transcript cueGroups found in YouTube API response`);
|
||||
}
|
||||
|
||||
transcriptLogger.debug(`Found ${cueGroups.length} cue groups in transcript`);
|
||||
|
||||
// Convert cue groups to segments - handle multiple formats
|
||||
const segments = cueGroups.map<TranscriptSegment | null>((cueGroup: unknown, index: number) => {
|
||||
// Traditional cueGroup format
|
||||
const cue = (cueGroup as {
|
||||
transcriptCueGroupRenderer?: {
|
||||
cues?: Array<{
|
||||
transcriptCueRenderer?: {
|
||||
cue?: { simpleText?: string };
|
||||
startOffsetMs?: string;
|
||||
durationMs?: string;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
}).transcriptCueGroupRenderer?.cues?.[0]?.transcriptCueRenderer;
|
||||
if (cue) {
|
||||
return {
|
||||
text: (cue.cue?.simpleText || '').trim(),
|
||||
start: parseInt(cue.startOffsetMs || '0') / 1000,
|
||||
duration: parseInt(cue.durationMs || '0') / 1000
|
||||
};
|
||||
}
|
||||
|
||||
// Alternative format: transcriptSegmentRenderer (from initialSegments)
|
||||
const segmentRenderer = (cueGroup as {
|
||||
transcriptSegmentRenderer?: {
|
||||
snippet?: { runs?: Array<{ text?: string }> };
|
||||
startMs?: string;
|
||||
endMs?: string;
|
||||
};
|
||||
}).transcriptSegmentRenderer;
|
||||
if (segmentRenderer) {
|
||||
const startMs = parseInt(segmentRenderer.startMs || '0');
|
||||
const endMs = parseInt(segmentRenderer.endMs || '0');
|
||||
return {
|
||||
text: (segmentRenderer.snippet?.runs?.[0]?.text || '').trim(),
|
||||
start: startMs / 1000,
|
||||
duration: (endMs - startMs) / 1000
|
||||
};
|
||||
}
|
||||
|
||||
if (index < 3) {
|
||||
const info = isRecord(cueGroup) ? Object.keys(cueGroup).join(', ') : 'non-object';
|
||||
transcriptLogger.debug(`Unknown cueGroup format at index ${index}: ${info}`);
|
||||
}
|
||||
return null;
|
||||
}).filter((segment): segment is TranscriptSegment => segment !== null && !!segment.text);
|
||||
|
||||
if (segments.length === 0) {
|
||||
throw new Error(`No valid transcript segments found in YouTube API response`);
|
||||
}
|
||||
|
||||
return segments;
|
||||
}
|
||||
|
||||
/**
|
||||
* MWEB client fallback: Uses YouTube's mobile-web client, which appears to have less strict
|
||||
* POT (Proof-of-Origin Token) enforcement than the WEB client for caption fetches.
|
||||
* clientName=MWEB, clientId=2
|
||||
*/
|
||||
private static async fetchViaPlayerApiMWEB(videoId: string, options: TranscriptOptions): Promise<TranscriptResult> {
|
||||
const lang = options.lang || 'en';
|
||||
const country = options.country || 'US';
|
||||
|
||||
const ytConfig = await this.getYouTubeConfig(videoId);
|
||||
transcriptLogger.debug('Player API (MWEB): attempting with MWEB client');
|
||||
|
||||
const playerUrl = `https://www.youtube.com/youtubei/v1/player?key=${ytConfig.apiKey}`;
|
||||
const playerBody = {
|
||||
context: {
|
||||
client: {
|
||||
clientName: 'MWEB',
|
||||
clientVersion: '2.20260408.02.00',
|
||||
hl: lang,
|
||||
gl: country,
|
||||
...(ytConfig.visitorData && { visitorData: ytConfig.visitorData })
|
||||
}
|
||||
},
|
||||
videoId
|
||||
};
|
||||
|
||||
transcriptLogger.debug(`Player API (MWEB): requesting caption tracks for ${videoId}`);
|
||||
|
||||
const playerResponse = await obsidianFetch(playerUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
|
||||
'Accept': 'application/json',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Content-Type': 'application/json',
|
||||
'Origin': 'https://m.youtube.com',
|
||||
'Referer': `https://m.youtube.com/watch?v=${videoId}`,
|
||||
'X-Youtube-Client-Name': '2', // 2 = MWEB
|
||||
'X-Youtube-Client-Version': '2.20260408.02.00',
|
||||
...(ytConfig.visitorData && { 'x-goog-visitor-id': ytConfig.visitorData }),
|
||||
...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore })
|
||||
},
|
||||
body: JSON.stringify(playerBody)
|
||||
});
|
||||
|
||||
if (!playerResponse.ok) {
|
||||
throw new Error(`Player API (MWEB) error: HTTP ${playerResponse.status}`);
|
||||
}
|
||||
|
||||
const playerData = await playerResponse.json() as UnknownRecord;
|
||||
|
||||
const videoDetails = (playerData as {
|
||||
videoDetails?: { title?: string; author?: string };
|
||||
}).videoDetails;
|
||||
|
||||
const metadata: TranscriptMetadata = {
|
||||
title: videoDetails?.title,
|
||||
author: videoDetails?.author
|
||||
};
|
||||
|
||||
const captions = (playerData as {
|
||||
captions?: {
|
||||
playerCaptionsTracklistRenderer?: {
|
||||
captionTracks?: Array<{ baseUrl?: string; languageCode?: string }>;
|
||||
};
|
||||
};
|
||||
}).captions?.playerCaptionsTracklistRenderer?.captionTracks;
|
||||
|
||||
if (!captions || !Array.isArray(captions) || captions.length === 0) {
|
||||
throw new Error('No caption tracks available from Player API (MWEB)');
|
||||
}
|
||||
|
||||
transcriptLogger.debug(`Player API (MWEB): found ${captions.length} caption tracks`);
|
||||
|
||||
const preferredTrack = captions.find(track => track.languageCode?.startsWith(lang)) || captions[0];
|
||||
if (!preferredTrack?.baseUrl) {
|
||||
throw new Error('Caption track missing baseUrl');
|
||||
}
|
||||
|
||||
// Use mobile Chrome UA consistently for the caption fetch
|
||||
const mwebUA = 'Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36';
|
||||
const segments = await this.fetchCaptionTrack(preferredTrack.baseUrl, undefined, mwebUA);
|
||||
transcriptLogger.debug(`Player API (MWEB): successfully extracted ${segments.length} segments`);
|
||||
return { segments, metadata };
|
||||
}
|
||||
|
||||
/**
|
||||
* ANDROID client fallback: Uses ANDROID client which bypasses WEB restrictions.
|
||||
* Reference: https://github.com/LuanRT/YouTube.js and ScrapeCreators research
|
||||
*/
|
||||
private static async fetchViaPlayerApiAndroid(videoId: string, options: TranscriptOptions): Promise<TranscriptResult> {
|
||||
const lang = options.lang || 'en';
|
||||
const country = options.country || 'US';
|
||||
|
||||
// Get API key from config
|
||||
const ytConfig = await this.getYouTubeConfig(videoId);
|
||||
transcriptLogger.debug('Player API (ANDROID): attempting with ANDROID client');
|
||||
|
||||
// ANDROID client configuration - bypasses WEB restrictions
|
||||
// Version 20.10.38 from youtube-transcript-api (actively maintained Python library)
|
||||
const playerUrl = `https://www.youtube.com/youtubei/v1/player?key=${ytConfig.apiKey}`;
|
||||
const playerBody = {
|
||||
context: {
|
||||
client: {
|
||||
clientName: 'ANDROID',
|
||||
clientVersion: '20.10.38',
|
||||
androidSdkVersion: 30,
|
||||
hl: lang,
|
||||
gl: country
|
||||
}
|
||||
},
|
||||
videoId
|
||||
};
|
||||
|
||||
transcriptLogger.debug(`Player API (ANDROID): requesting caption tracks for ${videoId}`);
|
||||
|
||||
const playerResponse = await obsidianFetch(playerUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'User-Agent': 'com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip',
|
||||
'Accept': 'application/json',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Content-Type': 'application/json',
|
||||
'Origin': 'https://www.youtube.com',
|
||||
'X-Youtube-Client-Name': '3', // 3 = ANDROID
|
||||
'X-Youtube-Client-Version': '20.10.38',
|
||||
...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore })
|
||||
},
|
||||
body: JSON.stringify(playerBody)
|
||||
});
|
||||
|
||||
if (!playerResponse.ok) {
|
||||
throw new Error(`Player API (ANDROID) error: HTTP ${playerResponse.status}`);
|
||||
}
|
||||
|
||||
const playerData = await playerResponse.json() as UnknownRecord;
|
||||
|
||||
const videoDetails = (playerData as {
|
||||
videoDetails?: { title?: string; author?: string };
|
||||
}).videoDetails;
|
||||
|
||||
const metadata: TranscriptMetadata = {
|
||||
title: videoDetails?.title,
|
||||
author: videoDetails?.author
|
||||
};
|
||||
|
||||
const captions = (playerData as {
|
||||
captions?: {
|
||||
playerCaptionsTracklistRenderer?: {
|
||||
captionTracks?: Array<{ baseUrl?: string; languageCode?: string }>;
|
||||
};
|
||||
};
|
||||
}).captions?.playerCaptionsTracklistRenderer?.captionTracks;
|
||||
|
||||
if (!captions || !Array.isArray(captions) || captions.length === 0) {
|
||||
throw new Error('No caption tracks available from Player API (ANDROID)');
|
||||
}
|
||||
|
||||
transcriptLogger.debug(`Player API (ANDROID): found ${captions.length} caption tracks`);
|
||||
|
||||
// Prefer matching language, otherwise first track
|
||||
const preferredTrack = captions.find(track => track.languageCode?.startsWith(lang)) || captions[0];
|
||||
if (!preferredTrack?.baseUrl) {
|
||||
throw new Error('Caption track missing baseUrl');
|
||||
}
|
||||
|
||||
// Pass the Android UA for the caption fetch — keeps a consistent client identity
|
||||
// end-to-end, which may help bypass POT requirements on the timedtext endpoint
|
||||
const androidUA = 'com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip';
|
||||
const segments = await this.fetchCaptionTrack(preferredTrack.baseUrl, undefined, androidUA);
|
||||
transcriptLogger.debug(`Player API (ANDROID): successfully extracted ${segments.length} segments`);
|
||||
return { segments, metadata };
|
||||
}
|
||||
|
||||
/**
|
||||
* Supadata API fallback: Uses Supadata's transcript API as a third fallback.
|
||||
* Requires an API key from supadata.ai.
|
||||
|
|
@ -1457,7 +691,6 @@ export class YouTubeTranscriptExtractor {
|
|||
'Origin': 'https://www.youtube.com',
|
||||
'Referer': 'https://www.youtube.com/',
|
||||
'DNT': '1',
|
||||
...(YouTubeTranscriptExtractor.cookieStore && { 'Cookie': YouTubeTranscriptExtractor.cookieStore })
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue