mirror of
https://github.com/core-hn/pseudobsidian-ization.git
synced 2026-07-22 06:57:06 +00:00
feat: export EAF (ELAN) depuis Markdown noScribe
- markdownToEaf() — génère un .eaf FORMAT 3.0 avec tiers par locuteur, TIME_SLOTs dédupliqués+triés, MEDIA_DESCRIPTOR optionnel, escape XML - Commande export-as-eaf + exportCurrentFileAsEaf() dans main.ts - Bouton dans l'onglet Exports (visible format vtt/html uniquement) - Clés i18n fr/en/es/ru - extractWordData() : toutes les cues incluses (plus seulement celles avec timestamps), wordTimestampRate par cue ; fileSequentialityRate() ajouté - 15 tests unitaires EafExporter (165 tests total, 0 échec)
This commit is contained in:
parent
928ff8d42b
commit
4ceb2a6a4f
10 changed files with 344 additions and 17 deletions
|
|
@ -202,6 +202,7 @@ Objectif : permettre la détection et le remplacement automatiques des entités
|
||||||
- [x] `MappingScanReviewModal` : modale de révision pour le scan par règles existantes
|
- [x] `MappingScanReviewModal` : modale de révision pour le scan par règles existantes
|
||||||
- [x] Scan par dictionnaire accessible depuis l'onglet Dictionnaires et commande Ctrl+P
|
- [x] Scan par dictionnaire accessible depuis l'onglet Dictionnaires et commande Ctrl+P
|
||||||
- [x] Suppression onglet "Candidats" — scan règles → Mappings · scan NER → NER
|
- [x] Suppression onglet "Candidats" — scan règles → Mappings · scan NER → NER
|
||||||
|
- [ ] Test de remplacement NER par Gliner PII (https://huggingface.co/nvidia/gliner-PII) ou EU PII (https://huggingface.co/bardsai/eu-pii-anonimization-multilang)
|
||||||
|
|
||||||
|
|
||||||
**Testable (v0.1.3) :** installer le dictionnaire communes → scanner un fichier → modale de révision → décocher les faux positifs → créer les règles → pseudonymiser.
|
**Testable (v0.1.3) :** installer le dictionnaire communes → scanner un fichier → modale de révision → décocher les faux positifs → créer les règles → pseudonymiser.
|
||||||
|
|
|
||||||
|
|
@ -90,6 +90,7 @@
|
||||||
"notice.wordsJsonMissing": "Timestamp file not found: {0}.words.json",
|
"notice.wordsJsonMissing": "Timestamp file not found: {0}.words.json",
|
||||||
"notice.vttMismatch": "Warning: Markdown cue count and words.json cue count differ — exported VTT may be incomplete.",
|
"notice.vttMismatch": "Warning: Markdown cue count and words.json cue count differ — exported VTT may be incomplete.",
|
||||||
"notice.vttExported": "VTT exported: {0}",
|
"notice.vttExported": "VTT exported: {0}",
|
||||||
|
"notice.eafExported": "EAF exported: {0}",
|
||||||
|
|
||||||
"command.organizeCorpus": "Organize corpus",
|
"command.organizeCorpus": "Organize corpus",
|
||||||
"command.addTranscription": "Add a transcription",
|
"command.addTranscription": "Add a transcription",
|
||||||
|
|
@ -221,10 +222,13 @@
|
||||||
"panel.exports.exportCha.desc": "Rebuilds the pseudonymized CHAT file from the Markdown.",
|
"panel.exports.exportCha.desc": "Rebuilds the pseudonymized CHAT file from the Markdown.",
|
||||||
"panel.exports.exportHtml": "Re-export as HTML",
|
"panel.exports.exportHtml": "Re-export as HTML",
|
||||||
"panel.exports.exportHtml.desc": "Rebuilds the pseudonymized noScribe HTML file from the Markdown and word timestamps (.words.json).",
|
"panel.exports.exportHtml.desc": "Rebuilds the pseudonymized noScribe HTML file from the Markdown and word timestamps (.words.json).",
|
||||||
|
"panel.exports.exportEaf": "Export as EAF (ELAN)",
|
||||||
|
"panel.exports.exportEaf.desc": "Generates an ELAN file (.eaf) with one tier per speaker, from the Markdown and word timestamps (.words.json).",
|
||||||
"notice.htmlExported": "HTML exported: {0}",
|
"notice.htmlExported": "HTML exported: {0}",
|
||||||
"command.exportAsSrt": "Export as SRT",
|
"command.exportAsSrt": "Export as SRT",
|
||||||
"command.exportAsCha": "Export as CHAT",
|
"command.exportAsCha": "Export as CHAT",
|
||||||
"command.exportAsHtml": "Export as HTML",
|
"command.exportAsHtml": "Export as HTML",
|
||||||
|
"command.exportAsEaf": "Export as EAF (ELAN)",
|
||||||
|
|
||||||
"panel.ner.scanBtn": "Identify candidates",
|
"panel.ner.scanBtn": "Identify candidates",
|
||||||
"panel.ner.hint": "Settings for the automatic named entity detection scanner.",
|
"panel.ner.hint": "Settings for the automatic named entity detection scanner.",
|
||||||
|
|
|
||||||
|
|
@ -84,6 +84,7 @@
|
||||||
"notice.wordsJsonMissing": "Archivo de marcas de tiempo no encontrado: {0}.words.json",
|
"notice.wordsJsonMissing": "Archivo de marcas de tiempo no encontrado: {0}.words.json",
|
||||||
"notice.vttMismatch": "Aviso: el número de cues del Markdown y del words.json difiere — el VTT exportado puede estar incompleto.",
|
"notice.vttMismatch": "Aviso: el número de cues del Markdown y del words.json difiere — el VTT exportado puede estar incompleto.",
|
||||||
"notice.vttExported": "VTT exportado: {0}",
|
"notice.vttExported": "VTT exportado: {0}",
|
||||||
|
"notice.eafExported": "EAF exportado: {0}",
|
||||||
|
|
||||||
"command.organizeCorpus": "Organizar corpus",
|
"command.organizeCorpus": "Organizar corpus",
|
||||||
"command.addTranscription": "Añadir una transcripción",
|
"command.addTranscription": "Añadir una transcripción",
|
||||||
|
|
@ -97,6 +98,7 @@
|
||||||
"command.exportAsSrt": "Exportar como SRT",
|
"command.exportAsSrt": "Exportar como SRT",
|
||||||
"command.exportAsCha": "Exportar como CHAT",
|
"command.exportAsCha": "Exportar como CHAT",
|
||||||
"command.exportAsHtml": "Exportar como HTML",
|
"command.exportAsHtml": "Exportar como HTML",
|
||||||
|
"command.exportAsEaf": "Exportar como EAF (ELAN)",
|
||||||
"command.openPanel": "Pseudonimización: abrir panel",
|
"command.openPanel": "Pseudonimización: abrir panel",
|
||||||
|
|
||||||
"contextMenu.declareException": "Declarar «{0}» como excepción aquí",
|
"contextMenu.declareException": "Declarar «{0}» como excepción aquí",
|
||||||
|
|
@ -223,6 +225,8 @@
|
||||||
"panel.exports.exportCha.desc": "Reconstruye el archivo CHAT pseudonimizado desde el Markdown.",
|
"panel.exports.exportCha.desc": "Reconstruye el archivo CHAT pseudonimizado desde el Markdown.",
|
||||||
"panel.exports.exportHtml": "Reexportar como HTML",
|
"panel.exports.exportHtml": "Reexportar como HTML",
|
||||||
"panel.exports.exportHtml.desc": "Reconstruye el archivo HTML noScribe pseudonimizado desde el Markdown y las marcas de tiempo (.words.json).",
|
"panel.exports.exportHtml.desc": "Reconstruye el archivo HTML noScribe pseudonimizado desde el Markdown y las marcas de tiempo (.words.json).",
|
||||||
|
"panel.exports.exportEaf": "Exportar como EAF (ELAN)",
|
||||||
|
"panel.exports.exportEaf.desc": "Genera un archivo ELAN (.eaf) con un tier por locutor, a partir del Markdown y las marcas de tiempo (.words.json).",
|
||||||
"notice.htmlExported": "HTML exportado: {0}",
|
"notice.htmlExported": "HTML exportado: {0}",
|
||||||
|
|
||||||
"panel.ner.scanBtn": "Identificar candidatos",
|
"panel.ner.scanBtn": "Identificar candidatos",
|
||||||
|
|
|
||||||
|
|
@ -90,6 +90,7 @@
|
||||||
"notice.wordsJsonMissing": "Fichier de timestamps introuvable : {0}.words.json",
|
"notice.wordsJsonMissing": "Fichier de timestamps introuvable : {0}.words.json",
|
||||||
"notice.vttMismatch": "Attention : le nombre de cues du Markdown et du words.json diffère — le VTT exporté peut être incomplet.",
|
"notice.vttMismatch": "Attention : le nombre de cues du Markdown et du words.json diffère — le VTT exporté peut être incomplet.",
|
||||||
"notice.vttExported": "VTT exporté : {0}",
|
"notice.vttExported": "VTT exporté : {0}",
|
||||||
|
"notice.eafExported": "EAF exporté : {0}",
|
||||||
|
|
||||||
"command.organizeCorpus": "Organiser le corpus",
|
"command.organizeCorpus": "Organiser le corpus",
|
||||||
"command.addTranscription": "Ajouter une transcription",
|
"command.addTranscription": "Ajouter une transcription",
|
||||||
|
|
@ -221,10 +222,13 @@
|
||||||
"panel.exports.exportCha.desc": "Reconstruit le fichier CHAT pseudonymisé à partir du Markdown.",
|
"panel.exports.exportCha.desc": "Reconstruit le fichier CHAT pseudonymisé à partir du Markdown.",
|
||||||
"panel.exports.exportHtml": "Re-exporter en HTML",
|
"panel.exports.exportHtml": "Re-exporter en HTML",
|
||||||
"panel.exports.exportHtml.desc": "Reconstruit le fichier HTML noScribe pseudonymisé à partir du Markdown et des timestamps (.words.json).",
|
"panel.exports.exportHtml.desc": "Reconstruit le fichier HTML noScribe pseudonymisé à partir du Markdown et des timestamps (.words.json).",
|
||||||
|
"panel.exports.exportEaf": "Exporter en EAF (ELAN)",
|
||||||
|
"panel.exports.exportEaf.desc": "Génère un fichier ELAN (.eaf) avec un tier par locuteur, à partir du Markdown et des timestamps (.words.json).",
|
||||||
"notice.htmlExported": "HTML exporté : {0}",
|
"notice.htmlExported": "HTML exporté : {0}",
|
||||||
"command.exportAsSrt": "Exporter en SRT",
|
"command.exportAsSrt": "Exporter en SRT",
|
||||||
"command.exportAsCha": "Exporter en CHAT",
|
"command.exportAsCha": "Exporter en CHAT",
|
||||||
"command.exportAsHtml": "Exporter en HTML",
|
"command.exportAsHtml": "Exporter en HTML",
|
||||||
|
"command.exportAsEaf": "Exporter en EAF (ELAN)",
|
||||||
|
|
||||||
"panel.ner.scanBtn": "Identifier des candidats",
|
"panel.ner.scanBtn": "Identifier des candidats",
|
||||||
"panel.ner.hint": "Paramètres du scanner de détection automatique des entités nommées.",
|
"panel.ner.hint": "Paramètres du scanner de détection automatique des entités nommées.",
|
||||||
|
|
|
||||||
|
|
@ -84,6 +84,7 @@
|
||||||
"notice.wordsJsonMissing": "Файл временных меток не найден: {0}.words.json",
|
"notice.wordsJsonMissing": "Файл временных меток не найден: {0}.words.json",
|
||||||
"notice.vttMismatch": "Предупреждение: количество реплик в Markdown и words.json различается — экспортированный VTT может быть неполным.",
|
"notice.vttMismatch": "Предупреждение: количество реплик в Markdown и words.json различается — экспортированный VTT может быть неполным.",
|
||||||
"notice.vttExported": "VTT экспортирован: {0}",
|
"notice.vttExported": "VTT экспортирован: {0}",
|
||||||
|
"notice.eafExported": "EAF экспортирован: {0}",
|
||||||
|
|
||||||
"command.organizeCorpus": "Организовать корпус",
|
"command.organizeCorpus": "Организовать корпус",
|
||||||
"command.addTranscription": "Добавить транскрипцию",
|
"command.addTranscription": "Добавить транскрипцию",
|
||||||
|
|
@ -97,6 +98,7 @@
|
||||||
"command.exportAsSrt": "Экспортировать как SRT",
|
"command.exportAsSrt": "Экспортировать как SRT",
|
||||||
"command.exportAsCha": "Экспортировать как CHAT",
|
"command.exportAsCha": "Экспортировать как CHAT",
|
||||||
"command.exportAsHtml": "Экспортировать как HTML",
|
"command.exportAsHtml": "Экспортировать как HTML",
|
||||||
|
"command.exportAsEaf": "Экспортировать как EAF (ELAN)",
|
||||||
"command.openPanel": "Псевдонимизация: открыть панель",
|
"command.openPanel": "Псевдонимизация: открыть панель",
|
||||||
|
|
||||||
"contextMenu.declareException": "Объявить «{0}» исключением здесь",
|
"contextMenu.declareException": "Объявить «{0}» исключением здесь",
|
||||||
|
|
@ -223,6 +225,8 @@
|
||||||
"panel.exports.exportCha.desc": "Восстанавливает псевдонимизированный CHAT из Markdown.",
|
"panel.exports.exportCha.desc": "Восстанавливает псевдонимизированный CHAT из Markdown.",
|
||||||
"panel.exports.exportHtml": "Повторно экспортировать как HTML",
|
"panel.exports.exportHtml": "Повторно экспортировать как HTML",
|
||||||
"panel.exports.exportHtml.desc": "Восстанавливает псевдонимизированный HTML noScribe из Markdown и временных меток (.words.json).",
|
"panel.exports.exportHtml.desc": "Восстанавливает псевдонимизированный HTML noScribe из Markdown и временных меток (.words.json).",
|
||||||
|
"panel.exports.exportEaf": "Экспортировать как EAF (ELAN)",
|
||||||
|
"panel.exports.exportEaf.desc": "Создаёт файл ELAN (.eaf) с одним tier на каждого говорящего, на основе Markdown и временных меток (.words.json).",
|
||||||
"notice.htmlExported": "HTML экспортирован: {0}",
|
"notice.htmlExported": "HTML экспортирован: {0}",
|
||||||
|
|
||||||
"panel.ner.scanBtn": "Определить кандидатов",
|
"panel.ner.scanBtn": "Определить кандидатов",
|
||||||
|
|
|
||||||
47
src/main.ts
47
src/main.ts
|
|
@ -22,7 +22,7 @@ import { ChatParser } from './parsers/ChatParser';
|
||||||
import { VttParser } from './parsers/VttParser';
|
import { VttParser } from './parsers/VttParser';
|
||||||
import { NoScribeHtmlParser } from './parsers/NoScribeHtmlParser';
|
import { NoScribeHtmlParser } from './parsers/NoScribeHtmlParser';
|
||||||
import { NoScribeVttParser } from './parsers/NoScribeVttParser';
|
import { NoScribeVttParser } from './parsers/NoScribeVttParser';
|
||||||
import { srtToMarkdown, chatToMarkdown, vttToMarkdown, noScribeHtmlToMarkdown, extractWordData, markdownToVtt, markdownToSrt, markdownToCha, markdownToNoScribeHtml, type VttCueData } from './parsers/TranscriptConverter';
|
import { srtToMarkdown, chatToMarkdown, vttToMarkdown, noScribeHtmlToMarkdown, extractWordData, markdownToVtt, markdownToSrt, markdownToCha, markdownToNoScribeHtml, markdownToEaf, type VttCueData } from './parsers/TranscriptConverter';
|
||||||
import { MappingStore } from './mappings/MappingStore';
|
import { MappingStore } from './mappings/MappingStore';
|
||||||
import { ScopeResolver } from './mappings/ScopeResolver';
|
import { ScopeResolver } from './mappings/ScopeResolver';
|
||||||
import { PseudonymizationEngine } from './pseudonymizer/PseudonymizationEngine';
|
import { PseudonymizationEngine } from './pseudonymizer/PseudonymizationEngine';
|
||||||
|
|
@ -189,6 +189,12 @@ export default class PseudObsPlugin extends Plugin {
|
||||||
callback: () => void this.exportCurrentFileAsHtml(),
|
callback: () => void this.exportCurrentFileAsHtml(),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
this.addCommand({
|
||||||
|
id: 'export-as-eaf',
|
||||||
|
name: t('command.exportAsEaf'),
|
||||||
|
callback: () => void this.exportCurrentFileAsEaf(),
|
||||||
|
});
|
||||||
|
|
||||||
this.addCommand({
|
this.addCommand({
|
||||||
id: 'pseudonymize-selection',
|
id: 'pseudonymize-selection',
|
||||||
name: t('command.pseudonymizeSelection'),
|
name: t('command.pseudonymizeSelection'),
|
||||||
|
|
@ -1392,6 +1398,45 @@ export default class PseudObsPlugin extends Plugin {
|
||||||
await this.writeExport(file, 'html', html, 'notice.htmlExported');
|
await this.writeExport(file, 'html', html, 'notice.htmlExported');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exporte le fichier Markdown noScribe actif au format EAF (ELAN).
|
||||||
|
* Lit le .words.json correspondant pour les timestamps précis.
|
||||||
|
*/
|
||||||
|
async exportCurrentFileAsEaf(): Promise<void> {
|
||||||
|
const file = this.getActiveOrLastFile();
|
||||||
|
if (!file || file.extension !== 'md') {
|
||||||
|
new Notice(t('notice.noActiveFile'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const content = await this.app.vault.read(file);
|
||||||
|
const formatMatch = /^pseudobs-format:\s*(\w+)/m.exec(content);
|
||||||
|
const format = formatMatch?.[1];
|
||||||
|
if (format !== 'vtt' && format !== 'html') {
|
||||||
|
new Notice(t('notice.notNoScribeFormat'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const rawBasename = file.basename.replace(/\.pseudonymized$/, '');
|
||||||
|
const wordsJson = await this.findWordsJson(rawBasename);
|
||||||
|
if (!wordsJson) {
|
||||||
|
new Notice(t('notice.wordsJsonMissing', rawBasename));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const audioMatch = /^pseudobs-audio:\s*"([^"]+)"/m.exec(content);
|
||||||
|
const audioSource = audioMatch?.[1];
|
||||||
|
|
||||||
|
const wordData = JSON.parse(wordsJson) as VttCueData[];
|
||||||
|
const { eaf, mismatch } = markdownToEaf(content, wordData, audioSource);
|
||||||
|
|
||||||
|
if (mismatch) {
|
||||||
|
new Notice(t('notice.vttMismatch'));
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.writeExport(file, 'eaf', eaf, 'notice.eafExported');
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Re-exporte le Markdown pseudonymisé courant au format d'origine (srt ou cha).
|
* Re-exporte le Markdown pseudonymisé courant au format d'origine (srt ou cha).
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -158,29 +158,45 @@ export function markdownToCha(mdContent: string): string {
|
||||||
return out.join('\n');
|
return out.join('\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Données word-level d'une cue — stockées dans <basename>.words.json. */
|
/** Données d'une cue — stockées dans <basename>.words.json. */
|
||||||
export interface VttCueData {
|
export interface VttCueData {
|
||||||
index: number;
|
index: number;
|
||||||
startTime: string;
|
startTime: string;
|
||||||
endTime: string;
|
endTime: string;
|
||||||
speaker?: string;
|
speaker?: string;
|
||||||
words: VttWord[];
|
words: VttWord[];
|
||||||
|
/** Ratio mots avec timestamp Whisper sur total (0.0–1.0). */
|
||||||
|
wordTimestampRate: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Extrait les données word-level d'un VttDocument pour le fichier .words.json.
|
* Extrait les données de toutes les cues d'un VttDocument pour le fichier .words.json.
|
||||||
* Seules les cues avec au moins un word timestamp sont incluses.
|
* Toutes les cues sont incluses (même sans word timestamps) pour garantir l'alignement
|
||||||
|
* par index avec le Markdown converti.
|
||||||
*/
|
*/
|
||||||
export function extractWordData(doc: VttDocument): VttCueData[] {
|
export function extractWordData(doc: VttDocument): VttCueData[] {
|
||||||
return doc.cues
|
return doc.cues.map((cue, index) => {
|
||||||
.map((cue, index) => ({
|
const total = cue.words.length;
|
||||||
|
const timed = cue.words.filter((w) => w.time !== '').length;
|
||||||
|
return {
|
||||||
index,
|
index,
|
||||||
startTime: cue.startTime,
|
startTime: cue.startTime,
|
||||||
endTime: cue.endTime,
|
endTime: cue.endTime,
|
||||||
speaker: cue.speaker,
|
speaker: cue.speaker,
|
||||||
words: cue.words,
|
words: cue.words,
|
||||||
}))
|
wordTimestampRate: total > 0 ? timed / total : 0,
|
||||||
.filter((c) => c.words.some((w) => w.time !== ''));
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Taux de séquentialité d'un fichier : ratio global de mots avec timestamp Whisper.
|
||||||
|
* Seuils indicatifs : ≥ 0.5 → .cha avec %tim / EAF word-level envisageable.
|
||||||
|
*/
|
||||||
|
export function fileSequentialityRate(data: VttCueData[]): number {
|
||||||
|
const total = data.reduce((n, c) => n + c.words.length, 0);
|
||||||
|
const timed = data.reduce((n, c) => n + c.words.filter((w) => w.time !== '').length, 0);
|
||||||
|
return total > 0 ? timed / total : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- Re-export VTT ----------------------------------------------------------
|
// ---- Re-export VTT ----------------------------------------------------------
|
||||||
|
|
@ -414,6 +430,99 @@ export function markdownToNoScribeHtml(
|
||||||
return { html, mismatch };
|
return { html, mismatch };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- Export EAF (ELAN) ------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Génère un fichier EAF (ELAN Annotation Format 3.0) depuis :
|
||||||
|
* - `mdContent` : le Markdown noScribe pseudonymisé (pseudobs-format: vtt ou html)
|
||||||
|
* - `wordData` : le contenu de <basename>.words.json (timestamps précis par cue)
|
||||||
|
* - `audioSource` : chemin audio optionnel (pour <MEDIA_DESCRIPTOR>)
|
||||||
|
*
|
||||||
|
* Produit un tier par locuteur, un ALIGNABLE_ANNOTATION par cue.
|
||||||
|
* Les TIME_SLOTs sont dédupliqués par valeur ms et triés en ordre croissant.
|
||||||
|
*/
|
||||||
|
export function markdownToEaf(
|
||||||
|
mdContent: string,
|
||||||
|
wordData: VttCueData[],
|
||||||
|
audioSource?: string,
|
||||||
|
): { eaf: string; mismatch: boolean } {
|
||||||
|
const bodyMatch = /^---\n[\s\S]*?\n---\n+([\s\S]*)$/.exec(mdContent);
|
||||||
|
const body = bodyMatch ? bodyMatch[1] : mdContent;
|
||||||
|
|
||||||
|
const cueLines = body.split('\n').filter((l) => MD_CUE_RE.test(l.trim()));
|
||||||
|
const mismatch = cueLines.length !== wordData.length;
|
||||||
|
const count = Math.min(cueLines.length, wordData.length);
|
||||||
|
|
||||||
|
interface EafAnnotation { id: string; ts1: string; ts2: string; text: string }
|
||||||
|
const tierMap = new Map<string, EafAnnotation[]>();
|
||||||
|
const timeSlotMap = new Map<number, string>(); // ms → TIME_SLOT_ID
|
||||||
|
let tsCounter = 0;
|
||||||
|
let annCounter = 0;
|
||||||
|
|
||||||
|
const getOrAddTs = (ms: number): string => {
|
||||||
|
if (timeSlotMap.has(ms)) return timeSlotMap.get(ms)!;
|
||||||
|
const id = `ts${++tsCounter}`;
|
||||||
|
timeSlotMap.set(ms, id);
|
||||||
|
return id;
|
||||||
|
};
|
||||||
|
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const m = MD_CUE_RE.exec(cueLines[i].trim())!;
|
||||||
|
const speaker = (m[1]?.trim() || wordData[i].speaker || '_transcript').trim();
|
||||||
|
const text = m[2]?.trim() ?? '';
|
||||||
|
if (!text) continue;
|
||||||
|
|
||||||
|
const ts1 = getOrAddTs(tsToMs(wordData[i].startTime));
|
||||||
|
const ts2 = getOrAddTs(tsToMs(wordData[i].endTime));
|
||||||
|
const ann: EafAnnotation = { id: `a${++annCounter}`, ts1, ts2, text };
|
||||||
|
|
||||||
|
if (!tierMap.has(speaker)) tierMap.set(speaker, []);
|
||||||
|
tierMap.get(speaker)!.push(ann);
|
||||||
|
}
|
||||||
|
|
||||||
|
const date = new Date().toISOString().replace(/\.\d{3}Z$/, '+00:00');
|
||||||
|
const out: string[] = [];
|
||||||
|
|
||||||
|
out.push('<?xml version="1.0" encoding="UTF-8"?>');
|
||||||
|
out.push(`<ANNOTATION_DOCUMENT AUTHOR="" DATE="${date}" FORMAT="3.0" VERSION="3.0"`);
|
||||||
|
out.push(' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"');
|
||||||
|
out.push(' xsi:noNamespaceSchemaLocation="http://www.mpi.nl/tools/elan/EAFv3.0.xsd">');
|
||||||
|
|
||||||
|
out.push(' <HEADER MEDIA_FILE="" TIME_UNITS="milliseconds">');
|
||||||
|
if (audioSource) {
|
||||||
|
out.push(` <MEDIA_DESCRIPTOR MEDIA_URL="${escapeHtml(audioSource)}" MIME_TYPE="audio/x-wav" RELATIVE_MEDIA_URL="" TIME_ORIGIN="0"/>`);
|
||||||
|
}
|
||||||
|
out.push(' </HEADER>');
|
||||||
|
|
||||||
|
out.push(' <TIME_ORDER>');
|
||||||
|
for (const [ms, id] of [...timeSlotMap.entries()].sort((a, b) => a[0] - b[0])) {
|
||||||
|
out.push(` <TIME_SLOT TIME_SLOT_ID="${id}" TIME_VALUE="${ms}"/>`);
|
||||||
|
}
|
||||||
|
out.push(' </TIME_ORDER>');
|
||||||
|
|
||||||
|
for (const [speaker, anns] of tierMap) {
|
||||||
|
out.push(` <TIER LINGUISTIC_TYPE_REF="default-lt" TIER_ID="${escapeHtml(speaker)}">`);
|
||||||
|
for (const ann of anns) {
|
||||||
|
out.push(' <ANNOTATION>');
|
||||||
|
out.push(` <ALIGNABLE_ANNOTATION ANNOTATION_ID="${ann.id}" TIME_SLOT_REF1="${ann.ts1}" TIME_SLOT_REF2="${ann.ts2}">`);
|
||||||
|
out.push(` <ANNOTATION_VALUE>${escapeHtml(ann.text)}</ANNOTATION_VALUE>`);
|
||||||
|
out.push(' </ALIGNABLE_ANNOTATION>');
|
||||||
|
out.push(' </ANNOTATION>');
|
||||||
|
}
|
||||||
|
out.push(' </TIER>');
|
||||||
|
}
|
||||||
|
|
||||||
|
out.push(' <LINGUISTIC_TYPE GRAPHIC_REFERENCES="false" LINGUISTIC_TYPE_ID="default-lt" TIME_ALIGNABLE="true"/>');
|
||||||
|
out.push(' <CONSTRAINT DESCRIPTION="Time subdivision of parent annotation\'s time interval, no time gaps allowed within this interval" STEREOTYPE="Time_Subdivision"/>');
|
||||||
|
out.push(' <CONSTRAINT DESCRIPTION="Symbolic subdivision of a parent annotation. Annotations refer to the same time slot as the parent" STEREOTYPE="Symbolic_Subdivision"/>');
|
||||||
|
out.push(' <CONSTRAINT DESCRIPTION="1-1 association with a parent annotation" STEREOTYPE="Symbolic_Association"/>');
|
||||||
|
out.push(' <CONSTRAINT DESCRIPTION="Time alignment of a parent annotation" STEREOTYPE="Included_In"/>');
|
||||||
|
out.push('</ANNOTATION_DOCUMENT>');
|
||||||
|
out.push('');
|
||||||
|
|
||||||
|
return { eaf: out.join('\n'), mismatch };
|
||||||
|
}
|
||||||
|
|
||||||
function lineGroup(line: ChatLine): 'structural' | 'turn' | null {
|
function lineGroup(line: ChatLine): 'structural' | 'turn' | null {
|
||||||
if (line.type === 'meta' || line.type === 'dependent') return 'structural';
|
if (line.type === 'meta' || line.type === 'dependent') return 'structural';
|
||||||
if (line.type === 'turn' || line.type === 'continuation') return 'turn';
|
if (line.type === 'turn' || line.type === 'continuation') return 'turn';
|
||||||
|
|
|
||||||
|
|
@ -728,6 +728,14 @@ export class PseudonymizationView extends ItemView {
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
new Setting(el)
|
||||||
|
.setName(t('panel.exports.exportEaf'))
|
||||||
|
.setDesc(t('panel.exports.exportEaf.desc'))
|
||||||
|
.addButton((btn) =>
|
||||||
|
btn.setButtonText(t('command.exportAsEaf')).onClick(() => {
|
||||||
|
void this.plugin.exportCurrentFileAsEaf();
|
||||||
|
})
|
||||||
|
);
|
||||||
} else if (format === 'srt') {
|
} else if (format === 'srt') {
|
||||||
new Setting(el)
|
new Setting(el)
|
||||||
.setName(t('panel.exports.exportSrt'))
|
.setName(t('panel.exports.exportSrt'))
|
||||||
|
|
|
||||||
137
tests/unit/EafExporter.test.ts
Normal file
137
tests/unit/EafExporter.test.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
||||||
|
import { markdownToEaf } from '../../src/parsers/TranscriptConverter';
|
||||||
|
import type { VttCueData } from '../../src/parsers/TranscriptConverter';
|
||||||
|
|
||||||
|
const WORDS_DATA: VttCueData[] = [
|
||||||
|
{
|
||||||
|
index: 0, startTime: '00:00:01.240', endTime: '00:00:05.800',
|
||||||
|
speaker: 'Expérimentateur',
|
||||||
|
words: [{ text: 'Bonjour,', time: '00:00:01.240' }, { text: ' Marie.', time: '00:00:02.100' }],
|
||||||
|
wordTimestampRate: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
index: 1, startTime: '00:00:06.100', endTime: '00:00:09.300',
|
||||||
|
speaker: 'SC01',
|
||||||
|
words: [{ text: 'Si je me présente ?', time: '' }],
|
||||||
|
wordTimestampRate: 0,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const MD = `---
|
||||||
|
pseudobs-format: vtt
|
||||||
|
pseudobs-source: "entretien.vtt"
|
||||||
|
---
|
||||||
|
|
||||||
|
**Expérimentateur** [00:00:01] : Bonjour, Marie.
|
||||||
|
|
||||||
|
**SC01** [00:00:06] : Si je me présente ?
|
||||||
|
`;
|
||||||
|
|
||||||
|
describe('markdownToEaf', () => {
|
||||||
|
it('produit une déclaration XML valide', () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
expect(eaf).toMatch(/^<\?xml version="1\.0" encoding="UTF-8"\?>/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('contient ANNOTATION_DOCUMENT FORMAT="3.0"', () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
expect(eaf).toContain('FORMAT="3.0"');
|
||||||
|
expect(eaf).toContain('<ANNOTATION_DOCUMENT');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('génère un TIME_SLOT par timestamp unique', () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
// 4 timestamps distincts : 1240, 5800, 6100, 9300
|
||||||
|
const slots = [...eaf.matchAll(/<TIME_SLOT /g)];
|
||||||
|
expect(slots).toHaveLength(4);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('les TIME_SLOTs sont triés en ordre croissant', () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
const values = [...eaf.matchAll(/TIME_VALUE="(\d+)"/g)].map((m) => parseInt(m[1], 10));
|
||||||
|
expect(values).toEqual([...values].sort((a, b) => a - b));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('crée un tier par locuteur', () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
expect(eaf).toContain('TIER_ID="Expérimentateur"');
|
||||||
|
expect(eaf).toContain('TIER_ID="SC01"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('intègre le texte pseudonymisé dans ANNOTATION_VALUE', () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
expect(eaf).toContain('<ANNOTATION_VALUE>Bonjour, Marie.</ANNOTATION_VALUE>');
|
||||||
|
expect(eaf).toContain('<ANNOTATION_VALUE>Si je me présente ?</ANNOTATION_VALUE>');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('les TIME_VALUE correspondent aux timestamps du words.json (en ms)', () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
expect(eaf).toContain('TIME_VALUE="1240"');
|
||||||
|
expect(eaf).toContain('TIME_VALUE="5800"');
|
||||||
|
expect(eaf).toContain('TIME_VALUE="6100"');
|
||||||
|
expect(eaf).toContain('TIME_VALUE="9300"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('déduplique les TIME_SLOTs partagés entre cues', () => {
|
||||||
|
const sharedData: VttCueData[] = [
|
||||||
|
{ index: 0, startTime: '00:00:01.000', endTime: '00:00:03.000',
|
||||||
|
speaker: 'S1', words: [], wordTimestampRate: 0 },
|
||||||
|
{ index: 1, startTime: '00:00:03.000', endTime: '00:00:05.000',
|
||||||
|
speaker: 'S1', words: [], wordTimestampRate: 0 },
|
||||||
|
];
|
||||||
|
const md = `---\npseudobs-format: vtt\npseudobs-source: "x.vtt"\n---\n\n**S1** [00:00:01] : Aa.\n\n**S1** [00:00:03] : Bb.\n`;
|
||||||
|
const { eaf } = markdownToEaf(md, sharedData);
|
||||||
|
// 3 slots : 1000, 3000 (partagé), 5000
|
||||||
|
const slots = [...eaf.matchAll(/<TIME_SLOT /g)];
|
||||||
|
expect(slots).toHaveLength(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('inclut MEDIA_DESCRIPTOR si audioSource fourni', () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA, '/audio/entretien.wav');
|
||||||
|
expect(eaf).toContain('<MEDIA_DESCRIPTOR');
|
||||||
|
expect(eaf).toContain('/audio/entretien.wav');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("n'inclut pas MEDIA_DESCRIPTOR si audioSource absent", () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
expect(eaf).not.toContain('<MEDIA_DESCRIPTOR');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('échappe les caractères spéciaux XML dans le texte', () => {
|
||||||
|
const dataXml: VttCueData[] = [
|
||||||
|
{ index: 0, startTime: '00:00:01.000', endTime: '00:00:02.000',
|
||||||
|
speaker: 'S1', words: [], wordTimestampRate: 0 },
|
||||||
|
];
|
||||||
|
const md = `---\npseudobs-format: vtt\npseudobs-source: "x.vtt"\n---\n\n**S1** [00:00:01] : a & b < c > d.\n`;
|
||||||
|
const { eaf } = markdownToEaf(md, dataXml);
|
||||||
|
expect(eaf).toContain('a & b < c > d.');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('signale un mismatch si le nombre de cues diffère', () => {
|
||||||
|
const { mismatch } = markdownToEaf(MD, WORDS_DATA.slice(0, 1));
|
||||||
|
expect(mismatch).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('pas de mismatch si les comptes sont égaux', () => {
|
||||||
|
const { mismatch } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
expect(mismatch).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('utilise _transcript comme tier si aucun locuteur', () => {
|
||||||
|
const dataNoSpk: VttCueData[] = [
|
||||||
|
{ index: 0, startTime: '00:00:01.000', endTime: '00:00:03.000',
|
||||||
|
speaker: undefined, words: [], wordTimestampRate: 0 },
|
||||||
|
];
|
||||||
|
const md = `---\npseudobs-format: vtt\npseudobs-source: "x.vtt"\n---\n\n[00:00:01] Texte sans locuteur.\n`;
|
||||||
|
const { eaf } = markdownToEaf(md, dataNoSpk);
|
||||||
|
expect(eaf).toContain('TIER_ID="_transcript"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('contient LINGUISTIC_TYPE et les quatre CONSTRAINT requis par ELAN', () => {
|
||||||
|
const { eaf } = markdownToEaf(MD, WORDS_DATA);
|
||||||
|
expect(eaf).toContain('<LINGUISTIC_TYPE');
|
||||||
|
expect(eaf).toContain('STEREOTYPE="Time_Subdivision"');
|
||||||
|
expect(eaf).toContain('STEREOTYPE="Symbolic_Subdivision"');
|
||||||
|
expect(eaf).toContain('STEREOTYPE="Symbolic_Association"');
|
||||||
|
expect(eaf).toContain('STEREOTYPE="Included_In"');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -4,7 +4,7 @@ import { SrtParser } from '../../src/parsers/SrtParser';
|
||||||
import { ChatParser } from '../../src/parsers/ChatParser';
|
import { ChatParser } from '../../src/parsers/ChatParser';
|
||||||
import { VttParser } from '../../src/parsers/VttParser';
|
import { VttParser } from '../../src/parsers/VttParser';
|
||||||
import { NoScribeHtmlParser } from '../../src/parsers/NoScribeHtmlParser';
|
import { NoScribeHtmlParser } from '../../src/parsers/NoScribeHtmlParser';
|
||||||
import { srtToMarkdown, chatToMarkdown, vttToMarkdown, noScribeHtmlToMarkdown, extractWordData, markdownToVtt } from '../../src/parsers/TranscriptConverter';
|
import { srtToMarkdown, chatToMarkdown, vttToMarkdown, noScribeHtmlToMarkdown, extractWordData, fileSequentialityRate, markdownToVtt } from '../../src/parsers/TranscriptConverter';
|
||||||
|
|
||||||
const SRT_FIXTURE = path.join(__dirname, '../fixtures/entretien_01.srt');
|
const SRT_FIXTURE = path.join(__dirname, '../fixtures/entretien_01.srt');
|
||||||
const CHA_FIXTURE = path.join(__dirname, '../fixtures/entretien_02.cha');
|
const CHA_FIXTURE = path.join(__dirname, '../fixtures/entretien_02.cha');
|
||||||
|
|
@ -191,21 +191,32 @@ describe('extractWordData', () => {
|
||||||
<v SC01>Si je me présente ?
|
<v SC01>Si je me présente ?
|
||||||
`;
|
`;
|
||||||
|
|
||||||
it('extrait les cues avec word timestamps', () => {
|
it('inclut toutes les cues (avec et sans word timestamps)', () => {
|
||||||
const doc = new VttParser().parse(VTT_WITH_WORDS);
|
const doc = new VttParser().parse(VTT_WITH_WORDS);
|
||||||
const data = extractWordData(doc);
|
const data = extractWordData(doc);
|
||||||
expect(data).toHaveLength(1); // cue 2 sans word timestamps exclue
|
expect(data).toHaveLength(2); // toutes les cues, index stable
|
||||||
expect(data[0].index).toBe(0);
|
expect(data[0].index).toBe(0);
|
||||||
expect(data[0].speaker).toBe('Expérimentateur');
|
expect(data[0].speaker).toBe('Expérimentateur');
|
||||||
expect(data[0].startTime).toBe('00:00:01.240');
|
expect(data[0].startTime).toBe('00:00:01.240');
|
||||||
expect(data[0].endTime).toBe('00:00:05.800');
|
expect(data[0].endTime).toBe('00:00:05.800');
|
||||||
expect(data[0].words.some((w) => w.time !== '')).toBe(true);
|
expect(data[1].index).toBe(1);
|
||||||
|
expect(data[1].speaker).toBe('SC01');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('exclut les cues sans word timestamps', () => {
|
it('calcule wordTimestampRate correctement', () => {
|
||||||
const doc = new VttParser().parse(VTT_WITH_WORDS);
|
const doc = new VttParser().parse(VTT_WITH_WORDS);
|
||||||
const data = extractWordData(doc);
|
const data = extractWordData(doc);
|
||||||
expect(data.every((c) => c.words.some((w) => w.time !== ''))).toBe(true);
|
// Cue 0 : 2 mots avec timestamps Whisper → rate = 1.0
|
||||||
|
expect(data[0].wordTimestampRate).toBe(1);
|
||||||
|
// Cue 1 : aucun word timestamp → rate = 0
|
||||||
|
expect(data[1].wordTimestampRate).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('fileSequentialityRate agrège correctement', () => {
|
||||||
|
const doc = new VttParser().parse(VTT_WITH_WORDS);
|
||||||
|
const data = extractWordData(doc);
|
||||||
|
// Cue 0 : 2 mots timés, cue 1 : 1 mot non timé → 2/3
|
||||||
|
expect(fileSequentialityRate(data)).toBeCloseTo(2 / 3);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('produit un JSON sérialisable', () => {
|
it('produit un JSON sérialisable', () => {
|
||||||
|
|
@ -220,11 +231,11 @@ describe('extractWordData', () => {
|
||||||
describe('markdownToVtt — re-export VTT', () => {
|
describe('markdownToVtt — re-export VTT', () => {
|
||||||
const WORDS_DATA = [
|
const WORDS_DATA = [
|
||||||
{ index: 0, startTime: '00:01:28.080', endTime: '00:01:33.536', speaker: 'S00',
|
{ index: 0, startTime: '00:01:28.080', endTime: '00:01:33.536', speaker: 'S00',
|
||||||
words: [{ text: 'original', time: '00:01:28.080' }] },
|
words: [{ text: 'original', time: '00:01:28.080' }], wordTimestampRate: 1 },
|
||||||
{ index: 1, startTime: '00:01:32.672', endTime: '00:01:34.816', speaker: 'S01',
|
{ index: 1, startTime: '00:01:32.672', endTime: '00:01:34.816', speaker: 'S01',
|
||||||
words: [{ text: 'original', time: '00:01:32.672' }] },
|
words: [{ text: 'original', time: '00:01:32.672' }], wordTimestampRate: 1 },
|
||||||
{ index: 2, startTime: '00:01:34.816', endTime: '00:01:45.920', speaker: 'S00',
|
{ index: 2, startTime: '00:01:34.816', endTime: '00:01:45.920', speaker: 'S00',
|
||||||
words: [{ text: 'original', time: '00:01:34.816' }] },
|
words: [{ text: 'original', time: '00:01:34.816' }], wordTimestampRate: 1 },
|
||||||
];
|
];
|
||||||
|
|
||||||
const MD_PSEUDONYMIZED = `---
|
const MD_PSEUDONYMIZED = `---
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue