obsidian-plugin/fancive_obsidian-parallel-reader
1
0
Fork 0
mirror of https://github.com/fancive/obsidian-parallel-reader.git synced 2026-07-22 06:53:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: parse multiline sse events

Browse source 
Change-Id: I9c013dbe6840a0892836eb5d71206c6fcde5ec78
This commit is contained in:
wujunchen 2026-04-26 18:27:28 +08:00
parent 846e32301a
commit bc51acd00e
3 changed files with 51 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
main.js
View file

File diff suppressed because one or more lines are too long

22
src/streaming.ts
View file

@ -44,16 +44,22 @@ export function deltaExtractorForFormat(format: string): DeltaExtractor | null {
*/
export function parseSseBuffer(buffer: string, extractDelta: DeltaExtractor): { deltas: string[]; rest: string } {
const deltas: string[] = [];
const lines = buffer.split('\n');
const rest = lines.pop() ?? ''; // keep incomplete line
const normalized = buffer.replace(/\r\n/g, '\n');
const chunks = normalized.split('\n\n');
const rest = normalized.endsWith('\n\n') ? '' : (chunks.pop() ?? '');
const eventChunks = normalized.endsWith('\n\n') ? chunks.slice(0, -1) : chunks;
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed || trimmed === 'data: [DONE]') continue;
if (trimmed.startsWith('event:')) continue;
if (!trimmed.startsWith('data:')) continue;
for (const eventChunk of eventChunks) {
const dataLines: string[] = [];
for (const line of eventChunk.split('\n')) {
if (!line.startsWith('data:')) continue;
const data = line.slice(line.startsWith('data: ') ? 6 : 5);
dataLines.push(data);
}
if (dataLines.length === 0) continue;
const data = trimmed.slice(trimmed.startsWith('data: ') ? 6 : 5);
const data = dataLines.join('\n');
if (data.trim() === '[DONE]') continue;
try {
const json = JSON.parse(data) as Record<string, unknown>;
const delta = extractDelta(json);

21
tests/modules.test.js
View file

@ -594,7 +594,7 @@ assert.strictEqual(t.deltaExtractorForFormat('google-generative-ai'), null, 'no
// parseSseBuffer
const sseResult = t.parseSseBuffer(
'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: {"choices":[{"delta":{"content":" there"}}]}\ndata: [DONE]\npartial',
'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: {"choices":[{"delta":{"content":" there"}}]}\n\ndata: [DONE]\n\npartial',
openaiExtract,
);
assert.deepStrictEqual(sseResult.deltas, ['hi', ' there'], 'SSE parser extracts deltas');
@ -618,21 +618,34 @@ assert.ok(ssePartial.rest.includes('"ok"'), 'partial line kept in rest');
// parseSseBuffer: multi-event in one chunk
const sseMulti = t.parseSseBuffer(
'data: {"choices":[{"delta":{"content":"a"}}]}\ndata: {"choices":[{"delta":{"content":"b"}}]}\ndata: {"choices":[{"delta":{"content":"c"}}]}\n',
'data: {"choices":[{"delta":{"content":"a"}}]}\n\ndata: {"choices":[{"delta":{"content":"b"}}]}\n\ndata: {"choices":[{"delta":{"content":"c"}}]}\n\n',
openaiExtract,
);
assert.deepStrictEqual(sseMulti.deltas, ['a', 'b', 'c'], 'three events in one chunk');
assert.strictEqual(sseMulti.rest, '', 'nothing left in rest when chunk ends with newline');
const sseMultilineData = t.parseSseBuffer(
'data: {"choices":[\ndata: {"delta":{"content":"joined"}}\ndata: ]}\n\n',
openaiExtract,
);
assert.deepStrictEqual(sseMultilineData.deltas, ['joined'], 'consecutive data lines are merged into one event');
const ssePartialEvent = t.parseSseBuffer(
'data: {"choices":[{"delta":{"content":"wait"}}]}\n',
openaiExtract,
);
assert.deepStrictEqual(ssePartialEvent.deltas, [], 'event without blank-line terminator is retained');
assert.strictEqual(ssePartialEvent.rest.includes('"wait"'), true, 'partial event remains in rest');
// parseSseBuffer: non-data lines are skipped
const sseMixed = t.parseSseBuffer(
': comment\nevent: ping\ndata: {"choices":[{"delta":{"content":"x"}}]}\n',
': comment\nevent: ping\ndata: {"choices":[{"delta":{"content":"x"}}]}\n\n',
openaiExtract,
);
assert.deepStrictEqual(sseMixed.deltas, ['x'], 'comment and event lines ignored');
// parseSseBuffer: malformed JSON is silently skipped
const sseBad = t.parseSseBuffer('data: not_json\ndata: {"choices":[{"delta":{"content":"y"}}]}\n', openaiExtract);
const sseBad = t.parseSseBuffer('data: not_json\n\ndata: {"choices":[{"delta":{"content":"y"}}]}\n\n', openaiExtract);
assert.deepStrictEqual(sseBad.deltas, ['y'], 'bad JSON line skipped, good line extracted');
// i18n: additional edge cases