From ec45d77ce92dcc9777561f0587e6fb00531cc058 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Sun, 23 Nov 2025 18:27:57 -0500 Subject: [PATCH] refac: sources and citations --- .../Messages/Citations/CitationModal.svelte | 2 +- .../components/chat/Messages/Markdown.svelte | 7 +- .../chat/Messages/Markdown/HTMLToken.svelte | 5 -- .../Markdown/MarkdownInlineTokens.svelte | 13 ++++ .../Messages/Markdown/MarkdownTokens.svelte | 8 ++- .../chat/Messages/Markdown/Source.svelte | 33 ++------- .../chat/Messages/Markdown/SourceToken.svelte | 70 +++++++++++++++++++ .../chat/Messages/ResponseMessage.svelte | 6 +- src/lib/utils/index.ts | 26 +------ src/lib/utils/marked/citation-extension.ts | 55 +++++++++++++++ src/lib/utils/marked/footnote-extension.ts | 38 ++++++++++ 11 files changed, 201 insertions(+), 62 deletions(-) create mode 100644 src/lib/components/chat/Messages/Markdown/SourceToken.svelte create mode 100644 src/lib/utils/marked/citation-extension.ts create mode 100644 src/lib/utils/marked/footnote-extension.ts diff --git a/src/lib/components/chat/Messages/Citations/CitationModal.svelte b/src/lib/components/chat/Messages/Citations/CitationModal.svelte index 114d4f48d2..3ec9ae21a0 100644 --- a/src/lib/components/chat/Messages/Citations/CitationModal.svelte +++ b/src/lib/components/chat/Messages/Citations/CitationModal.svelte @@ -169,7 +169,7 @@ > {:else} 
-                {document.document}
+                {document.document.trim()}
{/if} diff --git a/src/lib/components/chat/Messages/Markdown.svelte b/src/lib/components/chat/Messages/Markdown.svelte index b1ca46c79b..b97a31e4f4 100644 --- a/src/lib/components/chat/Messages/Markdown.svelte +++ b/src/lib/components/chat/Messages/Markdown.svelte @@ -9,6 +9,8 @@ import { mentionExtension } from '$lib/utils/marked/mention-extension'; import MarkdownTokens from './Markdown/MarkdownTokens.svelte'; + import footnoteExtension from '$lib/utils/marked/footnote-extension'; + import citationExtension from '$lib/utils/marked/citation-extension'; export let id = ''; export let content; @@ -39,6 +41,8 @@ marked.use(markedKatexExtension(options)); marked.use(markedExtension(options)); + marked.use(citationExtension(options)); + marked.use(footnoteExtension(options)); marked.use(disableSingleTilde); marked.use({ extensions: [mentionExtension({ triggerChar: '@' }), mentionExtension({ triggerChar: '#' })] @@ -47,7 +51,7 @@ $: (async () => { if (content) { tokens = marked.lexer( - replaceTokens(processResponseContent(content), sourceIds, model?.name, $user?.name) + replaceTokens(processResponseContent(content), model?.name, $user?.name) ); } })(); @@ -61,6 +65,7 @@ {save} {preview} {editCodeBlock} + {sourceIds} {topPadding} {onTaskClick} {onSourceClick} diff --git a/src/lib/components/chat/Messages/Markdown/HTMLToken.svelte b/src/lib/components/chat/Messages/Markdown/HTMLToken.svelte index fac4fad1de..da9690c631 100644 --- a/src/lib/components/chat/Messages/Markdown/HTMLToken.svelte +++ b/src/lib/components/chat/Messages/Markdown/HTMLToken.svelte @@ -3,14 +3,11 @@ import type { Token } from 'marked'; import { WEBUI_BASE_URL } from '$lib/constants'; - import Source from './Source.svelte'; import { settings } from '$lib/stores'; export let id: string; export let token: Token; - export let onSourceClick: Function = () => {}; - let html: string | null = null; $: if (token.type === 'html' && token?.text) { @@ -129,8 +126,6 @@ }} > {/if} - {:else if token.text.includes(` {:else if token.text.trim().match(/^$/i)}
{:else} diff --git a/src/lib/components/chat/Messages/Markdown/MarkdownInlineTokens.svelte b/src/lib/components/chat/Messages/Markdown/MarkdownInlineTokens.svelte index 85ba8740c1..a5e97ed9e8 100644 --- a/src/lib/components/chat/Messages/Markdown/MarkdownInlineTokens.svelte +++ b/src/lib/components/chat/Messages/Markdown/MarkdownInlineTokens.svelte @@ -17,10 +17,12 @@ import TextToken from './MarkdownInlineTokens/TextToken.svelte'; import CodespanToken from './MarkdownInlineTokens/CodespanToken.svelte'; import MentionToken from './MarkdownInlineTokens/MentionToken.svelte'; + import SourceToken from './SourceToken.svelte'; export let id: string; export let done = true; export let tokens: Token[]; + export let sourceIds = []; export let onSourceClick: Function = () => {}; @@ -68,6 +70,17 @@ > {:else if token.type === 'mention'} + {:else if token.type === 'footnote'} + {@html DOMPurify.sanitize( + `${token.escapedText}` + ) || ''} + {:else if token.type === 'citation'} + + {:else if token.type === 'text'} {/if} diff --git a/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte b/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte index 341861b4ff..143fe20541 100644 --- a/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte +++ b/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte @@ -21,7 +21,6 @@ import Tooltip from '$lib/components/common/Tooltip.svelte'; import Download from '$lib/components/icons/Download.svelte'; - import Source from './Source.svelte'; import HtmlToken from './HTMLToken.svelte'; import Clipboard from '$lib/components/icons/Clipboard.svelte'; @@ -29,6 +28,7 @@ export let tokens: Token[]; export let top = true; export let attributes = {}; + export let sourceIds = []; export let done = true; @@ -96,6 +96,7 @@ id={`${id}-${tokenIdx}-h`} tokens={token.tokens} {done} + {sourceIds} {onSourceClick} /> @@ -147,6 +148,7 @@ id={`${id}-${tokenIdx}-header-${headerIdx}`} tokens={header.tokens} {done} + {sourceIds} {onSourceClick} /> @@ -172,6 +174,7 @@ id={`${id}-${tokenIdx}-row-${rowIdx}-${cellIdx}`} tokens={cell.tokens} {done} + {sourceIds} {onSourceClick} /> @@ -348,6 +351,7 @@ id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} {done} + {sourceIds} {onSourceClick} />

@@ -359,6 +363,7 @@ id={`${id}-${tokenIdx}-t`} tokens={token.tokens} {done} + {sourceIds} {onSourceClick} /> {:else} @@ -370,6 +375,7 @@ id={`${id}-${tokenIdx}-p`} tokens={token.tokens ?? []} {done} + {sourceIds} {onSourceClick} /> {:else} diff --git a/src/lib/components/chat/Messages/Markdown/Source.svelte b/src/lib/components/chat/Messages/Markdown/Source.svelte index b298337320..d48525e070 100644 --- a/src/lib/components/chat/Messages/Markdown/Source.svelte +++ b/src/lib/components/chat/Messages/Markdown/Source.svelte @@ -1,23 +1,10 @@ -{#if attributes.title !== 'N/A'} +{#if title !== 'N/A'} {/if} diff --git a/src/lib/components/chat/Messages/Markdown/SourceToken.svelte b/src/lib/components/chat/Messages/Markdown/SourceToken.svelte new file mode 100644 index 0000000000..9d0bbcd6ba --- /dev/null +++ b/src/lib/components/chat/Messages/Markdown/SourceToken.svelte @@ -0,0 +1,70 @@ + + +{#if (token?.ids ?? []).length == 1} + +{:else} + + + + + +
+ {#each token.ids as sourceId} +
+ +
+ {/each} +
+ + +{/if} diff --git a/src/lib/components/chat/Messages/ResponseMessage.svelte b/src/lib/components/chat/Messages/ResponseMessage.svelte index 98a9dafcc7..2238b21abb 100644 --- a/src/lib/components/chat/Messages/ResponseMessage.svelte +++ b/src/lib/components/chat/Messages/ResponseMessage.svelte @@ -797,11 +797,11 @@ onTaskClick={async (e) => { console.log(e); }} - onSourceClick={async (id, idx) => { - console.log(id, idx); + onSourceClick={async (id) => { + console.log(id); if (citationsElement) { - citationsElement?.showSourceModal(idx - 1); + citationsElement?.showSourceModal(id); } }} onAddMessages={({ modelId, parentId, messages }) => { diff --git a/src/lib/utils/index.ts b/src/lib/utils/index.ts index f9fb4236f2..5b92da8b8e 100644 --- a/src/lib/utils/index.ts +++ b/src/lib/utils/index.ts @@ -32,7 +32,7 @@ function escapeRegExp(string: string): string { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } -export const replaceTokens = (content, sourceIds, char, user) => { +export const replaceTokens = (content, char, user) => { const tokens = [ { regex: /{{char}}/gi, replacement: char }, { regex: /{{user}}/gi, replacement: user }, @@ -67,30 +67,6 @@ export const replaceTokens = (content, sourceIds, char, user) => { } }); - if (Array.isArray(sourceIds)) { - // Match both [1], [2], and [1,2,3] forms - const multiRefRegex = /\[([\d,\s]+)\]/g; - segment = segment.replace(multiRefRegex, (match, group) => { - // Extract numbers like 1,2,3 - const indices = group - .split(',') - .map((n) => parseInt(n.trim(), 10)) - .filter((n) => !isNaN(n)); - - // Replace each index with a tag - const sources = indices - .map((idx) => { - const sourceId = sourceIds[idx - 1]; - return sourceId - ? `` - : `[${idx}]`; - }) - .join(''); - - return sources; - }); - } - return segment; }); diff --git a/src/lib/utils/marked/citation-extension.ts b/src/lib/utils/marked/citation-extension.ts new file mode 100644 index 0000000000..dad266c9f0 --- /dev/null +++ b/src/lib/utils/marked/citation-extension.ts @@ -0,0 +1,55 @@ +export function citationExtension() { + return { + name: 'citation', + level: 'inline' as const, + + start(src: string) { + // Trigger on any [number] + return src.search(/\[(\d[\d,\s]*)\]/); + }, + + tokenizer(src: string) { + // Avoid matching footnotes + if (/^\[\^/.test(src)) return; + + // Match ONE OR MORE adjacent [1] or [1,2] blocks + // Example matched: "[1][2,3][4]" + const rule = /^(\[(?:\d[\d,\s]*)\])+/; + const match = rule.exec(src); + if (!match) return; + + const raw = match[0]; + + // Extract ALL bracket groups inside the big match + const groupRegex = /\[([\d,\s]+)\]/g; + const ids: number[] = []; + let m: RegExpExecArray | null; + + while ((m = groupRegex.exec(raw))) { + const parsed = m[1] + .split(',') + .map((n) => parseInt(n.trim(), 10)) + .filter((n) => !isNaN(n)); + + ids.push(...parsed); + } + + return { + type: 'citation', + raw, + ids // merged list + }; + }, + + renderer(token: any) { + // e.g. "1,2,3" + return token.ids.join(','); + } + }; +} + +export default function () { + return { + extensions: [citationExtension()] + }; +} diff --git a/src/lib/utils/marked/footnote-extension.ts b/src/lib/utils/marked/footnote-extension.ts new file mode 100644 index 0000000000..4ef95be6a8 --- /dev/null +++ b/src/lib/utils/marked/footnote-extension.ts @@ -0,0 +1,38 @@ +// footnote-extension.ts +// Simple extension for marked to support footnote references like [^1], [^note] + +function escapeHtml(s: string) { + return s.replace( + /[&<>"']/g, + (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[c]! + ); +} + +export function footnoteExtension() { + return { + name: 'footnote', + level: 'inline' as const, + start(src: string) { + return src.search(/\[\^\s*[a-zA-Z0-9_-]+\s*\]/); + }, + tokenizer(src: string) { + const rule = /^\[\^\s*([a-zA-Z0-9_-]+)\s*\]/; + const match = rule.exec(src); + if (match) { + const escapedText = escapeHtml(match[1]); + return { + type: 'footnote', + raw: match[0], + text: match[1], + escapedText: escapedText + }; + } + } + }; +} + +export default function () { + return { + extensions: [footnoteExtension()] + }; +}