diff --git a/backend/open_webui/routers/notes.py b/backend/open_webui/routers/notes.py index 0c4842909f..375f59ff6c 100644 --- a/backend/open_webui/routers/notes.py +++ b/backend/open_webui/routers/notes.py @@ -6,6 +6,9 @@ from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Request, status, BackgroundTasks from pydantic import BaseModel +from open_webui.socket.main import sio + + from open_webui.models.users import Users, UserResponse from open_webui.models.notes import Notes, NoteModel, NoteForm, NoteUserResponse @@ -170,6 +173,12 @@ async def update_note_by_id( try: note = Notes.update_note_by_id(id, form_data) + await sio.emit( + "note-events", + note.model_dump(), + to=f"note:{note.id}", + ) + return note except Exception as e: log.exception(e) diff --git a/backend/open_webui/socket/main.py b/backend/open_webui/socket/main.py index e3f2731f4e..cc78bbb98d 100644 --- a/backend/open_webui/socket/main.py +++ b/backend/open_webui/socket/main.py @@ -316,6 +316,37 @@ async def join_channel(sid, data): await sio.enter_room(sid, f"channel:{channel.id}") +@sio.on("join-note") +async def join_note(sid, data): + auth = data["auth"] if "auth" in data else None + if not auth or "token" not in auth: + return + + token_data = decode_token(auth["token"]) + if token_data is None or "id" not in token_data: + return + + user = Users.get_user_by_id(token_data["id"]) + if not user: + return + + note = Notes.get_note_by_id(data["note_id"]) + if not note: + log.error(f"Note {data['note_id']} not found for user {user.id}") + return + + if ( + user.role != "admin" + and user.id != note.user_id + and not has_access(user.id, type="read", access_control=note.access_control) + ): + log.error(f"User {user.id} does not have access to note {data['note_id']}") + return + + log.debug(f"Joining note {note.id} for user {user.id}") + await sio.enter_room(sid, f"note:{note.id}") + + @sio.on("channel-events") async def channel_events(sid, data): room = f"channel:{data['channel_id']}" diff --git a/package-lock.json b/package-lock.json index a8e9ea7094..79ae4f62d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "@tiptap/core": "^3.0.7", "@tiptap/extension-bubble-menu": "^2.26.1", "@tiptap/extension-code-block-lowlight": "^3.0.7", + "@tiptap/extension-drag-handle": "^3.0.7", "@tiptap/extension-file-handler": "^3.0.7", "@tiptap/extension-floating-menu": "^2.26.1", "@tiptap/extension-highlight": "^3.0.7", @@ -30,6 +31,7 @@ "@tiptap/extension-list": "^3.0.7", "@tiptap/extension-table": "^3.0.7", "@tiptap/extension-typography": "^3.0.7", + "@tiptap/extension-youtube": "^3.0.7", "@tiptap/extensions": "^3.0.7", "@tiptap/pm": "^3.0.7", "@tiptap/starter-kit": "^3.0.7", @@ -3218,6 +3220,23 @@ "lowlight": "^2 || ^3" } }, + "node_modules/@tiptap/extension-collaboration": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@tiptap/extension-collaboration/-/extension-collaboration-3.0.7.tgz", + "integrity": "sha512-so59vQCAS1vy6k86byk96fYvAPM5w8u8/Yp3jKF1LPi9LH4wzS4hGnOP/dEbedxPU48an9WB1lSOczSKPECJaQ==", + "license": "MIT", + "peer": true, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "^3.0.7", + "@tiptap/pm": "^3.0.7", + "@tiptap/y-tiptap": "^3.0.0-beta.3", + "yjs": "^13" + } + }, "node_modules/@tiptap/extension-document": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@tiptap/extension-document/-/extension-document-3.0.7.tgz", @@ -3231,6 +3250,26 @@ "@tiptap/core": "^3.0.7" } }, + "node_modules/@tiptap/extension-drag-handle": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@tiptap/extension-drag-handle/-/extension-drag-handle-3.0.7.tgz", + "integrity": "sha512-rm8+0kPz5C5JTp4f1QY61Qd5d7zlJAxLeJtOvgC9RCnrNG1F7LCsmOkvy5fsU6Qk2YCCYOiSSMC4S4HKPrUJhw==", + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.6.13" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "^3.0.7", + "@tiptap/extension-collaboration": "^3.0.7", + "@tiptap/extension-node-range": "^3.0.7", + "@tiptap/pm": "^3.0.7", + "@tiptap/y-tiptap": "^3.0.0-beta.3" + } + }, "node_modules/@tiptap/extension-dropcursor": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@tiptap/extension-dropcursor/-/extension-dropcursor-3.0.7.tgz", @@ -3425,6 +3464,21 @@ "@tiptap/extension-list": "^3.0.7" } }, + "node_modules/@tiptap/extension-node-range": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@tiptap/extension-node-range/-/extension-node-range-3.0.7.tgz", + "integrity": "sha512-cHViNqtOUD9CLJxEj28rcj8tb8RYQZ7kwmtSvIye84Y3MJIzigRm4IUBNNOYnZfq5YAZIR97WKcJeFz3EU1VPg==", + "license": "MIT", + "peer": true, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "^3.0.7", + "@tiptap/pm": "^3.0.7" + } + }, "node_modules/@tiptap/extension-ordered-list": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@tiptap/extension-ordered-list/-/extension-ordered-list-3.0.7.tgz", @@ -3531,6 +3585,19 @@ "@tiptap/core": "^3.0.7" } }, + "node_modules/@tiptap/extension-youtube": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@tiptap/extension-youtube/-/extension-youtube-3.0.7.tgz", + "integrity": "sha512-BD4rc7Xoi3O+puXSEArHAbBVu4dhj+9TuuVYzEFgNHI+FN/py9J5AiNf4TXGKBSlMUOYPpODaEROwyGmqAmpuA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "^3.0.7" + } + }, "node_modules/@tiptap/extensions": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@tiptap/extensions/-/extensions-3.0.7.tgz", @@ -3611,6 +3678,27 @@ "url": "https://github.com/sponsors/ueberdosis" } }, + "node_modules/@tiptap/y-tiptap": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@tiptap/y-tiptap/-/y-tiptap-3.0.0.tgz", + "integrity": "sha512-HIeJZCj+KYJde2x6fONzo4o6kd7gW7eonwhQsv2p2VQnUgwNXMVhN+D6Z3AH/2i541Sq33y1PO4U/1ThCPjqbA==", + "license": "MIT", + "peer": true, + "dependencies": { + "lib0": "^0.2.100" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "peerDependencies": { + "prosemirror-model": "^1.7.1", + "prosemirror-state": "^1.2.3", + "prosemirror-view": "^1.9.10", + "y-protocols": "^1.0.1", + "yjs": "^13.5.38" + } + }, "node_modules/@types/cookie": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", diff --git a/package.json b/package.json index 97955b7fb4..8c26c794bf 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "@tiptap/core": "^3.0.7", "@tiptap/extension-bubble-menu": "^2.26.1", "@tiptap/extension-code-block-lowlight": "^3.0.7", + "@tiptap/extension-drag-handle": "^3.0.7", "@tiptap/extension-file-handler": "^3.0.7", "@tiptap/extension-floating-menu": "^2.26.1", "@tiptap/extension-highlight": "^3.0.7", @@ -74,6 +75,7 @@ "@tiptap/extension-list": "^3.0.7", "@tiptap/extension-table": "^3.0.7", "@tiptap/extension-typography": "^3.0.7", + "@tiptap/extension-youtube": "^3.0.7", "@tiptap/extensions": "^3.0.7", "@tiptap/pm": "^3.0.7", "@tiptap/starter-kit": "^3.0.7", diff --git a/src/lib/components/common/RichTextInput.svelte b/src/lib/components/common/RichTextInput.svelte index b7e7010d47..55aa5db2c9 100644 --- a/src/lib/components/common/RichTextInput.svelte +++ b/src/lib/components/common/RichTextInput.svelte @@ -84,6 +84,10 @@ import { ListKit } from '@tiptap/extension-list'; import { Placeholder, CharacterCount } from '@tiptap/extensions'; + import Image from './RichTextInput/Image/index.js'; + // import TiptapImage from '@tiptap/extension-image'; + + import FileHandler from '@tiptap/extension-file-handler'; import Typography from '@tiptap/extension-typography'; import Highlight from '@tiptap/extension-highlight'; import CodeBlockLowlight from '@tiptap/extension-code-block-lowlight'; @@ -106,11 +110,15 @@ export let socket = null; export let user = null; + export let files = []; + export let documentId = ''; export let className = 'input-prose'; export let placeholder = 'Type here...'; export let link = false; + export let image = false; + export let fileHandler = false; export let id = ''; export let value = ''; @@ -819,7 +827,9 @@ editor = new Editor({ element: element, extensions: [ - StarterKit, + StarterKit.configure({ + link: link + }), Placeholder.configure({ placeholder }), CodeBlockLowlight.configure({ @@ -838,6 +848,60 @@ }), CharacterCount.configure({}), + ...(image ? [Image] : []), + ...(fileHandler + ? [ + FileHandler.configure({ + allowedMimeTypes: ['image/png', 'image/jpeg', 'image/gif', 'image/webp'], + onDrop: (currentEditor, files, pos) => { + files.forEach((file) => { + const fileReader = new FileReader(); + + fileReader.readAsDataURL(file); + fileReader.onload = () => { + currentEditor + .chain() + .insertContentAt(pos, { + type: 'image', + attrs: { + src: fileReader.result + } + }) + .focus() + .run(); + }; + }); + }, + onPaste: (currentEditor, files, htmlContent) => { + files.forEach((file) => { + if (htmlContent) { + // if there is htmlContent, stop manual insertion & let other extensions handle insertion via inputRule + // you could extract the pasted file from this url string and upload it to a server for example + console.log(htmlContent); // eslint-disable-line no-console + return false; + } + + const fileReader = new FileReader(); + + fileReader.readAsDataURL(file); + fileReader.onload = () => { + currentEditor + .chain() + .insertContentAt(currentEditor.state.selection.anchor, { + type: 'image', + attrs: { + src: fileReader.result + } + }) + .focus() + .run(); + }; + }); + } + }) + ] + : []), + ...(autocomplete ? [ AIAutocompletion.configure({ @@ -1093,6 +1157,11 @@ return false; } } + }, + onBeforeCreate: ({ editor }) => { + if (files) { + editor.storage.files = files; + } } }); diff --git a/src/lib/components/common/RichTextInput/Image/image.ts b/src/lib/components/common/RichTextInput/Image/image.ts new file mode 100644 index 0000000000..c0fb995253 --- /dev/null +++ b/src/lib/components/common/RichTextInput/Image/image.ts @@ -0,0 +1,197 @@ +import { mergeAttributes, Node, nodeInputRule } from '@tiptap/core'; + +export interface ImageOptions { + /** + * Controls if the image node should be inline or not. + * @default false + * @example true + */ + inline: boolean; + + /** + * Controls if base64 images are allowed. Enable this if you want to allow + * base64 image urls in the `src` attribute. + * @default false + * @example true + */ + allowBase64: boolean; + + /** + * HTML attributes to add to the image element. + * @default {} + * @example { class: 'foo' } + */ + HTMLAttributes: Record; +} + +export interface SetImageOptions { + src: string; + alt?: string; + title?: string; + width?: number; + height?: number; +} + +declare module '@tiptap/core' { + interface Commands { + image: { + /** + * Add an image + * @param options The image attributes + * @example + * editor + * .commands + * .setImage({ src: 'https://tiptap.dev/logo.png', alt: 'tiptap', title: 'tiptap logo' }) + */ + setImage: (options: SetImageOptions) => ReturnType; + }; + } +} + +/** + * Matches an image to a ![image](src "title") on input. + */ +export const inputRegex = /(?:^|\s)(!\[(.+|:?)]\((\S+)(?:(?:\s+)["'](\S+)["'])?\))$/; + +/** + * This extension allows you to insert images. + * @see https://www.tiptap.dev/api/nodes/image + */ +export const Image = Node.create({ + name: 'image', + + addOptions() { + return { + inline: false, + allowBase64: false, + HTMLAttributes: {} + }; + }, + + inline() { + return this.options.inline; + }, + + group() { + return this.options.inline ? 'inline' : 'block'; + }, + + draggable: true, + + addAttributes() { + return { + file: { + default: null + }, + src: { + default: null + }, + alt: { + default: null + }, + title: { + default: null + }, + width: { + default: null + }, + height: { + default: null + } + }; + }, + + parseHTML() { + return [ + { + tag: this.options.allowBase64 ? 'img[src]' : 'img[src]:not([src^="data:"])' + } + ]; + }, + + renderHTML({ HTMLAttributes }) { + if (HTMLAttributes.file) { + delete HTMLAttributes.file; + } + + return ['img', mergeAttributes(this.options.HTMLAttributes, HTMLAttributes)]; + }, + + addNodeView() { + return ({ node, editor }) => { + const domImg = document.createElement('img'); + domImg.setAttribute('src', node.attrs.src || ''); + domImg.setAttribute('alt', node.attrs.alt || ''); + domImg.setAttribute('title', node.attrs.title || ''); + + const container = document.createElement('div'); + const img = document.createElement('img'); + + const fileId = node.attrs.src.replace('data://', ''); + img.setAttribute('id', `image:${fileId}`); + + img.classList.add('rounded-md', 'max-h-72', 'w-fit', 'object-contain'); + + const editorFiles = editor.storage?.files || []; + + if (editorFiles && node.attrs.src.startsWith('data://')) { + const file = editorFiles.find((f) => f.id === fileId); + if (file) { + img.setAttribute('src', file.url || ''); + } else { + img.setAttribute('src', node.attrs.src || ''); + } + } else { + img.setAttribute('src', node.attrs.src || ''); + } + + img.setAttribute('alt', node.attrs.alt || ''); + img.setAttribute('title', node.attrs.title || ''); + + img.addEventListener('data', (e) => { + const files = e?.files || []; + if (files && node.attrs.src.startsWith('data://')) { + const file = editorFiles.find((f) => f.id === fileId); + if (file) { + img.setAttribute('src', file.url || ''); + } else { + img.setAttribute('src', node.attrs.src || ''); + } + } + }); + + container.append(img); + return { + dom: img, + contentDOM: domImg + }; + }; + }, + + addCommands() { + return { + setImage: + (options) => + ({ commands }) => { + return commands.insertContent({ + type: this.name, + attrs: options + }); + } + }; + }, + + addInputRules() { + return [ + nodeInputRule({ + find: inputRegex, + type: this.type, + getAttributes: (match) => { + const [, , alt, src, title] = match; + + return { src, alt, title }; + } + }) + ]; + } +}); diff --git a/src/lib/components/common/RichTextInput/Image/index.ts b/src/lib/components/common/RichTextInput/Image/index.ts new file mode 100644 index 0000000000..356dc9198a --- /dev/null +++ b/src/lib/components/common/RichTextInput/Image/index.ts @@ -0,0 +1,5 @@ +import { Image } from './image.js'; + +export * from './image.js'; + +export default Image; diff --git a/src/lib/components/notes/NoteEditor.svelte b/src/lib/components/notes/NoteEditor.svelte index 571669022e..e7b6dd6fcb 100644 --- a/src/lib/components/notes/NoteEditor.svelte +++ b/src/lib/components/notes/NoteEditor.svelte @@ -432,6 +432,14 @@ ${content} note.data.files = null; } + editor.storage.files = files; + // open the settings panel if it is not open + selectedPanel = 'settings'; + + if (!showPanel) { + showPanel = true; + } + changeDebounceHandler(); }; @@ -504,20 +512,39 @@ ${content} imageUrl = await compressImageHandler(imageUrl, $settings, $config); - files = [ - ...files, - { - type: 'image', - url: `${imageUrl}` - } - ]; + const fileId = uuidv4(); + const fileItem = { + id: fileId, + type: 'image', + url: `${imageUrl}` + }; + files = [...files, fileItem]; note.data.files = files; + + if (imageUrl && editor) { + editor.storage.files = files; + editor + ?.chain() + .insertContentAt(editor.state.selection.$anchor.pos, { + type: 'image', + attrs: { + file: fileItem, + src: `data://${fileId}` + + // src: imageUrl + } + }) + .focus() + .run(); + } }; reader.readAsDataURL( file['type'] === 'image/heic' ? await heic2any({ blob: file, toType: 'image/jpeg' }) : file ); + + changeDebounceHandler(); } else { uploadFileHandler(file); } @@ -773,8 +800,47 @@ Provide the enhanced notes in markdown format. Use markdown syntax for headings, inputElement?.insertContent(content); }; + const noteEventHandler = async (_note) => { + console.log('noteEventHandler', _note); + if (_note.id !== id) return; + + if (_note.access_control && _note.access_control !== note.access_control) { + note.access_control = _note.access_control; + } + + if (_note.data && _note.data.files) { + files = _note.data.files; + note.data.files = files; + } + + if (_note.title && _note.title) { + note.title = _note.title; + } + + editor.storage.files = files; + await tick(); + + for (const file of files) { + if (file.type === 'image') { + const e = new CustomEvent('data', { files: files }); + + const img = document.getElementById(`image:${file.id}`); + if (img) { + img.dispatchEvent(e); + } + } + } + }; + onMount(async () => { await tick(); + $socket?.emit('join-note', { + note_id: id, + auth: { + token: localStorage.token + } + }); + $socket?.on('note-events', noteEventHandler); if ($settings?.models) { selectedModelId = $settings?.models[0]; @@ -807,6 +873,8 @@ Provide the enhanced notes in markdown format. Use markdown syntax for headings, onDestroy(() => { console.log('destroy'); + $socket?.off('note-events', noteEventHandler); + const dropzoneElement = document.getElementById('note-editor'); if (dropzoneElement) { @@ -1111,52 +1179,21 @@ Provide the enhanced notes in markdown format. Use markdown syntax for headings, > {/if} - {#if files && files.length > 0} -
- {#each files as file, fileIdx} -
- {#if file.type === 'image'} - { - files = files.filter((item, idx) => idx !== fileIdx); - note.data.files = files.length > 0 ? files : null; - }} - /> - {:else} - { - files = files.filter((item) => item?.id !== file.id); - note.data.files = files.length > 0 ? files : null; - }} - /> - {/if} -
- {/each} -
- {/if} - { @@ -1312,7 +1349,14 @@ Provide the enhanced notes in markdown format. Use markdown syntax for headings, scrollToBottomHandler={scrollToBottom} /> {:else if selectedPanel === 'settings'} - + { + changeDebounceHandler(); + }} + /> {/if} diff --git a/src/lib/components/notes/NoteEditor/Settings.svelte b/src/lib/components/notes/NoteEditor/Settings.svelte index cde5c7af85..738cffab1f 100644 --- a/src/lib/components/notes/NoteEditor/Settings.svelte +++ b/src/lib/components/notes/NoteEditor/Settings.svelte @@ -4,9 +4,17 @@ import XMark from '$lib/components/icons/XMark.svelte'; import { models } from '$lib/stores'; + import Collapsible from '$lib/components/common/Collapsible.svelte'; + import FileItem from '$lib/components/common/FileItem.svelte'; + import Image from '$lib/components/common/Image.svelte'; export let show = false; export let selectedModelId = ''; + export let files = []; + + export let onUpdate = (files: any[]) => { + // Default no-op function + };
@@ -23,13 +31,62 @@
- {$i18n.t('Settings')} + {$i18n.t('Controls')}
-
+
+ {#if files.length > 0} +
Files
+ +
+ {#each files.filter((file) => file.type !== 'image') as file, fileIdx} + { + // Remove the file from the files array + files = files.filter((item) => item.id !== file.id); + files = files; + + onUpdate(files); + }} + on:click={() => { + console.log(file); + }} + /> + {/each} + +
+ {#each files.filter((file) => file.type === 'image') as file, fileIdx} + { + files = files.filter((item) => item.id !== file.id); + files = files; + + onUpdate(files); + }} + /> + {/each} +
+
+ +
+ {/if} +
Model