diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index d2e6a840b1..63ae72f7bc 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -1606,6 +1606,7 @@ async def chat_completion( "user_id": user.id, "chat_id": form_data.pop("chat_id", None), "message_id": form_data.pop("id", None), + "parent_message": form_data.pop("parent_message", None), "parent_message_id": form_data.pop("parent_id", None), "session_id": form_data.pop("session_id", None), "filter_ids": form_data.pop("filter_ids", []), @@ -1630,15 +1631,38 @@ async def chat_completion( }, } - if metadata.get("chat_id") and (user and user.role != "admin"): - if not metadata["chat_id"].startswith("local:"): + if metadata.get("chat_id") and user: + if not metadata["chat_id"].startswith( + "local:" + ): # temporary chats are not stored + + # Verify chat ownership chat = Chats.get_chat_by_id_and_user_id(metadata["chat_id"], user.id) - if chat is None: + if chat is None and user.role != "admin": # admins can access any chat raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.DEFAULT(), ) + # Insert chat files from parent message if any + parent_message = metadata.get("parent_message", {}) + parent_message_files = parent_message.get("files", []) + if parent_message_files: + try: + Chats.insert_chat_files( + metadata["chat_id"], + parent_message.get("id"), + [ + file_item.get("id") + for file_item in parent_message_files + if file_item.get("type") == "file" + ], + user.id, + ) + except Exception as e: + log.debug(f"Error inserting chat files: {e}") + pass + request.state.metadata = metadata form_data["metadata"] = metadata diff --git a/backend/open_webui/migrations/versions/c440947495f3_add_chat_file_table.py b/backend/open_webui/migrations/versions/c440947495f3_add_chat_file_table.py new file mode 100644 index 0000000000..20f4a6d7b6 --- /dev/null +++ b/backend/open_webui/migrations/versions/c440947495f3_add_chat_file_table.py @@ -0,0 +1,57 @@ +"""Add chat_file table + +Revision ID: c440947495f3 +Revises: 81cc2ce44d79 +Create Date: 2025-12-21 20:27:41.694897 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c440947495f3" +down_revision: Union[str, None] = "81cc2ce44d79" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "chat_file", + sa.Column("id", sa.Text(), primary_key=True), + sa.Column("user_id", sa.Text(), nullable=False), + sa.Column( + "chat_id", + sa.Text(), + sa.ForeignKey("chat.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "file_id", + sa.Text(), + sa.ForeignKey("file.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("message_id", sa.Text(), nullable=True), + sa.Column("created_at", sa.BigInteger(), nullable=False), + sa.Column("updated_at", sa.BigInteger(), nullable=False), + # indexes + sa.Index("ix_chat_file_chat_id", "chat_id"), + sa.Index("ix_chat_file_file_id", "file_id"), + sa.Index("ix_chat_file_message_id", "message_id"), + sa.Index("ix_chat_file_user_id", "user_id"), + # unique constraints + sa.UniqueConstraint( + "chat_id", "file_id", name="uq_chat_file_chat_file" + ), # prevent duplicate entries + ) + pass + + +def downgrade() -> None: + op.drop_table("chat_file") + pass diff --git a/backend/open_webui/models/chats.py b/backend/open_webui/models/chats.py index 3c9507e0fc..1397a254c6 100644 --- a/backend/open_webui/models/chats.py +++ b/backend/open_webui/models/chats.py @@ -10,7 +10,17 @@ from open_webui.models.folders import Folders from open_webui.utils.misc import sanitize_data_for_db, sanitize_text_for_db from pydantic import BaseModel, ConfigDict -from sqlalchemy import BigInteger, Boolean, Column, String, Text, JSON, Index +from sqlalchemy import ( + BigInteger, + Boolean, + Column, + ForeignKey, + String, + Text, + JSON, + Index, + UniqueConstraint, +) from sqlalchemy import or_, func, select, and_, text from sqlalchemy.sql import exists from sqlalchemy.sql.expression import bindparam @@ -74,6 +84,38 @@ class ChatModel(BaseModel): folder_id: Optional[str] = None +class ChatFile(Base): + __tablename__ = "chat_file" + + id = Column(Text, unique=True, primary_key=True) + user_id = Column(Text, nullable=False) + + chat_id = Column(Text, ForeignKey("chat.id", ondelete="CASCADE"), nullable=False) + message_id = Column(Text, nullable=True) + file_id = Column(Text, ForeignKey("file.id", ondelete="CASCADE"), nullable=False) + + created_at = Column(BigInteger, nullable=False) + updated_at = Column(BigInteger, nullable=False) + + __table_args__ = ( + UniqueConstraint("chat_id", "file_id", name="uq_chat_file_chat_file"), + ) + + +class ChatFileModel(BaseModel): + id: str + user_id: str + + chat_id: str + message_id: Optional[str] = None + file_id: str + + created_at: int + updated_at: int + + model_config = ConfigDict(from_attributes=True) + + #################### # Forms #################### @@ -1219,5 +1261,81 @@ class ChatTable: except Exception: return False + def insert_chat_files( + self, chat_id: str, message_id: str, file_ids: list[str], user_id: str + ) -> Optional[list[ChatFileModel]]: + if not file_ids: + return None + + chat_message_file_ids = [ + item.id + for item in self.get_chat_files_by_chat_id_and_message_id( + chat_id, message_id + ) + ] + # Remove duplicates and existing file_ids + file_ids = list( + set( + [ + file_id + for file_id in file_ids + if file_id and file_id not in chat_message_file_ids + ] + ) + ) + if not file_ids: + return None + + try: + with get_db() as db: + now = int(time.time()) + + chat_files = [ + ChatFileModel( + id=str(uuid.uuid4()), + user_id=user_id, + chat_id=chat_id, + message_id=message_id, + file_id=file_id, + created_at=now, + updated_at=now, + ) + for file_id in file_ids + ] + + results = [ + ChatFile(**chat_file.model_dump()) for chat_file in chat_files + ] + + db.add_all(results) + db.commit() + + return chat_files + except Exception: + return None + + def get_chat_files_by_chat_id_and_message_id( + self, chat_id: str, message_id: str + ) -> list[ChatFileModel]: + with get_db() as db: + all_chat_files = ( + db.query(ChatFile) + .filter_by(chat_id=chat_id, message_id=message_id) + .order_by(ChatFile.created_at.asc()) + .all() + ) + return [ + ChatFileModel.model_validate(chat_file) for chat_file in all_chat_files + ] + + def delete_chat_file(self, chat_id: str, file_id: str) -> bool: + try: + with get_db() as db: + db.query(ChatFile).filter_by(chat_id=chat_id, file_id=file_id).delete() + db.commit() + return True + except Exception: + return False + Chats = ChatTable() diff --git a/backend/open_webui/utils/files.py b/backend/open_webui/utils/files.py index cd94a41144..2221d1707d 100644 --- a/backend/open_webui/utils/files.py +++ b/backend/open_webui/utils/files.py @@ -22,11 +22,43 @@ import base64 import io import re +import requests BASE64_IMAGE_URL_PREFIX = re.compile(r"data:image/\w+;base64,", re.IGNORECASE) MARKDOWN_IMAGE_URL_PATTERN = re.compile(r"!\[(.*?)\]\((.+?)\)", re.IGNORECASE) +def get_image_base64_from_url(url: str) -> Optional[str]: + try: + if url.startswith("http"): + # Download the image from the URL + response = requests.get(url) + response.raise_for_status() + image_data = response.content + encoded_string = base64.b64encode(image_data).decode("utf-8") + content_type = response.headers.get("Content-Type", "image/png") + return f"data:{content_type};base64,{encoded_string}" + else: + file = Files.get_file_by_id(url) + + if not file: + return None + + file_path = Storage.get_file(file.path) + file_path = Path(file_path) + + if file_path.is_file(): + with open(file_path, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode("utf-8") + content_type, _ = mimetypes.guess_type(file_path.name) + return f"data:{content_type};base64,{encoded_string}" + else: + return None + + except Exception as e: + return None + + def get_image_url_from_base64(request, base64_image_string, metadata, user): if BASE64_IMAGE_URL_PREFIX.match(base64_image_string): image_url = "" diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 5d0cda9e8f..865159208e 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -60,6 +60,7 @@ from open_webui.utils.webhook import post_webhook from open_webui.utils.files import ( convert_markdown_base64_images, get_file_url_from_base64, + get_image_base64_from_url, get_image_url_from_base64, ) @@ -1108,6 +1109,45 @@ def apply_params_to_form_data(form_data, model): return form_data +async def convert_url_images_to_base64(form_data): + messages = form_data.get("messages", []) + + for message in messages: + content = message.get("content") + if not isinstance(content, list): + continue + + new_content = [] + + for item in content: + if not isinstance(item, dict) or item.get("type") != "image_url": + new_content.append(item) + continue + + image_url = item.get("image_url", {}).get("url", "") + if image_url.startswith("data:image/"): + new_content.append(item) + continue + + try: + base64_data = await asyncio.to_thread( + get_image_base64_from_url, image_url + ) + new_content.append( + { + "type": "image_url", + "image_url": {"url": base64_data}, + } + ) + except Exception as e: + log.debug(f"Error converting image URL to base64: {e}") + new_content.append(item) + + message["content"] = new_content + + return form_data + + async def process_chat_payload(request, form_data, user, metadata, model): # Pipeline Inlet -> Filter Inlet -> Chat Memory -> Chat Web Search -> Chat Image Generation # -> Chat Code Interpreter (Form Data Update) -> (Default) Chat Tools Function Calling @@ -1125,6 +1165,8 @@ async def process_chat_payload(request, form_data, user, metadata, model): except: pass + form_data = await convert_url_images_to_base64(form_data) + event_emitter = get_event_emitter(metadata) event_caller = get_event_call(metadata) diff --git a/src/lib/components/channel/MessageInput.svelte b/src/lib/components/channel/MessageInput.svelte index de00b80c63..f962741130 100644 --- a/src/lib/components/channel/MessageInput.svelte +++ b/src/lib/components/channel/MessageInput.svelte @@ -116,7 +116,6 @@ // Check for known image types for (const type of item.types) { if (type.startsWith('image/')) { - // get as file const blob = await item.getType(type); const file = new File([blob], `clipboard-image.${type.split('/')[1]}`, { type: type @@ -486,7 +485,7 @@ fileItem.collection_name = uploadedFile?.meta?.collection_name || uploadedFile?.collection_name; fileItem.content_type = uploadedFile.meta?.content_type || uploadedFile.content_type; - fileItem.url = `${WEBUI_API_BASE_URL}/files/${uploadedFile.id}`; + fileItem.url = `${uploadedFile.id}`; files = files; } else { @@ -802,10 +801,14 @@
{#each files as file, fileIdx} {#if file.type === 'image' || (file?.content_type ?? '').startsWith('image/')} + {@const fileUrl = + file.url.startsWith('data') || file.url.startsWith('http') + ? file.url + : `${WEBUI_API_BASE_URL}/files/${file.url}${file?.content_type ? '/content' : ''}`}
@@ -928,8 +931,7 @@ for (const item of clipboardData.items) { const file = item.getAsFile(); if (file) { - const _files = [file]; - await inputFilesHandler(_files); + await inputFilesHandler([file]); e.preventDefault(); } } diff --git a/src/lib/components/channel/Messages/Message.svelte b/src/lib/components/channel/Messages/Message.svelte index cb1d280319..9a636eabc9 100644 --- a/src/lib/components/channel/Messages/Message.svelte +++ b/src/lib/components/channel/Messages/Message.svelte @@ -343,19 +343,15 @@ dir={$settings?.chatDirection ?? 'auto'} > {#each message?.data?.files as file} + {@const fileUrl = + file.url.startsWith('data') || file.url.startsWith('http') + ? file.url + : `${WEBUI_API_BASE_URL}/files/${file.url}${file?.content_type ? '/content' : ''}`}
{#if file.type === 'image' || (file?.content_type ?? '').startsWith('image/')} - {file.name} + {file.name} {:else if file.type === 'video' || (file?.content_type ?? '').startsWith('video/')} - + {:else} - ['doc', 'text', 'file', 'note', 'chat', 'folder', 'collection'].includes(item.type) + ..._files.filter( + (item) => + ['doc', 'text', 'note', 'chat', 'folder', 'collection'].includes(item.type) || + (item.type === 'file' && !item?.content_type?.startsWith('image/')) ) ); chatFiles = chatFiles.filter( @@ -1730,7 +1732,9 @@ if (model) { // If there are image files, check if model is vision capable const hasImages = createMessagesList(_history, parentId).some((message) => - message.files?.some((file) => file.type === 'image') + message.files?.some( + (file) => file.type === 'image' || file?.content_type?.startsWith('image/') + ) ); if (hasImages && !(model.info?.meta?.capabilities?.vision ?? true)) { @@ -1824,8 +1828,10 @@ let files = JSON.parse(JSON.stringify(chatFiles)); files.push( - ...(userMessage?.files ?? []).filter((item) => - ['doc', 'text', 'file', 'note', 'chat', 'collection'].includes(item.type) + ...(userMessage?.files ?? []).filter( + (item) => + ['doc', 'text', 'note', 'chat', 'collection'].includes(item.type) || + (item.type === 'file' && !item?.content_type.startsWith('image/')) ) ); // Remove duplicates @@ -1872,30 +1878,33 @@ ].filter((message) => message); messages = messages - .map((message, idx, arr) => ({ - role: message.role, - ...((message.files?.filter((file) => file.type === 'image').length > 0 ?? false) && - message.role === 'user' - ? { - content: [ - { - type: 'text', - text: message?.merged?.content ?? message.content - }, - ...message.files - .filter((file) => file.type === 'image') - .map((file) => ({ + .map((message, idx, arr) => { + const imageFiles = (message?.files ?? []).filter( + (file) => file.type === 'image' || (file?.content_type ?? '').startsWith('image/') + ); + + return { + role: message.role, + ...(message.role === 'user' && imageFiles.length > 0 + ? { + content: [ + { + type: 'text', + text: message?.merged?.content ?? message.content + }, + ...imageFiles.map((file) => ({ type: 'image_url', image_url: { url: file.url } })) - ] - } - : { - content: message?.merged?.content ?? message.content - }) - })) + ] + } + : { + content: message?.merged?.content ?? message.content + }) + }; + }) .filter((message) => message?.role === 'user' || message?.content?.trim()); const toolIds = []; @@ -1950,6 +1959,7 @@ id: responseMessageId, parent_id: userMessage?.id ?? null, + parent_message: userMessage, background_tasks: { ...(!$temporaryChatEnabled && diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index d5685d503b..3889f12cc5 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -191,17 +191,11 @@ for (const type of item.types) { if (type.startsWith('image/')) { const blob = await item.getType(type); - const reader = new FileReader(); - reader.onload = (event) => { - files = [ - ...files, - { - type: 'image', - url: event.target.result as string - } - ]; - }; - reader.readAsDataURL(blob); + const file = new File([blob], `clipboard-image.${type.split('/')[1]}`, { + type: type + }); + + inputFilesHandler([file]); } } } @@ -527,8 +521,9 @@ // Convert the canvas to a Base64 image URL const imageUrl = canvas.toDataURL('image/png'); - // Add the captured image to the files array to render it - files = [...files, { type: 'image', url: imageUrl }]; + const blob = await (await fetch(imageUrl)).blob(); + const file = new File([blob], `screen-capture-${Date.now()}.png`, { type: 'image/png' }); + inputFilesHandler([file]); // Clean memory: Clear video srcObject video.srcObject = null; } catch (error) { @@ -537,7 +532,7 @@ } }; - const uploadFileHandler = async (file, fullContext: boolean = false) => { + const uploadFileHandler = async (file, process = true, itemData = {}) => { if ($_user?.role !== 'admin' && !($_user?.permissions?.chat?.file_upload ?? true)) { toast.error($i18n.t('You do not have permission to upload files.')); return null; @@ -560,7 +555,7 @@ size: file.size, error: '', itemId: tempItemId, - ...(fullContext ? { context: 'full' } : {}) + ...itemData }; if (fileItem.size == 0) { @@ -584,7 +579,7 @@ } // During the file upload, file content is automatically extracted. - const uploadedFile = await uploadFile(localStorage.token, file, metadata); + const uploadedFile = await uploadFile(localStorage.token, file, metadata, process); if (uploadedFile) { console.log('File upload completed:', { @@ -603,7 +598,8 @@ fileItem.id = uploadedFile.id; fileItem.collection_name = uploadedFile?.meta?.collection_name || uploadedFile?.collection_name; - fileItem.url = `${WEBUI_API_BASE_URL}/files/${uploadedFile.id}`; + fileItem.content_type = uploadedFile.meta?.content_type || uploadedFile.content_type; + fileItem.url = `${uploadedFile.id}`; files = files; } else { @@ -726,19 +722,21 @@ }; let reader = new FileReader(); + reader.onload = async (event) => { let imageUrl = event.target.result; - imageUrl = await compressImageHandler(imageUrl, $settings, $config); + // Compress the image if settings or config require it + if ($settings?.imageCompression && $settings?.imageCompressionInChannels) { + imageUrl = await compressImageHandler(imageUrl, $settings, $config); + } - files = [ - ...files, - { - type: 'image', - url: `${imageUrl}` - } - ]; + const blob = await (await fetch(imageUrl)).blob(); + const compressedFile = new File([blob], file.name, { type: file.type }); + + uploadFileHandler(compressedFile, false); }; + reader.readAsDataURL(file['type'] === 'image/heic' ? await convertHeicToJpeg(file) : file); } else { uploadFileHandler(file); @@ -1146,11 +1144,15 @@ dir={$settings?.chatDirection ?? 'auto'} > {#each files as file, fileIdx} - {#if file.type === 'image'} + {#if file.type === 'image' || (file?.content_type ?? '').startsWith('image/')} + {@const fileUrl = + file.url.startsWith('data') || file.url.startsWith('http') + ? file.url + : `${WEBUI_API_BASE_URL}/files/${file.url}${file?.content_type ? '/content' : ''}`}
@@ -1392,29 +1394,7 @@ if (clipboardData && clipboardData.items) { for (const item of clipboardData.items) { - if (item.type.indexOf('image') !== -1) { - const blob = item.getAsFile(); - const reader = new FileReader(); - - reader.onload = function (e) { - files = [ - ...files, - { - type: 'image', - url: `${e.target.result}` - } - ]; - }; - - reader.readAsDataURL(blob); - } else if (item?.kind === 'file') { - const file = item.getAsFile(); - if (file) { - const _files = [file]; - await inputFilesHandler(_files); - e.preventDefault(); - } - } else if (item.type === 'text/plain') { + if (item.type === 'text/plain') { if (($settings?.largeTextAsFile ?? false) && !shiftKey) { const text = clipboardData.getData('text/plain'); @@ -1429,9 +1409,15 @@ } ); - await uploadFileHandler(file, true); + await uploadFileHandler(file, true, { context: 'full' }); } } + } else { + const file = item.getAsFile(); + if (file) { + await inputFilesHandler([file]); + e.preventDefault(); + } } } } diff --git a/src/lib/components/chat/Messages/UserMessage.svelte b/src/lib/components/chat/Messages/UserMessage.svelte index 41f205f2d1..fd2b3fb27a 100644 --- a/src/lib/components/chat/Messages/UserMessage.svelte +++ b/src/lib/components/chat/Messages/UserMessage.svelte @@ -193,9 +193,13 @@ dir={$settings?.chatDirection ?? 'auto'} > {#each message.files as file} + {@const fileUrl = + file.url.startsWith('data') || file.url.startsWith('http') + ? file.url + : `${WEBUI_API_BASE_URL}/files/${file.url}${file?.content_type ? '/content' : ''}`}
- {#if file.type === 'image'} - + {#if file.type === 'image' || (file?.content_type ?? '').startsWith('image/')} + {:else} import { createEventDispatcher, getContext } from 'svelte'; + import { WEBUI_API_BASE_URL } from '$lib/constants'; + import { formatFileSize } from '$lib/utils'; import { settings } from '$lib/stores'; @@ -60,7 +62,11 @@ } else { if (url) { if (type === 'file') { - window.open(`${url}/content`, '_blank').focus(); + if (url.startsWith('http')) { + window.open(`${url}/content`, '_blank').focus(); + } else { + window.open(`${WEBUI_API_BASE_URL}/files/${url}/content`, '_blank').focus(); + } } else { window.open(`${url}`, '_blank').focus(); } diff --git a/src/lib/components/common/FileItemModal.svelte b/src/lib/components/common/FileItemModal.svelte index 828eb43a9e..33a9d3b3f4 100644 --- a/src/lib/components/common/FileItemModal.svelte +++ b/src/lib/components/common/FileItemModal.svelte @@ -197,7 +197,11 @@ on:click|preventDefault={() => { if (!isPDF && item.url) { window.open( - item.type === 'file' ? `${item.url}/content` : `${item.url}`, + item.type === 'file' + ? item?.url?.startsWith('http') + ? item.url + : `${WEBUI_API_BASE_URL}/files/${item.url}/content` + : item.url, '_blank' ); } diff --git a/src/lib/components/notes/NoteEditor.svelte b/src/lib/components/notes/NoteEditor.svelte index 9e0c02b0fc..37620f75e1 100644 --- a/src/lib/components/notes/NoteEditor.svelte +++ b/src/lib/components/notes/NoteEditor.svelte @@ -442,7 +442,7 @@ ${content} fileItem.collection_name = uploadedFile?.meta?.collection_name || uploadedFile?.collection_name; - fileItem.url = `${WEBUI_API_BASE_URL}/files/${uploadedFile.id}`; + fileItem.url = `${uploadedFile.id}`; files = files; } else { diff --git a/src/lib/components/workspace/Models/Knowledge.svelte b/src/lib/components/workspace/Models/Knowledge.svelte index 82bbb12d69..eb19d767cf 100644 --- a/src/lib/components/workspace/Models/Knowledge.svelte +++ b/src/lib/components/workspace/Models/Knowledge.svelte @@ -80,7 +80,7 @@ fileItem.id = uploadedFile.id; fileItem.collection_name = uploadedFile?.meta?.collection_name || uploadedFile?.collection_name; - fileItem.url = `${WEBUI_API_BASE_URL}/files/${uploadedFile.id}`; + fileItem.url = `${uploadedFile.id}`; selectedItems = selectedItems; } else {