diff --git a/backend/open_webui/models/files.py b/backend/open_webui/models/files.py index 1ed743df87..79117b869a 100644 --- a/backend/open_webui/models/files.py +++ b/backend/open_webui/models/files.py @@ -238,6 +238,7 @@ class FilesTable: try: file = db.query(File).filter_by(id=id).first() file.hash = hash + file.updated_at = int(time.time()) db.commit() return FileModel.model_validate(file) @@ -249,6 +250,7 @@ class FilesTable: try: file = db.query(File).filter_by(id=id).first() file.data = {**(file.data if file.data else {}), **data} + file.updated_at = int(time.time()) db.commit() return FileModel.model_validate(file) except Exception as e: @@ -260,6 +262,7 @@ class FilesTable: try: file = db.query(File).filter_by(id=id).first() file.meta = {**(file.meta if file.meta else {}), **meta} + file.updated_at = int(time.time()) db.commit() return FileModel.model_validate(file) except Exception: diff --git a/backend/open_webui/models/knowledge.py b/backend/open_webui/models/knowledge.py index 2c72401181..cc3ef7adee 100644 --- a/backend/open_webui/models/knowledge.py +++ b/backend/open_webui/models/knowledge.py @@ -7,9 +7,14 @@ import uuid from open_webui.internal.db import Base, get_db from open_webui.env import SRC_LOG_LEVELS -from open_webui.models.files import File, FileModel, FileMetadataResponse +from open_webui.models.files import ( + File, + FileModel, + FileMetadataResponse, + FileModelResponse, +) from open_webui.models.groups import Groups -from open_webui.models.users import Users, UserResponse +from open_webui.models.users import User, UserModel, Users, UserResponse from pydantic import BaseModel, ConfigDict @@ -21,6 +26,7 @@ from sqlalchemy import ( Text, JSON, UniqueConstraint, + or_, ) from open_webui.utils.access_control import has_access @@ -135,6 +141,15 @@ class KnowledgeForm(BaseModel): access_control: Optional[dict] = None +class FileUserResponse(FileModelResponse): + user: Optional[UserResponse] = None + + +class KnowledgeFileListResponse(BaseModel): + items: list[FileUserResponse] + total: int + + class KnowledgeTable: def insert_new_knowledge( self, user_id: str, form_data: KnowledgeForm @@ -232,6 +247,88 @@ class KnowledgeTable: except Exception: return [] + def search_files_by_id( + self, + knowledge_id: str, + user_id: str, + filter: dict, + skip: int = 0, + limit: int = 30, + ) -> KnowledgeFileListResponse: + try: + with get_db() as db: + query = ( + db.query(File, User) + .join(KnowledgeFile, File.id == KnowledgeFile.file_id) + .outerjoin(User, User.id == KnowledgeFile.user_id) + .filter(KnowledgeFile.knowledge_id == knowledge_id) + ) + + if filter: + query_key = filter.get("query") + if query_key: + query = query.filter(or_(File.filename.ilike(f"%{query_key}%"))) + + view_option = filter.get("view_option") + if view_option == "created": + query = query.filter(KnowledgeFile.user_id == user_id) + elif view_option == "shared": + query = query.filter(KnowledgeFile.user_id != user_id) + + order_by = filter.get("order_by") + direction = filter.get("direction") + + if order_by == "name": + if direction == "asc": + query = query.order_by(File.filename.asc()) + else: + query = query.order_by(File.filename.desc()) + elif order_by == "created_at": + if direction == "asc": + query = query.order_by(File.created_at.asc()) + else: + query = query.order_by(File.created_at.desc()) + elif order_by == "updated_at": + if direction == "asc": + query = query.order_by(File.updated_at.asc()) + else: + query = query.order_by(File.updated_at.desc()) + else: + query = query.order_by(File.updated_at.desc()) + + else: + query = query.order_by(File.updated_at.desc()) + + # Count BEFORE pagination + total = query.count() + + if skip: + query = query.offset(skip) + if limit: + query = query.limit(limit) + + items = query.all() + + files = [] + for file, user in items: + files.append( + FileUserResponse( + **FileModel.model_validate(file).model_dump(), + user=( + UserResponse( + **UserModel.model_validate(user).model_dump() + ) + if user + else None + ), + ) + ) + + return KnowledgeFileListResponse(items=files, total=total) + except Exception as e: + print(e) + return KnowledgeFileListResponse(items=[], total=0) + def get_files_by_id(self, knowledge_id: str) -> list[FileModel]: try: with get_db() as db: diff --git a/backend/open_webui/models/notes.py b/backend/open_webui/models/notes.py index d61094b6ff..2ca932ff7f 100644 --- a/backend/open_webui/models/notes.py +++ b/backend/open_webui/models/notes.py @@ -302,9 +302,6 @@ class NoteTable: else: query = query.order_by(Note.updated_at.desc()) - for key, value in filter.items(): - query = query.filter(getattr(Note, key).ilike(f"%{value}%")) - # Count BEFORE pagination total = query.count() diff --git a/backend/open_webui/models/users.py b/backend/open_webui/models/users.py index 86f9d011e8..5807603a89 100644 --- a/backend/open_webui/models/users.py +++ b/backend/open_webui/models/users.py @@ -5,11 +5,11 @@ from open_webui.internal.db import Base, JSONField, get_db from open_webui.env import DATABASE_USER_ACTIVE_STATUS_UPDATE_INTERVAL + from open_webui.models.chats import Chats from open_webui.models.groups import Groups, GroupMember from open_webui.models.channels import ChannelMember - from open_webui.utils.misc import throttle diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 3bfc961ac3..f67390518b 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -5,6 +5,7 @@ from fastapi.concurrency import run_in_threadpool import logging from open_webui.models.knowledge import ( + KnowledgeFileListResponse, Knowledges, KnowledgeForm, KnowledgeResponse, @@ -264,6 +265,59 @@ async def update_knowledge_by_id( ) +############################ +# GetKnowledgeFilesById +############################ + + +@router.get("/{id}/files", response_model=KnowledgeFileListResponse) +async def get_knowledge_files_by_id( + id: str, + query: Optional[str] = None, + view_option: Optional[str] = None, + order_by: Optional[str] = None, + direction: Optional[str] = None, + page: Optional[int] = 1, + user=Depends(get_verified_user), +): + + knowledge = Knowledges.get_knowledge_by_id(id=id) + if not knowledge: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.NOT_FOUND, + ) + + if not ( + user.role == "admin" + or knowledge.user_id == user.id + or has_access(user.id, "read", knowledge.access_control) + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.ACCESS_PROHIBITED, + ) + + page = max(page, 1) + + limit = 30 + skip = (page - 1) * limit + + filter = {} + if query: + filter["query"] = query + if view_option: + filter["view_option"] = view_option + if order_by: + filter["order_by"] = order_by + if direction: + filter["direction"] = direction + + return Knowledges.search_files_by_id( + id, user.id, filter=filter, skip=skip, limit=limit + ) + + ############################ # AddFileToKnowledge ############################ diff --git a/src/lib/apis/knowledge/index.ts b/src/lib/apis/knowledge/index.ts index c01c986a2a..98b2c1e5ec 100644 --- a/src/lib/apis/knowledge/index.ts +++ b/src/lib/apis/knowledge/index.ts @@ -132,6 +132,56 @@ export const getKnowledgeById = async (token: string, id: string) => { return res; }; +export const searchKnowledgeFilesById = async ( + token: string, + id: string, + query?: string | null = null, + viewOption?: string | null = null, + orderBy?: string | null = null, + direction?: string | null = null, + page: number = 1 +) => { + let error = null; + + const searchParams = new URLSearchParams(); + if (query) searchParams.append('query', query); + if (viewOption) searchParams.append('view_option', viewOption); + if (orderBy) searchParams.append('order_by', orderBy); + if (direction) searchParams.append('direction', direction); + searchParams.append('page', page.toString()); + + const res = await fetch( + `${WEBUI_API_BASE_URL}/knowledge/${id}/files?${searchParams.toString()}`, + { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + } + ) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + type KnowledgeUpdateForm = { name?: string; description?: string; diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index d09e608d51..1dff4bd8f8 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -31,7 +31,8 @@ removeFileFromKnowledgeById, resetKnowledgeById, updateFileFromKnowledgeById, - updateKnowledgeById + updateKnowledgeById, + searchKnowledgeFilesById } from '$lib/apis/knowledge'; import { blobToFile } from '$lib/utils'; @@ -43,22 +44,25 @@ import AddTextContentModal from './KnowledgeBase/AddTextContentModal.svelte'; import SyncConfirmDialog from '../../common/ConfirmDialog.svelte'; - import RichTextInput from '$lib/components/common/RichTextInput.svelte'; - import EllipsisVertical from '$lib/components/icons/EllipsisVertical.svelte'; import Drawer from '$lib/components/common/Drawer.svelte'; import ChevronLeft from '$lib/components/icons/ChevronLeft.svelte'; import LockClosed from '$lib/components/icons/LockClosed.svelte'; import AccessControlModal from '../common/AccessControlModal.svelte'; import Search from '$lib/components/icons/Search.svelte'; - import Textarea from '$lib/components/common/Textarea.svelte'; import FilesOverlay from '$lib/components/chat/MessageInput/FilesOverlay.svelte'; + import DropdownOptions from '$lib/components/common/DropdownOptions.svelte'; + import Pagination from '$lib/components/common/Pagination.svelte'; let largeScreen = true; let pane; let showSidepanel = true; - let minSize = 0; + let showAddTextContentModal = false; + let showSyncConfirmModal = false; + let showAccessControlModal = false; + + let minSize = 0; type Knowledge = { id: string; name: string; @@ -71,52 +75,88 @@ let id = null; let knowledge: Knowledge | null = null; - let query = ''; - let showAddTextContentModal = false; - let showSyncConfirmModal = false; - let showAccessControlModal = false; + let selectedFileId = null; + let selectedFile = null; + let selectedFileContent = ''; let inputFiles = null; - let filteredItems = []; - $: if (knowledge && knowledge.files) { - fuse = new Fuse(knowledge.files, { - keys: ['meta.name', 'meta.description'] - }); + let query = ''; + let viewOption = null; + let sortKey = null; + let direction = null; + + let currentPage = 1; + let fileItems = null; + let fileItemsTotal = null; + + const reset = () => { + currentPage = 1; + }; + + const init = async () => { + reset(); + await getItemsPage(); + }; + + $: if ( + knowledge !== null && + query !== undefined && + viewOption !== undefined && + sortKey !== undefined && + direction !== undefined && + currentPage !== undefined + ) { + getItemsPage(); } - $: if (fuse) { - filteredItems = query - ? fuse.search(query).map((e) => { - return e.item; - }) - : (knowledge?.files ?? []); + $: if ( + query !== undefined && + viewOption !== undefined && + sortKey !== undefined && + direction !== undefined + ) { + reset(); } - let selectedFile = null; - let selectedFileId = null; - let selectedFileContent = ''; + const getItemsPage = async () => { + if (knowledge === null) return; - // Add cache object - let fileContentCache = new Map(); + fileItems = null; + fileItemsTotal = null; - $: if (selectedFileId) { - const file = (knowledge?.files ?? []).find((file) => file.id === selectedFileId); - if (file) { - fileSelectHandler(file); - } else { - selectedFile = null; + if (sortKey === null) { + direction = null; } - } else { - selectedFile = null; - } - let fuse = null; - let debounceTimeout = null; - let mediaQuery; - let dragged = false; - let isSaving = false; + const res = await searchKnowledgeFilesById( + localStorage.token, + knowledge.id, + query, + viewOption, + sortKey, + direction, + currentPage + ).catch(() => { + return null; + }); + + if (res) { + fileItems = res.items; + fileItemsTotal = res.total; + } + return res; + }; + + const fileSelectHandler = async (file) => { + try { + selectedFile = file; + selectedFileContent = selectedFile?.data?.content || ''; + } catch (e) { + toast.error($i18n.t('Failed to load file content.')); + } + }; const createFileFromText = (name, content) => { const blob = new Blob([content], { type: 'text/plain' }); @@ -163,8 +203,7 @@ return; } - knowledge.files = [...(knowledge.files ?? []), fileItem]; - + fileItems = [...(fileItems ?? []), fileItem]; try { // If the file is an audio file, provide the language for STT. let metadata = null; @@ -184,7 +223,7 @@ if (uploadedFile) { console.log(uploadedFile); - knowledge.files = knowledge.files.map((item) => { + fileItems = fileItems.map((item) => { if (item.itemId === tempItemId) { item.id = uploadedFile.id; } @@ -197,7 +236,7 @@ if (uploadedFile.error) { console.warn('File upload warning:', uploadedFile.error); toast.warning(uploadedFile.error); - knowledge.files = knowledge.files.filter((file) => file.id !== uploadedFile.id); + fileItems = fileItems.filter((file) => file.id !== uploadedFile.id); } else { await addFileHandler(uploadedFile.id); } @@ -413,7 +452,7 @@ toast.success($i18n.t('File added successfully.')); } else { toast.error($i18n.t('Failed to add file.')); - knowledge.files = knowledge.files.filter((file) => file.id !== fileId); + fileItems = fileItems.filter((file) => file.id !== fileId); } }; @@ -436,32 +475,38 @@ } }; + let debounceTimeout = null; + let mediaQuery; + + let dragged = false; + let isSaving = false; + const updateFileContentHandler = async () => { if (isSaving) { console.log('Save operation already in progress, skipping...'); return; } + isSaving = true; + try { - const fileId = selectedFile.id; - const content = selectedFileContent; - // Clear the cache for this file since we're updating it - fileContentCache.delete(fileId); - const res = await updateFileDataContentById(localStorage.token, fileId, content).catch( - (e) => { - toast.error(`${e}`); - } - ); - const updatedKnowledge = await updateFileFromKnowledgeById( + const res = await updateFileDataContentById( localStorage.token, - id, - fileId + selectedFile.id, + selectedFileContent ).catch((e) => { toast.error(`${e}`); + return null; }); - if (res && updatedKnowledge) { - knowledge = updatedKnowledge; + + if (res) { toast.success($i18n.t('File content updated successfully.')); + + selectedFileId = null; + selectedFile = null; + selectedFileContent = ''; + + await init(); } } finally { isSaving = false; @@ -504,29 +549,6 @@ } }; - const fileSelectHandler = async (file) => { - try { - selectedFile = file; - - // Check cache first - if (fileContentCache.has(file.id)) { - selectedFileContent = fileContentCache.get(file.id); - return; - } - - const response = await getFileById(localStorage.token, file.id); - if (response) { - selectedFileContent = response.data.content; - // Cache the content - fileContentCache.set(file.id, response.data.content); - } else { - toast.error($i18n.t('No content found in file.')); - } - } catch (e) { - toast.error($i18n.t('Failed to load file content.')); - } - }; - const onDragOver = (e) => { e.preventDefault(); @@ -705,32 +727,42 @@ }} /> -
+
{#if id && knowledge} { changeDebounceHandler(); }} accessRoles={['read', 'write']} /> -
+
-
-
+
+
{ changeDebounceHandler(); }} /> + +
+ {#if (knowledge?.files ?? []).length} +
+ {$i18n.t('{{count}} files', { + count: (knowledge?.files ?? []).length + })} +
+ {/if} +
@@ -750,7 +782,7 @@
-
+
-
- {#if largeScreen} -
- {#if selectedFile} -
-
- {#if !showSidepanel} -
- -
- {/if} +
+
+
+
+ +
+ { + selectedFileId = null; + }} + /> - - -
- -
-
- -
- {#key selectedFile.id} -