From ceae3d48e603f53313d5483abe94099e20e914e8 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Wed, 10 Dec 2025 23:19:19 -0500 Subject: [PATCH] enh/refac: kb pagination --- backend/open_webui/models/files.py | 5 + backend/open_webui/models/knowledge.py | 133 ++++++- backend/open_webui/routers/files.py | 1 - backend/open_webui/routers/knowledge.py | 144 +++++--- backend/open_webui/utils/db/access_control.py | 130 +++++++ src/lib/apis/knowledge/index.ts | 70 +++- .../MessageInput/CommandSuggestionList.svelte | 4 - .../MessageInput/Commands/Knowledge.svelte | 32 +- .../chat/MessageInput/InputMenu.svelte | 64 ++-- .../MessageInput/InputMenu/Knowledge.svelte | 342 ++++++++++++++---- src/lib/components/workspace/Knowledge.svelte | 280 +++++++------- .../Knowledge/CreateKnowledgeBase.svelte | 9 +- .../workspace/Knowledge/KnowledgeBase.svelte | 2 - .../Knowledge/KnowledgeBase/Files.svelte | 6 +- .../workspace/Models/Knowledge.svelte | 10 +- .../Models/Knowledge/KnowledgeSelector.svelte | 190 ++++++++++ .../Models/Knowledge/Selector.svelte | 186 ---------- .../workspace/Models/ModelEditor.svelte | 4 +- 18 files changed, 1086 insertions(+), 526 deletions(-) create mode 100644 backend/open_webui/utils/db/access_control.py create mode 100644 src/lib/components/workspace/Models/Knowledge/KnowledgeSelector.svelte delete mode 100644 src/lib/components/workspace/Models/Knowledge/Selector.svelte diff --git a/backend/open_webui/models/files.py b/backend/open_webui/models/files.py index 79117b869a..0eb106501a 100644 --- a/backend/open_webui/models/files.py +++ b/backend/open_webui/models/files.py @@ -104,6 +104,11 @@ class FileUpdateForm(BaseModel): meta: Optional[dict] = None +class FileListResponse(BaseModel): + items: list[FileModel] + total: int + + class FilesTable: def insert_new_file(self, user_id: str, form_data: FileForm) -> Optional[FileModel]: with get_db() as db: diff --git a/backend/open_webui/models/knowledge.py b/backend/open_webui/models/knowledge.py index 42370c0d7e..3775f18093 100644 --- a/backend/open_webui/models/knowledge.py +++ b/backend/open_webui/models/knowledge.py @@ -5,6 +5,7 @@ from typing import Optional import uuid from open_webui.internal.db import Base, get_db + from open_webui.env import SRC_LOG_LEVELS from open_webui.models.files import ( @@ -30,6 +31,8 @@ from sqlalchemy import ( ) from open_webui.utils.access_control import has_access +from open_webui.utils.db.access_control import has_permission + log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["MODELS"]) @@ -145,6 +148,11 @@ class FileUserResponse(FileModelResponse): user: Optional[UserResponse] = None +class KnowledgeListResponse(BaseModel): + items: list[KnowledgeUserModel] + total: int + + class KnowledgeFileListResponse(BaseModel): items: list[FileUserResponse] total: int @@ -177,12 +185,13 @@ class KnowledgeTable: except Exception: return None - def get_knowledge_bases(self) -> list[KnowledgeUserModel]: + def get_knowledge_bases( + self, skip: int = 0, limit: int = 30 + ) -> list[KnowledgeUserModel]: with get_db() as db: all_knowledge = ( db.query(Knowledge).order_by(Knowledge.updated_at.desc()).all() ) - user_ids = list(set(knowledge.user_id for knowledge in all_knowledge)) users = Users.get_users_by_user_ids(user_ids) if user_ids else [] @@ -201,6 +210,126 @@ class KnowledgeTable: ) return knowledge_bases + def search_knowledge_bases( + self, user_id: str, filter: dict, skip: int = 0, limit: int = 30 + ) -> KnowledgeListResponse: + try: + with get_db() as db: + query = db.query(Knowledge, User).outerjoin( + User, User.id == Knowledge.user_id + ) + + if filter: + query_key = filter.get("query") + if query_key: + query = query.filter( + or_( + Knowledge.name.ilike(f"%{query_key}%"), + Knowledge.description.ilike(f"%{query_key}%"), + ) + ) + + view_option = filter.get("view_option") + if view_option == "created": + query = query.filter(Knowledge.user_id == user_id) + elif view_option == "shared": + query = query.filter(Knowledge.user_id != user_id) + + query = has_permission(db, Knowledge, query, filter) + + query = query.order_by(Knowledge.updated_at.desc()) + + total = query.count() + if skip: + query = query.offset(skip) + if limit: + query = query.limit(limit) + + items = query.all() + + knowledge_bases = [] + for knowledge_base, user in items: + knowledge_bases.append( + KnowledgeUserModel.model_validate( + { + **KnowledgeModel.model_validate( + knowledge_base + ).model_dump(), + "user": ( + UserModel.model_validate(user).model_dump() + if user + else None + ), + } + ) + ) + + return KnowledgeListResponse(items=knowledge_bases, total=total) + except Exception as e: + print(e) + return KnowledgeListResponse(items=[], total=0) + + def search_knowledge_files( + self, filter: dict, skip: int = 0, limit: int = 30 + ) -> KnowledgeFileListResponse: + """ + Scalable version: search files across all knowledge bases the user has + READ access to, without loading all KBs or using large IN() lists. + """ + try: + with get_db() as db: + # Base query: join Knowledge → KnowledgeFile → File + query = ( + db.query(File, User) + .join(KnowledgeFile, File.id == KnowledgeFile.file_id) + .join(Knowledge, KnowledgeFile.knowledge_id == Knowledge.id) + .outerjoin(User, User.id == KnowledgeFile.user_id) + ) + + # Apply access-control directly to the joined query + # This makes the database handle filtering, even with 10k+ KBs + query = has_permission(db, Knowledge, query, filter) + + # Apply filename search + if filter: + q = filter.get("query") + if q: + query = query.filter(File.filename.ilike(f"%{q}%")) + + # Order by file changes + query = query.order_by(File.updated_at.desc()) + + # Count before pagination + total = query.count() + + if skip: + query = query.offset(skip) + if limit: + query = query.limit(limit) + + rows = query.all() + + items = [] + for file, user in rows: + items.append( + FileUserResponse( + **FileModel.model_validate(file).model_dump(), + user=( + UserResponse( + **UserModel.model_validate(user).model_dump() + ) + if user + else None + ), + ) + ) + + return KnowledgeFileListResponse(items=items, total=total) + + except Exception as e: + print("search_knowledge_files error:", e) + return KnowledgeFileListResponse(items=[], total=0) + def check_access_by_user_id(self, id, user_id, permission="write") -> bool: knowledge = self.get_knowledge_by_id(id) if not knowledge: diff --git a/backend/open_webui/routers/files.py b/backend/open_webui/routers/files.py index bbb144a9cf..3bb28b95d6 100644 --- a/backend/open_webui/routers/files.py +++ b/backend/open_webui/routers/files.py @@ -39,7 +39,6 @@ from open_webui.models.knowledge import Knowledges from open_webui.models.groups import Groups -from open_webui.routers.knowledge import get_knowledge, get_knowledge_list from open_webui.routers.retrieval import ProcessFileForm, process_file from open_webui.routers.audio import transcribe diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 4a1a5f01d7..0801deb470 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends, HTTPException, status, Request, Query from fastapi.concurrency import run_in_threadpool import logging +from open_webui.models.groups import Groups from open_webui.models.knowledge import ( KnowledgeFileListResponse, Knowledges, @@ -40,53 +41,115 @@ router = APIRouter() # getKnowledgeBases ############################ +PAGE_ITEM_COUNT = 30 + class KnowledgeAccessResponse(KnowledgeUserResponse): write_access: Optional[bool] = False -@router.get("/", response_model=list[KnowledgeAccessResponse]) -async def get_knowledge(user=Depends(get_verified_user)): - # Return knowledge bases with read access - knowledge_bases = [] - if user.role == "admin" and BYPASS_ADMIN_ACCESS_CONTROL: - knowledge_bases = Knowledges.get_knowledge_bases() - else: - knowledge_bases = Knowledges.get_knowledge_bases_by_user_id(user.id, "read") - - return [ - KnowledgeAccessResponse( - **knowledge_base.model_dump(), - files=Knowledges.get_file_metadatas_by_id(knowledge_base.id), - write_access=( - user.id == knowledge_base.user_id - or has_access(user.id, "write", knowledge_base.access_control) - ), - ) - for knowledge_base in knowledge_bases - ] +class KnowledgeAccessListResponse(BaseModel): + items: list[KnowledgeAccessResponse] + total: int -@router.get("/list", response_model=list[KnowledgeAccessResponse]) -async def get_knowledge_list(user=Depends(get_verified_user)): - # Return knowledge bases with write access - knowledge_bases = [] - if user.role == "admin" and BYPASS_ADMIN_ACCESS_CONTROL: - knowledge_bases = Knowledges.get_knowledge_bases() - else: - knowledge_bases = Knowledges.get_knowledge_bases_by_user_id(user.id, "read") +@router.get("/", response_model=KnowledgeAccessListResponse) +async def get_knowledge_bases(page: Optional[int] = 1, user=Depends(get_verified_user)): + page = max(page, 1) + limit = PAGE_ITEM_COUNT + skip = (page - 1) * limit - return [ - KnowledgeAccessResponse( - **knowledge_base.model_dump(), - files=Knowledges.get_file_metadatas_by_id(knowledge_base.id), - write_access=( - user.id == knowledge_base.user_id - or has_access(user.id, "write", knowledge_base.access_control) - ), - ) - for knowledge_base in knowledge_bases - ] + filter = {} + if not user.role == "admin" or not BYPASS_ADMIN_ACCESS_CONTROL: + groups = Groups.get_groups_by_member_id(user.id) + if groups: + filter["group_ids"] = [group.id for group in groups] + + filter["user_id"] = user.id + + result = Knowledges.search_knowledge_bases( + user.id, filter=filter, skip=skip, limit=limit + ) + + return KnowledgeAccessListResponse( + items=[ + KnowledgeAccessResponse( + **knowledge_base.model_dump(), + write_access=( + user.id == knowledge_base.user_id + or has_access(user.id, "write", knowledge_base.access_control) + ), + ) + for knowledge_base in result.items + ], + total=result.total, + ) + + +@router.get("/search", response_model=KnowledgeAccessListResponse) +async def search_knowledge_bases( + query: Optional[str] = None, + view_option: Optional[str] = None, + page: Optional[int] = 1, + user=Depends(get_verified_user), +): + page = max(page, 1) + limit = PAGE_ITEM_COUNT + skip = (page - 1) * limit + + filter = {} + if query: + filter["query"] = query + if view_option: + filter["view_option"] = view_option + + if not user.role == "admin" or not BYPASS_ADMIN_ACCESS_CONTROL: + groups = Groups.get_groups_by_member_id(user.id) + if groups: + filter["group_ids"] = [group.id for group in groups] + + filter["user_id"] = user.id + + result = Knowledges.search_knowledge_bases( + user.id, filter=filter, skip=skip, limit=limit + ) + + return KnowledgeAccessListResponse( + items=[ + KnowledgeAccessResponse( + **knowledge_base.model_dump(), + write_access=( + user.id == knowledge_base.user_id + or has_access(user.id, "write", knowledge_base.access_control) + ), + ) + for knowledge_base in result.items + ], + total=result.total, + ) + + +@router.get("/search/files", response_model=KnowledgeFileListResponse) +async def search_knowledge_files( + query: Optional[str] = None, + page: Optional[int] = 1, + user=Depends(get_verified_user), +): + page = max(page, 1) + limit = PAGE_ITEM_COUNT + skip = (page - 1) * limit + + filter = {} + if query: + filter["query"] = query + + groups = Groups.get_groups_by_member_id(user.id) + if groups: + filter["group_ids"] = [group.id for group in groups] + + filter["user_id"] = user.id + + return Knowledges.search_knowledge_files(filter=filter, skip=skip, limit=limit) ############################ @@ -198,7 +261,7 @@ async def reindex_knowledge_files(request: Request, user=Depends(get_verified_us class KnowledgeFilesResponse(KnowledgeResponse): - files: list[FileMetadataResponse] + files: Optional[list[FileMetadataResponse]] = None write_access: Optional[bool] = False @@ -215,7 +278,6 @@ async def get_knowledge_by_id(id: str, user=Depends(get_verified_user)): return KnowledgeFilesResponse( **knowledge.model_dump(), - files=Knowledges.get_file_metadatas_by_id(knowledge.id), write_access=( user.id == knowledge.user_id or has_access(user.id, "write", knowledge.access_control) diff --git a/backend/open_webui/utils/db/access_control.py b/backend/open_webui/utils/db/access_control.py new file mode 100644 index 0000000000..d2e6151e5b --- /dev/null +++ b/backend/open_webui/utils/db/access_control.py @@ -0,0 +1,130 @@ +from pydantic import BaseModel, ConfigDict +from sqlalchemy import BigInteger, Boolean, Column, String, Text, JSON +from sqlalchemy.dialects.postgresql import JSONB + + +from sqlalchemy import or_, func, select, and_, text, cast, or_, and_, func + + +def has_permission(db, DocumentModel, query, filter: dict, permission: str = "read"): + group_ids = filter.get("group_ids", []) + user_id = filter.get("user_id") + dialect_name = db.bind.dialect.name + + conditions = [] + + # Handle read_only permission separately + if permission == "read_only": + # For read_only, we want items where: + # 1. User has explicit read permission (via groups or user-level) + # 2. BUT does NOT have write permission + # 3. Public items are NOT considered read_only + + read_conditions = [] + + # Group-level read permission + if group_ids: + group_read_conditions = [] + for gid in group_ids: + if dialect_name == "sqlite": + group_read_conditions.append( + DocumentModel.access_control["read"]["group_ids"].contains( + [gid] + ) + ) + elif dialect_name == "postgresql": + group_read_conditions.append( + cast( + DocumentModel.access_control["read"]["group_ids"], + JSONB, + ).contains([gid]) + ) + + if group_read_conditions: + read_conditions.append(or_(*group_read_conditions)) + + # Combine read conditions + if read_conditions: + has_read = or_(*read_conditions) + else: + # If no read conditions, return empty result + return query.filter(False) + + # Now exclude items where user has write permission + write_exclusions = [] + + # Exclude items owned by user (they have implicit write) + if user_id: + write_exclusions.append(DocumentModel.user_id != user_id) + + # Exclude items where user has explicit write permission via groups + if group_ids: + group_write_conditions = [] + for gid in group_ids: + if dialect_name == "sqlite": + group_write_conditions.append( + DocumentModel.access_control["write"]["group_ids"].contains( + [gid] + ) + ) + elif dialect_name == "postgresql": + group_write_conditions.append( + cast( + DocumentModel.access_control["write"]["group_ids"], + JSONB, + ).contains([gid]) + ) + + if group_write_conditions: + # User should NOT have write permission + write_exclusions.append(~or_(*group_write_conditions)) + + # Exclude public items (items without access_control) + write_exclusions.append(DocumentModel.access_control.isnot(None)) + write_exclusions.append(cast(DocumentModel.access_control, String) != "null") + + # Combine: has read AND does not have write AND not public + if write_exclusions: + query = query.filter(and_(has_read, *write_exclusions)) + else: + query = query.filter(has_read) + + return query + + # Original logic for other permissions (read, write, etc.) + # Public access conditions + if group_ids or user_id: + conditions.extend( + [ + DocumentModel.access_control.is_(None), + cast(DocumentModel.access_control, String) == "null", + ] + ) + + # User-level permission (owner has all permissions) + if user_id: + conditions.append(DocumentModel.user_id == user_id) + + # Group-level permission + if group_ids: + group_conditions = [] + for gid in group_ids: + if dialect_name == "sqlite": + group_conditions.append( + DocumentModel.access_control[permission]["group_ids"].contains( + [gid] + ) + ) + elif dialect_name == "postgresql": + group_conditions.append( + cast( + DocumentModel.access_control[permission]["group_ids"], + JSONB, + ).contains([gid]) + ) + conditions.append(or_(*group_conditions)) + + if conditions: + query = query.filter(or_(*conditions)) + + return query diff --git a/src/lib/apis/knowledge/index.ts b/src/lib/apis/knowledge/index.ts index 98b2c1e5ec..9656a232ea 100644 --- a/src/lib/apis/knowledge/index.ts +++ b/src/lib/apis/knowledge/index.ts @@ -38,10 +38,13 @@ export const createNewKnowledge = async ( return res; }; -export const getKnowledgeBases = async (token: string = '') => { +export const getKnowledgeBases = async (token: string = '', page: number | null = null) => { let error = null; - const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/`, { + const searchParams = new URLSearchParams(); + if (page) searchParams.append('page', page.toString()); + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/?${searchParams.toString()}`, { method: 'GET', headers: { Accept: 'application/json', @@ -69,10 +72,20 @@ export const getKnowledgeBases = async (token: string = '') => { return res; }; -export const getKnowledgeBaseList = async (token: string = '') => { +export const searchKnowledgeBases = async ( + token: string = '', + query: string | null = null, + viewOption: string | null = null, + page: number | null = null +) => { let error = null; - const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/list`, { + const searchParams = new URLSearchParams(); + if (query) searchParams.append('query', query); + if (viewOption) searchParams.append('view_option', viewOption); + if (page) searchParams.append('page', page.toString()); + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/search?${searchParams.toString()}`, { method: 'GET', headers: { Accept: 'application/json', @@ -100,6 +113,55 @@ export const getKnowledgeBaseList = async (token: string = '') => { return res; }; +export const searchKnowledgeFiles = async ( + token: string, + query?: string | null = null, + viewOption?: string | null = null, + orderBy?: string | null = null, + direction?: string | null = null, + page: number = 1 +) => { + let error = null; + + const searchParams = new URLSearchParams(); + if (query) searchParams.append('query', query); + if (viewOption) searchParams.append('view_option', viewOption); + if (orderBy) searchParams.append('order_by', orderBy); + if (direction) searchParams.append('direction', direction); + searchParams.append('page', page.toString()); + + const res = await fetch( + `${WEBUI_API_BASE_URL}/knowledge/search/files?${searchParams.toString()}`, + { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + } + ) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + export const getKnowledgeById = async (token: string, id: string) => { let error = null; diff --git a/src/lib/components/chat/MessageInput/CommandSuggestionList.svelte b/src/lib/components/chat/MessageInput/CommandSuggestionList.svelte index d8a9e1e91a..75997f9d2a 100644 --- a/src/lib/components/chat/MessageInput/CommandSuggestionList.svelte +++ b/src/lib/components/chat/MessageInput/CommandSuggestionList.svelte @@ -28,9 +28,6 @@ await Promise.all([ (async () => { prompts.set(await getPrompts(localStorage.token)); - })(), - (async () => { - knowledge.set(await getKnowledgeBases(localStorage.token)); })() ]); loading = false; @@ -103,7 +100,6 @@ bind:this={suggestionElement} {query} bind:filteredItems - knowledge={$knowledge ?? []} onSelect={(e) => { const { type, data } = e; diff --git a/src/lib/components/chat/MessageInput/Commands/Knowledge.svelte b/src/lib/components/chat/MessageInput/Commands/Knowledge.svelte index 1b1f2aa9ed..77a6c12812 100644 --- a/src/lib/components/chat/MessageInput/Commands/Knowledge.svelte +++ b/src/lib/components/chat/MessageInput/Commands/Knowledge.svelte @@ -15,6 +15,7 @@ import Youtube from '$lib/components/icons/Youtube.svelte'; import { folders } from '$lib/stores'; import Folder from '$lib/components/icons/Folder.svelte'; + import { getFolders } from '$lib/apis/folders'; const i18n = getContext('i18n'); @@ -80,6 +81,10 @@ }; onMount(async () => { + if ($folders === null) { + await folders.set(await getFolders(localStorage.token)); + } + let collections = knowledge .filter((item) => !item?.meta?.document) .map((item) => ({ @@ -87,31 +92,6 @@ type: 'collection' })); - let collection_files = - knowledge.length > 0 - ? [ - ...knowledge - .reduce((a, item) => { - return [ - ...new Set([ - ...a, - ...(item?.files ?? []).map((file) => ({ - ...file, - collection: { name: item.name, description: item.description } // DO NOT REMOVE, USED IN FILE DESCRIPTION/ATTACHMENT - })) - ]) - ]; - }, []) - .map((file) => ({ - ...file, - name: file?.meta?.name, - description: `${file?.collection?.description}`, - knowledge: true, // DO NOT REMOVE, USED TO INDICATE KNOWLEDGE BASE FILE - type: 'file' - })) - ] - : []; - let folder_items = $folders.map((folder) => ({ ...folder, type: 'folder', @@ -119,7 +99,7 @@ title: folder.name })); - items = [...folder_items, ...collections, ...collection_files]; + items = [...folder_items, ...collections]; fuse = new Fuse(items, { keys: ['name', 'description'] }); diff --git a/src/lib/components/chat/MessageInput/InputMenu.svelte b/src/lib/components/chat/MessageInput/InputMenu.svelte index 82eeeaed58..83d6abc5a5 100644 --- a/src/lib/components/chat/MessageInput/InputMenu.svelte +++ b/src/lib/components/chat/MessageInput/InputMenu.svelte @@ -73,16 +73,6 @@ } }; - const init = async () => { - if ($knowledge === null) { - await knowledge.set(await getKnowledgeBases(localStorage.token)); - } - }; - - $: if (show) { - init(); - } - const onSelect = (item) => { if (files.find((f) => f.id === item.id)) { return; @@ -249,37 +239,35 @@ {/if} - {#if ($knowledge ?? []).length > 0} - + - - {/if} + +
+ +
+ + + {#if ($chats ?? []).length > 0} {}; let loaded = false; - let items = []; let selectedIdx = 0; - onMount(async () => { - if ($knowledge === null) { - await knowledge.set(await getKnowledgeBases(localStorage.token)); + let selectedItem = null; + + let selectedFileItemsPage = 1; + + let selectedFileItems = null; + let selectedFileItemsTotal = null; + + let selectedFileItemsLoading = false; + let selectedFileAllItemsLoaded = false; + + $: if (selectedItem) { + initSelectedFileItems(); + } + + const initSelectedFileItems = async () => { + selectedFileItemsPage = 1; + selectedFileItems = null; + selectedFileItemsTotal = null; + selectedFileAllItemsLoaded = false; + selectedFileItemsLoading = false; + await tick(); + await getSelectedFileItemsPage(); + }; + + const loadMoreSelectedFileItems = async () => { + if (selectedFileAllItemsLoaded) return; + selectedFileItemsPage += 1; + await getSelectedFileItemsPage(); + }; + + const getSelectedFileItemsPage = async () => { + if (!selectedItem) return; + selectedFileItemsLoading = true; + + const res = await searchKnowledgeFilesById( + localStorage.token, + selectedItem.id, + null, + null, + null, + null, + selectedFileItemsPage + ).catch(() => { + return null; + }); + + if (res) { + selectedFileItemsTotal = res.total; + const pageItems = res.items; + + if ((pageItems ?? []).length === 0) { + selectedFileAllItemsLoaded = true; + } else { + selectedFileAllItemsLoaded = false; + } + + if (selectedFileItems) { + selectedFileItems = [...selectedFileItems, ...pageItems]; + } else { + selectedFileItems = pageItems; + } } - let collections = $knowledge - .filter((item) => !item?.meta?.document) - .map((item) => ({ - ...item, - type: 'collection' - })); - ``; - let collection_files = - $knowledge.length > 0 - ? [ - ...$knowledge - .reduce((a, item) => { - return [ - ...new Set([ - ...a, - ...(item?.files ?? []).map((file) => ({ - ...file, - collection: { name: item.name, description: item.description } // DO NOT REMOVE, USED IN FILE DESCRIPTION/ATTACHMENT - })) - ]) - ]; - }, []) - .map((file) => ({ - ...file, - name: file?.meta?.name, - description: `${file?.collection?.name} - ${file?.collection?.description}`, - knowledge: true, // DO NOT REMOVE, USED TO INDICATE KNOWLEDGE BASE FILE - type: 'file' - })) - ] - : []; + selectedFileItemsLoading = false; + return res; + }; - items = [...collections, ...collection_files]; + let page = 1; + let items = null; + let total = null; + + let itemsLoading = false; + let allItemsLoaded = false; + + $: if (loaded) { + init(); + } + + const init = async () => { + reset(); await tick(); + await getItemsPage(); + }; + const reset = () => { + page = 1; + items = null; + total = null; + allItemsLoaded = false; + itemsLoading = false; + }; + + const loadMoreItems = async () => { + if (allItemsLoaded) return; + page += 1; + await getItemsPage(); + }; + + const getItemsPage = async () => { + itemsLoading = true; + const res = await getKnowledgeBases(localStorage.token, page).catch(() => { + return null; + }); + + if (res) { + console.log(res); + total = res.total; + const pageItems = res.items; + + if ((pageItems ?? []).length === 0) { + allItemsLoaded = true; + } else { + allItemsLoaded = false; + } + + if (items) { + items = [...items, ...pageItems]; + } else { + items = pageItems; + } + } + + itemsLoading = false; + return res; + }; + + onMount(async () => { + await tick(); loaded = true; }); -{#if loaded} +{#if loaded && items !== null}
- {#each items as item, idx} - + + +
- - {/each} + + {#if selectedItem && selectedItem.id === item.id} +
+ {#if selectedFileItems === null && selectedFileItemsTotal === null} +
+ +
+ {:else if selectedFileItemsTotal === 0} +
+ {$i18n.t('No files in this knowledge base.')} +
+ {:else} + {#each selectedFileItems as file, fileIdx (file.id)} + + {/each} + + {#if !selectedFileAllItemsLoaded && !selectedFileItemsLoading} + { + if (!selectedFileItemsLoading) { + await loadMoreSelectedFileItems(); + } + }} + > +
+ +
{$i18n.t('Loading...')}
+
+ + {/if} + {/if} +
+ {/if} + {/each} + + {#if !allItemsLoaded} + { + if (!itemsLoading) { + loadMoreItems(); + } + }} + > +
+ +
{$i18n.t('Loading...')}
+
+ + {/if} + {/if} {:else}
diff --git a/src/lib/components/workspace/Knowledge.svelte b/src/lib/components/workspace/Knowledge.svelte index c44d1d3e1b..104b00ff78 100644 --- a/src/lib/components/workspace/Knowledge.svelte +++ b/src/lib/components/workspace/Knowledge.svelte @@ -1,6 +1,4 @@ @@ -123,7 +132,7 @@
- {filteredItems.length} + {total}
@@ -192,96 +201,117 @@ - {#if (filteredItems ?? []).length !== 0} - -
- {#each filteredItems as item} - + {/each} +
+ + {#if !allItemsLoaded} + { + if (!itemsLoading) { + loadMoreItems(); + } + }} + > +
+ +
{$i18n.t('Loading...')}
+
+ + {/if} + {:else} +
+
+
😕
+
{$i18n.t('No knowledge found')}
+
+ {$i18n.t('Try adjusting your search or filter to find what you are looking for.')}
- - {/each} -
- {:else} -
-
-
😕
-
{$i18n.t('No knowledge found')}
-
- {$i18n.t('Try adjusting your search or filter to find what you are looking for.')}
+ {/if} + {:else} +
+
{/if}
diff --git a/src/lib/components/workspace/Knowledge/CreateKnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/CreateKnowledgeBase.svelte index 2e729f4968..3373e5a660 100644 --- a/src/lib/components/workspace/Knowledge/CreateKnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/CreateKnowledgeBase.svelte @@ -1,11 +1,13 @@ @@ -190,8 +187,7 @@ {#if loaded}
- { const item = e.detail; @@ -210,7 +206,7 @@ > {$i18n.t('Select Knowledge')}
- + {#if $user?.role === 'admin' || $user?.permissions?.chat?.file_upload} +
+ {/each} + {/if} + + + + diff --git a/src/lib/components/workspace/Models/Knowledge/Selector.svelte b/src/lib/components/workspace/Models/Knowledge/Selector.svelte deleted file mode 100644 index f40e48ef75..0000000000 --- a/src/lib/components/workspace/Models/Knowledge/Selector.svelte +++ /dev/null @@ -1,186 +0,0 @@ - - - { - if (e.detail === false) { - onClose(); - query = ''; - } - }} -> - - -
- -
-
-
- -
- -
-
- -
- {#if filteredItems.length === 0} -
- {$i18n.t('No knowledge found')} -
- {:else} - {#each filteredItems as item} - { - dispatch('select', item); - }} - > -
-
- {#if item.legacy} -
- Legacy -
- {:else if item?.meta?.document} -
- Document -
- {:else if item?.type === 'file'} -
- File -
- {:else if item?.type === 'note'} -
- Note -
- {:else} -
- Collection -
- {/if} - -
- {decodeString(item?.name)} -
-
- -
- {item?.description} -
-
- - {/each} - {/if} -
- -
- diff --git a/src/lib/components/workspace/Models/ModelEditor.svelte b/src/lib/components/workspace/Models/ModelEditor.svelte index a05896f8d8..636048e8c7 100644 --- a/src/lib/components/workspace/Models/ModelEditor.svelte +++ b/src/lib/components/workspace/Models/ModelEditor.svelte @@ -2,12 +2,11 @@ import { toast } from 'svelte-sonner'; import { onMount, getContext, tick } from 'svelte'; - import { models, tools, functions, knowledge as knowledgeCollections, user } from '$lib/stores'; + import { models, tools, functions, user } from '$lib/stores'; import { WEBUI_BASE_URL } from '$lib/constants'; import { getTools } from '$lib/apis/tools'; import { getFunctions } from '$lib/apis/functions'; - import { getKnowledgeBases } from '$lib/apis/knowledge'; import AdvancedParams from '$lib/components/chat/Settings/Advanced/AdvancedParams.svelte'; import Tags from '$lib/components/common/Tags.svelte'; @@ -223,7 +222,6 @@ onMount(async () => { await tools.set(await getTools(localStorage.token)); await functions.set(await getFunctions(localStorage.token)); - await knowledgeCollections.set([...(await getKnowledgeBases(localStorage.token))]); // Scroll to top 'workspace-container' element const workspaceContainer = document.getElementById('workspace-container');