diff --git a/CHANGELOG.md b/CHANGELOG.md index 72ffba1114..c42ab4f59c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +### [0.6.39] - 2025-11-25 + +### Added + +- 💬 A user list modal was added to channels, displaying all users with access and featuring search, sorting, and pagination capabilities. [Commit](https://github.com/open-webui/open-webui/commit/c0e120353824be00a2ef63cbde8be5d625bd6fd0) +- 💬 Channel navigation now displays the total number of users with access to the channel. [Commit](https://github.com/open-webui/open-webui/commit/3b5710d0cd445cf86423187f5ee7c40472a0df0b) +- 🔌 Tool servers and MCP connections now support function name filtering, allowing administrators to selectively enable or block specific functions using allow/block lists. [Commit](https://github.com/open-webui/open-webui/commit/743199f2d097ae1458381bce450d9025a0ab3f3d) +- ⚡ A toggle to disable parallel embedding processing was added via "ENABLE_ASYNC_EMBEDDING", allowing sequential processing for rate-limited or resource-constrained local embedding setups. [#19444](https://github.com/open-webui/open-webui/pull/19444) +- 🔄 Various improvements were implemented across the frontend and backend to enhance performance, stability, and security. +- 🌐 Localization improvements were made for German (de-DE) and Portuguese (Brazil) translations. + +### Fixed + +- 📝 Inline citations now render correctly within markdown lists and nested elements instead of displaying as "undefined" values. [#19452](https://github.com/open-webui/open-webui/issues/19452) +- 👥 Group member selection now works correctly without randomly selecting other users or causing the user list to jump around. [#19426](https://github.com/open-webui/open-webui/issues/19426) +- 👥 Admin panel user list now displays the correct total user count and properly paginates 30 items per page after fixing database query issues with group member joins. [#19429](https://github.com/open-webui/open-webui/issues/19429) +- 🔍 Knowledge base reindexing now works correctly after resolving async execution chain issues by implementing threadpool workers for embedding operations. [#19434](https://github.com/open-webui/open-webui/pull/19434) +- 🖼️ OpenAI image generation now works correctly after fixing a connection adapter error caused by incorrect URL formatting. [#19435](https://github.com/open-webui/open-webui/pull/19435) + +### Changed + +- 🔧 BREAKING: Docling configuration has been consolidated from individual environment variables into a single "DOCLING_PARAMS" JSON configuration and now supports API key authentication via "DOCLING_API_KEY", requiring users to migrate existing Docling settings to the new format. [#16841](https://github.com/open-webui/open-webui/issues/16841), [#19427](https://github.com/open-webui/open-webui/pull/19427) +- 🔧 The environment variable "REPLACE_IMAGE_URLS_IN_CHAT_RESPONSE" has been renamed to "ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION" for naming consistency. + ## [0.6.38] - 2025-11-24 ### Fixed diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index d0e693e319..5a9844c067 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -2538,6 +2538,12 @@ DOCLING_SERVER_URL = PersistentConfig( os.getenv("DOCLING_SERVER_URL", "http://docling:5001"), ) +DOCLING_API_KEY = PersistentConfig( + "DOCLING_API_KEY", + "rag.docling_api_key", + os.getenv("DOCLING_API_KEY", ""), +) + docling_params = os.getenv("DOCLING_PARAMS", "") try: docling_params = json.loads(docling_params) @@ -2550,88 +2556,6 @@ DOCLING_PARAMS = PersistentConfig( docling_params, ) -DOCLING_DO_OCR = PersistentConfig( - "DOCLING_DO_OCR", - "rag.docling_do_ocr", - os.getenv("DOCLING_DO_OCR", "True").lower() == "true", -) - -DOCLING_FORCE_OCR = PersistentConfig( - "DOCLING_FORCE_OCR", - "rag.docling_force_ocr", - os.getenv("DOCLING_FORCE_OCR", "False").lower() == "true", -) - -DOCLING_OCR_ENGINE = PersistentConfig( - "DOCLING_OCR_ENGINE", - "rag.docling_ocr_engine", - os.getenv("DOCLING_OCR_ENGINE", "tesseract"), -) - -DOCLING_OCR_LANG = PersistentConfig( - "DOCLING_OCR_LANG", - "rag.docling_ocr_lang", - os.getenv("DOCLING_OCR_LANG", "eng,fra,deu,spa"), -) - -DOCLING_PDF_BACKEND = PersistentConfig( - "DOCLING_PDF_BACKEND", - "rag.docling_pdf_backend", - os.getenv("DOCLING_PDF_BACKEND", "dlparse_v4"), -) - -DOCLING_TABLE_MODE = PersistentConfig( - "DOCLING_TABLE_MODE", - "rag.docling_table_mode", - os.getenv("DOCLING_TABLE_MODE", "accurate"), -) - -DOCLING_PIPELINE = PersistentConfig( - "DOCLING_PIPELINE", - "rag.docling_pipeline", - os.getenv("DOCLING_PIPELINE", "standard"), -) - -DOCLING_DO_PICTURE_DESCRIPTION = PersistentConfig( - "DOCLING_DO_PICTURE_DESCRIPTION", - "rag.docling_do_picture_description", - os.getenv("DOCLING_DO_PICTURE_DESCRIPTION", "False").lower() == "true", -) - -DOCLING_PICTURE_DESCRIPTION_MODE = PersistentConfig( - "DOCLING_PICTURE_DESCRIPTION_MODE", - "rag.docling_picture_description_mode", - os.getenv("DOCLING_PICTURE_DESCRIPTION_MODE", ""), -) - - -docling_picture_description_local = os.getenv("DOCLING_PICTURE_DESCRIPTION_LOCAL", "") -try: - docling_picture_description_local = json.loads(docling_picture_description_local) -except json.JSONDecodeError: - docling_picture_description_local = {} - - -DOCLING_PICTURE_DESCRIPTION_LOCAL = PersistentConfig( - "DOCLING_PICTURE_DESCRIPTION_LOCAL", - "rag.docling_picture_description_local", - docling_picture_description_local, -) - -docling_picture_description_api = os.getenv("DOCLING_PICTURE_DESCRIPTION_API", "") -try: - docling_picture_description_api = json.loads(docling_picture_description_api) -except json.JSONDecodeError: - docling_picture_description_api = {} - - -DOCLING_PICTURE_DESCRIPTION_API = PersistentConfig( - "DOCLING_PICTURE_DESCRIPTION_API", - "rag.docling_picture_description_api", - docling_picture_description_api, -) - - DOCUMENT_INTELLIGENCE_ENDPOINT = PersistentConfig( "DOCUMENT_INTELLIGENCE_ENDPOINT", "rag.document_intelligence_endpoint", @@ -2789,6 +2713,12 @@ RAG_EMBEDDING_BATCH_SIZE = PersistentConfig( ), ) +ENABLE_ASYNC_EMBEDDING = PersistentConfig( + "ENABLE_ASYNC_EMBEDDING", + "rag.enable_async_embedding", + os.environ.get("ENABLE_ASYNC_EMBEDDING", "True").lower() == "true", +) + RAG_EMBEDDING_QUERY_PREFIX = os.environ.get("RAG_EMBEDDING_QUERY_PREFIX", None) RAG_EMBEDDING_CONTENT_PREFIX = os.environ.get("RAG_EMBEDDING_CONTENT_PREFIX", None) diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 651629b950..e3c50ea8d1 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -561,7 +561,8 @@ else: #################################### ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION = ( - os.environ.get("REPLACE_IMAGE_URLS_IN_CHAT_RESPONSE", "False").lower() == "true" + os.environ.get("ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION", "False").lower() + == "true" ) CHAT_RESPONSE_STREAM_DELTA_CHUNK_SIZE = os.environ.get( diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index b14c18947c..af8e670a53 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -230,6 +230,7 @@ from open_webui.config import ( RAG_RERANKING_MODEL_TRUST_REMOTE_CODE, RAG_EMBEDDING_ENGINE, RAG_EMBEDDING_BATCH_SIZE, + ENABLE_ASYNC_EMBEDDING, RAG_TOP_K, RAG_TOP_K_RERANKER, RAG_RELEVANCE_THRESHOLD, @@ -268,18 +269,8 @@ from open_webui.config import ( EXTERNAL_DOCUMENT_LOADER_API_KEY, TIKA_SERVER_URL, DOCLING_SERVER_URL, + DOCLING_API_KEY, DOCLING_PARAMS, - DOCLING_DO_OCR, - DOCLING_FORCE_OCR, - DOCLING_OCR_ENGINE, - DOCLING_OCR_LANG, - DOCLING_PDF_BACKEND, - DOCLING_TABLE_MODE, - DOCLING_PIPELINE, - DOCLING_DO_PICTURE_DESCRIPTION, - DOCLING_PICTURE_DESCRIPTION_MODE, - DOCLING_PICTURE_DESCRIPTION_LOCAL, - DOCLING_PICTURE_DESCRIPTION_API, DOCUMENT_INTELLIGENCE_ENDPOINT, DOCUMENT_INTELLIGENCE_KEY, MISTRAL_OCR_API_BASE_URL, @@ -874,18 +865,8 @@ app.state.config.EXTERNAL_DOCUMENT_LOADER_URL = EXTERNAL_DOCUMENT_LOADER_URL app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY = EXTERNAL_DOCUMENT_LOADER_API_KEY app.state.config.TIKA_SERVER_URL = TIKA_SERVER_URL app.state.config.DOCLING_SERVER_URL = DOCLING_SERVER_URL +app.state.config.DOCLING_API_KEY = DOCLING_API_KEY app.state.config.DOCLING_PARAMS = DOCLING_PARAMS -app.state.config.DOCLING_DO_OCR = DOCLING_DO_OCR -app.state.config.DOCLING_FORCE_OCR = DOCLING_FORCE_OCR -app.state.config.DOCLING_OCR_ENGINE = DOCLING_OCR_ENGINE -app.state.config.DOCLING_OCR_LANG = DOCLING_OCR_LANG -app.state.config.DOCLING_PDF_BACKEND = DOCLING_PDF_BACKEND -app.state.config.DOCLING_TABLE_MODE = DOCLING_TABLE_MODE -app.state.config.DOCLING_PIPELINE = DOCLING_PIPELINE -app.state.config.DOCLING_DO_PICTURE_DESCRIPTION = DOCLING_DO_PICTURE_DESCRIPTION -app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE = DOCLING_PICTURE_DESCRIPTION_MODE -app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL = DOCLING_PICTURE_DESCRIPTION_LOCAL -app.state.config.DOCLING_PICTURE_DESCRIPTION_API = DOCLING_PICTURE_DESCRIPTION_API app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT = DOCUMENT_INTELLIGENCE_ENDPOINT app.state.config.DOCUMENT_INTELLIGENCE_KEY = DOCUMENT_INTELLIGENCE_KEY app.state.config.MISTRAL_OCR_API_BASE_URL = MISTRAL_OCR_API_BASE_URL @@ -904,6 +885,7 @@ app.state.config.CHUNK_OVERLAP = CHUNK_OVERLAP app.state.config.RAG_EMBEDDING_ENGINE = RAG_EMBEDDING_ENGINE app.state.config.RAG_EMBEDDING_MODEL = RAG_EMBEDDING_MODEL app.state.config.RAG_EMBEDDING_BATCH_SIZE = RAG_EMBEDDING_BATCH_SIZE +app.state.config.ENABLE_ASYNC_EMBEDDING = ENABLE_ASYNC_EMBEDDING app.state.config.RAG_RERANKING_ENGINE = RAG_RERANKING_ENGINE app.state.config.RAG_RERANKING_MODEL = RAG_RERANKING_MODEL diff --git a/backend/open_webui/models/channels.py b/backend/open_webui/models/channels.py index 2a14e7a2d5..5f4d1436d9 100644 --- a/backend/open_webui/models/channels.py +++ b/backend/open_webui/models/channels.py @@ -59,6 +59,7 @@ class ChannelModel(BaseModel): class ChannelResponse(ChannelModel): write_access: bool = False + user_count: Optional[int] = None class ChannelForm(BaseModel): diff --git a/backend/open_webui/models/groups.py b/backend/open_webui/models/groups.py index 1d96f5cfaa..e5c0612639 100644 --- a/backend/open_webui/models/groups.py +++ b/backend/open_webui/models/groups.py @@ -177,6 +177,23 @@ class GroupTable: return [m[0] for m in members] + def get_group_user_ids_by_ids(self, group_ids: list[str]) -> dict[str, list[str]]: + with get_db() as db: + members = ( + db.query(GroupMember.group_id, GroupMember.user_id) + .filter(GroupMember.group_id.in_(group_ids)) + .all() + ) + + group_user_ids: dict[str, list[str]] = { + group_id: [] for group_id in group_ids + } + + for group_id, user_id in members: + group_user_ids[group_id].append(user_id) + + return group_user_ids + def set_group_user_ids_by_id(self, group_id: str, user_ids: list[str]) -> None: with get_db() as db: # Delete existing members diff --git a/backend/open_webui/models/models.py b/backend/open_webui/models/models.py index f9390b405d..e902a978d1 100755 --- a/backend/open_webui/models/models.py +++ b/backend/open_webui/models/models.py @@ -290,10 +290,15 @@ class ModelsTable: models = [] for model, user in items: - model_model = ModelModel.model_validate(model) - user_model = UserResponse(**UserModel.model_validate(user).model_dump()) models.append( - ModelUserResponse(**model_model.model_dump(), user=user_model) + ModelUserResponse( + **ModelModel.model_validate(model).model_dump(), + user=( + UserResponse(**UserModel.model_validate(user).model_dump()) + if user + else None + ), + ) ) return ModelListResponse(items=models, total=total) diff --git a/backend/open_webui/models/users.py b/backend/open_webui/models/users.py index 256d3bc75e..a66a95a98c 100644 --- a/backend/open_webui/models/users.py +++ b/backend/open_webui/models/users.py @@ -11,8 +11,8 @@ from open_webui.utils.misc import throttle from pydantic import BaseModel, ConfigDict -from sqlalchemy import BigInteger, Column, String, Text, Date -from sqlalchemy import or_ +from sqlalchemy import BigInteger, Column, String, Text, Date, exists, select +from sqlalchemy import or_, case import datetime @@ -99,7 +99,16 @@ class UserGroupIdsModel(UserModel): group_ids: list[str] = [] +class UserModelResponse(UserModel): + model_config = ConfigDict(extra="allow") + + class UserListResponse(BaseModel): + users: list[UserModelResponse] + total: int + + +class UserGroupIdsListResponse(BaseModel): users: list[UserGroupIdsModel] total: int @@ -227,9 +236,7 @@ class UsersTable: ) -> dict: with get_db() as db: # Join GroupMember so we can order by group_id when requested - query = db.query(User).outerjoin( - GroupMember, GroupMember.user_id == User.id - ) + query = db.query(User) if filter: query_key = filter.get("query") @@ -241,23 +248,65 @@ class UsersTable: ) ) + user_ids = filter.get("user_ids") + group_ids = filter.get("group_ids") + + if isinstance(user_ids, list) and isinstance(group_ids, list): + # If both are empty lists, return no users + if not user_ids and not group_ids: + return {"users": [], "total": 0} + + if user_ids: + query = query.filter(User.id.in_(user_ids)) + + if group_ids: + query = query.filter( + exists( + select(GroupMember.id).where( + GroupMember.user_id == User.id, + GroupMember.group_id.in_(group_ids), + ) + ) + ) + + roles = filter.get("roles") + if roles: + include_roles = [role for role in roles if not role.startswith("!")] + exclude_roles = [role[1:] for role in roles if role.startswith("!")] + + if include_roles: + query = query.filter(User.role.in_(include_roles)) + if exclude_roles: + query = query.filter(~User.role.in_(exclude_roles)) + order_by = filter.get("order_by") direction = filter.get("direction") if order_by and order_by.startswith("group_id:"): group_id = order_by.split(":", 1)[1] - if direction == "asc": - query = query.order_by((GroupMember.group_id == group_id).asc()) - else: - query = query.order_by( - (GroupMember.group_id == group_id).desc() + # Subquery that checks if the user belongs to the group + membership_exists = exists( + select(GroupMember.id).where( + GroupMember.user_id == User.id, + GroupMember.group_id == group_id, ) + ) + + # CASE: user in group → 1, user not in group → 0 + group_sort = case((membership_exists, 1), else_=0) + + if direction == "asc": + query = query.order_by(group_sort.asc(), User.name.asc()) + else: + query = query.order_by(group_sort.desc(), User.name.asc()) + elif order_by == "name": if direction == "asc": query = query.order_by(User.name.asc()) else: query = query.order_by(User.name.desc()) + elif order_by == "email": if direction == "asc": query = query.order_by(User.email.asc()) @@ -291,11 +340,13 @@ class UsersTable: query = query.order_by(User.created_at.desc()) # Count BEFORE pagination + query = query.distinct(User.id) total = query.count() - if skip: + # correct pagination logic + if skip is not None: query = query.offset(skip) - if limit: + if limit is not None: query = query.limit(limit) users = query.all() diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index bbc3da9bc9..fcc507e088 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -132,8 +132,9 @@ class TikaLoader: class DoclingLoader: - def __init__(self, url, file_path=None, mime_type=None, params=None): + def __init__(self, url, api_key=None, file_path=None, mime_type=None, params=None): self.url = url.rstrip("/") + self.api_key = api_key self.file_path = file_path self.mime_type = mime_type @@ -141,6 +142,10 @@ class DoclingLoader: def load(self) -> list[Document]: with open(self.file_path, "rb") as f: + headers = {} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + files = { "files": ( self.file_path, @@ -149,60 +154,15 @@ class DoclingLoader: ) } - params = {"image_export_mode": "placeholder"} - - if self.params: - if self.params.get("do_picture_description"): - params["do_picture_description"] = self.params.get( - "do_picture_description" - ) - - picture_description_mode = self.params.get( - "picture_description_mode", "" - ).lower() - - if picture_description_mode == "local" and self.params.get( - "picture_description_local", {} - ): - params["picture_description_local"] = json.dumps( - self.params.get("picture_description_local", {}) - ) - - elif picture_description_mode == "api" and self.params.get( - "picture_description_api", {} - ): - params["picture_description_api"] = json.dumps( - self.params.get("picture_description_api", {}) - ) - - params["do_ocr"] = self.params.get("do_ocr") - - params["force_ocr"] = self.params.get("force_ocr") - - if ( - self.params.get("do_ocr") - and self.params.get("ocr_engine") - and self.params.get("ocr_lang") - ): - params["ocr_engine"] = self.params.get("ocr_engine") - params["ocr_lang"] = [ - lang.strip() - for lang in self.params.get("ocr_lang").split(",") - if lang.strip() - ] - - if self.params.get("pdf_backend"): - params["pdf_backend"] = self.params.get("pdf_backend") - - if self.params.get("table_mode"): - params["table_mode"] = self.params.get("table_mode") - - if self.params.get("pipeline"): - params["pipeline"] = self.params.get("pipeline") - - endpoint = f"{self.url}/v1/convert/file" - r = requests.post(endpoint, files=files, data=params) - + r = requests.post( + f"{self.url}/v1/convert/file", + files=files, + data={ + "image_export_mode": "placeholder", + **self.params, + }, + headers=headers, + ) if r.ok: result = r.json() document_data = result.get("document", {}) @@ -211,7 +171,6 @@ class DoclingLoader: metadata = {"Content-Type": self.mime_type} if self.mime_type else {} log.debug("Docling extracted text: %s", text) - return [Document(page_content=text, metadata=metadata)] else: error_msg = f"Error calling Docling API: {r.reason}" @@ -340,6 +299,7 @@ class Loader: loader = DoclingLoader( url=self.kwargs.get("DOCLING_SERVER_URL"), + api_key=self.kwargs.get("DOCLING_API_KEY", None), file_path=file_path, mime_type=file_content_type, params=params, diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index e8dc97209e..b041a00471 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -782,6 +782,7 @@ def get_embedding_function( key, embedding_batch_size, azure_api_version=None, + enable_async=True, ) -> Awaitable: if embedding_engine == "": # Sentence transformers: CPU-bound sync operation @@ -816,16 +817,26 @@ def get_embedding_function( query[i : i + embedding_batch_size] for i in range(0, len(query), embedding_batch_size) ] - log.debug( - f"generate_multiple_async: Processing {len(batches)} batches in parallel" - ) - # Execute all batches in parallel - tasks = [ - embedding_function(batch, prefix=prefix, user=user) - for batch in batches - ] - batch_results = await asyncio.gather(*tasks) + if enable_async: + log.debug( + f"generate_multiple_async: Processing {len(batches)} batches in parallel" + ) + # Execute all batches in parallel + tasks = [ + embedding_function(batch, prefix=prefix, user=user) + for batch in batches + ] + batch_results = await asyncio.gather(*tasks) + else: + log.debug( + f"generate_multiple_async: Processing {len(batches)} batches sequentially" + ) + batch_results = [] + for batch in batches: + batch_results.append( + await embedding_function(batch, prefix=prefix, user=user) + ) # Flatten results embeddings = [] diff --git a/backend/open_webui/retrieval/web/main.py b/backend/open_webui/retrieval/web/main.py index d8cfb11ba0..6d2fd1bc5a 100644 --- a/backend/open_webui/retrieval/web/main.py +++ b/backend/open_webui/retrieval/web/main.py @@ -5,7 +5,8 @@ from urllib.parse import urlparse from pydantic import BaseModel -from open_webui.retrieval.web.utils import is_string_allowed, resolve_hostname +from open_webui.retrieval.web.utils import resolve_hostname +from open_webui.utils.misc import is_string_allowed def get_filtered_results(results, filter_list): diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 127c703442..bdbde0b3a9 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -42,7 +42,7 @@ from open_webui.config import ( WEB_FETCH_FILTER_LIST, ) from open_webui.env import SRC_LOG_LEVELS - +from open_webui.utils.misc import is_string_allowed log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["RAG"]) @@ -59,39 +59,6 @@ def resolve_hostname(hostname): return ipv4_addresses, ipv6_addresses -def get_allow_block_lists(filter_list): - allow_list = [] - block_list = [] - - if filter_list: - for d in filter_list: - if d.startswith("!"): - # Domains starting with "!" → blocked - block_list.append(d[1:]) - else: - # Domains starting without "!" → allowed - allow_list.append(d) - - return allow_list, block_list - - -def is_string_allowed(string: str, filter_list: Optional[list[str]] = None) -> bool: - if not filter_list: - return True - - allow_list, block_list = get_allow_block_lists(filter_list) - # If allow list is non-empty, require domain to match one of them - if allow_list: - if not any(string.endswith(allowed) for allowed in allow_list): - return False - - # Block list always removes matches - if any(string.endswith(blocked) for blocked in block_list): - return False - - return True - - def validate_url(url: Union[str, Sequence[str]]): if isinstance(url, str): if isinstance(validators.url(url), validators.ValidationError): diff --git a/backend/open_webui/routers/channels.py b/backend/open_webui/routers/channels.py index fda0879594..e47c98554e 100644 --- a/backend/open_webui/routers/channels.py +++ b/backend/open_webui/routers/channels.py @@ -7,8 +7,17 @@ from fastapi import APIRouter, Depends, HTTPException, Request, status, Backgrou from pydantic import BaseModel -from open_webui.socket.main import sio, get_user_ids_from_room -from open_webui.models.users import Users, UserNameResponse +from open_webui.socket.main import ( + sio, + get_user_ids_from_room, + get_active_status_by_user_id, +) +from open_webui.models.users import ( + UserListResponse, + UserModelResponse, + Users, + UserNameResponse, +) from open_webui.models.groups import Groups from open_webui.models.channels import ( @@ -38,7 +47,11 @@ from open_webui.utils.chat import generate_chat_completion from open_webui.utils.auth import get_admin_user, get_verified_user -from open_webui.utils.access_control import has_access, get_users_with_access +from open_webui.utils.access_control import ( + has_access, + get_users_with_access, + get_permitted_group_and_user_ids, +) from open_webui.utils.webhook import post_webhook from open_webui.utils.channels import extract_mentions, replace_mentions @@ -105,14 +118,73 @@ async def get_channel_by_id(id: str, user=Depends(get_verified_user)): user.id, type="write", access_control=channel.access_control, strict=False ) + user_count = len(get_users_with_access("read", channel.access_control)) + return ChannelResponse( **{ **channel.model_dump(), "write_access": write_access or user.role == "admin", + "user_count": user_count, } ) +PAGE_ITEM_COUNT = 30 + + +@router.get("/{id}/users", response_model=UserListResponse) +async def get_channel_users_by_id( + id: str, + query: Optional[str] = None, + order_by: Optional[str] = None, + direction: Optional[str] = None, + page: Optional[int] = 1, + user=Depends(get_verified_user), +): + + channel = Channels.get_channel_by_id(id) + if not channel: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND + ) + + limit = PAGE_ITEM_COUNT + + page = max(1, page) + skip = (page - 1) * limit + + filter = { + "roles": ["!pending"], + } + + if query: + filter["query"] = query + if order_by: + filter["order_by"] = order_by + if direction: + filter["direction"] = direction + + permitted_ids = get_permitted_group_and_user_ids("read", channel.access_control) + if permitted_ids: + filter["user_ids"] = permitted_ids.get("user_ids") + filter["group_ids"] = permitted_ids.get("group_ids") + + result = Users.get_users(filter=filter, skip=skip, limit=limit) + + users = result["users"] + total = result["total"] + + return { + "users": [ + UserModelResponse( + **user.model_dump(), is_active=get_active_status_by_user_id(user.id) + ) + for user in users + ], + "total": total, + } + + ############################ # UpdateChannelById ############################ diff --git a/backend/open_webui/routers/images.py b/backend/open_webui/routers/images.py index 4cc2a99101..8aabf0f73b 100644 --- a/backend/open_webui/routers/images.py +++ b/backend/open_webui/routers/images.py @@ -549,9 +549,7 @@ async def image_generations( if ENABLE_FORWARD_USER_INFO_HEADERS: headers = include_user_info_headers(headers, user) - url = ( - f"{request.app.state.config.IMAGES_OPENAI_API_BASE_URL}/images/generations", - ) + url = f"{request.app.state.config.IMAGES_OPENAI_API_BASE_URL}/images/generations" if request.app.state.config.IMAGES_OPENAI_API_VERSION: url = f"{url}?api-version={request.app.state.config.IMAGES_OPENAI_API_VERSION}" diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 71722d706e..ad47fc1686 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -1,6 +1,7 @@ from typing import List, Optional from pydantic import BaseModel from fastapi import APIRouter, Depends, HTTPException, status, Request, Query +from fastapi.concurrency import run_in_threadpool import logging from open_webui.models.knowledge import ( @@ -223,7 +224,8 @@ async def reindex_knowledge_files(request: Request, user=Depends(get_verified_us failed_files = [] for file in files: try: - process_file( + await run_in_threadpool( + process_file, request, ProcessFileForm( file_id=file.id, collection_name=knowledge_base.id diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 358b8aca49..6080337250 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -241,13 +241,14 @@ class SearchForm(BaseModel): async def get_status(request: Request): return { "status": True, - "chunk_size": request.app.state.config.CHUNK_SIZE, - "chunk_overlap": request.app.state.config.CHUNK_OVERLAP, - "template": request.app.state.config.RAG_TEMPLATE, - "embedding_engine": request.app.state.config.RAG_EMBEDDING_ENGINE, - "embedding_model": request.app.state.config.RAG_EMBEDDING_MODEL, - "reranking_model": request.app.state.config.RAG_RERANKING_MODEL, - "embedding_batch_size": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE, + "CHUNK_SIZE": request.app.state.config.CHUNK_SIZE, + "CHUNK_OVERLAP": request.app.state.config.CHUNK_OVERLAP, + "RAG_TEMPLATE": request.app.state.config.RAG_TEMPLATE, + "RAG_EMBEDDING_ENGINE": request.app.state.config.RAG_EMBEDDING_ENGINE, + "RAG_EMBEDDING_MODEL": request.app.state.config.RAG_EMBEDDING_MODEL, + "RAG_RERANKING_MODEL": request.app.state.config.RAG_RERANKING_MODEL, + "RAG_EMBEDDING_BATCH_SIZE": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE, + "ENABLE_ASYNC_EMBEDDING": request.app.state.config.ENABLE_ASYNC_EMBEDDING, } @@ -255,9 +256,10 @@ async def get_status(request: Request): async def get_embedding_config(request: Request, user=Depends(get_admin_user)): return { "status": True, - "embedding_engine": request.app.state.config.RAG_EMBEDDING_ENGINE, - "embedding_model": request.app.state.config.RAG_EMBEDDING_MODEL, - "embedding_batch_size": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE, + "RAG_EMBEDDING_ENGINE": request.app.state.config.RAG_EMBEDDING_ENGINE, + "RAG_EMBEDDING_MODEL": request.app.state.config.RAG_EMBEDDING_MODEL, + "RAG_EMBEDDING_BATCH_SIZE": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE, + "ENABLE_ASYNC_EMBEDDING": request.app.state.config.ENABLE_ASYNC_EMBEDDING, "openai_config": { "url": request.app.state.config.RAG_OPENAI_API_BASE_URL, "key": request.app.state.config.RAG_OPENAI_API_KEY, @@ -294,18 +296,13 @@ class EmbeddingModelUpdateForm(BaseModel): openai_config: Optional[OpenAIConfigForm] = None ollama_config: Optional[OllamaConfigForm] = None azure_openai_config: Optional[AzureOpenAIConfigForm] = None - embedding_engine: str - embedding_model: str - embedding_batch_size: Optional[int] = 1 + RAG_EMBEDDING_ENGINE: str + RAG_EMBEDDING_MODEL: str + RAG_EMBEDDING_BATCH_SIZE: Optional[int] = 1 + ENABLE_ASYNC_EMBEDDING: Optional[bool] = True -@router.post("/embedding/update") -async def update_embedding_config( - request: Request, form_data: EmbeddingModelUpdateForm, user=Depends(get_admin_user) -): - log.info( - f"Updating embedding model: {request.app.state.config.RAG_EMBEDDING_MODEL} to {form_data.embedding_model}" - ) +def unload_embedding_model(request: Request): if request.app.state.config.RAG_EMBEDDING_ENGINE == "": # unloads current internal embedding model and clears VRAM cache request.app.state.ef = None @@ -318,9 +315,25 @@ async def update_embedding_config( if torch.cuda.is_available(): torch.cuda.empty_cache() + + +@router.post("/embedding/update") +async def update_embedding_config( + request: Request, form_data: EmbeddingModelUpdateForm, user=Depends(get_admin_user) +): + log.info( + f"Updating embedding model: {request.app.state.config.RAG_EMBEDDING_MODEL} to {form_data.RAG_EMBEDDING_MODEL}" + ) + unload_embedding_model(request) try: - request.app.state.config.RAG_EMBEDDING_ENGINE = form_data.embedding_engine - request.app.state.config.RAG_EMBEDDING_MODEL = form_data.embedding_model + request.app.state.config.RAG_EMBEDDING_ENGINE = form_data.RAG_EMBEDDING_ENGINE + request.app.state.config.RAG_EMBEDDING_MODEL = form_data.RAG_EMBEDDING_MODEL + request.app.state.config.RAG_EMBEDDING_BATCH_SIZE = ( + form_data.RAG_EMBEDDING_BATCH_SIZE + ) + request.app.state.config.ENABLE_ASYNC_EMBEDDING = ( + form_data.ENABLE_ASYNC_EMBEDDING + ) if request.app.state.config.RAG_EMBEDDING_ENGINE in [ "ollama", @@ -354,10 +367,6 @@ async def update_embedding_config( form_data.azure_openai_config.version ) - request.app.state.config.RAG_EMBEDDING_BATCH_SIZE = ( - form_data.embedding_batch_size - ) - request.app.state.ef = get_ef( request.app.state.config.RAG_EMBEDDING_ENGINE, request.app.state.config.RAG_EMBEDDING_MODEL, @@ -391,13 +400,15 @@ async def update_embedding_config( if request.app.state.config.RAG_EMBEDDING_ENGINE == "azure_openai" else None ), + enable_async=request.app.state.config.ENABLE_ASYNC_EMBEDDING, ) return { "status": True, - "embedding_engine": request.app.state.config.RAG_EMBEDDING_ENGINE, - "embedding_model": request.app.state.config.RAG_EMBEDDING_MODEL, - "embedding_batch_size": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE, + "RAG_EMBEDDING_ENGINE": request.app.state.config.RAG_EMBEDDING_ENGINE, + "RAG_EMBEDDING_MODEL": request.app.state.config.RAG_EMBEDDING_MODEL, + "RAG_EMBEDDING_BATCH_SIZE": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE, + "ENABLE_ASYNC_EMBEDDING": request.app.state.config.ENABLE_ASYNC_EMBEDDING, "openai_config": { "url": request.app.state.config.RAG_OPENAI_API_BASE_URL, "key": request.app.state.config.RAG_OPENAI_API_KEY, @@ -453,18 +464,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): "EXTERNAL_DOCUMENT_LOADER_API_KEY": request.app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY, "TIKA_SERVER_URL": request.app.state.config.TIKA_SERVER_URL, "DOCLING_SERVER_URL": request.app.state.config.DOCLING_SERVER_URL, + "DOCLING_API_KEY": request.app.state.config.DOCLING_API_KEY, "DOCLING_PARAMS": request.app.state.config.DOCLING_PARAMS, - "DOCLING_DO_OCR": request.app.state.config.DOCLING_DO_OCR, - "DOCLING_FORCE_OCR": request.app.state.config.DOCLING_FORCE_OCR, - "DOCLING_OCR_ENGINE": request.app.state.config.DOCLING_OCR_ENGINE, - "DOCLING_OCR_LANG": request.app.state.config.DOCLING_OCR_LANG, - "DOCLING_PDF_BACKEND": request.app.state.config.DOCLING_PDF_BACKEND, - "DOCLING_TABLE_MODE": request.app.state.config.DOCLING_TABLE_MODE, - "DOCLING_PIPELINE": request.app.state.config.DOCLING_PIPELINE, - "DOCLING_DO_PICTURE_DESCRIPTION": request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION, - "DOCLING_PICTURE_DESCRIPTION_MODE": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE, - "DOCLING_PICTURE_DESCRIPTION_LOCAL": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL, - "DOCLING_PICTURE_DESCRIPTION_API": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API, "DOCUMENT_INTELLIGENCE_ENDPOINT": request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT, "DOCUMENT_INTELLIGENCE_KEY": request.app.state.config.DOCUMENT_INTELLIGENCE_KEY, "MISTRAL_OCR_API_BASE_URL": request.app.state.config.MISTRAL_OCR_API_BASE_URL, @@ -642,18 +643,8 @@ class ConfigForm(BaseModel): TIKA_SERVER_URL: Optional[str] = None DOCLING_SERVER_URL: Optional[str] = None + DOCLING_API_KEY: Optional[str] = None DOCLING_PARAMS: Optional[dict] = None - DOCLING_DO_OCR: Optional[bool] = None - DOCLING_FORCE_OCR: Optional[bool] = None - DOCLING_OCR_ENGINE: Optional[str] = None - DOCLING_OCR_LANG: Optional[str] = None - DOCLING_PDF_BACKEND: Optional[str] = None - DOCLING_TABLE_MODE: Optional[str] = None - DOCLING_PIPELINE: Optional[str] = None - DOCLING_DO_PICTURE_DESCRIPTION: Optional[bool] = None - DOCLING_PICTURE_DESCRIPTION_MODE: Optional[str] = None - DOCLING_PICTURE_DESCRIPTION_LOCAL: Optional[dict] = None - DOCLING_PICTURE_DESCRIPTION_API: Optional[dict] = None DOCUMENT_INTELLIGENCE_ENDPOINT: Optional[str] = None DOCUMENT_INTELLIGENCE_KEY: Optional[str] = None MISTRAL_OCR_API_BASE_URL: Optional[str] = None @@ -831,68 +822,16 @@ async def update_rag_config( if form_data.DOCLING_SERVER_URL is not None else request.app.state.config.DOCLING_SERVER_URL ) + request.app.state.config.DOCLING_API_KEY = ( + form_data.DOCLING_API_KEY + if form_data.DOCLING_API_KEY is not None + else request.app.state.config.DOCLING_API_KEY + ) request.app.state.config.DOCLING_PARAMS = ( form_data.DOCLING_PARAMS if form_data.DOCLING_PARAMS is not None else request.app.state.config.DOCLING_PARAMS ) - request.app.state.config.DOCLING_DO_OCR = ( - form_data.DOCLING_DO_OCR - if form_data.DOCLING_DO_OCR is not None - else request.app.state.config.DOCLING_DO_OCR - ) - request.app.state.config.DOCLING_FORCE_OCR = ( - form_data.DOCLING_FORCE_OCR - if form_data.DOCLING_FORCE_OCR is not None - else request.app.state.config.DOCLING_FORCE_OCR - ) - request.app.state.config.DOCLING_OCR_ENGINE = ( - form_data.DOCLING_OCR_ENGINE - if form_data.DOCLING_OCR_ENGINE is not None - else request.app.state.config.DOCLING_OCR_ENGINE - ) - request.app.state.config.DOCLING_OCR_LANG = ( - form_data.DOCLING_OCR_LANG - if form_data.DOCLING_OCR_LANG is not None - else request.app.state.config.DOCLING_OCR_LANG - ) - request.app.state.config.DOCLING_PDF_BACKEND = ( - form_data.DOCLING_PDF_BACKEND - if form_data.DOCLING_PDF_BACKEND is not None - else request.app.state.config.DOCLING_PDF_BACKEND - ) - request.app.state.config.DOCLING_TABLE_MODE = ( - form_data.DOCLING_TABLE_MODE - if form_data.DOCLING_TABLE_MODE is not None - else request.app.state.config.DOCLING_TABLE_MODE - ) - request.app.state.config.DOCLING_PIPELINE = ( - form_data.DOCLING_PIPELINE - if form_data.DOCLING_PIPELINE is not None - else request.app.state.config.DOCLING_PIPELINE - ) - request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION = ( - form_data.DOCLING_DO_PICTURE_DESCRIPTION - if form_data.DOCLING_DO_PICTURE_DESCRIPTION is not None - else request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION - ) - - request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE = ( - form_data.DOCLING_PICTURE_DESCRIPTION_MODE - if form_data.DOCLING_PICTURE_DESCRIPTION_MODE is not None - else request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE - ) - request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL = ( - form_data.DOCLING_PICTURE_DESCRIPTION_LOCAL - if form_data.DOCLING_PICTURE_DESCRIPTION_LOCAL is not None - else request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL - ) - request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API = ( - form_data.DOCLING_PICTURE_DESCRIPTION_API - if form_data.DOCLING_PICTURE_DESCRIPTION_API is not None - else request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API - ) - request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT = ( form_data.DOCUMENT_INTELLIGENCE_ENDPOINT if form_data.DOCUMENT_INTELLIGENCE_ENDPOINT is not None @@ -1189,18 +1128,8 @@ async def update_rag_config( "EXTERNAL_DOCUMENT_LOADER_API_KEY": request.app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY, "TIKA_SERVER_URL": request.app.state.config.TIKA_SERVER_URL, "DOCLING_SERVER_URL": request.app.state.config.DOCLING_SERVER_URL, + "DOCLING_API_KEY": request.app.state.config.DOCLING_API_KEY, "DOCLING_PARAMS": request.app.state.config.DOCLING_PARAMS, - "DOCLING_DO_OCR": request.app.state.config.DOCLING_DO_OCR, - "DOCLING_FORCE_OCR": request.app.state.config.DOCLING_FORCE_OCR, - "DOCLING_OCR_ENGINE": request.app.state.config.DOCLING_OCR_ENGINE, - "DOCLING_OCR_LANG": request.app.state.config.DOCLING_OCR_LANG, - "DOCLING_PDF_BACKEND": request.app.state.config.DOCLING_PDF_BACKEND, - "DOCLING_TABLE_MODE": request.app.state.config.DOCLING_TABLE_MODE, - "DOCLING_PIPELINE": request.app.state.config.DOCLING_PIPELINE, - "DOCLING_DO_PICTURE_DESCRIPTION": request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION, - "DOCLING_PICTURE_DESCRIPTION_MODE": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE, - "DOCLING_PICTURE_DESCRIPTION_LOCAL": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL, - "DOCLING_PICTURE_DESCRIPTION_API": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API, "DOCUMENT_INTELLIGENCE_ENDPOINT": request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT, "DOCUMENT_INTELLIGENCE_KEY": request.app.state.config.DOCUMENT_INTELLIGENCE_KEY, "MISTRAL_OCR_API_BASE_URL": request.app.state.config.MISTRAL_OCR_API_BASE_URL, @@ -1512,6 +1441,9 @@ def process_file( form_data: ProcessFileForm, user=Depends(get_verified_user), ): + """ + Process a file and save its content to the vector database. + """ if user.role == "admin": file = Files.get_file_by_id(form_data.file_id) else: @@ -1607,20 +1539,8 @@ def process_file( EXTERNAL_DOCUMENT_LOADER_API_KEY=request.app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY, TIKA_SERVER_URL=request.app.state.config.TIKA_SERVER_URL, DOCLING_SERVER_URL=request.app.state.config.DOCLING_SERVER_URL, - DOCLING_PARAMS={ - "do_ocr": request.app.state.config.DOCLING_DO_OCR, - "force_ocr": request.app.state.config.DOCLING_FORCE_OCR, - "ocr_engine": request.app.state.config.DOCLING_OCR_ENGINE, - "ocr_lang": request.app.state.config.DOCLING_OCR_LANG, - "pdf_backend": request.app.state.config.DOCLING_PDF_BACKEND, - "table_mode": request.app.state.config.DOCLING_TABLE_MODE, - "pipeline": request.app.state.config.DOCLING_PIPELINE, - "do_picture_description": request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION, - "picture_description_mode": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE, - "picture_description_local": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL, - "picture_description_api": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API, - **request.app.state.config.DOCLING_PARAMS, - }, + DOCLING_API_KEY=request.app.state.config.DOCLING_API_KEY, + DOCLING_PARAMS=request.app.state.config.DOCLING_PARAMS, PDF_EXTRACT_IMAGES=request.app.state.config.PDF_EXTRACT_IMAGES, DOCUMENT_INTELLIGENCE_ENDPOINT=request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT, DOCUMENT_INTELLIGENCE_KEY=request.app.state.config.DOCUMENT_INTELLIGENCE_KEY, @@ -1750,7 +1670,7 @@ class ProcessTextForm(BaseModel): @router.post("/process/text") -def process_text( +async def process_text( request: Request, form_data: ProcessTextForm, user=Depends(get_verified_user), @@ -1768,7 +1688,9 @@ def process_text( text_content = form_data.content log.debug(f"text_content: {text_content}") - result = save_docs_to_vector_db(request, docs, collection_name, user=user) + result = await run_in_threadpool( + save_docs_to_vector_db, request, docs, collection_name, user + ) if result: return { "status": True, @@ -1784,7 +1706,7 @@ def process_text( @router.post("/process/youtube") @router.post("/process/web") -def process_web( +async def process_web( request: Request, form_data: ProcessUrlForm, user=Depends(get_verified_user) ): try: @@ -1792,16 +1714,14 @@ def process_web( if not collection_name: collection_name = calculate_sha256_string(form_data.url)[:63] - content, docs = get_content_from_url(request, form_data.url) + content, docs = await run_in_threadpool( + get_content_from_url, request, form_data.url + ) log.debug(f"text_content: {content}") if not request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: - save_docs_to_vector_db( - request, - docs, - collection_name, - overwrite=True, - user=user, + await run_in_threadpool( + save_docs_to_vector_db, request, docs, collection_name, True, user ) else: collection_name = None @@ -2488,7 +2408,7 @@ class BatchProcessFilesResponse(BaseModel): @router.post("/process/files/batch") -def process_files_batch( +async def process_files_batch( request: Request, form_data: BatchProcessFilesForm, user=Depends(get_verified_user), @@ -2543,12 +2463,8 @@ def process_files_batch( # Save all documents in one batch if all_docs: try: - save_docs_to_vector_db( - request=request, - docs=all_docs, - collection_name=collection_name, - add=True, - user=user, + await run_in_threadpool( + save_docs_to_vector_db, request, all_docs, collection_name, True, user ) # Update all files with collection name diff --git a/backend/open_webui/routers/users.py b/backend/open_webui/routers/users.py index f53b0e2749..0b44e4319a 100644 --- a/backend/open_webui/routers/users.py +++ b/backend/open_webui/routers/users.py @@ -17,7 +17,7 @@ from open_webui.models.chats import Chats from open_webui.models.users import ( UserModel, UserGroupIdsModel, - UserListResponse, + UserGroupIdsListResponse, UserInfoListResponse, UserIdNameListResponse, UserRoleUpdateForm, @@ -76,7 +76,7 @@ async def get_active_users( PAGE_ITEM_COUNT = 30 -@router.get("/", response_model=UserListResponse) +@router.get("/", response_model=UserGroupIdsListResponse) async def get_users( query: Optional[str] = None, order_by: Optional[str] = None, diff --git a/backend/open_webui/utils/access_control.py b/backend/open_webui/utils/access_control.py index af48bebfb4..97d0b41491 100644 --- a/backend/open_webui/utils/access_control.py +++ b/backend/open_webui/utils/access_control.py @@ -105,6 +105,22 @@ def has_permission( return get_permission(default_permissions, permission_hierarchy) +def get_permitted_group_and_user_ids( + type: str = "write", access_control: Optional[dict] = None +) -> Union[Dict[str, List[str]], None]: + if access_control is None: + return None + + permission_access = access_control.get(type, {}) + permitted_group_ids = permission_access.get("group_ids", []) + permitted_user_ids = permission_access.get("user_ids", []) + + return { + "group_ids": permitted_group_ids, + "user_ids": permitted_user_ids, + } + + def has_access( user_id: str, type: str = "write", @@ -122,9 +138,12 @@ def has_access( user_groups = Groups.get_groups_by_member_id(user_id) user_group_ids = {group.id for group in user_groups} - permission_access = access_control.get(type, {}) - permitted_group_ids = permission_access.get("group_ids", []) - permitted_user_ids = permission_access.get("user_ids", []) + permitted_ids = get_permitted_group_and_user_ids(type, access_control) + if permitted_ids is None: + return False + + permitted_group_ids = permitted_ids.get("group_ids", []) + permitted_user_ids = permitted_ids.get("user_ids", []) return user_id in permitted_user_ids or any( group_id in permitted_group_ids for group_id in user_group_ids @@ -136,18 +155,20 @@ def get_users_with_access( type: str = "write", access_control: Optional[dict] = None ) -> list[UserModel]: if access_control is None: - result = Users.get_users() + result = Users.get_users(filter={"roles": ["!pending"]}) return result.get("users", []) - permission_access = access_control.get(type, {}) - permitted_group_ids = permission_access.get("group_ids", []) - permitted_user_ids = permission_access.get("user_ids", []) + permitted_ids = get_permitted_group_and_user_ids(type, access_control) + if permitted_ids is None: + return [] + + permitted_group_ids = permitted_ids.get("group_ids", []) + permitted_user_ids = permitted_ids.get("user_ids", []) user_ids_with_access = set(permitted_user_ids) - for group_id in permitted_group_ids: - group_user_ids = Groups.get_group_user_ids_by_id(group_id) - if group_user_ids: - user_ids_with_access.update(group_user_ids) + group_user_ids_map = Groups.get_group_user_ids_by_ids(permitted_group_ids) + for user_ids in group_user_ids_map.values(): + user_ids_with_access.update(user_ids) return Users.get_users_by_user_ids(list(user_ids_with_access)) diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 4a4e0ea6be..efa187a382 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -24,6 +24,7 @@ from fastapi.responses import HTMLResponse from starlette.responses import Response, StreamingResponse, JSONResponse +from open_webui.utils.misc import is_string_allowed from open_webui.models.oauth_sessions import OAuthSessions from open_webui.models.chats import Chats from open_webui.models.folders import Folders @@ -1408,6 +1409,12 @@ async def process_chat_payload(request, form_data, user, metadata, model): headers=headers if headers else None, ) + function_name_filter_list = ( + mcp_server_connection.get("config", {}) + .get("function_name_filter_list", "") + .split(",") + ) + tool_specs = await mcp_clients[server_id].list_tool_specs() for tool_spec in tool_specs: @@ -1420,6 +1427,13 @@ async def process_chat_payload(request, form_data, user, metadata, model): return tool_function + if function_name_filter_list: + if not is_string_allowed( + tool_spec["name"], function_name_filter_list + ): + # Skip this function + continue + tool_function = make_tool_function( mcp_clients[server_id], tool_spec["name"] ) @@ -1460,6 +1474,7 @@ async def process_chat_payload(request, form_data, user, metadata, model): "__files__": metadata.get("files", []), }, ) + if mcp_tools_dict: tools_dict = {**tools_dict, **mcp_tools_dict} diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index ce16691365..5591fcdb3f 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -27,6 +27,47 @@ def deep_update(d, u): return d +def get_allow_block_lists(filter_list): + allow_list = [] + block_list = [] + + if filter_list: + for d in filter_list: + if d.startswith("!"): + # Domains starting with "!" → blocked + block_list.append(d[1:].strip()) + else: + # Domains starting without "!" → allowed + allow_list.append(d.strip()) + + return allow_list, block_list + + +def is_string_allowed(string: str, filter_list: Optional[list[str]] = None) -> bool: + """ + Checks if a string is allowed based on the provided filter list. + :param string: The string to check (e.g., domain or hostname). + :param filter_list: List of allowed/blocked strings. Strings starting with "!" are blocked. + :return: True if the string is allowed, False otherwise. + """ + if not filter_list: + return True + + allow_list, block_list = get_allow_block_lists(filter_list) + print(string, allow_list, block_list) + + # If allow list is non-empty, require domain to match one of them + if allow_list: + if not any(string.endswith(allowed) for allowed in allow_list): + return False + + # Block list always removes matches + if any(string.endswith(blocked) for blocked in block_list): + return False + + return True + + def get_message_list(messages_map, message_id): """ Reconstructs a list of messages in order up to the specified message_id. diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index fb623ed332..268624135d 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -34,6 +34,7 @@ from langchain_core.utils.function_calling import ( ) +from open_webui.utils.misc import is_string_allowed from open_webui.models.tools import Tools from open_webui.models.users import UserModel from open_webui.utils.plugin import load_tool_module_by_id @@ -149,8 +150,20 @@ async def get_tools( ) specs = tool_server_data.get("specs", []) + function_name_filter_list = ( + tool_server_connection.get("config", {}) + .get("function_name_filter_list", "") + .split(",") + ) + for spec in specs: function_name = spec["name"] + if function_name_filter_list: + if not is_string_allowed( + function_name, function_name_filter_list + ): + # Skip this function + continue auth_type = tool_server_connection.get("auth_type", "bearer") diff --git a/backend/requirements-min.txt b/backend/requirements-min.txt index bc4732fc1d..04befcf8c1 100644 --- a/backend/requirements-min.txt +++ b/backend/requirements-min.txt @@ -47,4 +47,5 @@ fake-useragent==2.2.0 chromadb==1.1.0 black==25.9.0 -pydub \ No newline at end of file +pydub +chardet==5.2.0 diff --git a/backend/requirements.txt b/backend/requirements.txt index db32255a89..999818f7a4 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -59,6 +59,7 @@ pyarrow==20.0.0 # fix: pin pyarrow version to 20 for rpi compatibility #15897 einops==0.8.1 ftfy==6.2.3 +chardet==5.2.0 pypdf==6.0.0 fpdf2==2.8.2 pymdown-extensions==10.14.2 diff --git a/package-lock.json b/package-lock.json index c2a27915b4..47850a35ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "open-webui", - "version": "0.6.38", + "version": "0.6.39", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "open-webui", - "version": "0.6.38", + "version": "0.6.39", "dependencies": { "@azure/msal-browser": "^4.5.0", "@codemirror/lang-javascript": "^6.2.2", diff --git a/package.json b/package.json index 3ee4b5680d..a887550f53 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-webui", - "version": "0.6.38", + "version": "0.6.39", "private": true, "scripts": { "dev": "npm run pyodide:fetch && vite dev --host", diff --git a/pyproject.toml b/pyproject.toml index 85a8044e3c..fb797a72cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ dependencies = [ "einops==0.8.1", "ftfy==6.2.3", + "chardet==5.2.0", "pypdf==6.0.0", "fpdf2==2.8.2", "pymdown-extensions==10.14.2", diff --git a/src/lib/apis/channels/index.ts b/src/lib/apis/channels/index.ts index ac51e5a5d0..2872bd89f8 100644 --- a/src/lib/apis/channels/index.ts +++ b/src/lib/apis/channels/index.ts @@ -101,6 +101,60 @@ export const getChannelById = async (token: string = '', channel_id: string) => return res; }; +export const getChannelUsersById = async ( + token: string, + channel_id: string, + query?: string, + orderBy?: string, + direction?: string, + page = 1 +) => { + let error = null; + let res = null; + + const searchParams = new URLSearchParams(); + + searchParams.set('page', `${page}`); + + if (query) { + searchParams.set('query', query); + } + + if (orderBy) { + searchParams.set('order_by', orderBy); + } + + if (direction) { + searchParams.set('direction', direction); + } + + res = await fetch( + `${WEBUI_API_BASE_URL}/channels/${channel_id}/users?${searchParams.toString()}`, + { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + } + } + ) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.error(err); + error = err.detail; + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + export const updateChannelById = async ( token: string = '', channel_id: string, diff --git a/src/lib/apis/retrieval/index.ts b/src/lib/apis/retrieval/index.ts index 6df927fec6..5cb0f60a72 100644 --- a/src/lib/apis/retrieval/index.ts +++ b/src/lib/apis/retrieval/index.ts @@ -295,42 +295,6 @@ export interface SearchDocument { filenames: string[]; } -export const processFile = async ( - token: string, - file_id: string, - collection_name: string | null = null -) => { - let error = null; - - const res = await fetch(`${RETRIEVAL_API_BASE_URL}/process/file`, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - authorization: `Bearer ${token}` - }, - body: JSON.stringify({ - file_id: file_id, - collection_name: collection_name ? collection_name : undefined - }) - }) - .then(async (res) => { - if (!res.ok) throw await res.json(); - return res.json(); - }) - .catch((err) => { - error = err.detail; - console.error(err); - return null; - }); - - if (error) { - throw error; - } - - return res; -}; - export const processYoutubeVideo = async (token: string, url: string) => { let error = null; diff --git a/src/lib/components/AddToolServerModal.svelte b/src/lib/components/AddToolServerModal.svelte index a2098de912..2b639b3e64 100644 --- a/src/lib/components/AddToolServerModal.svelte +++ b/src/lib/components/AddToolServerModal.svelte @@ -47,6 +47,7 @@ let key = ''; let headers = ''; + let functionNameFilterList = []; let accessControl = {}; let id = ''; @@ -303,7 +304,7 @@ key, config: { enable: enable, - + function_name_filter_list: functionNameFilterList, access_control: accessControl }, info: { @@ -333,9 +334,11 @@ id = ''; name = ''; description = ''; + oauthClientInfo = null; enable = true; + functionNameFilterList = []; accessControl = null; }; @@ -359,6 +362,7 @@ oauthClientInfo = connection.info?.oauth_client_info ?? null; enable = connection.config?.enable ?? true; + functionNameFilterList = connection.config?.function_name_filter_list ?? []; accessControl = connection.config?.access_control ?? null; } }; @@ -793,6 +797,25 @@ +
+ + +
+ +
+
+
diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index 83369ac993..548583ee8a 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -38,9 +38,11 @@ let showResetUploadDirConfirm = false; let showReindexConfirm = false; - let embeddingEngine = ''; - let embeddingModel = ''; - let embeddingBatchSize = 1; + let RAG_EMBEDDING_ENGINE = ''; + let RAG_EMBEDDING_MODEL = ''; + let RAG_EMBEDDING_BATCH_SIZE = 1; + let ENABLE_ASYNC_EMBEDDING = true; + let rerankingModel = ''; let OpenAIUrl = ''; @@ -64,7 +66,7 @@ let RAGConfig = null; const embeddingModelUpdateHandler = async () => { - if (embeddingEngine === '' && embeddingModel.split('/').length - 1 > 1) { + if (RAG_EMBEDDING_ENGINE === '' && RAG_EMBEDDING_MODEL.split('/').length - 1 > 1) { toast.error( $i18n.t( 'Model filesystem path detected. Model shortname is required for update, cannot continue.' @@ -72,7 +74,7 @@ ); return; } - if (embeddingEngine === 'ollama' && embeddingModel === '') { + if (RAG_EMBEDDING_ENGINE === 'ollama' && RAG_EMBEDDING_MODEL === '') { toast.error( $i18n.t( 'Model filesystem path detected. Model shortname is required for update, cannot continue.' @@ -81,7 +83,7 @@ return; } - if (embeddingEngine === 'openai' && embeddingModel === '') { + if (RAG_EMBEDDING_ENGINE === 'openai' && RAG_EMBEDDING_MODEL === '') { toast.error( $i18n.t( 'Model filesystem path detected. Model shortname is required for update, cannot continue.' @@ -91,20 +93,26 @@ } if ( - embeddingEngine === 'azure_openai' && + RAG_EMBEDDING_ENGINE === 'azure_openai' && (AzureOpenAIKey === '' || AzureOpenAIUrl === '' || AzureOpenAIVersion === '') ) { toast.error($i18n.t('OpenAI URL/Key required.')); return; } - console.debug('Update embedding model attempt:', embeddingModel); + console.debug('Update embedding model attempt:', { + RAG_EMBEDDING_ENGINE, + RAG_EMBEDDING_MODEL, + RAG_EMBEDDING_BATCH_SIZE, + ENABLE_ASYNC_EMBEDDING + }); updateEmbeddingModelLoading = true; const res = await updateEmbeddingConfig(localStorage.token, { - embedding_engine: embeddingEngine, - embedding_model: embeddingModel, - embedding_batch_size: embeddingBatchSize, + RAG_EMBEDDING_ENGINE: RAG_EMBEDDING_ENGINE, + RAG_EMBEDDING_MODEL: RAG_EMBEDDING_MODEL, + RAG_EMBEDDING_BATCH_SIZE: RAG_EMBEDDING_BATCH_SIZE, + ENABLE_ASYNC_EMBEDDING: ENABLE_ASYNC_EMBEDDING, ollama_config: { key: OllamaKey, url: OllamaUrl @@ -127,11 +135,6 @@ if (res) { console.debug('embeddingModelUpdateHandler:', res); - if (res.status === true) { - toast.success($i18n.t('Embedding model set to "{{embedding_model}}"', res), { - duration: 1000 * 10 - }); - } } }; @@ -151,26 +154,6 @@ toast.error($i18n.t('Docling Server URL required.')); return; } - if ( - RAGConfig.CONTENT_EXTRACTION_ENGINE === 'docling' && - RAGConfig.DOCLING_DO_OCR && - ((RAGConfig.DOCLING_OCR_ENGINE === '' && RAGConfig.DOCLING_OCR_LANG !== '') || - (RAGConfig.DOCLING_OCR_ENGINE !== '' && RAGConfig.DOCLING_OCR_LANG === '')) - ) { - toast.error( - $i18n.t('Both Docling OCR Engine and Language(s) must be provided or both left empty.') - ); - return; - } - if ( - RAGConfig.CONTENT_EXTRACTION_ENGINE === 'docling' && - RAGConfig.DOCLING_DO_OCR === false && - RAGConfig.DOCLING_FORCE_OCR === true - ) { - toast.error($i18n.t('In order to force OCR, performing OCR must be enabled.')); - return; - } - if ( RAGConfig.CONTENT_EXTRACTION_ENGINE === 'datalab_marker' && RAGConfig.DATALAB_MARKER_ADDITIONAL_CONFIG && @@ -238,12 +221,6 @@ ALLOWED_FILE_EXTENSIONS: RAGConfig.ALLOWED_FILE_EXTENSIONS.split(',') .map((ext) => ext.trim()) .filter((ext) => ext !== ''), - DOCLING_PICTURE_DESCRIPTION_LOCAL: JSON.parse( - RAGConfig.DOCLING_PICTURE_DESCRIPTION_LOCAL || '{}' - ), - DOCLING_PICTURE_DESCRIPTION_API: JSON.parse( - RAGConfig.DOCLING_PICTURE_DESCRIPTION_API || '{}' - ), DOCLING_PARAMS: typeof RAGConfig.DOCLING_PARAMS === 'string' && RAGConfig.DOCLING_PARAMS.trim() !== '' ? JSON.parse(RAGConfig.DOCLING_PARAMS) @@ -260,9 +237,10 @@ const embeddingConfig = await getEmbeddingConfig(localStorage.token); if (embeddingConfig) { - embeddingEngine = embeddingConfig.embedding_engine; - embeddingModel = embeddingConfig.embedding_model; - embeddingBatchSize = embeddingConfig.embedding_batch_size ?? 1; + RAG_EMBEDDING_ENGINE = embeddingConfig.RAG_EMBEDDING_ENGINE; + RAG_EMBEDDING_MODEL = embeddingConfig.RAG_EMBEDDING_MODEL; + RAG_EMBEDDING_BATCH_SIZE = embeddingConfig.RAG_EMBEDDING_BATCH_SIZE ?? 1; + ENABLE_ASYNC_EMBEDDING = embeddingConfig.ENABLE_ASYNC_EMBEDDING ?? true; OpenAIKey = embeddingConfig.openai_config.key; OpenAIUrl = embeddingConfig.openai_config.url; @@ -281,16 +259,6 @@ const config = await getRAGConfig(localStorage.token); config.ALLOWED_FILE_EXTENSIONS = (config?.ALLOWED_FILE_EXTENSIONS ?? []).join(', '); - config.DOCLING_PICTURE_DESCRIPTION_LOCAL = JSON.stringify( - config.DOCLING_PICTURE_DESCRIPTION_LOCAL ?? {}, - null, - 2 - ); - config.DOCLING_PICTURE_DESCRIPTION_API = JSON.stringify( - config.DOCLING_PICTURE_DESCRIPTION_API ?? {}, - null, - 2 - ); config.DOCLING_PARAMS = typeof config.DOCLING_PARAMS === 'object' ? JSON.stringify(config.DOCLING_PARAMS ?? {}, null, 2) @@ -589,174 +557,19 @@
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'docling'} -
+
+
-
-
-
- {$i18n.t('Perform OCR')} -
-
- -
-
-
- {#if RAGConfig.DOCLING_DO_OCR} -
- - -
- {/if} -
-
-
- {$i18n.t('Force OCR')} -
-
- -
-
-
-
-
- - {$i18n.t('PDF Backend')} - -
-
- -
-
-
-
- - {$i18n.t('Table Mode')} - -
-
- -
-
-
-
- - {$i18n.t('Pipeline')} - -
-
- -
-
-
-
-
- {$i18n.t('Describe Pictures in Documents')} -
-
- -
-
-
- {#if RAGConfig.DOCLING_DO_PICTURE_DESCRIPTION} -
-
- - {$i18n.t('Picture Description Mode')} - -
-
- -
-
- - {#if RAGConfig.DOCLING_PICTURE_DESCRIPTION_MODE === 'local'} -
-
-
- {$i18n.t('Picture Description Local Config')} -
-
- -