Merge branch 'dev' into feat/rate-limit-web-search

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -11,7 +11,9 @@ body:
 
         ## Important Notes
 
-        - **Before submitting a bug report**: Please check the [Issues](https://github.com/open-webui/open-webui/issues) and [Discussions](https://github.com/open-webui/open-webui/discussions) sections to see if a similar issue has already been reported. If unsure, start a discussion first, as this helps us efficiently focus on improving the project. Duplicates may be closed without notice. **Please search for existing issues and discussions.**
+        - **Before submitting a bug report**: Please check the [Issues](https://github.com/open-webui/open-webui/issues) and [Discussions](https://github.com/open-webui/open-webui/discussions) sections to see if a similar issue has already been reported. If unsure, start a discussion first, as this helps us efficiently focus on improving the project. Duplicates may be closed without notice. **Please search for existing issues AND discussions. No matter open or closed.**
+
+        - Check for opened, **but also for (recently) CLOSED issues** as the issue you are trying to report **might already have been fixed on the dev branch!**
 
         - **Respectful collaboration**: Open WebUI is a volunteer-driven project with a single maintainer and contributors who also have full-time jobs. Please be constructive and respectful in your communication.
 
@@ -19,6 +21,8 @@ body:
 
         - **Bug Reproducibility**: If a bug cannot be reproduced using a `:main` or `:dev` Docker setup or with `pip install` on Python 3.11, community assistance may be required. In such cases, we will move it to the "[Issues](https://github.com/open-webui/open-webui/discussions/categories/issues)" Discussions section. Your help is appreciated!
 
+        - **Scope**: If you want to report a SECURITY VULNERABILITY, then do so through our [GitHub security page](https://github.com/open-webui/open-webui/security).
+
   - type: checkboxes
     id: issue-check
     attributes:
@@ -29,6 +33,8 @@ body:
           required: true
         - label: I have searched for any existing and/or related discussions.
           required: true
+        - label: I have also searched in the CLOSED issues AND CLOSED discussions and found no related items (your issue might already be addressed on the development branch!).
+          required: true
         - label: I am using the latest version of Open WebUI.
           required: true
 

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -8,8 +8,19 @@ body:
       value: |
         ## Important Notes
         ### Before submitting
-        Please check the [Issues](https://github.com/open-webui/open-webui/issues) or [Discussions](https://github.com/open-webui/open-webui/discussions) to see if a similar request has been posted.
+        
+        Please check the **open AND closed** [Issues](https://github.com/open-webui/open-webui/issues) AND [Discussions](https://github.com/open-webui/open-webui/discussions) to see if a similar request has been posted.
         It's likely we're already tracking it! If you’re unsure, start a discussion post first.
+
+        #### Scope
+
+        If your feature request is likely to take more than a quick coding session to implement, test and verify, then open it in the **Ideas** section of the [Discussions](https://github.com/open-webui/open-webui/discussions) instead.
+        **We will close and force move your feature request to the Ideas section, if we believe your feature request is not trivial/quick to implement.**
+        This is to ensure the issues tab is used only for issues, quickly addressable feature requests and tracking tickets by the maintainers.
+        Other feature requests belong in the **Ideas** section of the [Discussions](https://github.com/open-webui/open-webui/discussions).
+        
+        If your feature request might impact others in the community, definitely open a discussion instead and evaluate whether and how to implement it.
+        
         This will help us efficiently focus on improving the project.
         
         ### Collaborate respectfully
@@ -22,7 +33,6 @@ body:
 
         We appreciate your time and ask that you **respect ours**.
 
-
         ### Contributing
         If you encounter an issue, we highly encourage you to submit a pull request or fork the project. We actively work to prevent contributor burnout to maintain the quality and continuity of Open WebUI.
 
@@ -35,14 +45,22 @@ body:
       label: Check Existing Issues
       description: Please confirm that you've checked for existing similar requests
       options:
-        - label: I have searched the existing issues and discussions.
+        - label: I have searched for all existing **open AND closed** issues and discussions for similar requests. I have found none that is comparable to my request.
+          required: true
+  - type: checkboxes
+    id: feature-scope
+    attributes:
+      label: Verify Feature Scope
+      description: Please confirm the feature's scope is within the described scope
+      options:
+        - label: I have read through and understood the scope definition for feature requests in the Issues section. I believe my feature request meets the definition and belongs in the Issues section instead of the Discussions.
           required: true
   - type: textarea
     id: problem-description
     attributes:
       label: Problem Description
       description: Is your feature request related to a problem? Please provide a clear and concise description of what the problem is.
-      placeholder: "Ex. I'm always frustrated when..."
+      placeholder: "Ex. I'm always frustrated when... / Not related to a problem"
     validations:
       required: true
   - type: textarea

.github/pull_request_template.md

@@ -1,16 +1,18 @@
 # Pull Request Checklist
 
-### Note to first-time contributors: Please open a discussion post in [Discussions](https://github.com/open-webui/open-webui/discussions) and describe your changes before submitting a pull request.
+### Note to first-time contributors: Please open a discussion post in [Discussions](https://github.com/open-webui/open-webui/discussions) to discuss your idea/fix with the community before creating a pull request, and describe your changes before submitting a pull request.
+
+This is to ensure large feature PRs are discussed with the community first, before starting work on it. If the community does not want this feature or it is not relevant for Open WebUI as a project, it can be identified in the discussion before working on the feature and submitting the PR.
 
 **Before submitting, make sure you've checked the following:**
 
-- [ ] **Target branch:** Verify that the pull request targets the `dev` branch. Not targeting the `dev` branch may lead to immediate closure of the PR.
-- [ ] **Description:** Provide a concise description of the changes made in this pull request.
+- [ ] **Target branch:** Verify that the pull request targets the `dev` branch. **Not targeting the `dev` branch will lead to immediate closure of the PR.**
+- [ ] **Description:** Provide a concise description of the changes made in this pull request down below.
 - [ ] **Changelog:** Ensure a changelog entry following the format of [Keep a Changelog](https://keepachangelog.com/) is added at the bottom of the PR description.
 - [ ] **Documentation:** If necessary, update relevant documentation [Open WebUI Docs](https://github.com/open-webui/docs) like environment variables, the tutorials, or other documentation sources.
 - [ ] **Dependencies:** Are there any new dependencies? Have you updated the dependency versions in the documentation?
-- [ ] **Testing:** Perform manual tests to verify the implemented fix/feature works as intended AND does not break any other functionality. Take this as an opportunity to make screenshots of the feature/fix and include it in the PR description.
-- [ ] **Agentic AI Code:**: Confirm this Pull Request is **not written by any AI Agent** or has at least gone through additional human review **and** manual testing. If any AI Agent is the co-author of this PR, it may lead to immediate closure of the PR.
+- [ ] **Testing:** Perform manual tests to **verify the implemented fix/feature works as intended AND does not break any other functionality**. Take this as an opportunity to **make screenshots of the feature/fix and include it in the PR description**.
+- [ ] **Agentic AI Code:** Confirm this Pull Request is **not written by any AI Agent** or has at least **gone through additional human review AND manual testing**. If any AI Agent is the co-author of this PR, it may lead to immediate closure of the PR.
 - [ ] **Code review:** Have you performed a self-review of your code, addressing any coding standard issues and ensuring adherence to the project's coding standards?
 - [ ] **Title Prefix:** To clearly categorize this pull request, prefix the pull request title using one of the following:
   - **BREAKING CHANGE**: Significant changes that may affect compatibility
@@ -75,3 +77,6 @@
 ### Contributor License Agreement
 
 By submitting this pull request, I confirm that I have read and fully agree to the [Contributor License Agreement (CLA)](https://github.com/open-webui/open-webui/blob/main/CONTRIBUTOR_LICENSE_AGREEMENT), and I am providing my contributions under its terms.
+
+> [!NOTE]
+> Deleting the CLA section will lead to immediate closure of your PR and it will not be merged in.

.github/workflows/docker-build.yaml

@@ -141,6 +141,9 @@ jobs:
           platform=${{ matrix.platform }}
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
+      - name: Delete huge unnecessary tools folder
+        run: rm -rf /opt/hostedtoolcache
+
       - name: Checkout repository
         uses: actions/checkout@v5
 
@@ -243,6 +246,9 @@ jobs:
           platform=${{ matrix.platform }}
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
+      - name: Delete huge unnecessary tools folder
+        run: rm -rf /opt/hostedtoolcache
+
       - name: Checkout repository
         uses: actions/checkout@v5
 

CHANGELOG.md

@@ -5,6 +5,331 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.6.40] - 2025-11-25
+
+### Fixed
+
+- 🗄️ A critical PostgreSQL user listing performance issue was resolved by removing a redundant count operation that caused severe database slowdowns and potential timeouts when viewing user lists in admin panels.
+
+## [0.6.39] - 2025-11-25
+
+### Added
+
+- 💬 A user list modal was added to channels, displaying all users with access and featuring search, sorting, and pagination capabilities. [Commit](https://github.com/open-webui/open-webui/commit/c0e120353824be00a2ef63cbde8be5d625bd6fd0)
+- 💬 Channel navigation now displays the total number of users with access to the channel. [Commit](https://github.com/open-webui/open-webui/commit/3b5710d0cd445cf86423187f5ee7c40472a0df0b)
+- 🔌 Tool servers and MCP connections now support function name filtering, allowing administrators to selectively enable or block specific functions using allow/block lists. [Commit](https://github.com/open-webui/open-webui/commit/743199f2d097ae1458381bce450d9025a0ab3f3d)
+- ⚡ A toggle to disable parallel embedding processing was added via "ENABLE_ASYNC_EMBEDDING", allowing sequential processing for rate-limited or resource-constrained local embedding setups. [#19444](https://github.com/open-webui/open-webui/pull/19444)
+- 🔄 Various improvements were implemented across the frontend and backend to enhance performance, stability, and security.
+- 🌐 Localization improvements were made for German (de-DE) and Portuguese (Brazil) translations.
+
+### Fixed
+
+- 📝 Inline citations now render correctly within markdown lists and nested elements instead of displaying as "undefined" values. [#19452](https://github.com/open-webui/open-webui/issues/19452)
+- 👥 Group member selection now works correctly without randomly selecting other users or causing the user list to jump around. [#19426](https://github.com/open-webui/open-webui/issues/19426)
+- 👥 Admin panel user list now displays the correct total user count and properly paginates 30 items per page after fixing database query issues with group member joins. [#19429](https://github.com/open-webui/open-webui/issues/19429)
+- 🔍 Knowledge base reindexing now works correctly after resolving async execution chain issues by implementing threadpool workers for embedding operations. [#19434](https://github.com/open-webui/open-webui/pull/19434)
+- 🖼️ OpenAI image generation now works correctly after fixing a connection adapter error caused by incorrect URL formatting. [#19435](https://github.com/open-webui/open-webui/pull/19435)
+
+### Changed
+
+- 🔧 BREAKING: Docling configuration has been consolidated from individual environment variables into a single "DOCLING_PARAMS" JSON configuration and now supports API key authentication via "DOCLING_API_KEY", requiring users to migrate existing Docling settings to the new format. [#16841](https://github.com/open-webui/open-webui/issues/16841), [#19427](https://github.com/open-webui/open-webui/pull/19427)
+- 🔧 The environment variable "REPLACE_IMAGE_URLS_IN_CHAT_RESPONSE" has been renamed to "ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION" for naming consistency.
+
+## [0.6.38] - 2025-11-24
+
+### Fixed
+
+- 🔍 Hybrid search now works reliably after recent changes.
+- 🛠️ Tool server saving now handles errors gracefully, preventing failed saves from impacting the UI.
+- 🔐 SSO/OIDC code fixed to improve login reliability and better handle edge cases.
+
+## [0.6.37] - 2025-11-24
+
+### Added
+
+- 🔐 Granular sharing permissions are now available with two-tiered control separating group sharing from public sharing, allowing administrators to independently configure whether users can share workspace items with groups or make them publicly accessible, with separate permission toggles for models, knowledge bases, prompts, tools, and notes, configurable via "USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_SHARING", "USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_PUBLIC_SHARING", and corresponding environment variables for other workspace item types, while groups can now be configured to opt-out of sharing via the "Allow Group Sharing" setting. [Commit](https://github.com/open-webui/open-webui/commit/7be750bcbb40da91912a0a66b7ab791effdcc3b6), [Commit](https://github.com/open-webui/open-webui/commit/f69e37a8507d6d57382d6670641b367f3127f90a)
+- 🔐 Password policy enforcement is now available with configurable validation rules, allowing administrators to require specific password complexity requirements via "ENABLE_PASSWORD_VALIDATION" and "PASSWORD_VALIDATION_REGEX_PATTERN" environment variables, with default pattern requiring minimum 8 characters including uppercase, lowercase, digit, and special character. [#17794](https://github.com/open-webui/open-webui/pull/17794)
+- 🔐 Granular import and export permissions are now available for workspace items, introducing six separate permission toggles for models, prompts, and tools that are disabled by default for enhanced security. [#19242](https://github.com/open-webui/open-webui/pull/19242)
+- 👥 Default group assignment is now available for new users, allowing administrators to automatically assign newly registered users to a specified group for streamlined access control to models, prompts, and tools, particularly useful for organizations with group-based model access policies. [#19325](https://github.com/open-webui/open-webui/pull/19325), [#17842](https://github.com/open-webui/open-webui/issues/17842)
+- 🔒 Password-based authentication can now be fully disabled via "ENABLE_PASSWORD_AUTH" environment variable, enforcing SSO-only authentication and preventing password login fallback when SSO is configured. [#19113](https://github.com/open-webui/open-webui/pull/19113)
+- 🖼️ Large stream chunk handling was implemented to support models that generate images directly in their output responses, with configurable buffer size via "CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE" environment variable, resolving compatibility issues with models like Gemini 2.5 Flash Image. [#18884](https://github.com/open-webui/open-webui/pull/18884), [#17626](https://github.com/open-webui/open-webui/issues/17626)
+- 🖼️ Streaming response middleware now handles images in delta updates with automatic base64 conversion, enabling proper display of images from models using the "choices[0].delta.images.image_url" format such as Gemini 2.5 Flash Image Preview on OpenRouter. [#19073](https://github.com/open-webui/open-webui/pull/19073), [#19019](https://github.com/open-webui/open-webui/issues/19019)
+- 📈 Model list API performance was optimized by pre-fetching user group memberships and removing profile image URLs from response payloads, significantly reducing both database queries and payload size for instances with large model lists, with profile images now served dynamically via dedicated endpoints. [#19097](https://github.com/open-webui/open-webui/pull/19097), [#18950](https://github.com/open-webui/open-webui/issues/18950)
+- ⏩ Batch file processing performance was improved by reducing database queries by 67% while ensuring data consistency between vector and relational databases. [#18953](https://github.com/open-webui/open-webui/pull/18953)
+- 🚀 Chat import performance was dramatically improved by replacing individual per-chat API requests with a bulk import endpoint, reducing import time by up to 95% for large chat collections and providing user feedback via toast notifications displaying the number of successfully imported chats. [#17861](https://github.com/open-webui/open-webui/pull/17861)
+- ⚡ Socket event broadcasting performance was optimized by implementing user-specific rooms, significantly reducing server overhead particularly for users with multiple concurrent sessions. [#18996](https://github.com/open-webui/open-webui/pull/18996)
+- 🗄️ Weaviate is now supported as a vector database option, providing an additional choice for RAG document storage alongside existing ChromaDB, Milvus, Qdrant, and OpenSearch integrations. [#14747](https://github.com/open-webui/open-webui/pull/14747)
+- 🗄️ PostgreSQL pgvector now supports HNSW index types and large dimensional embeddings exceeding 2000 dimensions through automatic halfvec type selection, with configurable index methods via "PGVECTOR_INDEX_METHOD", "PGVECTOR_HNSW_M", "PGVECTOR_HNSW_EF_CONSTRUCTION", and "PGVECTOR_IVFFLAT_LISTS" environment variables. [#19158](https://github.com/open-webui/open-webui/pull/19158), [#16890](https://github.com/open-webui/open-webui/issues/16890)
+- 🔍 Azure AI Search is now supported as a web search provider, enabling integration with Azure's cognitive search services via "AZURE_AI_SEARCH_API_KEY", "AZURE_AI_SEARCH_ENDPOINT", and "AZURE_AI_SEARCH_INDEX_NAME" configuration. [#19104](https://github.com/open-webui/open-webui/pull/19104)
+- ⚡ External embedding generation now processes API requests in parallel instead of sequential batches, reducing document processing time by 10-50x when using OpenAI, Azure OpenAI, or Ollama embedding providers, with large PDFs now processing in seconds instead of minutes. [#19296](https://github.com/open-webui/open-webui/pull/19296)
+- 💨 Base64 image conversion is now available for markdown content in chat responses, automatically uploading embedded images exceeding 1KB and replacing them with file URLs to reduce payload size and resource consumption, configurable via "REPLACE_IMAGE_URLS_IN_CHAT_RESPONSE" environment variable. [#19076](https://github.com/open-webui/open-webui/pull/19076)
+- 🎨 OpenAI image generation now supports additional API parameters including quality settings for GPT Image 1, configurable via "IMAGES_OPENAI_API_PARAMS" environment variable or through the admin interface, enabling cost-effective image generation with low, medium, or high quality options. [#19228](https://github.com/open-webui/open-webui/issues/19228)
+- 🖼️ Image editing can now be independently enabled or disabled via admin settings, allowing administrators to control whether sequential image prompts trigger image editing or new image generation, configurable via "ENABLE_IMAGE_EDIT" environment variable. [#19284](https://github.com/open-webui/open-webui/issues/19284)
+- 🔐 SSRF protection was implemented with a configurable URL blocklist that prevents access to cloud metadata endpoints and private networks, with default protections for AWS, Google Cloud, Azure, and Alibaba Cloud metadata services, customizable via "WEB_FETCH_FILTER_LIST" environment variable. [#19201](https://github.com/open-webui/open-webui/pull/19201)
+- ⚡ Workspace models page now supports server-side pagination dramatically improving load times and usability for instances with large numbers of workspace models.
+- 🔍 Hybrid search now indexes file metadata including filenames, titles, headings, sources, and snippets alongside document content, enabling keyword queries to surface documents where search terms appear only in metadata, configurable via "ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS" environment variable. [#19095](https://github.com/open-webui/open-webui/pull/19095)
+- 📂 Knowledge base upload page now supports folder drag-and-drop with recursive directory handling, enabling batch uploads of entire directory structures instead of requiring individual file selection. [#19320](https://github.com/open-webui/open-webui/pull/19320)
+- 🤖 Model cloning is now available in admin settings, allowing administrators to quickly create workspace models based on existing base models through a "Clone" option in the model dropdown menu. [#17937](https://github.com/open-webui/open-webui/pull/17937)
+- 🎨 UI scale adjustment is now available in interface settings, allowing users to increase the size of the entire interface from 1.0x to 1.5x for improved accessibility and readability, particularly beneficial for users with visual impairments. [#19186](https://github.com/open-webui/open-webui/pull/19186)
+- 📌 Default pinned models can now be configured by administrators for all new users, mirroring the behavior of default models where admin-configured defaults apply only to users who haven't customized their pinned models, configurable via "DEFAULT_PINNED_MODELS" environment variable. [#19273](https://github.com/open-webui/open-webui/pull/19273)
+- 🎙️ Text-to-Speech and Speech-to-Text services now receive user information headers when "ENABLE_FORWARD_USER_INFO_HEADERS" is enabled, allowing external TTS and STT providers to implement user-specific personalization, rate limiting, and usage tracking. [#19323](https://github.com/open-webui/open-webui/pull/19323), [#19312](https://github.com/open-webui/open-webui/issues/19312)
+- 🎙️ Voice mode now supports custom system prompts via "VOICE_MODE_PROMPT_TEMPLATE" configuration, allowing administrators to control response style and behavior for voice interactions. [#18607](https://github.com/open-webui/open-webui/pull/18607)
+- 🔧 WebSocket and Redis configuration options are now available including debug logging controls, custom ping timeout and interval settings, and arbitrary Redis connection options via "WEBSOCKET_SERVER_LOGGING", "WEBSOCKET_SERVER_ENGINEIO_LOGGING", "WEBSOCKET_SERVER_PING_TIMEOUT", "WEBSOCKET_SERVER_PING_INTERVAL", and "WEBSOCKET_REDIS_OPTIONS" environment variables. [#19091](https://github.com/open-webui/open-webui/pull/19091)
+- 🔧 MCP OAuth dynamic client registration now automatically detects and uses the appropriate token endpoint authentication method from server-supported options, enabling compatibility with OAuth servers that only support "client_secret_basic" instead of "client_secret_post". [#19193](https://github.com/open-webui/open-webui/issues/19193)
+- 🔧 Custom headers can now be configured for remote MCP and OpenAPI tool server connections, enabling integration with services that require additional authentication headers. [#18918](https://github.com/open-webui/open-webui/issues/18918)
+- 🔍 Perplexity Search now supports custom API endpoints via "PERPLEXITY_SEARCH_API_URL" configuration and automatically forwards user information headers to enable personalized search experiences. [#19147](https://github.com/open-webui/open-webui/pull/19147)
+- 🔍 User information headers can now be optionally forwarded to external web search engines when "ENABLE_FORWARD_USER_INFO_HEADERS" is enabled. [#19043](https://github.com/open-webui/open-webui/pull/19043)
+- 📊 Daily active user metric is now available for monitoring, tracking unique users active since midnight UTC via the "webui.users.active.today" Prometheus gauge. [#19236](https://github.com/open-webui/open-webui/pull/19236), [#19234](https://github.com/open-webui/open-webui/issues/19234)
+- 📊 Audit log file path is now configurable via "AUDIT_LOGS_FILE_PATH" environment variable, enabling storage in separate volumes or custom locations. [#19173](https://github.com/open-webui/open-webui/pull/19173)
+- 🎨 Sidebar collapse states for model lists and group information are now persistent across page refreshes, remembering user preferences through browser-based storage. [#19159](https://github.com/open-webui/open-webui/issues/19159)
+- 🎨 Background image display was enhanced with semi-transparent overlays for navbar and sidebar, creating a seamless and visually cohesive design across the entire interface. [#19157](https://github.com/open-webui/open-webui/issues/19157)
+- 📋 Tables in chat messages now include a copy button that appears on hover, enabling quick copying of table content alongside the existing CSV export functionality. [#19162](https://github.com/open-webui/open-webui/issues/19162)
+- 📝 Notes can now be created directly via the "/notes/new" URL endpoint with optional title and content query parameters, enabling faster note creation through bookmarks and shortcuts. [#19195](https://github.com/open-webui/open-webui/issues/19195)
+- 🏷️ Tag suggestions are now context-aware, displaying only relevant tags when creating or editing models versus chat conversations, preventing confusion between model and chat tags. [#19135](https://github.com/open-webui/open-webui/issues/19135)
+- ✍️ Prompt autocompletion is now available independently of the rich text input setting, improving accessibility to the feature. [#19150](https://github.com/open-webui/open-webui/issues/19150)
+- 🔄 Various improvements were implemented across the frontend and backend to enhance performance, stability, and security.
+- 🌐 Translations for Simplified Chinese, Traditional Chinese, Portuguese (Brazil), Catalan, Spanish (Spain), Finnish, Irish, Farsi, Swedish, Danish, German, Korean, and Thai were improved and expanded.
+
+### Fixed
+
+- 🤖 Model update functionality now works correctly, resolving a database parameter binding error that prevented saving changes to model configurations via the Save & Update button. [#19335](https://github.com/open-webui/open-webui/issues/19335)
+- 🖼️ Multiple input images for image editing and generation are now correctly passed as an array using the "image[]" parameter syntax, enabling proper multi-image reference functionality with models like GPT Image 1. [#19339](https://github.com/open-webui/open-webui/issues/19339)
+- 📱 PWA installations on iOS now properly refresh after server container restarts, resolving freezing issues by automatically unregistering service workers when version or deployment changes are detected. [#19316](https://github.com/open-webui/open-webui/pull/19316)
+- 🗄️ S3 Vectors collection detection now correctly handles buckets with more than 2000 indexes by using direct index lookup instead of paginated list scanning, improving performance by approximately 8x and enabling RAG queries to work reliably at scale. [#19238](https://github.com/open-webui/open-webui/pull/19238), [#19233](https://github.com/open-webui/open-webui/issues/19233)
+- 📈 Feedback retrieval performance was optimized by eliminating N+1 query patterns through database joins, adding server-side pagination and sorting, significantly reducing database load for instances with large feedback datasets. [#17976](https://github.com/open-webui/open-webui/pull/17976)
+- 🔍 Chat search now works correctly with PostgreSQL when chat data contains null bytes, with comprehensive sanitization preventing null bytes during data writes, cleaning existing data on read, and stripping null bytes during search queries to ensure reliable search functionality. [#15616](https://github.com/open-webui/open-webui/issues/15616)
+- 🔍 Hybrid search with reranking now correctly handles attribute validation, preventing errors when collection results lack expected structure. [#19025](https://github.com/open-webui/open-webui/pull/19025), [#17046](https://github.com/open-webui/open-webui/issues/17046)
+- 🔎 Reranking functionality now works correctly after recent refactoring, resolving crashes caused by incorrect function argument handling. [#19270](https://github.com/open-webui/open-webui/pull/19270)
+- 🤖 Azure OpenAI models now support the "reasoning_effort" parameter, enabling proper configuration of reasoning capabilities for models like GPT-5.1 which default to no reasoning without this setting. [#19290](https://github.com/open-webui/open-webui/issues/19290)
+- 🤖 Models with very long IDs can now be deleted correctly, resolving URL length limitations that previously prevented management operations on such models. [#18230](https://github.com/open-webui/open-webui/pull/18230)
+- 🤖 Model-level streaming settings now correctly apply to API requests, ensuring "Stream Chat Response" toggle properly controls the streaming parameter. [#19154](https://github.com/open-webui/open-webui/issues/19154)
+- 🖼️ Image editing configuration now correctly preserves independent OpenAI API endpoints and keys, preventing them from being overwritten by image generation settings. [#19003](https://github.com/open-webui/open-webui/issues/19003)
+- 🎨 Gemini image edit settings now display correctly in the admin panel, fixing an incorrect configuration key reference that prevented proper rendering of edit options. [#19200](https://github.com/open-webui/open-webui/pull/19200)
+- 🖌️ Image generation settings menu now loads correctly, resolving validation errors with AUTOMATIC1111 API authentication parameters. [#19187](https://github.com/open-webui/open-webui/issues/19187), [#19246](https://github.com/open-webui/open-webui/issues/19246)
+- 📅 Date formatting in chat search and admin user chat search now correctly respects the "DEFAULT_LOCALE" environment variable, displaying dates according to the configured locale instead of always using MM/DD/YYYY format. [#19305](https://github.com/open-webui/open-webui/pull/19305), [#19020](https://github.com/open-webui/open-webui/issues/19020)
+- 📝 RAG template query placeholder escaping logic was corrected to prevent unintended replacements of context values when query placeholders appear in retrieved content. [#19102](https://github.com/open-webui/open-webui/pull/19102), [#19101](https://github.com/open-webui/open-webui/issues/19101)
+- 📄 RAG template prompt duplication was eliminated by removing redundant user query section from the default template. [#19099](https://github.com/open-webui/open-webui/pull/19099), [#19098](https://github.com/open-webui/open-webui/issues/19098)
+- 📋 MinerU local mode configuration no longer incorrectly requires an API key, allowing proper use of local content extraction without external API credentials. [#19258](https://github.com/open-webui/open-webui/issues/19258)
+- 📊 Excel file uploads now work correctly with the addition of the missing msoffcrypto-tool dependency, resolving import errors introduced by the unstructured package upgrade. [#19153](https://github.com/open-webui/open-webui/issues/19153)
+- 📑 Docling parameters now properly handle JSON serialization, preventing exceptions and ensuring configuration changes are saved correctly. [#19072](https://github.com/open-webui/open-webui/pull/19072)
+- 🛠️ UserValves configuration now correctly isolates settings per tool, preventing configuration contamination when multiple tools with UserValves are used simultaneously. [#19185](https://github.com/open-webui/open-webui/pull/19185), [#15569](https://github.com/open-webui/open-webui/issues/15569)
+- 🔧 Tool selection prompt now correctly handles user messages without duplication, removing redundant query prefixes and improving prompt clarity. [#19122](https://github.com/open-webui/open-webui/pull/19122), [#19121](https://github.com/open-webui/open-webui/issues/19121)
+- 📝 Notes chat feature now correctly submits messages to the completions endpoint, resolving errors that prevented AI model interactions. [#19079](https://github.com/open-webui/open-webui/pull/19079)
+- 📝 Note PDF downloads now sanitize HTML content using DOMPurify before rendering, preventing potential DOM-based XSS attacks from malicious content in notes. [Commit](https://github.com/open-webui/open-webui/commit/03cc6ce8eb5c055115406e2304fbf7e3338b8dce)
+- 📁 Archived chats now have their folder associations automatically removed to prevent unintended deletion when their previous folder is deleted. [#14578](https://github.com/open-webui/open-webui/issues/14578)
+- 🔐 ElevenLabs API key is now properly obfuscated in the admin settings page, preventing plain text exposure of sensitive credentials. [#19262](https://github.com/open-webui/open-webui/pull/19262), [#19260](https://github.com/open-webui/open-webui/issues/19260)
+- 🔧 MCP OAuth server metadata discovery now follows the correct specification order, ensuring proper authentication flow compliance. [#19244](https://github.com/open-webui/open-webui/pull/19244)
+- 🔒 API key endpoint restrictions now properly enforce access controls for all endpoints including SCIM, preventing unintended access when "API_KEY_ALLOWED_ENDPOINTS" is configured. [#19168](https://github.com/open-webui/open-webui/issues/19168)
+- 🔓 OAuth role claim parsing now supports both flat and nested claim structures, enabling compatibility with OAuth providers that deliver claims as direct properties on the user object rather than nested structures. [#19286](https://github.com/open-webui/open-webui/pull/19286)
+- 🔑 OAuth MCP server verification now correctly extracts the access token value for authorization headers instead of sending the entire token dictionary. [#19149](https://github.com/open-webui/open-webui/pull/19149), [#19148](https://github.com/open-webui/open-webui/issues/19148)
+- ⚙️ OAuth dynamic client registration now correctly converts empty strings to None for optional fields, preventing validation failures in MCP package integration. [#19144](https://github.com/open-webui/open-webui/pull/19144), [#19129](https://github.com/open-webui/open-webui/issues/19129)
+- 🔐 OIDC authentication now correctly passes client credentials in access token requests, ensuring compatibility with providers that require these parameters per RFC 6749. [#19132](https://github.com/open-webui/open-webui/pull/19132), [#19131](https://github.com/open-webui/open-webui/issues/19131)
+- 🔗 OAuth client creation now respects configured token endpoint authentication methods instead of defaulting to basic authentication, preventing failures with servers that don't support basic auth. [#19165](https://github.com/open-webui/open-webui/pull/19165)
+- 📋 Text copied from chat responses in Chrome now pastes without background formatting, improving readability when pasting into word processors. [#19083](https://github.com/open-webui/open-webui/issues/19083)
+
+### Changed
+
+- 🗄️ Group membership data storage was refactored from JSON arrays to a dedicated relational database table, significantly improving query performance and scalability for instances with large numbers of users and groups, while API responses now return member counts instead of full user ID arrays. [#19239](https://github.com/open-webui/open-webui/pull/19239)
+- 📄 MinerU parameter handling was refactored to pass parameters directly to the API, improving flexibility and fixing VLM backend configuration. [#19105](https://github.com/open-webui/open-webui/pull/19105), [#18446](https://github.com/open-webui/open-webui/discussions/18446)
+- 🔐 API key creation is now controlled by granular user and group permissions, with the "ENABLE_API_KEY" environment variable renamed to "ENABLE_API_KEYS" and disabled by default, requiring explicit configuration at both the global and user permission levels, while related environment variables "ENABLE_API_KEY_ENDPOINT_RESTRICTIONS" and "API_KEY_ALLOWED_ENDPOINTS" were renamed to "ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS" and "API_KEYS_ALLOWED_ENDPOINTS" respectively. [#18336](https://github.com/open-webui/open-webui/pull/18336)
+
+## [0.6.36] - 2025-11-07
+
+### Added
+
+- 🔐 OAuth group parsing now supports configurable separators via the "OAUTH_GROUPS_SEPARATOR" environment variable, enabling proper handling of semicolon-separated group claims from providers like CILogon. [#18987](https://github.com/open-webui/open-webui/pull/18987), [#18979](https://github.com/open-webui/open-webui/issues/18979)
+
+### Fixed
+
+- 🛠️ Tool calling functionality is restored by correcting asynchronous function handling in tool parameter updates. [#18981](https://github.com/open-webui/open-webui/issues/18981)
+- 🖼️ The ComfyUI image edit workflow editor modal now opens correctly when clicking the Edit button. [#18978](https://github.com/open-webui/open-webui/issues/18978)
+- 🔥 Firecrawl import errors are resolved by implementing lazy loading and using the correct class name. [#18973](https://github.com/open-webui/open-webui/issues/18973)
+- 🔌 Socket.IO CORS warning is resolved by properly configuring CORS origins for Socket.IO connections. [Commit](https://github.com/open-webui/open-webui/commit/639d26252e528c9c37a5f553b11eb94376d8792d)
+
+## [0.6.35] - 2025-11-06
+
+### Added
+
+- 🖼️ Image generation system received a comprehensive overhaul with major new capabilities including full image editing support allowing users to modify existing images using text prompts with OpenAI, Gemini, or ComfyUI engines, adding Gemini 2.5 Flash Image (Nano Banana) support, Qwen Image Edit integration, resolution of base64-encoded image display issues, streamlined AUTOMATIC1111 configuration by consolidating parameters into a flexible JSON parameters field, and enhanced UI with a code editor modal for ComfyUI workflow management. [#17434](https://github.com/open-webui/open-webui/pull/17434), [#16976](https://github.com/open-webui/open-webui/issues/16976), [Commit](https://github.com/open-webui/open-webui/commit/8e5690aab4f632a57027e2acf880b8f89a8717c0), [Commit](https://github.com/open-webui/open-webui/commit/72f8539fd2e679fec0762945f22f4b8a6920afa0), [Commit](https://github.com/open-webui/open-webui/commit/8d34fcb586eeee1fac6da2f991518b8a68b00b72), [Commit](https://github.com/open-webui/open-webui/commit/72900cd686de1fa6be84b5a8a2fc857cff7b91b8)
+- 🔒 CORS origin validation was added to WebSocket connections as a defense-in-depth security measure against cross-site WebSocket hijacking attacks. [#18411](https://github.com/open-webui/open-webui/pull/18411), [#18410](https://github.com/open-webui/open-webui/issues/18410)
+- 🔄 Automatic page refresh now occurs when a version update is detected via WebSocket connection, ensuring users always run the latest version without cache issues. [Commit](https://github.com/open-webui/open-webui/commit/989f192c92d2fe55daa31336e7971e21798b96ae)
+- 🐍 Experimental initial preparations for Python 3.13 compatibility by updating dependencies with security enhancements and cryptographic improvements. [#18430](https://github.com/open-webui/open-webui/pull/18430), [#18424](https://github.com/open-webui/open-webui/pull/18424)
+- ⚡ Image compression now preserves the original image format instead of converting to PNG, significantly reducing file sizes and improving chat loading performance. [#18506](https://github.com/open-webui/open-webui/pull/18506)
+- 🎤 Mistral Voxtral model support was added for text-to-speech, including voxtral-small and voxtral-mini models with both transcription and chat completion API support. [#18934](https://github.com/open-webui/open-webui/pull/18934)
+- 🔊 Text-to-speech now uses a global audio queue system to prevent overlapping playback, ensuring only one TTS instance plays at a time with proper stop/start controls and automatic cleanup when switching between messages. [#16152](https://github.com/open-webui/open-webui/pull/16152), [#18744](https://github.com/open-webui/open-webui/pull/18744), [#16150](https://github.com/open-webui/open-webui/issues/16150)
+- 🔊 ELEVENLABS_API_BASE_URL environment variable now allows configuration of custom ElevenLabs API endpoints, enabling support for EU residency API requirements. [#18402](https://github.com/open-webui/open-webui/issues/18402)
+- 🔐 OAUTH_ROLES_SEPARATOR environment variable now allows custom role separators for OAuth roles that contain commas, useful for roles specified in LDAP syntax. [#18572](https://github.com/open-webui/open-webui/pull/18572)
+- 📄 External document loaders can now optionally forward user information headers when ENABLE_FORWARD_USER_INFO_HEADERS is enabled, enabling cost tracking, audit logs, and usage analytics for external services. [#18731](https://github.com/open-webui/open-webui/pull/18731)
+- 📄 MISTRAL_OCR_API_BASE_URL environment variable now allows configuration of custom Mistral OCR API endpoints for flexible deployment options. [Commit](https://github.com/open-webui/open-webui/commit/415b93c7c35c2e2db4425e6da1b88b3750f496b0)
+- ⌨️ Keyboard shortcut hints are now displayed on sidebar buttons with a refactored shortcuts modal that accurately reflects all available hotkeys across different keyboard layouts. [#18473](https://github.com/open-webui/open-webui/pull/18473)
+- 🛠️ Tooltips now display tool descriptions when hovering over tool names on the model edit page, improving usability and providing immediate context. [#18707](https://github.com/open-webui/open-webui/pull/18707)
+- 📝 "Create a new note" from the search modal now immediately creates a new private note and opens it in the editor instead of navigating to the generic notes page. [#18255](https://github.com/open-webui/open-webui/pull/18255)
+- 🖨️ Code block output now preserves whitespace formatting with monospace font to accurately reflect terminal behavior. [#18352](https://github.com/open-webui/open-webui/pull/18352)
+- ✏️ Edit button is now available in the three-dot menu of models in the workspace section for quick access to model editing, with the menu reorganized for better user experience and Edit, Clone, Copy Link, and Share options logically grouped. [#18574](https://github.com/open-webui/open-webui/pull/18574)
+- 📌 Sidebar models section is now collapsible, allowing users to expand and collapse the pinned models list for better sidebar organization. [Commit](https://github.com/open-webui/open-webui/commit/82c08a3b5d189f81c96b6548cc872198771015b0)
+- 🌙 Dark mode styles for select elements were added using Tailwind CSS classes, improving consistency across the interface. [#18636](https://github.com/open-webui/open-webui/pull/18636)
+- 🔄 Various improvements were implemented across the frontend and backend to enhance performance, stability, and security.
+- 🌐 Translations for Portuguese (Brazil), Greek, German, Traditional Chinese, Simplified Chinese, Spanish, Georgian, Danish, and Estonian were enhanced and expanded.
+
+### Fixed
+
+- 🔒 Server-Sent Event (SSE) code injection vulnerability in Direct Connections is resolved by blocking event emission from untrusted external model servers; event emitters from direct connected model servers are no longer supported, preventing arbitrary JavaScript execution in user browsers. [Commit](https://github.com/open-webui/open-webui/commit/8af6a4cf21b756a66cd58378a01c60f74c39b7ca)
+- 🛡️ DOM XSS vulnerability in "Insert Prompt as Rich Text" is resolved by sanitizing HTML content with DOMPurify before rendering. [Commit](https://github.com/open-webui/open-webui/commit/eb9c4c0e358c274aea35f21c2856c0a20051e5f1)
+- ⚙️ MCP server cancellation scope corruption is prevented by reversing disconnection order to follow LIFO and properly handling exceptions, resolving 100% CPU usage when resuming chats with expired tokens or using multiple streamable MCP servers. [#18537](https://github.com/open-webui/open-webui/pull/18537)
+- 🔧 UI freeze when querying models with knowledge bases containing inconsistent distance metrics is resolved by properly initializing the distances array in citations. [#18585](https://github.com/open-webui/open-webui/pull/18585)
+- 🤖 Duplicate model IDs from multiple OpenAI endpoints are now automatically deduplicated server-side, preventing frontend crashes for users with unified gateway proxies that aggregate multiple providers. [Commit](https://github.com/open-webui/open-webui/commit/fdf7ca11d4f3cc8fe63e81c98dc0d1e48e52ba36)
+- 🔐 Login failures with passwords longer than 72 bytes are resolved by safely truncating oversized passwords for bcrypt compatibility. [#18157](https://github.com/open-webui/open-webui/issues/18157)
+- 🔐 OAuth 2.1 MCP tool connections now automatically re-register clients when stored client IDs become stale, preventing unauthorized_client errors after editing tool endpoints and providing detailed error messages for callback failures. [#18415](https://github.com/open-webui/open-webui/pull/18415), [#18309](https://github.com/open-webui/open-webui/issues/18309)
+- 🔓 OAuth 2.1 discovery, metadata fetching, and dynamic client registration now correctly use HTTP proxy environment variables when trust_env is enabled. [Commit](https://github.com/open-webui/open-webui/commit/bafeb76c411483bd6b135f0edbcdce048120f264)
+- 🔌 MCP server connection failures now display clear error messages in the chat interface instead of silently failing. [#18892](https://github.com/open-webui/open-webui/pull/18892), [#18889](https://github.com/open-webui/open-webui/issues/18889)
+- 💬 Chat titles are now properly generated even when title auto-generation is disabled in interface settings, fixing an issue where chats would remain labeled as "New chat". [#18761](https://github.com/open-webui/open-webui/pull/18761), [#18717](https://github.com/open-webui/open-webui/issues/18717), [#6478](https://github.com/open-webui/open-webui/issues/6478)
+- 🔍 Chat query errors are prevented by properly validating and handling the "order_by" parameter to ensure requested columns exist. [#18400](https://github.com/open-webui/open-webui/pull/18400), [#18452](https://github.com/open-webui/open-webui/pull/18452)
+- 🔧 Root-level max_tokens parameter is no longer dropped when proxying to Ollama, properly converting to num_predict to limit output token length as intended. [#18618](https://github.com/open-webui/open-webui/issues/18618)
+- 🔑 Self-hosted Marker instances can now be used without requiring an API key, while keeping it optional for datalab Marker service users. [#18617](https://github.com/open-webui/open-webui/issues/18617)
+- 🔧 OpenAPI specification endpoint conflict between "/api/v1/models" and "/api/v1/models/" is resolved by changing the models router endpoint to "/list", preventing duplicate operationId errors when generating TypeScript API clients. [#18758](https://github.com/open-webui/open-webui/issues/18758)
+- 🏷️ Model tags are now de-duplicated case-insensitively in both the model selector and workspace models page, preventing duplicate entries with different capitalization from appearing in filter dropdowns. [#18716](https://github.com/open-webui/open-webui/pull/18716), [#18711](https://github.com/open-webui/open-webui/issues/18711)
+- 📄 Docling RAG parameter configuration is now correctly saved in the admin UI by fixing the typo in the "DOCLING_PARAMS" parameter name. [#18390](https://github.com/open-webui/open-webui/pull/18390)
+- 📃 Tika document processing now automatically detects content types instead of relying on potentially incorrect browser-provided mime-types, improving file handling accuracy for formats like RTF. [#18765](https://github.com/open-webui/open-webui/pull/18765), [#18683](https://github.com/open-webui/open-webui/issues/18683)
+- 🖼️ Image and video uploads to knowledge bases now display proper error messages instead of showing an infinite spinner when the content extraction engine does not support these file types. [#18514](https://github.com/open-webui/open-webui/issues/18514)
+- 📝 Notes PDF export now properly detects and applies dark mode styling consistently across both the notes list and individual note pages, with a shared utility function to eliminate code duplication. [#18526](https://github.com/open-webui/open-webui/issues/18526)
+- 💭 Details tags for reasoning content are now correctly identified and rendered even when the same tag is present in user messages. [#18840](https://github.com/open-webui/open-webui/pull/18840), [#18294](https://github.com/open-webui/open-webui/issues/18294)
+- 📊 Mermaid and Vega rendering errors now display inline with the code instead of showing repetitive toast notifications, improving user experience when models generate invalid diagram syntax. [Commit](https://github.com/open-webui/open-webui/commit/fdc0f04a8b7dd0bc9f9dc0e7e30854f7a0eea3e9)
+- 📈 Mermaid diagram rendering errors no longer cause UI unavailability or display error messages below the input box. [#18493](https://github.com/open-webui/open-webui/pull/18493), [#18340](https://github.com/open-webui/open-webui/issues/18340)
+- 🔗 Web search SSL verification is now asynchronous, preventing the website from hanging during web search operations. [#18714](https://github.com/open-webui/open-webui/pull/18714), [#18699](https://github.com/open-webui/open-webui/issues/18699)
+- 🌍 Web search results now correctly use HTTP proxy environment variables when WEB_SEARCH_TRUST_ENV is enabled. [#18667](https://github.com/open-webui/open-webui/pull/18667), [#7008](https://github.com/open-webui/open-webui/discussions/7008)
+- 🔍 Google Programmable Search Engine now properly includes referer headers, enabling API keys with HTTP referrer restrictions configured in Google Cloud Console. [#18871](https://github.com/open-webui/open-webui/pull/18871), [#18870](https://github.com/open-webui/open-webui/issues/18870)
+- ⚡ YouTube video transcript fetching now works correctly when using a proxy connection. [#18419](https://github.com/open-webui/open-webui/pull/18419)
+- 🎙️ Speech-to-text transcription no longer deletes or replaces existing text in the prompt input field, properly preserving any previously entered content. [#18540](https://github.com/open-webui/open-webui/issues/18540)
+- 🎙️ The "Instant Auto-Send After Voice Transcription" setting now functions correctly and automatically sends transcribed text when enabled. [#18466](https://github.com/open-webui/open-webui/issues/18466)
+- ⚙️ Chat settings now load properly when reopening a tab or starting a new session by initializing defaults when sessionStorage is empty. [#18438](https://github.com/open-webui/open-webui/pull/18438)
+- 🔎 Folder tag search in the sidebar now correctly handles folder names with multiple spaces by replacing all spaces with underscores. [Commit](https://github.com/open-webui/open-webui/commit/a8fe979af68e47e4e4bb3eb76e48d93d60cd2a45)
+- 🛠️ Functions page now updates immediately after deleting a function, removing the need for a manual page reload. [#18912](https://github.com/open-webui/open-webui/pull/18912), [#18908](https://github.com/open-webui/open-webui/issues/18908)
+- 🛠️ Native tool calling now properly supports sequential tool calls with shared context, allowing tools to access images and data from previous tool executions in the same conversation. [#18664](https://github.com/open-webui/open-webui/pull/18664)
+- 🎯 Globally enabled actions in the model editor now correctly apply as global instead of being treated as disabled. [#18577](https://github.com/open-webui/open-webui/pull/18577)
+- 📋 Clipboard images pasted via the "{{CLIPBOARD}}" prompt variable are now correctly converted to base64 format before being sent to the backend, resolving base64 encoding errors. [#18432](https://github.com/open-webui/open-webui/pull/18432), [#18425](https://github.com/open-webui/open-webui/issues/18425)
+- 📋 File list is now cleared when switching to models that do not support file uploads, preventing files from being sent to incompatible models. [#18496](https://github.com/open-webui/open-webui/pull/18496)
+- 📂 Move menu no longer displays when folders are empty. [#18484](https://github.com/open-webui/open-webui/pull/18484)
+- 📁 Folder and channel creation now validates that names are not empty, preventing creation of folders or channels with no name and showing an error toast if attempted. [#18564](https://github.com/open-webui/open-webui/pull/18564)
+- 🖊️ Rich text input no longer removes text between equals signs when pasting code with comparison operators. [#18551](https://github.com/open-webui/open-webui/issues/18551)
+- ⌨️ Keyboard shortcuts now display the correct keys for international and non-QWERTY keyboard layouts by detecting the user's layout using the Keyboard API. [#18533](https://github.com/open-webui/open-webui/pull/18533)
+- 🌐 "Attach Webpage" button now displays with correct disabled styling when a model does not support file uploads. [#18483](https://github.com/open-webui/open-webui/pull/18483)
+- 🎚️ Divider no longer displays in the integrations menu when no integrations are enabled. [#18487](https://github.com/open-webui/open-webui/pull/18487)
+- 📱 Chat controls button is now properly hidden on mobile for users without admin or explicit chat control permissions. [#18641](https://github.com/open-webui/open-webui/pull/18641)
+- 📍 User menu, download submenu, and move submenu are now repositioned to prevent overlap with the Chat Controls sidebar when it is open. [Commit](https://github.com/open-webui/open-webui/commit/414ab51cb6df1ab0d6c85ac6c1f2c5c9a5f8e2aa)
+- 🎯 Artifacts button no longer appears in the chat menu when there are no artifacts to display. [Commit](https://github.com/open-webui/open-webui/commit/ed6449d35f84f68dc75ee5c6b3f4748a3fda0096)
+- 🎨 Artifacts view now automatically displays when opening an existing conversation containing artifacts, improving user experience. [#18215](https://github.com/open-webui/open-webui/pull/18215)
+- 🖌️ Formatting toolbar is no longer hidden under images or code blocks in chat and now displays correctly above all message content.
+- 🎨 Layout shift near system instructions is prevented by properly rendering the chat component when system prompts are empty. [#18594](https://github.com/open-webui/open-webui/pull/18594)
+- 📐 Modal layout shift caused by scrollbar appearance is prevented by adding a stable scrollbar gutter. [#18591](https://github.com/open-webui/open-webui/pull/18591)
+- ✨ Spacing between icon and label in the user menu dropdown items is now consistent. [#18595](https://github.com/open-webui/open-webui/pull/18595)
+- 💬 Duplicate prompt suggestions no longer cause the webpage to freeze or throw JavaScript errors by implementing proper key management with composite keys. [#18841](https://github.com/open-webui/open-webui/pull/18841), [#18566](https://github.com/open-webui/open-webui/issues/18566)
+- 🔍 Chat preview loading in the search modal now works correctly for all search results by fixing an index boundary check that previously caused out-of-bounds errors. [#18911](https://github.com/open-webui/open-webui/pull/18911)
+- ♿ Screen reader support was enhanced by wrapping messages in semantic elements with descriptive aria-labels, adding "Assistant is typing" and "Response complete" announcements for improved accessibility. [#18735](https://github.com/open-webui/open-webui/pull/18735)
+- 🔒 Incorrect await call in the OAuth 2.1 flow is removed, eliminating a logged exception during authentication. [#18236](https://github.com/open-webui/open-webui/pull/18236)
+- 🛡️ Duplicate crossorigin attribute in the manifest file was removed. [#18413](https://github.com/open-webui/open-webui/pull/18413)
+
+### Changed
+
+- 🔄 Firecrawl integration was refactored to use the official Firecrawl SDK instead of direct HTTP requests and langchain_community FireCrawlLoader, improving reliability and performance with batch scraping support and enhanced error handling. [#18635](https://github.com/open-webui/open-webui/pull/18635)
+- 📄 MinerU content extraction engine now only supports PDF files following the upstream removal of LibreOffice document conversion in version 2.0.0; users needing to process office documents should convert them to PDF format first. [#18448](https://github.com/open-webui/open-webui/issues/18448)
+
+## [0.6.34] - 2025-10-16
+
+### Added
+
+- 📄 MinerU is now supported as a document parser backend, with support for both local and managed API deployments. [#18306](https://github.com/open-webui/open-webui/pull/18306)
+- 🔒 JWT token expiration default is now set to 4 weeks instead of never expiring, with security warnings displayed in backend logs and admin UI when set to unlimited. [#18261](https://github.com/open-webui/open-webui/pull/18261), [#18262](https://github.com/open-webui/open-webui/pull/18262)
+- ⚡ Page loading performance is improved by preventing unnecessary API requests when sidebar folders are not expanded. [#18179](https://github.com/open-webui/open-webui/pull/18179), [#17476](https://github.com/open-webui/open-webui/issues/17476)
+- 📁 File hash values are now included in the knowledge endpoint response, enabling efficient file synchronization through hash comparison. [#18284](https://github.com/open-webui/open-webui/pull/18284), [#18283](https://github.com/open-webui/open-webui/issues/18283)
+- 🎨 Chat dialog scrollbar visibility is improved by increasing its width, making it easier to use for navigation. [#18369](https://github.com/open-webui/open-webui/pull/18369), [#11782](https://github.com/open-webui/open-webui/issues/11782)
+- 🔄 Various improvements were implemented across the frontend and backend to enhance performance, stability, and security.
+- 🌐 Translations for Catalan, Chinese, Czech, Finnish, German, Kabyle, Korean, Portuguese (Brazil), Spanish, Thai, and Turkish were enhanced and expanded.
+
+### Fixed
+
+- 📚 Focused retrieval mode now works correctly, preventing the system from forcing full context mode and loading all documents in a knowledge base regardless of settings. [#18133](https://github.com/open-webui/open-webui/issues/18133)
+- 🔧 Filter inlet functions now correctly execute on tool call continuations, ensuring parameter persistence throughout tool interactions. [#18222](https://github.com/open-webui/open-webui/issues/18222)
+- 🛠️ External tool servers now properly support DELETE requests with body data. [#18289](https://github.com/open-webui/open-webui/pull/18289), [#18287](https://github.com/open-webui/open-webui/issues/18287)
+- 🗄️ Oracle23ai vector database client now correctly handles variable initialization, resolving UnboundLocalError when retrieving items from collections. [#18356](https://github.com/open-webui/open-webui/issues/18356)
+- 🔧 Model auto-pull functionality now works correctly even when user settings remain unmodified. [#18324](https://github.com/open-webui/open-webui/pull/18324)
+- 🎨 Duplicate HTML content in artifacts is now prevented by improving code block detection logic. [#18195](https://github.com/open-webui/open-webui/pull/18195), [#6154](https://github.com/open-webui/open-webui/issues/6154)
+- 💬 Pinned chats now appear in the Reference Chats list and can be referenced in conversations. [#18288](https://github.com/open-webui/open-webui/issues/18288)
+- 📝 Misleading knowledge base warning text in documents settings is clarified to correctly instruct users about reindexing vectors. [#18263](https://github.com/open-webui/open-webui/pull/18263)
+- 🔔 Toast notifications can now be dismissed even when a modal is open. [#18260](https://github.com/open-webui/open-webui/pull/18260)
+- 🔘 The "Chats" button in the sidebar now correctly toggles chat list visibility without navigating away from the current page. [#18232](https://github.com/open-webui/open-webui/pull/18232)
+- 🎯 The Integrations menu no longer closes prematurely when clicking outside the Valves modal. [#18310](https://github.com/open-webui/open-webui/pull/18310)
+- 🛠️ Tool ID display issues where "undefined" was incorrectly shown in the interface are now resolved. [#18178](https://github.com/open-webui/open-webui/pull/18178)
+- 🛠️ Model management issues caused by excessively long model IDs are now prevented through validation that limits model IDs to 256 characters. [#18125](https://github.com/open-webui/open-webui/issues/18125)
+
+## [0.6.33] - 2025-10-08
+
+### Added
+
+- 🎨 Workspace interface received a comprehensive redesign across Models, Knowledge, Prompts, and Tools sections, featuring reorganized controls, view filters for created vs shared items, tag selectors, improved visual hierarchy, and streamlined import/export functionality. [Commit](https://github.com/open-webui/open-webui/commit/2c59a288603d8c5f004f223ee00fef37cc763a8e), [Commit](https://github.com/open-webui/open-webui/commit/6050c86ab6ef6b8c96dd3f99c62a6867011b67a4), [Commit](https://github.com/open-webui/open-webui/commit/96ecb47bc71c072aa34ef2be10781b042bef4e8c), [Commit](https://github.com/open-webui/open-webui/commit/2250d102b28075a9611696e911536547abb8b38a), [Commit](https://github.com/open-webui/open-webui/commit/23c8f6d507bfee75ab0015a3e2972d5c26f7e9bf), [Commit](https://github.com/open-webui/open-webui/commit/a743b16728c6ae24b8befbc2d7f24eb9e20c4ad5)
+- 🛠️ Functions admin interface received a comprehensive redesign with creator attribution display, ownership filters for created vs shared items, improved organization, and refined styling. [Commit](https://github.com/open-webui/open-webui/commit/f5e1a42f51acc0b9d5b63a33c1ca2e42470239c1)
+- ⚡ Page initialization performance is significantly improved through parallel data loading and optimized folder API calls, reducing initial page load time. [#17559](https://github.com/open-webui/open-webui/pull/17559), [#17889](https://github.com/open-webui/open-webui/pull/17889)
+- ⚡ Chat overview component is now dynamically loaded on demand, reducing initial page bundle size by approximately 470KB and improving first-screen loading speed. [#17595](https://github.com/open-webui/open-webui/pull/17595)
+- 📁 Folders can now be attached to chats using the "#" command, automatically expanding to include all files within the folder for streamlined knowledge base integration. [Commit](https://github.com/open-webui/open-webui/commit/d2cb78179d66dc85188172a08622d4c97a2ea1ee)
+- 📱 Progressive Web App now supports Android share target functionality, allowing users to share web pages, YouTube videos, and text directly to Open WebUI from the system share menu. [#17633](https://github.com/open-webui/open-webui/pull/17633), [#17125](https://github.com/open-webui/open-webui/issues/17125)
+- 🗄️ Redis session storage is now available as an experimental option for OAuth authentication flows via the ENABLE_STAR_SESSIONS_MIDDLEWARE environment variable, providing shared session state across multi-replica deployments to address CSRF errors, though currently only basic Redis setups are supported. [#17223](https://github.com/open-webui/open-webui/pull/17223), [#15373](https://github.com/open-webui/open-webui/issues/15373), [Docs:Commit](https://github.com/open-webui/docs/commit/14052347f165d1b597615370373d7289ce44c7f9)
+- 📊 Vega and Vega-Lite chart visualization renderers are now supported in code blocks, enabling inline rendering of data visualizations with automatic compilation of Vega-Lite specifications. [#18033](https://github.com/open-webui/open-webui/pull/18033), [#18040](https://github.com/open-webui/open-webui/pull/18040), [#18022](https://github.com/open-webui/open-webui/issues/18022)
+- 🔗 OpenAI connections now support custom HTTP headers, enabling users to configure authentication and routing headers for specific deployment requirements. [#18021](https://github.com/open-webui/open-webui/pull/18021), [#9732](https://github.com/open-webui/open-webui/discussions/9732)
+- 🔐 OpenID Connect authentication now supports OIDC providers without email scope via the ENABLE_OAUTH_WITHOUT_EMAIL environment variable, enabling compatibility with identity providers that don't expose email addresses. [#18047](https://github.com/open-webui/open-webui/pull/18047), [#18045](https://github.com/open-webui/open-webui/issues/18045)
+- 🤖 Ollama model management modal now features individual model update cancellation, comprehensive tooltips for all buttons, and streamlined notification behavior to reduce toast spam. [#16863](https://github.com/open-webui/open-webui/pull/16863)
+- ☁️ OneDrive file picker now includes search functionality and "My Organization" pivot for business accounts, enabling easier file discovery across organizational content. [#17930](https://github.com/open-webui/open-webui/pull/17930), [#17929](https://github.com/open-webui/open-webui/issues/17929)
+- 📊 Chat overview flow diagram now supports toggling between vertical and horizontal layout orientations for improved visualization flexibility. [#17941](https://github.com/open-webui/open-webui/pull/17941)
+- 🔊 OpenAI Text-to-Speech engine now supports additional parameters, allowing users to customize TTS behavior with provider-specific options via JSON configuration. [#17985](https://github.com/open-webui/open-webui/issues/17985), [#17188](https://github.com/open-webui/open-webui/pull/17188)
+- 🛠️ Tool server list now displays server name, URL, and type (OpenAPI or MCP) for easier identification and management. [#18062](https://github.com/open-webui/open-webui/issues/18062)
+- 📁 Folders now remember the last selected model, automatically applying it when starting new chats within that folder. [#17836](https://github.com/open-webui/open-webui/issues/17836)
+- 🔢 Ollama embedding endpoint now supports the optional dimensions parameter for controlling embedding output size, compatible with Ollama v0.11.11 and later. [#17942](https://github.com/open-webui/open-webui/pull/17942)
+- ⚡ Workspace knowledge page load time is improved by removing redundant API calls, enhancing overall responsiveness. [#18057](https://github.com/open-webui/open-webui/pull/18057)
+- ⚡ File metadata query performance is enhanced by selecting only relevant columns instead of retrieving entire records, reducing database overhead. [#18013](https://github.com/open-webui/open-webui/pull/18013)
+- 📄 Note PDF exports now include titles and properly render in dark mode with appropriate background colors. [Commit](https://github.com/open-webui/open-webui/commit/216fb5c3db1a223ffe6e72d97aa9551fe0e2d028)
+- 📄 Docling document extraction now supports additional parameters for VLM pipeline configuration, enabling customized vision model settings. [#17363](https://github.com/open-webui/open-webui/pull/17363)
+- ⚙️ Server startup script now supports passing arbitrary arguments to uvicorn, enabling custom server configuration options. [#17919](https://github.com/open-webui/open-webui/pull/17919), [#17918](https://github.com/open-webui/open-webui/issues/17918)
+- 🔄 Various improvements were implemented across the frontend and backend to enhance performance, stability, and security.
+- 🌐 Translations for German, Danish, Spanish, Korean, Portuguese (Brazil), Simplified Chinese, and Traditional Chinese were enhanced and expanded.
+
+### Fixed
+
+- 💬 System prompts are no longer duplicated in chat requests, eliminating confusion and excessive token usage caused by repeated instructions being sent to models. [#17198](https://github.com/open-webui/open-webui/issues/17198), [#16855](https://github.com/open-webui/open-webui/issues/16855)
+- 🔐 MCP OAuth 2.1 authentication now complies with the standard by implementing PKCE with S256 code challenge method and explicitly passing client credentials during token authorization, resolving "code_challenge: Field required" and "client_id: Field required" errors when connecting to OAuth-secured MCP servers. [Commit](https://github.com/open-webui/open-webui/commit/911a114ad459f5deebd97543c13c2b90196efb54), [#18010](https://github.com/open-webui/open-webui/issues/18010), [#18087](https://github.com/open-webui/open-webui/pull/18087)
+- 🔐 OAuth signup flow now handles password hashing correctly by migrating from passlib to native bcrypt, preventing failures when passwords exceed 72 bytes. [#17917](https://github.com/open-webui/open-webui/issues/17917)
+- 🔐 OAuth token refresh errors are resolved by properly registering and storing OAuth clients, fixing "Constructor parameter should be str" exceptions for Google, Microsoft, and OIDC providers. [#17829](https://github.com/open-webui/open-webui/issues/17829)
+- 🔐 OAuth server metadata URL is now correctly accessed via the proper attribute, fixing automatic token refresh and logout functionality for Microsoft OAuth provider when OPENID_PROVIDER_URL is not set. [#18065](https://github.com/open-webui/open-webui/pull/18065)
+- 🔐 OAuth credential decryption failures now allow the application to start gracefully with clear error messages instead of crashing, preventing complete service outages when WEBUI_SECRET_KEY mismatches occur during database migrations or environment changes. [#18094](https://github.com/open-webui/open-webui/pull/18094), [#18092](https://github.com/open-webui/open-webui/issues/18092)
+- 🔐 OAuth 2.1 server discovery now correctly attempts all configured discovery URLs in sequence instead of only trying the first URL. [#17906](https://github.com/open-webui/open-webui/pull/17906), [#17904](https://github.com/open-webui/open-webui/issues/17904), [#18026](https://github.com/open-webui/open-webui/pull/18026)
+- 🔐 Login redirect now correctly honors the redirect query parameter after authentication, ensuring users are returned to their intended destination with query parameters intact instead of defaulting to the homepage. [#18071](https://github.com/open-webui/open-webui/issues/18071)
+- ☁️ OneDrive Business integration authentication regression is resolved, ensuring the popup now properly triggers when connecting to OneDrive accounts. [#17902](https://github.com/open-webui/open-webui/pull/17902), [#17825](https://github.com/open-webui/open-webui/discussions/17825), [#17816](https://github.com/open-webui/open-webui/issues/17816)
+- 👥 Default group settings now persist correctly after page navigation, ensuring configuration changes are properly saved and retained. [#17899](https://github.com/open-webui/open-webui/issues/17899), [#18003](https://github.com/open-webui/open-webui/issues/18003)
+- 📁 Folder data integrity is now verified on retrieval, automatically fixing orphaned folders with invalid parent references and ensuring proper cascading deletion of nested folder structures. [Commit](https://github.com/open-webui/open-webui/commit/5448618dd5ea181b9635b77040cef60926a902ff)
+- 🗄️ Redis Sentinel and Redis Cluster configurations with the experimental ENABLE_STAR_SESSIONS_MIDDLEWARE feature are now properly isolated by making the feature opt-in only, preventing ReadOnlyError failures when connecting to read replicas in multi-node Redis deployments. [#18073](https://github.com/open-webui/open-webui/issues/18073)
+- 📊 Mermaid and Vega diagram rendering now displays error toast notifications when syntax errors are detected, helping users identify and fix diagram issues instead of silently failing. [#18068](https://github.com/open-webui/open-webui/pull/18068)
+- 🤖 Reasoning models that return reasoning_content instead of content no longer cause NoneType errors during chat title generation, follow-up suggestions, and tag generation. [#18080](https://github.com/open-webui/open-webui/pull/18080)
+- 📚 Citation rendering now correctly handles multiple source references in a single bracket, parsing formats like [1,2] and [1, 2] into separate clickable citation links. [#18120](https://github.com/open-webui/open-webui/pull/18120)
+- 🔍 Web search now handles individual source failures gracefully, continuing to process remaining sources instead of failing entirely when a single URL is unreachable or returns an error. [Commit](https://github.com/open-webui/open-webui/commit/e000494e488090c5f66989a2b3f89d3eaeb7946b), [Commit](https://github.com/open-webui/open-webui/commit/53e98620bff38ab9280aee5165af0a704bdd99b9)
+- 🔍 Hybrid search with reranking now handles empty result sets gracefully instead of crashing with ValueError when all results are filtered out due to relevance thresholds. [#18096](https://github.com/open-webui/open-webui/issues/18096)
+- 🔍 Reranking models without defined padding tokens now work correctly by automatically falling back to eos_token_id as pad_token_id, fixing "Cannot handle batch sizes > 1" errors for models like Qwen3-Reranker. [#18108](https://github.com/open-webui/open-webui/pull/18108), [#16027](https://github.com/open-webui/open-webui/discussions/16027)
+- 🔍 Model selector search now correctly returns results for non-admin users by dynamically updating the search index when the model list changes, fixing a race condition that caused empty search results. [#17996](https://github.com/open-webui/open-webui/pull/17996), [#17960](https://github.com/open-webui/open-webui/pull/17960)
+- ⚡ Task model function calling performance is improved by excluding base64 image data from payloads, significantly reducing token count and memory usage when images are present in conversations. [#17897](https://github.com/open-webui/open-webui/pull/17897)
+- 🤖 Text selection "Ask" action now correctly recognizes and uses local models configured via direct connections instead of only showing external provider models. [#17896](https://github.com/open-webui/open-webui/issues/17896)
+- 🛑 Task cancellation API now returns accurate response status, correctly reporting successful cancellations instead of incorrectly indicating failures. [#17920](https://github.com/open-webui/open-webui/issues/17920)
+- 💬 Follow-up query suggestions are now generated and displayed in temporary chats, matching the behavior of saved chats. [#14987](https://github.com/open-webui/open-webui/issues/14987)
+- 🔊 Azure Text-to-Speech now properly escapes special characters like ampersands in SSML, preventing HTTP 400 errors and ensuring audio generation succeeds for all text content. [#17962](https://github.com/open-webui/open-webui/issues/17962)
+- 🛠️ OpenAPI tool server calls with optional parameters now execute successfully even when no arguments are provided, removing the incorrect requirement for a request body. [#18036](https://github.com/open-webui/open-webui/issues/18036)
+- 🛠️ MCP mode tool server connections no longer incorrectly validate the OpenAPI path field, allowing seamless switching between OpenAPI and MCP connection types. [#17989](https://github.com/open-webui/open-webui/pull/17989), [#17988](https://github.com/open-webui/open-webui/issues/17988)
+- 🛠️ Third-party tool responses containing non-UTF8 or invalid byte sequences are now handled gracefully without causing request failures. [#17882](https://github.com/open-webui/open-webui/pull/17882)
+- 🎨 Workspace filter dropdown now correctly renders model tags as strings instead of displaying individual characters, fixing broken filtering interface when models have multiple tags. [#18034](https://github.com/open-webui/open-webui/issues/18034)
+- ⌨️ Ctrl+Enter keyboard shortcut now correctly sends messages in mobile and narrow browser views on Chrome instead of inserting newlines. [#17975](https://github.com/open-webui/open-webui/issues/17975)
+- ⌨️ Tab characters are now preserved when pasting code or formatted text into the chat input box in plain text mode. [#17958](https://github.com/open-webui/open-webui/issues/17958)
+- 📋 Text selection copying from the chat input box now correctly copies only the selected text instead of the entire textbox content. [#17911](https://github.com/open-webui/open-webui/issues/17911)
+- 🔍 Web search query logging now uses debug level instead of info level, preventing user search queries from appearing in production logs. [#17888](https://github.com/open-webui/open-webui/pull/17888)
+- 📝 Debug print statements in middleware were removed to prevent excessive log pollution and respect configured logging levels. [#17943](https://github.com/open-webui/open-webui/issues/17943)
+
+### Changed
+
+- 🗄️ Milvus vector database dependency is updated from pymilvus 2.5.0 to 2.6.2, ensuring compatibility with newer Milvus versions but requiring users on older Milvus instances to either upgrade their database or manually downgrade the pymilvus package. [#18066](https://github.com/open-webui/open-webui/pull/18066)
+
 ## [0.6.32] - 2025-09-29
 
 ### Added

README.md

@@ -17,7 +17,7 @@ Passionate about open-source AI? [Join our team →](https://careers.openwebui.c
 ![Open WebUI Demo](./demo.gif)
 
 > [!TIP]  
-> **Looking for an [Enterprise Plan](https://docs.openwebui.com/enterprise)?** – **[Speak with Our Sales Team Today!](mailto:sales@openwebui.com)**
+> **Looking for an [Enterprise Plan](https://docs.openwebui.com/enterprise)?** – **[Speak with Our Sales Team Today!](https://docs.openwebui.com/enterprise)**
 >
 > Get **enhanced capabilities**, including **custom theming and branding**, **Service Level Agreement (SLA) support**, **Long-Term Support (LTS) versions**, and **more!**
 
@@ -31,32 +31,44 @@ For more information, be sure to check out our [Open WebUI Documentation](https:
 
 - 🛡️ **Granular Permissions and User Groups**: By allowing administrators to create detailed user roles and permissions, we ensure a secure user environment. This granularity not only enhances security but also allows for customized user experiences, fostering a sense of ownership and responsibility amongst users.
 
-- 🔄 **SCIM 2.0 Support**: Enterprise-grade user and group provisioning through SCIM 2.0 protocol, enabling seamless integration with identity providers like Okta, Azure AD, and Google Workspace for automated user lifecycle management.
-
 - 📱 **Responsive Design**: Enjoy a seamless experience across Desktop PC, Laptop, and Mobile devices.
 
 - 📱 **Progressive Web App (PWA) for Mobile**: Enjoy a native app-like experience on your mobile device with our PWA, providing offline access on localhost and a seamless user interface.
 
 - ✒️🔢 **Full Markdown and LaTeX Support**: Elevate your LLM experience with comprehensive Markdown and LaTeX capabilities for enriched interaction.
 
-- 🎤📹 **Hands-Free Voice/Video Call**: Experience seamless communication with integrated hands-free voice and video call features, allowing for a more dynamic and interactive chat environment.
+- 🎤📹 **Hands-Free Voice/Video Call**: Experience seamless communication with integrated hands-free voice and video call features using multiple Speech-to-Text providers (Local Whisper, OpenAI, Deepgram, Azure) and Text-to-Speech engines (Azure, ElevenLabs, OpenAI, Transformers, WebAPI), allowing for dynamic and interactive chat environments.
 
 - 🛠️ **Model Builder**: Easily create Ollama models via the Web UI. Create and add custom characters/agents, customize chat elements, and import models effortlessly through [Open WebUI Community](https://openwebui.com/) integration.
 
 - 🐍 **Native Python Function Calling Tool**: Enhance your LLMs with built-in code editor support in the tools workspace. Bring Your Own Function (BYOF) by simply adding your pure Python functions, enabling seamless integration with LLMs.
 
-- 📚 **Local RAG Integration**: Dive into the future of chat interactions with groundbreaking Retrieval Augmented Generation (RAG) support. This feature seamlessly integrates document interactions into your chat experience. You can load documents directly into the chat or add files to your document library, effortlessly accessing them using the `#` command before a query.
+- 💾 **Persistent Artifact Storage**: Built-in key-value storage API for artifacts, enabling features like journals, trackers, leaderboards, and collaborative tools with both personal and shared data scopes across sessions.
 
-- 🔍 **Web Search for RAG**: Perform web searches using providers like `SearXNG`, `Google PSE`, `Brave Search`, `serpstack`, `serper`, `Serply`, `DuckDuckGo`, `TavilySearch`, `SearchApi` and `Bing` and inject the results directly into your chat experience.
+- 📚 **Local RAG Integration**: Dive into the future of chat interactions with groundbreaking Retrieval Augmented Generation (RAG) support using your choice of 9 vector databases and multiple content extraction engines (Tika, Docling, Document Intelligence, Mistral OCR, External loaders). Load documents directly into chat or add files to your document library, effortlessly accessing them using the `#` command before a query.
+
+- 🔍 **Web Search for RAG**: Perform web searches using 15+ providers including `SearXNG`, `Google PSE`, `Brave Search`, `Kagi`, `Mojeek`, `Tavily`, `Perplexity`, `serpstack`, `serper`, `Serply`, `DuckDuckGo`, `SearchApi`, `SerpApi`, `Bing`, `Jina`, `Exa`, `Sougou`, `Azure AI Search`, and `Ollama Cloud`, injecting results directly into your chat experience.
 
 - 🌐 **Web Browsing Capability**: Seamlessly integrate websites into your chat experience using the `#` command followed by a URL. This feature allows you to incorporate web content directly into your conversations, enhancing the richness and depth of your interactions.
 
-- 🎨 **Image Generation Integration**: Seamlessly incorporate image generation capabilities using options such as AUTOMATIC1111 API or ComfyUI (local), and OpenAI's DALL-E (external), enriching your chat experience with dynamic visual content.
+- 🎨 **Image Generation & Editing Integration**: Create and edit images using multiple engines including OpenAI's DALL-E, Gemini, ComfyUI (local), and AUTOMATIC1111 (local), with support for both generation and prompt-based editing workflows.
 
 - ⚙️ **Many Models Conversations**: Effortlessly engage with various models simultaneously, harnessing their unique strengths for optimal responses. Enhance your experience by leveraging a diverse set of models in parallel.
 
 - 🔐 **Role-Based Access Control (RBAC)**: Ensure secure access with restricted permissions; only authorized individuals can access your Ollama, and exclusive model creation/pulling rights are reserved for administrators.
 
+- 🗄️ **Flexible Database & Storage Options**: Choose from SQLite (with optional encryption), PostgreSQL, or configure cloud storage backends (S3, Google Cloud Storage, Azure Blob Storage) for scalable deployments.
+
+- 🔍 **Advanced Vector Database Support**: Select from 9 vector database options including ChromaDB, PGVector, Qdrant, Milvus, Elasticsearch, OpenSearch, Pinecone, S3Vector, and Oracle 23ai for optimal RAG performance.
+
+- 🔐 **Enterprise Authentication**: Full support for LDAP/Active Directory integration, SCIM 2.0 automated provisioning, and SSO via trusted headers alongside OAuth providers. Enterprise-grade user and group provisioning through SCIM 2.0 protocol, enabling seamless integration with identity providers like Okta, Azure AD, and Google Workspace for automated user lifecycle management.
+
+- ☁️ **Cloud-Native Integration**: Native support for Google Drive and OneDrive/SharePoint file picking, enabling seamless document import from enterprise cloud storage.
+
+- 📊 **Production Observability**: Built-in OpenTelemetry support for traces, metrics, and logs, enabling comprehensive monitoring with your existing observability stack.
+
+- ⚖️ **Horizontal Scalability**: Redis-backed session management and WebSocket support for multi-worker and multi-node deployments behind load balancers.
+
 - 🌐🌍 **Multilingual Support**: Experience Open WebUI in your preferred language with our internationalization (i18n) support. Join us in expanding our supported languages! We're actively seeking contributors!
 
 - 🧩 **Pipelines, Open WebUI Plugin Support**: Seamlessly integrate custom logic and Python libraries into Open WebUI using [Pipelines Plugin Framework](https://github.com/open-webui/pipelines). Launch your Pipelines instance, set the OpenAI URL to the Pipelines URL, and explore endless possibilities. [Examples](https://github.com/open-webui/pipelines/tree/main/examples) include **Function Calling**, User **Rate Limiting** to control access, **Usage Monitoring** with tools like Langfuse, **Live Translation with LibreTranslate** for multilingual support, **Toxic Message Filtering** and much more.
@@ -65,43 +77,6 @@ For more information, be sure to check out our [Open WebUI Documentation](https:
 
 Want to learn more about Open WebUI's features? Check out our [Open WebUI documentation](https://docs.openwebui.com/features) for a comprehensive overview!
 
-## Sponsors 🙌
-
-#### Emerald
-
-<table>
-  <!-- <tr>
-    <td>
-      <a href="https://n8n.io/" target="_blank">
-        <img src="https://docs.openwebui.com/sponsors/logos/n8n.png" alt="n8n" style="width: 8rem; height: 8rem; border-radius: .75rem;" />
-      </a>
-    </td>
-    <td>
-      <a href="https://n8n.io/">n8n</a> • Does your interface have a backend yet?<br>Try <a href="https://n8n.io/">n8n</a>
-    </td>
-  </tr> -->
-  <tr>
-    <td>
-      <a href="https://tailscale.com/blog/self-host-a-local-ai-stack/?utm_source=OpenWebUI&utm_medium=paid-ad-placement&utm_campaign=OpenWebUI-Docs" target="_blank">
-        <img src="https://docs.openwebui.com/sponsors/logos/tailscale.png" alt="Tailscale" style="width: 8rem; height: 8rem; border-radius: .75rem;" />
-      </a>
-    </td>
-    <td>
-      <a href="https://tailscale.com/blog/self-host-a-local-ai-stack/?utm_source=OpenWebUI&utm_medium=paid-ad-placement&utm_campaign=OpenWebUI-Docs">Tailscale</a> • Connect self-hosted AI to any device with Tailscale
-    </td>
-  </tr>
-   <tr>
-    <td>
-      <a href="https://warp.dev/open-webui" target="_blank">
-        <img src="https://docs.openwebui.com/sponsors/logos/warp.png" alt="Warp" style="width: 8rem; height: 8rem; border-radius: .75rem;" />
-      </a>
-    </td>
-    <td>
-      <a href="https://warp.dev/open-webui">Warp</a> • The intelligent terminal for developers
-    </td>
-  </tr>
-</table>
-
 ---
 
 We are incredibly grateful for the generous support of our sponsors. Their contributions help us to maintain and improve our project, ensuring we can continue to deliver quality work to our community. Thank you!

backend/open_webui/config.py

@@ -287,29 +287,40 @@ class AppConfig:
 # WEBUI_AUTH (Required for security)
 ####################################
 
-ENABLE_API_KEY = PersistentConfig(
-    "ENABLE_API_KEY",
-    "auth.api_key.enable",
-    os.environ.get("ENABLE_API_KEY", "True").lower() == "true",
+ENABLE_API_KEYS = PersistentConfig(
+    "ENABLE_API_KEYS",
+    "auth.enable_api_keys",
+    os.environ.get("ENABLE_API_KEYS", "False").lower() == "true",
 )
 
-ENABLE_API_KEY_ENDPOINT_RESTRICTIONS = PersistentConfig(
-    "ENABLE_API_KEY_ENDPOINT_RESTRICTIONS",
+ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS = PersistentConfig(
+    "ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS",
     "auth.api_key.endpoint_restrictions",
-    os.environ.get("ENABLE_API_KEY_ENDPOINT_RESTRICTIONS", "False").lower() == "true",
+    os.environ.get(
+        "ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS",
+        os.environ.get("ENABLE_API_KEY_ENDPOINT_RESTRICTIONS", "False"),
+    ).lower()
+    == "true",
 )
 
-API_KEY_ALLOWED_ENDPOINTS = PersistentConfig(
-    "API_KEY_ALLOWED_ENDPOINTS",
+API_KEYS_ALLOWED_ENDPOINTS = PersistentConfig(
+    "API_KEYS_ALLOWED_ENDPOINTS",
     "auth.api_key.allowed_endpoints",
-    os.environ.get("API_KEY_ALLOWED_ENDPOINTS", ""),
+    os.environ.get(
+        "API_KEYS_ALLOWED_ENDPOINTS", os.environ.get("API_KEY_ALLOWED_ENDPOINTS", "")
+    ),
 )
 
-
 JWT_EXPIRES_IN = PersistentConfig(
-    "JWT_EXPIRES_IN", "auth.jwt_expiry", os.environ.get("JWT_EXPIRES_IN", "-1")
+    "JWT_EXPIRES_IN", "auth.jwt_expiry", os.environ.get("JWT_EXPIRES_IN", "4w")
 )
 
+if JWT_EXPIRES_IN.value == "-1":
+    log.warning(
+        "⚠️  SECURITY WARNING: JWT_EXPIRES_IN is set to '-1'\n"
+        "    See: https://docs.openwebui.com/getting-started/env-configuration\n"
+    )
+
 ####################################
 # OAuth config
 ####################################
@@ -564,25 +575,38 @@ OAUTH_BLOCKED_GROUPS = PersistentConfig(
     os.environ.get("OAUTH_BLOCKED_GROUPS", "[]"),
 )
 
+OAUTH_GROUPS_SEPARATOR = os.environ.get("OAUTH_GROUPS_SEPARATOR", ";")
+
 OAUTH_ROLES_CLAIM = PersistentConfig(
     "OAUTH_ROLES_CLAIM",
     "oauth.roles_claim",
     os.environ.get("OAUTH_ROLES_CLAIM", "roles"),
 )
 
+OAUTH_ROLES_SEPARATOR = os.environ.get("OAUTH_ROLES_SEPARATOR", ",")
+
 OAUTH_ALLOWED_ROLES = PersistentConfig(
     "OAUTH_ALLOWED_ROLES",
     "oauth.allowed_roles",
     [
         role.strip()
-        for role in os.environ.get("OAUTH_ALLOWED_ROLES", "user,admin").split(",")
+        for role in os.environ.get(
+            "OAUTH_ALLOWED_ROLES", f"user{OAUTH_ROLES_SEPARATOR}admin"
+        ).split(OAUTH_ROLES_SEPARATOR)
+        if role
     ],
 )
 
 OAUTH_ADMIN_ROLES = PersistentConfig(
     "OAUTH_ADMIN_ROLES",
     "oauth.admin_roles",
-    [role.strip() for role in os.environ.get("OAUTH_ADMIN_ROLES", "admin").split(",")],
+    [
+        role.strip()
+        for role in os.environ.get("OAUTH_ADMIN_ROLES", "admin").split(
+            OAUTH_ROLES_SEPARATOR
+        )
+        if role
+    ],
 )
 
 OAUTH_ALLOWED_DOMAINS = PersistentConfig(
@@ -600,13 +624,18 @@ OAUTH_UPDATE_PICTURE_ON_LOGIN = PersistentConfig(
     os.environ.get("OAUTH_UPDATE_PICTURE_ON_LOGIN", "False").lower() == "true",
 )
 
+OAUTH_ACCESS_TOKEN_REQUEST_INCLUDE_CLIENT_ID = (
+    os.environ.get("OAUTH_ACCESS_TOKEN_REQUEST_INCLUDE_CLIENT_ID", "False").lower()
+    == "true"
+)
+
 
 def load_oauth_providers():
     OAUTH_PROVIDERS.clear()
     if GOOGLE_CLIENT_ID.value and GOOGLE_CLIENT_SECRET.value:
 
         def google_oauth_register(oauth: OAuth):
-            return oauth.register(
+            client = oauth.register(
                 name="google",
                 client_id=GOOGLE_CLIENT_ID.value,
                 client_secret=GOOGLE_CLIENT_SECRET.value,
@@ -621,6 +650,7 @@ def load_oauth_providers():
                 },
                 redirect_uri=GOOGLE_REDIRECT_URI.value,
             )
+            return client
 
         OAUTH_PROVIDERS["google"] = {
             "redirect_uri": GOOGLE_REDIRECT_URI.value,
@@ -634,7 +664,7 @@ def load_oauth_providers():
     ):
 
         def microsoft_oauth_register(oauth: OAuth):
-            return oauth.register(
+            client = oauth.register(
                 name="microsoft",
                 client_id=MICROSOFT_CLIENT_ID.value,
                 client_secret=MICROSOFT_CLIENT_SECRET.value,
@@ -649,6 +679,7 @@ def load_oauth_providers():
                 },
                 redirect_uri=MICROSOFT_REDIRECT_URI.value,
             )
+            return client
 
         OAUTH_PROVIDERS["microsoft"] = {
             "redirect_uri": MICROSOFT_REDIRECT_URI.value,
@@ -659,7 +690,7 @@ def load_oauth_providers():
     if GITHUB_CLIENT_ID.value and GITHUB_CLIENT_SECRET.value:
 
         def github_oauth_register(oauth: OAuth):
-            return oauth.register(
+            client = oauth.register(
                 name="github",
                 client_id=GITHUB_CLIENT_ID.value,
                 client_secret=GITHUB_CLIENT_SECRET.value,
@@ -677,6 +708,7 @@ def load_oauth_providers():
                 },
                 redirect_uri=GITHUB_CLIENT_REDIRECT_URI.value,
             )
+            return client
 
         OAUTH_PROVIDERS["github"] = {
             "redirect_uri": GITHUB_CLIENT_REDIRECT_URI.value,
@@ -716,7 +748,7 @@ def load_oauth_providers():
                     % ("S256", OAUTH_CODE_CHALLENGE_METHOD.value)
                 )
 
-            return oauth.register(
+            client = oauth.register(
                 name="oidc",
                 client_id=OAUTH_CLIENT_ID.value,
                 client_secret=OAUTH_CLIENT_SECRET.value,
@@ -724,6 +756,7 @@ def load_oauth_providers():
                 client_kwargs=client_kwargs,
                 redirect_uri=OPENID_REDIRECT_URI.value,
             )
+            return client
 
         OAUTH_PROVIDERS["oidc"] = {
             "name": OAUTH_PROVIDER_NAME.value,
@@ -734,7 +767,7 @@ def load_oauth_providers():
     if FEISHU_CLIENT_ID.value and FEISHU_CLIENT_SECRET.value:
 
         def feishu_oauth_register(oauth: OAuth):
-            return oauth.register(
+            client = oauth.register(
                 name="feishu",
                 client_id=FEISHU_CLIENT_ID.value,
                 client_secret=FEISHU_CLIENT_SECRET.value,
@@ -752,6 +785,7 @@ def load_oauth_providers():
                 },
                 redirect_uri=FEISHU_REDIRECT_URI.value,
             )
+            return client
 
         OAUTH_PROVIDERS["feishu"] = {
             "register": feishu_oauth_register,
@@ -1104,6 +1138,7 @@ ENABLE_LOGIN_FORM = PersistentConfig(
     os.environ.get("ENABLE_LOGIN_FORM", "True").lower() == "true",
 )
 
+ENABLE_PASSWORD_AUTH = os.environ.get("ENABLE_PASSWORD_AUTH", "True").lower() == "true"
 
 DEFAULT_LOCALE = PersistentConfig(
     "DEFAULT_LOCALE",
@@ -1115,6 +1150,12 @@ DEFAULT_MODELS = PersistentConfig(
     "DEFAULT_MODELS", "ui.default_models", os.environ.get("DEFAULT_MODELS", None)
 )
 
+DEFAULT_PINNED_MODELS = PersistentConfig(
+    "DEFAULT_PINNED_MODELS",
+    "ui.default_pinned_models",
+    os.environ.get("DEFAULT_PINNED_MODELS", None),
+)
+
 try:
     default_prompt_suggestions = json.loads(
         os.environ.get("DEFAULT_PROMPT_SUGGESTIONS", "[]")
@@ -1171,6 +1212,12 @@ DEFAULT_USER_ROLE = PersistentConfig(
     os.getenv("DEFAULT_USER_ROLE", "pending"),
 )
 
+DEFAULT_GROUP_ID = PersistentConfig(
+    "DEFAULT_GROUP_ID",
+    "ui.default_group_id",
+    os.environ.get("DEFAULT_GROUP_ID", ""),
+)
+
 PENDING_USER_OVERLAY_TITLE = PersistentConfig(
     "PENDING_USER_OVERLAY_TITLE",
     "ui.pending_user_overlay_title",
@@ -1210,6 +1257,40 @@ USER_PERMISSIONS_WORKSPACE_TOOLS_ACCESS = (
     os.environ.get("USER_PERMISSIONS_WORKSPACE_TOOLS_ACCESS", "False").lower() == "true"
 )
 
+USER_PERMISSIONS_WORKSPACE_MODELS_IMPORT = (
+    os.environ.get("USER_PERMISSIONS_WORKSPACE_MODELS_IMPORT", "False").lower()
+    == "true"
+)
+
+USER_PERMISSIONS_WORKSPACE_MODELS_EXPORT = (
+    os.environ.get("USER_PERMISSIONS_WORKSPACE_MODELS_EXPORT", "False").lower()
+    == "true"
+)
+
+USER_PERMISSIONS_WORKSPACE_PROMPTS_IMPORT = (
+    os.environ.get("USER_PERMISSIONS_WORKSPACE_PROMPTS_IMPORT", "False").lower()
+    == "true"
+)
+
+USER_PERMISSIONS_WORKSPACE_PROMPTS_EXPORT = (
+    os.environ.get("USER_PERMISSIONS_WORKSPACE_PROMPTS_EXPORT", "False").lower()
+    == "true"
+)
+
+USER_PERMISSIONS_WORKSPACE_TOOLS_IMPORT = (
+    os.environ.get("USER_PERMISSIONS_WORKSPACE_TOOLS_IMPORT", "False").lower() == "true"
+)
+
+USER_PERMISSIONS_WORKSPACE_TOOLS_EXPORT = (
+    os.environ.get("USER_PERMISSIONS_WORKSPACE_TOOLS_EXPORT", "False").lower() == "true"
+)
+
+
+USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_SHARING = (
+    os.environ.get("USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_SHARING", "False").lower()
+    == "true"
+)
+
 USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_PUBLIC_SHARING = (
     os.environ.get(
         "USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_PUBLIC_SHARING", "False"
@@ -1217,8 +1298,10 @@ USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_PUBLIC_SHARING = (
     == "true"
 )
 
-USER_PERMISSIONS_NOTES_ALLOW_PUBLIC_SHARING = (
-    os.environ.get("USER_PERMISSIONS_NOTES_ALLOW_PUBLIC_SHARING", "False").lower()
+USER_PERMISSIONS_WORKSPACE_KNOWLEDGE_ALLOW_SHARING = (
+    os.environ.get(
+        "USER_PERMISSIONS_WORKSPACE_KNOWLEDGE_ALLOW_PUBLIC_SHARING", "False"
+    ).lower()
     == "true"
 )
 
@@ -1229,6 +1312,11 @@ USER_PERMISSIONS_WORKSPACE_KNOWLEDGE_ALLOW_PUBLIC_SHARING = (
     == "true"
 )
 
+USER_PERMISSIONS_WORKSPACE_PROMPTS_ALLOW_SHARING = (
+    os.environ.get("USER_PERMISSIONS_WORKSPACE_PROMPTS_ALLOW_SHARING", "False").lower()
+    == "true"
+)
+
 USER_PERMISSIONS_WORKSPACE_PROMPTS_ALLOW_PUBLIC_SHARING = (
     os.environ.get(
         "USER_PERMISSIONS_WORKSPACE_PROMPTS_ALLOW_PUBLIC_SHARING", "False"
@@ -1236,6 +1324,12 @@ USER_PERMISSIONS_WORKSPACE_PROMPTS_ALLOW_PUBLIC_SHARING = (
     == "true"
 )
 
+
+USER_PERMISSIONS_WORKSPACE_TOOLS_ALLOW_SHARING = (
+    os.environ.get("USER_PERMISSIONS_WORKSPACE_TOOLS_ALLOW_SHARING", "False").lower()
+    == "true"
+)
+
 USER_PERMISSIONS_WORKSPACE_TOOLS_ALLOW_PUBLIC_SHARING = (
     os.environ.get(
         "USER_PERMISSIONS_WORKSPACE_TOOLS_ALLOW_PUBLIC_SHARING", "False"
@@ -1244,6 +1338,17 @@ USER_PERMISSIONS_WORKSPACE_TOOLS_ALLOW_PUBLIC_SHARING = (
 )
 
 
+USER_PERMISSIONS_NOTES_ALLOW_SHARING = (
+    os.environ.get("USER_PERMISSIONS_NOTES_ALLOW_PUBLIC_SHARING", "False").lower()
+    == "true"
+)
+
+USER_PERMISSIONS_NOTES_ALLOW_PUBLIC_SHARING = (
+    os.environ.get("USER_PERMISSIONS_NOTES_ALLOW_PUBLIC_SHARING", "False").lower()
+    == "true"
+)
+
+
 USER_PERMISSIONS_CHAT_CONTROLS = (
     os.environ.get("USER_PERMISSIONS_CHAT_CONTROLS", "True").lower() == "true"
 )
@@ -1342,10 +1447,18 @@ USER_PERMISSIONS_FEATURES_CODE_INTERPRETER = (
     == "true"
 )
 
+USER_PERMISSIONS_FEATURES_FOLDERS = (
+    os.environ.get("USER_PERMISSIONS_FEATURES_FOLDERS", "True").lower() == "true"
+)
+
 USER_PERMISSIONS_FEATURES_NOTES = (
     os.environ.get("USER_PERMISSIONS_FEATURES_NOTES", "True").lower() == "true"
 )
 
+USER_PERMISSIONS_FEATURES_API_KEYS = (
+    os.environ.get("USER_PERMISSIONS_FEATURES_API_KEYS", "False").lower() == "true"
+)
+
 
 DEFAULT_USER_PERMISSIONS = {
     "workspace": {
@@ -1353,12 +1466,23 @@ DEFAULT_USER_PERMISSIONS = {
         "knowledge": USER_PERMISSIONS_WORKSPACE_KNOWLEDGE_ACCESS,
         "prompts": USER_PERMISSIONS_WORKSPACE_PROMPTS_ACCESS,
         "tools": USER_PERMISSIONS_WORKSPACE_TOOLS_ACCESS,
+        "models_import": USER_PERMISSIONS_WORKSPACE_MODELS_IMPORT,
+        "models_export": USER_PERMISSIONS_WORKSPACE_MODELS_EXPORT,
+        "prompts_import": USER_PERMISSIONS_WORKSPACE_PROMPTS_IMPORT,
+        "prompts_export": USER_PERMISSIONS_WORKSPACE_PROMPTS_EXPORT,
+        "tools_import": USER_PERMISSIONS_WORKSPACE_TOOLS_IMPORT,
+        "tools_export": USER_PERMISSIONS_WORKSPACE_TOOLS_EXPORT,
     },
     "sharing": {
+        "models": USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_SHARING,
         "public_models": USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_PUBLIC_SHARING,
+        "knowledge": USER_PERMISSIONS_WORKSPACE_KNOWLEDGE_ALLOW_SHARING,
         "public_knowledge": USER_PERMISSIONS_WORKSPACE_KNOWLEDGE_ALLOW_PUBLIC_SHARING,
+        "prompts": USER_PERMISSIONS_WORKSPACE_PROMPTS_ALLOW_SHARING,
         "public_prompts": USER_PERMISSIONS_WORKSPACE_PROMPTS_ALLOW_PUBLIC_SHARING,
+        "tools": USER_PERMISSIONS_WORKSPACE_TOOLS_ALLOW_SHARING,
         "public_tools": USER_PERMISSIONS_WORKSPACE_TOOLS_ALLOW_PUBLIC_SHARING,
+        "notes": USER_PERMISSIONS_NOTES_ALLOW_SHARING,
         "public_notes": USER_PERMISSIONS_NOTES_ALLOW_PUBLIC_SHARING,
     },
     "chat": {
@@ -1383,11 +1507,15 @@ DEFAULT_USER_PERMISSIONS = {
         "temporary_enforced": USER_PERMISSIONS_CHAT_TEMPORARY_ENFORCED,
     },
     "features": {
+        # General features
+        "api_keys": USER_PERMISSIONS_FEATURES_API_KEYS,
+        "folders": USER_PERMISSIONS_FEATURES_FOLDERS,
+        "notes": USER_PERMISSIONS_FEATURES_NOTES,
         "direct_tool_servers": USER_PERMISSIONS_FEATURES_DIRECT_TOOL_SERVERS,
+        # Chat features
         "web_search": USER_PERMISSIONS_FEATURES_WEB_SEARCH,
         "image_generation": USER_PERMISSIONS_FEATURES_IMAGE_GENERATION,
         "code_interpreter": USER_PERMISSIONS_FEATURES_CODE_INTERPRETER,
-        "notes": USER_PERMISSIONS_FEATURES_NOTES,
     },
 }
 
@@ -1397,6 +1525,12 @@ USER_PERMISSIONS = PersistentConfig(
     DEFAULT_USER_PERMISSIONS,
 )
 
+ENABLE_FOLDERS = PersistentConfig(
+    "ENABLE_FOLDERS",
+    "folders.enable",
+    os.environ.get("ENABLE_FOLDERS", "True").lower() == "true",
+)
+
 ENABLE_CHANNELS = PersistentConfig(
     "ENABLE_CHANNELS",
     "channels.enable",
@@ -1796,6 +1930,38 @@ Output:
 #### Output:
 """
 
+
+VOICE_MODE_PROMPT_TEMPLATE = PersistentConfig(
+    "VOICE_MODE_PROMPT_TEMPLATE",
+    "task.voice.prompt_template",
+    os.environ.get("VOICE_MODE_PROMPT_TEMPLATE", ""),
+)
+
+DEFAULT_VOICE_MODE_PROMPT_TEMPLATE = """You are a friendly, concise voice assistant.
+
+Everything you say will be spoken aloud.
+Keep responses short, clear, and natural.
+
+STYLE:
+- Use simple words and short sentences.
+- Sound warm and conversational.
+- Avoid long explanations, lists, or complex phrasing.
+
+BEHAVIOR:
+- Give the quickest helpful answer first.
+- Offer extra detail only if needed.
+- Ask for clarification only when necessary.
+
+VOICE OPTIMIZATION:
+- Break information into small, easy-to-hear chunks.
+- Avoid dense wording or anything that sounds like reading text.
+
+ERROR HANDLING:
+- If unsure, say so briefly and offer options.
+- If something is unsafe or impossible, decline kindly and suggest a safe alternative.
+
+Stay consistent, helpful, and easy to listen to."""
+
 TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE = PersistentConfig(
     "TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE",
     "task.tools.prompt_template",
@@ -2036,6 +2202,11 @@ ENABLE_QDRANT_MULTITENANCY_MODE = (
 )
 QDRANT_COLLECTION_PREFIX = os.environ.get("QDRANT_COLLECTION_PREFIX", "open-webui")
 
+WEAVIATE_HTTP_HOST = os.environ.get("WEAVIATE_HTTP_HOST", "")
+WEAVIATE_HTTP_PORT = int(os.environ.get("WEAVIATE_HTTP_PORT", "8080"))
+WEAVIATE_GRPC_PORT = int(os.environ.get("WEAVIATE_GRPC_PORT", "50051"))
+WEAVIATE_API_KEY = os.environ.get("WEAVIATE_API_KEY")
+
 # OpenSearch
 OPENSEARCH_URI = os.environ.get("OPENSEARCH_URI", "https://localhost:9200")
 OPENSEARCH_SSL = os.environ.get("OPENSEARCH_SSL", "true").lower() == "true"
@@ -2066,6 +2237,16 @@ PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH = int(
     os.environ.get("PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH", "1536")
 )
 
+PGVECTOR_USE_HALFVEC = os.getenv("PGVECTOR_USE_HALFVEC", "false").lower() == "true"
+
+if PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH > 2000 and not PGVECTOR_USE_HALFVEC:
+    raise ValueError(
+        "PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH is set to "
+        f"{PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH}, which exceeds the 2000 dimension limit of the "
+        "'vector' type. Set PGVECTOR_USE_HALFVEC=true to enable the 'halfvec' "
+        "type required for high-dimensional embeddings."
+    )
+
 PGVECTOR_CREATE_EXTENSION = (
     os.getenv("PGVECTOR_CREATE_EXTENSION", "true").lower() == "true"
 )
@@ -2115,6 +2296,40 @@ else:
     except Exception:
         PGVECTOR_POOL_RECYCLE = 3600
 
+PGVECTOR_INDEX_METHOD = os.getenv("PGVECTOR_INDEX_METHOD", "").strip().lower()
+if PGVECTOR_INDEX_METHOD not in ("ivfflat", "hnsw", ""):
+    PGVECTOR_INDEX_METHOD = ""
+
+PGVECTOR_HNSW_M = os.environ.get("PGVECTOR_HNSW_M", 16)
+
+if PGVECTOR_HNSW_M == "":
+    PGVECTOR_HNSW_M = 16
+else:
+    try:
+        PGVECTOR_HNSW_M = int(PGVECTOR_HNSW_M)
+    except Exception:
+        PGVECTOR_HNSW_M = 16
+
+PGVECTOR_HNSW_EF_CONSTRUCTION = os.environ.get("PGVECTOR_HNSW_EF_CONSTRUCTION", 64)
+
+if PGVECTOR_HNSW_EF_CONSTRUCTION == "":
+    PGVECTOR_HNSW_EF_CONSTRUCTION = 64
+else:
+    try:
+        PGVECTOR_HNSW_EF_CONSTRUCTION = int(PGVECTOR_HNSW_EF_CONSTRUCTION)
+    except Exception:
+        PGVECTOR_HNSW_EF_CONSTRUCTION = 64
+
+PGVECTOR_IVFFLAT_LISTS = os.environ.get("PGVECTOR_IVFFLAT_LISTS", 100)
+
+if PGVECTOR_IVFFLAT_LISTS == "":
+    PGVECTOR_IVFFLAT_LISTS = 100
+else:
+    try:
+        PGVECTOR_IVFFLAT_LISTS = int(PGVECTOR_IVFFLAT_LISTS)
+    except Exception:
+        PGVECTOR_IVFFLAT_LISTS = 100
+
 # Pinecone
 PINECONE_API_KEY = os.environ.get("PINECONE_API_KEY", None)
 PINECONE_ENVIRONMENT = os.environ.get("PINECONE_ENVIRONMENT", None)
@@ -2286,6 +2501,36 @@ DATALAB_MARKER_OUTPUT_FORMAT = PersistentConfig(
     os.environ.get("DATALAB_MARKER_OUTPUT_FORMAT", "markdown"),
 )
 
+MINERU_API_MODE = PersistentConfig(
+    "MINERU_API_MODE",
+    "rag.mineru_api_mode",
+    os.environ.get("MINERU_API_MODE", "local"),  # "local" or "cloud"
+)
+
+MINERU_API_URL = PersistentConfig(
+    "MINERU_API_URL",
+    "rag.mineru_api_url",
+    os.environ.get("MINERU_API_URL", "http://localhost:8000"),
+)
+
+MINERU_API_KEY = PersistentConfig(
+    "MINERU_API_KEY",
+    "rag.mineru_api_key",
+    os.environ.get("MINERU_API_KEY", ""),
+)
+
+mineru_params = os.getenv("MINERU_PARAMS", "")
+try:
+    mineru_params = json.loads(mineru_params)
+except json.JSONDecodeError:
+    mineru_params = {}
+
+MINERU_PARAMS = PersistentConfig(
+    "MINERU_PARAMS",
+    "rag.mineru_params",
+    mineru_params,
+)
+
 EXTERNAL_DOCUMENT_LOADER_URL = PersistentConfig(
     "EXTERNAL_DOCUMENT_LOADER_URL",
     "rag.external_document_loader_url",
@@ -2310,6 +2555,12 @@ DOCLING_SERVER_URL = PersistentConfig(
     os.getenv("DOCLING_SERVER_URL", "http://docling:5001"),
 )
 
+DOCLING_API_KEY = PersistentConfig(
+    "DOCLING_API_KEY",
+    "rag.docling_api_key",
+    os.getenv("DOCLING_API_KEY", ""),
+)
+
 docling_params = os.getenv("DOCLING_PARAMS", "")
 try:
     docling_params = json.loads(docling_params)
@@ -2322,88 +2573,6 @@ DOCLING_PARAMS = PersistentConfig(
     docling_params,
 )
 
-DOCLING_DO_OCR = PersistentConfig(
-    "DOCLING_DO_OCR",
-    "rag.docling_do_ocr",
-    os.getenv("DOCLING_DO_OCR", "True").lower() == "true",
-)
-
-DOCLING_FORCE_OCR = PersistentConfig(
-    "DOCLING_FORCE_OCR",
-    "rag.docling_force_ocr",
-    os.getenv("DOCLING_FORCE_OCR", "False").lower() == "true",
-)
-
-DOCLING_OCR_ENGINE = PersistentConfig(
-    "DOCLING_OCR_ENGINE",
-    "rag.docling_ocr_engine",
-    os.getenv("DOCLING_OCR_ENGINE", "tesseract"),
-)
-
-DOCLING_OCR_LANG = PersistentConfig(
-    "DOCLING_OCR_LANG",
-    "rag.docling_ocr_lang",
-    os.getenv("DOCLING_OCR_LANG", "eng,fra,deu,spa"),
-)
-
-DOCLING_PDF_BACKEND = PersistentConfig(
-    "DOCLING_PDF_BACKEND",
-    "rag.docling_pdf_backend",
-    os.getenv("DOCLING_PDF_BACKEND", "dlparse_v4"),
-)
-
-DOCLING_TABLE_MODE = PersistentConfig(
-    "DOCLING_TABLE_MODE",
-    "rag.docling_table_mode",
-    os.getenv("DOCLING_TABLE_MODE", "accurate"),
-)
-
-DOCLING_PIPELINE = PersistentConfig(
-    "DOCLING_PIPELINE",
-    "rag.docling_pipeline",
-    os.getenv("DOCLING_PIPELINE", "standard"),
-)
-
-DOCLING_DO_PICTURE_DESCRIPTION = PersistentConfig(
-    "DOCLING_DO_PICTURE_DESCRIPTION",
-    "rag.docling_do_picture_description",
-    os.getenv("DOCLING_DO_PICTURE_DESCRIPTION", "False").lower() == "true",
-)
-
-DOCLING_PICTURE_DESCRIPTION_MODE = PersistentConfig(
-    "DOCLING_PICTURE_DESCRIPTION_MODE",
-    "rag.docling_picture_description_mode",
-    os.getenv("DOCLING_PICTURE_DESCRIPTION_MODE", ""),
-)
-
-
-docling_picture_description_local = os.getenv("DOCLING_PICTURE_DESCRIPTION_LOCAL", "")
-try:
-    docling_picture_description_local = json.loads(docling_picture_description_local)
-except json.JSONDecodeError:
-    docling_picture_description_local = {}
-
-
-DOCLING_PICTURE_DESCRIPTION_LOCAL = PersistentConfig(
-    "DOCLING_PICTURE_DESCRIPTION_LOCAL",
-    "rag.docling_picture_description_local",
-    docling_picture_description_local,
-)
-
-docling_picture_description_api = os.getenv("DOCLING_PICTURE_DESCRIPTION_API", "")
-try:
-    docling_picture_description_api = json.loads(docling_picture_description_api)
-except json.JSONDecodeError:
-    docling_picture_description_api = {}
-
-
-DOCLING_PICTURE_DESCRIPTION_API = PersistentConfig(
-    "DOCLING_PICTURE_DESCRIPTION_API",
-    "rag.docling_picture_description_api",
-    docling_picture_description_api,
-)
-
-
 DOCUMENT_INTELLIGENCE_ENDPOINT = PersistentConfig(
     "DOCUMENT_INTELLIGENCE_ENDPOINT",
     "rag.document_intelligence_endpoint",
@@ -2416,6 +2585,12 @@ DOCUMENT_INTELLIGENCE_KEY = PersistentConfig(
     os.getenv("DOCUMENT_INTELLIGENCE_KEY", ""),
 )
 
+MISTRAL_OCR_API_BASE_URL = PersistentConfig(
+    "MISTRAL_OCR_API_BASE_URL",
+    "rag.MISTRAL_OCR_API_BASE_URL",
+    os.getenv("MISTRAL_OCR_API_BASE_URL", "https://api.mistral.ai/v1"),
+)
+
 MISTRAL_OCR_API_KEY = PersistentConfig(
     "MISTRAL_OCR_API_KEY",
     "rag.mistral_ocr_api_key",
@@ -2454,6 +2629,13 @@ ENABLE_RAG_HYBRID_SEARCH = PersistentConfig(
     os.environ.get("ENABLE_RAG_HYBRID_SEARCH", "").lower() == "true",
 )
 
+ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS = PersistentConfig(
+    "ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS",
+    "rag.enable_hybrid_search_enriched_texts",
+    os.environ.get("ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS", "False").lower()
+    == "true",
+)
+
 RAG_FULL_CONTEXT = PersistentConfig(
     "RAG_FULL_CONTEXT",
     "rag.full_context",
@@ -2548,6 +2730,12 @@ RAG_EMBEDDING_BATCH_SIZE = PersistentConfig(
     ),
 )
 
+ENABLE_ASYNC_EMBEDDING = PersistentConfig(
+    "ENABLE_ASYNC_EMBEDDING",
+    "rag.enable_async_embedding",
+    os.environ.get("ENABLE_ASYNC_EMBEDDING", "True").lower() == "true",
+)
+
 RAG_EMBEDDING_QUERY_PREFIX = os.environ.get("RAG_EMBEDDING_QUERY_PREFIX", None)
 
 RAG_EMBEDDING_CONTENT_PREFIX = os.environ.get("RAG_EMBEDDING_CONTENT_PREFIX", None)
@@ -2641,10 +2829,6 @@ Provide a clear and direct response to the user's query, including inline citati
 <context>
 {{CONTEXT}}
 </context>
-
-<user_query>
-{{QUERY}}
-</user_query>
 """
 
 RAG_TEMPLATE = PersistentConfig(
@@ -2697,6 +2881,26 @@ ENABLE_RAG_LOCAL_WEB_FETCH = (
     os.getenv("ENABLE_RAG_LOCAL_WEB_FETCH", "False").lower() == "true"
 )
 
+
+DEFAULT_WEB_FETCH_FILTER_LIST = [
+    "!169.254.169.254",
+    "!fd00:ec2::254",
+    "!metadata.google.internal",
+    "!metadata.azure.com",
+    "!100.100.100.200",
+]
+
+web_fetch_filter_list = os.getenv("WEB_FETCH_FILTER_LIST", "")
+if web_fetch_filter_list == "":
+    web_fetch_filter_list = []
+else:
+    web_fetch_filter_list = [
+        item.strip() for item in web_fetch_filter_list.split(",") if item.strip()
+    ]
+
+WEB_FETCH_FILTER_LIST = list(set(DEFAULT_WEB_FETCH_FILTER_LIST + web_fetch_filter_list))
+
+
 YOUTUBE_LOADER_LANGUAGE = PersistentConfig(
     "YOUTUBE_LOADER_LANGUAGE",
     "rag.youtube_loader_language",
@@ -2778,6 +2982,7 @@ WEB_SEARCH_DOMAIN_FILTER_LIST = PersistentConfig(
         # "wikipedia.com",
         # "wikimedia.org",
         # "wikidata.org",
+        # "!stackoverflow.com",
     ],
 )
 
@@ -2949,6 +3154,24 @@ BING_SEARCH_V7_SUBSCRIPTION_KEY = PersistentConfig(
     os.environ.get("BING_SEARCH_V7_SUBSCRIPTION_KEY", ""),
 )
 
+AZURE_AI_SEARCH_API_KEY = PersistentConfig(
+    "AZURE_AI_SEARCH_API_KEY",
+    "rag.web.search.azure_ai_search_api_key",
+    os.environ.get("AZURE_AI_SEARCH_API_KEY", ""),
+)
+
+AZURE_AI_SEARCH_ENDPOINT = PersistentConfig(
+    "AZURE_AI_SEARCH_ENDPOINT",
+    "rag.web.search.azure_ai_search_endpoint",
+    os.environ.get("AZURE_AI_SEARCH_ENDPOINT", ""),
+)
+
+AZURE_AI_SEARCH_INDEX_NAME = PersistentConfig(
+    "AZURE_AI_SEARCH_INDEX_NAME",
+    "rag.web.search.azure_ai_search_index_name",
+    os.environ.get("AZURE_AI_SEARCH_INDEX_NAME", ""),
+)
+
 EXA_API_KEY = PersistentConfig(
     "EXA_API_KEY",
     "rag.web.search.exa_api_key",
@@ -2973,6 +3196,12 @@ PERPLEXITY_SEARCH_CONTEXT_USAGE = PersistentConfig(
     os.getenv("PERPLEXITY_SEARCH_CONTEXT_USAGE", "medium"),
 )
 
+PERPLEXITY_SEARCH_API_URL = PersistentConfig(
+    "PERPLEXITY_SEARCH_API_URL",
+    "rag.web.search.perplexity_search_api_url",
+    os.getenv("PERPLEXITY_SEARCH_API_URL", "https://api.perplexity.ai/search"),
+)
+
 SOUGOU_API_SID = PersistentConfig(
     "SOUGOU_API_SID",
     "rag.web.search.sougou_api_sid",
@@ -3049,16 +3278,30 @@ EXTERNAL_WEB_LOADER_API_KEY = PersistentConfig(
 # Images
 ####################################
 
+ENABLE_IMAGE_GENERATION = PersistentConfig(
+    "ENABLE_IMAGE_GENERATION",
+    "image_generation.enable",
+    os.environ.get("ENABLE_IMAGE_GENERATION", "").lower() == "true",
+)
+
 IMAGE_GENERATION_ENGINE = PersistentConfig(
     "IMAGE_GENERATION_ENGINE",
     "image_generation.engine",
     os.getenv("IMAGE_GENERATION_ENGINE", "openai"),
 )
 
-ENABLE_IMAGE_GENERATION = PersistentConfig(
-    "ENABLE_IMAGE_GENERATION",
-    "image_generation.enable",
-    os.environ.get("ENABLE_IMAGE_GENERATION", "").lower() == "true",
+IMAGE_GENERATION_MODEL = PersistentConfig(
+    "IMAGE_GENERATION_MODEL",
+    "image_generation.model",
+    os.getenv("IMAGE_GENERATION_MODEL", ""),
+)
+
+IMAGE_SIZE = PersistentConfig(
+    "IMAGE_SIZE", "image_generation.size", os.getenv("IMAGE_SIZE", "512x512")
+)
+
+IMAGE_STEPS = PersistentConfig(
+    "IMAGE_STEPS", "image_generation.steps", int(os.getenv("IMAGE_STEPS", 50))
 )
 
 ENABLE_IMAGE_PROMPT_GENERATION = PersistentConfig(
@@ -3078,35 +3321,16 @@ AUTOMATIC1111_API_AUTH = PersistentConfig(
     os.getenv("AUTOMATIC1111_API_AUTH", ""),
 )
 
-AUTOMATIC1111_CFG_SCALE = PersistentConfig(
-    "AUTOMATIC1111_CFG_SCALE",
-    "image_generation.automatic1111.cfg_scale",
-    (
-        float(os.environ.get("AUTOMATIC1111_CFG_SCALE"))
-        if os.environ.get("AUTOMATIC1111_CFG_SCALE")
-        else None
-    ),
-)
+automatic1111_params = os.getenv("AUTOMATIC1111_PARAMS", "")
+try:
+    automatic1111_params = json.loads(automatic1111_params)
+except json.JSONDecodeError:
+    automatic1111_params = {}
 
-
-AUTOMATIC1111_SAMPLER = PersistentConfig(
-    "AUTOMATIC1111_SAMPLER",
-    "image_generation.automatic1111.sampler",
-    (
-        os.environ.get("AUTOMATIC1111_SAMPLER")
-        if os.environ.get("AUTOMATIC1111_SAMPLER")
-        else None
-    ),
-)
-
-AUTOMATIC1111_SCHEDULER = PersistentConfig(
-    "AUTOMATIC1111_SCHEDULER",
-    "image_generation.automatic1111.scheduler",
-    (
-        os.environ.get("AUTOMATIC1111_SCHEDULER")
-        if os.environ.get("AUTOMATIC1111_SCHEDULER")
-        else None
-    ),
+AUTOMATIC1111_PARAMS = PersistentConfig(
+    "AUTOMATIC1111_PARAMS",
+    "image_generation.automatic1111.api_params",
+    automatic1111_params,
 )
 
 COMFYUI_BASE_URL = PersistentConfig(
@@ -3261,6 +3485,18 @@ IMAGES_OPENAI_API_KEY = PersistentConfig(
     os.getenv("IMAGES_OPENAI_API_KEY", OPENAI_API_KEY),
 )
 
+images_openai_params = os.getenv("IMAGES_OPENAI_PARAMS", "")
+try:
+    images_openai_params = json.loads(images_openai_params)
+except json.JSONDecodeError:
+    images_openai_params = {}
+
+
+IMAGES_OPENAI_API_PARAMS = PersistentConfig(
+    "IMAGES_OPENAI_API_PARAMS", "image_generation.openai.params", images_openai_params
+)
+
+
 IMAGES_GEMINI_API_BASE_URL = PersistentConfig(
     "IMAGES_GEMINI_API_BASE_URL",
     "image_generation.gemini.api_base_url",
@@ -3272,18 +3508,84 @@ IMAGES_GEMINI_API_KEY = PersistentConfig(
     os.getenv("IMAGES_GEMINI_API_KEY", GEMINI_API_KEY),
 )
 
-IMAGE_SIZE = PersistentConfig(
-    "IMAGE_SIZE", "image_generation.size", os.getenv("IMAGE_SIZE", "512x512")
+IMAGES_GEMINI_ENDPOINT_METHOD = PersistentConfig(
+    "IMAGES_GEMINI_ENDPOINT_METHOD",
+    "image_generation.gemini.endpoint_method",
+    os.getenv("IMAGES_GEMINI_ENDPOINT_METHOD", ""),
 )
 
-IMAGE_STEPS = PersistentConfig(
-    "IMAGE_STEPS", "image_generation.steps", int(os.getenv("IMAGE_STEPS", 50))
+ENABLE_IMAGE_EDIT = PersistentConfig(
+    "ENABLE_IMAGE_EDIT",
+    "images.edit.enable",
+    os.environ.get("ENABLE_IMAGE_EDIT", "").lower() == "true",
 )
 
-IMAGE_GENERATION_MODEL = PersistentConfig(
-    "IMAGE_GENERATION_MODEL",
-    "image_generation.model",
-    os.getenv("IMAGE_GENERATION_MODEL", ""),
+IMAGE_EDIT_ENGINE = PersistentConfig(
+    "IMAGE_EDIT_ENGINE",
+    "images.edit.engine",
+    os.getenv("IMAGE_EDIT_ENGINE", "openai"),
+)
+
+IMAGE_EDIT_MODEL = PersistentConfig(
+    "IMAGE_EDIT_MODEL",
+    "images.edit.model",
+    os.getenv("IMAGE_EDIT_MODEL", ""),
+)
+
+IMAGE_EDIT_SIZE = PersistentConfig(
+    "IMAGE_EDIT_SIZE", "images.edit.size", os.getenv("IMAGE_EDIT_SIZE", "")
+)
+
+IMAGES_EDIT_OPENAI_API_BASE_URL = PersistentConfig(
+    "IMAGES_EDIT_OPENAI_API_BASE_URL",
+    "images.edit.openai.api_base_url",
+    os.getenv("IMAGES_EDIT_OPENAI_API_BASE_URL", OPENAI_API_BASE_URL),
+)
+IMAGES_EDIT_OPENAI_API_VERSION = PersistentConfig(
+    "IMAGES_EDIT_OPENAI_API_VERSION",
+    "images.edit.openai.api_version",
+    os.getenv("IMAGES_EDIT_OPENAI_API_VERSION", ""),
+)
+
+IMAGES_EDIT_OPENAI_API_KEY = PersistentConfig(
+    "IMAGES_EDIT_OPENAI_API_KEY",
+    "images.edit.openai.api_key",
+    os.getenv("IMAGES_EDIT_OPENAI_API_KEY", OPENAI_API_KEY),
+)
+
+IMAGES_EDIT_GEMINI_API_BASE_URL = PersistentConfig(
+    "IMAGES_EDIT_GEMINI_API_BASE_URL",
+    "images.edit.gemini.api_base_url",
+    os.getenv("IMAGES_EDIT_GEMINI_API_BASE_URL", GEMINI_API_BASE_URL),
+)
+IMAGES_EDIT_GEMINI_API_KEY = PersistentConfig(
+    "IMAGES_EDIT_GEMINI_API_KEY",
+    "images.edit.gemini.api_key",
+    os.getenv("IMAGES_EDIT_GEMINI_API_KEY", GEMINI_API_KEY),
+)
+
+
+IMAGES_EDIT_COMFYUI_BASE_URL = PersistentConfig(
+    "IMAGES_EDIT_COMFYUI_BASE_URL",
+    "images.edit.comfyui.base_url",
+    os.getenv("IMAGES_EDIT_COMFYUI_BASE_URL", ""),
+)
+IMAGES_EDIT_COMFYUI_API_KEY = PersistentConfig(
+    "IMAGES_EDIT_COMFYUI_API_KEY",
+    "images.edit.comfyui.api_key",
+    os.getenv("IMAGES_EDIT_COMFYUI_API_KEY", ""),
+)
+
+IMAGES_EDIT_COMFYUI_WORKFLOW = PersistentConfig(
+    "IMAGES_EDIT_COMFYUI_WORKFLOW",
+    "images.edit.comfyui.workflow",
+    os.getenv("IMAGES_EDIT_COMFYUI_WORKFLOW", ""),
+)
+
+IMAGES_EDIT_COMFYUI_WORKFLOW_NODES = PersistentConfig(
+    "IMAGES_EDIT_COMFYUI_WORKFLOW_NODES",
+    "images.edit.comfyui.nodes",
+    [],
 )
 
 ####################################
@@ -3318,6 +3620,10 @@ DEEPGRAM_API_KEY = PersistentConfig(
     os.getenv("DEEPGRAM_API_KEY", ""),
 )
 
+# ElevenLabs configuration
+ELEVENLABS_API_BASE_URL = os.getenv(
+    "ELEVENLABS_API_BASE_URL", "https://api.elevenlabs.io"
+)
 
 AUDIO_STT_OPENAI_API_BASE_URL = PersistentConfig(
     "AUDIO_STT_OPENAI_API_BASE_URL",
@@ -3385,6 +3691,24 @@ AUDIO_STT_AZURE_MAX_SPEAKERS = PersistentConfig(
     os.getenv("AUDIO_STT_AZURE_MAX_SPEAKERS", ""),
 )
 
+AUDIO_STT_MISTRAL_API_KEY = PersistentConfig(
+    "AUDIO_STT_MISTRAL_API_KEY",
+    "audio.stt.mistral.api_key",
+    os.getenv("AUDIO_STT_MISTRAL_API_KEY", ""),
+)
+
+AUDIO_STT_MISTRAL_API_BASE_URL = PersistentConfig(
+    "AUDIO_STT_MISTRAL_API_BASE_URL",
+    "audio.stt.mistral.api_base_url",
+    os.getenv("AUDIO_STT_MISTRAL_API_BASE_URL", "https://api.mistral.ai/v1"),
+)
+
+AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS = PersistentConfig(
+    "AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS",
+    "audio.stt.mistral.use_chat_completions",
+    os.getenv("AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS", "false").lower() == "true",
+)
+
 AUDIO_TTS_OPENAI_API_BASE_URL = PersistentConfig(
     "AUDIO_TTS_OPENAI_API_BASE_URL",
     "audio.tts.openai.api_base_url",

backend/open_webui/constants.py

@@ -38,13 +38,14 @@ class ERROR_MESSAGES(str, Enum):
     ID_TAKEN = "Uh-oh! This id is already registered. Please choose another id string."
     MODEL_ID_TAKEN = "Uh-oh! This model id is already registered. Please choose another model id string."
     NAME_TAG_TAKEN = "Uh-oh! This name tag is already registered. Please choose another name tag string."
+    MODEL_ID_TOO_LONG = "The model id is too long. Please make sure your model id is less than 256 characters long."
 
     INVALID_TOKEN = (
         "Your session has expired or the token is invalid. Please sign in again."
     )
     INVALID_CRED = "The email or password provided is incorrect. Please check for typos and try logging in again."
     INVALID_EMAIL_FORMAT = "The email format you entered is invalid. Please double-check and make sure you're using a valid email address (e.g., yourname@example.com)."
-    INVALID_PASSWORD = (
+    INCORRECT_PASSWORD = (
         "The password provided is incorrect. Please check for typos and try again."
     )
     INVALID_TRUSTED_HEADER = "Your provider has not provided a trusted header. Please contact your administrator for assistance."
@@ -104,6 +105,10 @@ class ERROR_MESSAGES(str, Enum):
     )
     FILE_NOT_PROCESSED = "Extracted content is not available for this file. Please ensure that the file is processed before proceeding."
 
+    INVALID_PASSWORD = lambda err="": (
+        err if err else "The password does not meet the required validation criteria."
+    )
+
 
 class TASKS(str, Enum):
     def __str__(self) -> str:

backend/open_webui/env.py

@@ -8,6 +8,8 @@ import shutil
 from uuid import uuid4
 from pathlib import Path
 from cryptography.hazmat.primitives import serialization
+import re
+
 
 import markdown
 from bs4 import BeautifulSoup
@@ -135,6 +137,9 @@ else:
         PACKAGE_DATA = {"version": "0.0.0"}
 
 VERSION = PACKAGE_DATA["version"]
+
+
+DEPLOYMENT_ID = os.environ.get("DEPLOYMENT_ID", "")
 INSTANCE_ID = os.environ.get("INSTANCE_ID", str(uuid4()))
 
 
@@ -212,6 +217,11 @@ ENABLE_FORWARD_USER_INFO_HEADERS = (
     os.environ.get("ENABLE_FORWARD_USER_INFO_HEADERS", "False").lower() == "true"
 )
 
+# Experimental feature, may be removed in future
+ENABLE_STAR_SESSIONS_MIDDLEWARE = (
+    os.environ.get("ENABLE_STAR_SESSIONS_MIDDLEWARE", "False").lower() == "true"
+)
+
 ####################################
 # WEBUI_BUILD_HASH
 ####################################
@@ -421,6 +431,17 @@ WEBUI_AUTH_TRUSTED_GROUPS_HEADER = os.environ.get(
 )
 
 
+ENABLE_PASSWORD_VALIDATION = (
+    os.environ.get("ENABLE_PASSWORD_VALIDATION", "False").lower() == "true"
+)
+PASSWORD_VALIDATION_REGEX_PATTERN = os.environ.get(
+    "PASSWORD_VALIDATION_REGEX_PATTERN",
+    "^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[^\w\s]).{8,}$",
+)
+
+PASSWORD_VALIDATION_REGEX_PATTERN = re.compile(PASSWORD_VALIDATION_REGEX_PATTERN)
+
+
 BYPASS_MODEL_ACCESS_CONTROL = (
     os.environ.get("BYPASS_MODEL_ACCESS_CONTROL", "False").lower() == "true"
 )
@@ -468,7 +489,9 @@ ENABLE_COMPRESSION_MIDDLEWARE = (
 ####################################
 # OAUTH Configuration
 ####################################
-
+ENABLE_OAUTH_EMAIL_FALLBACK = (
+    os.environ.get("ENABLE_OAUTH_EMAIL_FALLBACK", "False").lower() == "true"
+)
 
 ENABLE_OAUTH_ID_TOKEN_COOKIE = (
     os.environ.get("ENABLE_OAUTH_ID_TOKEN_COOKIE", "True").lower() == "true"
@@ -482,12 +505,14 @@ OAUTH_SESSION_TOKEN_ENCRYPTION_KEY = os.environ.get(
     "OAUTH_SESSION_TOKEN_ENCRYPTION_KEY", WEBUI_SECRET_KEY
 )
 
-
 ####################################
 # SCIM Configuration
 ####################################
 
-SCIM_ENABLED = os.environ.get("SCIM_ENABLED", "False").lower() == "true"
+ENABLE_SCIM = (
+    os.environ.get("ENABLE_SCIM", os.environ.get("SCIM_ENABLED", "False")).lower()
+    == "true"
+)
 SCIM_TOKEN = os.environ.get("SCIM_TOKEN", "")
 
 ####################################
@@ -535,6 +560,11 @@ else:
 # CHAT
 ####################################
 
+ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION = (
+    os.environ.get("ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION", "False").lower()
+    == "true"
+)
+
 CHAT_RESPONSE_STREAM_DELTA_CHUNK_SIZE = os.environ.get(
     "CHAT_RESPONSE_STREAM_DELTA_CHUNK_SIZE", "1"
 )
@@ -563,6 +593,21 @@ else:
         CHAT_RESPONSE_MAX_TOOL_CALL_RETRIES = 30
 
 
+CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE = os.environ.get(
+    "CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE", ""
+)
+
+if CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE == "":
+    CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE = None
+else:
+    try:
+        CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE = int(
+            CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE
+        )
+    except Exception:
+        CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE = None
+
+
 ####################################
 # WEBSOCKET SUPPORT
 ####################################
@@ -574,6 +619,17 @@ ENABLE_WEBSOCKET_SUPPORT = (
 
 WEBSOCKET_MANAGER = os.environ.get("WEBSOCKET_MANAGER", "")
 
+WEBSOCKET_REDIS_OPTIONS = os.environ.get("WEBSOCKET_REDIS_OPTIONS", "")
+if WEBSOCKET_REDIS_OPTIONS == "":
+    log.debug("No WEBSOCKET_REDIS_OPTIONS provided, defaulting to None")
+    WEBSOCKET_REDIS_OPTIONS = None
+else:
+    try:
+        WEBSOCKET_REDIS_OPTIONS = json.loads(WEBSOCKET_REDIS_OPTIONS)
+    except Exception:
+        log.warning("Invalid WEBSOCKET_REDIS_OPTIONS, defaulting to None")
+        WEBSOCKET_REDIS_OPTIONS = None
+
 WEBSOCKET_REDIS_URL = os.environ.get("WEBSOCKET_REDIS_URL", REDIS_URL)
 WEBSOCKET_REDIS_CLUSTER = (
     os.environ.get("WEBSOCKET_REDIS_CLUSTER", str(REDIS_CLUSTER)).lower() == "true"
@@ -588,6 +644,23 @@ except ValueError:
 
 WEBSOCKET_SENTINEL_HOSTS = os.environ.get("WEBSOCKET_SENTINEL_HOSTS", "")
 WEBSOCKET_SENTINEL_PORT = os.environ.get("WEBSOCKET_SENTINEL_PORT", "26379")
+WEBSOCKET_SERVER_LOGGING = (
+    os.environ.get("WEBSOCKET_SERVER_LOGGING", "False").lower() == "true"
+)
+WEBSOCKET_SERVER_ENGINEIO_LOGGING = (
+    os.environ.get("WEBSOCKET_SERVER_LOGGING", "False").lower() == "true"
+)
+WEBSOCKET_SERVER_PING_TIMEOUT = os.environ.get("WEBSOCKET_SERVER_PING_TIMEOUT", "20")
+try:
+    WEBSOCKET_SERVER_PING_TIMEOUT = int(WEBSOCKET_SERVER_PING_TIMEOUT)
+except ValueError:
+    WEBSOCKET_SERVER_PING_TIMEOUT = 20
+
+WEBSOCKET_SERVER_PING_INTERVAL = os.environ.get("WEBSOCKET_SERVER_PING_INTERVAL", "25")
+try:
+    WEBSOCKET_SERVER_PING_INTERVAL = int(WEBSOCKET_SERVER_PING_INTERVAL)
+except ValueError:
+    WEBSOCKET_SERVER_PING_INTERVAL = 25
 
 
 AIOHTTP_CLIENT_TIMEOUT = os.environ.get("AIOHTTP_CLIENT_TIMEOUT", "")
@@ -700,7 +773,9 @@ if OFFLINE_MODE:
 # AUDIT LOGGING
 ####################################
 # Where to store log file
-AUDIT_LOGS_FILE_PATH = f"{DATA_DIR}/audit.log"
+# Defaults to the DATA_DIR/audit.log. To set AUDIT_LOGS_FILE_PATH you need to
+# provide the whole path, like: /app/audit.log
+AUDIT_LOGS_FILE_PATH = os.getenv("AUDIT_LOGS_FILE_PATH", f"{DATA_DIR}/audit.log")
 # Maximum size of a file before rotating into a new log file
 AUDIT_LOG_FILE_ROTATION_SIZE = os.getenv("AUDIT_LOG_FILE_ROTATION_SIZE", "10MB")
 

backend/open_webui/main.py

@@ -61,11 +61,11 @@ from open_webui.utils import logger
 from open_webui.utils.audit import AuditLevel, AuditLoggingMiddleware
 from open_webui.utils.logger import start_logger
 from open_webui.socket.main import (
+    MODELS,
     app as socket_app,
     periodic_usage_pool_cleanup,
     get_event_emitter,
     get_models_in_use,
-    get_active_user_ids,
 )
 from open_webui.routers import (
     audio,
@@ -146,9 +146,7 @@ from open_webui.config import (
     # Image
     AUTOMATIC1111_API_AUTH,
     AUTOMATIC1111_BASE_URL,
-    AUTOMATIC1111_CFG_SCALE,
-    AUTOMATIC1111_SAMPLER,
-    AUTOMATIC1111_SCHEDULER,
+    AUTOMATIC1111_PARAMS,
     COMFYUI_BASE_URL,
     COMFYUI_API_KEY,
     COMFYUI_WORKFLOW,
@@ -162,8 +160,23 @@ from open_webui.config import (
     IMAGES_OPENAI_API_BASE_URL,
     IMAGES_OPENAI_API_VERSION,
     IMAGES_OPENAI_API_KEY,
+    IMAGES_OPENAI_API_PARAMS,
     IMAGES_GEMINI_API_BASE_URL,
     IMAGES_GEMINI_API_KEY,
+    IMAGES_GEMINI_ENDPOINT_METHOD,
+    ENABLE_IMAGE_EDIT,
+    IMAGE_EDIT_ENGINE,
+    IMAGE_EDIT_MODEL,
+    IMAGE_EDIT_SIZE,
+    IMAGES_EDIT_OPENAI_API_BASE_URL,
+    IMAGES_EDIT_OPENAI_API_KEY,
+    IMAGES_EDIT_OPENAI_API_VERSION,
+    IMAGES_EDIT_GEMINI_API_BASE_URL,
+    IMAGES_EDIT_GEMINI_API_KEY,
+    IMAGES_EDIT_COMFYUI_BASE_URL,
+    IMAGES_EDIT_COMFYUI_API_KEY,
+    IMAGES_EDIT_COMFYUI_WORKFLOW,
+    IMAGES_EDIT_COMFYUI_WORKFLOW_NODES,
     # Audio
     AUDIO_STT_ENGINE,
     AUDIO_STT_MODEL,
@@ -175,6 +188,9 @@ from open_webui.config import (
     AUDIO_STT_AZURE_LOCALES,
     AUDIO_STT_AZURE_BASE_URL,
     AUDIO_STT_AZURE_MAX_SPEAKERS,
+    AUDIO_STT_MISTRAL_API_KEY,
+    AUDIO_STT_MISTRAL_API_BASE_URL,
+    AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS,
     AUDIO_TTS_ENGINE,
     AUDIO_TTS_MODEL,
     AUDIO_TTS_VOICE,
@@ -214,6 +230,7 @@ from open_webui.config import (
     RAG_RERANKING_MODEL_TRUST_REMOTE_CODE,
     RAG_EMBEDDING_ENGINE,
     RAG_EMBEDDING_BATCH_SIZE,
+    ENABLE_ASYNC_EMBEDDING,
     RAG_TOP_K,
     RAG_TOP_K_RERANKER,
     RAG_RELEVANCE_THRESHOLD,
@@ -243,25 +260,20 @@ from open_webui.config import (
     DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION,
     DATALAB_MARKER_FORMAT_LINES,
     DATALAB_MARKER_OUTPUT_FORMAT,
+    MINERU_API_MODE,
+    MINERU_API_URL,
+    MINERU_API_KEY,
+    MINERU_PARAMS,
     DATALAB_MARKER_USE_LLM,
     EXTERNAL_DOCUMENT_LOADER_URL,
     EXTERNAL_DOCUMENT_LOADER_API_KEY,
     TIKA_SERVER_URL,
     DOCLING_SERVER_URL,
+    DOCLING_API_KEY,
     DOCLING_PARAMS,
-    DOCLING_DO_OCR,
-    DOCLING_FORCE_OCR,
-    DOCLING_OCR_ENGINE,
-    DOCLING_OCR_LANG,
-    DOCLING_PDF_BACKEND,
-    DOCLING_TABLE_MODE,
-    DOCLING_PIPELINE,
-    DOCLING_DO_PICTURE_DESCRIPTION,
-    DOCLING_PICTURE_DESCRIPTION_MODE,
-    DOCLING_PICTURE_DESCRIPTION_LOCAL,
-    DOCLING_PICTURE_DESCRIPTION_API,
     DOCUMENT_INTELLIGENCE_ENDPOINT,
     DOCUMENT_INTELLIGENCE_KEY,
+    MISTRAL_OCR_API_BASE_URL,
     MISTRAL_OCR_API_KEY,
     RAG_TEXT_SPLITTER,
     TIKTOKEN_ENCODING_NAME,
@@ -300,6 +312,7 @@ from open_webui.config import (
     PERPLEXITY_API_KEY,
     PERPLEXITY_MODEL,
     PERPLEXITY_SEARCH_CONTEXT_USAGE,
+    PERPLEXITY_SEARCH_API_URL,
     SOUGOU_API_SID,
     SOUGOU_API_SK,
     KAGI_SEARCH_API_KEY,
@@ -317,6 +330,7 @@ from open_webui.config import (
     ENABLE_ONEDRIVE_PERSONAL,
     ENABLE_ONEDRIVE_BUSINESS,
     ENABLE_RAG_HYBRID_SEARCH,
+    ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS,
     ENABLE_RAG_LOCAL_WEB_FETCH,
     ENABLE_WEB_LOADER_SSL_VERIFICATION,
     ENABLE_GOOGLE_DRIVE_INTEGRATION,
@@ -335,9 +349,10 @@ from open_webui.config import (
     JWT_EXPIRES_IN,
     ENABLE_SIGNUP,
     ENABLE_LOGIN_FORM,
-    ENABLE_API_KEY,
-    ENABLE_API_KEY_ENDPOINT_RESTRICTIONS,
-    API_KEY_ALLOWED_ENDPOINTS,
+    ENABLE_API_KEYS,
+    ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS,
+    API_KEYS_ALLOWED_ENDPOINTS,
+    ENABLE_FOLDERS,
     ENABLE_CHANNELS,
     ENABLE_NOTES,
     ENABLE_COMMUNITY_SHARING,
@@ -347,10 +362,12 @@ from open_webui.config import (
     BYPASS_ADMIN_ACCESS_CONTROL,
     USER_PERMISSIONS,
     DEFAULT_USER_ROLE,
+    DEFAULT_GROUP_ID,
     PENDING_USER_OVERLAY_CONTENT,
     PENDING_USER_OVERLAY_TITLE,
     DEFAULT_PROMPT_SUGGESTIONS,
     DEFAULT_MODELS,
+    DEFAULT_PINNED_MODELS,
     DEFAULT_ARENA_MODEL,
     MODEL_ORDER_LIST,
     EVALUATION_ARENA_MODELS,
@@ -409,6 +426,7 @@ from open_webui.config import (
     TAGS_GENERATION_PROMPT_TEMPLATE,
     IMAGE_PROMPT_GENERATION_PROMPT_TEMPLATE,
     TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
+    VOICE_MODE_PROMPT_TEMPLATE,
     QUERY_GENERATION_PROMPT_TEMPLATE,
     AUTOCOMPLETE_GENERATION_PROMPT_TEMPLATE,
     AUTOCOMPLETE_GENERATION_INPUT_MAX_LENGTH,
@@ -430,6 +448,7 @@ from open_webui.env import (
     SAFE_MODE,
     SRC_LOG_LEVELS,
     VERSION,
+    DEPLOYMENT_ID,
     INSTANCE_ID,
     WEBUI_BUILD_HASH,
     WEBUI_SECRET_KEY,
@@ -440,7 +459,7 @@ from open_webui.env import (
     WEBUI_AUTH_TRUSTED_NAME_HEADER,
     WEBUI_AUTH_SIGNOUT_REDIRECT_URL,
     # SCIM
-    SCIM_ENABLED,
+    ENABLE_SCIM,
     SCIM_TOKEN,
     ENABLE_COMPRESSION_MIDDLEWARE,
     ENABLE_WEBSOCKET_SUPPORT,
@@ -450,6 +469,7 @@ from open_webui.env import (
     ENABLE_OTEL,
     EXTERNAL_PWA_MANIFEST_URL,
     AIOHTTP_CLIENT_SESSION_SSL,
+    ENABLE_STAR_SESSIONS_MIDDLEWARE,
 )
 
 
@@ -477,9 +497,11 @@ from open_webui.utils.auth import (
 )
 from open_webui.utils.plugin import install_tool_and_function_dependencies
 from open_webui.utils.oauth import (
+    get_oauth_client_info_with_dynamic_client_registration,
+    encrypt_data,
+    decrypt_data,
     OAuthManager,
     OAuthClientManager,
-    decrypt_data,
     OAuthClientInformationFull,
 )
 from open_webui.utils.security_headers import SecurityHeadersMiddleware
@@ -693,7 +715,7 @@ app.state.config.ENABLE_DIRECT_CONNECTIONS = ENABLE_DIRECT_CONNECTIONS
 #
 ########################################
 
-app.state.SCIM_ENABLED = SCIM_ENABLED
+app.state.ENABLE_SCIM = ENABLE_SCIM
 app.state.SCIM_TOKEN = SCIM_TOKEN
 
 ########################################
@@ -715,11 +737,11 @@ app.state.config.WEBUI_URL = WEBUI_URL
 app.state.config.ENABLE_SIGNUP = ENABLE_SIGNUP
 app.state.config.ENABLE_LOGIN_FORM = ENABLE_LOGIN_FORM
 
-app.state.config.ENABLE_API_KEY = ENABLE_API_KEY
-app.state.config.ENABLE_API_KEY_ENDPOINT_RESTRICTIONS = (
-    ENABLE_API_KEY_ENDPOINT_RESTRICTIONS
+app.state.config.ENABLE_API_KEYS = ENABLE_API_KEYS
+app.state.config.ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS = (
+    ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS
 )
-app.state.config.API_KEY_ALLOWED_ENDPOINTS = API_KEY_ALLOWED_ENDPOINTS
+app.state.config.API_KEYS_ALLOWED_ENDPOINTS = API_KEYS_ALLOWED_ENDPOINTS
 
 app.state.config.JWT_EXPIRES_IN = JWT_EXPIRES_IN
 
@@ -728,8 +750,13 @@ app.state.config.ADMIN_EMAIL = ADMIN_EMAIL
 
 
 app.state.config.DEFAULT_MODELS = DEFAULT_MODELS
+app.state.config.DEFAULT_PINNED_MODELS = DEFAULT_PINNED_MODELS
+app.state.config.MODEL_ORDER_LIST = MODEL_ORDER_LIST
+
+
 app.state.config.DEFAULT_PROMPT_SUGGESTIONS = DEFAULT_PROMPT_SUGGESTIONS
 app.state.config.DEFAULT_USER_ROLE = DEFAULT_USER_ROLE
+app.state.config.DEFAULT_GROUP_ID = DEFAULT_GROUP_ID
 
 app.state.config.PENDING_USER_OVERLAY_CONTENT = PENDING_USER_OVERLAY_CONTENT
 app.state.config.PENDING_USER_OVERLAY_TITLE = PENDING_USER_OVERLAY_TITLE
@@ -739,9 +766,9 @@ app.state.config.RESPONSE_WATERMARK = RESPONSE_WATERMARK
 app.state.config.USER_PERMISSIONS = USER_PERMISSIONS
 app.state.config.WEBHOOK_URL = WEBHOOK_URL
 app.state.config.BANNERS = WEBUI_BANNERS
-app.state.config.MODEL_ORDER_LIST = MODEL_ORDER_LIST
 
 
+app.state.config.ENABLE_FOLDERS = ENABLE_FOLDERS
 app.state.config.ENABLE_CHANNELS = ENABLE_CHANNELS
 app.state.config.ENABLE_NOTES = ENABLE_NOTES
 app.state.config.ENABLE_COMMUNITY_SHARING = ENABLE_COMMUNITY_SHARING
@@ -817,6 +844,9 @@ app.state.config.FILE_IMAGE_COMPRESSION_HEIGHT = FILE_IMAGE_COMPRESSION_HEIGHT
 app.state.config.RAG_FULL_CONTEXT = RAG_FULL_CONTEXT
 app.state.config.BYPASS_EMBEDDING_AND_RETRIEVAL = BYPASS_EMBEDDING_AND_RETRIEVAL
 app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH
+app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS = (
+    ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS
+)
 app.state.config.ENABLE_WEB_LOADER_SSL_VERIFICATION = ENABLE_WEB_LOADER_SSL_VERIFICATION
 
 app.state.config.CONTENT_EXTRACTION_ENGINE = CONTENT_EXTRACTION_ENGINE
@@ -837,21 +867,16 @@ app.state.config.EXTERNAL_DOCUMENT_LOADER_URL = EXTERNAL_DOCUMENT_LOADER_URL
 app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY = EXTERNAL_DOCUMENT_LOADER_API_KEY
 app.state.config.TIKA_SERVER_URL = TIKA_SERVER_URL
 app.state.config.DOCLING_SERVER_URL = DOCLING_SERVER_URL
+app.state.config.DOCLING_API_KEY = DOCLING_API_KEY
 app.state.config.DOCLING_PARAMS = DOCLING_PARAMS
-app.state.config.DOCLING_DO_OCR = DOCLING_DO_OCR
-app.state.config.DOCLING_FORCE_OCR = DOCLING_FORCE_OCR
-app.state.config.DOCLING_OCR_ENGINE = DOCLING_OCR_ENGINE
-app.state.config.DOCLING_OCR_LANG = DOCLING_OCR_LANG
-app.state.config.DOCLING_PDF_BACKEND = DOCLING_PDF_BACKEND
-app.state.config.DOCLING_TABLE_MODE = DOCLING_TABLE_MODE
-app.state.config.DOCLING_PIPELINE = DOCLING_PIPELINE
-app.state.config.DOCLING_DO_PICTURE_DESCRIPTION = DOCLING_DO_PICTURE_DESCRIPTION
-app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE = DOCLING_PICTURE_DESCRIPTION_MODE
-app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL = DOCLING_PICTURE_DESCRIPTION_LOCAL
-app.state.config.DOCLING_PICTURE_DESCRIPTION_API = DOCLING_PICTURE_DESCRIPTION_API
 app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT = DOCUMENT_INTELLIGENCE_ENDPOINT
 app.state.config.DOCUMENT_INTELLIGENCE_KEY = DOCUMENT_INTELLIGENCE_KEY
+app.state.config.MISTRAL_OCR_API_BASE_URL = MISTRAL_OCR_API_BASE_URL
 app.state.config.MISTRAL_OCR_API_KEY = MISTRAL_OCR_API_KEY
+app.state.config.MINERU_API_MODE = MINERU_API_MODE
+app.state.config.MINERU_API_URL = MINERU_API_URL
+app.state.config.MINERU_API_KEY = MINERU_API_KEY
+app.state.config.MINERU_PARAMS = MINERU_PARAMS
 
 app.state.config.TEXT_SPLITTER = RAG_TEXT_SPLITTER
 app.state.config.TIKTOKEN_ENCODING_NAME = TIKTOKEN_ENCODING_NAME
@@ -862,6 +887,7 @@ app.state.config.CHUNK_OVERLAP = CHUNK_OVERLAP
 app.state.config.RAG_EMBEDDING_ENGINE = RAG_EMBEDDING_ENGINE
 app.state.config.RAG_EMBEDDING_MODEL = RAG_EMBEDDING_MODEL
 app.state.config.RAG_EMBEDDING_BATCH_SIZE = RAG_EMBEDDING_BATCH_SIZE
+app.state.config.ENABLE_ASYNC_EMBEDDING = ENABLE_ASYNC_EMBEDDING
 
 app.state.config.RAG_RERANKING_ENGINE = RAG_RERANKING_ENGINE
 app.state.config.RAG_RERANKING_MODEL = RAG_RERANKING_MODEL
@@ -931,6 +957,7 @@ app.state.config.EXA_API_KEY = EXA_API_KEY
 app.state.config.PERPLEXITY_API_KEY = PERPLEXITY_API_KEY
 app.state.config.PERPLEXITY_MODEL = PERPLEXITY_MODEL
 app.state.config.PERPLEXITY_SEARCH_CONTEXT_USAGE = PERPLEXITY_SEARCH_CONTEXT_USAGE
+app.state.config.PERPLEXITY_SEARCH_API_URL = PERPLEXITY_SEARCH_API_URL
 app.state.config.SOUGOU_API_SID = SOUGOU_API_SID
 app.state.config.SOUGOU_API_SK = SOUGOU_API_SK
 app.state.config.EXTERNAL_WEB_SEARCH_URL = EXTERNAL_WEB_SEARCH_URL
@@ -1053,27 +1080,42 @@ app.state.config.IMAGE_GENERATION_ENGINE = IMAGE_GENERATION_ENGINE
 app.state.config.ENABLE_IMAGE_GENERATION = ENABLE_IMAGE_GENERATION
 app.state.config.ENABLE_IMAGE_PROMPT_GENERATION = ENABLE_IMAGE_PROMPT_GENERATION
 
+app.state.config.IMAGE_GENERATION_MODEL = IMAGE_GENERATION_MODEL
+app.state.config.IMAGE_SIZE = IMAGE_SIZE
+app.state.config.IMAGE_STEPS = IMAGE_STEPS
+
 app.state.config.IMAGES_OPENAI_API_BASE_URL = IMAGES_OPENAI_API_BASE_URL
 app.state.config.IMAGES_OPENAI_API_VERSION = IMAGES_OPENAI_API_VERSION
 app.state.config.IMAGES_OPENAI_API_KEY = IMAGES_OPENAI_API_KEY
+app.state.config.IMAGES_OPENAI_API_PARAMS = IMAGES_OPENAI_API_PARAMS
 
 app.state.config.IMAGES_GEMINI_API_BASE_URL = IMAGES_GEMINI_API_BASE_URL
 app.state.config.IMAGES_GEMINI_API_KEY = IMAGES_GEMINI_API_KEY
-
-app.state.config.IMAGE_GENERATION_MODEL = IMAGE_GENERATION_MODEL
+app.state.config.IMAGES_GEMINI_ENDPOINT_METHOD = IMAGES_GEMINI_ENDPOINT_METHOD
 
 app.state.config.AUTOMATIC1111_BASE_URL = AUTOMATIC1111_BASE_URL
 app.state.config.AUTOMATIC1111_API_AUTH = AUTOMATIC1111_API_AUTH
-app.state.config.AUTOMATIC1111_CFG_SCALE = AUTOMATIC1111_CFG_SCALE
-app.state.config.AUTOMATIC1111_SAMPLER = AUTOMATIC1111_SAMPLER
-app.state.config.AUTOMATIC1111_SCHEDULER = AUTOMATIC1111_SCHEDULER
+app.state.config.AUTOMATIC1111_PARAMS = AUTOMATIC1111_PARAMS
+
 app.state.config.COMFYUI_BASE_URL = COMFYUI_BASE_URL
 app.state.config.COMFYUI_API_KEY = COMFYUI_API_KEY
 app.state.config.COMFYUI_WORKFLOW = COMFYUI_WORKFLOW
 app.state.config.COMFYUI_WORKFLOW_NODES = COMFYUI_WORKFLOW_NODES
 
-app.state.config.IMAGE_SIZE = IMAGE_SIZE
-app.state.config.IMAGE_STEPS = IMAGE_STEPS
+
+app.state.config.ENABLE_IMAGE_EDIT = ENABLE_IMAGE_EDIT
+app.state.config.IMAGE_EDIT_ENGINE = IMAGE_EDIT_ENGINE
+app.state.config.IMAGE_EDIT_MODEL = IMAGE_EDIT_MODEL
+app.state.config.IMAGE_EDIT_SIZE = IMAGE_EDIT_SIZE
+app.state.config.IMAGES_EDIT_OPENAI_API_BASE_URL = IMAGES_EDIT_OPENAI_API_BASE_URL
+app.state.config.IMAGES_EDIT_OPENAI_API_KEY = IMAGES_EDIT_OPENAI_API_KEY
+app.state.config.IMAGES_EDIT_OPENAI_API_VERSION = IMAGES_EDIT_OPENAI_API_VERSION
+app.state.config.IMAGES_EDIT_GEMINI_API_BASE_URL = IMAGES_EDIT_GEMINI_API_BASE_URL
+app.state.config.IMAGES_EDIT_GEMINI_API_KEY = IMAGES_EDIT_GEMINI_API_KEY
+app.state.config.IMAGES_EDIT_COMFYUI_BASE_URL = IMAGES_EDIT_COMFYUI_BASE_URL
+app.state.config.IMAGES_EDIT_COMFYUI_API_KEY = IMAGES_EDIT_COMFYUI_API_KEY
+app.state.config.IMAGES_EDIT_COMFYUI_WORKFLOW = IMAGES_EDIT_COMFYUI_WORKFLOW
+app.state.config.IMAGES_EDIT_COMFYUI_WORKFLOW_NODES = IMAGES_EDIT_COMFYUI_WORKFLOW_NODES
 
 
 ########################################
@@ -1099,6 +1141,12 @@ app.state.config.AUDIO_STT_AZURE_LOCALES = AUDIO_STT_AZURE_LOCALES
 app.state.config.AUDIO_STT_AZURE_BASE_URL = AUDIO_STT_AZURE_BASE_URL
 app.state.config.AUDIO_STT_AZURE_MAX_SPEAKERS = AUDIO_STT_AZURE_MAX_SPEAKERS
 
+app.state.config.AUDIO_STT_MISTRAL_API_KEY = AUDIO_STT_MISTRAL_API_KEY
+app.state.config.AUDIO_STT_MISTRAL_API_BASE_URL = AUDIO_STT_MISTRAL_API_BASE_URL
+app.state.config.AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS = (
+    AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS
+)
+
 app.state.config.TTS_ENGINE = AUDIO_TTS_ENGINE
 
 app.state.config.TTS_MODEL = AUDIO_TTS_MODEL
@@ -1160,6 +1208,7 @@ app.state.config.AUTOCOMPLETE_GENERATION_PROMPT_TEMPLATE = (
 app.state.config.AUTOCOMPLETE_GENERATION_INPUT_MAX_LENGTH = (
     AUTOCOMPLETE_GENERATION_INPUT_MAX_LENGTH
 )
+app.state.config.VOICE_MODE_PROMPT_TEMPLATE = VOICE_MODE_PROMPT_TEMPLATE
 
 
 ########################################
@@ -1168,7 +1217,11 @@ app.state.config.AUTOCOMPLETE_GENERATION_INPUT_MAX_LENGTH = (
 #
 ########################################
 
-app.state.MODELS = {}
+app.state.MODELS = MODELS
+
+# Add the middleware to the app
+if ENABLE_COMPRESSION_MIDDLEWARE:
+    app.add_middleware(CompressMiddleware)
 
 
 class RedirectMiddleware(BaseHTTPMiddleware):
@@ -1211,14 +1264,53 @@ class RedirectMiddleware(BaseHTTPMiddleware):
         return response
 
 
-# Add the middleware to the app
-if ENABLE_COMPRESSION_MIDDLEWARE:
-    app.add_middleware(CompressMiddleware)
-
 app.add_middleware(RedirectMiddleware)
 app.add_middleware(SecurityHeadersMiddleware)
 
 
+class APIKeyRestrictionMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        auth_header = request.headers.get("Authorization")
+        token = None
+
+        if auth_header:
+            scheme, token = auth_header.split(" ")
+
+        # Only apply restrictions if an sk- API key is used
+        if token and token.startswith("sk-"):
+            # Check if restrictions are enabled
+            if request.app.state.config.ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS:
+                allowed_paths = [
+                    path.strip()
+                    for path in str(
+                        request.app.state.config.API_KEYS_ALLOWED_ENDPOINTS
+                    ).split(",")
+                    if path.strip()
+                ]
+
+                request_path = request.url.path
+
+                # Match exact path or prefix path
+                is_allowed = any(
+                    request_path == allowed or request_path.startswith(allowed + "/")
+                    for allowed in allowed_paths
+                )
+
+                if not is_allowed:
+                    return JSONResponse(
+                        status_code=status.HTTP_403_FORBIDDEN,
+                        content={
+                            "detail": "API key not allowed to access this endpoint."
+                        },
+                    )
+
+        response = await call_next(request)
+        return response
+
+
+app.add_middleware(APIKeyRestrictionMiddleware)
+
+
 @app.middleware("http")
 async def commit_session_after_request(request: Request, call_next):
     response = await call_next(request)
@@ -1234,7 +1326,7 @@ async def check_url(request: Request, call_next):
         request.headers.get("Authorization")
     )
 
-    request.state.enable_api_key = app.state.config.ENABLE_API_KEY
+    request.state.enable_api_keys = app.state.config.ENABLE_API_KEYS
     response = await call_next(request)
     process_time = int(time.time()) - start_time
     response.headers["X-Process-Time"] = str(process_time)
@@ -1309,7 +1401,7 @@ app.include_router(
 app.include_router(utils.router, prefix="/api/v1/utils", tags=["utils"])
 
 # SCIM 2.0 API for identity management
-if SCIM_ENABLED:
+if ENABLE_SCIM:
     app.include_router(scim.router, prefix="/api/v1/scim/v2", tags=["scim"])
 
 
@@ -1346,6 +1438,10 @@ async def get_models(
         if "pipeline" in model and model["pipeline"].get("type", None) == "filter":
             continue
 
+        # Remove profile image URL to reduce payload size
+        if model.get("info", {}).get("meta", {}).get("profile_image_url"):
+            model["info"]["meta"].pop("profile_image_url", None)
+
         try:
             model_tags = [
                 tag.get("name")
@@ -1468,6 +1564,9 @@ async def chat_completion(
         reasoning_tags = form_data.get("params", {}).get("reasoning_tags")
 
         # Model Params
+        if model_info_params.get("stream_response") is not None:
+            form_data["stream"] = model_info_params.get("stream_response")
+
         if model_info_params.get("stream_delta_chunk_size"):
             stream_delta_chunk_size = model_info_params.get("stream_delta_chunk_size")
 
@@ -1478,6 +1577,7 @@ async def chat_completion(
             "user_id": user.id,
             "chat_id": form_data.pop("chat_id", None),
             "message_id": form_data.pop("id", None),
+            "parent_message_id": form_data.pop("parent_id", None),
             "session_id": form_data.pop("session_id", None),
             "filter_ids": form_data.pop("filter_ids", []),
             "tool_ids": form_data.get("tool_ids", None),
@@ -1534,6 +1634,7 @@ async def chat_completion(
                             metadata["chat_id"],
                             metadata["message_id"],
                             {
+                                "parentId": metadata.get("parent_message_id", None),
                                 "model": model_id,
                             },
                         )
@@ -1547,11 +1648,15 @@ async def chat_completion(
             log.info("Chat processing was cancelled")
             try:
                 event_emitter = get_event_emitter(metadata)
-                await event_emitter(
+                await asyncio.shield(
+                    event_emitter(
                         {"type": "chat:tasks:cancel"},
                     )
+                )
             except Exception as e:
                 pass
+            finally:
+                raise  # re-raise to ensure proper task cancellation handling
         except Exception as e:
             log.debug(f"Error processing chat payload: {e}")
             if metadata.get("chat_id") and metadata.get("message_id"):
@@ -1562,6 +1667,7 @@ async def chat_completion(
                             metadata["chat_id"],
                             metadata["message_id"],
                             {
+                                "parentId": metadata.get("parent_message_id", None),
                                 "error": {"content": str(e)},
                             },
                         )
@@ -1582,7 +1688,7 @@ async def chat_completion(
         finally:
             try:
                 if mcp_clients := metadata.get("mcp_clients"):
-                    for client in mcp_clients.values():
+                    for client in reversed(mcp_clients.values()):
                         await client.disconnect()
             except Exception as e:
                 log.debug(f"Error cleaning up: {e}")
@@ -1733,7 +1839,7 @@ async def get_app_config(request: Request):
             "auth_trusted_header": bool(app.state.AUTH_TRUSTED_EMAIL_HEADER),
             "enable_signup_password_confirmation": ENABLE_SIGNUP_PASSWORD_CONFIRMATION,
             "enable_ldap": app.state.config.ENABLE_LDAP,
-            "enable_api_key": app.state.config.ENABLE_API_KEY,
+            "enable_api_keys": app.state.config.ENABLE_API_KEYS,
             "enable_signup": app.state.config.ENABLE_SIGNUP,
             "enable_login_form": app.state.config.ENABLE_LOGIN_FORM,
             "enable_websocket": ENABLE_WEBSOCKET_SUPPORT,
@@ -1741,6 +1847,7 @@ async def get_app_config(request: Request):
             **(
                 {
                     "enable_direct_connections": app.state.config.ENABLE_DIRECT_CONNECTIONS,
+                    "enable_folders": app.state.config.ENABLE_FOLDERS,
                     "enable_channels": app.state.config.ENABLE_CHANNELS,
                     "enable_notes": app.state.config.ENABLE_NOTES,
                     "enable_web_search": app.state.config.ENABLE_WEB_SEARCH,
@@ -1771,6 +1878,7 @@ async def get_app_config(request: Request):
         **(
             {
                 "default_models": app.state.config.DEFAULT_MODELS,
+                "default_pinned_models": app.state.config.DEFAULT_PINNED_MODELS,
                 "default_prompt_suggestions": app.state.config.DEFAULT_PROMPT_SUGGESTIONS,
                 "user_count": user_count,
                 "code": {
@@ -1872,6 +1980,7 @@ async def update_webhook_url(form_data: UrlForm, user=Depends(get_admin_user)):
 async def get_app_version():
     return {
         "version": VERSION,
+        "deployment_id": DEPLOYMENT_ID,
     }
 
 
@@ -1911,7 +2020,10 @@ async def get_current_usage(user=Depends(get_verified_user)):
     This is an experimental endpoint and subject to change.
     """
     try:
-        return {"model_ids": get_models_in_use(), "user_ids": get_active_user_ids()}
+        return {
+            "model_ids": get_models_in_use(),
+            "user_count": Users.get_active_user_count(),
+        }
     except Exception as e:
         log.error(f"Error getting usage statistics: {e}")
         raise HTTPException(status_code=500, detail="Internal Server Error")
@@ -1928,18 +2040,26 @@ if len(app.state.config.TOOL_SERVER_CONNECTIONS) > 0:
         if tool_server_connection.get("type", "openapi") == "mcp":
             server_id = tool_server_connection.get("info", {}).get("id")
             auth_type = tool_server_connection.get("auth_type", "none")
+
             if server_id and auth_type == "oauth_2.1":
                 oauth_client_info = tool_server_connection.get("info", {}).get(
                     "oauth_client_info", ""
                 )
 
+                try:
                     oauth_client_info = decrypt_data(oauth_client_info)
                     app.state.oauth_client_manager.add_client(
-                    f"mcp:{server_id}", OAuthClientInformationFull(**oauth_client_info)
+                        f"mcp:{server_id}",
+                        OAuthClientInformationFull(**oauth_client_info),
                     )
+                except Exception as e:
+                    log.error(
+                        f"Error adding OAuth client for MCP tool server {server_id}: {e}"
+                    )
+                    pass
 
 try:
-    if REDIS_URL:
+    if ENABLE_STAR_SESSIONS_MIDDLEWARE:
         redis_session_store = RedisStore(
             url=REDIS_URL,
             prefix=(f"{REDIS_KEY_PREFIX}:session:" if REDIS_KEY_PREFIX else "session:"),
@@ -1966,6 +2086,64 @@ except Exception as e:
     )
 
 
+async def register_client(self, request, client_id: str) -> bool:
+    server_type, server_id = client_id.split(":", 1)
+
+    connection = None
+    connection_idx = None
+
+    for idx, conn in enumerate(request.app.state.config.TOOL_SERVER_CONNECTIONS or []):
+        if conn.get("type", "openapi") == server_type:
+            info = conn.get("info", {})
+            if info.get("id") == server_id:
+                connection = conn
+                connection_idx = idx
+                break
+
+    if connection is None or connection_idx is None:
+        log.warning(
+            f"Unable to locate MCP tool server configuration for client {client_id} during re-registration"
+        )
+        return False
+
+    server_url = connection.get("url")
+    oauth_server_key = (connection.get("config") or {}).get("oauth_server_key")
+
+    try:
+        oauth_client_info = (
+            await get_oauth_client_info_with_dynamic_client_registration(
+                request,
+                client_id,
+                server_url,
+                oauth_server_key,
+            )
+        )
+    except Exception as e:
+        log.error(f"Dynamic client re-registration failed for {client_id}: {e}")
+        return False
+
+    try:
+        request.app.state.config.TOOL_SERVER_CONNECTIONS[connection_idx] = {
+            **connection,
+            "info": {
+                **connection.get("info", {}),
+                "oauth_client_info": encrypt_data(
+                    oauth_client_info.model_dump(mode="json")
+                ),
+            },
+        }
+    except Exception as e:
+        log.error(
+            f"Failed to persist updated OAuth client info for tool server {client_id}: {e}"
+        )
+        return False
+
+    oauth_client_manager.remove_client(client_id)
+    oauth_client_manager.add_client(client_id, oauth_client_info)
+    log.info(f"Re-registered OAuth client {client_id} for tool server")
+    return True
+
+
 @app.get("/oauth/clients/{client_id}/authorize")
 async def oauth_client_authorize(
     client_id: str,
@@ -1973,6 +2151,41 @@ async def oauth_client_authorize(
     response: Response,
     user=Depends(get_verified_user),
 ):
+    # ensure_valid_client_registration
+    client = oauth_client_manager.get_client(client_id)
+    client_info = oauth_client_manager.get_client_info(client_id)
+    if client is None or client_info is None:
+        raise HTTPException(status.HTTP_404_NOT_FOUND)
+
+    if not await oauth_client_manager._preflight_authorization_url(client, client_info):
+        log.info(
+            "Detected invalid OAuth client %s; attempting re-registration",
+            client_id,
+        )
+
+        registered = await register_client(request, client_id)
+        if not registered:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="Failed to re-register OAuth client",
+            )
+
+        client = oauth_client_manager.get_client(client_id)
+        client_info = oauth_client_manager.get_client_info(client_id)
+        if client is None or client_info is None:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="OAuth client unavailable after re-registration",
+            )
+
+        if not await oauth_client_manager._preflight_authorization_url(
+            client, client_info
+        ):
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail="OAuth client registration is still invalid after re-registration",
+            )
+
     return await oauth_client_manager.handle_authorize(request, client_id=client_id)
 
 

backend/open_webui/migrations/versions/2f1211949ecc_update_message_and_channel_member_table.py

@@ -0,0 +1,103 @@
+"""Update messages and channel member table
+
+Revision ID: 2f1211949ecc
+Revises: 37f288994c47
+Create Date: 2025-11-27 03:07:56.200231
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import open_webui.internal.db
+
+
+# revision identifiers, used by Alembic.
+revision: str = "2f1211949ecc"
+down_revision: Union[str, None] = "37f288994c47"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # New columns to be added to channel_member table
+    op.add_column("channel_member", sa.Column("status", sa.Text(), nullable=True))
+    op.add_column(
+        "channel_member",
+        sa.Column(
+            "is_active",
+            sa.Boolean(),
+            nullable=False,
+            default=True,
+            server_default=sa.sql.expression.true(),
+        ),
+    )
+
+    op.add_column(
+        "channel_member",
+        sa.Column(
+            "is_channel_muted",
+            sa.Boolean(),
+            nullable=False,
+            default=False,
+            server_default=sa.sql.expression.false(),
+        ),
+    )
+    op.add_column(
+        "channel_member",
+        sa.Column(
+            "is_channel_pinned",
+            sa.Boolean(),
+            nullable=False,
+            default=False,
+            server_default=sa.sql.expression.false(),
+        ),
+    )
+
+    op.add_column("channel_member", sa.Column("data", sa.JSON(), nullable=True))
+    op.add_column("channel_member", sa.Column("meta", sa.JSON(), nullable=True))
+
+    op.add_column(
+        "channel_member", sa.Column("joined_at", sa.BigInteger(), nullable=False)
+    )
+    op.add_column(
+        "channel_member", sa.Column("left_at", sa.BigInteger(), nullable=True)
+    )
+
+    op.add_column(
+        "channel_member", sa.Column("last_read_at", sa.BigInteger(), nullable=True)
+    )
+
+    op.add_column(
+        "channel_member", sa.Column("updated_at", sa.BigInteger(), nullable=True)
+    )
+
+    # New columns to be added to message table
+    op.add_column(
+        "message",
+        sa.Column(
+            "is_pinned",
+            sa.Boolean(),
+            nullable=False,
+            default=False,
+            server_default=sa.sql.expression.false(),
+        ),
+    )
+    op.add_column("message", sa.Column("pinned_at", sa.BigInteger(), nullable=True))
+    op.add_column("message", sa.Column("pinned_by", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("channel_member", "updated_at")
+    op.drop_column("channel_member", "last_read_at")
+
+    op.drop_column("channel_member", "meta")
+    op.drop_column("channel_member", "data")
+
+    op.drop_column("channel_member", "is_channel_pinned")
+    op.drop_column("channel_member", "is_channel_muted")
+
+    op.drop_column("message", "pinned_by")
+    op.drop_column("message", "pinned_at")
+    op.drop_column("message", "is_pinned")

backend/open_webui/migrations/versions/37f288994c47_add_group_member_table.py

@@ -0,0 +1,146 @@
+"""add_group_member_table
+
+Revision ID: 37f288994c47
+Revises: a5c220713937
+Create Date: 2025-11-17 03:45:25.123939
+
+"""
+
+import uuid
+import time
+import json
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "37f288994c47"
+down_revision: Union[str, None] = "a5c220713937"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # 1. Create new table
+    op.create_table(
+        "group_member",
+        sa.Column("id", sa.Text(), primary_key=True, unique=True, nullable=False),
+        sa.Column(
+            "group_id",
+            sa.Text(),
+            sa.ForeignKey("group.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "user_id",
+            sa.Text(),
+            sa.ForeignKey("user.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("created_at", sa.BigInteger(), nullable=True),
+        sa.Column("updated_at", sa.BigInteger(), nullable=True),
+        sa.UniqueConstraint("group_id", "user_id", name="uq_group_member_group_user"),
+    )
+
+    connection = op.get_bind()
+
+    # 2. Read existing group with user_ids JSON column
+    group_table = sa.Table(
+        "group",
+        sa.MetaData(),
+        sa.Column("id", sa.Text()),
+        sa.Column("user_ids", sa.JSON()),  # JSON stored as text in SQLite + PG
+    )
+
+    results = connection.execute(
+        sa.select(group_table.c.id, group_table.c.user_ids)
+    ).fetchall()
+
+    print(results)
+
+    # 3. Insert members into group_member table
+    gm_table = sa.Table(
+        "group_member",
+        sa.MetaData(),
+        sa.Column("id", sa.Text()),
+        sa.Column("group_id", sa.Text()),
+        sa.Column("user_id", sa.Text()),
+        sa.Column("created_at", sa.BigInteger()),
+        sa.Column("updated_at", sa.BigInteger()),
+    )
+
+    now = int(time.time())
+    for group_id, user_ids in results:
+        if not user_ids:
+            continue
+
+        if isinstance(user_ids, str):
+            try:
+                user_ids = json.loads(user_ids)
+            except Exception:
+                continue  # skip invalid JSON
+
+        if not isinstance(user_ids, list):
+            continue
+
+        rows = [
+            {
+                "id": str(uuid.uuid4()),
+                "group_id": group_id,
+                "user_id": uid,
+                "created_at": now,
+                "updated_at": now,
+            }
+            for uid in user_ids
+        ]
+
+        if rows:
+            connection.execute(gm_table.insert(), rows)
+
+    # 4. Optionally drop the old column
+    with op.batch_alter_table("group") as batch:
+        batch.drop_column("user_ids")
+
+
+def downgrade():
+    # Reverse: restore user_ids column
+    with op.batch_alter_table("group") as batch:
+        batch.add_column(sa.Column("user_ids", sa.JSON()))
+
+    connection = op.get_bind()
+    gm_table = sa.Table(
+        "group_member",
+        sa.MetaData(),
+        sa.Column("group_id", sa.Text()),
+        sa.Column("user_id", sa.Text()),
+        sa.Column("created_at", sa.BigInteger()),
+        sa.Column("updated_at", sa.BigInteger()),
+    )
+
+    group_table = sa.Table(
+        "group",
+        sa.MetaData(),
+        sa.Column("id", sa.Text()),
+        sa.Column("user_ids", sa.JSON()),
+    )
+
+    # Build JSON arrays again
+    results = connection.execute(sa.select(group_table.c.id)).fetchall()
+
+    for (group_id,) in results:
+        members = connection.execute(
+            sa.select(gm_table.c.user_id).where(gm_table.c.group_id == group_id)
+        ).fetchall()
+
+        member_ids = [m[0] for m in members]
+
+        connection.execute(
+            group_table.update()
+            .where(group_table.c.id == group_id)
+            .values(user_ids=member_ids)
+        )
+
+    # Drop the new table
+    op.drop_table("group_member")

backend/open_webui/migrations/versions/38d63c18f30f_add_oauth_session_table.py

@@ -20,18 +20,46 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
+    # Ensure 'id' column in 'user' table is unique and primary key (ForeignKey constraint)
+    inspector = sa.inspect(op.get_bind())
+    columns = inspector.get_columns("user")
+
+    pk_columns = inspector.get_pk_constraint("user")["constrained_columns"]
+    id_column = next((col for col in columns if col["name"] == "id"), None)
+
+    if id_column and not id_column.get("unique", False):
+        unique_constraints = inspector.get_unique_constraints("user")
+        unique_columns = {tuple(u["column_names"]) for u in unique_constraints}
+
+        with op.batch_alter_table("user") as batch_op:
+            # If primary key is wrong, drop it
+            if pk_columns and pk_columns != ["id"]:
+                batch_op.drop_constraint(
+                    inspector.get_pk_constraint("user")["name"], type_="primary"
+                )
+
+            # Add unique constraint if missing
+            if ("id",) not in unique_columns:
+                batch_op.create_unique_constraint("uq_user_id", ["id"])
+
+            # Re-create correct primary key
+            batch_op.create_primary_key("pk_user_id", ["id"])
+
     # Create oauth_session table
     op.create_table(
         "oauth_session",
-        sa.Column("id", sa.Text(), nullable=False),
-        sa.Column("user_id", sa.Text(), nullable=False),
+        sa.Column("id", sa.Text(), primary_key=True, nullable=False, unique=True),
+        sa.Column(
+            "user_id",
+            sa.Text(),
+            sa.ForeignKey("user.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
         sa.Column("provider", sa.Text(), nullable=False),
         sa.Column("token", sa.Text(), nullable=False),
         sa.Column("expires_at", sa.BigInteger(), nullable=False),
         sa.Column("created_at", sa.BigInteger(), nullable=False),
         sa.Column("updated_at", sa.BigInteger(), nullable=False),
-        sa.PrimaryKeyConstraint("id"),
-        sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
     )
 
     # Create indexes for better performance

backend/open_webui/migrations/versions/b10670c03dd5_update_user_table.py

@@ -0,0 +1,251 @@
+"""Update user table
+
+Revision ID: b10670c03dd5
+Revises: 2f1211949ecc
+Create Date: 2025-11-28 04:55:31.737538
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+import open_webui.internal.db
+import json
+import time
+
+# revision identifiers, used by Alembic.
+revision: str = "b10670c03dd5"
+down_revision: Union[str, None] = "2f1211949ecc"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _drop_sqlite_indexes_for_column(table_name, column_name, conn):
+    """
+    SQLite requires manual removal of any indexes referencing a column
+    before ALTER TABLE ... DROP COLUMN can succeed.
+    """
+    indexes = conn.execute(sa.text(f"PRAGMA index_list('{table_name}')")).fetchall()
+
+    for idx in indexes:
+        index_name = idx[1]  # index name
+        # Get indexed columns
+        idx_info = conn.execute(
+            sa.text(f"PRAGMA index_info('{index_name}')")
+        ).fetchall()
+
+        indexed_cols = [row[2] for row in idx_info]  # col names
+        if column_name in indexed_cols:
+            conn.execute(sa.text(f"DROP INDEX IF EXISTS {index_name}"))
+
+
+def _convert_column_to_json(table: str, column: str):
+    conn = op.get_bind()
+    dialect = conn.dialect.name
+
+    # SQLite cannot ALTER COLUMN → must recreate column
+    if dialect == "sqlite":
+        # 1. Add temporary column
+        op.add_column(table, sa.Column(f"{column}_json", sa.JSON(), nullable=True))
+
+        # 2. Load old data
+        rows = conn.execute(sa.text(f'SELECT id, {column} FROM "{table}"')).fetchall()
+
+        for row in rows:
+            uid, raw = row
+            if raw is None:
+                parsed = None
+            else:
+                try:
+                    parsed = json.loads(raw)
+                except Exception:
+                    parsed = None  # fallback safe behavior
+
+            conn.execute(
+                sa.text(f'UPDATE "{table}" SET {column}_json = :val WHERE id = :id'),
+                {"val": json.dumps(parsed) if parsed else None, "id": uid},
+            )
+
+        # 3. Drop old TEXT column
+        op.drop_column(table, column)
+
+        # 4. Rename new JSON column → original name
+        op.alter_column(table, f"{column}_json", new_column_name=column)
+
+    else:
+        # PostgreSQL supports direct CAST
+        op.alter_column(
+            table,
+            column,
+            type_=sa.JSON(),
+            postgresql_using=f"{column}::json",
+        )
+
+
+def _convert_column_to_text(table: str, column: str):
+    conn = op.get_bind()
+    dialect = conn.dialect.name
+
+    if dialect == "sqlite":
+        op.add_column(table, sa.Column(f"{column}_text", sa.Text(), nullable=True))
+
+        rows = conn.execute(sa.text(f'SELECT id, {column} FROM "{table}"')).fetchall()
+
+        for uid, raw in rows:
+            conn.execute(
+                sa.text(f'UPDATE "{table}" SET {column}_text = :val WHERE id = :id'),
+                {"val": json.dumps(raw) if raw else None, "id": uid},
+            )
+
+        op.drop_column(table, column)
+        op.alter_column(table, f"{column}_text", new_column_name=column)
+
+    else:
+        op.alter_column(
+            table,
+            column,
+            type_=sa.Text(),
+            postgresql_using=f"to_json({column})::text",
+        )
+
+
+def upgrade() -> None:
+    op.add_column(
+        "user", sa.Column("profile_banner_image_url", sa.Text(), nullable=True)
+    )
+    op.add_column("user", sa.Column("timezone", sa.String(), nullable=True))
+
+    op.add_column("user", sa.Column("presence_state", sa.String(), nullable=True))
+    op.add_column("user", sa.Column("status_emoji", sa.String(), nullable=True))
+    op.add_column("user", sa.Column("status_message", sa.Text(), nullable=True))
+    op.add_column(
+        "user", sa.Column("status_expires_at", sa.BigInteger(), nullable=True)
+    )
+
+    op.add_column("user", sa.Column("oauth", sa.JSON(), nullable=True))
+
+    # Convert info (TEXT/JSONField) → JSON
+    _convert_column_to_json("user", "info")
+    # Convert settings (TEXT/JSONField) → JSON
+    _convert_column_to_json("user", "settings")
+
+    op.create_table(
+        "api_key",
+        sa.Column("id", sa.Text(), primary_key=True, unique=True),
+        sa.Column("user_id", sa.Text(), sa.ForeignKey("user.id", ondelete="CASCADE")),
+        sa.Column("key", sa.Text(), unique=True, nullable=False),
+        sa.Column("data", sa.JSON(), nullable=True),
+        sa.Column("expires_at", sa.BigInteger(), nullable=True),
+        sa.Column("last_used_at", sa.BigInteger(), nullable=True),
+        sa.Column("created_at", sa.BigInteger(), nullable=False),
+        sa.Column("updated_at", sa.BigInteger(), nullable=False),
+    )
+
+    conn = op.get_bind()
+    users = conn.execute(
+        sa.text('SELECT id, oauth_sub FROM "user" WHERE oauth_sub IS NOT NULL')
+    ).fetchall()
+
+    for uid, oauth_sub in users:
+        if oauth_sub:
+            # Example formats supported:
+            #   provider@sub
+            #   plain sub (stored as {"oidc": {"sub": sub}})
+            if "@" in oauth_sub:
+                provider, sub = oauth_sub.split("@", 1)
+            else:
+                provider, sub = "oidc", oauth_sub
+
+            oauth_json = json.dumps({provider: {"sub": sub}})
+            conn.execute(
+                sa.text('UPDATE "user" SET oauth = :oauth WHERE id = :id'),
+                {"oauth": oauth_json, "id": uid},
+            )
+
+    users_with_keys = conn.execute(
+        sa.text('SELECT id, api_key FROM "user" WHERE api_key IS NOT NULL')
+    ).fetchall()
+    now = int(time.time())
+
+    for uid, api_key in users_with_keys:
+        if api_key:
+            conn.execute(
+                sa.text(
+                    """
+                    INSERT INTO api_key (id, user_id, key, created_at, updated_at)
+                    VALUES (:id, :user_id, :key, :created_at, :updated_at)
+                """
+                ),
+                {
+                    "id": f"key_{uid}",
+                    "user_id": uid,
+                    "key": api_key,
+                    "created_at": now,
+                    "updated_at": now,
+                },
+            )
+
+    if conn.dialect.name == "sqlite":
+        _drop_sqlite_indexes_for_column("user", "api_key", conn)
+        _drop_sqlite_indexes_for_column("user", "oauth_sub", conn)
+
+    with op.batch_alter_table("user") as batch_op:
+        batch_op.drop_column("api_key")
+        batch_op.drop_column("oauth_sub")
+
+
+def downgrade() -> None:
+    # --- 1. Restore old oauth_sub column ---
+    op.add_column("user", sa.Column("oauth_sub", sa.Text(), nullable=True))
+
+    conn = op.get_bind()
+    users = conn.execute(
+        sa.text('SELECT id, oauth FROM "user" WHERE oauth IS NOT NULL')
+    ).fetchall()
+
+    for uid, oauth in users:
+        try:
+            data = json.loads(oauth)
+            provider = list(data.keys())[0]
+            sub = data[provider].get("sub")
+            oauth_sub = f"{provider}@{sub}"
+        except Exception:
+            oauth_sub = None
+
+        conn.execute(
+            sa.text('UPDATE "user" SET oauth_sub = :oauth_sub WHERE id = :id'),
+            {"oauth_sub": oauth_sub, "id": uid},
+        )
+
+    op.drop_column("user", "oauth")
+
+    # --- 2. Restore api_key field ---
+    op.add_column("user", sa.Column("api_key", sa.String(), nullable=True))
+
+    # Restore values from api_key
+    keys = conn.execute(sa.text("SELECT user_id, key FROM api_key")).fetchall()
+    for uid, key in keys:
+        conn.execute(
+            sa.text('UPDATE "user" SET api_key = :key WHERE id = :id'),
+            {"key": key, "id": uid},
+        )
+
+    # Drop new table
+    op.drop_table("api_key")
+
+    with op.batch_alter_table("user") as batch_op:
+        batch_op.drop_column("profile_banner_image_url")
+        batch_op.drop_column("timezone")
+
+        batch_op.drop_column("presence_state")
+        batch_op.drop_column("status_emoji")
+        batch_op.drop_column("status_message")
+        batch_op.drop_column("status_expires_at")
+
+    # Convert info (JSON) → TEXT
+    _convert_column_to_text("user", "info")
+    # Convert settings (JSON) → TEXT
+    _convert_column_to_text("user", "settings")

backend/open_webui/models/auths.py

@@ -3,11 +3,10 @@ import uuid
 from typing import Optional
 
 from open_webui.internal.db import Base, get_db
-from open_webui.models.users import UserModel, Users
+from open_webui.models.users import UserModel, UserProfileImageResponse, Users
 from open_webui.env import SRC_LOG_LEVELS
 from pydantic import BaseModel
 from sqlalchemy import Boolean, Column, String, Text
-from open_webui.utils.auth import verify_password
 
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["MODELS"])
@@ -20,7 +19,7 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 class Auth(Base):
     __tablename__ = "auth"
 
-    id = Column(String, primary_key=True)
+    id = Column(String, primary_key=True, unique=True)
     email = Column(String)
     password = Column(Text)
     active = Column(Boolean)
@@ -47,15 +46,7 @@ class ApiKey(BaseModel):
     api_key: Optional[str] = None
 
 
-class UserResponse(BaseModel):
-    id: str
-    email: str
-    name: str
-    role: str
-    profile_image_url: str
-
-
-class SigninResponse(Token, UserResponse):
+class SigninResponse(Token, UserProfileImageResponse):
     pass
 
 
@@ -97,7 +88,7 @@ class AuthsTable:
         name: str,
         profile_image_url: str = "/user.png",
         role: str = "pending",
-        oauth_sub: Optional[str] = None,
+        oauth: Optional[dict] = None,
     ) -> Optional[UserModel]:
         with get_db() as db:
             log.info("insert_new_auth")
@@ -111,7 +102,7 @@ class AuthsTable:
             db.add(result)
 
             user = Users.insert_new_user(
-                id, name, email, profile_image_url, role, oauth_sub
+                id, name, email, profile_image_url, role, oauth=oauth
             )
 
             db.commit()
@@ -122,7 +113,9 @@ class AuthsTable:
             else:
                 return None
 
-    def authenticate_user(self, email: str, password: str) -> Optional[UserModel]:
+    def authenticate_user(
+        self, email: str, verify_password: callable
+    ) -> Optional[UserModel]:
         log.info(f"authenticate_user: {email}")
 
         user = Users.get_user_by_email(email)
@@ -133,7 +126,7 @@ class AuthsTable:
             with get_db() as db:
                 auth = db.query(Auth).filter_by(id=user.id, active=True).first()
                 if auth:
-                    if verify_password(password, auth.password):
+                    if verify_password(auth.password):
                         return user
                     else:
                         return None

backend/open_webui/models/channels.py

@@ -7,7 +7,7 @@ from open_webui.internal.db import Base, get_db
 from open_webui.utils.access_control import has_access
 
 from pydantic import BaseModel, ConfigDict
-from sqlalchemy import BigInteger, Boolean, Column, String, Text, JSON
+from sqlalchemy import BigInteger, Boolean, Column, String, Text, JSON, case
 from sqlalchemy import or_, func, select, and_, text
 from sqlalchemy.sql import exists
 
@@ -19,7 +19,7 @@ from sqlalchemy.sql import exists
 class Channel(Base):
     __tablename__ = "channel"
 
-    id = Column(Text, primary_key=True)
+    id = Column(Text, primary_key=True, unique=True)
     user_id = Column(Text)
     type = Column(Text, nullable=True)
 
@@ -48,8 +48,58 @@ class ChannelModel(BaseModel):
     meta: Optional[dict] = None
     access_control: Optional[dict] = None
 
-    created_at: int  # timestamp in epoch
-    updated_at: int  # timestamp in epoch
+    created_at: int  # timestamp in epoch (time_ns)
+    updated_at: int  # timestamp in epoch (time_ns)
+
+
+class ChannelMember(Base):
+    __tablename__ = "channel_member"
+
+    id = Column(Text, primary_key=True, unique=True)
+    channel_id = Column(Text, nullable=False)
+    user_id = Column(Text, nullable=False)
+
+    status = Column(Text, nullable=True)
+    is_active = Column(Boolean, nullable=False, default=True)
+
+    is_channel_muted = Column(Boolean, nullable=False, default=False)
+    is_channel_pinned = Column(Boolean, nullable=False, default=False)
+
+    data = Column(JSON, nullable=True)
+    meta = Column(JSON, nullable=True)
+
+    joined_at = Column(BigInteger)
+    left_at = Column(BigInteger, nullable=True)
+
+    last_read_at = Column(BigInteger, nullable=True)
+
+    created_at = Column(BigInteger)
+    updated_at = Column(BigInteger)
+
+
+class ChannelMemberModel(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
+    id: str
+    channel_id: str
+    user_id: str
+
+    status: Optional[str] = None
+    is_active: bool = True
+
+    is_channel_muted: bool = False
+    is_channel_pinned: bool = False
+
+    data: Optional[dict] = None
+    meta: Optional[dict] = None
+
+    joined_at: Optional[int] = None  # timestamp in epoch (time_ns)
+    left_at: Optional[int] = None  # timestamp in epoch (time_ns)
+
+    last_read_at: Optional[int] = None  # timestamp in epoch (time_ns)
+
+    created_at: Optional[int] = None  # timestamp in epoch (time_ns)
+    updated_at: Optional[int] = None  # timestamp in epoch (time_ns)
 
 
 ####################
@@ -59,25 +109,28 @@ class ChannelModel(BaseModel):
 
 class ChannelResponse(ChannelModel):
     write_access: bool = False
+    user_count: Optional[int] = None
 
 
 class ChannelForm(BaseModel):
+    type: Optional[str] = None
     name: str
     description: Optional[str] = None
     data: Optional[dict] = None
     meta: Optional[dict] = None
     access_control: Optional[dict] = None
+    user_ids: Optional[list[str]] = None
 
 
 class ChannelTable:
     def insert_new_channel(
-        self, type: Optional[str], form_data: ChannelForm, user_id: str
+        self, form_data: ChannelForm, user_id: str
     ) -> Optional[ChannelModel]:
         with get_db() as db:
             channel = ChannelModel(
                 **{
                     **form_data.model_dump(),
-                    "type": type,
+                    "type": form_data.type if form_data.type else None,
                     "name": form_data.name.lower(),
                     "id": str(uuid.uuid4()),
                     "user_id": user_id,
@@ -85,9 +138,34 @@ class ChannelTable:
                     "updated_at": int(time.time_ns()),
                 }
             )
-
             new_channel = Channel(**channel.model_dump())
 
+            if form_data.type == "dm":
+                # For direct message channels, automatically add the specified users as members
+                user_ids = form_data.user_ids or []
+                if user_id not in user_ids:
+                    user_ids.append(user_id)  # Ensure the creator is also a member
+
+                for uid in user_ids:
+                    channel_member = ChannelMemberModel(
+                        **{
+                            "id": str(uuid.uuid4()),
+                            "channel_id": channel.id,
+                            "user_id": uid,
+                            "status": "joined",
+                            "is_active": True,
+                            "is_channel_muted": False,
+                            "is_channel_pinned": False,
+                            "joined_at": int(time.time_ns()),
+                            "left_at": None,
+                            "last_read_at": int(time.time_ns()),
+                            "created_at": int(time.time_ns()),
+                            "updated_at": int(time.time_ns()),
+                        }
+                    )
+                    new_membership = ChannelMember(**channel_member.model_dump())
+                    db.add(new_membership)
+
             db.add(new_channel)
             db.commit()
             return channel
@@ -101,13 +179,211 @@ class ChannelTable:
         self, user_id: str, permission: str = "read"
     ) -> list[ChannelModel]:
         channels = self.get_channels()
+
+        channel_list = []
+        for channel in channels:
+            if channel.type == "dm":
+                membership = self.get_member_by_channel_and_user_id(channel.id, user_id)
+                if membership and membership.is_active:
+                    channel_list.append(channel)
+            else:
+                if channel.user_id == user_id or has_access(
+                    user_id, permission, channel.access_control
+                ):
+                    channel_list.append(channel)
+
+        return channel_list
+
+    def get_dm_channel_by_user_ids(self, user_ids: list[str]) -> Optional[ChannelModel]:
+        with get_db() as db:
+            # Ensure uniqueness in case a list with duplicates is passed
+            unique_user_ids = list(set(user_ids))
+
+            match_count = func.sum(
+                case(
+                    (ChannelMember.user_id.in_(unique_user_ids), 1),
+                    else_=0,
+                )
+            )
+
+            subquery = (
+                db.query(ChannelMember.channel_id)
+                .group_by(ChannelMember.channel_id)
+                # 1. Channel must have exactly len(user_ids) members
+                .having(func.count(ChannelMember.user_id) == len(unique_user_ids))
+                # 2. All those members must be in unique_user_ids
+                .having(match_count == len(unique_user_ids))
+                .subquery()
+            )
+
+            channel = (
+                db.query(Channel)
+                .filter(
+                    Channel.id.in_(subquery),
+                    Channel.type == "dm",
+                )
+                .first()
+            )
+
+            return ChannelModel.model_validate(channel) if channel else None
+
+    def join_channel(
+        self, channel_id: str, user_id: str
+    ) -> Optional[ChannelMemberModel]:
+        with get_db() as db:
+            # Check if the membership already exists
+            existing_membership = (
+                db.query(ChannelMember)
+                .filter(
+                    ChannelMember.channel_id == channel_id,
+                    ChannelMember.user_id == user_id,
+                )
+                .first()
+            )
+            if existing_membership:
+                return ChannelMemberModel.model_validate(existing_membership)
+
+            # Create new membership
+            channel_member = ChannelMemberModel(
+                **{
+                    "id": str(uuid.uuid4()),
+                    "channel_id": channel_id,
+                    "user_id": user_id,
+                    "status": "joined",
+                    "is_active": True,
+                    "is_channel_muted": False,
+                    "is_channel_pinned": False,
+                    "joined_at": int(time.time_ns()),
+                    "left_at": None,
+                    "last_read_at": int(time.time_ns()),
+                    "created_at": int(time.time_ns()),
+                    "updated_at": int(time.time_ns()),
+                }
+            )
+            new_membership = ChannelMember(**channel_member.model_dump())
+
+            db.add(new_membership)
+            db.commit()
+            return channel_member
+
+    def leave_channel(self, channel_id: str, user_id: str) -> bool:
+        with get_db() as db:
+            membership = (
+                db.query(ChannelMember)
+                .filter(
+                    ChannelMember.channel_id == channel_id,
+                    ChannelMember.user_id == user_id,
+                )
+                .first()
+            )
+            if not membership:
+                return False
+
+            membership.status = "left"
+            membership.is_active = False
+            membership.left_at = int(time.time_ns())
+            membership.updated_at = int(time.time_ns())
+
+            db.commit()
+            return True
+
+    def get_member_by_channel_and_user_id(
+        self, channel_id: str, user_id: str
+    ) -> Optional[ChannelMemberModel]:
+        with get_db() as db:
+            membership = (
+                db.query(ChannelMember)
+                .filter(
+                    ChannelMember.channel_id == channel_id,
+                    ChannelMember.user_id == user_id,
+                )
+                .first()
+            )
+            return ChannelMemberModel.model_validate(membership) if membership else None
+
+    def get_members_by_channel_id(self, channel_id: str) -> list[ChannelMemberModel]:
+        with get_db() as db:
+            memberships = (
+                db.query(ChannelMember)
+                .filter(ChannelMember.channel_id == channel_id)
+                .all()
+            )
             return [
-            channel
-            for channel in channels
-            if channel.user_id == user_id
-            or has_access(user_id, permission, channel.access_control)
+                ChannelMemberModel.model_validate(membership)
+                for membership in memberships
             ]
 
+    def pin_channel(self, channel_id: str, user_id: str, is_pinned: bool) -> bool:
+        with get_db() as db:
+            membership = (
+                db.query(ChannelMember)
+                .filter(
+                    ChannelMember.channel_id == channel_id,
+                    ChannelMember.user_id == user_id,
+                )
+                .first()
+            )
+            if not membership:
+                return False
+
+            membership.is_channel_pinned = is_pinned
+            membership.updated_at = int(time.time_ns())
+
+            db.commit()
+            return True
+
+    def update_member_last_read_at(self, channel_id: str, user_id: str) -> bool:
+        with get_db() as db:
+            membership = (
+                db.query(ChannelMember)
+                .filter(
+                    ChannelMember.channel_id == channel_id,
+                    ChannelMember.user_id == user_id,
+                )
+                .first()
+            )
+            if not membership:
+                return False
+
+            membership.last_read_at = int(time.time_ns())
+            membership.updated_at = int(time.time_ns())
+
+            db.commit()
+            return True
+
+    def update_member_active_status(
+        self, channel_id: str, user_id: str, is_active: bool
+    ) -> bool:
+        with get_db() as db:
+            membership = (
+                db.query(ChannelMember)
+                .filter(
+                    ChannelMember.channel_id == channel_id,
+                    ChannelMember.user_id == user_id,
+                )
+                .first()
+            )
+            if not membership:
+                return False
+
+            membership.is_active = is_active
+            membership.updated_at = int(time.time_ns())
+
+            db.commit()
+            return True
+
+    def is_user_channel_member(self, channel_id: str, user_id: str) -> bool:
+        with get_db() as db:
+            membership = (
+                db.query(ChannelMember)
+                .filter(
+                    ChannelMember.channel_id == channel_id,
+                    ChannelMember.user_id == user_id,
+                )
+                .first()
+            )
+            return membership is not None
+
     def get_channel_by_id(self, id: str) -> Optional[ChannelModel]:
         with get_db() as db:
             channel = db.query(Channel).filter(Channel.id == id).first()

backend/open_webui/models/chats.py

@@ -26,7 +26,7 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 class Chat(Base):
     __tablename__ = "chat"
 
-    id = Column(String, primary_key=True)
+    id = Column(String, primary_key=True, unique=True)
     user_id = Column(String)
     title = Column(Text)
     chat = Column(JSON)
@@ -92,6 +92,10 @@ class ChatImportForm(ChatForm):
     updated_at: Optional[int] = None
 
 
+class ChatsImportForm(BaseModel):
+    chats: list[ChatImportForm]
+
+
 class ChatTitleMessagesForm(BaseModel):
     title: str
     messages: list[dict]
@@ -123,6 +127,43 @@ class ChatTitleIdResponse(BaseModel):
 
 
 class ChatTable:
+    def _clean_null_bytes(self, obj):
+        """
+        Recursively remove actual null bytes (\x00) and unicode escape \\u0000
+        from strings inside dict/list structures.
+        Safe for JSON objects.
+        """
+        if isinstance(obj, str):
+            return obj.replace("\x00", "").replace("\u0000", "")
+        elif isinstance(obj, dict):
+            return {k: self._clean_null_bytes(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [self._clean_null_bytes(v) for v in obj]
+        return obj
+
+    def _sanitize_chat_row(self, chat_item):
+        """
+        Clean a Chat SQLAlchemy model's title + chat JSON,
+        and return True if anything changed.
+        """
+        changed = False
+
+        # Clean title
+        if chat_item.title:
+            cleaned = self._clean_null_bytes(chat_item.title)
+            if cleaned != chat_item.title:
+                chat_item.title = cleaned
+                changed = True
+
+        # Clean JSON
+        if chat_item.chat:
+            cleaned = self._clean_null_bytes(chat_item.chat)
+            if cleaned != chat_item.chat:
+                chat_item.chat = cleaned
+                changed = True
+
+        return changed
+
     def insert_new_chat(self, user_id: str, form_data: ChatForm) -> Optional[ChatModel]:
         with get_db() as db:
             id = str(uuid.uuid4())
@@ -130,68 +171,76 @@ class ChatTable:
                 **{
                     "id": id,
                     "user_id": user_id,
-                    "title": (
+                    "title": self._clean_null_bytes(
                         form_data.chat["title"]
                         if "title" in form_data.chat
                         else "New Chat"
                     ),
-                    "chat": form_data.chat,
+                    "chat": self._clean_null_bytes(form_data.chat),
                     "folder_id": form_data.folder_id,
                     "created_at": int(time.time()),
                     "updated_at": int(time.time()),
                 }
             )
 
-            result = Chat(**chat.model_dump())
-            db.add(result)
+            chat_item = Chat(**chat.model_dump())
+            db.add(chat_item)
             db.commit()
-            db.refresh(result)
-            return ChatModel.model_validate(result) if result else None
+            db.refresh(chat_item)
+            return ChatModel.model_validate(chat_item) if chat_item else None
 
-    def import_chat(
+    def _chat_import_form_to_chat_model(
         self, user_id: str, form_data: ChatImportForm
-    ) -> Optional[ChatModel]:
-        with get_db() as db:
+    ) -> ChatModel:
         id = str(uuid.uuid4())
         chat = ChatModel(
             **{
                 "id": id,
                 "user_id": user_id,
-                    "title": (
-                        form_data.chat["title"]
-                        if "title" in form_data.chat
-                        else "New Chat"
+                "title": self._clean_null_bytes(
+                    form_data.chat["title"] if "title" in form_data.chat else "New Chat"
                 ),
-                    "chat": form_data.chat,
+                "chat": self._clean_null_bytes(form_data.chat),
                 "meta": form_data.meta,
                 "pinned": form_data.pinned,
                 "folder_id": form_data.folder_id,
                 "created_at": (
-                        form_data.created_at
-                        if form_data.created_at
-                        else int(time.time())
+                    form_data.created_at if form_data.created_at else int(time.time())
                 ),
                 "updated_at": (
-                        form_data.updated_at
-                        if form_data.updated_at
-                        else int(time.time())
+                    form_data.updated_at if form_data.updated_at else int(time.time())
                 ),
             }
         )
+        return chat
 
-            result = Chat(**chat.model_dump())
-            db.add(result)
+    def import_chats(
+        self, user_id: str, chat_import_forms: list[ChatImportForm]
+    ) -> list[ChatModel]:
+        with get_db() as db:
+            chats = []
+
+            for form_data in chat_import_forms:
+                chat = self._chat_import_form_to_chat_model(user_id, form_data)
+                chats.append(Chat(**chat.model_dump()))
+
+            db.add_all(chats)
             db.commit()
-            db.refresh(result)
-            return ChatModel.model_validate(result) if result else None
+            return [ChatModel.model_validate(chat) for chat in chats]
 
     def update_chat_by_id(self, id: str, chat: dict) -> Optional[ChatModel]:
         try:
             with get_db() as db:
                 chat_item = db.get(Chat, id)
-                chat_item.chat = chat
-                chat_item.title = chat["title"] if "title" in chat else "New Chat"
+                chat_item.chat = self._clean_null_bytes(chat)
+                chat_item.title = (
+                    self._clean_null_bytes(chat["title"])
+                    if "title" in chat
+                    else "New Chat"
+                )
+
                 chat_item.updated_at = int(time.time())
+
                 db.commit()
                 db.refresh(chat_item)
 
@@ -297,6 +346,27 @@ class ChatTable:
         chat["history"] = history
         return self.update_chat_by_id(id, chat)
 
+    def add_message_files_by_id_and_message_id(
+        self, id: str, message_id: str, files: list[dict]
+    ) -> list[dict]:
+        chat = self.get_chat_by_id(id)
+        if chat is None:
+            return None
+
+        chat = chat.chat
+        history = chat.get("history", {})
+
+        message_files = []
+
+        if message_id in history.get("messages", {}):
+            message_files = history["messages"][message_id].get("files", [])
+            message_files = message_files + files
+            history["messages"][message_id]["files"] = message_files
+
+        chat["history"] = history
+        self.update_chat_by_id(id, chat)
+        return message_files
+
     def insert_shared_chat_by_chat_id(self, chat_id: str) -> Optional[ChatModel]:
         with get_db() as db:
             # Get the existing chat to share
@@ -405,6 +475,7 @@ class ChatTable:
             with get_db() as db:
                 chat = db.get(Chat, id)
                 chat.archived = not chat.archived
+                chat.folder_id = None
                 chat.updated_at = int(time.time())
                 db.commit()
                 db.refresh(chat)
@@ -440,7 +511,10 @@ class ChatTable:
                 order_by = filter.get("order_by")
                 direction = filter.get("direction")
 
-                if order_by and direction and getattr(Chat, order_by):
+                if order_by and direction:
+                    if not getattr(Chat, order_by, None):
+                        raise ValueError("Invalid order_by field")
+
                     if direction.lower() == "asc":
                         query = query.order_by(getattr(Chat, order_by).asc())
                     elif direction.lower() == "desc":
@@ -502,6 +576,7 @@ class ChatTable:
         user_id: str,
         include_archived: bool = False,
         include_folders: bool = False,
+        include_pinned: bool = False,
         skip: Optional[int] = None,
         limit: Optional[int] = None,
     ) -> list[ChatTitleIdResponse]:
@@ -511,6 +586,7 @@ class ChatTable:
             if not include_folders:
                 query = query.filter_by(folder_id=None)
 
+            if not include_pinned:
                 query = query.filter(or_(Chat.pinned == False, Chat.pinned == None))
 
             if not include_archived:
@@ -556,8 +632,15 @@ class ChatTable:
     def get_chat_by_id(self, id: str) -> Optional[ChatModel]:
         try:
             with get_db() as db:
-                chat = db.get(Chat, id)
-                return ChatModel.model_validate(chat)
+                chat_item = db.get(Chat, id)
+                if chat_item is None:
+                    return None
+
+                if self._sanitize_chat_row(chat_item):
+                    db.commit()
+                    db.refresh(chat_item)
+
+                return ChatModel.model_validate(chat_item)
         except Exception:
             return None
 
@@ -760,21 +843,32 @@ class ChatTable:
                     )
 
             elif dialect_name == "postgresql":
-                # PostgreSQL relies on proper JSON query for search
-                postgres_content_sql = (
-                    "EXISTS ("
-                    "    SELECT 1 "
-                    "    FROM json_array_elements(Chat.chat->'messages') AS message "
-                    "    WHERE LOWER(message->>'content') LIKE '%' || :content_key || '%'"
-                    ")"
+                # PostgreSQL doesn't allow null bytes in text. We filter those out by checking
+                # the JSON representation for \u0000 before attempting text extraction
+
+                # Safety filter: JSON field must not contain \u0000
+                query = query.filter(text("Chat.chat::text NOT LIKE '%\\\\u0000%'"))
+
+                # Safety filter: title must not contain actual null bytes
+                query = query.filter(text("Chat.title::text NOT LIKE '%\\x00%'"))
+
+                postgres_content_sql = """
+                EXISTS (
+                    SELECT 1
+                    FROM json_array_elements(Chat.chat->'messages') AS message
+                    WHERE json_typeof(message->'content') = 'string'
+                    AND LOWER(message->>'content') LIKE '%' || :content_key || '%'
                 )
+                """
+
                 postgres_content_clause = text(postgres_content_sql)
+
                 query = query.filter(
                     or_(
                         Chat.title.ilike(bindparam("title_key")),
                         postgres_content_clause,
-                    ).params(title_key=f"%{search_text}%", content_key=search_text)
                     )
+                ).params(title_key=f"%{search_text}%", content_key=search_text.lower())
 
                 # Check if there are any tags to filter, it should have all the tags
                 if "none" in tag_ids:
@@ -1049,6 +1143,20 @@ class ChatTable:
         except Exception:
             return False
 
+    def move_chats_by_user_id_and_folder_id(
+        self, user_id: str, folder_id: str, new_folder_id: Optional[str]
+    ) -> bool:
+        try:
+            with get_db() as db:
+                db.query(Chat).filter_by(user_id=user_id, folder_id=folder_id).update(
+                    {"folder_id": new_folder_id}
+                )
+                db.commit()
+
+                return True
+        except Exception:
+            return False
+
     def delete_shared_chats_by_user_id(self, user_id: str) -> bool:
         try:
             with get_db() as db:

backend/open_webui/models/feedbacks.py

@@ -4,7 +4,7 @@ import uuid
 from typing import Optional
 
 from open_webui.internal.db import Base, get_db
-from open_webui.models.chats import Chats
+from open_webui.models.users import User
 
 from open_webui.env import SRC_LOG_LEVELS
 from pydantic import BaseModel, ConfigDict
@@ -21,7 +21,7 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 
 class Feedback(Base):
     __tablename__ = "feedback"
-    id = Column(Text, primary_key=True)
+    id = Column(Text, primary_key=True, unique=True)
     user_id = Column(Text)
     version = Column(BigInteger, default=0)
     type = Column(Text)
@@ -92,6 +92,28 @@ class FeedbackForm(BaseModel):
     model_config = ConfigDict(extra="allow")
 
 
+class UserResponse(BaseModel):
+    id: str
+    name: str
+    email: str
+    role: str = "pending"
+
+    last_active_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
+    created_at: int  # timestamp in epoch
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class FeedbackUserResponse(FeedbackResponse):
+    user: Optional[UserResponse] = None
+
+
+class FeedbackListResponse(BaseModel):
+    items: list[FeedbackUserResponse]
+    total: int
+
+
 class FeedbackTable:
     def insert_new_feedback(
         self, user_id: str, form_data: FeedbackForm
@@ -143,6 +165,70 @@ class FeedbackTable:
         except Exception:
             return None
 
+    def get_feedback_items(
+        self, filter: dict = {}, skip: int = 0, limit: int = 30
+    ) -> FeedbackListResponse:
+        with get_db() as db:
+            query = db.query(Feedback, User).join(User, Feedback.user_id == User.id)
+
+            if filter:
+                order_by = filter.get("order_by")
+                direction = filter.get("direction")
+
+                if order_by == "username":
+                    if direction == "asc":
+                        query = query.order_by(User.name.asc())
+                    else:
+                        query = query.order_by(User.name.desc())
+                elif order_by == "model_id":
+                    # it's stored in feedback.data['model_id']
+                    if direction == "asc":
+                        query = query.order_by(
+                            Feedback.data["model_id"].as_string().asc()
+                        )
+                    else:
+                        query = query.order_by(
+                            Feedback.data["model_id"].as_string().desc()
+                        )
+                elif order_by == "rating":
+                    # it's stored in feedback.data['rating']
+                    if direction == "asc":
+                        query = query.order_by(
+                            Feedback.data["rating"].as_string().asc()
+                        )
+                    else:
+                        query = query.order_by(
+                            Feedback.data["rating"].as_string().desc()
+                        )
+                elif order_by == "updated_at":
+                    if direction == "asc":
+                        query = query.order_by(Feedback.updated_at.asc())
+                    else:
+                        query = query.order_by(Feedback.updated_at.desc())
+
+            else:
+                query = query.order_by(Feedback.created_at.desc())
+
+            # Count BEFORE pagination
+            total = query.count()
+
+            if skip:
+                query = query.offset(skip)
+            if limit:
+                query = query.limit(limit)
+
+            items = query.all()
+
+            feedbacks = []
+            for feedback, user in items:
+                feedback_model = FeedbackModel.model_validate(feedback)
+                user_model = UserResponse.model_validate(user)
+                feedbacks.append(
+                    FeedbackUserResponse(**feedback_model.model_dump(), user=user_model)
+                )
+
+            return FeedbackListResponse(items=feedbacks, total=total)
+
     def get_all_feedbacks(self) -> list[FeedbackModel]:
         with get_db() as db:
             return [

backend/open_webui/models/files.py

@@ -17,7 +17,7 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 
 class File(Base):
     __tablename__ = "file"
-    id = Column(String, primary_key=True)
+    id = Column(String, primary_key=True, unique=True)
     user_id = Column(String)
     hash = Column(Text, nullable=True)
 
@@ -82,6 +82,7 @@ class FileModelResponse(BaseModel):
 
 class FileMetadataResponse(BaseModel):
     id: str
+    hash: Optional[str] = None
     meta: dict
     created_at: int  # timestamp in epoch
     updated_at: int  # timestamp in epoch
@@ -97,6 +98,12 @@ class FileForm(BaseModel):
     access_control: Optional[dict] = None
 
 
+class FileUpdateForm(BaseModel):
+    hash: Optional[str] = None
+    data: Optional[dict] = None
+    meta: Optional[dict] = None
+
+
 class FilesTable:
     def insert_new_file(self, user_id: str, form_data: FileForm) -> Optional[FileModel]:
         with get_db() as db:
@@ -147,6 +154,7 @@ class FilesTable:
                 file = db.get(File, id)
                 return FileMetadataResponse(
                     id=file.id,
+                    hash=file.hash,
                     meta=file.meta,
                     created_at=file.created_at,
                     updated_at=file.updated_at,
@@ -182,11 +190,14 @@ class FilesTable:
             return [
                 FileMetadataResponse(
                     id=file.id,
+                    hash=file.hash,
                     meta=file.meta,
                     created_at=file.created_at,
                     updated_at=file.updated_at,
                 )
-                for file in db.query(File)
+                for file in db.query(
+                    File.id, File.hash, File.meta, File.created_at, File.updated_at
+                )
                 .filter(File.id.in_(ids))
                 .order_by(File.updated_at.desc())
                 .all()
@@ -199,6 +210,29 @@ class FilesTable:
                 for file in db.query(File).filter_by(user_id=user_id).all()
             ]
 
+    def update_file_by_id(
+        self, id: str, form_data: FileUpdateForm
+    ) -> Optional[FileModel]:
+        with get_db() as db:
+            try:
+                file = db.query(File).filter_by(id=id).first()
+
+                if form_data.hash is not None:
+                    file.hash = form_data.hash
+
+                if form_data.data is not None:
+                    file.data = {**(file.data if file.data else {}), **form_data.data}
+
+                if form_data.meta is not None:
+                    file.meta = {**(file.meta if file.meta else {}), **form_data.meta}
+
+                file.updated_at = int(time.time())
+                db.commit()
+                return FileModel.model_validate(file)
+            except Exception as e:
+                log.exception(f"Error updating file completely by id: {e}")
+                return None
+
     def update_file_hash_by_id(self, id: str, hash: str) -> Optional[FileModel]:
         with get_db() as db:
             try:

backend/open_webui/models/folders.py

@@ -23,7 +23,7 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 
 class Folder(Base):
     __tablename__ = "folder"
-    id = Column(Text, primary_key=True)
+    id = Column(Text, primary_key=True, unique=True)
     parent_id = Column(Text, nullable=True)
     user_id = Column(Text)
     name = Column(Text)

backend/open_webui/models/functions.py

@@ -3,7 +3,7 @@ import time
 from typing import Optional
 
 from open_webui.internal.db import Base, JSONField, get_db
-from open_webui.models.users import Users
+from open_webui.models.users import Users, UserModel
 from open_webui.env import SRC_LOG_LEVELS
 from pydantic import BaseModel, ConfigDict
 from sqlalchemy import BigInteger, Boolean, Column, String, Text, Index
@@ -19,7 +19,7 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 class Function(Base):
     __tablename__ = "function"
 
-    id = Column(String, primary_key=True)
+    id = Column(String, primary_key=True, unique=True)
     user_id = Column(String)
     name = Column(Text)
     type = Column(Text)
@@ -76,6 +76,10 @@ class FunctionWithValvesModel(BaseModel):
 ####################
 
 
+class FunctionUserResponse(FunctionModel):
+    user: Optional[UserModel] = None
+
+
 class FunctionResponse(BaseModel):
     id: str
     user_id: str
@@ -203,6 +207,28 @@ class FunctionsTable:
                     FunctionModel.model_validate(function) for function in functions
                 ]
 
+    def get_function_list(self) -> list[FunctionUserResponse]:
+        with get_db() as db:
+            functions = db.query(Function).order_by(Function.updated_at.desc()).all()
+            user_ids = list(set(func.user_id for func in functions))
+
+            users = Users.get_users_by_user_ids(user_ids) if user_ids else []
+            users_dict = {user.id: user for user in users}
+
+            return [
+                FunctionUserResponse.model_validate(
+                    {
+                        **FunctionModel.model_validate(func).model_dump(),
+                        "user": (
+                            users_dict.get(func.user_id).model_dump()
+                            if func.user_id in users_dict
+                            else None
+                        ),
+                    }
+                )
+                for func in functions
+            ]
+
     def get_functions_by_type(
         self, type: str, active_only=False
     ) -> list[FunctionModel]:

backend/open_webui/models/groups.py

@@ -11,7 +11,18 @@ from open_webui.models.files import FileMetadataResponse
 
 
 from pydantic import BaseModel, ConfigDict
-from sqlalchemy import BigInteger, Column, String, Text, JSON, func
+from sqlalchemy import (
+    BigInteger,
+    Column,
+    String,
+    Text,
+    JSON,
+    and_,
+    func,
+    ForeignKey,
+    cast,
+    or_,
+)
 
 
 log = logging.getLogger(__name__)
@@ -35,14 +46,12 @@ class Group(Base):
     meta = Column(JSON, nullable=True)
 
     permissions = Column(JSON, nullable=True)
-    user_ids = Column(JSON, nullable=True)
 
     created_at = Column(BigInteger)
     updated_at = Column(BigInteger)
 
 
 class GroupModel(BaseModel):
-    model_config = ConfigDict(from_attributes=True)
     id: str
     user_id: str
 
@@ -53,44 +62,64 @@ class GroupModel(BaseModel):
     meta: Optional[dict] = None
 
     permissions: Optional[dict] = None
-    user_ids: list[str] = []
 
     created_at: int  # timestamp in epoch
     updated_at: int  # timestamp in epoch
 
+    model_config = ConfigDict(from_attributes=True)
+
+
+class GroupMember(Base):
+    __tablename__ = "group_member"
+
+    id = Column(Text, unique=True, primary_key=True)
+    group_id = Column(
+        Text,
+        ForeignKey("group.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    user_id = Column(Text, nullable=False)
+    created_at = Column(BigInteger, nullable=True)
+    updated_at = Column(BigInteger, nullable=True)
+
+
+class GroupMemberModel(BaseModel):
+    id: str
+    group_id: str
+    user_id: str
+    created_at: Optional[int] = None  # timestamp in epoch
+    updated_at: Optional[int] = None  # timestamp in epoch
+
 
 ####################
 # Forms
 ####################
 
 
-class GroupResponse(BaseModel):
-    id: str
-    user_id: str
-    name: str
-    description: str
-    permissions: Optional[dict] = None
-    data: Optional[dict] = None
-    meta: Optional[dict] = None
-    user_ids: list[str] = []
-    created_at: int  # timestamp in epoch
-    updated_at: int  # timestamp in epoch
+class GroupResponse(GroupModel):
+    member_count: Optional[int] = None
 
 
 class GroupForm(BaseModel):
     name: str
     description: str
     permissions: Optional[dict] = None
+    data: Optional[dict] = None
 
 
 class UserIdsForm(BaseModel):
     user_ids: Optional[list[str]] = None
 
 
-class GroupUpdateForm(GroupForm, UserIdsForm):
+class GroupUpdateForm(GroupForm):
     pass
 
 
+class GroupListResponse(BaseModel):
+    items: list[GroupResponse] = []
+    total: int = 0
+
+
 class GroupTable:
     def insert_new_group(
         self, user_id: str, form_data: GroupForm
@@ -119,24 +148,94 @@ class GroupTable:
             except Exception:
                 return None
 
-    def get_groups(self) -> list[GroupModel]:
+    def get_all_groups(self) -> list[GroupModel]:
         with get_db() as db:
+            groups = db.query(Group).order_by(Group.updated_at.desc()).all()
+            return [GroupModel.model_validate(group) for group in groups]
+
+    def get_groups(self, filter) -> list[GroupResponse]:
+        with get_db() as db:
+            query = db.query(Group)
+
+            if filter:
+                if "query" in filter:
+                    query = query.filter(Group.name.ilike(f"%{filter['query']}%"))
+                if "member_id" in filter:
+                    query = query.join(
+                        GroupMember, GroupMember.group_id == Group.id
+                    ).filter(GroupMember.user_id == filter["member_id"])
+
+                if "share" in filter:
+                    share_value = filter["share"]
+                    json_share = Group.data["config"]["share"].as_boolean()
+
+                    if share_value:
+                        query = query.filter(
+                            or_(
+                                Group.data.is_(None),
+                                json_share.is_(None),
+                                json_share == True,
+                            )
+                        )
+                    else:
+                        query = query.filter(
+                            and_(Group.data.isnot(None), json_share == False)
+                        )
+            groups = query.order_by(Group.updated_at.desc()).all()
             return [
-                GroupModel.model_validate(group)
-                for group in db.query(Group).order_by(Group.updated_at.desc()).all()
+                GroupResponse.model_validate(
+                    {
+                        **GroupModel.model_validate(group).model_dump(),
+                        "member_count": self.get_group_member_count_by_id(group.id),
+                    }
+                )
+                for group in groups
             ]
 
+    def search_groups(
+        self, filter: Optional[dict] = None, skip: int = 0, limit: int = 30
+    ) -> GroupListResponse:
+        with get_db() as db:
+            query = db.query(Group)
+
+            if filter:
+                if "query" in filter:
+                    query = query.filter(Group.name.ilike(f"%{filter['query']}%"))
+                if "member_id" in filter:
+                    query = query.join(
+                        GroupMember, GroupMember.group_id == Group.id
+                    ).filter(GroupMember.user_id == filter["member_id"])
+
+                if "share" in filter:
+                    #  'share' is stored in data JSON, support both sqlite and postgres
+                    share_value = filter["share"]
+                    print("Filtering by share:", share_value)
+                    query = query.filter(
+                        Group.data.op("->>")("share") == str(share_value)
+                    )
+
+            total = query.count()
+            query = query.order_by(Group.updated_at.desc())
+            groups = query.offset(skip).limit(limit).all()
+
+            return {
+                "items": [
+                    GroupResponse.model_validate(
+                        **GroupModel.model_validate(group).model_dump(),
+                        member_count=self.get_group_member_count_by_id(group.id),
+                    )
+                    for group in groups
+                ],
+                "total": total,
+            }
+
     def get_groups_by_member_id(self, user_id: str) -> list[GroupModel]:
         with get_db() as db:
             return [
                 GroupModel.model_validate(group)
                 for group in db.query(Group)
-                .filter(
-                    func.json_array_length(Group.user_ids) > 0
-                )  # Ensure array exists
-                .filter(
-                    Group.user_ids.cast(String).like(f'%"{user_id}"%')
-                )  # String-based check
+                .join(GroupMember, GroupMember.group_id == Group.id)
+                .filter(GroupMember.user_id == user_id)
                 .order_by(Group.updated_at.desc())
                 .all()
             ]
@@ -149,13 +248,64 @@ class GroupTable:
         except Exception:
             return None
 
-    def get_group_user_ids_by_id(self, id: str) -> Optional[str]:
-        group = self.get_group_by_id(id)
-        if group:
-            return group.user_ids
-        else:
+    def get_group_user_ids_by_id(self, id: str) -> Optional[list[str]]:
+        with get_db() as db:
+            members = (
+                db.query(GroupMember.user_id).filter(GroupMember.group_id == id).all()
+            )
+
+            if not members:
                 return None
 
+            return [m[0] for m in members]
+
+    def get_group_user_ids_by_ids(self, group_ids: list[str]) -> dict[str, list[str]]:
+        with get_db() as db:
+            members = (
+                db.query(GroupMember.group_id, GroupMember.user_id)
+                .filter(GroupMember.group_id.in_(group_ids))
+                .all()
+            )
+
+            group_user_ids: dict[str, list[str]] = {
+                group_id: [] for group_id in group_ids
+            }
+
+            for group_id, user_id in members:
+                group_user_ids[group_id].append(user_id)
+
+            return group_user_ids
+
+    def set_group_user_ids_by_id(self, group_id: str, user_ids: list[str]) -> None:
+        with get_db() as db:
+            # Delete existing members
+            db.query(GroupMember).filter(GroupMember.group_id == group_id).delete()
+
+            # Insert new members
+            now = int(time.time())
+            new_members = [
+                GroupMember(
+                    id=str(uuid.uuid4()),
+                    group_id=group_id,
+                    user_id=user_id,
+                    created_at=now,
+                    updated_at=now,
+                )
+                for user_id in user_ids
+            ]
+
+            db.add_all(new_members)
+            db.commit()
+
+    def get_group_member_count_by_id(self, id: str) -> int:
+        with get_db() as db:
+            count = (
+                db.query(func.count(GroupMember.user_id))
+                .filter(GroupMember.group_id == id)
+                .scalar()
+            )
+            return count if count else 0
+
     def update_group_by_id(
         self, id: str, form_data: GroupUpdateForm, overwrite: bool = False
     ) -> Optional[GroupModel]:
@@ -195,20 +345,29 @@ class GroupTable:
     def remove_user_from_all_groups(self, user_id: str) -> bool:
         with get_db() as db:
             try:
-                groups = self.get_groups_by_member_id(user_id)
-
-                for group in groups:
-                    group.user_ids.remove(user_id)
-                    db.query(Group).filter_by(id=group.id).update(
-                        {
-                            "user_ids": group.user_ids,
-                            "updated_at": int(time.time()),
-                        }
+                # Find all groups the user belongs to
+                groups = (
+                    db.query(Group)
+                    .join(GroupMember, GroupMember.group_id == Group.id)
+                    .filter(GroupMember.user_id == user_id)
+                    .all()
                 )
-                    db.commit()
 
+                # Remove the user from each group
+                for group in groups:
+                    db.query(GroupMember).filter(
+                        GroupMember.group_id == group.id, GroupMember.user_id == user_id
+                    ).delete()
+
+                    db.query(Group).filter_by(id=group.id).update(
+                        {"updated_at": int(time.time())}
+                    )
+
+                db.commit()
                 return True
+
             except Exception:
+                db.rollback()
                 return False
 
     def create_groups_by_group_names(
@@ -216,7 +375,7 @@ class GroupTable:
     ) -> list[GroupModel]:
 
         # check for existing groups
-        existing_groups = self.get_groups()
+        existing_groups = self.get_all_groups()
         existing_group_names = {group.name for group in existing_groups}
 
         new_groups = []
@@ -246,37 +405,61 @@ class GroupTable:
     def sync_groups_by_group_names(self, user_id: str, group_names: list[str]) -> bool:
         with get_db() as db:
             try:
-                groups = db.query(Group).filter(Group.name.in_(group_names)).all()
-                group_ids = [group.id for group in groups]
+                now = int(time.time())
 
-                # Remove user from groups not in the new list
-                existing_groups = self.get_groups_by_member_id(user_id)
+                # 1. Groups that SHOULD contain the user
+                target_groups = (
+                    db.query(Group).filter(Group.name.in_(group_names)).all()
+                )
+                target_group_ids = {g.id for g in target_groups}
 
-                for group in existing_groups:
-                    if group.id not in group_ids:
-                        group.user_ids.remove(user_id)
-                        db.query(Group).filter_by(id=group.id).update(
-                            {
-                                "user_ids": group.user_ids,
-                                "updated_at": int(time.time()),
+                # 2. Groups the user is CURRENTLY in
+                existing_group_ids = {
+                    g.id
+                    for g in db.query(Group)
+                    .join(GroupMember, GroupMember.group_id == Group.id)
+                    .filter(GroupMember.user_id == user_id)
+                    .all()
                 }
+
+                # 3. Determine adds + removals
+                groups_to_add = target_group_ids - existing_group_ids
+                groups_to_remove = existing_group_ids - target_group_ids
+
+                # 4. Remove in one bulk delete
+                if groups_to_remove:
+                    db.query(GroupMember).filter(
+                        GroupMember.user_id == user_id,
+                        GroupMember.group_id.in_(groups_to_remove),
+                    ).delete(synchronize_session=False)
+
+                    db.query(Group).filter(Group.id.in_(groups_to_remove)).update(
+                        {"updated_at": now}, synchronize_session=False
                     )
 
-                # Add user to new groups
-                for group in groups:
-                    if user_id not in group.user_ids:
-                        group.user_ids.append(user_id)
-                        db.query(Group).filter_by(id=group.id).update(
-                            {
-                                "user_ids": group.user_ids,
-                                "updated_at": int(time.time()),
-                            }
+                # 5. Bulk insert missing memberships
+                for group_id in groups_to_add:
+                    db.add(
+                        GroupMember(
+                            id=str(uuid.uuid4()),
+                            group_id=group_id,
+                            user_id=user_id,
+                            created_at=now,
+                            updated_at=now,
+                        )
+                    )
+
+                if groups_to_add:
+                    db.query(Group).filter(Group.id.in_(groups_to_add)).update(
+                        {"updated_at": now}, synchronize_session=False
                     )
 
                 db.commit()
                 return True
+
             except Exception as e:
                 log.exception(e)
+                db.rollback()
                 return False
 
     def add_users_to_group(
@@ -288,21 +471,31 @@ class GroupTable:
                 if not group:
                     return None
 
-                group_user_ids = group.user_ids
-                if not group_user_ids or not isinstance(group_user_ids, list):
-                    group_user_ids = []
+                now = int(time.time())
 
-                group_user_ids = list(set(group_user_ids))  # Deduplicate
+                for user_id in user_ids or []:
+                    try:
+                        db.add(
+                            GroupMember(
+                                id=str(uuid.uuid4()),
+                                group_id=id,
+                                user_id=user_id,
+                                created_at=now,
+                                updated_at=now,
+                            )
+                        )
+                        db.flush()  # Detect unique constraint violation early
+                    except Exception:
+                        db.rollback()  # Clear failed INSERT
+                        db.begin()  # Start a new transaction
+                        continue  # Duplicate → ignore
 
-                for user_id in user_ids:
-                    if user_id not in group_user_ids:
-                        group_user_ids.append(user_id)
-
-                group.user_ids = group_user_ids
-                group.updated_at = int(time.time())
+                group.updated_at = now
                 db.commit()
                 db.refresh(group)
+
                 return GroupModel.model_validate(group)
+
         except Exception as e:
             log.exception(e)
             return None
@@ -316,23 +509,22 @@ class GroupTable:
                 if not group:
                     return None
 
-                group_user_ids = group.user_ids
-
-                if not group_user_ids or not isinstance(group_user_ids, list):
+                if not user_ids:
                     return GroupModel.model_validate(group)
 
-                group_user_ids = list(set(group_user_ids))  # Deduplicate
-
+                # Remove each user from group_member
                 for user_id in user_ids:
-                    if user_id in group_user_ids:
-                        group_user_ids.remove(user_id)
+                    db.query(GroupMember).filter(
+                        GroupMember.group_id == id, GroupMember.user_id == user_id
+                    ).delete()
 
-                group.user_ids = group_user_ids
+                # Update group timestamp
                 group.updated_at = int(time.time())
 
                 db.commit()
                 db.refresh(group)
                 return GroupModel.model_validate(group)
+
         except Exception as e:
             log.exception(e)
             return None

backend/open_webui/models/memories.py

@@ -14,7 +14,7 @@ from sqlalchemy import BigInteger, Column, String, Text
 class Memory(Base):
     __tablename__ = "memory"
 
-    id = Column(String, primary_key=True)
+    id = Column(String, primary_key=True, unique=True)
     user_id = Column(String)
     content = Column(Text)
     updated_at = Column(BigInteger)

backend/open_webui/models/messages.py

@@ -5,7 +5,8 @@ from typing import Optional
 
 from open_webui.internal.db import Base, get_db
 from open_webui.models.tags import TagModel, Tag, Tags
-from open_webui.models.users import Users, UserNameResponse
+from open_webui.models.users import Users, User, UserNameResponse
+from open_webui.models.channels import Channels, ChannelMember
 
 
 from pydantic import BaseModel, ConfigDict
@@ -20,7 +21,7 @@ from sqlalchemy.sql import exists
 
 class MessageReaction(Base):
     __tablename__ = "message_reaction"
-    id = Column(Text, primary_key=True)
+    id = Column(Text, primary_key=True, unique=True)
     user_id = Column(Text)
     message_id = Column(Text)
     name = Column(Text)
@@ -39,7 +40,7 @@ class MessageReactionModel(BaseModel):
 
 class Message(Base):
     __tablename__ = "message"
-    id = Column(Text, primary_key=True)
+    id = Column(Text, primary_key=True, unique=True)
 
     user_id = Column(Text)
     channel_id = Column(Text, nullable=True)
@@ -47,6 +48,11 @@ class Message(Base):
     reply_to_id = Column(Text, nullable=True)
     parent_id = Column(Text, nullable=True)
 
+    # Pins
+    is_pinned = Column(Boolean, nullable=False, default=False)
+    pinned_at = Column(BigInteger, nullable=True)
+    pinned_by = Column(Text, nullable=True)
+
     content = Column(Text)
     data = Column(JSON, nullable=True)
     meta = Column(JSON, nullable=True)
@@ -65,12 +71,17 @@ class MessageModel(BaseModel):
     reply_to_id: Optional[str] = None
     parent_id: Optional[str] = None
 
+    # Pins
+    is_pinned: bool = False
+    pinned_by: Optional[str] = None
+    pinned_at: Optional[int] = None  # timestamp in epoch (time_ns)
+
     content: str
     data: Optional[dict] = None
     meta: Optional[dict] = None
 
-    created_at: int  # timestamp in epoch
-    updated_at: int  # timestamp in epoch
+    created_at: int  # timestamp in epoch (time_ns)
+    updated_at: int  # timestamp in epoch (time_ns)
 
 
 ####################
@@ -79,6 +90,7 @@ class MessageModel(BaseModel):
 
 
 class MessageForm(BaseModel):
+    temp_id: Optional[str] = None
     content: str
     reply_to_id: Optional[str] = None
     parent_id: Optional[str] = None
@@ -88,7 +100,7 @@ class MessageForm(BaseModel):
 
 class Reactions(BaseModel):
     name: str
-    user_ids: list[str]
+    users: list[dict]
     count: int
 
 
@@ -100,6 +112,10 @@ class MessageReplyToResponse(MessageUserResponse):
     reply_to_message: Optional[MessageUserResponse] = None
 
 
+class MessageWithReactionsResponse(MessageUserResponse):
+    reactions: list[Reactions]
+
+
 class MessageResponse(MessageReplyToResponse):
     latest_reply_at: Optional[int]
     reply_count: int
@@ -111,9 +127,11 @@ class MessageTable:
         self, form_data: MessageForm, channel_id: str, user_id: str
     ) -> Optional[MessageModel]:
         with get_db() as db:
-            id = str(uuid.uuid4())
+            channel_member = Channels.join_channel(channel_id, user_id)
 
+            id = str(uuid.uuid4())
             ts = int(time.time_ns())
+
             message = MessageModel(
                 **{
                     "id": id,
@@ -121,6 +139,9 @@ class MessageTable:
                     "channel_id": channel_id,
                     "reply_to_id": form_data.reply_to_id,
                     "parent_id": form_data.parent_id,
+                    "is_pinned": False,
+                    "pinned_at": None,
+                    "pinned_by": None,
                     "content": form_data.content,
                     "data": form_data.data,
                     "meta": form_data.meta,
@@ -128,8 +149,8 @@ class MessageTable:
                     "updated_at": ts,
                 }
             )
-
             result = Message(**message.model_dump())
+
             db.add(result)
             db.commit()
             db.refresh(result)
@@ -280,6 +301,30 @@ class MessageTable:
                 )
             return messages
 
+    def get_last_message_by_channel_id(self, channel_id: str) -> Optional[MessageModel]:
+        with get_db() as db:
+            message = (
+                db.query(Message)
+                .filter_by(channel_id=channel_id)
+                .order_by(Message.created_at.desc())
+                .first()
+            )
+            return MessageModel.model_validate(message) if message else None
+
+    def get_pinned_messages_by_channel_id(
+        self, channel_id: str, skip: int = 0, limit: int = 50
+    ) -> list[MessageModel]:
+        with get_db() as db:
+            all_messages = (
+                db.query(Message)
+                .filter_by(channel_id=channel_id, is_pinned=True)
+                .order_by(Message.pinned_at.desc())
+                .offset(skip)
+                .limit(limit)
+                .all()
+            )
+            return [MessageModel.model_validate(message) for message in all_messages]
+
     def update_message_by_id(
         self, id: str, form_data: MessageForm
     ) -> Optional[MessageModel]:
@@ -299,10 +344,44 @@ class MessageTable:
             db.refresh(message)
             return MessageModel.model_validate(message) if message else None
 
+    def update_is_pinned_by_id(
+        self, id: str, is_pinned: bool, pinned_by: Optional[str] = None
+    ) -> Optional[MessageModel]:
+        with get_db() as db:
+            message = db.get(Message, id)
+            message.is_pinned = is_pinned
+            message.pinned_at = int(time.time_ns()) if is_pinned else None
+            message.pinned_by = pinned_by if is_pinned else None
+            db.commit()
+            db.refresh(message)
+            return MessageModel.model_validate(message) if message else None
+
+    def get_unread_message_count(
+        self, channel_id: str, user_id: str, last_read_at: Optional[int] = None
+    ) -> int:
+        with get_db() as db:
+            query = db.query(Message).filter(
+                Message.channel_id == channel_id,
+                Message.parent_id == None,  # only count top-level messages
+                Message.created_at > (last_read_at if last_read_at else 0),
+            )
+            if user_id:
+                query = query.filter(Message.user_id != user_id)
+            return query.count()
+
     def add_reaction_to_message(
         self, id: str, user_id: str, name: str
     ) -> Optional[MessageReactionModel]:
         with get_db() as db:
+            # check for existing reaction
+            existing_reaction = (
+                db.query(MessageReaction)
+                .filter_by(message_id=id, user_id=user_id, name=name)
+                .first()
+            )
+            if existing_reaction:
+                return MessageReactionModel.model_validate(existing_reaction)
+
             reaction_id = str(uuid.uuid4())
             reaction = MessageReactionModel(
                 id=reaction_id,
@@ -319,17 +398,30 @@ class MessageTable:
 
     def get_reactions_by_message_id(self, id: str) -> list[Reactions]:
         with get_db() as db:
-            all_reactions = db.query(MessageReaction).filter_by(message_id=id).all()
+            # JOIN User so all user info is fetched in one query
+            results = (
+                db.query(MessageReaction, User)
+                .join(User, MessageReaction.user_id == User.id)
+                .filter(MessageReaction.message_id == id)
+                .all()
+            )
 
             reactions = {}
-            for reaction in all_reactions:
+
+            for reaction, user in results:
                 if reaction.name not in reactions:
                     reactions[reaction.name] = {
                         "name": reaction.name,
-                        "user_ids": [],
+                        "users": [],
                         "count": 0,
                     }
-                reactions[reaction.name]["user_ids"].append(reaction.user_id)
+
+                reactions[reaction.name]["users"].append(
+                    {
+                        "id": user.id,
+                        "name": user.name,
+                    }
+                )
                 reactions[reaction.name]["count"] += 1
 
             return [Reactions(**reaction) for reaction in reactions.values()]

backend/open_webui/models/models.py

@@ -6,12 +6,12 @@ from open_webui.internal.db import Base, JSONField, get_db
 from open_webui.env import SRC_LOG_LEVELS
 
 from open_webui.models.groups import Groups
-from open_webui.models.users import Users, UserResponse
+from open_webui.models.users import User, UserModel, Users, UserResponse
 
 
 from pydantic import BaseModel, ConfigDict
 
-from sqlalchemy import or_, and_, func
+from sqlalchemy import String, cast, or_, and_, func
 from sqlalchemy.dialects import postgresql, sqlite
 from sqlalchemy import BigInteger, Column, Text, JSON, Boolean
 
@@ -53,7 +53,7 @@ class ModelMeta(BaseModel):
 class Model(Base):
     __tablename__ = "model"
 
-    id = Column(Text, primary_key=True)
+    id = Column(Text, primary_key=True, unique=True)
     """
         The model's id as used in the API. If set to an existing model, it will override the model.
     """
@@ -133,6 +133,11 @@ class ModelResponse(ModelModel):
     pass
 
 
+class ModelListResponse(BaseModel):
+    items: list[ModelUserResponse]
+    total: int
+
+
 class ModelForm(BaseModel):
     id: str
     base_model_id: Optional[str] = None
@@ -215,6 +220,117 @@ class ModelsTable:
             or has_access(user_id, permission, model.access_control, user_group_ids)
         ]
 
+    def _has_write_permission(self, query, filter: dict):
+        group_ids = filter.get("group_ids", [])
+        user_id = filter.get("user_id")
+
+        json_group_ids = Model.access_control["write"]["group_ids"]
+
+        conditions = []
+        if group_ids or user_id:
+            conditions.append(Model.access_control.is_(None))
+
+        if user_id:
+            conditions.append(Model.user_id == user_id)
+
+        if group_ids:
+            group_conditions = []
+
+            for gid in group_ids:
+                # CASE: gid IN JSON array
+                # SQLite → json_extract(access_control, '$.write.group_ids') LIKE '%gid%'
+                # Postgres → access_control->'write'->'group_ids' @> '[gid]'
+                group_conditions.append(json_group_ids.contains([gid]))
+
+            conditions.append(or_(*group_conditions))
+
+        if conditions:
+            query = query.filter(or_(*conditions))
+
+        return query
+
+    def search_models(
+        self, user_id: str, filter: dict = {}, skip: int = 0, limit: int = 30
+    ) -> ModelListResponse:
+        with get_db() as db:
+            # Join GroupMember so we can order by group_id when requested
+            query = db.query(Model, User).outerjoin(User, User.id == Model.user_id)
+            query = query.filter(Model.base_model_id != None)
+
+            if filter:
+                query_key = filter.get("query")
+                if query_key:
+                    query = query.filter(
+                        or_(
+                            Model.name.ilike(f"%{query_key}%"),
+                            Model.base_model_id.ilike(f"%{query_key}%"),
+                        )
+                    )
+
+                # Apply access control filtering
+                query = self._has_write_permission(query, filter)
+
+                view_option = filter.get("view_option")
+                if view_option == "created":
+                    query = query.filter(Model.user_id == user_id)
+                elif view_option == "shared":
+                    query = query.filter(Model.user_id != user_id)
+
+                tag = filter.get("tag")
+                if tag:
+                    # TODO: This is a simple implementation and should be improved for performance
+                    like_pattern = f'%"{tag.lower()}"%'  # `"tag"` inside JSON array
+                    meta_text = func.lower(cast(Model.meta, String))
+
+                    query = query.filter(meta_text.like(like_pattern))
+
+                order_by = filter.get("order_by")
+                direction = filter.get("direction")
+
+                if order_by == "name":
+                    if direction == "asc":
+                        query = query.order_by(Model.name.asc())
+                    else:
+                        query = query.order_by(Model.name.desc())
+                elif order_by == "created_at":
+                    if direction == "asc":
+                        query = query.order_by(Model.created_at.asc())
+                    else:
+                        query = query.order_by(Model.created_at.desc())
+                elif order_by == "updated_at":
+                    if direction == "asc":
+                        query = query.order_by(Model.updated_at.asc())
+                    else:
+                        query = query.order_by(Model.updated_at.desc())
+
+            else:
+                query = query.order_by(Model.created_at.desc())
+
+            # Count BEFORE pagination
+            total = query.count()
+
+            if skip:
+                query = query.offset(skip)
+            if limit:
+                query = query.limit(limit)
+
+            items = query.all()
+
+            models = []
+            for model, user in items:
+                models.append(
+                    ModelUserResponse(
+                        **ModelModel.model_validate(model).model_dump(),
+                        user=(
+                            UserResponse(**UserModel.model_validate(user).model_dump())
+                            if user
+                            else None
+                        ),
+                    )
+                )
+
+            return ModelListResponse(items=models, total=total)
+
     def get_model_by_id(self, id: str) -> Optional[ModelModel]:
         try:
             with get_db() as db:
@@ -244,11 +360,9 @@ class ModelsTable:
         try:
             with get_db() as db:
                 # update only the fields that are present in the model
-                result = (
-                    db.query(Model)
-                    .filter_by(id=id)
-                    .update(model.model_dump(exclude={"id"}))
-                )
+                data = model.model_dump(exclude={"id"})
+                result = db.query(Model).filter_by(id=id).update(data)
+
                 db.commit()
 
                 model = db.get(Model, id)

backend/open_webui/models/notes.py

@@ -23,7 +23,7 @@ from sqlalchemy.sql import exists
 class Note(Base):
     __tablename__ = "note"
 
-    id = Column(Text, primary_key=True)
+    id = Column(Text, primary_key=True, unique=True)
     user_id = Column(Text)
 
     title = Column(Text)

backend/open_webui/models/oauth_sessions.py

@@ -25,7 +25,7 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 class OAuthSession(Base):
     __tablename__ = "oauth_session"
 
-    id = Column(Text, primary_key=True)
+    id = Column(Text, primary_key=True, unique=True)
     user_id = Column(Text, nullable=False)
     provider = Column(Text, nullable=False)
     token = Column(
@@ -262,5 +262,16 @@ class OAuthSessionTable:
             log.error(f"Error deleting OAuth sessions by user ID: {e}")
             return False
 
+    def delete_sessions_by_provider(self, provider: str) -> bool:
+        """Delete all OAuth sessions for a provider"""
+        try:
+            with get_db() as db:
+                db.query(OAuthSession).filter_by(provider=provider).delete()
+                db.commit()
+                return True
+        except Exception as e:
+            log.error(f"Error deleting OAuth sessions by provider {provider}: {e}")
+            return False
+
 
 OAuthSessions = OAuthSessionTable()

backend/open_webui/models/tools.py

@@ -24,7 +24,7 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 class Tool(Base):
     __tablename__ = "tool"
 
-    id = Column(String, primary_key=True)
+    id = Column(String, primary_key=True, unique=True)
     user_id = Column(String)
     name = Column(Text)
     content = Column(Text)

backend/open_webui/models/users.py

@@ -6,13 +6,23 @@ from open_webui.internal.db import Base, JSONField, get_db
 
 from open_webui.env import DATABASE_USER_ACTIVE_STATUS_UPDATE_INTERVAL
 from open_webui.models.chats import Chats
-from open_webui.models.groups import Groups
+from open_webui.models.groups import Groups, GroupMember
 from open_webui.utils.misc import throttle
 
 
 from pydantic import BaseModel, ConfigDict
-from sqlalchemy import BigInteger, Column, String, Text, Date
-from sqlalchemy import or_
+from sqlalchemy import (
+    BigInteger,
+    JSON,
+    Column,
+    String,
+    Boolean,
+    Text,
+    Date,
+    exists,
+    select,
+)
+from sqlalchemy import or_, case
 
 import datetime
 
@@ -21,59 +31,71 @@ import datetime
 ####################
 
 
-class User(Base):
-    __tablename__ = "user"
-
-    id = Column(String, primary_key=True)
-    name = Column(String)
-
-    email = Column(String)
-    username = Column(String(50), nullable=True)
-
-    role = Column(String)
-    profile_image_url = Column(Text)
-
-    bio = Column(Text, nullable=True)
-    gender = Column(Text, nullable=True)
-    date_of_birth = Column(Date, nullable=True)
-
-    info = Column(JSONField, nullable=True)
-    settings = Column(JSONField, nullable=True)
-
-    api_key = Column(String, nullable=True, unique=True)
-    oauth_sub = Column(Text, unique=True)
-
-    last_active_at = Column(BigInteger)
-
-    updated_at = Column(BigInteger)
-    created_at = Column(BigInteger)
-
-
 class UserSettings(BaseModel):
     ui: Optional[dict] = {}
     model_config = ConfigDict(extra="allow")
     pass
 
 
+class User(Base):
+    __tablename__ = "user"
+
+    id = Column(String, primary_key=True, unique=True)
+    email = Column(String)
+    username = Column(String(50), nullable=True)
+    role = Column(String)
+
+    name = Column(String)
+
+    profile_image_url = Column(Text)
+    profile_banner_image_url = Column(Text, nullable=True)
+
+    bio = Column(Text, nullable=True)
+    gender = Column(Text, nullable=True)
+    date_of_birth = Column(Date, nullable=True)
+    timezone = Column(String, nullable=True)
+
+    presence_state = Column(String, nullable=True)
+    status_emoji = Column(String, nullable=True)
+    status_message = Column(Text, nullable=True)
+    status_expires_at = Column(BigInteger, nullable=True)
+
+    info = Column(JSON, nullable=True)
+    settings = Column(JSON, nullable=True)
+
+    oauth = Column(JSON, nullable=True)
+
+    last_active_at = Column(BigInteger)
+    updated_at = Column(BigInteger)
+    created_at = Column(BigInteger)
+
+
 class UserModel(BaseModel):
     id: str
-    name: str
 
     email: str
     username: Optional[str] = None
-
     role: str = "pending"
+
+    name: str
+
     profile_image_url: str
+    profile_banner_image_url: Optional[str] = None
 
     bio: Optional[str] = None
     gender: Optional[str] = None
     date_of_birth: Optional[datetime.date] = None
+    timezone: Optional[str] = None
+
+    presence_state: Optional[str] = None
+    status_emoji: Optional[str] = None
+    status_message: Optional[str] = None
+    status_expires_at: Optional[int] = None
 
     info: Optional[dict] = None
     settings: Optional[UserSettings] = None
 
-    api_key: Optional[str] = None
-    oauth_sub: Optional[str] = None
+    oauth: Optional[dict] = None
 
     last_active_at: int  # timestamp in epoch
     updated_at: int  # timestamp in epoch
@@ -82,6 +104,32 @@ class UserModel(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
 
+class ApiKey(Base):
+    __tablename__ = "api_key"
+
+    id = Column(Text, primary_key=True, unique=True)
+    user_id = Column(Text, nullable=False)
+    key = Column(Text, unique=True, nullable=False)
+    data = Column(JSON, nullable=True)
+    expires_at = Column(BigInteger, nullable=True)
+    last_used_at = Column(BigInteger, nullable=True)
+    created_at = Column(BigInteger, nullable=False)
+    updated_at = Column(BigInteger, nullable=False)
+
+
+class ApiKeyModel(BaseModel):
+    id: str
+    user_id: str
+    key: str
+    data: Optional[dict] = None
+    expires_at: Optional[int] = None
+    last_used_at: Optional[int] = None
+    created_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
+
+    model_config = ConfigDict(from_attributes=True)
+
+
 ####################
 # Forms
 ####################
@@ -95,8 +143,21 @@ class UpdateProfileForm(BaseModel):
     date_of_birth: Optional[datetime.date] = None
 
 
+class UserGroupIdsModel(UserModel):
+    group_ids: list[str] = []
+
+
+class UserModelResponse(UserModel):
+    model_config = ConfigDict(extra="allow")
+
+
 class UserListResponse(BaseModel):
-    users: list[UserModel]
+    users: list[UserModelResponse]
+    total: int
+
+
+class UserGroupIdsListResponse(BaseModel):
+    users: list[UserGroupIdsModel]
     total: int
 
 
@@ -112,6 +173,12 @@ class UserIdNameResponse(BaseModel):
     name: str
 
 
+class UserIdNameStatusResponse(BaseModel):
+    id: str
+    name: str
+    is_active: bool = False
+
+
 class UserInfoListResponse(BaseModel):
     users: list[UserInfoResponse]
     total: int
@@ -122,18 +189,18 @@ class UserIdNameListResponse(BaseModel):
     total: int
 
 
-class UserResponse(BaseModel):
-    id: str
-    name: str
-    email: str
-    role: str
-    profile_image_url: str
-
-
 class UserNameResponse(BaseModel):
     id: str
     name: str
     role: str
+
+
+class UserResponse(UserNameResponse):
+    email: str
+
+
+class UserProfileImageResponse(UserNameResponse):
+    email: str
     profile_image_url: str
 
 
@@ -158,20 +225,20 @@ class UsersTable:
         email: str,
         profile_image_url: str = "/user.png",
         role: str = "pending",
-        oauth_sub: Optional[str] = None,
+        oauth: Optional[dict] = None,
     ) -> Optional[UserModel]:
         with get_db() as db:
             user = UserModel(
                 **{
                     "id": id,
-                    "name": name,
                     "email": email,
+                    "name": name,
                     "role": role,
                     "profile_image_url": profile_image_url,
                     "last_active_at": int(time.time()),
                     "created_at": int(time.time()),
                     "updated_at": int(time.time()),
-                    "oauth_sub": oauth_sub,
+                    "oauth": oauth,
                 }
             )
             result = User(**user.model_dump())
@@ -194,8 +261,13 @@ class UsersTable:
     def get_user_by_api_key(self, api_key: str) -> Optional[UserModel]:
         try:
             with get_db() as db:
-                user = db.query(User).filter_by(api_key=api_key).first()
-                return UserModel.model_validate(user)
+                user = (
+                    db.query(User)
+                    .join(ApiKey, User.id == ApiKey.user_id)
+                    .filter(ApiKey.key == api_key)
+                    .first()
+                )
+                return UserModel.model_validate(user) if user else None
         except Exception:
             return None
 
@@ -207,11 +279,15 @@ class UsersTable:
         except Exception:
             return None
 
-    def get_user_by_oauth_sub(self, sub: str) -> Optional[UserModel]:
+    def get_user_by_oauth_sub(self, provider: str, sub: str) -> Optional[UserModel]:
         try:
             with get_db() as db:
-                user = db.query(User).filter_by(oauth_sub=sub).first()
-                return UserModel.model_validate(user)
+                user = (
+                    db.query(User)
+                    .filter(User.oauth.contains({provider: {"sub": sub}}))
+                    .first()
+                )
+                return UserModel.model_validate(user) if user else None
         except Exception:
             return None
 
@@ -222,6 +298,7 @@ class UsersTable:
         limit: Optional[int] = None,
     ) -> dict:
         with get_db() as db:
+            # Join GroupMember so we can order by group_id when requested
             query = db.query(User)
 
             if filter:
@@ -234,14 +311,65 @@ class UsersTable:
                         )
                     )
 
+                user_ids = filter.get("user_ids")
+                group_ids = filter.get("group_ids")
+
+                if isinstance(user_ids, list) and isinstance(group_ids, list):
+                    # If both are empty lists, return no users
+                    if not user_ids and not group_ids:
+                        return {"users": [], "total": 0}
+
+                if user_ids:
+                    query = query.filter(User.id.in_(user_ids))
+
+                if group_ids:
+                    query = query.filter(
+                        exists(
+                            select(GroupMember.id).where(
+                                GroupMember.user_id == User.id,
+                                GroupMember.group_id.in_(group_ids),
+                            )
+                        )
+                    )
+
+                roles = filter.get("roles")
+                if roles:
+                    include_roles = [role for role in roles if not role.startswith("!")]
+                    exclude_roles = [role[1:] for role in roles if role.startswith("!")]
+
+                    if include_roles:
+                        query = query.filter(User.role.in_(include_roles))
+                    if exclude_roles:
+                        query = query.filter(~User.role.in_(exclude_roles))
+
                 order_by = filter.get("order_by")
                 direction = filter.get("direction")
 
-                if order_by == "name":
+                if order_by and order_by.startswith("group_id:"):
+                    group_id = order_by.split(":", 1)[1]
+
+                    # Subquery that checks if the user belongs to the group
+                    membership_exists = exists(
+                        select(GroupMember.id).where(
+                            GroupMember.user_id == User.id,
+                            GroupMember.group_id == group_id,
+                        )
+                    )
+
+                    # CASE: user in group → 1, user not in group → 0
+                    group_sort = case((membership_exists, 1), else_=0)
+
+                    if direction == "asc":
+                        query = query.order_by(group_sort.asc(), User.name.asc())
+                    else:
+                        query = query.order_by(group_sort.desc(), User.name.asc())
+
+                elif order_by == "name":
                     if direction == "asc":
                         query = query.order_by(User.name.asc())
                     else:
                         query = query.order_by(User.name.desc())
+
                 elif order_by == "email":
                     if direction == "asc":
                         query = query.order_by(User.email.asc())
@@ -274,15 +402,19 @@ class UsersTable:
             else:
                 query = query.order_by(User.created_at.desc())
 
-            if skip:
+            # Count BEFORE pagination
+            total = query.count()
+
+            # correct pagination logic
+            if skip is not None:
                 query = query.offset(skip)
-            if limit:
+            if limit is not None:
                 query = query.limit(limit)
 
             users = query.all()
             return {
                 "users": [UserModel.model_validate(user) for user in users],
-                "total": db.query(User).count(),
+                "total": total,
             }
 
     def get_users_by_user_ids(self, user_ids: list[str]) -> list[UserModel]:
@@ -322,6 +454,15 @@ class UsersTable:
         except Exception:
             return None
 
+    def get_num_users_active_today(self) -> Optional[int]:
+        with get_db() as db:
+            current_timestamp = int(datetime.datetime.now().timestamp())
+            today_midnight_timestamp = current_timestamp - (current_timestamp % 86400)
+            query = db.query(User).filter(
+                User.last_active_at > today_midnight_timestamp
+            )
+            return query.count()
+
     def update_user_role_by_id(self, id: str, role: str) -> Optional[UserModel]:
         try:
             with get_db() as db:
@@ -348,7 +489,7 @@ class UsersTable:
             return None
 
     @throttle(DATABASE_USER_ACTIVE_STATUS_UPDATE_INTERVAL)
-    def update_user_last_active_by_id(self, id: str) -> Optional[UserModel]:
+    def update_last_active_by_id(self, id: str) -> Optional[UserModel]:
         try:
             with get_db() as db:
                 db.query(User).filter_by(id=id).update(
@@ -361,16 +502,35 @@ class UsersTable:
         except Exception:
             return None
 
-    def update_user_oauth_sub_by_id(
-        self, id: str, oauth_sub: str
+    def update_user_oauth_by_id(
+        self, id: str, provider: str, sub: str
     ) -> Optional[UserModel]:
+        """
+        Update or insert an OAuth provider/sub pair into the user's oauth JSON field.
+        Example resulting structure:
+            {
+                "google": { "sub": "123" },
+                "github": { "sub": "abc" }
+            }
+        """
         try:
             with get_db() as db:
-                db.query(User).filter_by(id=id).update({"oauth_sub": oauth_sub})
+                user = db.query(User).filter_by(id=id).first()
+                if not user:
+                    return None
+
+                # Load existing oauth JSON or create empty
+                oauth = user.oauth or {}
+
+                # Update or insert provider entry
+                oauth[provider] = {"sub": sub}
+
+                # Persist updated JSON
+                db.query(User).filter_by(id=id).update({"oauth": oauth})
                 db.commit()
 
-                user = db.query(User).filter_by(id=id).first()
                 return UserModel.model_validate(user)
+
         except Exception:
             return None
 
@@ -424,23 +584,45 @@ class UsersTable:
         except Exception:
             return False
 
-    def update_user_api_key_by_id(self, id: str, api_key: str) -> bool:
-        try:
-            with get_db() as db:
-                result = db.query(User).filter_by(id=id).update({"api_key": api_key})
-                db.commit()
-                return True if result == 1 else False
-        except Exception:
-            return False
-
     def get_user_api_key_by_id(self, id: str) -> Optional[str]:
         try:
             with get_db() as db:
-                user = db.query(User).filter_by(id=id).first()
-                return user.api_key
+                api_key = db.query(ApiKey).filter_by(user_id=id).first()
+                return api_key.key if api_key else None
         except Exception:
             return None
 
+    def update_user_api_key_by_id(self, id: str, api_key: str) -> bool:
+        try:
+            with get_db() as db:
+                db.query(ApiKey).filter_by(user_id=id).delete()
+                db.commit()
+
+                now = int(time.time())
+                new_api_key = ApiKey(
+                    id=f"key_{id}",
+                    user_id=id,
+                    key=api_key,
+                    created_at=now,
+                    updated_at=now,
+                )
+                db.add(new_api_key)
+                db.commit()
+
+                return True
+
+        except Exception:
+            return False
+
+    def delete_user_api_key_by_id(self, id: str) -> bool:
+        try:
+            with get_db() as db:
+                db.query(ApiKey).filter_by(user_id=id).delete()
+                db.commit()
+                return True
+        except Exception:
+            return False
+
     def get_valid_user_ids(self, user_ids: list[str]) -> list[str]:
         with get_db() as db:
             users = db.query(User).filter(User.id.in_(user_ids)).all()
@@ -454,5 +636,23 @@ class UsersTable:
             else:
                 return None
 
+    def get_active_user_count(self) -> int:
+        with get_db() as db:
+            # Consider user active if last_active_at within the last 3 minutes
+            three_minutes_ago = int(time.time()) - 180
+            count = (
+                db.query(User).filter(User.last_active_at >= three_minutes_ago).count()
+            )
+            return count
+
+    def is_user_active(self, user_id: str) -> bool:
+        with get_db() as db:
+            user = db.query(User).filter_by(id=user_id).first()
+            if user and user.last_active_at:
+                # Consider user active if last_active_at within the last 3 minutes
+                three_minutes_ago = int(time.time()) - 180
+                return user.last_active_at >= three_minutes_ago
+            return False
+
 
 Users = UsersTable()

backend/open_webui/retrieval/loaders/external_document.py

@@ -5,6 +5,7 @@ from urllib.parse import quote
 
 from langchain_core.document_loaders import BaseLoader
 from langchain_core.documents import Document
+from open_webui.utils.headers import include_user_info_headers
 from open_webui.env import SRC_LOG_LEVELS
 
 log = logging.getLogger(__name__)
@@ -18,6 +19,7 @@ class ExternalDocumentLoader(BaseLoader):
         url: str,
         api_key: str,
         mime_type=None,
+        user=None,
         **kwargs,
     ) -> None:
         self.url = url
@@ -26,6 +28,8 @@ class ExternalDocumentLoader(BaseLoader):
         self.file_path = file_path
         self.mime_type = mime_type
 
+        self.user = user
+
     def load(self) -> List[Document]:
         with open(self.file_path, "rb") as f:
             data = f.read()
@@ -42,6 +46,9 @@ class ExternalDocumentLoader(BaseLoader):
         except:
             pass
 
+        if self.user is not None:
+            headers = include_user_info_headers(headers, self.user)
+
         url = self.url
         if url.endswith("/"):
             url = url[:-1]

backend/open_webui/retrieval/loaders/main.py

@@ -27,6 +27,7 @@ from open_webui.retrieval.loaders.external_document import ExternalDocumentLoade
 
 from open_webui.retrieval.loaders.mistral import MistralLoader
 from open_webui.retrieval.loaders.datalab_marker import DatalabMarkerLoader
+from open_webui.retrieval.loaders.mineru import MinerULoader
 
 
 from open_webui.env import SRC_LOG_LEVELS, GLOBAL_LOG_LEVEL
@@ -131,8 +132,9 @@ class TikaLoader:
 
 
 class DoclingLoader:
-    def __init__(self, url, file_path=None, mime_type=None, params=None):
+    def __init__(self, url, api_key=None, file_path=None, mime_type=None, params=None):
         self.url = url.rstrip("/")
+        self.api_key = api_key
         self.file_path = file_path
         self.mime_type = mime_type
 
@@ -140,6 +142,10 @@ class DoclingLoader:
 
     def load(self) -> list[Document]:
         with open(self.file_path, "rb") as f:
+            headers = {}
+            if self.api_key:
+                headers["Authorization"] = f"Bearer {self.api_key}"
+
             files = {
                 "files": (
                     self.file_path,
@@ -148,60 +154,15 @@ class DoclingLoader:
                 )
             }
 
-            params = {"image_export_mode": "placeholder"}
-
-            if self.params:
-                if self.params.get("do_picture_description"):
-                    params["do_picture_description"] = self.params.get(
-                        "do_picture_description"
+            r = requests.post(
+                f"{self.url}/v1/convert/file",
+                files=files,
+                data={
+                    "image_export_mode": "placeholder",
+                    **self.params,
+                },
+                headers=headers,
             )
-
-                    picture_description_mode = self.params.get(
-                        "picture_description_mode", ""
-                    ).lower()
-
-                    if picture_description_mode == "local" and self.params.get(
-                        "picture_description_local", {}
-                    ):
-                        params["picture_description_local"] = json.dumps(
-                            self.params.get("picture_description_local", {})
-                        )
-
-                    elif picture_description_mode == "api" and self.params.get(
-                        "picture_description_api", {}
-                    ):
-                        params["picture_description_api"] = json.dumps(
-                            self.params.get("picture_description_api", {})
-                        )
-
-                params["do_ocr"] = self.params.get("do_ocr")
-
-                params["force_ocr"] = self.params.get("force_ocr")
-
-                if (
-                    self.params.get("do_ocr")
-                    and self.params.get("ocr_engine")
-                    and self.params.get("ocr_lang")
-                ):
-                    params["ocr_engine"] = self.params.get("ocr_engine")
-                    params["ocr_lang"] = [
-                        lang.strip()
-                        for lang in self.params.get("ocr_lang").split(",")
-                        if lang.strip()
-                    ]
-
-                if self.params.get("pdf_backend"):
-                    params["pdf_backend"] = self.params.get("pdf_backend")
-
-                if self.params.get("table_mode"):
-                    params["table_mode"] = self.params.get("table_mode")
-
-                if self.params.get("pipeline"):
-                    params["pipeline"] = self.params.get("pipeline")
-
-            endpoint = f"{self.url}/v1/convert/file"
-            r = requests.post(endpoint, files=files, data=params)
-
         if r.ok:
             result = r.json()
             document_data = result.get("document", {})
@@ -210,7 +171,6 @@ class DoclingLoader:
             metadata = {"Content-Type": self.mime_type} if self.mime_type else {}
 
             log.debug("Docling extracted text: %s", text)
-
             return [Document(page_content=text, metadata=metadata)]
         else:
             error_msg = f"Error calling Docling API: {r.reason}"
@@ -227,6 +187,7 @@ class DoclingLoader:
 class Loader:
     def __init__(self, engine: str = "", **kwargs):
         self.engine = engine
+        self.user = kwargs.get("user", None)
         self.kwargs = kwargs
 
     def load(
@@ -263,6 +224,7 @@ class Loader:
                 url=self.kwargs.get("EXTERNAL_DOCUMENT_LOADER_URL"),
                 api_key=self.kwargs.get("EXTERNAL_DOCUMENT_LOADER_API_KEY"),
                 mime_type=file_content_type,
+                user=self.user,
             )
         elif self.engine == "tika" and self.kwargs.get("TIKA_SERVER_URL"):
             if self._is_text_file(file_ext, file_content_type):
@@ -271,7 +233,6 @@ class Loader:
                 loader = TikaLoader(
                     url=self.kwargs.get("TIKA_SERVER_URL"),
                     file_path=file_path,
-                    mime_type=file_content_type,
                     extract_images=self.kwargs.get("PDF_EXTRACT_IMAGES"),
                 )
         elif (
@@ -338,6 +299,7 @@ class Loader:
 
                 loader = DoclingLoader(
                     url=self.kwargs.get("DOCLING_SERVER_URL"),
+                    api_key=self.kwargs.get("DOCLING_API_KEY", None),
                     file_path=file_path,
                     mime_type=file_content_type,
                     params=params,
@@ -367,6 +329,16 @@ class Loader:
                     api_endpoint=self.kwargs.get("DOCUMENT_INTELLIGENCE_ENDPOINT"),
                     azure_credential=DefaultAzureCredential(),
                 )
+        elif self.engine == "mineru" and file_ext in [
+            "pdf"
+        ]:  # MinerU currently only supports PDF
+            loader = MinerULoader(
+                file_path=file_path,
+                api_mode=self.kwargs.get("MINERU_API_MODE", "local"),
+                api_url=self.kwargs.get("MINERU_API_URL", "http://localhost:8000"),
+                api_key=self.kwargs.get("MINERU_API_KEY", ""),
+                params=self.kwargs.get("MINERU_PARAMS", {}),
+            )
         elif (
             self.engine == "mistral_ocr"
             and self.kwargs.get("MISTRAL_OCR_API_KEY") != ""
@@ -374,16 +346,9 @@ class Loader:
             in ["pdf"]  # Mistral OCR currently only supports PDF and images
         ):
             loader = MistralLoader(
-                api_key=self.kwargs.get("MISTRAL_OCR_API_KEY"), file_path=file_path
-            )
-        elif (
-            self.engine == "external"
-            and self.kwargs.get("MISTRAL_OCR_API_KEY") != ""
-            and file_ext
-            in ["pdf"]  # Mistral OCR currently only supports PDF and images
-        ):
-            loader = MistralLoader(
-                api_key=self.kwargs.get("MISTRAL_OCR_API_KEY"), file_path=file_path
+                base_url=self.kwargs.get("MISTRAL_OCR_API_BASE_URL"),
+                api_key=self.kwargs.get("MISTRAL_OCR_API_KEY"),
+                file_path=file_path,
             )
         else:
             if file_ext == "pdf":

backend/open_webui/retrieval/loaders/mineru.py

@@ -0,0 +1,522 @@
+import os
+import time
+import requests
+import logging
+import tempfile
+import zipfile
+from typing import List, Optional
+from langchain_core.documents import Document
+from fastapi import HTTPException, status
+
+log = logging.getLogger(__name__)
+
+
+class MinerULoader:
+    """
+    MinerU document parser loader supporting both Cloud API and Local API modes.
+
+    Cloud API: Uses MinerU managed service with async task-based processing
+    Local API: Uses self-hosted MinerU API with synchronous processing
+    """
+
+    def __init__(
+        self,
+        file_path: str,
+        api_mode: str = "local",
+        api_url: str = "http://localhost:8000",
+        api_key: str = "",
+        params: dict = None,
+    ):
+        self.file_path = file_path
+        self.api_mode = api_mode.lower()
+        self.api_url = api_url.rstrip("/")
+        self.api_key = api_key
+
+        # Parse params dict with defaults
+        self.params = params or {}
+        self.enable_ocr = params.get("enable_ocr", False)
+        self.enable_formula = params.get("enable_formula", True)
+        self.enable_table = params.get("enable_table", True)
+        self.language = params.get("language", "en")
+        self.model_version = params.get("model_version", "pipeline")
+
+        self.page_ranges = self.params.pop("page_ranges", "")
+
+        # Validate API mode
+        if self.api_mode not in ["local", "cloud"]:
+            raise ValueError(
+                f"Invalid API mode: {self.api_mode}. Must be 'local' or 'cloud'"
+            )
+
+        # Validate Cloud API requirements
+        if self.api_mode == "cloud" and not self.api_key:
+            raise ValueError("API key is required for Cloud API mode")
+
+    def load(self) -> List[Document]:
+        """
+        Main entry point for loading and parsing the document.
+        Routes to Cloud or Local API based on api_mode.
+        """
+        try:
+            if self.api_mode == "cloud":
+                return self._load_cloud_api()
+            else:
+                return self._load_local_api()
+        except Exception as e:
+            log.error(f"Error loading document with MinerU: {e}")
+            raise
+
+    def _load_local_api(self) -> List[Document]:
+        """
+        Load document using Local API (synchronous).
+        Posts file to /file_parse endpoint and gets immediate response.
+        """
+        log.info(f"Using MinerU Local API at {self.api_url}")
+
+        filename = os.path.basename(self.file_path)
+
+        # Build form data for Local API
+        form_data = {
+            **self.params,
+            "return_md": "true",
+        }
+
+        # Page ranges (Local API uses start_page_id and end_page_id)
+        if self.page_ranges:
+            # For simplicity, if page_ranges is specified, log a warning
+            # Full page range parsing would require parsing the string
+            log.warning(
+                f"Page ranges '{self.page_ranges}' specified but Local API uses different format. "
+                "Consider using start_page_id/end_page_id parameters if needed."
+            )
+
+        try:
+            with open(self.file_path, "rb") as f:
+                files = {"files": (filename, f, "application/octet-stream")}
+
+                log.info(f"Sending file to MinerU Local API: {filename}")
+                log.debug(f"Local API parameters: {form_data}")
+
+                response = requests.post(
+                    f"{self.api_url}/file_parse",
+                    data=form_data,
+                    files=files,
+                    timeout=300,  # 5 minute timeout for large documents
+                )
+                response.raise_for_status()
+
+        except FileNotFoundError:
+            raise HTTPException(
+                status.HTTP_404_NOT_FOUND, detail=f"File not found: {self.file_path}"
+            )
+        except requests.Timeout:
+            raise HTTPException(
+                status.HTTP_504_GATEWAY_TIMEOUT,
+                detail="MinerU Local API request timed out",
+            )
+        except requests.HTTPError as e:
+            error_detail = f"MinerU Local API request failed: {e}"
+            if e.response is not None:
+                try:
+                    error_data = e.response.json()
+                    error_detail += f" - {error_data}"
+                except:
+                    error_detail += f" - {e.response.text}"
+            raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=error_detail)
+        except Exception as e:
+            raise HTTPException(
+                status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Error calling MinerU Local API: {str(e)}",
+            )
+
+        # Parse response
+        try:
+            result = response.json()
+        except ValueError as e:
+            raise HTTPException(
+                status.HTTP_502_BAD_GATEWAY,
+                detail=f"Invalid JSON response from MinerU Local API: {e}",
+            )
+
+        # Extract markdown content from response
+        if "results" not in result:
+            raise HTTPException(
+                status.HTTP_502_BAD_GATEWAY,
+                detail="MinerU Local API response missing 'results' field",
+            )
+
+        results = result["results"]
+        if not results:
+            raise HTTPException(
+                status.HTTP_400_BAD_REQUEST,
+                detail="MinerU returned empty results",
+            )
+
+        # Get the first (and typically only) result
+        file_result = list(results.values())[0]
+        markdown_content = file_result.get("md_content", "")
+
+        if not markdown_content:
+            raise HTTPException(
+                status.HTTP_400_BAD_REQUEST,
+                detail="MinerU returned empty markdown content",
+            )
+
+        log.info(f"Successfully parsed document with MinerU Local API: {filename}")
+
+        # Create metadata
+        metadata = {
+            "source": filename,
+            "api_mode": "local",
+            "backend": result.get("backend", "unknown"),
+            "version": result.get("version", "unknown"),
+        }
+
+        return [Document(page_content=markdown_content, metadata=metadata)]
+
+    def _load_cloud_api(self) -> List[Document]:
+        """
+        Load document using Cloud API (asynchronous).
+        Uses batch upload endpoint to avoid need for public file URLs.
+        """
+        log.info(f"Using MinerU Cloud API at {self.api_url}")
+
+        filename = os.path.basename(self.file_path)
+
+        # Step 1: Request presigned upload URL
+        batch_id, upload_url = self._request_upload_url(filename)
+
+        # Step 2: Upload file to presigned URL
+        self._upload_to_presigned_url(upload_url)
+
+        # Step 3: Poll for results
+        result = self._poll_batch_status(batch_id, filename)
+
+        # Step 4: Download and extract markdown from ZIP
+        markdown_content = self._download_and_extract_zip(
+            result["full_zip_url"], filename
+        )
+
+        log.info(f"Successfully parsed document with MinerU Cloud API: {filename}")
+
+        # Create metadata
+        metadata = {
+            "source": filename,
+            "api_mode": "cloud",
+            "batch_id": batch_id,
+        }
+
+        return [Document(page_content=markdown_content, metadata=metadata)]
+
+    def _request_upload_url(self, filename: str) -> tuple:
+        """
+        Request presigned upload URL from Cloud API.
+        Returns (batch_id, upload_url).
+        """
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        # Build request body
+        request_body = {
+            **self.params,
+            "files": [
+                {
+                    "name": filename,
+                    "is_ocr": self.enable_ocr,
+                }
+            ],
+        }
+
+        # Add page ranges if specified
+        if self.page_ranges:
+            request_body["files"][0]["page_ranges"] = self.page_ranges
+
+        log.info(f"Requesting upload URL for: {filename}")
+        log.debug(f"Cloud API request body: {request_body}")
+
+        try:
+            response = requests.post(
+                f"{self.api_url}/file-urls/batch",
+                headers=headers,
+                json=request_body,
+                timeout=30,
+            )
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            error_detail = f"Failed to request upload URL: {e}"
+            if e.response is not None:
+                try:
+                    error_data = e.response.json()
+                    error_detail += f" - {error_data.get('msg', error_data)}"
+                except:
+                    error_detail += f" - {e.response.text}"
+            raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=error_detail)
+        except Exception as e:
+            raise HTTPException(
+                status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Error requesting upload URL: {str(e)}",
+            )
+
+        try:
+            result = response.json()
+        except ValueError as e:
+            raise HTTPException(
+                status.HTTP_502_BAD_GATEWAY,
+                detail=f"Invalid JSON response: {e}",
+            )
+
+        # Check for API error response
+        if result.get("code") != 0:
+            raise HTTPException(
+                status.HTTP_400_BAD_REQUEST,
+                detail=f"MinerU Cloud API error: {result.get('msg', 'Unknown error')}",
+            )
+
+        data = result.get("data", {})
+        batch_id = data.get("batch_id")
+        file_urls = data.get("file_urls", [])
+
+        if not batch_id or not file_urls:
+            raise HTTPException(
+                status.HTTP_502_BAD_GATEWAY,
+                detail="MinerU Cloud API response missing batch_id or file_urls",
+            )
+
+        upload_url = file_urls[0]
+        log.info(f"Received upload URL for batch: {batch_id}")
+
+        return batch_id, upload_url
+
+    def _upload_to_presigned_url(self, upload_url: str) -> None:
+        """
+        Upload file to presigned URL (no authentication needed).
+        """
+        log.info(f"Uploading file to presigned URL")
+
+        try:
+            with open(self.file_path, "rb") as f:
+                response = requests.put(
+                    upload_url,
+                    data=f,
+                    timeout=300,  # 5 minute timeout for large files
+                )
+                response.raise_for_status()
+        except FileNotFoundError:
+            raise HTTPException(
+                status.HTTP_404_NOT_FOUND, detail=f"File not found: {self.file_path}"
+            )
+        except requests.Timeout:
+            raise HTTPException(
+                status.HTTP_504_GATEWAY_TIMEOUT,
+                detail="File upload to presigned URL timed out",
+            )
+        except requests.HTTPError as e:
+            raise HTTPException(
+                status.HTTP_400_BAD_REQUEST,
+                detail=f"Failed to upload file to presigned URL: {e}",
+            )
+        except Exception as e:
+            raise HTTPException(
+                status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Error uploading file: {str(e)}",
+            )
+
+        log.info("File uploaded successfully")
+
+    def _poll_batch_status(self, batch_id: str, filename: str) -> dict:
+        """
+        Poll batch status until completion.
+        Returns the result dict for the file.
+        """
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+        }
+
+        max_iterations = 300  # 10 minutes max (2 seconds per iteration)
+        poll_interval = 2  # seconds
+
+        log.info(f"Polling batch status: {batch_id}")
+
+        for iteration in range(max_iterations):
+            try:
+                response = requests.get(
+                    f"{self.api_url}/extract-results/batch/{batch_id}",
+                    headers=headers,
+                    timeout=30,
+                )
+                response.raise_for_status()
+            except requests.HTTPError as e:
+                error_detail = f"Failed to poll batch status: {e}"
+                if e.response is not None:
+                    try:
+                        error_data = e.response.json()
+                        error_detail += f" - {error_data.get('msg', error_data)}"
+                    except:
+                        error_detail += f" - {e.response.text}"
+                raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=error_detail)
+            except Exception as e:
+                raise HTTPException(
+                    status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=f"Error polling batch status: {str(e)}",
+                )
+
+            try:
+                result = response.json()
+            except ValueError as e:
+                raise HTTPException(
+                    status.HTTP_502_BAD_GATEWAY,
+                    detail=f"Invalid JSON response while polling: {e}",
+                )
+
+            # Check for API error response
+            if result.get("code") != 0:
+                raise HTTPException(
+                    status.HTTP_400_BAD_REQUEST,
+                    detail=f"MinerU Cloud API error: {result.get('msg', 'Unknown error')}",
+                )
+
+            data = result.get("data", {})
+            extract_result = data.get("extract_result", [])
+
+            # Find our file in the batch results
+            file_result = None
+            for item in extract_result:
+                if item.get("file_name") == filename:
+                    file_result = item
+                    break
+
+            if not file_result:
+                raise HTTPException(
+                    status.HTTP_502_BAD_GATEWAY,
+                    detail=f"File {filename} not found in batch results",
+                )
+
+            state = file_result.get("state")
+
+            if state == "done":
+                log.info(f"Processing complete for {filename}")
+                return file_result
+            elif state == "failed":
+                error_msg = file_result.get("err_msg", "Unknown error")
+                raise HTTPException(
+                    status.HTTP_400_BAD_REQUEST,
+                    detail=f"MinerU processing failed: {error_msg}",
+                )
+            elif state in ["waiting-file", "pending", "running", "converting"]:
+                # Still processing
+                if iteration % 10 == 0:  # Log every 20 seconds
+                    log.info(
+                        f"Processing status: {state} (iteration {iteration + 1}/{max_iterations})"
+                    )
+                time.sleep(poll_interval)
+            else:
+                log.warning(f"Unknown state: {state}")
+                time.sleep(poll_interval)
+
+        # Timeout
+        raise HTTPException(
+            status.HTTP_504_GATEWAY_TIMEOUT,
+            detail="MinerU processing timed out after 10 minutes",
+        )
+
+    def _download_and_extract_zip(self, zip_url: str, filename: str) -> str:
+        """
+        Download ZIP file from CDN and extract markdown content.
+        Returns the markdown content as a string.
+        """
+        log.info(f"Downloading results from: {zip_url}")
+
+        try:
+            response = requests.get(zip_url, timeout=60)
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            raise HTTPException(
+                status.HTTP_400_BAD_REQUEST,
+                detail=f"Failed to download results ZIP: {e}",
+            )
+        except Exception as e:
+            raise HTTPException(
+                status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Error downloading results: {str(e)}",
+            )
+
+        # Save ZIP to temporary file and extract
+        try:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as tmp_zip:
+                tmp_zip.write(response.content)
+                tmp_zip_path = tmp_zip.name
+
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                # Extract ZIP
+                with zipfile.ZipFile(tmp_zip_path, "r") as zip_ref:
+                    zip_ref.extractall(tmp_dir)
+
+                # Find markdown file - search recursively for any .md file
+                markdown_content = None
+                found_md_path = None
+
+                # First, list all files in the ZIP for debugging
+                all_files = []
+                for root, dirs, files in os.walk(tmp_dir):
+                    for file in files:
+                        full_path = os.path.join(root, file)
+                        all_files.append(full_path)
+                        # Look for any .md file
+                        if file.endswith(".md"):
+                            found_md_path = full_path
+                            log.info(f"Found markdown file at: {full_path}")
+                            try:
+                                with open(full_path, "r", encoding="utf-8") as f:
+                                    markdown_content = f.read()
+                                if (
+                                    markdown_content
+                                ):  # Use the first non-empty markdown file
+                                    break
+                            except Exception as e:
+                                log.warning(f"Failed to read {full_path}: {e}")
+                    if markdown_content:
+                        break
+
+                if markdown_content is None:
+                    log.error(f"Available files in ZIP: {all_files}")
+                    # Try to provide more helpful error message
+                    md_files = [f for f in all_files if f.endswith(".md")]
+                    if md_files:
+                        error_msg = (
+                            f"Found .md files but couldn't read them: {md_files}"
+                        )
+                    else:
+                        error_msg = (
+                            f"No .md files found in ZIP. Available files: {all_files}"
+                        )
+                    raise HTTPException(
+                        status.HTTP_502_BAD_GATEWAY,
+                        detail=error_msg,
+                    )
+
+            # Clean up temporary ZIP file
+            os.unlink(tmp_zip_path)
+
+        except zipfile.BadZipFile as e:
+            raise HTTPException(
+                status.HTTP_502_BAD_GATEWAY,
+                detail=f"Invalid ZIP file received: {e}",
+            )
+        except Exception as e:
+            raise HTTPException(
+                status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Error extracting ZIP: {str(e)}",
+            )
+
+        if not markdown_content:
+            raise HTTPException(
+                status.HTTP_400_BAD_REQUEST,
+                detail="Extracted markdown content is empty",
+            )
+
+        log.info(
+            f"Successfully extracted markdown content ({len(markdown_content)} characters)"
+        )
+        return markdown_content

backend/open_webui/retrieval/loaders/mistral.py

@@ -30,10 +30,9 @@ class MistralLoader:
     - Enhanced error handling with retryable error classification
     """
 
-    BASE_API_URL = "https://api.mistral.ai/v1"
-
     def __init__(
         self,
+        base_url: str,
         api_key: str,
         file_path: str,
         timeout: int = 300,  # 5 minutes default
@@ -55,6 +54,9 @@ class MistralLoader:
         if not os.path.exists(file_path):
             raise FileNotFoundError(f"File not found at {file_path}")
 
+        self.base_url = (
+            base_url.rstrip("/") if base_url else "https://api.mistral.ai/v1"
+        )
         self.api_key = api_key
         self.file_path = file_path
         self.timeout = timeout
@@ -240,7 +242,7 @@ class MistralLoader:
         in a context manager to minimize memory usage duration.
         """
         log.info("Uploading file to Mistral API")
-        url = f"{self.BASE_API_URL}/files"
+        url = f"{self.base_url}/files"
 
         def upload_request():
             # MEMORY OPTIMIZATION: Use context manager to minimize file handle lifetime
@@ -275,7 +277,7 @@ class MistralLoader:
 
     async def _upload_file_async(self, session: aiohttp.ClientSession) -> str:
         """Async file upload with streaming for better memory efficiency."""
-        url = f"{self.BASE_API_URL}/files"
+        url = f"{self.base_url}/files"
 
         async def upload_request():
             # Create multipart writer for streaming upload
@@ -321,7 +323,7 @@ class MistralLoader:
     def _get_signed_url(self, file_id: str) -> str:
         """Retrieves a temporary signed URL for the uploaded file (sync version)."""
         log.info(f"Getting signed URL for file ID: {file_id}")
-        url = f"{self.BASE_API_URL}/files/{file_id}/url"
+        url = f"{self.base_url}/files/{file_id}/url"
         params = {"expiry": 1}
         signed_url_headers = {**self.headers, "Accept": "application/json"}
 
@@ -346,7 +348,7 @@ class MistralLoader:
         self, session: aiohttp.ClientSession, file_id: str
     ) -> str:
         """Async signed URL retrieval."""
-        url = f"{self.BASE_API_URL}/files/{file_id}/url"
+        url = f"{self.base_url}/files/{file_id}/url"
         params = {"expiry": 1}
 
         headers = {**self.headers, "Accept": "application/json"}
@@ -373,7 +375,7 @@ class MistralLoader:
     def _process_ocr(self, signed_url: str) -> Dict[str, Any]:
         """Sends the signed URL to the OCR endpoint for processing (sync version)."""
         log.info("Processing OCR via Mistral API")
-        url = f"{self.BASE_API_URL}/ocr"
+        url = f"{self.base_url}/ocr"
         ocr_headers = {
             **self.headers,
             "Content-Type": "application/json",
@@ -407,7 +409,7 @@ class MistralLoader:
         self, session: aiohttp.ClientSession, signed_url: str
     ) -> Dict[str, Any]:
         """Async OCR processing with timing metrics."""
-        url = f"{self.BASE_API_URL}/ocr"
+        url = f"{self.base_url}/ocr"
 
         headers = {
             **self.headers,
@@ -446,7 +448,7 @@ class MistralLoader:
     def _delete_file(self, file_id: str) -> None:
         """Deletes the file from Mistral storage (sync version)."""
         log.info(f"Deleting uploaded file ID: {file_id}")
-        url = f"{self.BASE_API_URL}/files/{file_id}"
+        url = f"{self.base_url}/files/{file_id}"
 
         try:
             response = requests.delete(
@@ -467,7 +469,7 @@ class MistralLoader:
             async def delete_request():
                 self._debug_log(f"Deleting file ID: {file_id}")
                 async with session.delete(
-                    url=f"{self.BASE_API_URL}/files/{file_id}",
+                    url=f"{self.base_url}/files/{file_id}",
                     headers=self.headers,
                     timeout=aiohttp.ClientTimeout(
                         total=self.cleanup_timeout

backend/open_webui/retrieval/loaders/youtube.py

@@ -83,6 +83,7 @@ class YoutubeLoader:
                 TranscriptsDisabled,
                 YouTubeTranscriptApi,
             )
+            from youtube_transcript_api.proxies import GenericProxyConfig
         except ImportError:
             raise ImportError(
                 'Could not import "youtube_transcript_api" Python package. '
@@ -90,10 +91,9 @@ class YoutubeLoader:
             )
 
         if self.proxy_url:
-            youtube_proxies = {
-                "http": self.proxy_url,
-                "https": self.proxy_url,
-            }
+            youtube_proxies = GenericProxyConfig(
+                http_url=self.proxy_url, https_url=self.proxy_url
+            )
             log.debug(f"Using proxy URL: {self.proxy_url[:14]}...")
         else:
             youtube_proxies = None
@@ -157,3 +157,10 @@ class YoutubeLoader:
             f"No transcript found for any of the specified languages: {languages_tried}. Verify if the video has transcripts, add more languages if needed."
         )
         raise NoTranscriptFound(self.video_id, self.language, list(transcript_list))
+
+    async def aload(self) -> Generator[Document, None, None]:
+        """Asynchronously load YouTube transcripts into `Document` objects."""
+        import asyncio
+
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, self.load)

backend/open_webui/retrieval/models/external.py

@@ -6,6 +6,7 @@ from urllib.parse import quote
 
 from open_webui.env import ENABLE_FORWARD_USER_INFO_HEADERS, SRC_LOG_LEVELS
 from open_webui.retrieval.models.base_reranker import BaseReranker
+from open_webui.utils.headers import include_user_info_headers
 
 
 log = logging.getLogger(__name__)
@@ -40,22 +41,17 @@ class ExternalReranker(BaseReranker):
             log.info(f"ExternalReranker:predict:model {self.model}")
             log.info(f"ExternalReranker:predict:query {query}")
 
-            r = requests.post(
-                f"{self.url}",
-                headers={
+            headers = {
                 "Content-Type": "application/json",
                 "Authorization": f"Bearer {self.api_key}",
-                    **(
-                        {
-                            "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                            "X-OpenWebUI-User-Id": user.id,
-                            "X-OpenWebUI-User-Email": user.email,
-                            "X-OpenWebUI-User-Role": user.role,
             }
-                        if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                        else {}
-                    ),
-                },
+
+            if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+                headers = include_user_info_headers(headers, user)
+
+            r = requests.post(
+                f"{self.url}",
+                headers=headers,
                 json=payload,
             )
 

backend/open_webui/retrieval/utils.py

@@ -1,11 +1,14 @@
 import logging
 import os
-from typing import Optional, Union
+from typing import Awaitable, Optional, Union
 
 import requests
+import aiohttp
+import asyncio
 import hashlib
 from concurrent.futures import ThreadPoolExecutor
 import time
+import re
 
 from urllib.parse import quote
 from huggingface_hub import snapshot_download
@@ -16,6 +19,7 @@ from langchain_core.documents import Document
 from open_webui.config import VECTOR_DB
 from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT
 
+
 from open_webui.models.users import UserModel
 from open_webui.models.files import Files
 from open_webui.models.knowledge import Knowledges
@@ -25,8 +29,12 @@ from open_webui.models.notes import Notes
 
 from open_webui.retrieval.vector.main import GetResult
 from open_webui.utils.access_control import has_access
+from open_webui.utils.headers import include_user_info_headers
 from open_webui.utils.misc import get_message_list
 
+from open_webui.retrieval.web.utils import get_web_loader
+from open_webui.retrieval.loaders.youtube import YoutubeLoader
+
 
 from open_webui.env import (
     SRC_LOG_LEVELS,
@@ -49,20 +57,63 @@ from langchain_core.callbacks import CallbackManagerForRetrieverRun
 from langchain_core.retrievers import BaseRetriever
 
 
+def is_youtube_url(url: str) -> bool:
+    youtube_regex = r"^(https?://)?(www\.)?(youtube\.com|youtu\.be)/.+$"
+    return re.match(youtube_regex, url) is not None
+
+
+def get_loader(request, url: str):
+    if is_youtube_url(url):
+        return YoutubeLoader(
+            url,
+            language=request.app.state.config.YOUTUBE_LOADER_LANGUAGE,
+            proxy_url=request.app.state.config.YOUTUBE_LOADER_PROXY_URL,
+        )
+    else:
+        return get_web_loader(
+            url,
+            verify_ssl=request.app.state.config.ENABLE_WEB_LOADER_SSL_VERIFICATION,
+            requests_per_second=request.app.state.config.WEB_LOADER_CONCURRENT_REQUESTS,
+            trust_env=request.app.state.config.WEB_SEARCH_TRUST_ENV,
+        )
+
+
+def get_content_from_url(request, url: str) -> str:
+    loader = get_loader(request, url)
+    docs = loader.load()
+    content = " ".join([doc.page_content for doc in docs])
+    return content, docs
+
+
 class VectorSearchRetriever(BaseRetriever):
     collection_name: Any
     embedding_function: Any
     top_k: int
 
     def _get_relevant_documents(
+        self, query: str, *, run_manager: CallbackManagerForRetrieverRun
+    ) -> list[Document]:
+        """Get documents relevant to a query.
+
+        Args:
+            query: String to find relevant documents for.
+            run_manager: The callback handler to use.
+
+        Returns:
+            List of relevant documents.
+        """
+        return []
+
+    async def _aget_relevant_documents(
         self,
         query: str,
         *,
         run_manager: CallbackManagerForRetrieverRun,
     ) -> list[Document]:
+        embedding = await self.embedding_function(query, RAG_EMBEDDING_QUERY_PREFIX)
         result = VECTOR_DB_CLIENT.search(
             collection_name=self.collection_name,
-            vectors=[self.embedding_function(query, RAG_EMBEDDING_QUERY_PREFIX)],
+            vectors=[embedding],
             limit=self.top_k,
         )
 
@@ -115,7 +166,45 @@ def get_doc(collection_name: str, user: UserModel = None):
         raise e
 
 
-def query_doc_with_hybrid_search(
+def get_enriched_texts(collection_result: GetResult) -> list[str]:
+    enriched_texts = []
+    for idx, text in enumerate(collection_result.documents[0]):
+        metadata = collection_result.metadatas[0][idx]
+        metadata_parts = [text]
+
+        # Add filename (repeat twice for extra weight in BM25 scoring)
+        if metadata.get("name"):
+            filename = metadata["name"]
+            filename_tokens = (
+                filename.replace("_", " ").replace("-", " ").replace(".", " ")
+            )
+            metadata_parts.append(
+                f"Filename: {filename} {filename_tokens} {filename_tokens}"
+            )
+
+        # Add title if available
+        if metadata.get("title"):
+            metadata_parts.append(f"Title: {metadata['title']}")
+
+        # Add document section headings if available (from markdown splitter)
+        if metadata.get("headings") and isinstance(metadata["headings"], list):
+            headings = " > ".join(str(h) for h in metadata["headings"])
+            metadata_parts.append(f"Section: {headings}")
+
+        # Add source URL/path if available
+        if metadata.get("source"):
+            metadata_parts.append(f"Source: {metadata['source']}")
+
+        # Add snippet for web search results
+        if metadata.get("snippet"):
+            metadata_parts.append(f"Snippet: {metadata['snippet']}")
+
+        enriched_texts.append(" ".join(metadata_parts))
+
+    return enriched_texts
+
+
+async def query_doc_with_hybrid_search(
     collection_name: str,
     collection_result: GetResult,
     query: str,
@@ -125,12 +214,21 @@ def query_doc_with_hybrid_search(
     k_reranker: int,
     r: float,
     hybrid_bm25_weight: float,
+    enable_enriched_texts: bool = False,
 ) -> dict:
     try:
+        # First check if collection_result has the required attributes
         if (
             not collection_result
             or not hasattr(collection_result, "documents")
-            or not collection_result.documents
+            or not hasattr(collection_result, "metadatas")
+        ):
+            log.warning(f"query_doc_with_hybrid_search:no_docs {collection_name}")
+            return {"documents": [], "metadatas": [], "distances": []}
+
+        # Now safely check the documents content after confirming attributes exist
+        if (
+            not collection_result.documents
             or len(collection_result.documents) == 0
             or not collection_result.documents[0]
         ):
@@ -139,8 +237,14 @@ def query_doc_with_hybrid_search(
 
         log.debug(f"query_doc_with_hybrid_search:doc {collection_name}")
 
+        bm25_texts = (
+            get_enriched_texts(collection_result)
+            if enable_enriched_texts
+            else collection_result.documents[0]
+        )
+
         bm25_retriever = BM25Retriever.from_texts(
-            texts=collection_result.documents[0],
+            texts=bm25_texts,
             metadatas=collection_result.metadatas[0],
         )
         bm25_retriever.k = k
@@ -176,7 +280,7 @@ def query_doc_with_hybrid_search(
             base_compressor=compressor, base_retriever=ensemble_retriever
         )
 
-        result = compression_retriever.invoke(query)
+        result = await compression_retriever.ainvoke(query)
 
         distances = [d.metadata.get("score") for d in result]
         documents = [d.page_content for d in result]
@@ -188,7 +292,11 @@ def query_doc_with_hybrid_search(
                 zip(distances, metadatas, documents), key=lambda x: x[0], reverse=True
             )
             sorted_items = sorted_items[:k]
+
+            if sorted_items:
                 distances, documents, metadatas = map(list, zip(*sorted_items))
+            else:
+                distances, documents, metadatas = [], [], []
 
         result = {
             "distances": [distances],
@@ -232,6 +340,13 @@ def merge_and_sort_query_results(query_results: list[dict], k: int) -> dict:
     combined = dict()  # To store documents with unique document hashes
 
     for data in query_results:
+        if (
+            len(data.get("distances", [])) == 0
+            or len(data.get("documents", [])) == 0
+            or len(data.get("metadatas", [])) == 0
+        ):
+            continue
+
         distances = data["distances"][0]
         documents = data["documents"][0]
         metadatas = data["metadatas"][0]
@@ -284,7 +399,7 @@ def get_all_items_from_collections(collection_names: list[str]) -> dict:
     return merge_get_results(results)
 
 
-def query_collection(
+async def query_collection(
     collection_names: list[str],
     queries: list[str],
     embedding_function,
@@ -309,7 +424,9 @@ def query_collection(
             return None, e
 
     # Generate all query embeddings (in one call)
-    query_embeddings = embedding_function(queries, prefix=RAG_EMBEDDING_QUERY_PREFIX)
+    query_embeddings = await embedding_function(
+        queries, prefix=RAG_EMBEDDING_QUERY_PREFIX
+    )
     log.debug(
         f"query_collection: processing {len(queries)} queries across {len(collection_names)} collections"
     )
@@ -336,7 +453,7 @@ def query_collection(
     return merge_and_sort_query_results(results, k=k)
 
 
-def query_collection_with_hybrid_search(
+async def query_collection_with_hybrid_search(
     collection_names: list[str],
     queries: list[str],
     embedding_function,
@@ -345,6 +462,7 @@ def query_collection_with_hybrid_search(
     k_reranker: int,
     r: float,
     hybrid_bm25_weight: float,
+    enable_enriched_texts: bool = False,
 ) -> dict:
     results = []
     error = False
@@ -367,9 +485,9 @@ def query_collection_with_hybrid_search(
         f"Starting hybrid search for {len(queries)} queries in {len(collection_names)} collections..."
     )
 
-    def process_query(collection_name, query):
+    async def process_query(collection_name, query):
         try:
-            result = query_doc_with_hybrid_search(
+            result = await query_doc_with_hybrid_search(
                 collection_name=collection_name,
                 collection_result=collection_results[collection_name],
                 query=query,
@@ -379,6 +497,7 @@ def query_collection_with_hybrid_search(
                 k_reranker=k_reranker,
                 r=r,
                 hybrid_bm25_weight=hybrid_bm25_weight,
+                enable_enriched_texts=enable_enriched_texts,
             )
             return result, None
         except Exception as e:
@@ -388,15 +507,16 @@ def query_collection_with_hybrid_search(
     # Prepare tasks for all collections and queries
     # Avoid running any tasks for collections that failed to fetch data (have assigned None)
     tasks = [
-        (cn, q)
-        for cn in collection_names
-        if collection_results[cn] is not None
-        for q in queries
+        (collection_name, query)
+        for collection_name in collection_names
+        if collection_results[collection_name] is not None
+        for query in queries
     ]
 
-    with ThreadPoolExecutor() as executor:
-        future_results = [executor.submit(process_query, cn, q) for cn, q in tasks]
-        task_results = [future.result() for future in future_results]
+    # Run all queries in parallel using asyncio.gather
+    task_results = await asyncio.gather(
+        *[process_query(collection_name, query) for collection_name, query in tasks]
+    )
 
     for result, err in task_results:
         if err is not None:
@@ -412,6 +532,248 @@ def query_collection_with_hybrid_search(
     return merge_and_sort_query_results(results, k=k)
 
 
+def generate_openai_batch_embeddings(
+    model: str,
+    texts: list[str],
+    url: str = "https://api.openai.com/v1",
+    key: str = "",
+    prefix: str = None,
+    user: UserModel = None,
+) -> Optional[list[list[float]]]:
+    try:
+        log.debug(
+            f"generate_openai_batch_embeddings:model {model} batch size: {len(texts)}"
+        )
+        json_data = {"input": texts, "model": model}
+        if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
+            json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {key}",
+        }
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
+        r = requests.post(
+            f"{url}/embeddings",
+            headers=headers,
+            json=json_data,
+        )
+        r.raise_for_status()
+        data = r.json()
+        if "data" in data:
+            return [elem["embedding"] for elem in data["data"]]
+        else:
+            raise "Something went wrong :/"
+    except Exception as e:
+        log.exception(f"Error generating openai batch embeddings: {e}")
+        return None
+
+
+async def agenerate_openai_batch_embeddings(
+    model: str,
+    texts: list[str],
+    url: str = "https://api.openai.com/v1",
+    key: str = "",
+    prefix: str = None,
+    user: UserModel = None,
+) -> Optional[list[list[float]]]:
+    try:
+        log.debug(
+            f"agenerate_openai_batch_embeddings:model {model} batch size: {len(texts)}"
+        )
+        form_data = {"input": texts, "model": model}
+        if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
+            form_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {key}",
+        }
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
+        async with aiohttp.ClientSession(trust_env=True) as session:
+            async with session.post(
+                f"{url}/embeddings", headers=headers, json=form_data
+            ) as r:
+                r.raise_for_status()
+                data = await r.json()
+                if "data" in data:
+                    return [item["embedding"] for item in data["data"]]
+                else:
+                    raise Exception("Something went wrong :/")
+    except Exception as e:
+        log.exception(f"Error generating openai batch embeddings: {e}")
+        return None
+
+
+def generate_azure_openai_batch_embeddings(
+    model: str,
+    texts: list[str],
+    url: str,
+    key: str = "",
+    version: str = "",
+    prefix: str = None,
+    user: UserModel = None,
+) -> Optional[list[list[float]]]:
+    try:
+        log.debug(
+            f"generate_azure_openai_batch_embeddings:deployment {model} batch size: {len(texts)}"
+        )
+        json_data = {"input": texts}
+        if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
+            json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
+
+        url = f"{url}/openai/deployments/{model}/embeddings?api-version={version}"
+
+        for _ in range(5):
+            headers = {
+                "Content-Type": "application/json",
+                "api-key": key,
+            }
+            if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+                headers = include_user_info_headers(headers, user)
+
+            r = requests.post(
+                url,
+                headers=headers,
+                json=json_data,
+            )
+            if r.status_code == 429:
+                retry = float(r.headers.get("Retry-After", "1"))
+                time.sleep(retry)
+                continue
+            r.raise_for_status()
+            data = r.json()
+            if "data" in data:
+                return [elem["embedding"] for elem in data["data"]]
+            else:
+                raise Exception("Something went wrong :/")
+        return None
+    except Exception as e:
+        log.exception(f"Error generating azure openai batch embeddings: {e}")
+        return None
+
+
+async def agenerate_azure_openai_batch_embeddings(
+    model: str,
+    texts: list[str],
+    url: str,
+    key: str = "",
+    version: str = "",
+    prefix: str = None,
+    user: UserModel = None,
+) -> Optional[list[list[float]]]:
+    try:
+        log.debug(
+            f"agenerate_azure_openai_batch_embeddings:deployment {model} batch size: {len(texts)}"
+        )
+        form_data = {"input": texts}
+        if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
+            form_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
+
+        full_url = f"{url}/openai/deployments/{model}/embeddings?api-version={version}"
+
+        headers = {
+            "Content-Type": "application/json",
+            "api-key": key,
+        }
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
+        async with aiohttp.ClientSession(trust_env=True) as session:
+            async with session.post(full_url, headers=headers, json=form_data) as r:
+                r.raise_for_status()
+                data = await r.json()
+                if "data" in data:
+                    return [item["embedding"] for item in data["data"]]
+                else:
+                    raise Exception("Something went wrong :/")
+    except Exception as e:
+        log.exception(f"Error generating azure openai batch embeddings: {e}")
+        return None
+
+
+def generate_ollama_batch_embeddings(
+    model: str,
+    texts: list[str],
+    url: str,
+    key: str = "",
+    prefix: str = None,
+    user: UserModel = None,
+) -> Optional[list[list[float]]]:
+    try:
+        log.debug(
+            f"generate_ollama_batch_embeddings:model {model} batch size: {len(texts)}"
+        )
+        json_data = {"input": texts, "model": model}
+        if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
+            json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {key}",
+        }
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
+        r = requests.post(
+            f"{url}/api/embed",
+            headers=headers,
+            json=json_data,
+        )
+        r.raise_for_status()
+        data = r.json()
+
+        if "embeddings" in data:
+            return data["embeddings"]
+        else:
+            raise "Something went wrong :/"
+    except Exception as e:
+        log.exception(f"Error generating ollama batch embeddings: {e}")
+        return None
+
+
+async def agenerate_ollama_batch_embeddings(
+    model: str,
+    texts: list[str],
+    url: str,
+    key: str = "",
+    prefix: str = None,
+    user: UserModel = None,
+) -> Optional[list[list[float]]]:
+    try:
+        log.debug(
+            f"agenerate_ollama_batch_embeddings:model {model} batch size: {len(texts)}"
+        )
+        form_data = {"input": texts, "model": model}
+        if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
+            form_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {key}",
+        }
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
+        async with aiohttp.ClientSession(trust_env=True) as session:
+            async with session.post(
+                f"{url}/api/embed", headers=headers, json=form_data
+            ) as r:
+                r.raise_for_status()
+                data = await r.json()
+                if "embeddings" in data:
+                    return data["embeddings"]
+                else:
+                    raise Exception("Something went wrong :/")
+    except Exception as e:
+        log.exception(f"Error generating ollama batch embeddings: {e}")
+        return None
+
+
 def get_embedding_function(
     embedding_engine,
     embedding_model,
@@ -420,13 +782,24 @@ def get_embedding_function(
     key,
     embedding_batch_size,
     azure_api_version=None,
-):
+    enable_async=True,
+) -> Awaitable:
     if embedding_engine == "":
-        return lambda query, prefix=None, user=None: embedding_function.encode(
+        # Sentence transformers: CPU-bound sync operation
+        async def async_embedding_function(query, prefix=None, user=None):
+            return await asyncio.to_thread(
+                (
+                    lambda query, prefix=None: embedding_function.encode(
                         query, **({"prompt": prefix} if prefix else {})
                     ).tolist()
+                ),
+                query,
+                prefix,
+            )
+
+        return async_embedding_function
     elif embedding_engine in ["ollama", "openai", "azure_openai"]:
-        func = lambda query, prefix=None, user=None: generate_embeddings(
+        embedding_function = lambda query, prefix=None, user=None: generate_embeddings(
             engine=embedding_engine,
             model=embedding_model,
             text=query,
@@ -437,41 +810,114 @@ def get_embedding_function(
             azure_api_version=azure_api_version,
         )
 
-        def generate_multiple(query, prefix, user, func):
+        async def async_embedding_function(query, prefix=None, user=None):
             if isinstance(query, list):
-                embeddings = []
-                for i in range(0, len(query), embedding_batch_size):
-                    batch_embeddings = func(
-                        query[i : i + embedding_batch_size],
-                        prefix=prefix,
-                        user=user,
+                # Create batches
+                batches = [
+                    query[i : i + embedding_batch_size]
+                    for i in range(0, len(query), embedding_batch_size)
+                ]
+
+                if enable_async:
+                    log.debug(
+                        f"generate_multiple_async: Processing {len(batches)} batches in parallel"
+                    )
+                    # Execute all batches in parallel
+                    tasks = [
+                        embedding_function(batch, prefix=prefix, user=user)
+                        for batch in batches
+                    ]
+                    batch_results = await asyncio.gather(*tasks)
+                else:
+                    log.debug(
+                        f"generate_multiple_async: Processing {len(batches)} batches sequentially"
+                    )
+                    batch_results = []
+                    for batch in batches:
+                        batch_results.append(
+                            await embedding_function(batch, prefix=prefix, user=user)
                         )
 
+                # Flatten results
+                embeddings = []
+                for batch_embeddings in batch_results:
                     if isinstance(batch_embeddings, list):
                         embeddings.extend(batch_embeddings)
+
+                log.debug(
+                    f"generate_multiple_async: Generated {len(embeddings)} embeddings from {len(batches)} parallel batches"
+                )
                 return embeddings
             else:
-                return func(query, prefix, user)
+                return await embedding_function(query, prefix, user)
 
-        return lambda query, prefix=None, user=None: generate_multiple(
-            query, prefix, user, func
-        )
+        return async_embedding_function
     else:
         raise ValueError(f"Unknown embedding engine: {embedding_engine}")
 
 
+async def generate_embeddings(
+    engine: str,
+    model: str,
+    text: Union[str, list[str]],
+    prefix: Union[str, None] = None,
+    **kwargs,
+):
+    url = kwargs.get("url", "")
+    key = kwargs.get("key", "")
+    user = kwargs.get("user")
+
+    if prefix is not None and RAG_EMBEDDING_PREFIX_FIELD_NAME is None:
+        if isinstance(text, list):
+            text = [f"{prefix}{text_element}" for text_element in text]
+        else:
+            text = f"{prefix}{text}"
+
+    if engine == "ollama":
+        embeddings = await agenerate_ollama_batch_embeddings(
+            **{
+                "model": model,
+                "texts": text if isinstance(text, list) else [text],
+                "url": url,
+                "key": key,
+                "prefix": prefix,
+                "user": user,
+            }
+        )
+        return embeddings[0] if isinstance(text, str) else embeddings
+    elif engine == "openai":
+        embeddings = await agenerate_openai_batch_embeddings(
+            model, text if isinstance(text, list) else [text], url, key, prefix, user
+        )
+        return embeddings[0] if isinstance(text, str) else embeddings
+    elif engine == "azure_openai":
+        azure_api_version = kwargs.get("azure_api_version", "")
+        embeddings = await agenerate_azure_openai_batch_embeddings(
+            model,
+            text if isinstance(text, list) else [text],
+            url,
+            key,
+            azure_api_version,
+            prefix,
+            user,
+        )
+        return embeddings[0] if isinstance(text, str) else embeddings
+
+
 def get_reranking_function(reranking_engine, reranking_model, reranking_function):
     if reranking_function is None:
         return None
     if reranking_engine == "external":
-        return lambda sentences, user=None: reranking_function.predict(
-            sentences, user=user
+        return lambda query, documents, user=None: reranking_function.predict(
+            [(query, doc.page_content) for doc in documents], user=user
         )
     else:
-        return lambda sentences, user=None: reranking_function.predict(sentences)
+        return lambda query, documents, user=None: reranking_function.predict(
+            [(query, doc.page_content) for doc in documents]
+        )
 
 
-def get_sources_from_items(
+async def get_sources_from_items(
     request,
     items,
     queries,
@@ -571,6 +1017,13 @@ def get_sources_from_items(
                         "metadatas": [[{"file_id": chat.id, "name": chat.title}]],
                     }
 
+        elif item.get("type") == "url":
+            content, docs = get_content_from_url(request, item.get("url"))
+            if docs:
+                query_result = {
+                    "documents": [[content]],
+                    "metadatas": [[{"url": item.get("url"), "name": item.get("url")}]],
+                }
         elif item.get("type") == "file":
             if (
                 item.get("context") == "full"
@@ -618,13 +1071,18 @@ def get_sources_from_items(
                     collection_names.append(f"file-{item['id']}")
 
         elif item.get("type") == "collection":
+            # Manual Full Mode Toggle for Collection
+            knowledge_base = Knowledges.get_knowledge_by_id(item.get("id"))
+
+            if knowledge_base and (
+                user.role == "admin"
+                or knowledge_base.user_id == user.id
+                or has_access(user.id, "read", knowledge_base.access_control)
+            ):
                 if (
                     item.get("context") == "full"
                     or request.app.state.config.BYPASS_EMBEDDING_AND_RETRIEVAL
                 ):
-                # Manual Full Mode Toggle for Collection
-                knowledge_base = Knowledges.get_knowledge_by_id(item.get("id"))
-
                     if knowledge_base and (
                         user.role == "admin"
                         or knowledge_base.user_id == user.id
@@ -687,7 +1145,7 @@ def get_sources_from_items(
                     query_result = None  # Initialize to None
                     if hybrid_search:
                         try:
-                            query_result = query_collection_with_hybrid_search(
+                            query_result = await query_collection_with_hybrid_search(
                                 collection_names=collection_names,
                                 queries=queries,
                                 embedding_function=embedding_function,
@@ -696,6 +1154,7 @@ def get_sources_from_items(
                                 k_reranker=k_reranker,
                                 r=r,
                                 hybrid_bm25_weight=hybrid_bm25_weight,
+                                enable_enriched_texts=request.app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS,
                             )
                         except Exception as e:
                             log.debug(
@@ -704,7 +1163,7 @@ def get_sources_from_items(
 
                     # fallback to non-hybrid search
                     if not hybrid_search and query_result is None:
-                        query_result = query_collection(
+                        query_result = await query_collection(
                             collection_names=collection_names,
                             queries=queries,
                             embedding_function=embedding_function,
@@ -736,7 +1195,6 @@ def get_sources_from_items(
                     sources.append(source)
         except Exception as e:
             log.exception(e)
-
     return sources
 
 
@@ -781,199 +1239,6 @@ def get_model_path(model: str, update_model: bool = False):
         return model
 
 
-def generate_openai_batch_embeddings(
-    model: str,
-    texts: list[str],
-    url: str = "https://api.openai.com/v1",
-    key: str = "",
-    prefix: str = None,
-    user: UserModel = None,
-) -> Optional[list[list[float]]]:
-    try:
-        log.debug(
-            f"generate_openai_batch_embeddings:model {model} batch size: {len(texts)}"
-        )
-        json_data = {"input": texts, "model": model}
-        if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
-            json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
-
-        r = requests.post(
-            f"{url}/embeddings",
-            headers={
-                "Content-Type": "application/json",
-                "Authorization": f"Bearer {key}",
-                **(
-                    {
-                        "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                        "X-OpenWebUI-User-Id": user.id,
-                        "X-OpenWebUI-User-Email": user.email,
-                        "X-OpenWebUI-User-Role": user.role,
-                    }
-                    if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                    else {}
-                ),
-            },
-            json=json_data,
-        )
-        r.raise_for_status()
-        data = r.json()
-        if "data" in data:
-            return [elem["embedding"] for elem in data["data"]]
-        else:
-            raise "Something went wrong :/"
-    except Exception as e:
-        log.exception(f"Error generating openai batch embeddings: {e}")
-        return None
-
-
-def generate_azure_openai_batch_embeddings(
-    model: str,
-    texts: list[str],
-    url: str,
-    key: str = "",
-    version: str = "",
-    prefix: str = None,
-    user: UserModel = None,
-) -> Optional[list[list[float]]]:
-    try:
-        log.debug(
-            f"generate_azure_openai_batch_embeddings:deployment {model} batch size: {len(texts)}"
-        )
-        json_data = {"input": texts}
-        if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
-            json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
-
-        url = f"{url}/openai/deployments/{model}/embeddings?api-version={version}"
-
-        for _ in range(5):
-            r = requests.post(
-                url,
-                headers={
-                    "Content-Type": "application/json",
-                    "api-key": key,
-                    **(
-                        {
-                            "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                            "X-OpenWebUI-User-Id": user.id,
-                            "X-OpenWebUI-User-Email": user.email,
-                            "X-OpenWebUI-User-Role": user.role,
-                        }
-                        if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                        else {}
-                    ),
-                },
-                json=json_data,
-            )
-            if r.status_code == 429:
-                retry = float(r.headers.get("Retry-After", "1"))
-                time.sleep(retry)
-                continue
-            r.raise_for_status()
-            data = r.json()
-            if "data" in data:
-                return [elem["embedding"] for elem in data["data"]]
-            else:
-                raise Exception("Something went wrong :/")
-        return None
-    except Exception as e:
-        log.exception(f"Error generating azure openai batch embeddings: {e}")
-        return None
-
-
-def generate_ollama_batch_embeddings(
-    model: str,
-    texts: list[str],
-    url: str,
-    key: str = "",
-    prefix: str = None,
-    user: UserModel = None,
-) -> Optional[list[list[float]]]:
-    try:
-        log.debug(
-            f"generate_ollama_batch_embeddings:model {model} batch size: {len(texts)}"
-        )
-        json_data = {"input": texts, "model": model}
-        if isinstance(RAG_EMBEDDING_PREFIX_FIELD_NAME, str) and isinstance(prefix, str):
-            json_data[RAG_EMBEDDING_PREFIX_FIELD_NAME] = prefix
-
-        r = requests.post(
-            f"{url}/api/embed",
-            headers={
-                "Content-Type": "application/json",
-                "Authorization": f"Bearer {key}",
-                **(
-                    {
-                        "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                        "X-OpenWebUI-User-Id": user.id,
-                        "X-OpenWebUI-User-Email": user.email,
-                        "X-OpenWebUI-User-Role": user.role,
-                    }
-                    if ENABLE_FORWARD_USER_INFO_HEADERS
-                    else {}
-                ),
-            },
-            json=json_data,
-        )
-        r.raise_for_status()
-        data = r.json()
-
-        if "embeddings" in data:
-            return data["embeddings"]
-        else:
-            raise "Something went wrong :/"
-    except Exception as e:
-        log.exception(f"Error generating ollama batch embeddings: {e}")
-        return None
-
-
-def generate_embeddings(
-    engine: str,
-    model: str,
-    text: Union[str, list[str]],
-    prefix: Union[str, None] = None,
-    **kwargs,
-):
-    url = kwargs.get("url", "")
-    key = kwargs.get("key", "")
-    user = kwargs.get("user")
-
-    if prefix is not None and RAG_EMBEDDING_PREFIX_FIELD_NAME is None:
-        if isinstance(text, list):
-            text = [f"{prefix}{text_element}" for text_element in text]
-        else:
-            text = f"{prefix}{text}"
-
-    if engine == "ollama":
-        embeddings = generate_ollama_batch_embeddings(
-            **{
-                "model": model,
-                "texts": text if isinstance(text, list) else [text],
-                "url": url,
-                "key": key,
-                "prefix": prefix,
-                "user": user,
-            }
-        )
-        return embeddings[0] if isinstance(text, str) else embeddings
-    elif engine == "openai":
-        embeddings = generate_openai_batch_embeddings(
-            model, text if isinstance(text, list) else [text], url, key, prefix, user
-        )
-        return embeddings[0] if isinstance(text, str) else embeddings
-    elif engine == "azure_openai":
-        azure_api_version = kwargs.get("azure_api_version", "")
-        embeddings = generate_azure_openai_batch_embeddings(
-            model,
-            text if isinstance(text, list) else [text],
-            url,
-            key,
-            azure_api_version,
-            prefix,
-            user,
-        )
-        return embeddings[0] if isinstance(text, str) else embeddings
-
-
 import operator
 from typing import Optional, Sequence
 
@@ -996,19 +1261,38 @@ class RerankCompressor(BaseDocumentCompressor):
         documents: Sequence[Document],
         query: str,
         callbacks: Optional[Callbacks] = None,
+    ) -> Sequence[Document]:
+        """Compress retrieved documents given the query context.
+
+        Args:
+            documents: The retrieved documents.
+            query: The query context.
+            callbacks: Optional callbacks to run during compression.
+
+        Returns:
+            The compressed documents.
+
+        """
+        return []
+
+    async def acompress_documents(
+        self,
+        documents: Sequence[Document],
+        query: str,
+        callbacks: Optional[Callbacks] = None,
     ) -> Sequence[Document]:
         reranking = self.reranking_function is not None
 
         scores = None
         if reranking:
-            scores = self.reranking_function(
-                [(query, doc.page_content) for doc in documents]
-            )
+            scores = self.reranking_function(query, documents)
         else:
             from sentence_transformers import util
 
-            query_embedding = self.embedding_function(query, RAG_EMBEDDING_QUERY_PREFIX)
-            document_embedding = self.embedding_function(
+            query_embedding = await self.embedding_function(
+                query, RAG_EMBEDDING_QUERY_PREFIX
+            )
+            document_embedding = await self.embedding_function(
                 [doc.page_content for doc in documents], RAG_EMBEDDING_CONTENT_PREFIX
             )
             scores = util.cos_sim(query_embedding, document_embedding)[0]

backend/open_webui/retrieval/vector/dbs/oracle23ai.py

@@ -717,7 +717,7 @@ class Oracle23aiClient(VectorDBBase):
         )
 
         try:
-            limit = limit or 1000
+            limit = 1000  # Hardcoded limit for get operation
 
             with self.get_connection() as connection:
                 with connection.cursor() as cursor:

backend/open_webui/retrieval/vector/dbs/pgvector.py

@@ -1,4 +1,4 @@
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Dict, Any, Tuple
 import logging
 import json
 from sqlalchemy import (
@@ -22,7 +22,7 @@ from sqlalchemy.pool import NullPool, QueuePool
 
 from sqlalchemy.orm import declarative_base, scoped_session, sessionmaker
 from sqlalchemy.dialects.postgresql import JSONB, array
-from pgvector.sqlalchemy import Vector
+from pgvector.sqlalchemy import Vector, HALFVEC
 from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.exc import NoSuchTableError
 
@@ -44,11 +44,20 @@ from open_webui.config import (
     PGVECTOR_POOL_MAX_OVERFLOW,
     PGVECTOR_POOL_TIMEOUT,
     PGVECTOR_POOL_RECYCLE,
+    PGVECTOR_INDEX_METHOD,
+    PGVECTOR_HNSW_M,
+    PGVECTOR_HNSW_EF_CONSTRUCTION,
+    PGVECTOR_IVFFLAT_LISTS,
+    PGVECTOR_USE_HALFVEC,
 )
 
 from open_webui.env import SRC_LOG_LEVELS
 
 VECTOR_LENGTH = PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH
+USE_HALFVEC = PGVECTOR_USE_HALFVEC
+
+VECTOR_TYPE_FACTORY = HALFVEC if USE_HALFVEC else Vector
+VECTOR_OPCLASS = "halfvec_cosine_ops" if USE_HALFVEC else "vector_cosine_ops"
 Base = declarative_base()
 
 log = logging.getLogger(__name__)
@@ -67,7 +76,7 @@ class DocumentChunk(Base):
     __tablename__ = "document_chunk"
 
     id = Column(Text, primary_key=True)
-    vector = Column(Vector(dim=VECTOR_LENGTH), nullable=True)
+    vector = Column(VECTOR_TYPE_FACTORY(dim=VECTOR_LENGTH), nullable=True)
     collection_name = Column(Text, nullable=False)
 
     if PGVECTOR_PGCRYPTO:
@@ -157,13 +166,9 @@ class PgvectorClient(VectorDBBase):
             connection = self.session.connection()
             Base.metadata.create_all(bind=connection)
 
-            # Create an index on the vector column if it doesn't exist
-            self.session.execute(
-                text(
-                    "CREATE INDEX IF NOT EXISTS idx_document_chunk_vector "
-                    "ON document_chunk USING ivfflat (vector vector_cosine_ops) WITH (lists = 100);"
-                )
-            )
+            index_method, index_options = self._vector_index_configuration()
+            self._ensure_vector_index(index_method, index_options)
+
             self.session.execute(
                 text(
                     "CREATE INDEX IF NOT EXISTS idx_document_chunk_collection_name "
@@ -177,6 +182,78 @@ class PgvectorClient(VectorDBBase):
             log.exception(f"Error during initialization: {e}")
             raise
 
+    @staticmethod
+    def _extract_index_method(index_def: Optional[str]) -> Optional[str]:
+        if not index_def:
+            return None
+        try:
+            after_using = index_def.lower().split("using ", 1)[1]
+            return after_using.split()[0]
+        except (IndexError, AttributeError):
+            return None
+
+    def _vector_index_configuration(self) -> Tuple[str, str]:
+        if PGVECTOR_INDEX_METHOD:
+            index_method = PGVECTOR_INDEX_METHOD
+            log.info(
+                "Using vector index method '%s' from PGVECTOR_INDEX_METHOD.",
+                index_method,
+            )
+        elif USE_HALFVEC:
+            index_method = "hnsw"
+            log.info(
+                "VECTOR_LENGTH=%s exceeds 2000; using halfvec column type with hnsw index.",
+                VECTOR_LENGTH,
+            )
+        else:
+            index_method = "ivfflat"
+
+        if index_method == "hnsw":
+            index_options = f"WITH (m = {PGVECTOR_HNSW_M}, ef_construction = {PGVECTOR_HNSW_EF_CONSTRUCTION})"
+        else:
+            index_options = f"WITH (lists = {PGVECTOR_IVFFLAT_LISTS})"
+
+        return index_method, index_options
+
+    def _ensure_vector_index(self, index_method: str, index_options: str) -> None:
+        index_name = "idx_document_chunk_vector"
+        existing_index_def = self.session.execute(
+            text(
+                """
+                SELECT indexdef
+                FROM pg_indexes
+                WHERE schemaname = current_schema()
+                  AND tablename = 'document_chunk'
+                  AND indexname = :index_name
+                """
+            ),
+            {"index_name": index_name},
+        ).scalar()
+
+        existing_method = self._extract_index_method(existing_index_def)
+        if existing_method and existing_method != index_method:
+            raise RuntimeError(
+                f"Existing pgvector index '{index_name}' uses method '{existing_method}' but configuration now "
+                f"requires '{index_method}'. Automatic rebuild is disabled to prevent long-running maintenance. "
+                "Drop the index manually (optionally after tuning maintenance_work_mem/max_parallel_maintenance_workers) "
+                "and recreate it with the new method before restarting Open WebUI."
+            )
+
+        if not existing_index_def:
+            index_sql = (
+                f"CREATE INDEX IF NOT EXISTS {index_name} "
+                f"ON document_chunk USING {index_method} (vector {VECTOR_OPCLASS})"
+            )
+            if index_options:
+                index_sql = f"{index_sql} {index_options}"
+            self.session.execute(text(index_sql))
+            log.info(
+                "Ensured vector index '%s' using %s%s.",
+                index_name,
+                index_method,
+                f" {index_options}" if index_options else "",
+            )
+
     def check_vector_length(self) -> None:
         """
         Check if the VECTOR_LENGTH matches the existing vector column dimension in the database.
@@ -196,17 +273,20 @@ class PgvectorClient(VectorDBBase):
         if "vector" in document_chunk_table.columns:
             vector_column = document_chunk_table.columns["vector"]
             vector_type = vector_column.type
-            if isinstance(vector_type, Vector):
-                db_vector_length = vector_type.dim
-                if db_vector_length != VECTOR_LENGTH:
+            expected_type = HALFVEC if USE_HALFVEC else Vector
+
+            if not isinstance(vector_type, expected_type):
+                raise Exception(
+                    "The 'vector' column type does not match the expected type "
+                    f"('{expected_type.__name__}') for VECTOR_LENGTH {VECTOR_LENGTH}."
+                )
+
+            db_vector_length = getattr(vector_type, "dim", None)
+            if db_vector_length is not None and db_vector_length != VECTOR_LENGTH:
                 raise Exception(
                     f"VECTOR_LENGTH {VECTOR_LENGTH} does not match existing vector column dimension {db_vector_length}. "
                     "Cannot change vector size after initialization without migrating the data."
                 )
-            else:
-                raise Exception(
-                    "The 'vector' column exists but is not of type 'Vector'."
-                )
         else:
             raise Exception(
                 "The 'vector' column does not exist in the 'document_chunk' table."
@@ -360,11 +440,11 @@ class PgvectorClient(VectorDBBase):
             num_queries = len(vectors)
 
             def vector_expr(vector):
-                return cast(array(vector), Vector(VECTOR_LENGTH))
+                return cast(array(vector), VECTOR_TYPE_FACTORY(VECTOR_LENGTH))
 
             # Create the values for query vectors
             qid_col = column("qid", Integer)
-            q_vector_col = column("q_vector", Vector(VECTOR_LENGTH))
+            q_vector_col = column("q_vector", VECTOR_TYPE_FACTORY(VECTOR_LENGTH))
             query_vectors = (
                 values(qid_col, q_vector_col)
                 .data(

backend/open_webui/retrieval/vector/dbs/s3vector.py

@@ -117,15 +117,16 @@ class S3VectorClient(VectorDBBase):
 
     def has_collection(self, collection_name: str) -> bool:
         """
-        Check if a vector index (collection) exists in the S3 vector bucket.
+        Check if a vector index exists using direct lookup.
+        This avoids pagination issues with list_indexes() and is significantly faster.
         """
-
         try:
-            response = self.client.list_indexes(vectorBucketName=self.bucket_name)
-            indexes = response.get("indexes", [])
-            return any(idx.get("indexName") == collection_name for idx in indexes)
+            self.client.get_index(
+                vectorBucketName=self.bucket_name, indexName=collection_name
+            )
+            return True
         except Exception as e:
-            log.error(f"Error listing indexes: {e}")
+            log.error(f"Error checking if index '{collection_name}' exists: {e}")
             return False
 
     def delete_collection(self, collection_name: str) -> None:

backend/open_webui/retrieval/vector/dbs/weaviate.py

@@ -0,0 +1,340 @@
+import weaviate
+import re
+import uuid
+from typing import Any, Dict, List, Optional, Union
+
+from open_webui.retrieval.vector.main import (
+    VectorDBBase,
+    VectorItem,
+    SearchResult,
+    GetResult,
+)
+from open_webui.retrieval.vector.utils import process_metadata
+from open_webui.config import (
+    WEAVIATE_HTTP_HOST,
+    WEAVIATE_HTTP_PORT,
+    WEAVIATE_GRPC_PORT,
+    WEAVIATE_API_KEY,
+)
+
+
+def _convert_uuids_to_strings(obj: Any) -> Any:
+    """
+    Recursively convert UUID objects to strings in nested data structures.
+
+    This function handles:
+    - UUID objects -> string
+    - Dictionaries with UUID values
+    - Lists/Tuples with UUID values
+    - Nested combinations of the above
+
+    Args:
+        obj: Any object that might contain UUIDs
+
+    Returns:
+        The same object structure with UUIDs converted to strings
+    """
+    if isinstance(obj, uuid.UUID):
+        return str(obj)
+    elif isinstance(obj, dict):
+        return {key: _convert_uuids_to_strings(value) for key, value in obj.items()}
+    elif isinstance(obj, (list, tuple)):
+        return type(obj)(_convert_uuids_to_strings(item) for item in obj)
+    elif isinstance(obj, (str, int, float, bool, type(None))):
+        return obj
+    else:
+        return obj
+
+
+class WeaviateClient(VectorDBBase):
+    def __init__(self):
+        self.url = WEAVIATE_HTTP_HOST
+        try:
+            # Build connection parameters
+            connection_params = {
+                "host": WEAVIATE_HTTP_HOST,
+                "port": WEAVIATE_HTTP_PORT,
+                "grpc_port": WEAVIATE_GRPC_PORT,
+            }
+
+            # Only add auth_credentials if WEAVIATE_API_KEY exists and is not empty
+            if WEAVIATE_API_KEY:
+                connection_params["auth_credentials"] = (
+                    weaviate.classes.init.Auth.api_key(WEAVIATE_API_KEY)
+                )
+
+            self.client = weaviate.connect_to_local(**connection_params)
+            self.client.connect()
+        except Exception as e:
+            raise ConnectionError(f"Failed to connect to Weaviate: {e}") from e
+
+    def _sanitize_collection_name(self, collection_name: str) -> str:
+        """Sanitize collection name to be a valid Weaviate class name."""
+        if not isinstance(collection_name, str) or not collection_name.strip():
+            raise ValueError("Collection name must be a non-empty string")
+
+        # Requirements for a valid Weaviate class name:
+        # The collection name must begin with a capital letter.
+        # The name can only contain letters, numbers, and the underscore (_) character. Spaces are not allowed.
+
+        # Replace hyphens with underscores and keep only alphanumeric characters
+        name = re.sub(r"[^a-zA-Z0-9_]", "", collection_name.replace("-", "_"))
+        name = name.strip("_")
+
+        if not name:
+            raise ValueError(
+                "Could not sanitize collection name to be a valid Weaviate class name"
+            )
+
+        # Ensure it starts with a letter and is capitalized
+        if not name[0].isalpha():
+            name = "C" + name
+
+        return name[0].upper() + name[1:]
+
+    def has_collection(self, collection_name: str) -> bool:
+        sane_collection_name = self._sanitize_collection_name(collection_name)
+        return self.client.collections.exists(sane_collection_name)
+
+    def delete_collection(self, collection_name: str) -> None:
+        sane_collection_name = self._sanitize_collection_name(collection_name)
+        if self.client.collections.exists(sane_collection_name):
+            self.client.collections.delete(sane_collection_name)
+
+    def _create_collection(self, collection_name: str) -> None:
+        self.client.collections.create(
+            name=collection_name,
+            vector_config=weaviate.classes.config.Configure.Vectors.self_provided(),
+            properties=[
+                weaviate.classes.config.Property(
+                    name="text", data_type=weaviate.classes.config.DataType.TEXT
+                ),
+            ],
+        )
+
+    def insert(self, collection_name: str, items: List[VectorItem]) -> None:
+        sane_collection_name = self._sanitize_collection_name(collection_name)
+        if not self.client.collections.exists(sane_collection_name):
+            self._create_collection(sane_collection_name)
+
+        collection = self.client.collections.get(sane_collection_name)
+
+        with collection.batch.fixed_size(batch_size=100) as batch:
+            for item in items:
+                item_uuid = str(uuid.uuid4()) if not item["id"] else str(item["id"])
+
+                properties = {"text": item["text"]}
+                if item["metadata"]:
+                    clean_metadata = _convert_uuids_to_strings(
+                        process_metadata(item["metadata"])
+                    )
+                    clean_metadata.pop("text", None)
+                    properties.update(clean_metadata)
+
+                batch.add_object(
+                    properties=properties, uuid=item_uuid, vector=item["vector"]
+                )
+
+    def upsert(self, collection_name: str, items: List[VectorItem]) -> None:
+        sane_collection_name = self._sanitize_collection_name(collection_name)
+        if not self.client.collections.exists(sane_collection_name):
+            self._create_collection(sane_collection_name)
+
+        collection = self.client.collections.get(sane_collection_name)
+
+        with collection.batch.fixed_size(batch_size=100) as batch:
+            for item in items:
+                item_uuid = str(item["id"]) if item["id"] else None
+
+                properties = {"text": item["text"]}
+                if item["metadata"]:
+                    clean_metadata = _convert_uuids_to_strings(
+                        process_metadata(item["metadata"])
+                    )
+                    clean_metadata.pop("text", None)
+                    properties.update(clean_metadata)
+
+                batch.add_object(
+                    properties=properties, uuid=item_uuid, vector=item["vector"]
+                )
+
+    def search(
+        self, collection_name: str, vectors: List[List[Union[float, int]]], limit: int
+    ) -> Optional[SearchResult]:
+        sane_collection_name = self._sanitize_collection_name(collection_name)
+        if not self.client.collections.exists(sane_collection_name):
+            return None
+
+        collection = self.client.collections.get(sane_collection_name)
+
+        result_ids, result_documents, result_metadatas, result_distances = (
+            [],
+            [],
+            [],
+            [],
+        )
+
+        for vector_embedding in vectors:
+            try:
+                response = collection.query.near_vector(
+                    near_vector=vector_embedding,
+                    limit=limit,
+                    return_metadata=weaviate.classes.query.MetadataQuery(distance=True),
+                )
+
+                ids = [str(obj.uuid) for obj in response.objects]
+                documents = []
+                metadatas = []
+                distances = []
+
+                for obj in response.objects:
+                    properties = dict(obj.properties) if obj.properties else {}
+                    documents.append(properties.pop("text", ""))
+                    metadatas.append(_convert_uuids_to_strings(properties))
+
+                # Weaviate has cosine distance, 2 (worst) -> 0 (best). Re-ordering to 0 -> 1
+                raw_distances = [
+                    (
+                        obj.metadata.distance
+                        if obj.metadata and obj.metadata.distance
+                        else 2.0
+                    )
+                    for obj in response.objects
+                ]
+                distances = [(2 - dist) / 2 for dist in raw_distances]
+
+                result_ids.append(ids)
+                result_documents.append(documents)
+                result_metadatas.append(metadatas)
+                result_distances.append(distances)
+            except Exception:
+                result_ids.append([])
+                result_documents.append([])
+                result_metadatas.append([])
+                result_distances.append([])
+
+        return SearchResult(
+            **{
+                "ids": result_ids,
+                "documents": result_documents,
+                "metadatas": result_metadatas,
+                "distances": result_distances,
+            }
+        )
+
+    def query(
+        self, collection_name: str, filter: Dict, limit: Optional[int] = None
+    ) -> Optional[GetResult]:
+        sane_collection_name = self._sanitize_collection_name(collection_name)
+        if not self.client.collections.exists(sane_collection_name):
+            return None
+
+        collection = self.client.collections.get(sane_collection_name)
+
+        weaviate_filter = None
+        if filter:
+            for key, value in filter.items():
+                prop_filter = weaviate.classes.query.Filter.by_property(name=key).equal(
+                    value
+                )
+                weaviate_filter = (
+                    prop_filter
+                    if weaviate_filter is None
+                    else weaviate.classes.query.Filter.all_of(
+                        [weaviate_filter, prop_filter]
+                    )
+                )
+
+        try:
+            response = collection.query.fetch_objects(
+                filters=weaviate_filter, limit=limit
+            )
+
+            ids = [str(obj.uuid) for obj in response.objects]
+            documents = []
+            metadatas = []
+
+            for obj in response.objects:
+                properties = dict(obj.properties) if obj.properties else {}
+                documents.append(properties.pop("text", ""))
+                metadatas.append(_convert_uuids_to_strings(properties))
+
+            return GetResult(
+                **{
+                    "ids": [ids],
+                    "documents": [documents],
+                    "metadatas": [metadatas],
+                }
+            )
+        except Exception:
+            return None
+
+    def get(self, collection_name: str) -> Optional[GetResult]:
+        sane_collection_name = self._sanitize_collection_name(collection_name)
+        if not self.client.collections.exists(sane_collection_name):
+            return None
+
+        collection = self.client.collections.get(sane_collection_name)
+        ids, documents, metadatas = [], [], []
+
+        try:
+            for item in collection.iterator():
+                ids.append(str(item.uuid))
+                properties = dict(item.properties) if item.properties else {}
+                documents.append(properties.pop("text", ""))
+                metadatas.append(_convert_uuids_to_strings(properties))
+
+            if not ids:
+                return None
+
+            return GetResult(
+                **{
+                    "ids": [ids],
+                    "documents": [documents],
+                    "metadatas": [metadatas],
+                }
+            )
+        except Exception:
+            return None
+
+    def delete(
+        self,
+        collection_name: str,
+        ids: Optional[List[str]] = None,
+        filter: Optional[Dict] = None,
+    ) -> None:
+        sane_collection_name = self._sanitize_collection_name(collection_name)
+        if not self.client.collections.exists(sane_collection_name):
+            return
+
+        collection = self.client.collections.get(sane_collection_name)
+
+        try:
+            if ids:
+                for item_id in ids:
+                    collection.data.delete_by_id(uuid=item_id)
+            elif filter:
+                weaviate_filter = None
+                for key, value in filter.items():
+                    prop_filter = weaviate.classes.query.Filter.by_property(
+                        name=key
+                    ).equal(value)
+                    weaviate_filter = (
+                        prop_filter
+                        if weaviate_filter is None
+                        else weaviate.classes.query.Filter.all_of(
+                            [weaviate_filter, prop_filter]
+                        )
+                    )
+
+                if weaviate_filter:
+                    collection.data.delete_many(where=weaviate_filter)
+        except Exception:
+            pass
+
+    def reset(self) -> None:
+        try:
+            for collection_name in self.client.collections.list_all().keys():
+                self.client.collections.delete(collection_name)
+        except Exception:
+            pass

backend/open_webui/retrieval/vector/factory.py

@@ -67,6 +67,10 @@ class Vector:
                 from open_webui.retrieval.vector.dbs.oracle23ai import Oracle23aiClient
 
                 return Oracle23aiClient()
+            case VectorType.WEAVIATE:
+                from open_webui.retrieval.vector.dbs.weaviate import WeaviateClient
+
+                return WeaviateClient()
             case _:
                 raise ValueError(f"Unsupported vector type: {vector_type}")
 

backend/open_webui/retrieval/vector/type.py

@@ -11,3 +11,4 @@ class VectorType(StrEnum):
     PGVECTOR = "pgvector"
     ORACLE23AI = "oracle23ai"
     S3VECTOR = "s3vector"
+    WEAVIATE = "weaviate"

backend/open_webui/retrieval/web/azure.py

@@ -0,0 +1,128 @@
+import logging
+from typing import Optional
+from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+from open_webui.env import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+"""
+Azure AI Search integration for Open WebUI.
+Documentation: https://learn.microsoft.com/en-us/python/api/overview/azure/search-documents-readme?view=azure-python
+
+Required package: azure-search-documents
+Install: pip install azure-search-documents
+"""
+
+
+def search_azure(
+    api_key: str,
+    endpoint: str,
+    index_name: str,
+    query: str,
+    count: int,
+    filter_list: Optional[list[str]] = None,
+) -> list[SearchResult]:
+    """
+    Search using Azure AI Search.
+
+    Args:
+        api_key: Azure Search API key (query key or admin key)
+        endpoint: Azure Search service endpoint (e.g., https://myservice.search.windows.net)
+        index_name: Name of the search index to query
+        query: Search query string
+        count: Number of results to return
+        filter_list: Optional list of domains to filter results
+
+    Returns:
+        List of SearchResult objects with link, title, and snippet
+    """
+    try:
+        from azure.core.credentials import AzureKeyCredential
+        from azure.search.documents import SearchClient
+    except ImportError:
+        log.error(
+            "azure-search-documents package is not installed. "
+            "Install it with: pip install azure-search-documents"
+        )
+        raise ImportError(
+            "azure-search-documents is required for Azure AI Search. "
+            "Install it with: pip install azure-search-documents"
+        )
+
+    try:
+        # Create search client with API key authentication
+        credential = AzureKeyCredential(api_key)
+        search_client = SearchClient(
+            endpoint=endpoint, index_name=index_name, credential=credential
+        )
+
+        # Perform the search
+        results = search_client.search(search_text=query, top=count)
+
+        # Convert results to list and extract fields
+        search_results = []
+        for result in results:
+            # Azure AI Search returns documents with custom schemas
+            # We need to extract common fields that might represent URL, title, and content
+            # Common field names to look for:
+            result_dict = dict(result)
+
+            # Try to find URL field (common names)
+            link = (
+                result_dict.get("url")
+                or result_dict.get("link")
+                or result_dict.get("uri")
+                or result_dict.get("metadata_storage_path")
+                or ""
+            )
+
+            # Try to find title field (common names)
+            title = (
+                result_dict.get("title")
+                or result_dict.get("name")
+                or result_dict.get("metadata_title")
+                or result_dict.get("metadata_storage_name")
+                or None
+            )
+
+            # Try to find content/snippet field (common names)
+            snippet = (
+                result_dict.get("content")
+                or result_dict.get("snippet")
+                or result_dict.get("description")
+                or result_dict.get("summary")
+                or result_dict.get("text")
+                or None
+            )
+
+            # Truncate snippet if too long
+            if snippet and len(snippet) > 500:
+                snippet = snippet[:497] + "..."
+
+            if link:  # Only add if we found a valid link
+                search_results.append(
+                    {
+                        "link": link,
+                        "title": title,
+                        "snippet": snippet,
+                    }
+                )
+
+        # Apply domain filtering if specified
+        if filter_list:
+            search_results = get_filtered_results(search_results, filter_list)
+
+        # Convert to SearchResult objects
+        return [
+            SearchResult(
+                link=result["link"],
+                title=result.get("title"),
+                snippet=result.get("snippet"),
+            )
+            for result in search_results
+        ]
+
+    except Exception as ex:
+        log.error(f"Azure AI Search error: {ex}")
+        raise ex

backend/open_webui/retrieval/web/external.py

@@ -2,27 +2,42 @@ import logging
 from typing import Optional, List
 
 import requests
-from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+
+from fastapi import Request
+
 from open_webui.env import SRC_LOG_LEVELS
 
+from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+from open_webui.utils.headers import include_user_info_headers
+
+
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["RAG"])
 
 
 def search_external(
+    request: Request,
     external_url: str,
     external_api_key: str,
     query: str,
     count: int,
     filter_list: Optional[List[str]] = None,
+    user=None,
 ) -> List[SearchResult]:
     try:
-        response = requests.post(
-            external_url,
-            headers={
+        headers = {
             "User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot",
             "Authorization": f"Bearer {external_api_key}",
-            },
+        }
+        headers = include_user_info_headers(headers, user)
+
+        chat_id = getattr(request.state, "chat_id", None)
+        if chat_id:
+            headers["X-OpenWebUI-Chat-Id"] = str(chat_id)
+
+        response = requests.post(
+            external_url,
+            headers=headers,
             json={
                 "query": query,
                 "count": count,

backend/open_webui/retrieval/web/firecrawl.py

@@ -1,11 +1,10 @@
 import logging
 from typing import Optional, List
-from urllib.parse import urljoin
 
-import requests
 from open_webui.retrieval.web.main import SearchResult, get_filtered_results
 from open_webui.env import SRC_LOG_LEVELS
 
+
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["RAG"])
 
@@ -18,27 +17,20 @@ def search_firecrawl(
     filter_list: Optional[List[str]] = None,
 ) -> List[SearchResult]:
     try:
-        firecrawl_search_url = urljoin(firecrawl_url, "/v1/search")
-        response = requests.post(
-            firecrawl_search_url,
-            headers={
-                "User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot",
-                "Authorization": f"Bearer {firecrawl_api_key}",
-            },
-            json={
-                "query": query,
-                "limit": count,
-            },
+        from firecrawl import FirecrawlApp
+
+        firecrawl = FirecrawlApp(api_key=firecrawl_api_key, api_url=firecrawl_url)
+        response = firecrawl.search(
+            query=query, limit=count, ignore_invalid_urls=True, timeout=count * 3
         )
-        response.raise_for_status()
-        results = response.json().get("data", [])
+        results = response.web
         if filter_list:
             results = get_filtered_results(results, filter_list)
         results = [
             SearchResult(
-                link=result.get("url"),
-                title=result.get("title"),
-                snippet=result.get("description"),
+                link=result.url,
+                title=result.title,
+                snippet=result.description,
             )
             for result in results[:count]
         ]

backend/open_webui/retrieval/web/google_pse.py

@@ -15,6 +15,7 @@ def search_google_pse(
     query: str,
     count: int,
     filter_list: Optional[list[str]] = None,
+    referer: Optional[str] = None,
 ) -> list[SearchResult]:
     """Search using Google's Programmable Search Engine API and return the results as a list of SearchResult objects.
     Handles pagination for counts greater than 10.
@@ -30,7 +31,11 @@ def search_google_pse(
         list[SearchResult]: A list of SearchResult objects.
     """
     url = "https://www.googleapis.com/customsearch/v1"
+
     headers = {"Content-Type": "application/json"}
+    if referer:
+        headers["Referer"] = referer
+
     all_results = []
     start_index = 1  # Google PSE start parameter is 1-based
 

backend/open_webui/retrieval/web/main.py

@@ -5,18 +5,38 @@ from urllib.parse import urlparse
 
 from pydantic import BaseModel
 
+from open_webui.retrieval.web.utils import resolve_hostname
+from open_webui.utils.misc import is_string_allowed
+
 
 def get_filtered_results(results, filter_list):
     if not filter_list:
         return results
+
     filtered_results = []
+
     for result in results:
         url = result.get("url") or result.get("link", "") or result.get("href", "")
         if not validators.url(url):
             continue
+
         domain = urlparse(url).netloc
-        if any(domain.endswith(filtered_domain) for filtered_domain in filter_list):
+        if not domain:
+            continue
+
+        hostnames = [domain]
+
+        try:
+            ipv4_addresses, ipv6_addresses = resolve_hostname(domain)
+            hostnames.extend(ipv4_addresses)
+            hostnames.extend(ipv6_addresses)
+        except Exception:
+            pass
+
+        if any(is_string_allowed(hostname, filter_list) for hostname in hostnames):
             filtered_results.append(result)
+            continue
+
     return filtered_results
 
 

backend/open_webui/retrieval/web/perplexity_search.py

@@ -3,6 +3,7 @@ from typing import Optional, Literal
 import requests
 
 from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+from open_webui.utils.headers import include_user_info_headers
 from open_webui.env import SRC_LOG_LEVELS
 
 
@@ -15,6 +16,8 @@ def search_perplexity_search(
     query: str,
     count: int,
     filter_list: Optional[list[str]] = None,
+    api_url: str = "https://api.perplexity.ai/search",
+    user=None,
 ) -> list[SearchResult]:
     """Search using Perplexity API and return the results as a list of SearchResult objects.
 
@@ -23,6 +26,8 @@ def search_perplexity_search(
       query (str): The query to search for
       count (int): Maximum number of results to return
       filter_list (Optional[list[str]]): List of domains to filter results
+      api_url (str): Custom API URL (defaults to https://api.perplexity.ai/search)
+      user: Optional user object for forwarding user info headers
 
     """
 
@@ -30,8 +35,11 @@ def search_perplexity_search(
     if hasattr(api_key, "__str__"):
         api_key = str(api_key)
 
+    if hasattr(api_url, "__str__"):
+        api_url = str(api_url)
+
     try:
-        url = "https://api.perplexity.ai/search"
+        url = api_url
 
         # Create payload for the API call
         payload = {
@@ -44,6 +52,10 @@ def search_perplexity_search(
             "Content-Type": "application/json",
         }
 
+        # Forward user info headers if user is provided
+        if user is not None:
+            headers = include_user_info_headers(headers, user)
+
         # Make the API request
         response = requests.request("POST", url, json=payload, headers=headers)
         # Parse the JSON response

backend/open_webui/retrieval/web/utils.py

@@ -4,7 +4,6 @@ import socket
 import ssl
 import urllib.parse
 import urllib.request
-from collections import defaultdict
 from datetime import datetime, time, timedelta
 from typing import (
     Any,
@@ -17,13 +16,15 @@ from typing import (
     Union,
     Literal,
 )
+
+from fastapi.concurrency import run_in_threadpool
 import aiohttp
 import certifi
 import validators
 from langchain_community.document_loaders import PlaywrightURLLoader, WebBaseLoader
-from langchain_community.document_loaders.firecrawl import FireCrawlLoader
 from langchain_community.document_loaders.base import BaseLoader
 from langchain_core.documents import Document
+
 from open_webui.retrieval.loaders.tavily import TavilyLoader
 from open_webui.retrieval.loaders.external_web import ExternalWebLoader
 from open_webui.constants import ERROR_MESSAGES
@@ -38,17 +39,46 @@ from open_webui.config import (
     TAVILY_EXTRACT_DEPTH,
     EXTERNAL_WEB_LOADER_URL,
     EXTERNAL_WEB_LOADER_API_KEY,
+    WEB_FETCH_FILTER_LIST,
 )
-from open_webui.env import SRC_LOG_LEVELS, AIOHTTP_CLIENT_SESSION_SSL
+from open_webui.env import SRC_LOG_LEVELS
+from open_webui.utils.misc import is_string_allowed
 
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["RAG"])
 
 
+def resolve_hostname(hostname):
+    # Get address information
+    addr_info = socket.getaddrinfo(hostname, None)
+
+    # Extract IP addresses from address information
+    ipv4_addresses = [info[4][0] for info in addr_info if info[0] == socket.AF_INET]
+    ipv6_addresses = [info[4][0] for info in addr_info if info[0] == socket.AF_INET6]
+
+    return ipv4_addresses, ipv6_addresses
+
+
 def validate_url(url: Union[str, Sequence[str]]):
     if isinstance(url, str):
         if isinstance(validators.url(url), validators.ValidationError):
             raise ValueError(ERROR_MESSAGES.INVALID_URL)
+
+        parsed_url = urllib.parse.urlparse(url)
+
+        # Protocol validation - only allow http/https
+        if parsed_url.scheme not in ["http", "https"]:
+            log.warning(
+                f"Blocked non-HTTP(S) protocol: {parsed_url.scheme} in URL: {url}"
+            )
+            raise ValueError(ERROR_MESSAGES.INVALID_URL)
+
+        # Blocklist check using unified filtering logic
+        if WEB_FETCH_FILTER_LIST:
+            if not is_string_allowed(url, WEB_FETCH_FILTER_LIST):
+                log.warning(f"URL blocked by filter list: {url}")
+                raise ValueError(ERROR_MESSAGES.INVALID_URL)
+
         if not ENABLE_RAG_LOCAL_WEB_FETCH:
             # Local web fetch is disabled, filter out any URLs that resolve to private IP addresses
             parsed_url = urllib.parse.urlparse(url)
@@ -75,22 +105,12 @@ def safe_validate_urls(url: Sequence[str]) -> Sequence[str]:
         try:
             if validate_url(u):
                 valid_urls.append(u)
-        except ValueError:
+        except Exception as e:
+            log.debug(f"Invalid URL {u}: {str(e)}")
             continue
     return valid_urls
 
 
-def resolve_hostname(hostname):
-    # Get address information
-    addr_info = socket.getaddrinfo(hostname, None)
-
-    # Extract IP addresses from address information
-    ipv4_addresses = [info[4][0] for info in addr_info if info[0] == socket.AF_INET]
-    ipv6_addresses = [info[4][0] for info in addr_info if info[0] == socket.AF_INET6]
-
-    return ipv4_addresses, ipv6_addresses
-
-
 def extract_metadata(soup, url):
     metadata = {"source": url}
     if title := soup.find("title"):
@@ -141,13 +161,13 @@ class RateLimitMixin:
 
 
 class URLProcessingMixin:
-    def _verify_ssl_cert(self, url: str) -> bool:
+    async def _verify_ssl_cert(self, url: str) -> bool:
         """Verify SSL certificate for a URL."""
-        return verify_ssl_cert(url)
+        return await run_in_threadpool(verify_ssl_cert, url)
 
     async def _safe_process_url(self, url: str) -> bool:
         """Perform safety checks before processing a URL."""
-        if self.verify_ssl and not self._verify_ssl_cert(url):
+        if self.verify_ssl and not await self._verify_ssl_cert(url):
             raise ValueError(f"SSL certificate verification failed for {url}")
         await self._wait_for_rate_limit()
         return True
@@ -188,13 +208,12 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
                 (uses FIRE_CRAWL_API_KEY environment variable if not provided).
             api_url: Base URL for FireCrawl API. Defaults to official API endpoint.
             mode: Operation mode selection:
-                - 'crawl': Website crawling mode (default)
-                - 'scrape': Direct page scraping
+                - 'crawl': Website crawling mode
+                - 'scrape': Direct page scraping (default)
                 - 'map': Site map generation
             proxy: Proxy override settings for the FireCrawl API.
             params: The parameters to pass to the Firecrawl API.
-                Examples include crawlerOptions.
-                For more details, visit: https://github.com/mendableai/firecrawl-py
+                For more details, visit: https://docs.firecrawl.dev/sdks/python#batch-scrape
         """
         proxy_server = proxy.get("server") if proxy else None
         if trust_env and not proxy_server:
@@ -214,50 +233,88 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
         self.api_key = api_key
         self.api_url = api_url
         self.mode = mode
-        self.params = params
+        self.params = params or {}
 
     def lazy_load(self) -> Iterator[Document]:
-        """Load documents concurrently using FireCrawl."""
-        for url in self.web_paths:
-            try:
-                self._safe_process_url_sync(url)
-                loader = FireCrawlLoader(
-                    url=url,
-                    api_key=self.api_key,
-                    api_url=self.api_url,
-                    mode=self.mode,
-                    params=self.params,
+        """Load documents using FireCrawl batch_scrape."""
+        log.debug(
+            "Starting FireCrawl batch scrape for %d URLs, mode: %s, params: %s",
+            len(self.web_paths),
+            self.mode,
+            self.params,
         )
-                for document in loader.lazy_load():
-                    if not document.metadata.get("source"):
-                        document.metadata["source"] = document.metadata.get("sourceURL")
-                    yield document
+        try:
+            from firecrawl import FirecrawlApp
+
+            firecrawl = FirecrawlApp(api_key=self.api_key, api_url=self.api_url)
+            result = firecrawl.batch_scrape(
+                self.web_paths,
+                formats=["markdown"],
+                skip_tls_verification=not self.verify_ssl,
+                ignore_invalid_urls=True,
+                remove_base64_images=True,
+                max_age=300000,  # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
+                wait_timeout=len(self.web_paths) * 3,
+                **self.params,
+            )
+
+            if result.status != "completed":
+                raise RuntimeError(
+                    f"FireCrawl batch scrape did not complete successfully. result: {result}"
+                )
+
+            for data in result.data:
+                metadata = data.metadata or {}
+                yield Document(
+                    page_content=data.markdown or "",
+                    metadata={"source": metadata.url or metadata.source_url or ""},
+                )
+
         except Exception as e:
             if self.continue_on_failure:
-                    log.exception(f"Error loading {url}: {e}")
-                    continue
+                log.exception(f"Error extracting content from URLs: {e}")
+            else:
                 raise e
 
     async def alazy_load(self):
         """Async version of lazy_load."""
-        for url in self.web_paths:
-            try:
-                await self._safe_process_url(url)
-                loader = FireCrawlLoader(
-                    url=url,
-                    api_key=self.api_key,
-                    api_url=self.api_url,
-                    mode=self.mode,
-                    params=self.params,
+        log.debug(
+            "Starting FireCrawl batch scrape for %d URLs, mode: %s, params: %s",
+            len(self.web_paths),
+            self.mode,
+            self.params,
         )
-                async for document in loader.alazy_load():
-                    if not document.metadata.get("source"):
-                        document.metadata["source"] = document.metadata.get("sourceURL")
-                    yield document
+        try:
+            from firecrawl import FirecrawlApp
+
+            firecrawl = FirecrawlApp(api_key=self.api_key, api_url=self.api_url)
+            result = firecrawl.batch_scrape(
+                self.web_paths,
+                formats=["markdown"],
+                skip_tls_verification=not self.verify_ssl,
+                ignore_invalid_urls=True,
+                remove_base64_images=True,
+                max_age=300000,  # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
+                wait_timeout=len(self.web_paths) * 3,
+                **self.params,
+            )
+
+            if result.status != "completed":
+                raise RuntimeError(
+                    f"FireCrawl batch scrape did not complete successfully. result: {result}"
+                )
+
+            for data in result.data:
+                metadata = data.metadata or {}
+                yield Document(
+                    page_content=data.markdown or "",
+                    metadata={"source": metadata.url or metadata.source_url or ""},
+                )
+
         except Exception as e:
             if self.continue_on_failure:
-                    log.exception(f"Error loading {url}: {e}")
-                    continue
+                log.exception(f"Error extracting content from URLs: {e}")
+            else:
                 raise e
 
 
@@ -603,6 +660,10 @@ def get_web_loader(
     # Check if the URLs are valid
     safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
 
+    if not safe_urls:
+        log.warning(f"All provided URLs were blocked or invalid: {urls}")
+        raise ValueError(ERROR_MESSAGES.INVALID_URL)
+
     web_loader_args = {
         "web_paths": safe_urls,
         "verify_ssl": verify_ssl,

backend/open_webui/routers/audio.py

@@ -4,6 +4,7 @@ import logging
 import os
 import uuid
 import html
+import base64
 from functools import lru_cache
 from pydub import AudioSegment
 from pydub.silence import split_on_silence
@@ -15,7 +16,6 @@ import aiohttp
 import aiofiles
 import requests
 import mimetypes
-from urllib.parse import urljoin, quote
 
 from fastapi import (
     Depends,
@@ -34,18 +34,20 @@ from pydantic import BaseModel
 
 
 from open_webui.utils.auth import get_admin_user, get_verified_user
+from open_webui.utils.headers import include_user_info_headers
 from open_webui.config import (
     WHISPER_MODEL_AUTO_UPDATE,
     WHISPER_MODEL_DIR,
     CACHE_DIR,
     WHISPER_LANGUAGE,
+    ELEVENLABS_API_BASE_URL,
 )
 
 from open_webui.constants import ERROR_MESSAGES
 from open_webui.env import (
+    ENV,
     AIOHTTP_CLIENT_SESSION_SSL,
     AIOHTTP_CLIENT_TIMEOUT,
-    ENV,
     SRC_LOG_LEVELS,
     DEVICE_TYPE,
     ENABLE_FORWARD_USER_INFO_HEADERS,
@@ -178,6 +180,9 @@ class STTConfigForm(BaseModel):
     AZURE_LOCALES: str
     AZURE_BASE_URL: str
     AZURE_MAX_SPEAKERS: str
+    MISTRAL_API_KEY: str
+    MISTRAL_API_BASE_URL: str
+    MISTRAL_USE_CHAT_COMPLETIONS: bool
 
 
 class AudioConfigUpdateForm(BaseModel):
@@ -214,6 +219,9 @@ async def get_audio_config(request: Request, user=Depends(get_admin_user)):
             "AZURE_LOCALES": request.app.state.config.AUDIO_STT_AZURE_LOCALES,
             "AZURE_BASE_URL": request.app.state.config.AUDIO_STT_AZURE_BASE_URL,
             "AZURE_MAX_SPEAKERS": request.app.state.config.AUDIO_STT_AZURE_MAX_SPEAKERS,
+            "MISTRAL_API_KEY": request.app.state.config.AUDIO_STT_MISTRAL_API_KEY,
+            "MISTRAL_API_BASE_URL": request.app.state.config.AUDIO_STT_MISTRAL_API_BASE_URL,
+            "MISTRAL_USE_CHAT_COMPLETIONS": request.app.state.config.AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS,
         },
     }
 
@@ -255,6 +263,13 @@ async def update_audio_config(
     request.app.state.config.AUDIO_STT_AZURE_MAX_SPEAKERS = (
         form_data.stt.AZURE_MAX_SPEAKERS
     )
+    request.app.state.config.AUDIO_STT_MISTRAL_API_KEY = form_data.stt.MISTRAL_API_KEY
+    request.app.state.config.AUDIO_STT_MISTRAL_API_BASE_URL = (
+        form_data.stt.MISTRAL_API_BASE_URL
+    )
+    request.app.state.config.AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS = (
+        form_data.stt.MISTRAL_USE_CHAT_COMPLETIONS
+    )
 
     if request.app.state.config.STT_ENGINE == "":
         request.app.state.faster_whisper_model = set_faster_whisper_model(
@@ -290,6 +305,9 @@ async def update_audio_config(
             "AZURE_LOCALES": request.app.state.config.AUDIO_STT_AZURE_LOCALES,
             "AZURE_BASE_URL": request.app.state.config.AUDIO_STT_AZURE_BASE_URL,
             "AZURE_MAX_SPEAKERS": request.app.state.config.AUDIO_STT_AZURE_MAX_SPEAKERS,
+            "MISTRAL_API_KEY": request.app.state.config.AUDIO_STT_MISTRAL_API_KEY,
+            "MISTRAL_API_BASE_URL": request.app.state.config.AUDIO_STT_MISTRAL_API_BASE_URL,
+            "MISTRAL_USE_CHAT_COMPLETIONS": request.app.state.config.AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS,
         },
     }
 
@@ -346,23 +364,17 @@ async def speech(request: Request, user=Depends(get_verified_user)):
                     **(request.app.state.config.TTS_OPENAI_PARAMS or {}),
                 }
 
+                headers = {
+                    "Content-Type": "application/json",
+                    "Authorization": f"Bearer {request.app.state.config.TTS_OPENAI_API_KEY}",
+                }
+                if ENABLE_FORWARD_USER_INFO_HEADERS:
+                    headers = include_user_info_headers(headers, user)
+
                 r = await session.post(
                     url=f"{request.app.state.config.TTS_OPENAI_API_BASE_URL}/audio/speech",
                     json=payload,
-                    headers={
-                        "Content-Type": "application/json",
-                        "Authorization": f"Bearer {request.app.state.config.TTS_OPENAI_API_KEY}",
-                        **(
-                            {
-                                "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                                "X-OpenWebUI-User-Id": user.id,
-                                "X-OpenWebUI-User-Email": user.email,
-                                "X-OpenWebUI-User-Role": user.role,
-                            }
-                            if ENABLE_FORWARD_USER_INFO_HEADERS
-                            else {}
-                        ),
-                    },
+                    headers=headers,
                     ssl=AIOHTTP_CLIENT_SESSION_SSL,
                 )
 
@@ -413,7 +425,7 @@ async def speech(request: Request, user=Depends(get_verified_user)):
                 timeout=timeout, trust_env=True
             ) as session:
                 async with session.post(
-                    f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}",
+                    f"{ELEVENLABS_API_BASE_URL}/v1/text-to-speech/{voice_id}",
                     json={
                         "text": payload["input"],
                         "model_id": request.app.state.config.TTS_MODEL,
@@ -552,7 +564,7 @@ async def speech(request: Request, user=Depends(get_verified_user)):
         return FileResponse(file_path)
 
 
-def transcription_handler(request, file_path, metadata):
+def transcription_handler(request, file_path, metadata, user=None):
     filename = os.path.basename(file_path)
     file_dir = os.path.dirname(file_path)
     id = filename.split(".")[0]
@@ -603,11 +615,15 @@ def transcription_handler(request, file_path, metadata):
                 if language:
                     payload["language"] = language
 
+                headers = {
+                    "Authorization": f"Bearer {request.app.state.config.STT_OPENAI_API_KEY}"
+                }
+                if user and ENABLE_FORWARD_USER_INFO_HEADERS:
+                    headers = include_user_info_headers(headers, user)
+
                 r = requests.post(
                     url=f"{request.app.state.config.STT_OPENAI_API_BASE_URL}/audio/transcriptions",
-                    headers={
-                        "Authorization": f"Bearer {request.app.state.config.STT_OPENAI_API_KEY}"
-                    },
+                    headers=headers,
                     files={"file": (filename, open(file_path, "rb"))},
                     data=payload,
                 )
@@ -828,8 +844,190 @@ def transcription_handler(request, file_path, metadata):
                 detail=detail if detail else "Open WebUI: Server Connection Error",
             )
 
+    elif request.app.state.config.STT_ENGINE == "mistral":
+        # Check file exists
+        if not os.path.exists(file_path):
+            raise HTTPException(status_code=400, detail="Audio file not found")
 
-def transcribe(request: Request, file_path: str, metadata: Optional[dict] = None):
+        # Check file size
+        file_size = os.path.getsize(file_path)
+        if file_size > MAX_FILE_SIZE:
+            raise HTTPException(
+                status_code=400,
+                detail=f"File size exceeds limit of {MAX_FILE_SIZE_MB}MB",
+            )
+
+        api_key = request.app.state.config.AUDIO_STT_MISTRAL_API_KEY
+        api_base_url = (
+            request.app.state.config.AUDIO_STT_MISTRAL_API_BASE_URL
+            or "https://api.mistral.ai/v1"
+        )
+        use_chat_completions = (
+            request.app.state.config.AUDIO_STT_MISTRAL_USE_CHAT_COMPLETIONS
+        )
+
+        if not api_key:
+            raise HTTPException(
+                status_code=400,
+                detail="Mistral API key is required for Mistral STT",
+            )
+
+        r = None
+        try:
+            # Use voxtral-mini-latest as the default model for transcription
+            model = request.app.state.config.STT_MODEL or "voxtral-mini-latest"
+
+            log.info(
+                f"Mistral STT - model: {model}, "
+                f"method: {'chat_completions' if use_chat_completions else 'transcriptions'}"
+            )
+
+            if use_chat_completions:
+                # Use chat completions API with audio input
+                # This method requires mp3 or wav format
+                audio_file_to_use = file_path
+
+                if is_audio_conversion_required(file_path):
+                    log.debug("Converting audio to mp3 for chat completions API")
+                    converted_path = convert_audio_to_mp3(file_path)
+                    if converted_path:
+                        audio_file_to_use = converted_path
+                    else:
+                        log.error("Audio conversion failed")
+                        raise HTTPException(
+                            status_code=500,
+                            detail="Audio conversion failed. Chat completions API requires mp3 or wav format.",
+                        )
+
+                # Read and encode audio file as base64
+                with open(audio_file_to_use, "rb") as audio_file:
+                    audio_base64 = base64.b64encode(audio_file.read()).decode("utf-8")
+
+                # Prepare chat completions request
+                url = f"{api_base_url}/chat/completions"
+
+                # Add language instruction if specified
+                language = metadata.get("language", None) if metadata else None
+                if language:
+                    text_instruction = f"Transcribe this audio exactly as spoken in {language}. Do not translate it."
+                else:
+                    text_instruction = "Transcribe this audio exactly as spoken in its original language. Do not translate it to another language."
+
+                payload = {
+                    "model": model,
+                    "messages": [
+                        {
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "input_audio",
+                                    "input_audio": audio_base64,
+                                },
+                                {"type": "text", "text": text_instruction},
+                            ],
+                        }
+                    ],
+                }
+
+                r = requests.post(
+                    url=url,
+                    json=payload,
+                    headers={
+                        "Authorization": f"Bearer {api_key}",
+                        "Content-Type": "application/json",
+                    },
+                )
+
+                r.raise_for_status()
+                response = r.json()
+
+                # Extract transcript from chat completion response
+                transcript = (
+                    response.get("choices", [{}])[0]
+                    .get("message", {})
+                    .get("content", "")
+                    .strip()
+                )
+                if not transcript:
+                    raise ValueError("Empty transcript in response")
+
+                data = {"text": transcript}
+
+            else:
+                # Use dedicated transcriptions API
+                url = f"{api_base_url}/audio/transcriptions"
+
+                # Determine the MIME type
+                mime_type, _ = mimetypes.guess_type(file_path)
+                if not mime_type:
+                    mime_type = "audio/webm"
+
+                # Use context manager to ensure file is properly closed
+                with open(file_path, "rb") as audio_file:
+                    files = {"file": (filename, audio_file, mime_type)}
+                    data_form = {"model": model}
+
+                    # Add language if specified in metadata
+                    language = metadata.get("language", None) if metadata else None
+                    if language:
+                        data_form["language"] = language
+
+                    r = requests.post(
+                        url=url,
+                        files=files,
+                        data=data_form,
+                        headers={
+                            "Authorization": f"Bearer {api_key}",
+                        },
+                    )
+
+                r.raise_for_status()
+                response = r.json()
+
+                # Extract transcript from response
+                transcript = response.get("text", "").strip()
+                if not transcript:
+                    raise ValueError("Empty transcript in response")
+
+                data = {"text": transcript}
+
+            # Save transcript to json file (consistent with other providers)
+            transcript_file = f"{file_dir}/{id}.json"
+            with open(transcript_file, "w") as f:
+                json.dump(data, f)
+
+            log.debug(data)
+            return data
+
+        except ValueError as e:
+            log.exception("Error parsing Mistral response")
+            raise HTTPException(
+                status_code=500,
+                detail=f"Failed to parse Mistral response: {str(e)}",
+            )
+        except requests.exceptions.RequestException as e:
+            log.exception(e)
+            detail = None
+
+            try:
+                if r is not None and r.status_code != 200:
+                    res = r.json()
+                    if "error" in res:
+                        detail = f"External: {res['error'].get('message', '')}"
+                    else:
+                        detail = f"External: {r.text}"
+            except Exception:
+                detail = f"External: {e}"
+
+            raise HTTPException(
+                status_code=getattr(r, "status_code", 500) if r else 500,
+                detail=detail if detail else "Open WebUI: Server Connection Error",
+            )
+
+
+def transcribe(
+    request: Request, file_path: str, metadata: Optional[dict] = None, user=None
+):
     log.info(f"transcribe: {file_path} {metadata}")
 
     if is_audio_conversion_required(file_path):
@@ -856,7 +1054,9 @@ def transcribe(request: Request, file_path: str, metadata: Optional[dict] = None
         with ThreadPoolExecutor() as executor:
             # Submit tasks for each chunk_path
             futures = [
-                executor.submit(transcription_handler, request, chunk_path, metadata)
+                executor.submit(
+                    transcription_handler, request, chunk_path, metadata, user
+                )
                 for chunk_path in chunk_paths
             ]
             # Gather results as they complete
@@ -991,7 +1191,7 @@ def transcription(
             if language:
                 metadata = {"language": language}
 
-            result = transcribe(request, file_path, metadata)
+            result = transcribe(request, file_path, metadata, user)
 
             return {
                 **result,
@@ -1037,7 +1237,7 @@ def get_available_models(request: Request) -> list[dict]:
     elif request.app.state.config.TTS_ENGINE == "elevenlabs":
         try:
             response = requests.get(
-                "https://api.elevenlabs.io/v1/models",
+                f"{ELEVENLABS_API_BASE_URL}/v1/models",
                 headers={
                     "xi-api-key": request.app.state.config.TTS_API_KEY,
                     "Content-Type": "application/json",
@@ -1141,7 +1341,7 @@ def get_elevenlabs_voices(api_key: str) -> dict:
     try:
         # TODO: Add retries
         response = requests.get(
-            "https://api.elevenlabs.io/v1/voices",
+            f"{ELEVENLABS_API_BASE_URL}/v1/voices",
             headers={
                 "xi-api-key": api_key,
                 "Content-Type": "application/json",

backend/open_webui/routers/auths.py

@@ -4,6 +4,7 @@ import time
 import datetime
 import logging
 from aiohttp import ClientSession
+import urllib
 
 from open_webui.models.auths import (
     AddUserForm,
@@ -15,9 +16,8 @@ from open_webui.models.auths import (
     SigninResponse,
     SignupForm,
     UpdatePasswordForm,
-    UserResponse,
 )
-from open_webui.models.users import Users, UpdateProfileForm
+from open_webui.models.users import UserProfileImageResponse, Users, UpdateProfileForm
 from open_webui.models.groups import Groups
 from open_webui.models.oauth_sessions import OAuthSessions
 
@@ -35,12 +35,20 @@ from open_webui.env import (
 )
 from fastapi import APIRouter, Depends, HTTPException, Request, status
 from fastapi.responses import RedirectResponse, Response, JSONResponse
-from open_webui.config import OPENID_PROVIDER_URL, ENABLE_OAUTH_SIGNUP, ENABLE_LDAP
+from open_webui.config import (
+    OPENID_PROVIDER_URL,
+    ENABLE_OAUTH_SIGNUP,
+    ENABLE_LDAP,
+    ENABLE_PASSWORD_AUTH,
+)
 from pydantic import BaseModel
 
 from open_webui.utils.misc import parse_duration, validate_email_format
 from open_webui.utils.auth import (
+    validate_password,
+    verify_password,
     decode_token,
+    invalidate_token,
     create_api_key,
     create_token,
     get_admin_user,
@@ -50,7 +58,7 @@ from open_webui.utils.auth import (
     get_http_authorization_cred,
 )
 from open_webui.utils.webhook import post_webhook
-from open_webui.utils.access_control import get_permissions
+from open_webui.utils.access_control import get_permissions, has_permission
 
 from typing import Optional, List
 
@@ -69,7 +77,7 @@ log.setLevel(SRC_LOG_LEVELS["MAIN"])
 ############################
 
 
-class SessionUserResponse(Token, UserResponse):
+class SessionUserResponse(Token, UserProfileImageResponse):
     expires_at: Optional[int] = None
     permissions: Optional[dict] = None
 
@@ -140,7 +148,7 @@ async def get_session_user(
 ############################
 
 
-@router.post("/update/profile", response_model=UserResponse)
+@router.post("/update/profile", response_model=UserProfileImageResponse)
 async def update_profile(
     form_data: UpdateProfileForm, session_user=Depends(get_verified_user)
 ):
@@ -169,13 +177,19 @@ async def update_password(
     if WEBUI_AUTH_TRUSTED_EMAIL_HEADER:
         raise HTTPException(400, detail=ERROR_MESSAGES.ACTION_PROHIBITED)
     if session_user:
-        user = Auths.authenticate_user(session_user.email, form_data.password)
+        user = Auths.authenticate_user(
+            session_user.email, lambda pw: verify_password(form_data.password, pw)
+        )
 
         if user:
+            try:
+                validate_password(form_data.password)
+            except Exception as e:
+                raise HTTPException(400, detail=str(e))
             hashed = get_password_hash(form_data.new_password)
             return Auths.update_user_password_by_id(user.id, hashed)
         else:
-            raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_PASSWORD)
+            raise HTTPException(400, detail=ERROR_MESSAGES.INCORRECT_PASSWORD)
     else:
         raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
 
@@ -185,7 +199,17 @@ async def update_password(
 ############################
 @router.post("/ldap", response_model=SessionUserResponse)
 async def ldap_auth(request: Request, response: Response, form_data: LdapForm):
-    ENABLE_LDAP = request.app.state.config.ENABLE_LDAP
+    # Security checks FIRST - before loading any config
+    if not request.app.state.config.ENABLE_LDAP:
+        raise HTTPException(400, detail="LDAP authentication is not enabled")
+
+    if not ENABLE_PASSWORD_AUTH:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=ERROR_MESSAGES.ACTION_PROHIBITED,
+        )
+
+    # NOW load LDAP config variables
     LDAP_SERVER_LABEL = request.app.state.config.LDAP_SERVER_LABEL
     LDAP_SERVER_HOST = request.app.state.config.LDAP_SERVER_HOST
     LDAP_SERVER_PORT = request.app.state.config.LDAP_SERVER_PORT
@@ -206,9 +230,6 @@ async def ldap_auth(request: Request, response: Response, form_data: LdapForm):
         else "ALL"
     )
 
-    if not ENABLE_LDAP:
-        raise HTTPException(400, detail="LDAP authentication is not enabled")
-
     try:
         tls = Tls(
             validate=LDAP_VALIDATE_CERT,
@@ -463,6 +484,12 @@ async def ldap_auth(request: Request, response: Response, form_data: LdapForm):
 
 @router.post("/signin", response_model=SessionUserResponse)
 async def signin(request: Request, response: Response, form_data: SigninForm):
+    if not ENABLE_PASSWORD_AUTH:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=ERROR_MESSAGES.ACTION_PROHIBITED,
+        )
+
     if WEBUI_AUTH_TRUSTED_EMAIL_HEADER:
         if WEBUI_AUTH_TRUSTED_EMAIL_HEADER not in request.headers:
             raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_TRUSTED_HEADER)
@@ -472,6 +499,10 @@ async def signin(request: Request, response: Response, form_data: SigninForm):
 
         if WEBUI_AUTH_TRUSTED_NAME_HEADER:
             name = request.headers.get(WEBUI_AUTH_TRUSTED_NAME_HEADER, email)
+            try:
+                name = urllib.parse.unquote(name, encoding="utf-8")
+            except Exception as e:
+                pass
 
         if not Users.get_user_by_email(email.lower()):
             await signup(
@@ -495,7 +526,9 @@ async def signin(request: Request, response: Response, form_data: SigninForm):
         admin_password = "admin"
 
         if Users.get_user_by_email(admin_email.lower()):
-            user = Auths.authenticate_user(admin_email.lower(), admin_password)
+            user = Auths.authenticate_user(
+                admin_email.lower(), lambda pw: verify_password(admin_password, pw)
+            )
         else:
             if Users.has_users():
                 raise HTTPException(400, detail=ERROR_MESSAGES.EXISTING_USERS)
@@ -506,9 +539,22 @@ async def signin(request: Request, response: Response, form_data: SigninForm):
                 SignupForm(email=admin_email, password=admin_password, name="User"),
             )
 
-            user = Auths.authenticate_user(admin_email.lower(), admin_password)
+            user = Auths.authenticate_user(
+                admin_email.lower(), lambda pw: verify_password(admin_password, pw)
+            )
     else:
-        user = Auths.authenticate_user(form_data.email.lower(), form_data.password)
+        password_bytes = form_data.password.encode("utf-8")
+        if len(password_bytes) > 72:
+            # TODO: Implement other hashing algorithms that support longer passwords
+            log.info("Password too long, truncating to 72 bytes for bcrypt")
+            password_bytes = password_bytes[:72]
+
+            # decode safely — ignore incomplete UTF-8 sequences
+            form_data.password = password_bytes.decode("utf-8", errors="ignore")
+
+        user = Auths.authenticate_user(
+            form_data.email.lower(), lambda pw: verify_password(form_data.password, pw)
+        )
 
     if user:
 
@@ -590,16 +636,14 @@ async def signup(request: Request, response: Response, form_data: SignupForm):
         raise HTTPException(400, detail=ERROR_MESSAGES.EMAIL_TAKEN)
 
     try:
-        role = "admin" if not has_users else request.app.state.config.DEFAULT_USER_ROLE
-
-        # The password passed to bcrypt must be 72 bytes or fewer. If it is longer, it will be truncated before hashing.
-        if len(form_data.password.encode("utf-8")) > 72:
-            raise HTTPException(
-                status.HTTP_400_BAD_REQUEST,
-                detail=ERROR_MESSAGES.PASSWORD_TOO_LONG,
-            )
+        try:
+            validate_password(form_data.password)
+        except Exception as e:
+            raise HTTPException(400, detail=str(e))
 
         hashed = get_password_hash(form_data.password)
+
+        role = "admin" if not has_users else request.app.state.config.DEFAULT_USER_ROLE
         user = Auths.insert_new_auth(
             form_data.email.lower(),
             hashed,
@@ -655,6 +699,10 @@ async def signup(request: Request, response: Response, form_data: SignupForm):
                 # Disable signup after the first user is created
                 request.app.state.config.ENABLE_SIGNUP = False
 
+            default_group_id = getattr(request.app.state.config, "DEFAULT_GROUP_ID", "")
+            if default_group_id and default_group_id:
+                Groups.add_users_to_group(default_group_id, [user.id])
+
             return {
                 "token": token,
                 "token_type": "Bearer",
@@ -675,6 +723,19 @@ async def signup(request: Request, response: Response, form_data: SignupForm):
 
 @router.get("/signout")
 async def signout(request: Request, response: Response):
+
+    # get auth token from headers or cookies
+    token = None
+    auth_header = request.headers.get("Authorization")
+    if auth_header:
+        auth_cred = get_http_authorization_cred(auth_header)
+        token = auth_cred.credentials
+    else:
+        token = request.cookies.get("token")
+
+    if token:
+        await invalidate_token(request, token)
+
     response.delete_cookie("token")
     response.delete_cookie("oui-session")
     response.delete_cookie("oauth_id_token")
@@ -755,6 +816,11 @@ async def add_user(form_data: AddUserForm, user=Depends(get_admin_user)):
         raise HTTPException(400, detail=ERROR_MESSAGES.EMAIL_TAKEN)
 
     try:
+        try:
+            validate_password(form_data.password)
+        except Exception as e:
+            raise HTTPException(400, detail=str(e))
+
         hashed = get_password_hash(form_data.password)
         user = Auths.insert_new_auth(
             form_data.email.lower(),
@@ -826,13 +892,15 @@ async def get_admin_config(request: Request, user=Depends(get_admin_user)):
         "SHOW_ADMIN_DETAILS": request.app.state.config.SHOW_ADMIN_DETAILS,
         "WEBUI_URL": request.app.state.config.WEBUI_URL,
         "ENABLE_SIGNUP": request.app.state.config.ENABLE_SIGNUP,
-        "ENABLE_API_KEY": request.app.state.config.ENABLE_API_KEY,
-        "ENABLE_API_KEY_ENDPOINT_RESTRICTIONS": request.app.state.config.ENABLE_API_KEY_ENDPOINT_RESTRICTIONS,
-        "API_KEY_ALLOWED_ENDPOINTS": request.app.state.config.API_KEY_ALLOWED_ENDPOINTS,
+        "ENABLE_API_KEYS": request.app.state.config.ENABLE_API_KEYS,
+        "ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS": request.app.state.config.ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS,
+        "API_KEYS_ALLOWED_ENDPOINTS": request.app.state.config.API_KEYS_ALLOWED_ENDPOINTS,
         "DEFAULT_USER_ROLE": request.app.state.config.DEFAULT_USER_ROLE,
+        "DEFAULT_GROUP_ID": request.app.state.config.DEFAULT_GROUP_ID,
         "JWT_EXPIRES_IN": request.app.state.config.JWT_EXPIRES_IN,
         "ENABLE_COMMUNITY_SHARING": request.app.state.config.ENABLE_COMMUNITY_SHARING,
         "ENABLE_MESSAGE_RATING": request.app.state.config.ENABLE_MESSAGE_RATING,
+        "ENABLE_FOLDERS": request.app.state.config.ENABLE_FOLDERS,
         "ENABLE_CHANNELS": request.app.state.config.ENABLE_CHANNELS,
         "ENABLE_NOTES": request.app.state.config.ENABLE_NOTES,
         "ENABLE_USER_WEBHOOKS": request.app.state.config.ENABLE_USER_WEBHOOKS,
@@ -846,13 +914,15 @@ class AdminConfig(BaseModel):
     SHOW_ADMIN_DETAILS: bool
     WEBUI_URL: str
     ENABLE_SIGNUP: bool
-    ENABLE_API_KEY: bool
-    ENABLE_API_KEY_ENDPOINT_RESTRICTIONS: bool
-    API_KEY_ALLOWED_ENDPOINTS: str
+    ENABLE_API_KEYS: bool
+    ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS: bool
+    API_KEYS_ALLOWED_ENDPOINTS: str
     DEFAULT_USER_ROLE: str
+    DEFAULT_GROUP_ID: str
     JWT_EXPIRES_IN: str
     ENABLE_COMMUNITY_SHARING: bool
     ENABLE_MESSAGE_RATING: bool
+    ENABLE_FOLDERS: bool
     ENABLE_CHANNELS: bool
     ENABLE_NOTES: bool
     ENABLE_USER_WEBHOOKS: bool
@@ -869,20 +939,23 @@ async def update_admin_config(
     request.app.state.config.WEBUI_URL = form_data.WEBUI_URL
     request.app.state.config.ENABLE_SIGNUP = form_data.ENABLE_SIGNUP
 
-    request.app.state.config.ENABLE_API_KEY = form_data.ENABLE_API_KEY
-    request.app.state.config.ENABLE_API_KEY_ENDPOINT_RESTRICTIONS = (
-        form_data.ENABLE_API_KEY_ENDPOINT_RESTRICTIONS
+    request.app.state.config.ENABLE_API_KEYS = form_data.ENABLE_API_KEYS
+    request.app.state.config.ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS = (
+        form_data.ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS
     )
-    request.app.state.config.API_KEY_ALLOWED_ENDPOINTS = (
-        form_data.API_KEY_ALLOWED_ENDPOINTS
+    request.app.state.config.API_KEYS_ALLOWED_ENDPOINTS = (
+        form_data.API_KEYS_ALLOWED_ENDPOINTS
     )
 
+    request.app.state.config.ENABLE_FOLDERS = form_data.ENABLE_FOLDERS
     request.app.state.config.ENABLE_CHANNELS = form_data.ENABLE_CHANNELS
     request.app.state.config.ENABLE_NOTES = form_data.ENABLE_NOTES
 
     if form_data.DEFAULT_USER_ROLE in ["pending", "user", "admin"]:
         request.app.state.config.DEFAULT_USER_ROLE = form_data.DEFAULT_USER_ROLE
 
+    request.app.state.config.DEFAULT_GROUP_ID = form_data.DEFAULT_GROUP_ID
+
     pattern = r"^(-1|0|(-?\d+(\.\d+)?)(ms|s|m|h|d|w))$"
 
     # Check if the input string matches the pattern
@@ -909,13 +982,15 @@ async def update_admin_config(
         "SHOW_ADMIN_DETAILS": request.app.state.config.SHOW_ADMIN_DETAILS,
         "WEBUI_URL": request.app.state.config.WEBUI_URL,
         "ENABLE_SIGNUP": request.app.state.config.ENABLE_SIGNUP,
-        "ENABLE_API_KEY": request.app.state.config.ENABLE_API_KEY,
-        "ENABLE_API_KEY_ENDPOINT_RESTRICTIONS": request.app.state.config.ENABLE_API_KEY_ENDPOINT_RESTRICTIONS,
-        "API_KEY_ALLOWED_ENDPOINTS": request.app.state.config.API_KEY_ALLOWED_ENDPOINTS,
+        "ENABLE_API_KEYS": request.app.state.config.ENABLE_API_KEYS,
+        "ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS": request.app.state.config.ENABLE_API_KEYS_ENDPOINT_RESTRICTIONS,
+        "API_KEYS_ALLOWED_ENDPOINTS": request.app.state.config.API_KEYS_ALLOWED_ENDPOINTS,
         "DEFAULT_USER_ROLE": request.app.state.config.DEFAULT_USER_ROLE,
+        "DEFAULT_GROUP_ID": request.app.state.config.DEFAULT_GROUP_ID,
         "JWT_EXPIRES_IN": request.app.state.config.JWT_EXPIRES_IN,
         "ENABLE_COMMUNITY_SHARING": request.app.state.config.ENABLE_COMMUNITY_SHARING,
         "ENABLE_MESSAGE_RATING": request.app.state.config.ENABLE_MESSAGE_RATING,
+        "ENABLE_FOLDERS": request.app.state.config.ENABLE_FOLDERS,
         "ENABLE_CHANNELS": request.app.state.config.ENABLE_CHANNELS,
         "ENABLE_NOTES": request.app.state.config.ENABLE_NOTES,
         "ENABLE_USER_WEBHOOKS": request.app.state.config.ENABLE_USER_WEBHOOKS,
@@ -1036,9 +1111,11 @@ async def update_ldap_config(
 # create api key
 @router.post("/api_key", response_model=ApiKey)
 async def generate_api_key(request: Request, user=Depends(get_current_user)):
-    if not request.app.state.config.ENABLE_API_KEY:
+    if not request.app.state.config.ENABLE_API_KEYS or not has_permission(
+        user.id, "features.api_keys", request.app.state.config.USER_PERMISSIONS
+    ):
         raise HTTPException(
-            status.HTTP_403_FORBIDDEN,
+            status_code=status.HTTP_403_FORBIDDEN,
             detail=ERROR_MESSAGES.API_KEY_CREATION_NOT_ALLOWED,
         )
 
@@ -1056,8 +1133,7 @@ async def generate_api_key(request: Request, user=Depends(get_current_user)):
 # delete api key
 @router.delete("/api_key", response_model=bool)
 async def delete_api_key(user=Depends(get_current_user)):
-    success = Users.update_user_api_key_by_id(user.id, None)
-    return success
+    return Users.delete_user_api_key_by_id(user.id)
 
 
 # get api key

backend/open_webui/routers/channels.py

@@ -7,8 +7,18 @@ from fastapi import APIRouter, Depends, HTTPException, Request, status, Backgrou
 from pydantic import BaseModel
 
 
-from open_webui.socket.main import sio, get_user_ids_from_room
-from open_webui.models.users import Users, UserNameResponse
+from open_webui.socket.main import (
+    sio,
+    get_user_ids_from_room,
+)
+from open_webui.models.users import (
+    UserIdNameResponse,
+    UserIdNameStatusResponse,
+    UserListResponse,
+    UserModelResponse,
+    Users,
+    UserNameResponse,
+)
 
 from open_webui.models.groups import Groups
 from open_webui.models.channels import (
@@ -21,6 +31,7 @@ from open_webui.models.messages import (
     Messages,
     MessageModel,
     MessageResponse,
+    MessageWithReactionsResponse,
     MessageForm,
 )
 
@@ -38,7 +49,11 @@ from open_webui.utils.chat import generate_chat_completion
 
 
 from open_webui.utils.auth import get_admin_user, get_verified_user
-from open_webui.utils.access_control import has_access, get_users_with_access
+from open_webui.utils.access_control import (
+    has_access,
+    get_users_with_access,
+    get_permitted_group_and_user_ids,
+)
 from open_webui.utils.webhook import post_webhook
 from open_webui.utils.channels import extract_mentions, replace_mentions
 
@@ -52,9 +67,54 @@ router = APIRouter()
 ############################
 
 
-@router.get("/", response_model=list[ChannelModel])
+class ChannelListItemResponse(ChannelModel):
+    user_ids: Optional[list[str]] = None  # 'dm' channels only
+    users: Optional[list[UserIdNameStatusResponse]] = None  # 'dm' channels only
+
+    last_message_at: Optional[int] = None  # timestamp in epoch (time_ns)
+    unread_count: int = 0
+
+
+@router.get("/", response_model=list[ChannelListItemResponse])
 async def get_channels(user=Depends(get_verified_user)):
-    return Channels.get_channels_by_user_id(user.id)
+
+    channels = Channels.get_channels_by_user_id(user.id)
+
+    channel_list = []
+    for channel in channels:
+        last_message = Messages.get_last_message_by_channel_id(channel.id)
+        last_message_at = last_message.created_at if last_message else None
+
+        channel_member = Channels.get_member_by_channel_and_user_id(channel.id, user.id)
+        unread_count = Messages.get_unread_message_count(
+            channel.id, user.id, channel_member.last_read_at if channel_member else None
+        )
+
+        user_ids = None
+        users = None
+        if channel.type == "dm":
+            user_ids = [
+                member.user_id
+                for member in Channels.get_members_by_channel_id(channel.id)
+            ]
+            users = [
+                UserIdNameStatusResponse(
+                    **{**user.model_dump(), "is_active": Users.is_user_active(user.id)}
+                )
+                for user in Users.get_users_by_user_ids(user_ids)
+            ]
+
+        channel_list.append(
+            ChannelListItemResponse(
+                **channel.model_dump(),
+                user_ids=user_ids,
+                users=users,
+                last_message_at=last_message_at,
+                unread_count=unread_count,
+            )
+        )
+
+    return channel_list
 
 
 @router.get("/list", response_model=list[ChannelModel])
@@ -72,7 +132,15 @@ async def get_all_channels(user=Depends(get_verified_user)):
 @router.post("/create", response_model=Optional[ChannelModel])
 async def create_new_channel(form_data: ChannelForm, user=Depends(get_admin_user)):
     try:
-        channel = Channels.insert_new_channel(None, form_data, user.id)
+        if form_data.type == "dm":
+            existing_channel = Channels.get_dm_channel_by_user_ids(
+                [user.id, *form_data.user_ids]
+            )
+            if existing_channel:
+                Channels.update_member_active_status(existing_channel.id, user.id, True)
+                return ChannelModel(**existing_channel.model_dump())
+
+        channel = Channels.insert_new_channel(form_data, user.id)
         return ChannelModel(**channel.model_dump())
     except Exception as e:
         log.exception(e)
@@ -86,7 +154,15 @@ async def create_new_channel(form_data: ChannelForm, user=Depends(get_admin_user
 ############################
 
 
-@router.get("/{id}", response_model=Optional[ChannelResponse])
+class ChannelFullResponse(ChannelResponse):
+    user_ids: Optional[list[str]] = None  # 'dm' channels only
+    users: Optional[list[UserIdNameResponse]] = None  # 'dm' channels only
+
+    last_read_at: Optional[int] = None  # timestamp in epoch (time_ns)
+    unread_count: int = 0
+
+
+@router.get("/{id}", response_model=Optional[ChannelFullResponse])
 async def get_channel_by_id(id: str, user=Depends(get_verified_user)):
     channel = Channels.get_channel_by_id(id)
     if not channel:
@@ -94,6 +170,41 @@ async def get_channel_by_id(id: str, user=Depends(get_verified_user)):
             status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
         )
 
+    user_ids = None
+    users = None
+
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+
+        user_ids = [
+            member.user_id for member in Channels.get_members_by_channel_id(channel.id)
+        ]
+        users = [
+            UserIdNameResponse(**user.model_dump())
+            for user in Users.get_users_by_user_ids(user_ids)
+        ]
+
+        channel_member = Channels.get_member_by_channel_and_user_id(channel.id, user.id)
+        unread_count = Messages.get_unread_message_count(
+            channel.id, user.id, channel_member.last_read_at if channel_member else None
+        )
+
+        return ChannelFullResponse(
+            **{
+                **channel.model_dump(),
+                "user_ids": user_ids,
+                "users": users,
+                "write_access": True,
+                "user_count": len(user_ids),
+                "last_read_at": channel_member.last_read_at if channel_member else None,
+                "unread_count": unread_count,
+            }
+        )
+
+    else:
         if user.role != "admin" and not has_access(
             user.id, type="read", access_control=channel.access_control
         ):
@@ -105,14 +216,141 @@ async def get_channel_by_id(id: str, user=Depends(get_verified_user)):
             user.id, type="write", access_control=channel.access_control, strict=False
         )
 
-    return ChannelResponse(
+        user_count = len(get_users_with_access("read", channel.access_control))
+
+        channel_member = Channels.get_member_by_channel_and_user_id(channel.id, user.id)
+        unread_count = Messages.get_unread_message_count(
+            channel.id, user.id, channel_member.last_read_at if channel_member else None
+        )
+
+        return ChannelFullResponse(
             **{
                 **channel.model_dump(),
+                "user_ids": user_ids,
+                "users": users,
                 "write_access": write_access or user.role == "admin",
+                "user_count": user_count,
+                "last_read_at": channel_member.last_read_at if channel_member else None,
+                "unread_count": unread_count,
             }
         )
 
 
+############################
+# GetChannelMembersById
+############################
+
+
+PAGE_ITEM_COUNT = 30
+
+
+@router.get("/{id}/members", response_model=UserListResponse)
+async def get_channel_members_by_id(
+    id: str,
+    query: Optional[str] = None,
+    order_by: Optional[str] = None,
+    direction: Optional[str] = None,
+    page: Optional[int] = 1,
+    user=Depends(get_verified_user),
+):
+
+    channel = Channels.get_channel_by_id(id)
+    if not channel:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
+        )
+
+    limit = PAGE_ITEM_COUNT
+
+    page = max(1, page)
+    skip = (page - 1) * limit
+
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+
+        user_ids = [
+            member.user_id for member in Channels.get_members_by_channel_id(channel.id)
+        ]
+        users = Users.get_users_by_user_ids(user_ids)
+
+        total = len(users)
+
+        return {
+            "users": [
+                UserModelResponse(
+                    **user.model_dump(), is_active=Users.is_user_active(user.id)
+                )
+                for user in users
+            ],
+            "total": total,
+        }
+
+    else:
+        filter = {
+            "roles": ["!pending"],
+        }
+
+        if query:
+            filter["query"] = query
+        if order_by:
+            filter["order_by"] = order_by
+        if direction:
+            filter["direction"] = direction
+
+        permitted_ids = get_permitted_group_and_user_ids("read", channel.access_control)
+        if permitted_ids:
+            filter["user_ids"] = permitted_ids.get("user_ids")
+            filter["group_ids"] = permitted_ids.get("group_ids")
+
+        result = Users.get_users(filter=filter, skip=skip, limit=limit)
+
+        users = result["users"]
+        total = result["total"]
+
+        return {
+            "users": [
+                UserModelResponse(
+                    **user.model_dump(), is_active=Users.is_user_active(user.id)
+                )
+                for user in users
+            ],
+            "total": total,
+        }
+
+
+#################################################
+# UpdateIsActiveMemberByIdAndUserId
+#################################################
+
+
+class UpdateActiveMemberForm(BaseModel):
+    is_active: bool
+
+
+@router.post("/{id}/members/active", response_model=bool)
+async def update_is_active_member_by_id_and_user_id(
+    id: str,
+    form_data: UpdateActiveMemberForm,
+    user=Depends(get_verified_user),
+):
+    channel = Channels.get_channel_by_id(id)
+    if not channel:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
+        )
+
+    if not Channels.is_user_channel_member(channel.id, user.id):
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
+        )
+
+    Channels.update_member_active_status(channel.id, user.id, form_data.is_active)
+    return True
+
+
 ############################
 # UpdateChannelById
 ############################
@@ -180,6 +418,12 @@ async def get_channel_messages(
             status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
         )
 
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
         if user.role != "admin" and not has_access(
             user.id, type="read", access_control=channel.access_control
         ):
@@ -187,6 +431,10 @@ async def get_channel_messages(
                 status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
             )
 
+        channel_member = Channels.join_channel(
+            id, user.id
+        )  # Ensure user is a member of the channel
+
     message_list = Messages.get_messages_by_channel_id(id, skip, limit)
     users = {}
 
@@ -216,6 +464,62 @@ async def get_channel_messages(
     return messages
 
 
+############################
+# GetPinnedChannelMessages
+############################
+
+PAGE_ITEM_COUNT_PINNED = 20
+
+
+@router.get("/{id}/messages/pinned", response_model=list[MessageWithReactionsResponse])
+async def get_pinned_channel_messages(
+    id: str, page: int = 1, user=Depends(get_verified_user)
+):
+    channel = Channels.get_channel_by_id(id)
+    if not channel:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
+        )
+
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
+        if user.role != "admin" and not has_access(
+            user.id, type="read", access_control=channel.access_control
+        ):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+
+    page = max(1, page)
+    skip = (page - 1) * PAGE_ITEM_COUNT_PINNED
+    limit = PAGE_ITEM_COUNT_PINNED
+
+    message_list = Messages.get_pinned_messages_by_channel_id(id, skip, limit)
+    users = {}
+
+    messages = []
+    for message in message_list:
+        if message.user_id not in users:
+            user = Users.get_user_by_id(message.user_id)
+            users[message.user_id] = user
+
+        messages.append(
+            MessageWithReactionsResponse(
+                **{
+                    **message.model_dump(),
+                    "reactions": Messages.get_reactions_by_message_id(message.id),
+                    "user": UserNameResponse(**users[message.user_id].model_dump()),
+                }
+            )
+        )
+
+    return messages
+
+
 ############################
 # PostNewMessage
 ############################
@@ -225,7 +529,9 @@ async def send_notification(name, webui_url, channel, message, active_user_ids):
     users = get_users_with_access("read", channel.access_control)
 
     for user in users:
-        if user.id not in active_user_ids:
+        if (user.id not in active_user_ids) and Channels.is_user_channel_member(
+            channel.id, user.id
+        ):
             if user.settings:
                 webhook_url = user.settings.ui.get("notifications", {}).get(
                     "webhook_url", None
@@ -340,11 +646,12 @@ async def model_response_handler(request, channel, message, user):
                         if file.get("type", "") == "image":
                             images.append(file.get("url", ""))
 
+                thread_history_string = "\n\n".join(thread_history)
                 system_message = {
                     "role": "system",
                     "content": f"You are {model.get('name', model_id)}, participating in a threaded conversation. Be concise and conversational."
                     + (
-                        f"Here's the thread history:\n\n{''.join([f'{msg}' for msg in thread_history])}\n\nContinue the conversation naturally as {model.get('name', model_id)}, addressing the most recent message while being aware of the full context."
+                        f"Here's the thread history:\n\n\n{thread_history_string}\n\n\nContinue the conversation naturally as {model.get('name', model_id)}, addressing the most recent message while being aware of the full context."
                         if thread_history
                         else ""
                     ),
@@ -384,6 +691,7 @@ async def model_response_handler(request, channel, message, user):
                 )
 
                 if res:
+                    if res.get("choices", []) and len(res["choices"]) > 0:
                         await update_message_by_id(
                             channel.id,
                             response_message.id,
@@ -397,6 +705,20 @@ async def model_response_handler(request, channel, message, user):
                             ),
                             user,
                         )
+                    elif res.get("error", None):
+                        await update_message_by_id(
+                            channel.id,
+                            response_message.id,
+                            MessageForm(
+                                **{
+                                    "content": f"Error: {res['error']}",
+                                    "meta": {
+                                        "done": True,
+                                    },
+                                }
+                            ),
+                            user,
+                        )
             except Exception as e:
                 log.info(e)
                 pass
@@ -413,6 +735,12 @@ async def new_message_handler(
             status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
         )
 
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
         if user.role != "admin" and not has_access(
             user.id, type="write", access_control=channel.access_control, strict=False
         ):
@@ -423,13 +751,21 @@ async def new_message_handler(
     try:
         message = Messages.insert_new_message(form_data, channel.id, user.id)
         if message:
+            if channel.type == "dm":
+                members = Channels.get_members_by_channel_id(channel.id)
+                for member in members:
+                    if not member.is_active:
+                        Channels.update_member_active_status(
+                            channel.id, member.user_id, True
+                        )
+
             message = Messages.get_message_by_id(message.id)
             event_data = {
                 "channel_id": channel.id,
                 "message_id": message.id,
                 "data": {
                     "type": "message",
-                    "data": message.model_dump(),
+                    "data": {"temp_id": form_data.temp_id, **message.model_dump()},
                 },
                 "user": UserNameResponse(**user.model_dump()).model_dump(),
                 "channel": channel.model_dump(),
@@ -521,6 +857,12 @@ async def get_channel_message(
             status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
         )
 
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
         if user.role != "admin" and not has_access(
             user.id, type="read", access_control=channel.access_control
         ):
@@ -549,6 +891,69 @@ async def get_channel_message(
     )
 
 
+############################
+# PinChannelMessage
+############################
+
+
+class PinMessageForm(BaseModel):
+    is_pinned: bool
+
+
+@router.post(
+    "/{id}/messages/{message_id}/pin", response_model=Optional[MessageUserResponse]
+)
+async def pin_channel_message(
+    id: str, message_id: str, form_data: PinMessageForm, user=Depends(get_verified_user)
+):
+    channel = Channels.get_channel_by_id(id)
+    if not channel:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
+        )
+
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
+        if user.role != "admin" and not has_access(
+            user.id, type="read", access_control=channel.access_control
+        ):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+
+    message = Messages.get_message_by_id(message_id)
+    if not message:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
+        )
+
+    if message.channel_id != id:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT()
+        )
+
+    try:
+        Messages.update_is_pinned_by_id(message_id, form_data.is_pinned, user.id)
+        message = Messages.get_message_by_id(message_id)
+        return MessageUserResponse(
+            **{
+                **message.model_dump(),
+                "user": UserNameResponse(
+                    **Users.get_user_by_id(message.user_id).model_dump()
+                ),
+            }
+        )
+    except Exception as e:
+        log.exception(e)
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT()
+        )
+
+
 ############################
 # GetChannelThreadMessages
 ############################
@@ -570,6 +975,12 @@ async def get_channel_thread_messages(
             status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
         )
 
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
         if user.role != "admin" and not has_access(
             user.id, type="read", access_control=channel.access_control
         ):
@@ -629,10 +1040,18 @@ async def update_message_by_id(
             status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT()
         )
 
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
         if (
             user.role != "admin"
             and message.user_id != user.id
-        and not has_access(user.id, type="read", access_control=channel.access_control)
+            and not has_access(
+                user.id, type="read", access_control=channel.access_control
+            )
         ):
             raise HTTPException(
                 status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
@@ -685,6 +1104,12 @@ async def add_reaction_to_message(
             status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
         )
 
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
         if user.role != "admin" and not has_access(
             user.id, type="write", access_control=channel.access_control, strict=False
         ):
@@ -748,6 +1173,12 @@ async def remove_reaction_by_id_and_user_id_and_name(
             status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
         )
 
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
         if user.role != "admin" and not has_access(
             user.id, type="write", access_control=channel.access_control, strict=False
         ):
@@ -825,11 +1256,20 @@ async def delete_message_by_id(
             status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT()
         )
 
+    if channel.type == "dm":
+        if not Channels.is_user_channel_member(channel.id, user.id):
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()
+            )
+    else:
         if (
             user.role != "admin"
             and message.user_id != user.id
             and not has_access(
-            user.id, type="write", access_control=channel.access_control, strict=False
+                user.id,
+                type="write",
+                access_control=channel.access_control,
+                strict=False,
             )
         ):
             raise HTTPException(

backend/open_webui/routers/chats.py

@@ -7,6 +7,7 @@ from open_webui.socket.main import get_event_emitter
 from open_webui.models.chats import (
     ChatForm,
     ChatImportForm,
+    ChatsImportForm,
     ChatResponse,
     Chats,
     ChatTitleIdResponse,
@@ -39,6 +40,7 @@ router = APIRouter()
 def get_session_user_chat_list(
     user=Depends(get_verified_user),
     page: Optional[int] = None,
+    include_pinned: Optional[bool] = False,
     include_folders: Optional[bool] = False,
 ):
     try:
@@ -47,11 +49,15 @@ def get_session_user_chat_list(
             skip = (page - 1) * limit
 
             return Chats.get_chat_title_id_list_by_user_id(
-                user.id, include_folders=include_folders, skip=skip, limit=limit
+                user.id,
+                include_folders=include_folders,
+                include_pinned=include_pinned,
+                skip=skip,
+                limit=limit,
             )
         else:
             return Chats.get_chat_title_id_list_by_user_id(
-                user.id, include_folders=include_folders
+                user.id, include_folders=include_folders, include_pinned=include_pinned
             )
     except Exception as e:
         log.exception(e)
@@ -137,26 +143,15 @@ async def create_new_chat(form_data: ChatForm, user=Depends(get_verified_user)):
 
 
 ############################
-# ImportChat
+# ImportChats
 ############################
 
 
-@router.post("/import", response_model=Optional[ChatResponse])
-async def import_chat(form_data: ChatImportForm, user=Depends(get_verified_user)):
+@router.post("/import", response_model=list[ChatResponse])
+async def import_chats(form_data: ChatsImportForm, user=Depends(get_verified_user)):
     try:
-        chat = Chats.import_chat(user.id, form_data)
-        if chat:
-            tags = chat.meta.get("tags", [])
-            for tag_id in tags:
-                tag_id = tag_id.replace(" ", "_").lower()
-                tag_name = " ".join([word.capitalize() for word in tag_id.split("_")])
-                if (
-                    tag_id != "none"
-                    and Tags.get_tag_by_name_and_user_id(tag_name, user.id) is None
-                ):
-                    Tags.insert_new_tag(tag_name, user.id)
-
-        return ChatResponse(**chat.model_dump())
+        chats = Chats.import_chats(user.id, form_data.chats)
+        return chats
     except Exception as e:
         log.exception(e)
         raise HTTPException(
@@ -223,7 +218,7 @@ async def get_chat_list_by_folder_id(
     folder_id: str, page: Optional[int] = 1, user=Depends(get_verified_user)
 ):
     try:
-        limit = 60
+        limit = 10
         skip = (page - 1) * limit
 
         return [
@@ -653,8 +648,9 @@ async def clone_chat_by_id(
             "title": form_data.title if form_data.title else f"Clone of {chat.title}",
         }
 
-        chat = Chats.import_chat(
+        chats = Chats.import_chats(
             user.id,
+            [
                 ChatImportForm(
                     **{
                         "chat": updated_chat,
@@ -662,10 +658,18 @@ async def clone_chat_by_id(
                         "pinned": chat.pinned,
                         "folder_id": chat.folder_id,
                     }
-            ),
+                )
+            ],
         )
 
+        if chats:
+            chat = chats[0]
             return ChatResponse(**chat.model_dump())
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=ERROR_MESSAGES.DEFAULT(),
+            )
     else:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED, detail=ERROR_MESSAGES.DEFAULT()
@@ -693,8 +697,9 @@ async def clone_shared_chat_by_id(id: str, user=Depends(get_verified_user)):
             "title": f"Clone of {chat.title}",
         }
 
-        chat = Chats.import_chat(
+        chats = Chats.import_chats(
             user.id,
+            [
                 ChatImportForm(
                     **{
                         "chat": updated_chat,
@@ -702,9 +707,18 @@ async def clone_shared_chat_by_id(id: str, user=Depends(get_verified_user)):
                         "pinned": chat.pinned,
                         "folder_id": chat.folder_id,
                     }
-            ),
                 )
+            ],
+        )
+
+        if chats:
+            chat = chats[0]
             return ChatResponse(**chat.model_dump())
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=ERROR_MESSAGES.DEFAULT(),
+            )
     else:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED, detail=ERROR_MESSAGES.DEFAULT()

backend/open_webui/routers/configs.py

@@ -1,4 +1,5 @@
 import logging
+import copy
 from fastapi import APIRouter, Depends, Request, HTTPException
 from pydantic import BaseModel, ConfigDict
 import aiohttp
@@ -15,6 +16,7 @@ from open_webui.utils.tools import (
     set_tool_servers,
 )
 from open_webui.utils.mcp.client import MCPClient
+from open_webui.models.oauth_sessions import OAuthSessions
 
 from open_webui.env import SRC_LOG_LEVELS
 
@@ -142,6 +144,7 @@ class ToolServerConnection(BaseModel):
     path: str
     type: Optional[str] = "openapi"  # openapi, mcp
     auth_type: Optional[str]
+    headers: Optional[dict | str] = None
     key: Optional[str]
     config: Optional[dict]
 
@@ -165,6 +168,21 @@ async def set_tool_servers_config(
     form_data: ToolServersConfigForm,
     user=Depends(get_admin_user),
 ):
+    for connection in request.app.state.config.TOOL_SERVER_CONNECTIONS:
+        server_type = connection.get("type", "openapi")
+        auth_type = connection.get("auth_type", "none")
+
+        if auth_type == "oauth_2.1":
+            # Remove existing OAuth clients for tool servers
+            server_id = connection.get("info", {}).get("id")
+            client_key = f"{server_type}:{server_id}"
+
+            try:
+                request.app.state.oauth_client_manager.remove_client(client_key)
+            except:
+                pass
+
+    # Set new tool server connections
     request.app.state.config.TOOL_SERVER_CONNECTIONS = [
         connection.model_dump() for connection in form_data.TOOL_SERVER_CONNECTIONS
     ]
@@ -176,6 +194,7 @@ async def set_tool_servers_config(
         if server_type == "mcp":
             server_id = connection.get("info", {}).get("id")
             auth_type = connection.get("auth_type", "none")
+
             if auth_type == "oauth_2.1" and server_id:
                 try:
                     oauth_client_info = connection.get("info", {}).get(
@@ -183,7 +202,7 @@ async def set_tool_servers_config(
                     )
                     oauth_client_info = decrypt_data(oauth_client_info)
 
-                    await request.app.state.oauth_client_manager.add_client(
+                    request.app.state.oauth_client_manager.add_client(
                         f"{server_type}:{server_id}",
                         OAuthClientInformationFull(**oauth_client_info),
                     )
@@ -211,7 +230,7 @@ async def verify_tool_servers_config(
                     log.debug(
                         f"Trying to fetch OAuth 2.1 discovery document from {discovery_url}"
                     )
-                    async with aiohttp.ClientSession() as session:
+                    async with aiohttp.ClientSession(trust_env=True) as session:
                         async with session.get(
                             discovery_url
                         ) as oauth_server_metadata_response:
@@ -234,7 +253,7 @@ async def verify_tool_servers_config(
                                     )
                                     raise HTTPException(
                                         status_code=400,
-                                        detail=f"Failed to parse OAuth 2.1 discovery document from {discovery_urls[0]}",
+                                        detail=f"Failed to parse OAuth 2.1 discovery document from {discovery_url}",
                                     )
 
                 raise HTTPException(
@@ -252,18 +271,26 @@ async def verify_tool_servers_config(
                     elif form_data.auth_type == "session":
                         token = request.state.token.credentials
                     elif form_data.auth_type == "system_oauth":
+                        oauth_token = None
                         try:
                             if request.cookies.get("oauth_session_id", None):
-                                token = await request.app.state.oauth_manager.get_oauth_token(
+                                oauth_token = await request.app.state.oauth_manager.get_oauth_token(
                                     user.id,
                                     request.cookies.get("oauth_session_id", None),
                                 )
+
+                                if oauth_token:
+                                    token = oauth_token.get("access_token", "")
                         except Exception as e:
                             pass
-
                     if token:
                         headers = {"Authorization": f"Bearer {token}"}
 
+                    if form_data.headers and isinstance(form_data.headers, dict):
+                        if headers is None:
+                            headers = {}
+                        headers.update(form_data.headers)
+
                     await client.connect(form_data.url, headers=headers)
                     specs = await client.list_tool_specs()
                     return {
@@ -281,6 +308,7 @@ async def verify_tool_servers_config(
                         await client.disconnect()
         else:  # openapi
             token = None
+            headers = None
             if form_data.auth_type == "bearer":
                 token = form_data.key
             elif form_data.auth_type == "session":
@@ -288,15 +316,29 @@ async def verify_tool_servers_config(
             elif form_data.auth_type == "system_oauth":
                 try:
                     if request.cookies.get("oauth_session_id", None):
-                        token = await request.app.state.oauth_manager.get_oauth_token(
+                        oauth_token = (
+                            await request.app.state.oauth_manager.get_oauth_token(
                                 user.id,
                                 request.cookies.get("oauth_session_id", None),
                             )
+                        )
+
+                        if oauth_token:
+                            token = oauth_token.get("access_token", "")
+
                 except Exception as e:
                     pass
 
+            if token:
+                headers = {"Authorization": f"Bearer {token}"}
+
+            if form_data.headers and isinstance(form_data.headers, dict):
+                if headers is None:
+                    headers = {}
+                headers.update(form_data.headers)
+
             url = get_tool_server_url(form_data.url, form_data.path)
-            return await get_tool_server_data(token, url)
+            return await get_tool_server_data(url, headers=headers)
     except HTTPException as e:
         raise e
     except Exception as e:
@@ -421,6 +463,7 @@ async def set_code_execution_config(
 ############################
 class ModelsConfigForm(BaseModel):
     DEFAULT_MODELS: Optional[str]
+    DEFAULT_PINNED_MODELS: Optional[str]
     MODEL_ORDER_LIST: Optional[list[str]]
 
 
@@ -428,6 +471,7 @@ class ModelsConfigForm(BaseModel):
 async def get_models_config(request: Request, user=Depends(get_admin_user)):
     return {
         "DEFAULT_MODELS": request.app.state.config.DEFAULT_MODELS,
+        "DEFAULT_PINNED_MODELS": request.app.state.config.DEFAULT_PINNED_MODELS,
         "MODEL_ORDER_LIST": request.app.state.config.MODEL_ORDER_LIST,
     }
 
@@ -437,9 +481,11 @@ async def set_models_config(
     request: Request, form_data: ModelsConfigForm, user=Depends(get_admin_user)
 ):
     request.app.state.config.DEFAULT_MODELS = form_data.DEFAULT_MODELS
+    request.app.state.config.DEFAULT_PINNED_MODELS = form_data.DEFAULT_PINNED_MODELS
     request.app.state.config.MODEL_ORDER_LIST = form_data.MODEL_ORDER_LIST
     return {
         "DEFAULT_MODELS": request.app.state.config.DEFAULT_MODELS,
+        "DEFAULT_PINNED_MODELS": request.app.state.config.DEFAULT_PINNED_MODELS,
         "MODEL_ORDER_LIST": request.app.state.config.MODEL_ORDER_LIST,
     }
 

backend/open_webui/routers/evaluations.py

@@ -7,6 +7,8 @@ from open_webui.models.feedbacks import (
     FeedbackModel,
     FeedbackResponse,
     FeedbackForm,
+    FeedbackUserResponse,
+    FeedbackListResponse,
     Feedbacks,
 )
 
@@ -56,35 +58,10 @@ async def update_config(
     }
 
 
-class UserResponse(BaseModel):
-    id: str
-    name: str
-    email: str
-    role: str = "pending"
-
-    last_active_at: int  # timestamp in epoch
-    updated_at: int  # timestamp in epoch
-    created_at: int  # timestamp in epoch
-
-
-class FeedbackUserResponse(FeedbackResponse):
-    user: Optional[UserResponse] = None
-
-
-@router.get("/feedbacks/all", response_model=list[FeedbackUserResponse])
+@router.get("/feedbacks/all", response_model=list[FeedbackResponse])
 async def get_all_feedbacks(user=Depends(get_admin_user)):
     feedbacks = Feedbacks.get_all_feedbacks()
-
-    feedback_list = []
-    for feedback in feedbacks:
-        user = Users.get_user_by_id(feedback.user_id)
-        feedback_list.append(
-            FeedbackUserResponse(
-                **feedback.model_dump(),
-                user=UserResponse(**user.model_dump()) if user else None,
-            )
-        )
-    return feedback_list
+    return feedbacks
 
 
 @router.delete("/feedbacks/all")
@@ -111,6 +88,31 @@ async def delete_feedbacks(user=Depends(get_verified_user)):
     return success
 
 
+PAGE_ITEM_COUNT = 30
+
+
+@router.get("/feedbacks/list", response_model=FeedbackListResponse)
+async def get_feedbacks(
+    order_by: Optional[str] = None,
+    direction: Optional[str] = None,
+    page: Optional[int] = 1,
+    user=Depends(get_admin_user),
+):
+    limit = PAGE_ITEM_COUNT
+
+    page = max(1, page)
+    skip = (page - 1) * limit
+
+    filter = {}
+    if order_by:
+        filter["order_by"] = order_by
+    if direction:
+        filter["direction"] = direction
+
+    result = Feedbacks.get_feedback_items(filter=filter, skip=skip, limit=limit)
+    return result
+
+
 @router.post("/feedback", response_model=FeedbackModel)
 async def create_feedback(
     request: Request,

backend/open_webui/routers/files.py

@@ -102,7 +102,7 @@ def process_uploaded_file(request, file, file_path, file_item, file_metadata, us
                 )
             ):
                 file_path = Storage.get_file(file_path)
-                result = transcribe(request, file_path, file_metadata)
+                result = transcribe(request, file_path, file_metadata, user)
 
                 process_file(
                     request,
@@ -115,6 +115,10 @@ def process_uploaded_file(request, file, file_path, file_item, file_metadata, us
                 request.app.state.config.CONTENT_EXTRACTION_ENGINE == "external"
             ):
                 process_file(request, ProcessFileForm(file_id=file_item.id), user=user)
+            else:
+                raise Exception(
+                    f"File type {file.content_type} is not supported for processing"
+                )
         else:
             log.info(
                 f"File type {file.content_type} is not provided, but trying to process anyway"

backend/open_webui/routers/folders.py

@@ -46,11 +46,35 @@ router = APIRouter()
 
 
 @router.get("/", response_model=list[FolderNameIdResponse])
-async def get_folders(user=Depends(get_verified_user)):
+async def get_folders(request: Request, user=Depends(get_verified_user)):
+    if request.app.state.config.ENABLE_FOLDERS is False:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
+        )
+
+    if user.role != "admin" and not has_permission(
+        user.id,
+        "features.folders",
+        request.app.state.config.USER_PERMISSIONS,
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
+        )
+
     folders = Folders.get_folders_by_user_id(user.id)
 
     # Verify folder data integrity
+    folder_list = []
     for folder in folders:
+        if folder.parent_id and not Folders.get_folder_by_id_and_user_id(
+            folder.parent_id, user.id
+        ):
+            folder = Folders.update_folder_parent_id_by_id_and_user_id(
+                folder.id, user.id, None
+            )
+
         if folder.data:
             if "files" in folder.data:
                 valid_files = []
@@ -74,12 +98,9 @@ async def get_folders(user=Depends(get_verified_user)):
                     folder.id, user.id, FolderUpdateForm(data=folder.data)
                 )
 
-    return [
-        {
-            **folder.model_dump(),
-        }
-        for folder in folders
-    ]
+        folder_list.append(FolderNameIdResponse(**folder.model_dump()))
+
+    return folder_list
 
 
 ############################
@@ -253,7 +274,10 @@ async def update_folder_is_expanded_by_id(
 
 @router.delete("/{id}")
 async def delete_folder_by_id(
-    request: Request, id: str, user=Depends(get_verified_user)
+    request: Request,
+    id: str,
+    delete_contents: Optional[bool] = True,
+    user=Depends(get_verified_user),
 ):
     if Chats.count_chats_by_folder_id_and_user_id(id, user.id):
         chat_delete_permission = has_permission(
@@ -265,12 +289,21 @@ async def delete_folder_by_id(
                 detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
             )
 
-    folder = Folders.get_folder_by_id_and_user_id(id, user.id)
+    folders = []
+    folders.append(Folders.get_folder_by_id_and_user_id(id, user.id))
+    while folders:
+        folder = folders.pop()
         if folder:
             try:
                 folder_ids = Folders.delete_folder_by_id_and_user_id(id, user.id)
+
                 for folder_id in folder_ids:
+                    if delete_contents:
                         Chats.delete_chats_by_user_id_and_folder_id(user.id, folder_id)
+                    else:
+                        Chats.move_chats_by_user_id_and_folder_id(
+                            user.id, folder_id, None
+                        )
 
                 return True
             except Exception as e:
@@ -280,6 +313,13 @@ async def delete_folder_by_id(
                     status_code=status.HTTP_400_BAD_REQUEST,
                     detail=ERROR_MESSAGES.DEFAULT("Error deleting folder"),
                 )
+            finally:
+                # Get all subfolders
+                subfolders = Folders.get_folders_by_parent_id_and_user_id(
+                    folder.id, user.id
+                )
+                folders.extend(subfolders)
+
     else:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,

backend/open_webui/routers/functions.py

@@ -10,6 +10,7 @@ from open_webui.models.functions import (
     FunctionForm,
     FunctionModel,
     FunctionResponse,
+    FunctionUserResponse,
     FunctionWithValvesModel,
     Functions,
 )
@@ -42,6 +43,11 @@ async def get_functions(user=Depends(get_verified_user)):
     return Functions.get_functions()
 
 
+@router.get("/list", response_model=list[FunctionUserResponse])
+async def get_function_list(user=Depends(get_admin_user)):
+    return Functions.get_function_list()
+
+
 ############################
 # ExportFunctions
 ############################

backend/open_webui/routers/groups.py

@@ -31,11 +31,18 @@ router = APIRouter()
 
 
 @router.get("/", response_model=list[GroupResponse])
-async def get_groups(user=Depends(get_verified_user)):
-    if user.role == "admin":
-        return Groups.get_groups()
-    else:
-        return Groups.get_groups_by_member_id(user.id)
+async def get_groups(share: Optional[bool] = None, user=Depends(get_verified_user)):
+
+    filter = {}
+    if user.role != "admin":
+        filter["member_id"] = user.id
+
+    if share is not None:
+        filter["share"] = share
+
+    groups = Groups.get_groups(filter=filter)
+
+    return groups
 
 
 ############################
@@ -48,7 +55,10 @@ async def create_new_group(form_data: GroupForm, user=Depends(get_admin_user)):
     try:
         group = Groups.insert_new_group(user.id, form_data)
         if group:
-            return group
+            return GroupResponse(
+                **group.model_dump(),
+                member_count=Groups.get_group_member_count_by_id(group.id),
+            )
         else:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
@@ -71,7 +81,36 @@ async def create_new_group(form_data: GroupForm, user=Depends(get_admin_user)):
 async def get_group_by_id(id: str, user=Depends(get_admin_user)):
     group = Groups.get_group_by_id(id)
     if group:
-        return group
+        return GroupResponse(
+            **group.model_dump(),
+            member_count=Groups.get_group_member_count_by_id(group.id),
+        )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# ExportGroupById
+############################
+
+
+class GroupExportResponse(GroupResponse):
+    user_ids: list[str] = []
+    pass
+
+
+@router.get("/id/{id}/export", response_model=Optional[GroupExportResponse])
+async def export_group_by_id(id: str, user=Depends(get_admin_user)):
+    group = Groups.get_group_by_id(id)
+    if group:
+        return GroupExportResponse(
+            **group.model_dump(),
+            member_count=Groups.get_group_member_count_by_id(group.id),
+            user_ids=Groups.get_group_user_ids_by_id(group.id),
+        )
     else:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
@@ -89,12 +128,12 @@ async def update_group_by_id(
     id: str, form_data: GroupUpdateForm, user=Depends(get_admin_user)
 ):
     try:
-        if form_data.user_ids:
-            form_data.user_ids = Users.get_valid_user_ids(form_data.user_ids)
-
         group = Groups.update_group_by_id(id, form_data)
         if group:
-            return group
+            return GroupResponse(
+                **group.model_dump(),
+                member_count=Groups.get_group_member_count_by_id(group.id),
+            )
         else:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
@@ -123,7 +162,10 @@ async def add_user_to_group(
 
         group = Groups.add_users_to_group(id, form_data.user_ids)
         if group:
-            return group
+            return GroupResponse(
+                **group.model_dump(),
+                member_count=Groups.get_group_member_count_by_id(group.id),
+            )
         else:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
@@ -144,7 +186,10 @@ async def remove_users_from_group(
     try:
         group = Groups.remove_users_from_group(id, form_data.user_ids)
         if group:
-            return group
+            return GroupResponse(
+                **group.model_dump(),
+                member_count=Groups.get_group_member_count_by_id(group.id),
+            )
         else:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,

backend/open_webui/routers/knowledge.py

@@ -1,6 +1,7 @@
 from typing import List, Optional
 from pydantic import BaseModel
 from fastapi import APIRouter, Depends, HTTPException, status, Request, Query
+from fastapi.concurrency import run_in_threadpool
 import logging
 
 from open_webui.models.knowledge import (
@@ -223,7 +224,8 @@ async def reindex_knowledge_files(request: Request, user=Depends(get_verified_us
             failed_files = []
             for file in files:
                 try:
-                    process_file(
+                    await run_in_threadpool(
+                        process_file,
                         request,
                         ProcessFileForm(
                             file_id=file.id, collection_name=knowledge_base.id
@@ -548,7 +550,11 @@ def remove_file_from_knowledge_by_id(
     try:
         VECTOR_DB_CLIENT.delete(
             collection_name=knowledge.id, filter={"file_id": form_data.file_id}
-        )
+        )  # Remove by file_id first
+
+        VECTOR_DB_CLIENT.delete(
+            collection_name=knowledge.id, filter={"hash": file.hash}
+        )  # Remove by hash as well in case of duplicates
     except Exception as e:
         log.debug("This was most likely caused by bypassing embedding processing")
         log.debug(e)
@@ -577,7 +583,6 @@ def remove_file_from_knowledge_by_id(
             data["file_ids"] = file_ids
 
             knowledge = Knowledges.update_knowledge_data_by_id(id=id, data=data)
-
             if knowledge:
                 files = Files.get_file_metadatas_by_ids(file_ids)
 
@@ -706,7 +711,7 @@ async def reset_knowledge_by_id(id: str, user=Depends(get_verified_user)):
 
 
 @router.post("/{id}/files/batch/add", response_model=Optional[KnowledgeFilesResponse])
-def add_files_to_knowledge_batch(
+async def add_files_to_knowledge_batch(
     request: Request,
     id: str,
     form_data: list[KnowledgeFileIdForm],
@@ -746,7 +751,7 @@ def add_files_to_knowledge_batch(
 
     # Process files
     try:
-        result = process_files_batch(
+        result = await process_files_batch(
             request=request,
             form_data=BatchProcessFilesForm(files=files, collection_name=id),
             user=user,

backend/open_webui/routers/memories.py

@@ -1,6 +1,7 @@
 from fastapi import APIRouter, Depends, HTTPException, Request
 from pydantic import BaseModel
 import logging
+import asyncio
 from typing import Optional
 
 from open_webui.models.memories import Memories, MemoryModel
@@ -17,7 +18,7 @@ router = APIRouter()
 
 @router.get("/ef")
 async def get_embeddings(request: Request):
-    return {"result": request.app.state.EMBEDDING_FUNCTION("hello world")}
+    return {"result": await request.app.state.EMBEDDING_FUNCTION("hello world")}
 
 
 ############################
@@ -51,15 +52,15 @@ async def add_memory(
 ):
     memory = Memories.insert_new_memory(user.id, form_data.content)
 
+    vector = await request.app.state.EMBEDDING_FUNCTION(memory.content, user=user)
+
     VECTOR_DB_CLIENT.upsert(
         collection_name=f"user-memory-{user.id}",
         items=[
             {
                 "id": memory.id,
                 "text": memory.content,
-                "vector": request.app.state.EMBEDDING_FUNCTION(
-                    memory.content, user=user
-                ),
+                "vector": vector,
                 "metadata": {"created_at": memory.created_at},
             }
         ],
@@ -86,9 +87,11 @@ async def query_memory(
     if not memories:
         raise HTTPException(status_code=404, detail="No memories found for user")
 
+    vector = await request.app.state.EMBEDDING_FUNCTION(form_data.content, user=user)
+
     results = VECTOR_DB_CLIENT.search(
         collection_name=f"user-memory-{user.id}",
-        vectors=[request.app.state.EMBEDDING_FUNCTION(form_data.content, user=user)],
+        vectors=[vector],
         limit=form_data.k,
     )
 
@@ -105,21 +108,28 @@ async def reset_memory_from_vector_db(
     VECTOR_DB_CLIENT.delete_collection(f"user-memory-{user.id}")
 
     memories = Memories.get_memories_by_user_id(user.id)
+
+    # Generate vectors in parallel
+    vectors = await asyncio.gather(
+        *[
+            request.app.state.EMBEDDING_FUNCTION(memory.content, user=user)
+            for memory in memories
+        ]
+    )
+
     VECTOR_DB_CLIENT.upsert(
         collection_name=f"user-memory-{user.id}",
         items=[
             {
                 "id": memory.id,
                 "text": memory.content,
-                "vector": request.app.state.EMBEDDING_FUNCTION(
-                    memory.content, user=user
-                ),
+                "vector": vectors[idx],
                 "metadata": {
                     "created_at": memory.created_at,
                     "updated_at": memory.updated_at,
                 },
             }
-            for memory in memories
+            for idx, memory in enumerate(memories)
         ],
     )
 
@@ -164,15 +174,15 @@ async def update_memory_by_id(
         raise HTTPException(status_code=404, detail="Memory not found")
 
     if form_data.content is not None:
+        vector = await request.app.state.EMBEDDING_FUNCTION(memory.content, user=user)
+
         VECTOR_DB_CLIENT.upsert(
             collection_name=f"user-memory-{user.id}",
             items=[
                 {
                     "id": memory.id,
                     "text": memory.content,
-                    "vector": request.app.state.EMBEDDING_FUNCTION(
-                        memory.content, user=user
-                    ),
+                    "vector": vector,
                     "metadata": {
                         "created_at": memory.created_at,
                         "updated_at": memory.updated_at,

backend/open_webui/routers/models.py

@@ -5,11 +5,12 @@ import json
 import asyncio
 import logging
 
+from open_webui.models.groups import Groups
 from open_webui.models.models import (
     ModelForm,
     ModelModel,
     ModelResponse,
-    ModelUserResponse,
+    ModelListResponse,
     Models,
 )
 
@@ -35,17 +36,56 @@ log = logging.getLogger(__name__)
 router = APIRouter()
 
 
+def is_valid_model_id(model_id: str) -> bool:
+    return model_id and len(model_id) <= 256
+
+
 ###########################
 # GetModels
 ###########################
 
 
-@router.get("/", response_model=list[ModelUserResponse])
-async def get_models(id: Optional[str] = None, user=Depends(get_verified_user)):
-    if user.role == "admin" and BYPASS_ADMIN_ACCESS_CONTROL:
-        return Models.get_models()
-    else:
-        return Models.get_models_by_user_id(user.id)
+PAGE_ITEM_COUNT = 30
+
+
+@router.get(
+    "/list", response_model=ModelListResponse
+)  # do NOT use "/" as path, conflicts with main.py
+async def get_models(
+    query: Optional[str] = None,
+    view_option: Optional[str] = None,
+    tag: Optional[str] = None,
+    order_by: Optional[str] = None,
+    direction: Optional[str] = None,
+    page: Optional[int] = 1,
+    user=Depends(get_verified_user),
+):
+
+    limit = PAGE_ITEM_COUNT
+
+    page = max(1, page)
+    skip = (page - 1) * limit
+
+    filter = {}
+    if query:
+        filter["query"] = query
+    if view_option:
+        filter["view_option"] = view_option
+    if tag:
+        filter["tag"] = tag
+    if order_by:
+        filter["order_by"] = order_by
+    if direction:
+        filter["direction"] = direction
+
+    if not user.role == "admin" or not BYPASS_ADMIN_ACCESS_CONTROL:
+        groups = Groups.get_groups_by_member_id(user.id)
+        if groups:
+            filter["group_ids"] = [group.id for group in groups]
+
+        filter["user_id"] = user.id
+
+    return Models.search_models(user.id, filter=filter, skip=skip, limit=limit)
 
 
 ###########################
@@ -58,6 +98,30 @@ async def get_base_models(user=Depends(get_admin_user)):
     return Models.get_base_models()
 
 
+###########################
+# GetModelTags
+###########################
+
+
+@router.get("/tags", response_model=list[str])
+async def get_model_tags(user=Depends(get_verified_user)):
+    if user.role == "admin" and BYPASS_ADMIN_ACCESS_CONTROL:
+        models = Models.get_models()
+    else:
+        models = Models.get_models_by_user_id(user.id)
+
+    tags_set = set()
+    for model in models:
+        if model.meta:
+            meta = model.meta.model_dump()
+            for tag in meta.get("tags", []):
+                tags_set.add((tag.get("name")))
+
+    tags = [tag for tag in tags_set]
+    tags.sort()
+    return tags
+
+
 ############################
 # CreateNewModel
 ############################
@@ -84,6 +148,12 @@ async def create_new_model(
             detail=ERROR_MESSAGES.MODEL_ID_TAKEN,
         )
 
+    if not is_valid_model_id(form_data.id):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.MODEL_ID_TOO_LONG,
+        )
+
     else:
         model = Models.insert_new_model(form_data, user.id)
         if model:
@@ -101,8 +171,19 @@ async def create_new_model(
 
 
 @router.get("/export", response_model=list[ModelModel])
-async def export_models(user=Depends(get_admin_user)):
+async def export_models(request: Request, user=Depends(get_verified_user)):
+    if user.role != "admin" and not has_permission(
+        user.id, "workspace.models_export", request.app.state.config.USER_PERMISSIONS
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=ERROR_MESSAGES.UNAUTHORIZED,
+        )
+
+    if user.role == "admin" and BYPASS_ADMIN_ACCESS_CONTROL:
         return Models.get_models()
+    else:
+        return Models.get_models_by_user_id(user.id)
 
 
 ############################
@@ -116,15 +197,25 @@ class ModelsImportForm(BaseModel):
 
 @router.post("/import", response_model=bool)
 async def import_models(
-    user: str = Depends(get_admin_user), form_data: ModelsImportForm = (...)
+    request: Request,
+    user=Depends(get_verified_user),
+    form_data: ModelsImportForm = (...),
 ):
+    if user.role != "admin" and not has_permission(
+        user.id, "workspace.models_import", request.app.state.config.USER_PERMISSIONS
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=ERROR_MESSAGES.UNAUTHORIZED,
+        )
     try:
         data = form_data.models
         if isinstance(data, list):
             for model_data in data:
                 # Here, you can add logic to validate model_data if needed
                 model_id = model_data.get("id")
-                if model_id:
+
+                if model_id and is_valid_model_id(model_id):
                     existing_model = Models.get_model_by_id(model_id)
                     if existing_model:
                         # Update existing model
@@ -170,6 +261,10 @@ async def sync_models(
 ###########################
 
 
+class ModelIdForm(BaseModel):
+    id: str
+
+
 # Note: We're not using the typical url path param here, but instead using a query parameter to allow '/' in the id
 @router.get("/model", response_model=Optional[ModelResponse])
 async def get_model_by_id(id: str, user=Depends(get_verified_user)):
@@ -216,6 +311,7 @@ async def get_model_profile_image(id: str, user=Depends(get_verified_user)):
                     )
                 except Exception as e:
                     pass
+
         return FileResponse(f"{STATIC_DIR}/favicon.png")
     else:
         return FileResponse(f"{STATIC_DIR}/favicon.png")
@@ -263,12 +359,10 @@ async def toggle_model_by_id(id: str, user=Depends(get_verified_user)):
 
 @router.post("/model/update", response_model=Optional[ModelModel])
 async def update_model_by_id(
-    id: str,
     form_data: ModelForm,
     user=Depends(get_verified_user),
 ):
-    model = Models.get_model_by_id(id)
-
+    model = Models.get_model_by_id(form_data.id)
     if not model:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
@@ -285,7 +379,7 @@ async def update_model_by_id(
             detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
         )
 
-    model = Models.update_model_by_id(id, form_data)
+    model = Models.update_model_by_id(form_data.id, ModelForm(**form_data.model_dump()))
     return model
 
 
@@ -294,9 +388,9 @@ async def update_model_by_id(
 ############################
 
 
-@router.delete("/model/delete", response_model=bool)
-async def delete_model_by_id(id: str, user=Depends(get_verified_user)):
-    model = Models.get_model_by_id(id)
+@router.post("/model/delete", response_model=bool)
+async def delete_model_by_id(form_data: ModelIdForm, user=Depends(get_verified_user)):
+    model = Models.get_model_by_id(form_data.id)
     if not model:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
@@ -313,7 +407,7 @@ async def delete_model_by_id(id: str, user=Depends(get_verified_user)):
             detail=ERROR_MESSAGES.UNAUTHORIZED,
         )
 
-    result = Models.delete_model_by_id(id)
+    result = Models.delete_model_by_id(form_data.id)
     return result
 
 

backend/open_webui/routers/ollama.py

@@ -16,8 +16,8 @@ from urllib.parse import urlparse
 import aiohttp
 from aiocache import cached
 import requests
-from urllib.parse import quote
 
+from open_webui.utils.headers import include_user_info_headers
 from open_webui.models.chats import Chats
 from open_webui.models.users import UserModel
 
@@ -82,22 +82,17 @@ async def send_get_request(url, key=None, user: UserModel = None):
     timeout = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST)
     try:
         async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
-            async with session.get(
-                url,
-                headers={
+            headers = {
                 "Content-Type": "application/json",
                 **({"Authorization": f"Bearer {key}"} if key else {}),
-                    **(
-                        {
-                            "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                            "X-OpenWebUI-User-Id": user.id,
-                            "X-OpenWebUI-User-Email": user.email,
-                            "X-OpenWebUI-User-Role": user.role,
             }
-                        if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                        else {}
-                    ),
-                },
+
+            if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+                headers = include_user_info_headers(headers, user)
+
+            async with session.get(
+                url,
+                headers=headers,
                 ssl=AIOHTTP_CLIENT_SESSION_SSL,
             ) as response:
                 return await response.json()
@@ -133,28 +128,20 @@ async def send_post_request(
             trust_env=True, timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT)
         )
 
+        headers = {
+            "Content-Type": "application/json",
+            **({"Authorization": f"Bearer {key}"} if key else {}),
+        }
+
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+            if metadata and metadata.get("chat_id"):
+                headers["X-OpenWebUI-Chat-Id"] = metadata.get("chat_id")
+
         r = await session.post(
             url,
             data=payload,
-            headers={
-                "Content-Type": "application/json",
-                **({"Authorization": f"Bearer {key}"} if key else {}),
-                **(
-                    {
-                        "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                        "X-OpenWebUI-User-Id": user.id,
-                        "X-OpenWebUI-User-Email": user.email,
-                        "X-OpenWebUI-User-Role": user.role,
-                        **(
-                            {"X-OpenWebUI-Chat-Id": metadata.get("chat_id")}
-                            if metadata and metadata.get("chat_id")
-                            else {}
-                        ),
-                    }
-                    if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                    else {}
-                ),
-            },
+            headers=headers,
             ssl=AIOHTTP_CLIENT_SESSION_SSL,
         )
 
@@ -246,21 +233,16 @@ async def verify_connection(
         timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST),
     ) as session:
         try:
+            headers = {
+                **({"Authorization": f"Bearer {key}"} if key else {}),
+            }
+
+            if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+                headers = include_user_info_headers(headers, user)
+
             async with session.get(
                 f"{url}/api/version",
-                headers={
-                    **({"Authorization": f"Bearer {key}"} if key else {}),
-                    **(
-                        {
-                            "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                            "X-OpenWebUI-User-Id": user.id,
-                            "X-OpenWebUI-User-Email": user.email,
-                            "X-OpenWebUI-User-Role": user.role,
-                        }
-                        if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                        else {}
-                    ),
-                },
+                headers=headers,
                 ssl=AIOHTTP_CLIENT_SESSION_SSL,
             ) as r:
                 if r.status != 200:
@@ -469,22 +451,17 @@ async def get_ollama_tags(
 
         r = None
         try:
+            headers = {
+                **({"Authorization": f"Bearer {key}"} if key else {}),
+            }
+
+            if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+                headers = include_user_info_headers(headers, user)
+
             r = requests.request(
                 method="GET",
                 url=f"{url}/api/tags",
-                headers={
-                    **({"Authorization": f"Bearer {key}"} if key else {}),
-                    **(
-                        {
-                            "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                            "X-OpenWebUI-User-Id": user.id,
-                            "X-OpenWebUI-User-Email": user.email,
-                            "X-OpenWebUI-User-Role": user.role,
-                        }
-                        if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                        else {}
-                    ),
-                },
+                headers=headers,
             )
             r.raise_for_status()
 
@@ -838,23 +815,18 @@ async def copy_model(
     key = get_api_key(url_idx, url, request.app.state.config.OLLAMA_API_CONFIGS)
 
     try:
+        headers = {
+            "Content-Type": "application/json",
+            **({"Authorization": f"Bearer {key}"} if key else {}),
+        }
+
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
         r = requests.request(
             method="POST",
             url=f"{url}/api/copy",
-            headers={
-                "Content-Type": "application/json",
-                **({"Authorization": f"Bearer {key}"} if key else {}),
-                **(
-                    {
-                        "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                        "X-OpenWebUI-User-Id": user.id,
-                        "X-OpenWebUI-User-Email": user.email,
-                        "X-OpenWebUI-User-Role": user.role,
-                    }
-                    if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                    else {}
-                ),
-            },
+            headers=headers,
             data=form_data.model_dump_json(exclude_none=True).encode(),
         )
         r.raise_for_status()
@@ -907,25 +879,21 @@ async def delete_model(
     url = request.app.state.config.OLLAMA_BASE_URLS[url_idx]
     key = get_api_key(url_idx, url, request.app.state.config.OLLAMA_API_CONFIGS)
 
+    r = None
     try:
+        headers = {
+            "Content-Type": "application/json",
+            **({"Authorization": f"Bearer {key}"} if key else {}),
+        }
+
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
         r = requests.request(
             method="DELETE",
             url=f"{url}/api/delete",
-            data=json.dumps(form_data).encode(),
-            headers={
-                "Content-Type": "application/json",
-                **({"Authorization": f"Bearer {key}"} if key else {}),
-                **(
-                    {
-                        "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                        "X-OpenWebUI-User-Id": user.id,
-                        "X-OpenWebUI-User-Email": user.email,
-                        "X-OpenWebUI-User-Role": user.role,
-                    }
-                    if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                    else {}
-                ),
-            },
+            headers=headers,
+            json=form_data,
         )
         r.raise_for_status()
 
@@ -973,24 +941,16 @@ async def show_model_info(
     key = get_api_key(url_idx, url, request.app.state.config.OLLAMA_API_CONFIGS)
 
     try:
-        r = requests.request(
-            method="POST",
-            url=f"{url}/api/show",
-            headers={
+        headers = {
             "Content-Type": "application/json",
             **({"Authorization": f"Bearer {key}"} if key else {}),
-                **(
-                    {
-                        "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                        "X-OpenWebUI-User-Id": user.id,
-                        "X-OpenWebUI-User-Email": user.email,
-                        "X-OpenWebUI-User-Role": user.role,
         }
-                    if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                    else {}
-                ),
-            },
-            data=json.dumps(form_data).encode(),
+
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
+        r = requests.request(
+            method="POST", url=f"{url}/api/show", headers=headers, json=form_data
         )
         r.raise_for_status()
 
@@ -1064,23 +1024,18 @@ async def embed(
         form_data.model = form_data.model.replace(f"{prefix_id}.", "")
 
     try:
+        headers = {
+            "Content-Type": "application/json",
+            **({"Authorization": f"Bearer {key}"} if key else {}),
+        }
+
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
         r = requests.request(
             method="POST",
             url=f"{url}/api/embed",
-            headers={
-                "Content-Type": "application/json",
-                **({"Authorization": f"Bearer {key}"} if key else {}),
-                **(
-                    {
-                        "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                        "X-OpenWebUI-User-Id": user.id,
-                        "X-OpenWebUI-User-Email": user.email,
-                        "X-OpenWebUI-User-Role": user.role,
-                    }
-                    if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                    else {}
-                ),
-            },
+            headers=headers,
             data=form_data.model_dump_json(exclude_none=True).encode(),
         )
         r.raise_for_status()
@@ -1151,23 +1106,18 @@ async def embeddings(
         form_data.model = form_data.model.replace(f"{prefix_id}.", "")
 
     try:
+        headers = {
+            "Content-Type": "application/json",
+            **({"Authorization": f"Bearer {key}"} if key else {}),
+        }
+
+        if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+            headers = include_user_info_headers(headers, user)
+
         r = requests.request(
             method="POST",
             url=f"{url}/api/embeddings",
-            headers={
-                "Content-Type": "application/json",
-                **({"Authorization": f"Bearer {key}"} if key else {}),
-                **(
-                    {
-                        "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                        "X-OpenWebUI-User-Id": user.id,
-                        "X-OpenWebUI-User-Email": user.email,
-                        "X-OpenWebUI-User-Role": user.role,
-                    }
-                    if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                    else {}
-                ),
-            },
+            headers=headers,
             data=form_data.model_dump_json(exclude_none=True).encode(),
         )
         r.raise_for_status()

backend/open_webui/routers/openai.py

@@ -7,7 +7,6 @@ from typing import Optional
 import aiohttp
 from aiocache import cached
 import requests
-from urllib.parse import quote
 
 from azure.identity import DefaultAzureCredential, get_bearer_token_provider
 
@@ -45,10 +44,12 @@ from open_webui.utils.payload import (
 )
 from open_webui.utils.misc import (
     convert_logit_bias_input_to_json,
+    stream_chunks_handler,
 )
 
 from open_webui.utils.auth import get_admin_user, get_verified_user
 from open_webui.utils.access_control import has_access
+from open_webui.utils.headers import include_user_info_headers
 
 
 log = logging.getLogger(__name__)
@@ -66,21 +67,16 @@ async def send_get_request(url, key=None, user: UserModel = None):
     timeout = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST)
     try:
         async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
+            headers = {
+                **({"Authorization": f"Bearer {key}"} if key else {}),
+            }
+
+            if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+                headers = include_user_info_headers(headers, user)
+
             async with session.get(
                 url,
-                headers={
-                    **({"Authorization": f"Bearer {key}"} if key else {}),
-                    **(
-                        {
-                            "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                            "X-OpenWebUI-User-Id": user.id,
-                            "X-OpenWebUI-User-Email": user.email,
-                            "X-OpenWebUI-User-Role": user.role,
-                        }
-                        if ENABLE_FORWARD_USER_INFO_HEADERS and user
-                        else {}
-                    ),
-                },
+                headers=headers,
                 ssl=AIOHTTP_CLIENT_SESSION_SSL,
             ) as response:
                 return await response.json()
@@ -140,23 +136,13 @@ async def get_headers_and_cookies(
             if "openrouter.ai" in url
             else {}
         ),
-        **(
-            {
-                "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
-                "X-OpenWebUI-User-Id": user.id,
-                "X-OpenWebUI-User-Email": user.email,
-                "X-OpenWebUI-User-Role": user.role,
-                **(
-                    {"X-OpenWebUI-Chat-Id": metadata.get("chat_id")}
-                    if metadata and metadata.get("chat_id")
-                    else {}
-                ),
-            }
-            if ENABLE_FORWARD_USER_INFO_HEADERS
-            else {}
-        ),
     }
 
+    if ENABLE_FORWARD_USER_INFO_HEADERS and user:
+        headers = include_user_info_headers(headers, user)
+        if metadata and metadata.get("chat_id"):
+            headers["X-OpenWebUI-Chat-Id"] = metadata.get("chat_id")
+
     token = None
     auth_type = config.get("auth_type")
 
@@ -190,6 +176,9 @@ async def get_headers_and_cookies(
     if token:
         headers["Authorization"] = f"Bearer {token}"
 
+    if config.get("headers") and isinstance(config.get("headers"), dict):
+        headers = {**headers, **config.get("headers")}
+
     return headers, cookies
 
 
@@ -498,30 +487,9 @@ async def get_all_models(request: Request, user: UserModel) -> dict[str, list]:
             return response
         return None
 
-    def merge_models_lists(model_lists):
-        log.debug(f"merge_models_lists {model_lists}")
-        merged_list = []
-
-        for idx, models in enumerate(model_lists):
-            if models is not None and "error" not in models:
-
-                merged_list.extend(
-                    [
-                        {
-                            **model,
-                            "name": model.get("name", model["id"]),
-                            "owned_by": "openai",
-                            "openai": model,
-                            "connection_type": model.get("connection_type", "external"),
-                            "urlIdx": idx,
-                        }
-                        for model in models
-                        if (model.get("id") or model.get("name"))
-                        and (
-                            "api.openai.com"
-                            not in request.app.state.config.OPENAI_API_BASE_URLS[idx]
-                            or not any(
-                                name in model["id"]
+    def is_supported_openai_models(model_id):
+        if any(
+            name in model_id
             for name in [
                 "babbage",
                 "dall-e",
@@ -530,18 +498,44 @@ async def get_all_models(request: Request, user: UserModel) -> dict[str, list]:
                 "tts",
                 "whisper",
             ]
-                            )
-                        )
-                    ]
-                )
+        ):
+            return False
+        return True
 
-        return merged_list
+    def get_merged_models(model_lists):
+        log.debug(f"merge_models_lists {model_lists}")
+        models = {}
 
-    models = {"data": merge_models_lists(map(extract_data, responses))}
+        for idx, model_list in enumerate(model_lists):
+            if model_list is not None and "error" not in model_list:
+                for model in model_list:
+                    model_id = model.get("id") or model.get("name")
+
+                    if (
+                        "api.openai.com"
+                        in request.app.state.config.OPENAI_API_BASE_URLS[idx]
+                        and not is_supported_openai_models(model_id)
+                    ):
+                        # Skip unwanted OpenAI models
+                        continue
+
+                    if model_id and model_id not in models:
+                        models[model_id] = {
+                            **model,
+                            "name": model.get("name", model_id),
+                            "owned_by": "openai",
+                            "openai": model,
+                            "connection_type": model.get("connection_type", "external"),
+                            "urlIdx": idx,
+                        }
+
+        return models
+
+    models = get_merged_models(map(extract_data, responses))
     log.debug(f"models: {models}")
 
-    request.app.state.OPENAI_MODELS = {model["id"]: model for model in models["data"]}
-    return models
+    request.app.state.OPENAI_MODELS = models
+    return {"data": list(models.values())}
 
 
 @router.get("/models")
@@ -754,6 +748,7 @@ def get_azure_allowed_params(api_version: str) -> set[str]:
         "response_format",
         "seed",
         "max_completion_tokens",
+        "reasoning_effort",
     }
 
     try:
@@ -944,7 +939,7 @@ async def generate_chat_completion(
         if "text/event-stream" in r.headers.get("Content-Type", ""):
             streaming = True
             return StreamingResponse(
-                r.content,
+                stream_chunks_handler(r.content),
                 status_code=r.status,
                 headers=dict(r.headers),
                 background=BackgroundTask(

backend/open_webui/routers/prompts.py

@@ -48,8 +48,15 @@ async def get_prompt_list(user=Depends(get_verified_user)):
 async def create_new_prompt(
     request: Request, form_data: PromptForm, user=Depends(get_verified_user)
 ):
-    if user.role != "admin" and not has_permission(
+    if user.role != "admin" and not (
+        has_permission(
             user.id, "workspace.prompts", request.app.state.config.USER_PERMISSIONS
+        )
+        or has_permission(
+            user.id,
+            "workspace.prompts_import",
+            request.app.state.config.USER_PERMISSIONS,
+        )
     ):
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,

backend/open_webui/routers/retrieval.py

@@ -6,6 +6,7 @@ import shutil
 import asyncio
 import time
 
+import re
 import uuid
 from datetime import datetime, timedelta
 from multiprocessing import Value
@@ -36,7 +37,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter, TokenTextSpl
 from langchain_text_splitters import MarkdownHeaderTextSplitter
 from langchain_core.documents import Document
 
-from open_webui.models.files import FileModel, Files
+from open_webui.models.files import FileModel, FileUpdateForm, Files
 from open_webui.models.knowledge import Knowledges
 from open_webui.storage.provider import Storage
 
@@ -68,6 +69,7 @@ from open_webui.retrieval.web.serply import search_serply
 from open_webui.retrieval.web.serpstack import search_serpstack
 from open_webui.retrieval.web.tavily import search_tavily
 from open_webui.retrieval.web.bing import search_bing
+from open_webui.retrieval.web.azure import search_azure
 from open_webui.retrieval.web.exa import search_exa
 from open_webui.retrieval.web.perplexity import search_perplexity
 from open_webui.retrieval.web.sougou import search_sougou
@@ -75,6 +77,7 @@ from open_webui.retrieval.web.firecrawl import search_firecrawl
 from open_webui.retrieval.web.external import search_external
 
 from open_webui.retrieval.utils import (
+    get_content_from_url,
     get_embedding_function,
     get_reranking_function,
     get_model_path,
@@ -194,6 +197,26 @@ def get_rf(
                     log.error(f"CrossEncoder: {e}")
                     raise Exception(ERROR_MESSAGES.DEFAULT("CrossEncoder error"))
 
+                # Safely adjust pad_token_id if missing as some models do not have this in config
+                try:
+                    model_cfg = getattr(rf, "model", None)
+                    if model_cfg and hasattr(model_cfg, "config"):
+                        cfg = model_cfg.config
+                        if getattr(cfg, "pad_token_id", None) is None:
+                            # Fallback to eos_token_id when available
+                            eos = getattr(cfg, "eos_token_id", None)
+                            if eos is not None:
+                                cfg.pad_token_id = eos
+                                log.debug(
+                                    f"Missing pad_token_id detected; set to eos_token_id={eos}"
+                                )
+                            else:
+                                log.warning(
+                                    "Neither pad_token_id nor eos_token_id present in model config"
+                                )
+                except Exception as e2:
+                    log.warning(f"Failed to adjust pad_token_id on CrossEncoder: {e2}")
+
     return rf
 
 
@@ -223,13 +246,14 @@ class SearchForm(BaseModel):
 async def get_status(request: Request):
     return {
         "status": True,
-        "chunk_size": request.app.state.config.CHUNK_SIZE,
-        "chunk_overlap": request.app.state.config.CHUNK_OVERLAP,
-        "template": request.app.state.config.RAG_TEMPLATE,
-        "embedding_engine": request.app.state.config.RAG_EMBEDDING_ENGINE,
-        "embedding_model": request.app.state.config.RAG_EMBEDDING_MODEL,
-        "reranking_model": request.app.state.config.RAG_RERANKING_MODEL,
-        "embedding_batch_size": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE,
+        "CHUNK_SIZE": request.app.state.config.CHUNK_SIZE,
+        "CHUNK_OVERLAP": request.app.state.config.CHUNK_OVERLAP,
+        "RAG_TEMPLATE": request.app.state.config.RAG_TEMPLATE,
+        "RAG_EMBEDDING_ENGINE": request.app.state.config.RAG_EMBEDDING_ENGINE,
+        "RAG_EMBEDDING_MODEL": request.app.state.config.RAG_EMBEDDING_MODEL,
+        "RAG_RERANKING_MODEL": request.app.state.config.RAG_RERANKING_MODEL,
+        "RAG_EMBEDDING_BATCH_SIZE": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE,
+        "ENABLE_ASYNC_EMBEDDING": request.app.state.config.ENABLE_ASYNC_EMBEDDING,
     }
 
 
@@ -237,9 +261,10 @@ async def get_status(request: Request):
 async def get_embedding_config(request: Request, user=Depends(get_admin_user)):
     return {
         "status": True,
-        "embedding_engine": request.app.state.config.RAG_EMBEDDING_ENGINE,
-        "embedding_model": request.app.state.config.RAG_EMBEDDING_MODEL,
-        "embedding_batch_size": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE,
+        "RAG_EMBEDDING_ENGINE": request.app.state.config.RAG_EMBEDDING_ENGINE,
+        "RAG_EMBEDDING_MODEL": request.app.state.config.RAG_EMBEDDING_MODEL,
+        "RAG_EMBEDDING_BATCH_SIZE": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE,
+        "ENABLE_ASYNC_EMBEDDING": request.app.state.config.ENABLE_ASYNC_EMBEDDING,
         "openai_config": {
             "url": request.app.state.config.RAG_OPENAI_API_BASE_URL,
             "key": request.app.state.config.RAG_OPENAI_API_KEY,
@@ -276,18 +301,13 @@ class EmbeddingModelUpdateForm(BaseModel):
     openai_config: Optional[OpenAIConfigForm] = None
     ollama_config: Optional[OllamaConfigForm] = None
     azure_openai_config: Optional[AzureOpenAIConfigForm] = None
-    embedding_engine: str
-    embedding_model: str
-    embedding_batch_size: Optional[int] = 1
+    RAG_EMBEDDING_ENGINE: str
+    RAG_EMBEDDING_MODEL: str
+    RAG_EMBEDDING_BATCH_SIZE: Optional[int] = 1
+    ENABLE_ASYNC_EMBEDDING: Optional[bool] = True
 
 
-@router.post("/embedding/update")
-async def update_embedding_config(
-    request: Request, form_data: EmbeddingModelUpdateForm, user=Depends(get_admin_user)
-):
-    log.info(
-        f"Updating embedding model: {request.app.state.config.RAG_EMBEDDING_MODEL} to {form_data.embedding_model}"
-    )
+def unload_embedding_model(request: Request):
     if request.app.state.config.RAG_EMBEDDING_ENGINE == "":
         # unloads current internal embedding model and clears VRAM cache
         request.app.state.ef = None
@@ -300,9 +320,25 @@ async def update_embedding_config(
 
             if torch.cuda.is_available():
                 torch.cuda.empty_cache()
+
+
+@router.post("/embedding/update")
+async def update_embedding_config(
+    request: Request, form_data: EmbeddingModelUpdateForm, user=Depends(get_admin_user)
+):
+    log.info(
+        f"Updating embedding model: {request.app.state.config.RAG_EMBEDDING_MODEL} to {form_data.RAG_EMBEDDING_MODEL}"
+    )
+    unload_embedding_model(request)
     try:
-        request.app.state.config.RAG_EMBEDDING_ENGINE = form_data.embedding_engine
-        request.app.state.config.RAG_EMBEDDING_MODEL = form_data.embedding_model
+        request.app.state.config.RAG_EMBEDDING_ENGINE = form_data.RAG_EMBEDDING_ENGINE
+        request.app.state.config.RAG_EMBEDDING_MODEL = form_data.RAG_EMBEDDING_MODEL
+        request.app.state.config.RAG_EMBEDDING_BATCH_SIZE = (
+            form_data.RAG_EMBEDDING_BATCH_SIZE
+        )
+        request.app.state.config.ENABLE_ASYNC_EMBEDDING = (
+            form_data.ENABLE_ASYNC_EMBEDDING
+        )
 
         if request.app.state.config.RAG_EMBEDDING_ENGINE in [
             "ollama",
@@ -336,10 +372,6 @@ async def update_embedding_config(
                     form_data.azure_openai_config.version
                 )
 
-            request.app.state.config.RAG_EMBEDDING_BATCH_SIZE = (
-                form_data.embedding_batch_size
-            )
-
         request.app.state.ef = get_ef(
             request.app.state.config.RAG_EMBEDDING_ENGINE,
             request.app.state.config.RAG_EMBEDDING_MODEL,
@@ -373,13 +405,15 @@ async def update_embedding_config(
                 if request.app.state.config.RAG_EMBEDDING_ENGINE == "azure_openai"
                 else None
             ),
+            enable_async=request.app.state.config.ENABLE_ASYNC_EMBEDDING,
         )
 
         return {
             "status": True,
-            "embedding_engine": request.app.state.config.RAG_EMBEDDING_ENGINE,
-            "embedding_model": request.app.state.config.RAG_EMBEDDING_MODEL,
-            "embedding_batch_size": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE,
+            "RAG_EMBEDDING_ENGINE": request.app.state.config.RAG_EMBEDDING_ENGINE,
+            "RAG_EMBEDDING_MODEL": request.app.state.config.RAG_EMBEDDING_MODEL,
+            "RAG_EMBEDDING_BATCH_SIZE": request.app.state.config.RAG_EMBEDDING_BATCH_SIZE,
+            "ENABLE_ASYNC_EMBEDDING": request.app.state.config.ENABLE_ASYNC_EMBEDDING,
             "openai_config": {
                 "url": request.app.state.config.RAG_OPENAI_API_BASE_URL,
                 "key": request.app.state.config.RAG_OPENAI_API_KEY,
@@ -413,6 +447,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
         "RAG_FULL_CONTEXT": request.app.state.config.RAG_FULL_CONTEXT,
         # Hybrid search settings
         "ENABLE_RAG_HYBRID_SEARCH": request.app.state.config.ENABLE_RAG_HYBRID_SEARCH,
+        "ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS": request.app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS,
         "TOP_K_RERANKER": request.app.state.config.TOP_K_RERANKER,
         "RELEVANCE_THRESHOLD": request.app.state.config.RELEVANCE_THRESHOLD,
         "HYBRID_BM25_WEIGHT": request.app.state.config.HYBRID_BM25_WEIGHT,
@@ -434,21 +469,17 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
         "EXTERNAL_DOCUMENT_LOADER_API_KEY": request.app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
         "TIKA_SERVER_URL": request.app.state.config.TIKA_SERVER_URL,
         "DOCLING_SERVER_URL": request.app.state.config.DOCLING_SERVER_URL,
+        "DOCLING_API_KEY": request.app.state.config.DOCLING_API_KEY,
         "DOCLING_PARAMS": request.app.state.config.DOCLING_PARAMS,
-        "DOCLING_DO_OCR": request.app.state.config.DOCLING_DO_OCR,
-        "DOCLING_FORCE_OCR": request.app.state.config.DOCLING_FORCE_OCR,
-        "DOCLING_OCR_ENGINE": request.app.state.config.DOCLING_OCR_ENGINE,
-        "DOCLING_OCR_LANG": request.app.state.config.DOCLING_OCR_LANG,
-        "DOCLING_PDF_BACKEND": request.app.state.config.DOCLING_PDF_BACKEND,
-        "DOCLING_TABLE_MODE": request.app.state.config.DOCLING_TABLE_MODE,
-        "DOCLING_PIPELINE": request.app.state.config.DOCLING_PIPELINE,
-        "DOCLING_DO_PICTURE_DESCRIPTION": request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION,
-        "DOCLING_PICTURE_DESCRIPTION_MODE": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE,
-        "DOCLING_PICTURE_DESCRIPTION_LOCAL": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL,
-        "DOCLING_PICTURE_DESCRIPTION_API": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API,
         "DOCUMENT_INTELLIGENCE_ENDPOINT": request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT,
         "DOCUMENT_INTELLIGENCE_KEY": request.app.state.config.DOCUMENT_INTELLIGENCE_KEY,
+        "MISTRAL_OCR_API_BASE_URL": request.app.state.config.MISTRAL_OCR_API_BASE_URL,
         "MISTRAL_OCR_API_KEY": request.app.state.config.MISTRAL_OCR_API_KEY,
+        # MinerU settings
+        "MINERU_API_MODE": request.app.state.config.MINERU_API_MODE,
+        "MINERU_API_URL": request.app.state.config.MINERU_API_URL,
+        "MINERU_API_KEY": request.app.state.config.MINERU_API_KEY,
+        "MINERU_PARAMS": request.app.state.config.MINERU_PARAMS,
         # Reranking settings
         "RAG_RERANKING_MODEL": request.app.state.config.RAG_RERANKING_MODEL,
         "RAG_RERANKING_ENGINE": request.app.state.config.RAG_RERANKING_ENGINE,
@@ -505,6 +536,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
             "PERPLEXITY_API_KEY": request.app.state.config.PERPLEXITY_API_KEY,
             "PERPLEXITY_MODEL": request.app.state.config.PERPLEXITY_MODEL,
             "PERPLEXITY_SEARCH_CONTEXT_USAGE": request.app.state.config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
+            "PERPLEXITY_SEARCH_API_URL": request.app.state.config.PERPLEXITY_SEARCH_API_URL,
             "SOUGOU_API_SID": request.app.state.config.SOUGOU_API_SID,
             "SOUGOU_API_SK": request.app.state.config.SOUGOU_API_SK,
             "WEB_LOADER_ENGINE": request.app.state.config.WEB_LOADER_ENGINE,
@@ -562,6 +594,7 @@ class WebConfig(BaseModel):
     PERPLEXITY_API_KEY: Optional[str] = None
     PERPLEXITY_MODEL: Optional[str] = None
     PERPLEXITY_SEARCH_CONTEXT_USAGE: Optional[str] = None
+    PERPLEXITY_SEARCH_API_URL: Optional[str] = None
     SOUGOU_API_SID: Optional[str] = None
     SOUGOU_API_SK: Optional[str] = None
     WEB_LOADER_ENGINE: Optional[str] = None
@@ -589,6 +622,7 @@ class ConfigForm(BaseModel):
 
     # Hybrid search settings
     ENABLE_RAG_HYBRID_SEARCH: Optional[bool] = None
+    ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS: Optional[bool] = None
     TOP_K_RERANKER: Optional[int] = None
     RELEVANCE_THRESHOLD: Optional[float] = None
     HYBRID_BM25_WEIGHT: Optional[float] = None
@@ -614,22 +648,19 @@ class ConfigForm(BaseModel):
 
     TIKA_SERVER_URL: Optional[str] = None
     DOCLING_SERVER_URL: Optional[str] = None
+    DOCLING_API_KEY: Optional[str] = None
     DOCLING_PARAMS: Optional[dict] = None
-    DOCLING_DO_OCR: Optional[bool] = None
-    DOCLING_FORCE_OCR: Optional[bool] = None
-    DOCLING_OCR_ENGINE: Optional[str] = None
-    DOCLING_OCR_LANG: Optional[str] = None
-    DOCLING_PDF_BACKEND: Optional[str] = None
-    DOCLING_TABLE_MODE: Optional[str] = None
-    DOCLING_PIPELINE: Optional[str] = None
-    DOCLING_DO_PICTURE_DESCRIPTION: Optional[bool] = None
-    DOCLING_PICTURE_DESCRIPTION_MODE: Optional[str] = None
-    DOCLING_PICTURE_DESCRIPTION_LOCAL: Optional[dict] = None
-    DOCLING_PICTURE_DESCRIPTION_API: Optional[dict] = None
     DOCUMENT_INTELLIGENCE_ENDPOINT: Optional[str] = None
     DOCUMENT_INTELLIGENCE_KEY: Optional[str] = None
+    MISTRAL_OCR_API_BASE_URL: Optional[str] = None
     MISTRAL_OCR_API_KEY: Optional[str] = None
 
+    # MinerU settings
+    MINERU_API_MODE: Optional[str] = None
+    MINERU_API_URL: Optional[str] = None
+    MINERU_API_KEY: Optional[str] = None
+    MINERU_PARAMS: Optional[dict] = None
+
     # Reranking settings
     RAG_RERANKING_MODEL: Optional[str] = None
     RAG_RERANKING_ENGINE: Optional[str] = None
@@ -688,6 +719,11 @@ async def update_rag_config(
         if form_data.ENABLE_RAG_HYBRID_SEARCH is not None
         else request.app.state.config.ENABLE_RAG_HYBRID_SEARCH
     )
+    request.app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS = (
+        form_data.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS
+        if form_data.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS is not None
+        else request.app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS
+    )
 
     request.app.state.config.TOP_K_RERANKER = (
         form_data.TOP_K_RERANKER
@@ -791,68 +827,16 @@ async def update_rag_config(
         if form_data.DOCLING_SERVER_URL is not None
         else request.app.state.config.DOCLING_SERVER_URL
     )
+    request.app.state.config.DOCLING_API_KEY = (
+        form_data.DOCLING_API_KEY
+        if form_data.DOCLING_API_KEY is not None
+        else request.app.state.config.DOCLING_API_KEY
+    )
     request.app.state.config.DOCLING_PARAMS = (
         form_data.DOCLING_PARAMS
         if form_data.DOCLING_PARAMS is not None
         else request.app.state.config.DOCLING_PARAMS
     )
-    request.app.state.config.DOCLING_DO_OCR = (
-        form_data.DOCLING_DO_OCR
-        if form_data.DOCLING_DO_OCR is not None
-        else request.app.state.config.DOCLING_DO_OCR
-    )
-    request.app.state.config.DOCLING_FORCE_OCR = (
-        form_data.DOCLING_FORCE_OCR
-        if form_data.DOCLING_FORCE_OCR is not None
-        else request.app.state.config.DOCLING_FORCE_OCR
-    )
-    request.app.state.config.DOCLING_OCR_ENGINE = (
-        form_data.DOCLING_OCR_ENGINE
-        if form_data.DOCLING_OCR_ENGINE is not None
-        else request.app.state.config.DOCLING_OCR_ENGINE
-    )
-    request.app.state.config.DOCLING_OCR_LANG = (
-        form_data.DOCLING_OCR_LANG
-        if form_data.DOCLING_OCR_LANG is not None
-        else request.app.state.config.DOCLING_OCR_LANG
-    )
-    request.app.state.config.DOCLING_PDF_BACKEND = (
-        form_data.DOCLING_PDF_BACKEND
-        if form_data.DOCLING_PDF_BACKEND is not None
-        else request.app.state.config.DOCLING_PDF_BACKEND
-    )
-    request.app.state.config.DOCLING_TABLE_MODE = (
-        form_data.DOCLING_TABLE_MODE
-        if form_data.DOCLING_TABLE_MODE is not None
-        else request.app.state.config.DOCLING_TABLE_MODE
-    )
-    request.app.state.config.DOCLING_PIPELINE = (
-        form_data.DOCLING_PIPELINE
-        if form_data.DOCLING_PIPELINE is not None
-        else request.app.state.config.DOCLING_PIPELINE
-    )
-    request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION = (
-        form_data.DOCLING_DO_PICTURE_DESCRIPTION
-        if form_data.DOCLING_DO_PICTURE_DESCRIPTION is not None
-        else request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION
-    )
-
-    request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE = (
-        form_data.DOCLING_PICTURE_DESCRIPTION_MODE
-        if form_data.DOCLING_PICTURE_DESCRIPTION_MODE is not None
-        else request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE
-    )
-    request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL = (
-        form_data.DOCLING_PICTURE_DESCRIPTION_LOCAL
-        if form_data.DOCLING_PICTURE_DESCRIPTION_LOCAL is not None
-        else request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL
-    )
-    request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API = (
-        form_data.DOCLING_PICTURE_DESCRIPTION_API
-        if form_data.DOCLING_PICTURE_DESCRIPTION_API is not None
-        else request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API
-    )
-
     request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT = (
         form_data.DOCUMENT_INTELLIGENCE_ENDPOINT
         if form_data.DOCUMENT_INTELLIGENCE_ENDPOINT is not None
@@ -863,12 +847,40 @@ async def update_rag_config(
         if form_data.DOCUMENT_INTELLIGENCE_KEY is not None
         else request.app.state.config.DOCUMENT_INTELLIGENCE_KEY
     )
+
+    request.app.state.config.MISTRAL_OCR_API_BASE_URL = (
+        form_data.MISTRAL_OCR_API_BASE_URL
+        if form_data.MISTRAL_OCR_API_BASE_URL is not None
+        else request.app.state.config.MISTRAL_OCR_API_BASE_URL
+    )
     request.app.state.config.MISTRAL_OCR_API_KEY = (
         form_data.MISTRAL_OCR_API_KEY
         if form_data.MISTRAL_OCR_API_KEY is not None
         else request.app.state.config.MISTRAL_OCR_API_KEY
     )
 
+    # MinerU settings
+    request.app.state.config.MINERU_API_MODE = (
+        form_data.MINERU_API_MODE
+        if form_data.MINERU_API_MODE is not None
+        else request.app.state.config.MINERU_API_MODE
+    )
+    request.app.state.config.MINERU_API_URL = (
+        form_data.MINERU_API_URL
+        if form_data.MINERU_API_URL is not None
+        else request.app.state.config.MINERU_API_URL
+    )
+    request.app.state.config.MINERU_API_KEY = (
+        form_data.MINERU_API_KEY
+        if form_data.MINERU_API_KEY is not None
+        else request.app.state.config.MINERU_API_KEY
+    )
+    request.app.state.config.MINERU_PARAMS = (
+        form_data.MINERU_PARAMS
+        if form_data.MINERU_PARAMS is not None
+        else request.app.state.config.MINERU_PARAMS
+    )
+
     # Reranking settings
     if request.app.state.config.RAG_RERANKING_ENGINE == "":
         # Unloading the internal reranker and clear VRAM memory
@@ -1050,6 +1062,9 @@ async def update_rag_config(
         request.app.state.config.PERPLEXITY_SEARCH_CONTEXT_USAGE = (
             form_data.web.PERPLEXITY_SEARCH_CONTEXT_USAGE
         )
+        request.app.state.config.PERPLEXITY_SEARCH_API_URL = (
+            form_data.web.PERPLEXITY_SEARCH_API_URL
+        )
         request.app.state.config.SOUGOU_API_SID = form_data.web.SOUGOU_API_SID
         request.app.state.config.SOUGOU_API_SK = form_data.web.SOUGOU_API_SK
 
@@ -1118,21 +1133,17 @@ async def update_rag_config(
         "EXTERNAL_DOCUMENT_LOADER_API_KEY": request.app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
         "TIKA_SERVER_URL": request.app.state.config.TIKA_SERVER_URL,
         "DOCLING_SERVER_URL": request.app.state.config.DOCLING_SERVER_URL,
+        "DOCLING_API_KEY": request.app.state.config.DOCLING_API_KEY,
         "DOCLING_PARAMS": request.app.state.config.DOCLING_PARAMS,
-        "DOCLING_DO_OCR": request.app.state.config.DOCLING_DO_OCR,
-        "DOCLING_FORCE_OCR": request.app.state.config.DOCLING_FORCE_OCR,
-        "DOCLING_OCR_ENGINE": request.app.state.config.DOCLING_OCR_ENGINE,
-        "DOCLING_OCR_LANG": request.app.state.config.DOCLING_OCR_LANG,
-        "DOCLING_PDF_BACKEND": request.app.state.config.DOCLING_PDF_BACKEND,
-        "DOCLING_TABLE_MODE": request.app.state.config.DOCLING_TABLE_MODE,
-        "DOCLING_PIPELINE": request.app.state.config.DOCLING_PIPELINE,
-        "DOCLING_DO_PICTURE_DESCRIPTION": request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION,
-        "DOCLING_PICTURE_DESCRIPTION_MODE": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE,
-        "DOCLING_PICTURE_DESCRIPTION_LOCAL": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL,
-        "DOCLING_PICTURE_DESCRIPTION_API": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API,
         "DOCUMENT_INTELLIGENCE_ENDPOINT": request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT,
         "DOCUMENT_INTELLIGENCE_KEY": request.app.state.config.DOCUMENT_INTELLIGENCE_KEY,
+        "MISTRAL_OCR_API_BASE_URL": request.app.state.config.MISTRAL_OCR_API_BASE_URL,
         "MISTRAL_OCR_API_KEY": request.app.state.config.MISTRAL_OCR_API_KEY,
+        # MinerU settings
+        "MINERU_API_MODE": request.app.state.config.MINERU_API_MODE,
+        "MINERU_API_URL": request.app.state.config.MINERU_API_URL,
+        "MINERU_API_KEY": request.app.state.config.MINERU_API_KEY,
+        "MINERU_PARAMS": request.app.state.config.MINERU_PARAMS,
         # Reranking settings
         "RAG_RERANKING_MODEL": request.app.state.config.RAG_RERANKING_MODEL,
         "RAG_RERANKING_ENGINE": request.app.state.config.RAG_RERANKING_ENGINE,
@@ -1189,6 +1200,7 @@ async def update_rag_config(
             "PERPLEXITY_API_KEY": request.app.state.config.PERPLEXITY_API_KEY,
             "PERPLEXITY_MODEL": request.app.state.config.PERPLEXITY_MODEL,
             "PERPLEXITY_SEARCH_CONTEXT_USAGE": request.app.state.config.PERPLEXITY_SEARCH_CONTEXT_USAGE,
+            "PERPLEXITY_SEARCH_API_URL": request.app.state.config.PERPLEXITY_SEARCH_API_URL,
             "SOUGOU_API_SID": request.app.state.config.SOUGOU_API_SID,
             "SOUGOU_API_SK": request.app.state.config.SOUGOU_API_SK,
             "WEB_LOADER_ENGINE": request.app.state.config.WEB_LOADER_ENGINE,
@@ -1242,7 +1254,7 @@ def save_docs_to_vector_db(
 
         return ", ".join(docs_info)
 
-    log.info(
+    log.debug(
         f"save_docs_to_vector_db: document {_get_docs_info(docs)} {collection_name}"
     )
 
@@ -1389,11 +1401,14 @@ def save_docs_to_vector_db(
             ),
         )
 
-        embeddings = embedding_function(
+        # Run async embedding in sync context
+        embeddings = asyncio.run(
+            embedding_function(
                 list(map(lambda x: x.replace("\n", " "), texts)),
                 prefix=RAG_EMBEDDING_CONTENT_PREFIX,
                 user=user,
             )
+        )
         log.info(f"embeddings generated {len(embeddings)} for {len(texts)} items")
 
         items = [
@@ -1431,6 +1446,9 @@ def process_file(
     form_data: ProcessFileForm,
     user=Depends(get_verified_user),
 ):
+    """
+    Process a file and save its content to the vector database.
+    """
     if user.role == "admin":
         file = Files.get_file_by_id(form_data.file_id)
     else:
@@ -1510,6 +1528,7 @@ def process_file(
                     file_path = Storage.get_file(file_path)
                     loader = Loader(
                         engine=request.app.state.config.CONTENT_EXTRACTION_ENGINE,
+                        user=user,
                         DATALAB_MARKER_API_KEY=request.app.state.config.DATALAB_MARKER_API_KEY,
                         DATALAB_MARKER_API_BASE_URL=request.app.state.config.DATALAB_MARKER_API_BASE_URL,
                         DATALAB_MARKER_ADDITIONAL_CONFIG=request.app.state.config.DATALAB_MARKER_ADDITIONAL_CONFIG,
@@ -1525,24 +1544,17 @@ def process_file(
                         EXTERNAL_DOCUMENT_LOADER_API_KEY=request.app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
                         TIKA_SERVER_URL=request.app.state.config.TIKA_SERVER_URL,
                         DOCLING_SERVER_URL=request.app.state.config.DOCLING_SERVER_URL,
-                        DOCLING_PARAMS={
-                            "do_ocr": request.app.state.config.DOCLING_DO_OCR,
-                            "force_ocr": request.app.state.config.DOCLING_FORCE_OCR,
-                            "ocr_engine": request.app.state.config.DOCLING_OCR_ENGINE,
-                            "ocr_lang": request.app.state.config.DOCLING_OCR_LANG,
-                            "pdf_backend": request.app.state.config.DOCLING_PDF_BACKEND,
-                            "table_mode": request.app.state.config.DOCLING_TABLE_MODE,
-                            "pipeline": request.app.state.config.DOCLING_PIPELINE,
-                            "do_picture_description": request.app.state.config.DOCLING_DO_PICTURE_DESCRIPTION,
-                            "picture_description_mode": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_MODE,
-                            "picture_description_local": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_LOCAL,
-                            "picture_description_api": request.app.state.config.DOCLING_PICTURE_DESCRIPTION_API,
-                            **request.app.state.config.DOCLING_PARAMS,
-                        },
+                        DOCLING_API_KEY=request.app.state.config.DOCLING_API_KEY,
+                        DOCLING_PARAMS=request.app.state.config.DOCLING_PARAMS,
                         PDF_EXTRACT_IMAGES=request.app.state.config.PDF_EXTRACT_IMAGES,
                         DOCUMENT_INTELLIGENCE_ENDPOINT=request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT,
                         DOCUMENT_INTELLIGENCE_KEY=request.app.state.config.DOCUMENT_INTELLIGENCE_KEY,
+                        MISTRAL_OCR_API_BASE_URL=request.app.state.config.MISTRAL_OCR_API_BASE_URL,
                         MISTRAL_OCR_API_KEY=request.app.state.config.MISTRAL_OCR_API_KEY,
+                        MINERU_API_MODE=request.app.state.config.MINERU_API_MODE,
+                        MINERU_API_URL=request.app.state.config.MINERU_API_URL,
+                        MINERU_API_KEY=request.app.state.config.MINERU_API_KEY,
+                        MINERU_PARAMS=request.app.state.config.MINERU_PARAMS,
                     )
                     docs = loader.load(
                         file.filename, file.meta.get("content_type"), file_path
@@ -1663,7 +1675,7 @@ class ProcessTextForm(BaseModel):
 
 
 @router.post("/process/text")
-def process_text(
+async def process_text(
     request: Request,
     form_data: ProcessTextForm,
     user=Depends(get_verified_user),
@@ -1681,7 +1693,9 @@ def process_text(
     text_content = form_data.content
     log.debug(f"text_content: {text_content}")
 
-    result = save_docs_to_vector_db(request, docs, collection_name, user=user)
+    result = await run_in_threadpool(
+        save_docs_to_vector_db, request, docs, collection_name, user=user
+    )
     if result:
         return {
             "status": True,
@@ -1696,51 +1710,8 @@ def process_text(
 
 
 @router.post("/process/youtube")
-def process_youtube_video(
-    request: Request, form_data: ProcessUrlForm, user=Depends(get_verified_user)
-):
-    try:
-        collection_name = form_data.collection_name
-        if not collection_name:
-            collection_name = calculate_sha256_string(form_data.url)[:63]
-
-        loader = YoutubeLoader(
-            form_data.url,
-            language=request.app.state.config.YOUTUBE_LOADER_LANGUAGE,
-            proxy_url=request.app.state.config.YOUTUBE_LOADER_PROXY_URL,
-        )
-
-        docs = loader.load()
-        content = " ".join([doc.page_content for doc in docs])
-        log.debug(f"text_content: {content}")
-
-        save_docs_to_vector_db(
-            request, docs, collection_name, overwrite=True, user=user
-        )
-
-        return {
-            "status": True,
-            "collection_name": collection_name,
-            "filename": form_data.url,
-            "file": {
-                "data": {
-                    "content": content,
-                },
-                "meta": {
-                    "name": form_data.url,
-                },
-            },
-        }
-    except Exception as e:
-        log.exception(e)
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail=ERROR_MESSAGES.DEFAULT(e),
-        )
-
-
 @router.post("/process/web")
-def process_web(
+async def process_web(
     request: Request, form_data: ProcessUrlForm, user=Depends(get_verified_user)
 ):
     try:
@@ -1748,19 +1719,19 @@ def process_web(
         if not collection_name:
             collection_name = calculate_sha256_string(form_data.url)[:63]
 
-        loader = get_web_loader(
-            form_data.url,
-            verify_ssl=request.app.state.config.ENABLE_WEB_LOADER_SSL_VERIFICATION,
-            requests_per_second=request.app.state.config.WEB_LOADER_CONCURRENT_REQUESTS,
+        content, docs = await run_in_threadpool(
+            get_content_from_url, request, form_data.url
         )
-        docs = loader.load()
-        content = " ".join([doc.page_content for doc in docs])
-
         log.debug(f"text_content: {content}")
 
         if not request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL:
-            save_docs_to_vector_db(
-                request, docs, collection_name, overwrite=True, user=user
+            await run_in_threadpool(
+                save_docs_to_vector_db,
+                request,
+                docs,
+                collection_name,
+                overwrite=True,
+                user=user,
             )
         else:
             collection_name = None
@@ -1786,7 +1757,6 @@ def process_web(
             detail=ERROR_MESSAGES.DEFAULT(e),
         )
 
-
 from open_webui.config import (
     WEB_SEARCH_RATE_LIMIT_MAX_REQUESTS,
     WEB_SEARCH_RATE_LIMIT_MIN_SECONDS,
@@ -1843,7 +1813,9 @@ def search_rate_limit(max_calls: int, period: float):
     int(WEB_SEARCH_RATE_LIMIT_MAX_REQUESTS.value),
     int(WEB_SEARCH_RATE_LIMIT_MIN_SECONDS.value),
 )
-async def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
+def search_web(
+    request: Request, engine: str, query: str, user=None
+) -> list[SearchResult]:
     """Search the web using a search engine and return the results as a list of SearchResult objects.
     Will look for a search engine API key in environment variables in the following order:
     - SEARXNG_QUERY_URL
@@ -1884,6 +1856,8 @@ async def search_web(request: Request, engine: str, query: str) -> list[SearchRe
                 query,
                 request.app.state.config.WEB_SEARCH_RESULT_COUNT,
                 request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
+                request.app.state.config.PERPLEXITY_SEARCH_API_URL,
+                user,
             )
         else:
             raise Exception("No PERPLEXITY_API_KEY found in environment variables")
@@ -1920,6 +1894,7 @@ async def search_web(request: Request, engine: str, query: str) -> list[SearchRe
                 query,
                 request.app.state.config.WEB_SEARCH_RESULT_COUNT,
                 request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
+                referer=request.app.state.config.WEBUI_URL,
             )
         else:
             raise Exception(
@@ -2060,6 +2035,24 @@ async def search_web(request: Request, engine: str, query: str) -> list[SearchRe
             request.app.state.config.WEB_SEARCH_RESULT_COUNT,
             request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
         )
+    elif engine == "azure":
+        if (
+            request.app.state.config.AZURE_AI_SEARCH_API_KEY
+            and request.app.state.config.AZURE_AI_SEARCH_ENDPOINT
+            and request.app.state.config.AZURE_AI_SEARCH_INDEX_NAME
+        ):
+            return search_azure(
+                request.app.state.config.AZURE_AI_SEARCH_API_KEY,
+                request.app.state.config.AZURE_AI_SEARCH_ENDPOINT,
+                request.app.state.config.AZURE_AI_SEARCH_INDEX_NAME,
+                query,
+                request.app.state.config.WEB_SEARCH_RESULT_COUNT,
+                request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
+            )
+        else:
+            raise Exception(
+                "AZURE_AI_SEARCH_API_KEY, AZURE_AI_SEARCH_ENDPOINT, and AZURE_AI_SEARCH_INDEX_NAME are required for Azure AI Search"
+            )
     elif engine == "exa":
         return search_exa(
             request.app.state.config.EXA_API_KEY,
@@ -2102,11 +2095,13 @@ async def search_web(request: Request, engine: str, query: str) -> list[SearchRe
         )
     elif engine == "external":
         return search_external(
+            request,
             request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
             request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
             query,
             request.app.state.config.WEB_SEARCH_RESULT_COUNT,
             request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
+            user=user,
         )
     else:
         raise Exception("No search engine API key found in environment variables")
@@ -2131,6 +2126,7 @@ async def process_web_search(
                 request,
                 request.app.state.config.WEB_SEARCH_ENGINE,
                 query,
+                user,
             )
             for query in form_data.queries
         ]
@@ -2156,6 +2152,12 @@ async def process_web_search(
             detail=ERROR_MESSAGES.WEB_SEARCH_ERROR(e),
         )
 
+    if len(urls) == 0:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.DEFAULT("No results found from web search"),
+        )
+
     try:
         if request.app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER:
             search_results = [
@@ -2251,7 +2253,7 @@ class QueryDocForm(BaseModel):
 
 
 @router.post("/query/doc")
-def query_doc_handler(
+async def query_doc_handler(
     request: Request,
     form_data: QueryDocForm,
     user=Depends(get_verified_user),
@@ -2264,7 +2266,7 @@ def query_doc_handler(
             collection_results[form_data.collection_name] = VECTOR_DB_CLIENT.get(
                 collection_name=form_data.collection_name
             )
-            return query_doc_with_hybrid_search(
+            return await query_doc_with_hybrid_search(
                 collection_name=form_data.collection_name,
                 collection_result=collection_results[form_data.collection_name],
                 query=form_data.query,
@@ -2274,8 +2276,8 @@ def query_doc_handler(
                 k=form_data.k if form_data.k else request.app.state.config.TOP_K,
                 reranking_function=(
                     (
-                        lambda sentences: request.app.state.RERANKING_FUNCTION(
-                            sentences, user=user
+                        lambda query, documents: request.app.state.RERANKING_FUNCTION(
+                            query, documents, user=user
                         )
                     )
                     if request.app.state.RERANKING_FUNCTION
@@ -2296,11 +2298,12 @@ def query_doc_handler(
                 user=user,
             )
         else:
+            query_embedding = await request.app.state.EMBEDDING_FUNCTION(
+                form_data.query, prefix=RAG_EMBEDDING_QUERY_PREFIX, user=user
+            )
             return query_doc(
                 collection_name=form_data.collection_name,
-                query_embedding=request.app.state.EMBEDDING_FUNCTION(
-                    form_data.query, prefix=RAG_EMBEDDING_QUERY_PREFIX, user=user
-                ),
+                query_embedding=query_embedding,
                 k=form_data.k if form_data.k else request.app.state.config.TOP_K,
                 user=user,
             )
@@ -2320,10 +2323,11 @@ class QueryCollectionsForm(BaseModel):
     r: Optional[float] = None
     hybrid: Optional[bool] = None
     hybrid_bm25_weight: Optional[float] = None
+    enable_enriched_texts: Optional[bool] = None
 
 
 @router.post("/query/collection")
-def query_collection_handler(
+async def query_collection_handler(
     request: Request,
     form_data: QueryCollectionsForm,
     user=Depends(get_verified_user),
@@ -2332,7 +2336,7 @@ def query_collection_handler(
         if request.app.state.config.ENABLE_RAG_HYBRID_SEARCH and (
             form_data.hybrid is None or form_data.hybrid
         ):
-            return query_collection_with_hybrid_search(
+            return await query_collection_with_hybrid_search(
                 collection_names=form_data.collection_names,
                 queries=[form_data.query],
                 embedding_function=lambda query, prefix: request.app.state.EMBEDDING_FUNCTION(
@@ -2341,8 +2345,8 @@ def query_collection_handler(
                 k=form_data.k if form_data.k else request.app.state.config.TOP_K,
                 reranking_function=(
                     (
-                        lambda sentences: request.app.state.RERANKING_FUNCTION(
-                            sentences, user=user
+                        lambda query, documents: request.app.state.RERANKING_FUNCTION(
+                            query, documents, user=user
                         )
                     )
                     if request.app.state.RERANKING_FUNCTION
@@ -2360,9 +2364,14 @@ def query_collection_handler(
                     if form_data.hybrid_bm25_weight
                     else request.app.state.config.HYBRID_BM25_WEIGHT
                 ),
+                enable_enriched_texts=(
+                    form_data.enable_enriched_texts
+                    if form_data.enable_enriched_texts is not None
+                    else request.app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS
+                ),
             )
         else:
-            return query_collection(
+            return await query_collection(
                 collection_names=form_data.collection_names,
                 queries=[form_data.query],
                 embedding_function=lambda query, prefix: request.app.state.EMBEDDING_FUNCTION(
@@ -2444,7 +2453,7 @@ if ENV == "dev":
     @router.get("/ef/{text}")
     async def get_embeddings(request: Request, text: Optional[str] = "Hello World!"):
         return {
-            "result": request.app.state.EMBEDDING_FUNCTION(
+            "result": await request.app.state.EMBEDDING_FUNCTION(
                 text, prefix=RAG_EMBEDDING_QUERY_PREFIX
             )
         }
@@ -2467,7 +2476,7 @@ class BatchProcessFilesResponse(BaseModel):
 
 
 @router.post("/process/files/batch")
-def process_files_batch(
+async def process_files_batch(
     request: Request,
     form_data: BatchProcessFilesForm,
     user=Depends(get_verified_user),
@@ -2475,16 +2484,19 @@ def process_files_batch(
     """
     Process a batch of files and save them to the vector database.
     """
-    results: List[BatchProcessFilesResult] = []
-    errors: List[BatchProcessFilesResult] = []
+
     collection_name = form_data.collection_name
 
+    file_results: List[BatchProcessFilesResult] = []
+    file_errors: List[BatchProcessFilesResult] = []
+    file_updates: List[FileUpdateForm] = []
+
     # Prepare all documents first
     all_docs: List[Document] = []
+
     for file in form_data.files:
         try:
             text_content = file.data.get("content", "")
-
             docs: List[Document] = [
                 Document(
                     page_content=text_content.replace("<br/>", "\n"),
@@ -2498,45 +2510,49 @@ def process_files_batch(
                 )
             ]
 
-            hash = calculate_sha256_string(text_content)
-            Files.update_file_hash_by_id(file.id, hash)
-            Files.update_file_data_by_id(file.id, {"content": text_content})
-
             all_docs.extend(docs)
-            results.append(BatchProcessFilesResult(file_id=file.id, status="prepared"))
+
+            file_updates.append(
+                FileUpdateForm(
+                    hash=calculate_sha256_string(text_content),
+                    data={"content": text_content},
+                )
+            )
+            file_results.append(
+                BatchProcessFilesResult(file_id=file.id, status="prepared")
+            )
 
         except Exception as e:
             log.error(f"process_files_batch: Error processing file {file.id}: {str(e)}")
-            errors.append(
+            file_errors.append(
                 BatchProcessFilesResult(file_id=file.id, status="failed", error=str(e))
             )
 
     # Save all documents in one batch
     if all_docs:
         try:
-            save_docs_to_vector_db(
-                request=request,
-                docs=all_docs,
-                collection_name=collection_name,
+            await run_in_threadpool(
+                save_docs_to_vector_db,
+                request,
+                all_docs,
+                collection_name,
                 add=True,
                 user=user,
             )
 
             # Update all files with collection name
-            for result in results:
-                Files.update_file_metadata_by_id(
-                    result.file_id, {"collection_name": collection_name}
-                )
-                result.status = "completed"
+            for file_update, file_result in zip(file_updates, file_results):
+                Files.update_file_by_id(id=file_result.file_id, form_data=file_update)
+                file_result.status = "completed"
 
         except Exception as e:
             log.error(
                 f"process_files_batch: Error saving documents to vector DB: {str(e)}"
             )
-            for result in results:
-                result.status = "failed"
-                errors.append(
-                    BatchProcessFilesResult(file_id=result.file_id, error=str(e))
+            for file_result in file_results:
+                file_result.status = "failed"
+                file_errors.append(
+                    BatchProcessFilesResult(file_id=file_result.file_id, error=str(e))
                 )
 
-    return BatchProcessFilesResponse(results=results, errors=errors)
+    return BatchProcessFilesResponse(results=file_results, errors=file_errors)

backend/open_webui/routers/scim.py

@@ -256,15 +256,16 @@ def get_scim_auth(
             )
 
         # Check if SCIM is enabled
-        scim_enabled = getattr(request.app.state, "SCIM_ENABLED", False)
+        enable_scim = getattr(request.app.state, "ENABLE_SCIM", False)
         log.info(
-            f"SCIM auth check - raw SCIM_ENABLED: {scim_enabled}, type: {type(scim_enabled)}"
+            f"SCIM auth check - raw ENABLE_SCIM: {enable_scim}, type: {type(enable_scim)}"
         )
+
         # Handle both PersistentConfig and direct value
-        if hasattr(scim_enabled, "value"):
-            scim_enabled = scim_enabled.value
-        log.info(f"SCIM enabled status after conversion: {scim_enabled}")
-        if not scim_enabled:
+        if hasattr(enable_scim, "value"):
+            enable_scim = enable_scim.value
+
+        if not enable_scim:
             raise HTTPException(
                 status_code=status.HTTP_403_FORBIDDEN,
                 detail="SCIM is not enabled",
@@ -348,8 +349,10 @@ def user_to_scim(user: UserModel, request: Request) -> SCIMUser:
 
 def group_to_scim(group: GroupModel, request: Request) -> SCIMGroup:
     """Convert internal Group model to SCIM Group"""
+    member_ids = Groups.get_group_user_ids_by_id(group.id)
     members = []
-    for user_id in group.user_ids:
+
+    for user_id in member_ids:
         user = Users.get_user_by_id(user_id)
         if user:
             members.append(
@@ -716,7 +719,7 @@ async def get_groups(
 ):
     """List SCIM Groups"""
     # Get all groups
-    groups_list = Groups.get_groups()
+    groups_list = Groups.get_all_groups()
 
     # Apply pagination
     total = len(groups_list)
@@ -795,9 +798,11 @@ async def create_group(
         update_form = GroupUpdateForm(
             name=new_group.name,
             description=new_group.description,
-            user_ids=member_ids,
         )
+
         Groups.update_group_by_id(new_group.id, update_form)
+        Groups.set_group_user_ids_by_id(new_group.id, member_ids)
+
         new_group = Groups.get_group_by_id(new_group.id)
 
     return group_to_scim(new_group, request)
@@ -829,7 +834,7 @@ async def update_group(
     # Handle members if provided
     if group_data.members is not None:
         member_ids = [member.value for member in group_data.members]
-        update_form.user_ids = member_ids
+        Groups.set_group_user_ids_by_id(group_id, member_ids)
 
     # Update group
     updated_group = Groups.update_group_by_id(group_id, update_form)
@@ -862,7 +867,6 @@ async def patch_group(
     update_form = GroupUpdateForm(
         name=group.name,
         description=group.description,
-        user_ids=group.user_ids.copy() if group.user_ids else [],
     )
 
     for operation in patch_data.Operations:
@@ -875,21 +879,22 @@ async def patch_group(
                 update_form.name = value
             elif path == "members":
                 # Replace all members
-                update_form.user_ids = [member["value"] for member in value]
+                Groups.set_group_user_ids_by_id(
+                    group_id, [member["value"] for member in value]
+                )
+
         elif op == "add":
             if path == "members":
                 # Add members
                 if isinstance(value, list):
                     for member in value:
                         if isinstance(member, dict) and "value" in member:
-                            if member["value"] not in update_form.user_ids:
-                                update_form.user_ids.append(member["value"])
+                            Groups.add_users_to_group(group_id, [member["value"]])
         elif op == "remove":
             if path and path.startswith("members[value eq"):
                 # Remove specific member
                 member_id = path.split('"')[1]
-                if member_id in update_form.user_ids:
-                    update_form.user_ids.remove(member_id)
+                Groups.remove_users_from_group(group_id, [member_id])
 
     # Update group
     updated_group = Groups.update_group_by_id(group_id, update_form)

backend/open_webui/routers/tasks.py

@@ -33,6 +33,7 @@ from open_webui.config import (
     DEFAULT_AUTOCOMPLETE_GENERATION_PROMPT_TEMPLATE,
     DEFAULT_EMOJI_GENERATION_PROMPT_TEMPLATE,
     DEFAULT_MOA_GENERATION_PROMPT_TEMPLATE,
+    DEFAULT_VOICE_MODE_PROMPT_TEMPLATE,
 )
 from open_webui.env import SRC_LOG_LEVELS
 
@@ -68,6 +69,7 @@ async def get_task_config(request: Request, user=Depends(get_verified_user)):
         "ENABLE_RETRIEVAL_QUERY_GENERATION": request.app.state.config.ENABLE_RETRIEVAL_QUERY_GENERATION,
         "QUERY_GENERATION_PROMPT_TEMPLATE": request.app.state.config.QUERY_GENERATION_PROMPT_TEMPLATE,
         "TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE": request.app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
+        "VOICE_MODE_PROMPT_TEMPLATE": request.app.state.config.VOICE_MODE_PROMPT_TEMPLATE,
     }
 
 
@@ -87,6 +89,7 @@ class TaskConfigForm(BaseModel):
     ENABLE_RETRIEVAL_QUERY_GENERATION: bool
     QUERY_GENERATION_PROMPT_TEMPLATE: str
     TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE: str
+    VOICE_MODE_PROMPT_TEMPLATE: Optional[str]
 
 
 @router.post("/config/update")
@@ -136,6 +139,10 @@ async def update_task_config(
         form_data.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE
     )
 
+    request.app.state.config.VOICE_MODE_PROMPT_TEMPLATE = (
+        form_data.VOICE_MODE_PROMPT_TEMPLATE
+    )
+
     return {
         "TASK_MODEL": request.app.state.config.TASK_MODEL,
         "TASK_MODEL_EXTERNAL": request.app.state.config.TASK_MODEL_EXTERNAL,
@@ -152,6 +159,7 @@ async def update_task_config(
         "ENABLE_RETRIEVAL_QUERY_GENERATION": request.app.state.config.ENABLE_RETRIEVAL_QUERY_GENERATION,
         "QUERY_GENERATION_PROMPT_TEMPLATE": request.app.state.config.QUERY_GENERATION_PROMPT_TEMPLATE,
         "TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE": request.app.state.config.TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
+        "VOICE_MODE_PROMPT_TEMPLATE": request.app.state.config.VOICE_MODE_PROMPT_TEMPLATE,
     }
 
 

backend/open_webui/routers/tools.py

@@ -247,9 +247,19 @@ async def load_tool_from_url(
 
 
 @router.get("/export", response_model=list[ToolModel])
-async def export_tools(user=Depends(get_admin_user)):
-    tools = Tools.get_tools()
-    return tools
+async def export_tools(request: Request, user=Depends(get_verified_user)):
+    if user.role != "admin" and not has_permission(
+        user.id, "workspace.tools_export", request.app.state.config.USER_PERMISSIONS
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=ERROR_MESSAGES.UNAUTHORIZED,
+        )
+
+    if user.role == "admin" and BYPASS_ADMIN_ACCESS_CONTROL:
+        return Tools.get_tools()
+    else:
+        return Tools.get_tools_by_user_id(user.id, "read")
 
 
 ############################
@@ -263,8 +273,13 @@ async def create_new_tools(
     form_data: ToolForm,
     user=Depends(get_verified_user),
 ):
-    if user.role != "admin" and not has_permission(
+    if user.role != "admin" and not (
+        has_permission(
             user.id, "workspace.tools", request.app.state.config.USER_PERMISSIONS
+        )
+        or has_permission(
+            user.id, "workspace.tools_import", request.app.state.config.USER_PERMISSIONS
+        )
     ):
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,

backend/open_webui/routers/users.py

@@ -6,7 +6,7 @@ import io
 
 from fastapi import APIRouter, Depends, HTTPException, Request, status
 from fastapi.responses import Response, StreamingResponse, FileResponse
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 from open_webui.models.auths import Auths
@@ -16,7 +16,8 @@ from open_webui.models.groups import Groups
 from open_webui.models.chats import Chats
 from open_webui.models.users import (
     UserModel,
-    UserListResponse,
+    UserGroupIdsModel,
+    UserGroupIdsListResponse,
     UserInfoListResponse,
     UserIdNameListResponse,
     UserRoleUpdateForm,
@@ -25,17 +26,16 @@ from open_webui.models.users import (
     UserUpdateForm,
 )
 
-
-from open_webui.socket.main import (
-    get_active_status_by_user_id,
-    get_active_user_ids,
-    get_user_active_status,
-)
 from open_webui.constants import ERROR_MESSAGES
 from open_webui.env import SRC_LOG_LEVELS, STATIC_DIR
 
 
-from open_webui.utils.auth import get_admin_user, get_password_hash, get_verified_user
+from open_webui.utils.auth import (
+    get_admin_user,
+    get_password_hash,
+    get_verified_user,
+    validate_password,
+)
 from open_webui.utils.access_control import get_permissions, has_permission
 
 
@@ -45,23 +45,6 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 router = APIRouter()
 
 
-############################
-# GetActiveUsers
-############################
-
-
-@router.get("/active")
-async def get_active_users(
-    user=Depends(get_verified_user),
-):
-    """
-    Get a list of active users.
-    """
-    return {
-        "user_ids": get_active_user_ids(),
-    }
-
-
 ############################
 # GetUsers
 ############################
@@ -70,7 +53,7 @@ async def get_active_users(
 PAGE_ITEM_COUNT = 30
 
 
-@router.get("/", response_model=UserListResponse)
+@router.get("/", response_model=UserGroupIdsListResponse)
 async def get_users(
     query: Optional[str] = None,
     order_by: Optional[str] = None,
@@ -91,7 +74,25 @@ async def get_users(
     if direction:
         filter["direction"] = direction
 
-    return Users.get_users(filter=filter, skip=skip, limit=limit)
+    result = Users.get_users(filter=filter, skip=skip, limit=limit)
+
+    users = result["users"]
+    total = result["total"]
+
+    return {
+        "users": [
+            UserGroupIdsModel(
+                **{
+                    **user.model_dump(),
+                    "group_ids": [
+                        group.id for group in Groups.get_groups_by_member_id(user.id)
+                    ],
+                }
+            )
+            for user in users
+        ],
+        "total": total,
+    }
 
 
 @router.get("/all", response_model=UserInfoListResponse)
@@ -150,13 +151,24 @@ class WorkspacePermissions(BaseModel):
     knowledge: bool = False
     prompts: bool = False
     tools: bool = False
+    models_import: bool = False
+    models_export: bool = False
+    prompts_import: bool = False
+    prompts_export: bool = False
+    tools_import: bool = False
+    tools_export: bool = False
 
 
 class SharingPermissions(BaseModel):
-    public_models: bool = True
-    public_knowledge: bool = True
-    public_prompts: bool = True
+    models: bool = False
+    public_models: bool = False
+    knowledge: bool = False
+    public_knowledge: bool = False
+    prompts: bool = False
+    public_prompts: bool = False
+    tools: bool = False
     public_tools: bool = True
+    notes: bool = False
     public_notes: bool = True
 
 
@@ -183,11 +195,14 @@ class ChatPermissions(BaseModel):
 
 
 class FeaturesPermissions(BaseModel):
+    api_keys: bool = False
+    folders: bool = True
+    notes: bool = True
     direct_tool_servers: bool = False
+
     web_search: bool = True
     image_generation: bool = True
     code_interpreter: bool = True
-    notes: bool = True
 
 
 class UserPermissions(BaseModel):
@@ -323,13 +338,14 @@ async def update_user_info_by_session_user(
 ############################
 
 
-class UserResponse(BaseModel):
+class UserActiveResponse(BaseModel):
     name: str
-    profile_image_url: str
-    active: Optional[bool] = None
+    profile_image_url: Optional[str] = None
+    is_active: bool
+    model_config = ConfigDict(extra="allow")
 
 
-@router.get("/{user_id}", response_model=UserResponse)
+@router.get("/{user_id}", response_model=UserActiveResponse)
 async def get_user_by_id(user_id: str, user=Depends(get_verified_user)):
     # Check if user_id is a shared chat
     # If it is, get the user_id from the chat
@@ -347,11 +363,11 @@ async def get_user_by_id(user_id: str, user=Depends(get_verified_user)):
     user = Users.get_user_by_id(user_id)
 
     if user:
-        return UserResponse(
+        return UserActiveResponse(
             **{
+                "id": user.id,
                 "name": user.name,
-                "profile_image_url": user.profile_image_url,
-                "active": get_active_status_by_user_id(user_id),
+                "is_active": Users.is_user_active(user_id),
             }
         )
     else:
@@ -361,7 +377,7 @@ async def get_user_by_id(user_id: str, user=Depends(get_verified_user)):
         )
 
 
-@router.get("/{user_id}/oauth/sessions", response_model=Optional[dict])
+@router.get("/{user_id}/oauth/sessions")
 async def get_user_oauth_sessions_by_id(user_id: str, user=Depends(get_admin_user)):
     sessions = OAuthSessions.get_sessions_by_user_id(user_id)
     if sessions and len(sessions) > 0:
@@ -418,7 +434,7 @@ async def get_user_profile_image_by_id(user_id: str, user=Depends(get_verified_u
 @router.get("/{user_id}/active", response_model=dict)
 async def get_user_active_status_by_id(user_id: str, user=Depends(get_verified_user)):
     return {
-        "active": get_user_active_status(user_id),
+        "active": Users.is_user_active(user_id),
     }
 
 
@@ -471,8 +487,12 @@ async def update_user_by_id(
                 )
 
         if form_data.password:
+            try:
+                validate_password(form_data.password)
+            except Exception as e:
+                raise HTTPException(400, detail=str(e))
+
             hashed = get_password_hash(form_data.password)
-            log.debug(f"hashed: {hashed}")
             Auths.update_user_password_by_id(user_id, hashed)
 
         Auths.update_email_by_id(user_id, form_data.email.lower())

backend/open_webui/routers/utils.py

@@ -124,12 +124,3 @@ async def download_db(user=Depends(get_admin_user)):
         media_type="application/octet-stream",
         filename="webui.db",
     )
-
-
-@router.get("/litellm/config")
-async def download_litellm_config_yaml(user=Depends(get_admin_user)):
-    return FileResponse(
-        f"{DATA_DIR}/litellm/config.yaml",
-        media_type="application/octet-stream",
-        filename="config.yaml",
-    )

backend/open_webui/socket/main.py

@@ -18,7 +18,12 @@ from open_webui.utils.redis import (
     get_sentinel_url_from_env,
 )
 
+from open_webui.config import (
+    CORS_ALLOW_ORIGIN,
+)
+
 from open_webui.env import (
+    VERSION,
     ENABLE_WEBSOCKET_SUPPORT,
     WEBSOCKET_MANAGER,
     WEBSOCKET_REDIS_URL,
@@ -27,6 +32,11 @@ from open_webui.env import (
     WEBSOCKET_SENTINEL_PORT,
     WEBSOCKET_SENTINEL_HOSTS,
     REDIS_KEY_PREFIX,
+    WEBSOCKET_REDIS_OPTIONS,
+    WEBSOCKET_SERVER_PING_TIMEOUT,
+    WEBSOCKET_SERVER_PING_INTERVAL,
+    WEBSOCKET_SERVER_LOGGING,
+    WEBSOCKET_SERVER_ENGINEIO_LOGGING,
 )
 from open_webui.utils.auth import decode_token
 from open_webui.socket.utils import RedisDict, RedisLock, YdocManager
@@ -48,30 +58,44 @@ log.setLevel(SRC_LOG_LEVELS["SOCKET"])
 
 REDIS = None
 
+# Configure CORS for Socket.IO
+SOCKETIO_CORS_ORIGINS = "*" if CORS_ALLOW_ORIGIN == ["*"] else CORS_ALLOW_ORIGIN
+
 if WEBSOCKET_MANAGER == "redis":
     if WEBSOCKET_SENTINEL_HOSTS:
         mgr = socketio.AsyncRedisManager(
             get_sentinel_url_from_env(
                 WEBSOCKET_REDIS_URL, WEBSOCKET_SENTINEL_HOSTS, WEBSOCKET_SENTINEL_PORT
-            )
+            ),
+            redis_options=WEBSOCKET_REDIS_OPTIONS,
         )
     else:
-        mgr = socketio.AsyncRedisManager(WEBSOCKET_REDIS_URL)
+        mgr = socketio.AsyncRedisManager(
+            WEBSOCKET_REDIS_URL, redis_options=WEBSOCKET_REDIS_OPTIONS
+        )
     sio = socketio.AsyncServer(
-        cors_allowed_origins=[],
+        cors_allowed_origins=SOCKETIO_CORS_ORIGINS,
         async_mode="asgi",
         transports=(["websocket"] if ENABLE_WEBSOCKET_SUPPORT else ["polling"]),
         allow_upgrades=ENABLE_WEBSOCKET_SUPPORT,
         always_connect=True,
         client_manager=mgr,
+        logger=WEBSOCKET_SERVER_LOGGING,
+        ping_interval=WEBSOCKET_SERVER_PING_INTERVAL,
+        ping_timeout=WEBSOCKET_SERVER_PING_TIMEOUT,
+        engineio_logger=WEBSOCKET_SERVER_ENGINEIO_LOGGING,
     )
 else:
     sio = socketio.AsyncServer(
-        cors_allowed_origins=[],
+        cors_allowed_origins=SOCKETIO_CORS_ORIGINS,
         async_mode="asgi",
         transports=(["websocket"] if ENABLE_WEBSOCKET_SUPPORT else ["polling"]),
         allow_upgrades=ENABLE_WEBSOCKET_SUPPORT,
         always_connect=True,
+        logger=WEBSOCKET_SERVER_LOGGING,
+        ping_interval=WEBSOCKET_SERVER_PING_INTERVAL,
+        ping_timeout=WEBSOCKET_SERVER_PING_TIMEOUT,
+        engineio_logger=WEBSOCKET_SERVER_ENGINEIO_LOGGING,
     )
 
 
@@ -94,14 +118,16 @@ if WEBSOCKET_MANAGER == "redis":
     redis_sentinels = get_sentinels_from_env(
         WEBSOCKET_SENTINEL_HOSTS, WEBSOCKET_SENTINEL_PORT
     )
-    SESSION_POOL = RedisDict(
-        f"{REDIS_KEY_PREFIX}:session_pool",
+
+    MODELS = RedisDict(
+        f"{REDIS_KEY_PREFIX}:models",
         redis_url=WEBSOCKET_REDIS_URL,
         redis_sentinels=redis_sentinels,
         redis_cluster=WEBSOCKET_REDIS_CLUSTER,
     )
-    USER_POOL = RedisDict(
-        f"{REDIS_KEY_PREFIX}:user_pool",
+
+    SESSION_POOL = RedisDict(
+        f"{REDIS_KEY_PREFIX}:session_pool",
         redis_url=WEBSOCKET_REDIS_URL,
         redis_sentinels=redis_sentinels,
         redis_cluster=WEBSOCKET_REDIS_CLUSTER,
@@ -124,8 +150,9 @@ if WEBSOCKET_MANAGER == "redis":
     renew_func = clean_up_lock.renew_lock
     release_func = clean_up_lock.release_lock
 else:
+    MODELS = {}
+
     SESSION_POOL = {}
-    USER_POOL = {}
     USAGE_POOL = {}
 
     aquire_func = release_func = renew_func = lambda: True
@@ -201,16 +228,6 @@ def get_models_in_use():
     return models_in_use
 
 
-def get_active_user_ids():
-    """Get the list of active user IDs."""
-    return list(USER_POOL.keys())
-
-
-def get_user_active_status(user_id):
-    """Check if a user is currently active."""
-    return user_id in USER_POOL
-
-
 def get_user_id_from_session_pool(sid):
     user = SESSION_POOL.get(sid)
     if user:
@@ -236,12 +253,6 @@ def get_user_ids_from_room(room):
     return active_user_ids
 
 
-def get_active_status_by_user_id(user_id):
-    if user_id in USER_POOL:
-        return True
-    return False
-
-
 @sio.on("usage")
 async def usage(sid, data):
     if sid in SESSION_POOL:
@@ -269,10 +280,7 @@ async def connect(sid, environ, auth):
             SESSION_POOL[sid] = user.model_dump(
                 exclude=["date_of_birth", "bio", "gender"]
             )
-            if user.id in USER_POOL:
-                USER_POOL[user.id] = USER_POOL[user.id] + [sid]
-            else:
-                USER_POOL[user.id] = [sid]
+            await sio.enter_room(sid, f"user:{user.id}")
 
 
 @sio.on("user-join")
@@ -290,12 +298,17 @@ async def user_join(sid, data):
     if not user:
         return
 
-    SESSION_POOL[sid] = user.model_dump(exclude=["date_of_birth", "bio", "gender"])
-    if user.id in USER_POOL:
-        USER_POOL[user.id] = USER_POOL[user.id] + [sid]
-    else:
-        USER_POOL[user.id] = [sid]
+    SESSION_POOL[sid] = user.model_dump(
+        exclude=[
+            "profile_image_url",
+            "profile_banner_image_url",
+            "date_of_birth",
+            "bio",
+            "gender",
+        ]
+    )
 
+    await sio.enter_room(sid, f"user:{user.id}")
     # Join all the channels
     channels = Channels.get_channels_by_user_id(user.id)
     log.debug(f"{channels=}")
@@ -304,6 +317,13 @@ async def user_join(sid, data):
     return {"id": user.id, "name": user.name}
 
 
+@sio.on("heartbeat")
+async def heartbeat(sid, data):
+    user = SESSION_POOL.get(sid)
+    if user:
+        Users.update_last_active_by_id(user["id"])
+
+
 @sio.on("join-channels")
 async def join_channel(sid, data):
     auth = data["auth"] if "auth" in data else None
@@ -371,6 +391,11 @@ async def channel_events(sid, data):
     event_data = data["data"]
     event_type = event_data["type"]
 
+    user = SESSION_POOL.get(sid)
+
+    if not user:
+        return
+
     if event_type == "typing":
         await sio.emit(
             "events:channel",
@@ -378,10 +403,12 @@ async def channel_events(sid, data):
                 "channel_id": data["channel_id"],
                 "message_id": data.get("message_id", None),
                 "data": event_data,
-                "user": UserNameResponse(**SESSION_POOL[sid]).model_dump(),
+                "user": UserNameResponse(**user).model_dump(),
             },
             room=room,
         )
+    elif event_type == "last_read_at":
+        Channels.update_member_last_read_at(data["channel_id"], user["id"])
 
 
 @sio.on("ydoc:document:join")
@@ -625,13 +652,6 @@ async def disconnect(sid):
     if sid in SESSION_POOL:
         user = SESSION_POOL[sid]
         del SESSION_POOL[sid]
-
-        user_id = user["id"]
-        USER_POOL[user_id] = [_sid for _sid in USER_POOL[user_id] if _sid != sid]
-
-        if len(USER_POOL[user_id]) == 0:
-            del USER_POOL[user_id]
-
         await YDOC_MANAGER.remove_user_from_all_documents(sid)
     else:
         pass
@@ -641,40 +661,24 @@ async def disconnect(sid):
 def get_event_emitter(request_info, update_db=True):
     async def __event_emitter__(event_data):
         user_id = request_info["user_id"]
+        chat_id = request_info["chat_id"]
+        message_id = request_info["message_id"]
 
-        session_ids = list(
-            set(
-                USER_POOL.get(user_id, [])
-                + (
-                    [request_info.get("session_id")]
-                    if request_info.get("session_id")
-                    else []
-                )
-            )
-        )
-
-        chat_id = request_info.get("chat_id", None)
-        message_id = request_info.get("message_id", None)
-
-        emit_tasks = [
-            sio.emit(
+        await sio.emit(
             "events",
             {
                 "chat_id": chat_id,
                 "message_id": message_id,
                 "data": event_data,
             },
-                to=session_id,
+            room=f"user:{user_id}",
         )
-            for session_id in session_ids
-        ]
-
-        await asyncio.gather(*emit_tasks)
         if (
             update_db
             and message_id
             and not request_info.get("chat_id", "").startswith("local:")
         ):
+
             if "type" in event_data and event_data["type"] == "status":
                 Chats.add_message_status_to_chat_by_id_and_message_id(
                     request_info["chat_id"],
@@ -764,7 +768,14 @@ def get_event_emitter(request_info, update_db=True):
                         },
                     )
 
+    if (
+        "user_id" in request_info
+        and "chat_id" in request_info
+        and "message_id" in request_info
+    ):
         return __event_emitter__
+    else:
+        return None
 
 
 def get_event_call(request_info):
@@ -780,7 +791,14 @@ def get_event_call(request_info):
         )
         return response
 
+    if (
+        "session_id" in request_info
+        and "chat_id" in request_info
+        and "message_id" in request_info
+    ):
         return __event_caller__
+    else:
+        return None
 
 
 get_event_caller = get_event_call

backend/open_webui/socket/utils.py

@@ -86,6 +86,15 @@ class RedisDict:
     def items(self):
         return [(k, json.loads(v)) for k, v in self.redis.hgetall(self.name).items()]
 
+    def set(self, mapping: dict):
+        pipe = self.redis.pipeline()
+
+        pipe.delete(self.name)
+        if mapping:
+            pipe.hset(self.name, mapping={k: json.dumps(v) for k, v in mapping.items()})
+
+        pipe.execute()
+
     def get(self, key, default=None):
         try:
             return self[key]

backend/open_webui/utils/access_control.py

@@ -105,6 +105,22 @@ def has_permission(
     return get_permission(default_permissions, permission_hierarchy)
 
 
+def get_permitted_group_and_user_ids(
+    type: str = "write", access_control: Optional[dict] = None
+) -> Union[Dict[str, List[str]], None]:
+    if access_control is None:
+        return None
+
+    permission_access = access_control.get(type, {})
+    permitted_group_ids = permission_access.get("group_ids", [])
+    permitted_user_ids = permission_access.get("user_ids", [])
+
+    return {
+        "group_ids": permitted_group_ids,
+        "user_ids": permitted_user_ids,
+    }
+
+
 def has_access(
     user_id: str,
     type: str = "write",
@@ -122,9 +138,12 @@ def has_access(
         user_groups = Groups.get_groups_by_member_id(user_id)
         user_group_ids = {group.id for group in user_groups}
 
-    permission_access = access_control.get(type, {})
-    permitted_group_ids = permission_access.get("group_ids", [])
-    permitted_user_ids = permission_access.get("user_ids", [])
+    permitted_ids = get_permitted_group_and_user_ids(type, access_control)
+    if permitted_ids is None:
+        return False
+
+    permitted_group_ids = permitted_ids.get("group_ids", [])
+    permitted_user_ids = permitted_ids.get("user_ids", [])
 
     return user_id in permitted_user_ids or any(
         group_id in permitted_group_ids for group_id in user_group_ids
@@ -136,18 +155,20 @@ def get_users_with_access(
     type: str = "write", access_control: Optional[dict] = None
 ) -> list[UserModel]:
     if access_control is None:
-        result = Users.get_users()
+        result = Users.get_users(filter={"roles": ["!pending"]})
         return result.get("users", [])
 
-    permission_access = access_control.get(type, {})
-    permitted_group_ids = permission_access.get("group_ids", [])
-    permitted_user_ids = permission_access.get("user_ids", [])
+    permitted_ids = get_permitted_group_and_user_ids(type, access_control)
+    if permitted_ids is None:
+        return []
+
+    permitted_group_ids = permitted_ids.get("group_ids", [])
+    permitted_user_ids = permitted_ids.get("user_ids", [])
 
     user_ids_with_access = set(permitted_user_ids)
 
-    for group_id in permitted_group_ids:
-        group_user_ids = Groups.get_group_user_ids_by_id(group_id)
-        if group_user_ids:
-            user_ids_with_access.update(group_user_ids)
+    group_user_ids_map = Groups.get_group_user_ids_by_ids(permitted_group_ids)
+    for user_ids in group_user_ids_map.values():
+        user_ids_with_access.update(user_ids)
 
     return Users.get_users_by_user_ids(list(user_ids_with_access))

backend/open_webui/utils/auth.py

@@ -21,13 +21,18 @@ from typing import Optional, Union, List, Dict
 
 from opentelemetry import trace
 
+
+from open_webui.utils.access_control import has_permission
 from open_webui.models.users import Users
 
 from open_webui.constants import ERROR_MESSAGES
 
 from open_webui.env import (
+    ENABLE_PASSWORD_VALIDATION,
     OFFLINE_MODE,
     LICENSE_BLOB,
+    PASSWORD_VALIDATION_REGEX_PATTERN,
+    REDIS_KEY_PREFIX,
     pk,
     WEBUI_SECRET_KEY,
     TRUSTED_SIGNATURE_KEY,
@@ -159,6 +164,20 @@ def get_password_hash(password: str) -> str:
     return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8")
 
 
+def validate_password(password: str) -> bool:
+    # The password passed to bcrypt must be 72 bytes or fewer. If it is longer, it will be truncated before hashing.
+    if len(password.encode("utf-8")) > 72:
+        raise Exception(
+            ERROR_MESSAGES.PASSWORD_TOO_LONG,
+        )
+
+    if ENABLE_PASSWORD_VALIDATION:
+        if not PASSWORD_VALIDATION_REGEX_PATTERN.match(password):
+            raise Exception(ERROR_MESSAGES.INVALID_PASSWORD())
+
+    return True
+
+
 def verify_password(plain_password: str, hashed_password: str) -> bool:
     """Verify a password against its hash"""
     return (
@@ -178,6 +197,9 @@ def create_token(data: dict, expires_delta: Union[timedelta, None] = None) -> st
         expire = datetime.now(UTC) + expires_delta
         payload.update({"exp": expire})
 
+    jti = str(uuid.uuid4())
+    payload.update({"jti": jti})
+
     encoded_jwt = jwt.encode(payload, SESSION_SECRET, algorithm=ALGORITHM)
     return encoded_jwt
 
@@ -190,6 +212,43 @@ def decode_token(token: str) -> Optional[dict]:
         return None
 
 
+async def is_valid_token(request, decoded) -> bool:
+    # Require Redis to check revoked tokens
+    if request.app.state.redis:
+        jti = decoded.get("jti")
+
+        if jti:
+            revoked = await request.app.state.redis.get(
+                f"{REDIS_KEY_PREFIX}:auth:token:{jti}:revoked"
+            )
+            if revoked:
+                return False
+
+    return True
+
+
+async def invalidate_token(request, token):
+    decoded = decode_token(token)
+
+    # Require Redis to store revoked tokens
+    if request.app.state.redis:
+        jti = decoded.get("jti")
+        exp = decoded.get("exp")
+
+        if jti:
+            ttl = exp - int(
+                datetime.now(UTC).timestamp()
+            )  # Calculate time-to-live for the token
+
+            if ttl > 0:
+                # Store the revoked token in Redis with an expiration time
+                await request.app.state.redis.set(
+                    f"{REDIS_KEY_PREFIX}:auth:token:{jti}:revoked",
+                    "1",
+                    ex=ttl,
+                )
+
+
 def extract_token_from_auth_header(auth_header: str):
     return auth_header[len("Bearer ") :]
 
@@ -209,7 +268,7 @@ def get_http_authorization_cred(auth_header: Optional[str]):
         return None
 
 
-def get_current_user(
+async def get_current_user(
     request: Request,
     response: Response,
     background_tasks: BackgroundTasks,
@@ -228,30 +287,7 @@ def get_current_user(
 
     # auth by api key
     if token.startswith("sk-"):
-        if not request.state.enable_api_key:
-            raise HTTPException(
-                status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.API_KEY_NOT_ALLOWED
-            )
-
-        if request.app.state.config.ENABLE_API_KEY_ENDPOINT_RESTRICTIONS:
-            allowed_paths = [
-                path.strip()
-                for path in str(
-                    request.app.state.config.API_KEY_ALLOWED_ENDPOINTS
-                ).split(",")
-            ]
-
-            # Check if the request path matches any allowed endpoint.
-            if not any(
-                request.url.path == allowed
-                or request.url.path.startswith(allowed + "/")
-                for allowed in allowed_paths
-            ):
-                raise HTTPException(
-                    status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.API_KEY_NOT_ALLOWED
-                )
-
-        user = get_current_user_by_api_key(token)
+        user = get_current_user_by_api_key(request, token)
 
         # Add user info to current span
         current_span = trace.get_current_span()
@@ -264,7 +300,6 @@ def get_current_user(
         return user
 
     # auth by jwt token
-
     try:
         try:
             data = decode_token(token)
@@ -275,6 +310,12 @@ def get_current_user(
             )
 
         if data is not None and "id" in data:
+            if data.get("jti") and not await is_valid_token(request, data):
+                raise HTTPException(
+                    status_code=status.HTTP_401_UNAUTHORIZED,
+                    detail="Invalid token",
+                )
+
             user = Users.get_user_by_id(data["id"])
             if user is None:
                 raise HTTPException(
@@ -303,9 +344,7 @@ def get_current_user(
                 # Refresh the user's last active timestamp asynchronously
                 # to prevent blocking the request
                 if background_tasks:
-                    background_tasks.add_task(
-                        Users.update_user_last_active_by_id, user.id
-                    )
+                    background_tasks.add_task(Users.update_last_active_by_id, user.id)
             return user
         else:
             raise HTTPException(
@@ -327,7 +366,7 @@ def get_current_user(
         raise e
 
 
-def get_current_user_by_api_key(api_key: str):
+def get_current_user_by_api_key(request, api_key: str):
     user = Users.get_user_by_api_key(api_key)
 
     if user is None:
@@ -335,7 +374,19 @@ def get_current_user_by_api_key(api_key: str):
             status_code=status.HTTP_401_UNAUTHORIZED,
             detail=ERROR_MESSAGES.INVALID_TOKEN,
         )
-    else:
+
+    if not request.state.enable_api_keys or (
+        user.role != "admin"
+        and not has_permission(
+            user.id,
+            "features.api_keys",
+            request.app.state.config.USER_PERMISSIONS,
+        )
+    ):
+        raise HTTPException(
+            status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.API_KEY_NOT_ALLOWED
+        )
+
     # Add user info to current span
     current_span = trace.get_current_span()
     if current_span:
@@ -344,8 +395,7 @@ def get_current_user_by_api_key(api_key: str):
         current_span.set_attribute("client.user.role", user.role)
         current_span.set_attribute("client.auth.type", "api_key")
 
-        Users.update_user_last_active_by_id(user.id)
-
+    Users.update_last_active_by_id(user.id)
     return user
 
 

backend/open_webui/utils/files.py

@@ -1,5 +1,5 @@
 from open_webui.routers.images import (
-    load_b64_image_data,
+    get_image_data,
     upload_image,
 )
 
@@ -16,13 +16,18 @@ from open_webui.routers.files import upload_file_handler
 import mimetypes
 import base64
 import io
+import re
+
+
+BASE64_IMAGE_URL_PREFIX = re.compile(r"data:image/\w+;base64,", re.IGNORECASE)
+MARKDOWN_IMAGE_URL_PATTERN = re.compile(r"!\[(.*?)\]\((.+?)\)", re.IGNORECASE)
 
 
 def get_image_url_from_base64(request, base64_image_string, metadata, user):
-    if "data:image/png;base64" in base64_image_string:
+    if BASE64_IMAGE_URL_PREFIX.match(base64_image_string):
         image_url = ""
         # Extract base64 image data from the line
-        image_data, content_type = load_b64_image_data(base64_image_string)
+        image_data, content_type = get_image_data(base64_image_string)
         if image_data is not None:
             image_url = upload_image(
                 request,
@@ -35,6 +40,19 @@ def get_image_url_from_base64(request, base64_image_string, metadata, user):
     return None
 
 
+def convert_markdown_base64_images(request, content: str, metadata, user):
+    def replace(match):
+        base64_string = match.group(2)
+        MIN_REPLACEMENT_URL_LENGTH = 1024
+        if len(base64_string) > MIN_REPLACEMENT_URL_LENGTH:
+            url = get_image_url_from_base64(request, base64_string, metadata, user)
+            if url:
+                return f"![{match.group(1)}]({url})"
+        return match.group(0)
+
+    return MARKDOWN_IMAGE_URL_PATTERN.sub(replace, content)
+
+
 def load_b64_audio_data(b64_str):
     try:
         if "," in b64_str:

backend/open_webui/utils/headers.py

@@ -0,0 +1,11 @@
+from urllib.parse import quote
+
+
+def include_user_info_headers(headers, user):
+    return {
+        **headers,
+        "X-OpenWebUI-User-Name": quote(user.name, safe=" "),
+        "X-OpenWebUI-User-Id": user.id,
+        "X-OpenWebUI-User-Email": user.email,
+        "X-OpenWebUI-User-Role": user.role,
+    }

backend/open_webui/utils/images/comfyui.py

@@ -2,6 +2,8 @@ import asyncio
 import json
 import logging
 import random
+import requests
+import aiohttp
 import urllib.parse
 import urllib.request
 from typing import Optional
@@ -91,6 +93,25 @@ def get_images(ws, prompt, client_id, base_url, api_key):
     return {"data": output_images}
 
 
+async def comfyui_upload_image(image_file_item, base_url, api_key):
+    url = f"{base_url}/api/upload/image"
+    headers = {}
+
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    _, (filename, file_bytes, mime_type) = image_file_item
+
+    form = aiohttp.FormData()
+    form.add_field("image", file_bytes, filename=filename, content_type=mime_type)
+    form.add_field("type", "input")  # required by ComfyUI
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, data=form, headers=headers) as resp:
+            resp.raise_for_status()
+            return await resp.json()
+
+
 class ComfyUINodeInput(BaseModel):
     type: Optional[str] = None
     node_ids: list[str] = []
@@ -103,7 +124,7 @@ class ComfyUIWorkflow(BaseModel):
     nodes: list[ComfyUINodeInput]
 
 
-class ComfyUIGenerateImageForm(BaseModel):
+class ComfyUICreateImageForm(BaseModel):
     workflow: ComfyUIWorkflow
 
     prompt: str
@@ -116,8 +137,8 @@ class ComfyUIGenerateImageForm(BaseModel):
     seed: Optional[int] = None
 
 
-async def comfyui_generate_image(
-    model: str, payload: ComfyUIGenerateImageForm, client_id, base_url, api_key
+async def comfyui_create_image(
+    model: str, payload: ComfyUICreateImageForm, client_id, base_url, api_key
 ):
     ws_url = base_url.replace("http://", "ws://").replace("https://", "wss://")
     workflow = json.loads(payload.workflow.workflow)
@@ -191,3 +212,102 @@ async def comfyui_generate_image(
     ws.close()
 
     return images
+
+
+class ComfyUIEditImageForm(BaseModel):
+    workflow: ComfyUIWorkflow
+
+    image: str | list[str]
+    prompt: str
+    width: Optional[int] = None
+    height: Optional[int] = None
+    n: Optional[int] = None
+
+    steps: Optional[int] = None
+    seed: Optional[int] = None
+
+
+async def comfyui_edit_image(
+    model: str, payload: ComfyUIEditImageForm, client_id, base_url, api_key
+):
+    ws_url = base_url.replace("http://", "ws://").replace("https://", "wss://")
+    workflow = json.loads(payload.workflow.workflow)
+
+    for node in payload.workflow.nodes:
+        if node.type:
+            if node.type == "model":
+                for node_id in node.node_ids:
+                    workflow[node_id]["inputs"][node.key] = model
+            elif node.type == "image":
+                if isinstance(payload.image, list):
+                    # check if multiple images are provided
+                    for idx, node_id in enumerate(node.node_ids):
+                        if idx < len(payload.image):
+                            workflow[node_id]["inputs"][node.key] = payload.image[idx]
+                else:
+                    for node_id in node.node_ids:
+                        workflow[node_id]["inputs"][node.key] = payload.image
+            elif node.type == "prompt":
+                for node_id in node.node_ids:
+                    workflow[node_id]["inputs"][
+                        node.key if node.key else "text"
+                    ] = payload.prompt
+            elif node.type == "negative_prompt":
+                for node_id in node.node_ids:
+                    workflow[node_id]["inputs"][
+                        node.key if node.key else "text"
+                    ] = payload.negative_prompt
+            elif node.type == "width":
+                for node_id in node.node_ids:
+                    workflow[node_id]["inputs"][
+                        node.key if node.key else "width"
+                    ] = payload.width
+            elif node.type == "height":
+                for node_id in node.node_ids:
+                    workflow[node_id]["inputs"][
+                        node.key if node.key else "height"
+                    ] = payload.height
+            elif node.type == "n":
+                for node_id in node.node_ids:
+                    workflow[node_id]["inputs"][
+                        node.key if node.key else "batch_size"
+                    ] = payload.n
+            elif node.type == "steps":
+                for node_id in node.node_ids:
+                    workflow[node_id]["inputs"][
+                        node.key if node.key else "steps"
+                    ] = payload.steps
+            elif node.type == "seed":
+                seed = (
+                    payload.seed
+                    if payload.seed
+                    else random.randint(0, 1125899906842624)
+                )
+                for node_id in node.node_ids:
+                    workflow[node_id]["inputs"][node.key] = seed
+        else:
+            for node_id in node.node_ids:
+                workflow[node_id]["inputs"][node.key] = node.value
+
+    try:
+        ws = websocket.WebSocket()
+        headers = {"Authorization": f"Bearer {api_key}"}
+        ws.connect(f"{ws_url}/ws?clientId={client_id}", header=headers)
+        log.info("WebSocket connection established.")
+    except Exception as e:
+        log.exception(f"Failed to connect to WebSocket server: {e}")
+        return None
+
+    try:
+        log.info("Sending workflow to WebSocket server.")
+        log.info(f"Workflow: {workflow}")
+        images = await asyncio.to_thread(
+            get_images, ws, workflow, client_id, base_url, api_key
+        )
+    except Exception as e:
+        log.exception(f"Error while receiving images: {e}")
+        images = None
+
+    ws.close()
+
+    return images

backend/open_webui/utils/mcp/client.py

@@ -2,6 +2,8 @@ import asyncio
 from typing import Optional
 from contextlib import AsyncExitStack
 
+import anyio
+
 from mcp import ClientSession
 from mcp.client.auth import OAuthClientProvider, TokenStorage
 from mcp.client.streamable_http import streamablehttp_client
@@ -11,25 +13,28 @@ from mcp.shared.auth import OAuthClientInformationFull, OAuthClientMetadata, OAu
 class MCPClient:
     def __init__(self):
         self.session: Optional[ClientSession] = None
-        self.exit_stack = AsyncExitStack()
+        self.exit_stack = None
 
     async def connect(self, url: str, headers: Optional[dict] = None):
+        async with AsyncExitStack() as exit_stack:
             try:
                 self._streams_context = streamablehttp_client(url, headers=headers)
 
-            transport = await self.exit_stack.enter_async_context(self._streams_context)
+                transport = await exit_stack.enter_async_context(self._streams_context)
                 read_stream, write_stream, _ = transport
 
                 self._session_context = ClientSession(
                     read_stream, write_stream
                 )  # pylint: disable=W0201
 
-            self.session = await self.exit_stack.enter_async_context(
+                self.session = await exit_stack.enter_async_context(
                     self._session_context
                 )
+                with anyio.fail_after(10):
                     await self.session.initialize()
+                self.exit_stack = exit_stack.pop_all()
             except Exception as e:
-            await self.disconnect()
+                await asyncio.shield(self.disconnect())
                 raise e
 
     async def list_tool_specs(self) -> Optional[dict]:

backend/open_webui/utils/middleware.py

@@ -24,6 +24,7 @@ from fastapi.responses import HTMLResponse
 from starlette.responses import Response, StreamingResponse, JSONResponse
 
 
+from open_webui.utils.misc import is_string_allowed
 from open_webui.models.oauth_sessions import OAuthSessions
 from open_webui.models.chats import Chats
 from open_webui.models.folders import Folders
@@ -31,7 +32,6 @@ from open_webui.models.users import Users
 from open_webui.socket.main import (
     get_event_call,
     get_event_emitter,
-    get_active_status_by_user_id,
 )
 from open_webui.routers.tasks import (
     generate_queries,
@@ -40,12 +40,15 @@ from open_webui.routers.tasks import (
     generate_image_prompt,
     generate_chat_tags,
 )
-from open_webui.routers.retrieval import process_web_search, SearchForm
+from open_webui.routers.retrieval import (
+    process_web_search,
+    SearchForm,
+)
 from open_webui.routers.images import (
-    load_b64_image_data,
     image_generations,
-    GenerateImageForm,
-    upload_image,
+    CreateImageForm,
+    image_edits,
+    EditImageForm,
 )
 from open_webui.routers.pipelines import (
     process_pipeline_inlet_filter,
@@ -55,7 +58,7 @@ from open_webui.routers.memories import query_memory, QueryMemoryForm
 
 from open_webui.utils.webhook import post_webhook
 from open_webui.utils.files import (
-    get_audio_url_from_base64,
+    convert_markdown_base64_images,
     get_file_url_from_base64,
     get_image_url_from_base64,
 )
@@ -76,6 +79,7 @@ from open_webui.utils.task import (
 )
 from open_webui.utils.misc import (
     deep_update,
+    extract_urls,
     get_message_list,
     add_or_update_system_message,
     add_or_update_user_message,
@@ -87,7 +91,7 @@ from open_webui.utils.misc import (
     convert_logit_bias_input_to_json,
     get_content_from_message,
 )
-from open_webui.utils.tools import get_tools
+from open_webui.utils.tools import get_tools, get_updated_tool_function
 from open_webui.utils.plugin import load_function_module_by_id
 from open_webui.utils.filter import (
     get_sorted_filter_ids,
@@ -100,6 +104,7 @@ from open_webui.utils.mcp.client import MCPClient
 
 from open_webui.config import (
     CACHE_DIR,
+    DEFAULT_VOICE_MODE_PROMPT_TEMPLATE,
     DEFAULT_TOOLS_FUNCTION_CALLING_PROMPT_TEMPLATE,
     DEFAULT_CODE_INTERPRETER_PROMPT,
     CODE_INTERPRETER_BLOCKED_MODULES,
@@ -107,6 +112,7 @@ from open_webui.config import (
 from open_webui.env import (
     SRC_LOG_LEVELS,
     GLOBAL_LOG_LEVEL,
+    ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION,
     CHAT_RESPONSE_STREAM_DELTA_CHUNK_SIZE,
     CHAT_RESPONSE_MAX_TOOL_CALL_RETRIES,
     BYPASS_MODEL_ACCESS_CONTROL,
@@ -298,19 +304,27 @@ async def chat_completion_tools_handler(
     def get_tools_function_calling_payload(messages, task_model_id, content):
         user_message = get_last_user_message(messages)
 
+        if user_message and messages and messages[-1]["role"] == "user":
+            # Remove the last user message to avoid duplication
+            messages = messages[:-1]
+
         recent_messages = messages[-4:] if len(messages) > 4 else messages
         chat_history = "\n".join(
             f"{message['role'].upper()}: \"\"\"{get_content_from_message(message)}\"\"\""
             for message in recent_messages
         )
 
-        prompt = f"History:\n{chat_history}\nQuery: {user_message}"
+        prompt = (
+            f"History:\n{chat_history}\nQuery: {user_message}"
+            if chat_history
+            else f"Query: {user_message}"
+        )
 
         return {
             "model": task_model_id,
             "messages": [
                 {"role": "system", "content": content},
-                {"role": "user", "content": f"Query: {prompt}"},
+                {"role": "user", "content": prompt},
             ],
             "stream": False,
             "metadata": {"task": str(TASKS.FUNCTION_CALLING)},
@@ -714,10 +728,56 @@ async def chat_web_search_handler(
     return form_data
 
 
+def get_last_images(message_list):
+    images = []
+    for message in reversed(message_list):
+        images_flag = False
+        for file in message.get("files", []):
+            if file.get("type") == "image":
+                images.append(file.get("url"))
+                images_flag = True
+
+        if images_flag:
+            break
+
+    return images
+
+
+def get_image_urls(delta_images, request, metadata, user) -> list[str]:
+    if not isinstance(delta_images, list):
+        return []
+
+    image_urls = []
+    for img in delta_images:
+        if not isinstance(img, dict) or img.get("type") != "image_url":
+            continue
+
+        url = img.get("image_url", {}).get("url")
+        if not url:
+            continue
+
+        if url.startswith("data:image/png;base64"):
+            url = get_image_url_from_base64(request, url, metadata, user)
+
+        image_urls.append(url)
+
+    return image_urls
+
+
 async def chat_image_generation_handler(
     request: Request, form_data: dict, extra_params: dict, user
 ):
+    metadata = extra_params.get("__metadata__", {})
+    chat_id = metadata.get("chat_id", None)
+    if not chat_id:
+        return form_data
+
     __event_emitter__ = extra_params["__event_emitter__"]
+
+    if chat_id.startswith("local:"):
+        message_list = form_data.get("messages", [])
+    else:
+        chat = Chats.get_chat_by_id_and_user_id(chat_id, user.id)
         await __event_emitter__(
             {
                 "type": "status",
@@ -725,48 +785,23 @@ async def chat_image_generation_handler(
             }
         )
 
-    messages = form_data["messages"]
-    user_message = get_last_user_message(messages)
+        messages_map = chat.chat.get("history", {}).get("messages", {})
+        message_id = chat.chat.get("history", {}).get("currentId")
+        message_list = get_message_list(messages_map, message_id)
+
+    user_message = get_last_user_message(message_list)
 
     prompt = user_message
-    negative_prompt = ""
-
-    if request.app.state.config.ENABLE_IMAGE_PROMPT_GENERATION:
-        try:
-            res = await generate_image_prompt(
-                request,
-                {
-                    "model": form_data["model"],
-                    "messages": messages,
-                },
-                user,
-            )
-
-            response = res["choices"][0]["message"]["content"]
-
-            try:
-                bracket_start = response.find("{")
-                bracket_end = response.rfind("}") + 1
-
-                if bracket_start == -1 or bracket_end == -1:
-                    raise Exception("No JSON object found in the response")
-
-                response = response[bracket_start:bracket_end]
-                response = json.loads(response)
-                prompt = response.get("prompt", [])
-            except Exception as e:
-                prompt = user_message
-
-        except Exception as e:
-            log.exception(e)
-            prompt = user_message
+    input_images = get_last_images(message_list)
 
     system_message_content = ""
 
+    if len(input_images) > 0 and request.app.state.config.ENABLE_IMAGE_EDIT:
+        # Edit image(s)
         try:
-        images = await image_generations(
+            images = await image_edits(
                 request=request,
-            form_data=GenerateImageForm(**{"prompt": prompt}),
+                form_data=EditImageForm(**{"prompt": prompt, "image": input_images}),
                 user=user,
             )
 
@@ -792,9 +827,17 @@ async def chat_image_generation_handler(
                 }
             )
 
-        system_message_content = "<context>User is shown the generated image, tell the user that the image has been generated</context>"
+            system_message_content = "<context>The requested image has been created and is now being shown to the user. Let them know that it has been generated.</context>"
         except Exception as e:
-        log.exception(e)
+            log.debug(e)
+
+            error_message = ""
+            if isinstance(e, HTTPException):
+                if e.detail and isinstance(e.detail, dict):
+                    error_message = e.detail.get("message", str(e.detail))
+                else:
+                    error_message = str(e.detail)
+
             await __event_emitter__(
                 {
                     "type": "status",
@@ -805,7 +848,91 @@ async def chat_image_generation_handler(
                 }
             )
 
-        system_message_content = "<context>Unable to generate an image, tell the user that an error occurred</context>"
+            system_message_content = f"<context>Image generation was attempted but failed. The system is currently unable to generate the image. Tell the user that the following error occurred: {error_message}</context>"
+
+    else:
+        # Create image(s)
+        if request.app.state.config.ENABLE_IMAGE_PROMPT_GENERATION:
+            try:
+                res = await generate_image_prompt(
+                    request,
+                    {
+                        "model": form_data["model"],
+                        "messages": form_data["messages"],
+                    },
+                    user,
+                )
+
+                response = res["choices"][0]["message"]["content"]
+
+                try:
+                    bracket_start = response.find("{")
+                    bracket_end = response.rfind("}") + 1
+
+                    if bracket_start == -1 or bracket_end == -1:
+                        raise Exception("No JSON object found in the response")
+
+                    response = response[bracket_start:bracket_end]
+                    response = json.loads(response)
+                    prompt = response.get("prompt", [])
+                except Exception as e:
+                    prompt = user_message
+
+            except Exception as e:
+                log.exception(e)
+                prompt = user_message
+
+        try:
+            images = await image_generations(
+                request=request,
+                form_data=CreateImageForm(**{"prompt": prompt}),
+                user=user,
+            )
+
+            await __event_emitter__(
+                {
+                    "type": "status",
+                    "data": {"description": "Image created", "done": True},
+                }
+            )
+
+            await __event_emitter__(
+                {
+                    "type": "files",
+                    "data": {
+                        "files": [
+                            {
+                                "type": "image",
+                                "url": image["url"],
+                            }
+                            for image in images
+                        ]
+                    },
+                }
+            )
+
+            system_message_content = "<context>The requested image has been created by the system successfully and is now being shown to the user. Let the user know that the image they requested has been generated and is now shown in the chat.</context>"
+        except Exception as e:
+            log.debug(e)
+
+            error_message = ""
+            if isinstance(e, HTTPException):
+                if e.detail and isinstance(e.detail, dict):
+                    error_message = e.detail.get("message", str(e.detail))
+                else:
+                    error_message = str(e.detail)
+
+            await __event_emitter__(
+                {
+                    "type": "status",
+                    "data": {
+                        "description": f"An error occurred while generating an image",
+                        "done": True,
+                    },
+                }
+            )
+
+            system_message_content = f"<context>Image generation was attempted but failed because of an error. The system is currently unable to generate the image. Tell the user that the following error occurred: {error_message}</context>"
 
     if system_message_content:
         form_data["messages"] = add_or_update_system_message(
@@ -855,10 +982,6 @@ async def chat_completion_files_handler(
             except:
                 pass
 
-        if len(queries) == 0:
-            queries = [get_last_user_message(body["messages"])]
-
-        if not all_full_context:
             await __event_emitter__(
                 {
                     "type": "status",
@@ -870,13 +993,12 @@ async def chat_completion_files_handler(
                 }
             )
 
+        if len(queries) == 0:
+            queries = [get_last_user_message(body["messages"])]
+
         try:
-            # Offload get_sources_from_items to a separate thread
-            loop = asyncio.get_running_loop()
-            with ThreadPoolExecutor() as executor:
-                sources = await loop.run_in_executor(
-                    executor,
-                    lambda: get_sources_from_items(
+            # Directly await async get_sources_from_items (no thread needed - fully async now)
+            sources = await get_sources_from_items(
                 request=request,
                 items=files,
                 queries=queries,
@@ -886,8 +1008,8 @@ async def chat_completion_files_handler(
                 k=request.app.state.config.TOP_K,
                 reranking_function=(
                     (
-                                lambda sentences: request.app.state.RERANKING_FUNCTION(
-                                    sentences, user=user
+                        lambda query, documents: request.app.state.RERANKING_FUNCTION(
+                            query, documents, user=user
                         )
                     )
                     if request.app.state.RERANKING_FUNCTION
@@ -900,7 +1022,6 @@ async def chat_completion_files_handler(
                 full_context=all_full_context
                 or request.app.state.config.RAG_FULL_CONTEXT,
                 user=user,
-                    ),
             )
         except Exception as e:
             log.exception(e)
@@ -908,7 +1029,6 @@ async def chat_completion_files_handler(
         log.debug(f"rag_contexts:sources: {sources}")
 
         unique_ids = set()
-
         for source in sources or []:
             if not source or len(source.keys()) == 0:
                 continue
@@ -927,7 +1047,6 @@ async def chat_completion_files_handler(
                 unique_ids.add(_id)
 
         sources_count = len(unique_ids)
-
         await __event_emitter__(
             {
                 "type": "status",
@@ -1001,16 +1120,16 @@ async def process_chat_payload(request, form_data, user, metadata, model):
     log.debug(f"form_data: {form_data}")
 
     system_message = get_system_message(form_data.get("messages", []))
-    if system_message:
+    if system_message:  # Chat Controls/User Settings
         try:
             form_data = apply_system_prompt_to_body(
                 system_message.get("content"), form_data, metadata, user, replace=True
-            )
+            )  # Required to handle system prompt variables
         except:
             pass
 
     event_emitter = get_event_emitter(metadata)
-    event_call = get_event_call(metadata)
+    event_caller = get_event_call(metadata)
 
     oauth_token = None
     try:
@@ -1024,14 +1143,13 @@ async def process_chat_payload(request, form_data, user, metadata, model):
 
     extra_params = {
         "__event_emitter__": event_emitter,
-        "__event_call__": event_call,
+        "__event_call__": event_caller,
         "__user__": user.model_dump() if isinstance(user, UserModel) else {},
         "__metadata__": metadata,
+        "__oauth_token__": oauth_token,
         "__request__": request,
         "__model__": model,
-        "__oauth_token__": oauth_token,
     }
-
     # Initialize events to store additional event to be sent to the client
     # Initialize contexts and citation
     if getattr(request.state, "direct", False) and hasattr(request.state, "model"):
@@ -1142,6 +1260,18 @@ async def process_chat_payload(request, form_data, user, metadata, model):
 
     features = form_data.pop("features", None)
     if features:
+        if "voice" in features and features["voice"]:
+            if request.app.state.config.VOICE_MODE_PROMPT_TEMPLATE != None:
+                if request.app.state.config.VOICE_MODE_PROMPT_TEMPLATE != "":
+                    template = request.app.state.config.VOICE_MODE_PROMPT_TEMPLATE
+                else:
+                    template = DEFAULT_VOICE_MODE_PROMPT_TEMPLATE
+
+                form_data["messages"] = add_or_update_system_message(
+                    template,
+                    form_data["messages"],
+                )
+
         if "memory" in features and features["memory"]:
             form_data = await chat_memory_handler(
                 request, form_data, extra_params, user
@@ -1170,8 +1300,28 @@ async def process_chat_payload(request, form_data, user, metadata, model):
     tool_ids = form_data.pop("tool_ids", None)
     files = form_data.pop("files", None)
 
-    # Remove files duplicates
+    prompt = get_last_user_message(form_data["messages"])
+    # TODO: re-enable URL extraction from prompt
+    # urls = []
+    # if prompt and len(prompt or "") < 500 and (not files or len(files) == 0):
+    #     urls = extract_urls(prompt)
+
     if files:
+        if not files:
+            files = []
+
+        for file_item in files:
+            if file_item.get("type", "file") == "folder":
+                # Get folder files
+                folder_id = file_item.get("id", None)
+                if folder_id:
+                    folder = Folders.get_folder_by_id_and_user_id(folder_id, user.id)
+                    if folder and folder.data and "files" in folder.data:
+                        files = [f for f in files if f.get("id", None) != folder_id]
+                        files = [*files, *folder.data["files"]]
+
+        # files = [*files, *[{"type": "url", "url": url, "name": url} for url in urls]]
+        # Remove duplicate files based on their content
         files = list({json.dumps(f, sort_keys=True): f for f in files}.values())
 
     metadata = {
@@ -1216,7 +1366,6 @@ async def process_chat_payload(request, form_data, user, metadata, model):
                         continue
 
                     auth_type = mcp_server_connection.get("auth_type", "")
-
                     headers = {}
                     if auth_type == "bearer":
                         headers["Authorization"] = (
@@ -1252,20 +1401,29 @@ async def process_chat_payload(request, form_data, user, metadata, model):
                             log.error(f"Error getting OAuth token: {e}")
                             oauth_token = None
 
+                    connection_headers = mcp_server_connection.get("headers", None)
+                    if connection_headers and isinstance(connection_headers, dict):
+                        for key, value in connection_headers.items():
+                            headers[key] = value
+
                     mcp_clients[server_id] = MCPClient()
                     await mcp_clients[server_id].connect(
                         url=mcp_server_connection.get("url", ""),
                         headers=headers if headers else None,
                     )
 
+                    function_name_filter_list = mcp_server_connection.get(
+                        "config", {}
+                    ).get("function_name_filter_list", "")
+
+                    if isinstance(function_name_filter_list, str):
+                        function_name_filter_list = function_name_filter_list.split(",")
+
                     tool_specs = await mcp_clients[server_id].list_tool_specs()
                     for tool_spec in tool_specs:
 
                         def make_tool_function(client, function_name):
                             async def tool_function(**kwargs):
-                                print(kwargs)
-                                print(client)
-                                print(await client.list_tool_specs())
                                 return await client.call_tool(
                                     function_name,
                                     function_args=kwargs,
@@ -1273,6 +1431,13 @@ async def process_chat_payload(request, form_data, user, metadata, model):
 
                             return tool_function
 
+                        if function_name_filter_list:
+                            if not is_string_allowed(
+                                tool_spec["name"], function_name_filter_list
+                            ):
+                                # Skip this function
+                                continue
+
                         tool_function = make_tool_function(
                             mcp_clients[server_id], tool_spec["name"]
                         )
@@ -1289,6 +1454,17 @@ async def process_chat_payload(request, form_data, user, metadata, model):
                         }
                 except Exception as e:
                     log.debug(e)
+                    if event_emitter:
+                        await event_emitter(
+                            {
+                                "type": "chat:message:error",
+                                "data": {
+                                    "error": {
+                                        "content": f"Failed to connect to MCP server '{server_id}'"
+                                    }
+                                },
+                            }
+                        )
                     continue
 
         tools_dict = await get_tools(
@@ -1302,6 +1478,7 @@ async def process_chat_payload(request, form_data, user, metadata, model):
                 "__files__": metadata.get("files", []),
             },
         )
+
         if mcp_tools_dict:
             tools_dict = {**tools_dict, **mcp_tools_dict}
 
@@ -1372,8 +1549,6 @@ async def process_chat_payload(request, form_data, user, metadata, model):
                     )
 
         context_string = context_string.strip()
-
-        prompt = get_last_user_message(form_data["messages"])
         if prompt is None:
             raise Exception("No user message found")
 
@@ -1471,8 +1646,6 @@ async def process_chat_response(
                     TASKS.FOLLOW_UP_GENERATION in tasks
                     and tasks[TASKS.FOLLOW_UP_GENERATION]
                 ):
-
-                    print("Generating follow ups")
                     res = await generate_follow_ups(
                         request,
                         {
@@ -1486,11 +1659,13 @@ async def process_chat_response(
 
                     if res and isinstance(res, dict):
                         if len(res.get("choices", [])) == 1:
-                            follow_ups_string = (
-                                res.get("choices", [])[0]
-                                .get("message", {})
-                                .get("content", "")
+                            response_message = res.get("choices", [])[0].get(
+                                "message", {}
                             )
+
+                            follow_ups_string = response_message.get(
+                                "content"
+                            ) or response_message.get("reasoning_content", "")
                         else:
                             follow_ups_string = ""
 
@@ -1527,16 +1702,13 @@ async def process_chat_response(
                 if not metadata.get("chat_id", "").startswith(
                     "local:"
                 ):  # Only update titles and tags for non-temp chats
-                    if (
-                        TASKS.TITLE_GENERATION in tasks
-                        and tasks[TASKS.TITLE_GENERATION]
-                    ):
+                    if TASKS.TITLE_GENERATION in tasks:
                         user_message = get_last_user_message(messages)
                         if user_message and len(user_message) > 100:
                             user_message = user_message[:100] + "..."
 
+                        title = None
                         if tasks[TASKS.TITLE_GENERATION]:
-
                             res = await generate_title(
                                 request,
                                 {
@@ -1549,13 +1721,16 @@ async def process_chat_response(
 
                             if res and isinstance(res, dict):
                                 if len(res.get("choices", [])) == 1:
-                                    title_string = (
-                                        res.get("choices", [])[0]
-                                        .get("message", {})
-                                        .get(
-                                            "content",
-                                            message.get("content", user_message),
+                                    response_message = res.get("choices", [])[0].get(
+                                        "message", {}
                                     )
+
+                                    title_string = (
+                                        response_message.get("content")
+                                        or response_message.get(
+                                            "reasoning_content",
+                                        )
+                                        or message.get("content", user_message)
                                     )
                                 else:
                                     title_string = ""
@@ -1584,7 +1759,8 @@ async def process_chat_response(
                                         "data": title,
                                     }
                                 )
-                        elif len(messages) == 2:
+
+                        if title == None and len(messages) == 2:
                             title = messages[0].get("content", user_message)
 
                             Chats.update_chat_title_by_id(metadata["chat_id"], title)
@@ -1609,11 +1785,13 @@ async def process_chat_response(
 
                         if res and isinstance(res, dict):
                             if len(res.get("choices", [])) == 1:
-                                tags_string = (
-                                    res.get("choices", [])[0]
-                                    .get("message", {})
-                                    .get("content", "")
+                                response_message = res.get("choices", [])[0].get(
+                                    "message", {}
                                 )
+
+                                tags_string = response_message.get(
+                                    "content"
+                                ) or response_message.get("reasoning_content", "")
                             else:
                                 tags_string = ""
 
@@ -1740,7 +1918,7 @@ async def process_chat_response(
                             )
 
                             # Send a webhook notification if the user is not active
-                            if not get_active_status_by_user_id(user.id):
+                            if not Users.is_user_active(user.id):
                                 webhook_url = Users.get_user_webhook_url_by_id(user.id)
                                 if webhook_url:
                                     await post_webhook(
@@ -1918,10 +2096,12 @@ async def process_chat_response(
                                 content = f"{content}{tool_calls_display_content}"
 
                     elif block["type"] == "reasoning":
-                        reasoning_display_content = "\n".join(
+                        reasoning_display_content = html.escape(
+                            "\n".join(
                                 (f"> {line}" if not line.startswith(">") else line)
                                 for line in block["content"].splitlines()
                             )
+                        )
 
                         reasoning_duration = block.get("duration", None)
 
@@ -2328,7 +2508,9 @@ async def process_chat_response(
                             )
 
                             if data:
-                                if "event" in data:
+                                if "event" in data and not getattr(
+                                    request.state, "direct", False
+                                ):
                                     await event_emitter(data.get("event", {}))
 
                                 if "selected_model_id" in data:
@@ -2436,6 +2618,26 @@ async def process_chat_response(
                                                             "arguments"
                                                         ] += delta_arguments
 
+                                    image_urls = get_image_urls(
+                                        delta.get("images", []), request, metadata, user
+                                    )
+                                    if image_urls:
+                                        message_files = Chats.add_message_files_by_id_and_message_id(
+                                            metadata["chat_id"],
+                                            metadata["message_id"],
+                                            [
+                                                {"type": "image", "url": url}
+                                                for url in image_urls
+                                            ],
+                                        )
+
+                                        await event_emitter(
+                                            {
+                                                "type": "files",
+                                                "data": {"files": message_files},
+                                            }
+                                        )
+
                                     value = delta.get("content")
 
                                     reasoning_content = (
@@ -2494,6 +2696,11 @@ async def process_chat_response(
                                                 }
                                             )
 
+                                        if ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION:
+                                            value = convert_markdown_base64_images(
+                                                request, value, metadata, user
+                                            )
+
                                         content = f"{content}{value}"
                                         if not content_blocks:
                                             content_blocks.append(
@@ -2645,8 +2852,6 @@ async def process_chat_response(
                     results = []
 
                     for tool_call in response_tool_calls:
-
-                        print("tool_call", tool_call)
                         tool_call_id = tool_call.get("id", "")
                         tool_function_name = tool_call.get("function", {}).get(
                             "name", ""
@@ -2721,7 +2926,16 @@ async def process_chat_response(
                                     )
 
                                 else:
-                                    tool_function = tool["callable"]
+                                    tool_function = get_updated_tool_function(
+                                        function=tool["callable"],
+                                        extra_params={
+                                            "__messages__": form_data.get(
+                                                "messages", []
+                                            ),
+                                            "__files__": metadata.get("files", []),
+                                        },
+                                    )
+
                                     tool_result = await tool_function(
                                         **tool_function_params
                                     )
@@ -2777,9 +2991,9 @@ async def process_chat_response(
 
                     try:
                         new_form_data = {
+                            **form_data,
                             "model": model_id,
                             "stream": True,
-                            "tools": form_data["tools"],
                             "messages": [
                                 *form_data["messages"],
                                 *convert_content_blocks_to_messages(
@@ -2953,6 +3167,7 @@ async def process_chat_response(
 
                         try:
                             new_form_data = {
+                                **form_data,
                                 "model": model_id,
                                 "stream": True,
                                 "messages": [
@@ -2998,7 +3213,7 @@ async def process_chat_response(
                     )
 
                 # Send a webhook notification if the user is not active
-                if not get_active_status_by_user_id(user.id):
+                if not Users.is_user_active(user.id):
                     webhook_url = Users.get_user_webhook_url_by_id(user.id)
                     if webhook_url:
                         await post_webhook(

backend/open_webui/utils/misc.py

@@ -8,10 +8,11 @@ from datetime import timedelta
 from pathlib import Path
 from typing import Callable, Optional
 import json
+import aiohttp
 
 
 import collections.abc
-from open_webui.env import SRC_LOG_LEVELS
+from open_webui.env import SRC_LOG_LEVELS, CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE
 
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["MAIN"])
@@ -26,6 +27,46 @@ def deep_update(d, u):
     return d
 
 
+def get_allow_block_lists(filter_list):
+    allow_list = []
+    block_list = []
+
+    if filter_list:
+        for d in filter_list:
+            if d.startswith("!"):
+                # Domains starting with "!" → blocked
+                block_list.append(d[1:].strip())
+            else:
+                # Domains starting without "!" → allowed
+                allow_list.append(d.strip())
+
+    return allow_list, block_list
+
+
+def is_string_allowed(string: str, filter_list: Optional[list[str]] = None) -> bool:
+    """
+    Checks if a string is allowed based on the provided filter list.
+    :param string: The string to check (e.g., domain or hostname).
+    :param filter_list: List of allowed/blocked strings. Strings starting with "!" are blocked.
+    :return: True if the string is allowed, False otherwise.
+    """
+    if not filter_list:
+        return True
+
+    allow_list, block_list = get_allow_block_lists(filter_list)
+
+    # If allow list is non-empty, require domain to match one of them
+    if allow_list:
+        if not any(string.endswith(allowed) for allowed in allow_list):
+            return False
+
+    # Block list always removes matches
+    if any(string.endswith(blocked) for blocked in block_list):
+        return False
+
+    return True
+
+
 def get_message_list(messages_map, message_id):
     """
     Reconstructs a list of messages in order up to the specified message_id.
@@ -531,3 +572,76 @@ def throttle(interval: float = 10.0):
         return wrapper
 
     return decorator
+
+
+def extract_urls(text: str) -> list[str]:
+    # Regex pattern to match URLs
+    url_pattern = re.compile(
+        r"(https?://[^\s]+)", re.IGNORECASE
+    )  # Matches http and https URLs
+    return url_pattern.findall(text)
+
+
+def stream_chunks_handler(stream: aiohttp.StreamReader):
+    """
+    Handle stream response chunks, supporting large data chunks that exceed the original 16kb limit.
+    When a single line exceeds max_buffer_size, returns an empty JSON string {} and skips subsequent data
+    until encountering normally sized data.
+
+    :param stream: The stream reader to handle.
+    :return: An async generator that yields the stream data.
+    """
+
+    max_buffer_size = CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE
+    if max_buffer_size is None or max_buffer_size <= 0:
+        return stream
+
+    async def yield_safe_stream_chunks():
+        buffer = b""
+        skip_mode = False
+
+        async for data, _ in stream.iter_chunks():
+            if not data:
+                continue
+
+            # In skip_mode, if buffer already exceeds the limit, clear it (it's part of an oversized line)
+            if skip_mode and len(buffer) > max_buffer_size:
+                buffer = b""
+
+            lines = (buffer + data).split(b"\n")
+
+            # Process complete lines (except the last possibly incomplete fragment)
+            for i in range(len(lines) - 1):
+                line = lines[i]
+
+                if skip_mode:
+                    # Skip mode: check if current line is small enough to exit skip mode
+                    if len(line) <= max_buffer_size:
+                        skip_mode = False
+                        yield line
+                    else:
+                        yield b"data: {}"
+                else:
+                    # Normal mode: check if line exceeds limit
+                    if len(line) > max_buffer_size:
+                        skip_mode = True
+                        yield b"data: {}"
+                        log.info(f"Skip mode triggered, line size: {len(line)}")
+                    else:
+                        yield line
+
+            # Save the last incomplete fragment
+            buffer = lines[-1]
+
+            # Check if buffer exceeds limit
+            if not skip_mode and len(buffer) > max_buffer_size:
+                skip_mode = True
+                log.info(f"Skip mode triggered, buffer size: {len(buffer)}")
+                # Clear oversized buffer to prevent unlimited growth
+                buffer = b""
+
+        # Process remaining buffer data
+        if buffer and not skip_mode:
+            yield buffer
+
+    return yield_safe_stream_chunks()

backend/open_webui/utils/models.py

@@ -6,12 +6,14 @@ import sys
 from aiocache import cached
 from fastapi import Request
 
+from open_webui.socket.utils import RedisDict
 from open_webui.routers import openai, ollama
 from open_webui.functions import get_function_models
 
 
 from open_webui.models.functions import Functions
 from open_webui.models.models import Models
+from open_webui.models.groups import Groups
 
 
 from open_webui.utils.plugin import (
@@ -166,7 +168,8 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
                         action_ids = []
                         filter_ids = []
 
-                        if "info" in model and "meta" in model["info"]:
+                        if "info" in model:
+                            if "meta" in model["info"]:
                                 action_ids.extend(
                                     model["info"]["meta"].get("actionIds", [])
                                 )
@@ -174,6 +177,10 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
                                     model["info"]["meta"].get("filterIds", [])
                                 )
 
+                            if "params" in model["info"]:
+                                # Remove params to avoid exposing sensitive info
+                                del model["info"]["params"]
+
                         model["action_ids"] = action_ids
                         model["filter_ids"] = filter_ids
                     else:
@@ -182,22 +189,40 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
         elif custom_model.is_active and (
             custom_model.id not in [model["id"] for model in models]
         ):
+            # Custom model based on a base model
             owned_by = "openai"
             pipe = None
 
+            for m in models:
+                if (
+                    custom_model.base_model_id == m["id"]
+                    or custom_model.base_model_id == m["id"].split(":")[0]
+                ):
+                    owned_by = m.get("owned_by", "unknown")
+                    if "pipe" in m:
+                        pipe = m["pipe"]
+                    break
+
+            model = {
+                "id": f"{custom_model.id}",
+                "name": custom_model.name,
+                "object": "model",
+                "created": custom_model.created_at,
+                "owned_by": owned_by,
+                "preset": True,
+                **({"pipe": pipe} if pipe is not None else {}),
+            }
+
+            info = custom_model.model_dump()
+            if "params" in info:
+                # Remove params to avoid exposing sensitive info
+                del info["params"]
+
+            model["info"] = info
+
             action_ids = []
             filter_ids = []
 
-            for model in models:
-                if (
-                    custom_model.base_model_id == model["id"]
-                    or custom_model.base_model_id == model["id"].split(":")[0]
-                ):
-                    owned_by = model.get("owned_by", "unknown owner")
-                    if "pipe" in model:
-                        pipe = model["pipe"]
-                    break
-
             if custom_model.meta:
                 meta = custom_model.meta.model_dump()
 
@@ -207,20 +232,10 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
                 if "filterIds" in meta:
                     filter_ids.extend(meta["filterIds"])
 
-            models.append(
-                {
-                    "id": f"{custom_model.id}",
-                    "name": custom_model.name,
-                    "object": "model",
-                    "created": custom_model.created_at,
-                    "owned_by": owned_by,
-                    "info": custom_model.model_dump(),
-                    "preset": True,
-                    **({"pipe": pipe} if pipe is not None else {}),
-                    "action_ids": action_ids,
-                    "filter_ids": filter_ids,
-                }
-            )
+            model["action_ids"] = action_ids
+            model["filter_ids"] = filter_ids
+
+            models.append(model)
 
     # Process action_ids to get the actions
     def get_action_items_from_module(function, module):
@@ -309,7 +324,12 @@ async def get_all_models(request, refresh: bool = False, user: UserModel = None)
 
     log.debug(f"get_all_models() returned {len(models)} models")
 
-    request.app.state.MODELS = {model["id"]: model for model in models}
+    models_dict = {model["id"]: model for model in models}
+    if isinstance(request.app.state.MODELS, RedisDict):
+        request.app.state.MODELS.set(models_dict)
+    else:
+        request.app.state.MODELS = models_dict
+
     return models
 
 
@@ -343,6 +363,7 @@ def get_filtered_models(models, user):
         or (user.role == "admin" and not BYPASS_ADMIN_ACCESS_CONTROL)
     ) and not BYPASS_MODEL_ACCESS_CONTROL:
         filtered_models = []
+        user_group_ids = {group.id for group in Groups.get_groups_by_member_id(user.id)}
         for model in models:
             if model.get("arena"):
                 if has_access(
@@ -351,6 +372,7 @@ def get_filtered_models(models, user):
                     access_control=model.get("info", {})
                     .get("meta", {})
                     .get("access_control", {}),
+                    user_group_ids=user_group_ids,
                 ):
                     filtered_models.append(model)
                 continue
@@ -364,6 +386,7 @@ def get_filtered_models(models, user):
                         user.id,
                         type="read",
                         access_control=model_info.access_control,
+                        user_group_ids=user_group_ids,
                     )
                 ):
                     filtered_models.append(model)

backend/open_webui/utils/oauth.py

@@ -1,4 +1,5 @@
 import base64
+import copy
 import hashlib
 import logging
 import mimetypes
@@ -13,7 +14,7 @@ import fnmatch
 import time
 import secrets
 from cryptography.fernet import Fernet
-
+from typing import Literal
 
 import aiohttp
 from authlib.integrations.starlette_client import OAuth
@@ -41,6 +42,8 @@ from open_webui.config import (
     ENABLE_OAUTH_GROUP_MANAGEMENT,
     ENABLE_OAUTH_GROUP_CREATION,
     OAUTH_BLOCKED_GROUPS,
+    OAUTH_GROUPS_SEPARATOR,
+    OAUTH_ROLES_SEPARATOR,
     OAUTH_ROLES_CLAIM,
     OAUTH_SUB_CLAIM,
     OAUTH_GROUPS_CLAIM,
@@ -51,6 +54,7 @@ from open_webui.config import (
     OAUTH_ADMIN_ROLES,
     OAUTH_ALLOWED_DOMAINS,
     OAUTH_UPDATE_PICTURE_ON_LOGIN,
+    OAUTH_ACCESS_TOKEN_REQUEST_INCLUDE_CLIENT_ID,
     WEBHOOK_URL,
     JWT_EXPIRES_IN,
     AppConfig,
@@ -62,6 +66,7 @@ from open_webui.env import (
     WEBUI_AUTH_COOKIE_SAME_SITE,
     WEBUI_AUTH_COOKIE_SECURE,
     ENABLE_OAUTH_ID_TOKEN_COOKIE,
+    ENABLE_OAUTH_EMAIL_FALLBACK,
     OAUTH_CLIENT_INFO_ENCRYPTION_KEY,
 )
 from open_webui.utils.misc import parse_duration
@@ -69,10 +74,19 @@ from open_webui.utils.auth import get_password_hash, create_token
 from open_webui.utils.webhook import post_webhook
 
 from mcp.shared.auth import (
-    OAuthClientMetadata,
+    OAuthClientMetadata as MCPOAuthClientMetadata,
     OAuthMetadata,
 )
 
+from authlib.oauth2.rfc6749.errors import OAuth2Error
+
+
+class OAuthClientMetadata(MCPOAuthClientMetadata):
+    token_endpoint_auth_method: Literal[
+        "none", "client_secret_basic", "client_secret_post"
+    ] = "client_secret_post"
+    pass
+
 
 class OAuthClientInformationFull(OAuthClientMetadata):
     issuer: Optional[str] = None  # URL of the OAuth server that issued this client
@@ -82,6 +96,8 @@ class OAuthClientInformationFull(OAuthClientMetadata):
     client_id_issued_at: int | None = None
     client_secret_expires_at: int | None = None
 
+    server_metadata: Optional[OAuthMetadata] = None  # Fetched from the OAuth server
+
 
 from open_webui.env import SRC_LOG_LEVELS, GLOBAL_LOG_LEVEL
 
@@ -147,6 +163,37 @@ def decrypt_data(data: str):
         raise
 
 
+def _build_oauth_callback_error_message(e: Exception) -> str:
+    """
+    Produce a user-facing callback error string with actionable context.
+    Keeps the message short and strips newlines for safe redirect usage.
+    """
+    if isinstance(e, OAuth2Error):
+        parts = [p for p in [e.error, e.description] if p]
+        detail = " - ".join(parts)
+    elif isinstance(e, HTTPException):
+        detail = e.detail if isinstance(e.detail, str) else str(e.detail)
+    elif isinstance(e, aiohttp.ClientResponseError):
+        detail = f"Upstream provider returned {e.status}: {e.message}"
+    elif isinstance(e, aiohttp.ClientError):
+        detail = str(e)
+    elif isinstance(e, KeyError):
+        missing = str(e).strip("'")
+        if missing.lower() == "state":
+            detail = "Missing state parameter in callback (session may have expired)"
+        else:
+            detail = f"Missing expected key '{missing}' in OAuth response"
+    else:
+        detail = str(e)
+
+    detail = detail.replace("\n", " ").strip()
+    if not detail:
+        detail = e.__class__.__name__
+
+    message = f"OAuth callback failed: {detail}"
+    return message[:197] + "..." if len(message) > 200 else message
+
+
 def is_in_blocked_groups(group_name: str, groups: list) -> bool:
     """
     Check if a group name matches any blocked pattern.
@@ -200,22 +247,31 @@ def get_parsed_and_base_url(server_url) -> tuple[urllib.parse.ParseResult, str]:
 def get_discovery_urls(server_url) -> list[str]:
     parsed, base_url = get_parsed_and_base_url(server_url)
 
-    urls = [
+    urls = []
+
+    if parsed.path and parsed.path != "/":
+        # Generate discovery URLs based on https://modelcontextprotocol.io/specification/draft/basic/authorization#authorization-server-metadata-discovery
+        tenant = parsed.path.rstrip("/")
+        urls.extend(
+            [
+                urllib.parse.urljoin(
+                    base_url,
+                    f"/.well-known/oauth-authorization-server{tenant}",
+                ),
+                urllib.parse.urljoin(
+                    base_url, f"/.well-known/openid-configuration{tenant}"
+                ),
+                urllib.parse.urljoin(
+                    base_url, f"{tenant}/.well-known/openid-configuration"
+                ),
+            ]
+        )
+
+    urls.extend(
+        [
             urllib.parse.urljoin(base_url, "/.well-known/oauth-authorization-server"),
             urllib.parse.urljoin(base_url, "/.well-known/openid-configuration"),
         ]
-
-    if parsed.path and parsed.path != "/":
-        urls.append(
-            urllib.parse.urljoin(
-                base_url,
-                f"/.well-known/oauth-authorization-server{parsed.path.rstrip('/')}",
-            )
-        )
-        urls.append(
-            urllib.parse.urljoin(
-                base_url, f"/.well-known/openid-configuration{parsed.path.rstrip('/')}"
-            )
     )
 
     return urls
@@ -242,13 +298,12 @@ async def get_oauth_client_info_with_dynamic_client_registration(
             redirect_uris=[f"{redirect_base_url}/oauth/clients/{client_id}/callback"],
             grant_types=["authorization_code", "refresh_token"],
             response_types=["code"],
-            token_endpoint_auth_method="client_secret_post",
         )
 
         # Attempt to fetch OAuth server metadata to get registration endpoint & scopes
         discovery_urls = get_discovery_urls(oauth_server_url)
         for url in discovery_urls:
-            async with aiohttp.ClientSession() as session:
+            async with aiohttp.ClientSession(trust_env=True) as session:
                 async with session.get(
                     url, ssl=AIOHTTP_CLIENT_SESSION_SSL
                 ) as oauth_server_metadata_response:
@@ -265,6 +320,17 @@ async def get_oauth_client_info_with_dynamic_client_registration(
                                 oauth_client_metadata.scope = " ".join(
                                     oauth_server_metadata.scopes_supported
                                 )
+
+                            if (
+                                oauth_server_metadata.token_endpoint_auth_methods_supported
+                                and oauth_client_metadata.token_endpoint_auth_method
+                                not in oauth_server_metadata.token_endpoint_auth_methods_supported
+                            ):
+                                # Pick the first supported method from the server
+                                oauth_client_metadata.token_endpoint_auth_method = oauth_server_metadata.token_endpoint_auth_methods_supported[
+                                    0
+                                ]
+
                             break
                         except Exception as e:
                             log.error(f"Error parsing OAuth metadata from {url}: {e}")
@@ -284,7 +350,7 @@ async def get_oauth_client_info_with_dynamic_client_registration(
         )
 
         # Perform dynamic client registration and return client info
-        async with aiohttp.ClientSession() as session:
+        async with aiohttp.ClientSession(trust_env=True) as session:
             async with session.post(
                 registration_url, json=registration_data, ssl=AIOHTTP_CLIENT_SESSION_SSL
             ) as oauth_client_registration_response:
@@ -292,10 +358,18 @@ async def get_oauth_client_info_with_dynamic_client_registration(
                     registration_response_json = (
                         await oauth_client_registration_response.json()
                     )
+
+                    # The mcp package requires optional unset values to be None. If an empty string is passed, it gets validated and fails.
+                    # This replaces all empty strings with None.
+                    registration_response_json = {
+                        k: (None if v == "" else v)
+                        for k, v in registration_response_json.items()
+                    }
                     oauth_client_info = OAuthClientInformationFull.model_validate(
                         {
                             **registration_response_json,
                             **{"issuer": oauth_server_metadata_url},
+                            **{"server_metadata": oauth_server_metadata},
                         }
                     )
                     log.info(
@@ -331,20 +405,45 @@ class OAuthClientManager:
         self.clients = {}
 
     def add_client(self, client_id, oauth_client_info: OAuthClientInformationFull):
-        self.clients[client_id] = {
-            "client": self.oauth.register(
-                name=client_id,
-                client_id=oauth_client_info.client_id,
-                client_secret=oauth_client_info.client_secret,
-                client_kwargs=(
+        kwargs = {
+            "name": client_id,
+            "client_id": oauth_client_info.client_id,
+            "client_secret": oauth_client_info.client_secret,
+            "client_kwargs": {
+                **(
                     {"scope": oauth_client_info.scope}
                     if oauth_client_info.scope
                     else {}
                 ),
-                server_metadata_url=(
+                **(
+                    {
+                        "token_endpoint_auth_method": oauth_client_info.token_endpoint_auth_method
+                    }
+                    if oauth_client_info.token_endpoint_auth_method
+                    else {}
+                ),
+            },
+            "server_metadata_url": (
                 oauth_client_info.issuer if oauth_client_info.issuer else None
             ),
-            ),
+        }
+
+        if (
+            oauth_client_info.server_metadata
+            and oauth_client_info.server_metadata.code_challenge_methods_supported
+        ):
+            if (
+                isinstance(
+                    oauth_client_info.server_metadata.code_challenge_methods_supported,
+                    list,
+                )
+                and "S256"
+                in oauth_client_info.server_metadata.code_challenge_methods_supported
+            ):
+                kwargs["code_challenge_method"] = "S256"
+
+        self.clients[client_id] = {
+            "client": self.oauth.register(**kwargs),
             "client_info": oauth_client_info,
         }
         return self.clients[client_id]
@@ -353,6 +452,82 @@ class OAuthClientManager:
         if client_id in self.clients:
             del self.clients[client_id]
             log.info(f"Removed OAuth client {client_id}")
+
+        if hasattr(self.oauth, "_clients"):
+            if client_id in self.oauth._clients:
+                self.oauth._clients.pop(client_id, None)
+
+        if hasattr(self.oauth, "_registry"):
+            if client_id in self.oauth._registry:
+                self.oauth._registry.pop(client_id, None)
+
+        return True
+
+    async def _preflight_authorization_url(
+        self, client, client_info: OAuthClientInformationFull
+    ) -> bool:
+        # TODO: Replace this logic with a more robust OAuth client registration validation
+        # Only perform preflight checks for Starlette OAuth clients
+        if not hasattr(client, "create_authorization_url"):
+            return True
+
+        redirect_uri = None
+        if client_info.redirect_uris:
+            redirect_uri = str(client_info.redirect_uris[0])
+
+        try:
+            auth_data = await client.create_authorization_url(redirect_uri=redirect_uri)
+            authorization_url = auth_data.get("url")
+
+            if not authorization_url:
+                return True
+        except Exception as e:
+            log.debug(
+                f"Skipping OAuth preflight for client {client_info.client_id}: {e}",
+            )
+            return True
+
+        try:
+            async with aiohttp.ClientSession(trust_env=True) as session:
+                async with session.get(
+                    authorization_url,
+                    allow_redirects=False,
+                    ssl=AIOHTTP_CLIENT_SESSION_SSL,
+                ) as resp:
+                    if resp.status < 400:
+                        return True
+                    response_text = await resp.text()
+
+                    error = None
+                    error_description = ""
+
+                    content_type = resp.headers.get("content-type", "")
+                    if "application/json" in content_type:
+                        try:
+                            payload = json.loads(response_text)
+                            error = payload.get("error")
+                            error_description = payload.get("error_description", "")
+                        except:
+                            pass
+                    else:
+                        error_description = response_text
+
+                    error_message = f"{error or ''} {error_description or ''}".lower()
+
+                    if any(
+                        keyword in error_message
+                        for keyword in ("invalid_client", "invalid client", "client id")
+                    ):
+                        log.warning(
+                            f"OAuth client preflight detected invalid registration for {client_info.client_id}: {error} {error_description}"
+                        )
+
+                        return False
+        except Exception as e:
+            log.debug(
+                f"Skipping OAuth preflight network check for client {client_info.client_id}: {e}"
+            )
+
         return True
 
     def get_client(self, client_id):
@@ -367,8 +542,8 @@ class OAuthClientManager:
         if client_id in self.clients:
             client = self.clients[client_id]
             return (
-                client.server_metadata_url
-                if hasattr(client, "server_metadata_url")
+                client._server_metadata_url
+                if hasattr(client, "_server_metadata_url")
                 else None
             )
         return None
@@ -543,7 +718,6 @@ class OAuthClientManager:
         client = self.get_client(client_id)
         if client is None:
             raise HTTPException(404)
-
         client_info = self.get_client_info(client_id)
         if client_info is None:
             raise HTTPException(404)
@@ -551,7 +725,8 @@ class OAuthClientManager:
         redirect_uri = (
             client_info.redirect_uris[0] if client_info.redirect_uris else None
         )
-        return await client.authorize_redirect(request, str(redirect_uri))
+        redirect_uri_str = str(redirect_uri) if redirect_uri else None
+        return await client.authorize_redirect(request, redirect_uri_str)
 
     async def handle_callback(self, request, client_id: str, user_id: str, response):
         client = self.get_client(client_id)
@@ -560,7 +735,18 @@ class OAuthClientManager:
 
         error_message = None
         try:
-            token = await client.authorize_access_token(request)
+            client_info = self.get_client_info(client_id)
+
+            auth_params = {}
+            if (
+                client_info
+                and hasattr(client_info, "client_id")
+                and hasattr(client_info, "client_secret")
+            ):
+                auth_params["client_id"] = client_info.client_id
+                auth_params["client_secret"] = client_info.client_secret
+
+            token = await client.authorize_access_token(request, **auth_params)
             if token:
                 try:
                     # Add timestamp for tracking
@@ -593,8 +779,14 @@ class OAuthClientManager:
                 error_message = "Failed to obtain OAuth token"
                 log.warning(error_message)
         except Exception as e:
-            error_message = "OAuth callback error"
-            log.warning(f"OAuth callback error: {e}")
+            error_message = _build_oauth_callback_error_message(e)
+            log.warning(
+                "OAuth callback error for user_id=%s client_id=%s: %s",
+                user_id,
+                client_id,
+                error_message,
+                exc_info=True,
+            )
 
         redirect_url = (
             str(request.app.state.config.WEBUI_URL or request.base_url)
@@ -602,7 +794,9 @@ class OAuthClientManager:
 
         if error_message:
             log.debug(error_message)
-            redirect_url = f"{redirect_url}/?error={error_message}"
+            redirect_url = (
+                f"{redirect_url}/?error={urllib.parse.quote_plus(error_message)}"
+            )
             return RedirectResponse(url=redirect_url, headers=response.headers)
 
         response = RedirectResponse(url=redirect_url, headers=response.headers)
@@ -633,8 +827,8 @@ class OAuthManager:
         if provider_name in self._clients:
             client = self._clients[provider_name]
             return (
-                client.server_metadata_url
-                if hasattr(client, "server_metadata_url")
+                client._server_metadata_url
+                if hasattr(client, "_server_metadata_url")
                 else None
             )
         return None
@@ -831,11 +1025,21 @@ class OAuthManager:
                 for nested_claim in nested_claims:
                     claim_data = claim_data.get(nested_claim, {})
 
+                # Try flat claim structure as alternative
+                if not claim_data:
+                    claim_data = user_data.get(oauth_claim, {})
+
                 oauth_roles = []
 
                 if isinstance(claim_data, list):
                     oauth_roles = claim_data
-                if isinstance(claim_data, str) or isinstance(claim_data, int):
+                elif isinstance(claim_data, str):
+                    # Split by the configured separator if present
+                    if OAUTH_ROLES_SEPARATOR and OAUTH_ROLES_SEPARATOR in claim_data:
+                        oauth_roles = claim_data.split(OAUTH_ROLES_SEPARATOR)
+                    else:
+                        oauth_roles = [claim_data]
+                elif isinstance(claim_data, int):
                     oauth_roles = [str(claim_data)]
 
             log.debug(f"Oauth Roles claim: {oauth_claim}")
@@ -889,12 +1093,16 @@ class OAuthManager:
             if isinstance(claim_data, list):
                 user_oauth_groups = claim_data
             elif isinstance(claim_data, str):
+                # Split by the configured separator if present
+                if OAUTH_GROUPS_SEPARATOR in claim_data:
+                    user_oauth_groups = claim_data.split(OAUTH_GROUPS_SEPARATOR)
+                else:
                     user_oauth_groups = [claim_data]
             else:
                 user_oauth_groups = []
 
         user_current_groups: list[GroupModel] = Groups.get_groups_by_member_id(user.id)
-        all_available_groups: list[GroupModel] = Groups.get_groups()
+        all_available_groups: list[GroupModel] = Groups.get_all_groups()
 
         # Create groups if they don't exist and creation is enabled
         if auth_manager_config.ENABLE_OAUTH_GROUP_CREATION:
@@ -938,7 +1146,7 @@ class OAuthManager:
 
             # Refresh the list of all available groups if any were created
             if groups_created:
-                all_available_groups = Groups.get_groups()
+                all_available_groups = Groups.get_all_groups()
                 log.debug("Refreshed list of all available groups after creation.")
 
         log.debug(f"Oauth Groups claim: {oauth_claim}")
@@ -960,22 +1168,21 @@ class OAuthManager:
                     f"Removing user from group {group_model.name} as it is no longer in their oauth groups"
                 )
 
-                user_ids = group_model.user_ids
-                user_ids = [i for i in user_ids if i != user.id]
+                Groups.remove_users_from_group(group_model.id, [user.id])
 
                 # In case a group is created, but perms are never assigned to the group by hitting "save"
                 group_permissions = group_model.permissions
                 if not group_permissions:
                     group_permissions = default_permissions
 
-                update_form = GroupUpdateForm(
+                Groups.update_group_by_id(
+                    id=group_model.id,
+                    form_data=GroupUpdateForm(
                         name=group_model.name,
                         description=group_model.description,
                         permissions=group_permissions,
-                    user_ids=user_ids,
-                )
-                Groups.update_group_by_id(
-                    id=group_model.id, form_data=update_form, overwrite=False
+                    ),
+                    overwrite=False,
                 )
 
         # Add user to new groups
@@ -991,22 +1198,21 @@ class OAuthManager:
                     f"Adding user to group {group_model.name} as it was found in their oauth groups"
                 )
 
-                user_ids = group_model.user_ids
-                user_ids.append(user.id)
+                Groups.add_users_to_group(group_model.id, [user.id])
 
                 # In case a group is created, but perms are never assigned to the group by hitting "save"
                 group_permissions = group_model.permissions
                 if not group_permissions:
                     group_permissions = default_permissions
 
-                update_form = GroupUpdateForm(
+                Groups.update_group_by_id(
+                    id=group_model.id,
+                    form_data=GroupUpdateForm(
                         name=group_model.name,
                         description=group_model.description,
                         permissions=group_permissions,
-                    user_ids=user_ids,
-                )
-                Groups.update_group_by_id(
-                    id=group_model.id, form_data=update_form, overwrite=False
+                    ),
+                    overwrite=False,
                 )
 
     async def _process_picture_url(
@@ -1073,10 +1279,26 @@ class OAuthManager:
         error_message = None
         try:
             client = self.get_client(provider)
+
+            auth_params = {}
+
+            if client:
+                if (
+                    hasattr(client, "client_id")
+                    and OAUTH_ACCESS_TOKEN_REQUEST_INCLUDE_CLIENT_ID
+                ):
+                    auth_params["client_id"] = client.client_id
+
             try:
-                token = await client.authorize_access_token(request)
+                token = await client.authorize_access_token(request, **auth_params)
             except Exception as e:
-                log.warning(f"OAuth callback error: {e}")
+                detailed_error = _build_oauth_callback_error_message(e)
+                log.warning(
+                    "OAuth callback error during authorize_access_token for provider %s: %s",
+                    provider,
+                    detailed_error,
+                    exc_info=True,
+                )
                 raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
 
             # Try to get userinfo from the token first, some providers include it there
@@ -1107,7 +1329,10 @@ class OAuthManager:
                 log.warning(f"OAuth callback failed, sub is missing: {user_data}")
                 raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
 
-            provider_sub = f"{provider}@{sub}"
+            oauth_data = {}
+            oauth_data[provider] = {
+                "sub": sub,
+            }
 
             # Email extraction
             email_claim = auth_manager_config.OAUTH_EMAIL_CLAIM
@@ -1153,11 +1378,13 @@ class OAuthManager:
                     except Exception as e:
                         log.warning(f"Error fetching GitHub email: {e}")
                         raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
+                elif ENABLE_OAUTH_EMAIL_FALLBACK:
+                    email = f"{provider}@{sub}.local"
                 else:
                     log.warning(f"OAuth callback failed, email is missing: {user_data}")
                     raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
-            email = email.lower()
 
+            email = email.lower()
             # If allowed domains are configured, check if the email domain is in the list
             if (
                 "*" not in auth_manager_config.OAUTH_ALLOWED_DOMAINS
@@ -1170,7 +1397,7 @@ class OAuthManager:
                 raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED)
 
             # Check if the user exists
-            user = Users.get_user_by_oauth_sub(provider_sub)
+            user = Users.get_user_by_oauth_sub(provider, sub)
             if not user:
                 # If the user does not exist, check if merging is enabled
                 if auth_manager_config.OAUTH_MERGE_ACCOUNTS_BY_EMAIL:
@@ -1178,12 +1405,15 @@ class OAuthManager:
                     user = Users.get_user_by_email(email)
                     if user:
                         # Update the user with the new oauth sub
-                        Users.update_user_oauth_sub_by_id(user.id, provider_sub)
+                        Users.update_user_oauth_by_id(user.id, provider, sub)
 
             if user:
                 determined_role = self.get_user_role(user, user_data)
                 if user.role != determined_role:
                     Users.update_user_role_by_id(user.id, determined_role)
+                    # Update the user object in memory as well,
+                    # to avoid problems with the ENABLE_OAUTH_GROUP_MANAGEMENT check below
+                    user.role = determined_role
                 # Update profile picture if enabled and different from current
                 if auth_manager_config.OAUTH_UPDATE_PICTURE_ON_LOGIN:
                     picture_claim = auth_manager_config.OAUTH_PICTURE_CLAIM
@@ -1234,7 +1464,7 @@ class OAuthManager:
                         name=name,
                         profile_image_url=picture_url,
                         role=self.get_user_role(None, user_data),
-                        oauth_sub=provider_sub,
+                        oauth=oauth_data,
                     )
 
                     if auth_manager_config.WEBHOOK_URL:

backend/open_webui/utils/payload.py

@@ -297,6 +297,10 @@ def convert_payload_openai_to_ollama(openai_payload: dict) -> dict:
     if "tools" in openai_payload:
         ollama_payload["tools"] = openai_payload["tools"]
 
+    if "max_tokens" in openai_payload:
+        ollama_payload["num_predict"] = openai_payload["max_tokens"]
+        del openai_payload["max_tokens"]
+
     # If there are advanced parameters in the payload, format them in Ollama's options field
     if openai_payload.get("options"):
         ollama_payload["options"] = openai_payload["options"]

backend/open_webui/utils/task.py

@@ -208,20 +208,21 @@ def rag_template(template: str, context: str, query: str):
     if "[query]" in context:
         query_placeholder = "{{QUERY" + str(uuid.uuid4()) + "}}"
         template = template.replace("[query]", query_placeholder)
-        query_placeholders.append(query_placeholder)
+        query_placeholders.append((query_placeholder, "[query]"))
 
     if "{{QUERY}}" in context:
         query_placeholder = "{{QUERY" + str(uuid.uuid4()) + "}}"
         template = template.replace("{{QUERY}}", query_placeholder)
-        query_placeholders.append(query_placeholder)
+        query_placeholders.append((query_placeholder, "{{QUERY}}"))
 
     template = template.replace("[context]", context)
     template = template.replace("{{CONTEXT}}", context)
+
     template = template.replace("[query]", query)
     template = template.replace("{{QUERY}}", query)
 
-    for query_placeholder in query_placeholders:
-        template = template.replace(query_placeholder, query)
+    for query_placeholder, original_placeholder in query_placeholders:
+        template = template.replace(query_placeholder, original_placeholder)
 
     return template
 

backend/open_webui/utils/telemetry/metrics.py

@@ -45,7 +45,6 @@ from open_webui.env import (
     OTEL_METRICS_OTLP_SPAN_EXPORTER,
     OTEL_METRICS_EXPORTER_OTLP_INSECURE,
 )
-from open_webui.socket.main import get_active_user_ids
 from open_webui.models.users import Users
 
 _EXPORT_INTERVAL_MILLIS = 10_000  # 10 seconds
@@ -99,6 +98,9 @@ def _build_meter_provider(resource: Resource) -> MeterProvider:
         View(
             instrument_name="webui.users.active",
         ),
+        View(
+            instrument_name="webui.users.active.today",
+        ),
     ]
 
     provider = MeterProvider(
@@ -132,7 +134,7 @@ def setup_metrics(app: FastAPI, resource: Resource) -> None:
     ) -> Sequence[metrics.Observation]:
         return [
             metrics.Observation(
-                value=len(get_active_user_ids()),
+                value=Users.get_active_user_count(),
             )
         ]
 
@@ -159,6 +161,18 @@ def setup_metrics(app: FastAPI, resource: Resource) -> None:
         callbacks=[observe_active_users],
     )
 
+    def observe_users_active_today(
+        options: metrics.CallbackOptions,
+    ) -> Sequence[metrics.Observation]:
+        return [metrics.Observation(value=Users.get_num_users_active_today())]
+
+    meter.create_observable_gauge(
+        name="webui.users.active.today",
+        description="Number of users active since midnight today",
+        unit="users",
+        callbacks=[observe_users_active_today],
+    )
+
     # FastAPI middleware
     @app.middleware("http")
     async def _metrics_middleware(request: Request, call_next):

backend/open_webui/utils/tools.py

@@ -34,6 +34,7 @@ from langchain_core.utils.function_calling import (
 )
 
 
+from open_webui.utils.misc import is_string_allowed
 from open_webui.models.tools import Tools
 from open_webui.models.users import UserModel
 from open_webui.utils.plugin import load_tool_module_by_id
@@ -85,9 +86,26 @@ def get_async_tool_function_and_apply_extra_params(
     update_wrapper(new_function, function)
     new_function.__signature__ = new_sig
 
+    new_function.__function__ = function  # type: ignore
+    new_function.__extra_params__ = extra_params  # type: ignore
+
     return new_function
 
 
+def get_updated_tool_function(function: Callable, extra_params: dict):
+    # Get the original function and merge updated params
+    __function__ = getattr(function, "__function__", None)
+    __extra_params__ = getattr(function, "__extra_params__", None)
+
+    if __function__ is not None and __extra_params__ is not None:
+        return get_async_tool_function_and_apply_extra_params(
+            __function__,
+            {**__extra_params__, **extra_params},
+        )
+
+    return function
+
+
 async def get_tools(
     request: Request, tool_ids: list[str], user: UserModel, extra_params: dict
 ) -> dict[str, dict]:
@@ -132,13 +150,28 @@ async def get_tools(
                     )
 
                     specs = tool_server_data.get("specs", [])
+                    function_name_filter_list = tool_server_connection.get(
+                        "config", {}
+                    ).get("function_name_filter_list", "")
+
+                    if isinstance(function_name_filter_list, str):
+                        function_name_filter_list = function_name_filter_list.split(",")
+
                     for spec in specs:
                         function_name = spec["name"]
+                        if function_name_filter_list:
+                            if not is_string_allowed(
+                                function_name, function_name_filter_list
+                            ):
+                                # Skip this function
+                                continue
 
                         auth_type = tool_server_connection.get("auth_type", "bearer")
 
                         cookies = {}
-                        headers = {}
+                        headers = {
+                            "Content-Type": "application/json",
+                        }
 
                         if auth_type == "bearer":
                             headers["Authorization"] = (
@@ -160,7 +193,10 @@ async def get_tools(
                                     f"Bearer {oauth_token.get('access_token', '')}"
                                 )
 
-                        headers["Content-Type"] = "application/json"
+                        connection_headers = tool_server_connection.get("headers", None)
+                        if connection_headers and isinstance(connection_headers, dict):
+                            for key, value in connection_headers.items():
+                                headers[key] = value
 
                         def make_tool_function(
                             function_name, tool_server_data, headers
@@ -215,14 +251,16 @@ async def get_tools(
                 module, _ = load_tool_module_by_id(tool_id)
                 request.app.state.TOOLS[tool_id] = module
 
-            extra_params["__id__"] = tool_id
+            __user__ = {
+                **extra_params["__user__"],
+            }
 
             # Set valves for the tool
             if hasattr(module, "valves") and hasattr(module, "Valves"):
                 valves = Tools.get_tool_valves_by_id(tool_id) or {}
                 module.valves = module.Valves(**valves)
             if hasattr(module, "UserValves"):
-                extra_params["__user__"]["valves"] = module.UserValves(  # type: ignore
+                __user__["valves"] = module.UserValves(  # type: ignore
                     **Tools.get_user_valves_by_id_and_user_id(tool_id, user.id)
                 )
 
@@ -244,7 +282,12 @@ async def get_tools(
                 function_name = spec["name"]
                 tool_function = getattr(module, function_name)
                 callable = get_async_tool_function_and_apply_extra_params(
-                    tool_function, extra_params
+                    tool_function,
+                    {
+                        **extra_params,
+                        "__id__": tool_id,
+                        "__user__": __user__,
+                    },
                 )
 
                 # TODO: Support Pydantic models as parameters
@@ -544,20 +587,21 @@ async def get_tool_servers(request: Request):
     return tool_servers
 
 
-async def get_tool_server_data(token: str, url: str) -> Dict[str, Any]:
-    headers = {
+async def get_tool_server_data(url: str, headers: Optional[dict]) -> Dict[str, Any]:
+    _headers = {
         "Accept": "application/json",
         "Content-Type": "application/json",
     }
-    if token:
-        headers["Authorization"] = f"Bearer {token}"
+
+    if headers:
+        _headers.update(headers)
 
     error = None
     try:
         timeout = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER_DATA)
         async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
             async with session.get(
-                url, headers=headers, ssl=AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL
+                url, headers=_headers, ssl=AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL
             ) as response:
                 if response.status != 200:
                     error_body = await response.json()
@@ -627,7 +671,10 @@ async def get_tool_servers_data(servers: List[Dict[str, Any]]) -> List[Dict[str,
                 openapi_path = server.get("path", "openapi.json")
                 spec_url = get_tool_server_url(server_url, openapi_path)
                 # Fetch from URL
-                task = get_tool_server_data(token, spec_url)
+                task = get_tool_server_data(
+                    spec_url,
+                    {"Authorization": f"Bearer {token}"} if token else None,
+                )
             elif spec_type == "json" and server.get("spec", ""):
                 # Use provided JSON spec
                 spec_json = None
@@ -748,17 +795,13 @@ async def execute_tool_server(
         if operation.get("requestBody", {}).get("content"):
             if params:
                 body_params = params
-            else:
-                raise Exception(
-                    f"Request body expected for operation '{name}' but none found."
-                )
 
         async with aiohttp.ClientSession(
             trust_env=True, timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT)
         ) as session:
             request_method = getattr(session, http_method.lower())
 
-            if http_method in ["post", "put", "patch"]:
+            if http_method in ["post", "put", "patch", "delete"]:
                 async with request_method(
                     final_url,
                     json=body_params,

backend/open_webui/utils/webhook.py

@@ -51,7 +51,7 @@ async def post_webhook(name: str, url: str, message: str, event_data: dict) -> b
             payload = {**event_data}
 
         log.debug(f"payload: {payload}")
-        async with aiohttp.ClientSession() as session:
+        async with aiohttp.ClientSession(trust_env=True) as session:
             async with session.post(url, json=payload) as r:
                 r_text = await r.text()
                 r.raise_for_status()

backend/requirements-min.txt

@@ -0,0 +1,51 @@
+# Minimal requirements for backend to run
+# WIP: use this as a reference to build a minimal docker image
+
+fastapi==0.118.0
+uvicorn[standard]==0.37.0
+pydantic==2.11.9
+python-multipart==0.0.20
+itsdangerous==2.2.0
+
+python-socketio==5.15.0
+python-jose==3.5.0
+cryptography
+bcrypt==5.0.0
+argon2-cffi==25.1.0
+PyJWT[crypto]==2.10.1
+authlib==1.6.5
+
+requests==2.32.5
+aiohttp==3.12.15
+async-timeout
+aiocache
+aiofiles
+starlette-compress==1.6.0
+httpx[socks,http2,zstd,cli,brotli]==0.28.1
+starsessions[redis]==2.2.1
+
+sqlalchemy==2.0.38
+alembic==1.14.0
+peewee==3.18.1
+peewee-migrate==1.12.2
+
+pycrdt==0.12.25
+redis
+
+APScheduler==3.10.4
+RestrictedPython==8.0
+
+loguru==0.7.3
+asgiref==3.8.1
+
+mcp==1.21.2
+openai
+
+langchain==0.3.27
+langchain-community==0.3.29
+fake-useragent==2.2.0
+
+chromadb==1.1.0
+black==25.9.0
+pydub
+chardet==5.2.0

backend/requirements.txt

@@ -4,13 +4,13 @@ pydantic==2.11.9
 python-multipart==0.0.20
 itsdangerous==2.2.0
 
-python-socketio==5.13.0
-python-jose==3.4.0
+python-socketio==5.15.0
+python-jose==3.5.0
 cryptography
 bcrypt==5.0.0
 argon2-cffi==25.1.0
 PyJWT[crypto]==2.10.1
-authlib==1.6.3
+authlib==1.6.5
 
 requests==2.32.5
 aiohttp==3.12.15
@@ -29,14 +29,6 @@ peewee-migrate==1.12.2
 pycrdt==0.12.25
 redis
 
-pymongo
-
-psycopg2-binary==2.9.10
-pgvector==0.4.1
-
-PyMySQL==1.1.1
-boto3==1.40.5
-
 APScheduler==3.10.4
 RestrictedPython==8.0
 
@@ -45,11 +37,11 @@ asgiref==3.8.1
 
 # AI libraries
 tiktoken
-mcp==1.14.1
+mcp==1.21.2
 
 openai
 anthropic
-google-genai==1.38.0
+google-genai==1.52.0
 google-generativeai==0.8.5
 
 langchain==0.3.27
@@ -57,31 +49,24 @@ langchain-community==0.3.29
 
 fake-useragent==2.2.0
 chromadb==1.1.0
+weaviate-client==4.17.0
 opensearch-py==2.8.0
 
-pymilvus==2.5.0
-qdrant-client==1.14.3
-playwright==1.49.1 # Caution: version must match docker-compose.playwright.yaml
-elasticsearch==9.1.0
-pinecone==6.0.2
-oracledb==3.2.0
-
-av==14.0.1 # Caution: Set due to FATAL FIPS SELFTEST FAILURE, see discussion https://github.com/open-webui/open-webui/discussions/15720
-transformers
-sentence-transformers==5.1.1
+transformers==4.57.3
+sentence-transformers==5.1.2
 accelerate
 pyarrow==20.0.0 # fix: pin pyarrow version to 20 for rpi compatibility #15897
 einops==0.8.1 
 
-colbert-ai==0.2.21
-
 ftfy==6.2.3
-pypdf==6.0.0
+chardet==5.2.0
+pypdf==6.4.0
 fpdf2==2.8.2
 pymdown-extensions==10.14.2
 docx2txt==0.8
 python-pptx==1.0.2
-unstructured==0.16.17
+unstructured==0.18.18
+msoffcrypto-tool==5.4.2
 nltk==3.9.1
 Markdown==3.9
 pypandoc==1.15
@@ -93,7 +78,6 @@ validators==0.35.0
 psutil
 sentencepiece
 soundfile==0.13.1
-azure-ai-documentintelligence==1.0.2
 
 pillow==11.3.0
 opencv-python-headless==4.11.0.86
@@ -103,7 +87,6 @@ rank-bm25==0.2.2
 onnxruntime==1.20.1
 faster-whisper==1.1.1
 
-
 black==25.9.0
 youtube-transcript-api==1.2.2
 pytube==15.0.0
@@ -111,31 +94,49 @@ pytube==15.0.0
 pydub
 ddgs==9.0.0
 
+azure-ai-documentintelligence==1.0.2
+azure-identity==1.25.0
+azure-storage-blob==12.24.1
+azure-search-documents==11.6.0
+
 ## Google Drive
 google-api-python-client
 google-auth-httplib2
 google-auth-oauthlib
 
+googleapis-common-protos==1.70.0
+google-cloud-storage==2.19.0
+
+## Databases
+pymongo
+psycopg2-binary==2.9.10
+pgvector==0.4.1
+
+PyMySQL==1.1.1
+boto3==1.40.5
+
+pymilvus==2.6.4
+qdrant-client==1.14.3
+playwright==1.49.1 # Caution: version must match docker-compose.playwright.yaml
+elasticsearch==9.1.0
+pinecone==6.0.2
+oracledb==3.2.0
+
+av==14.0.1 # Caution: Set due to FATAL FIPS SELFTEST FAILURE, see discussion https://github.com/open-webui/open-webui/discussions/15720
+
+colbert-ai==0.2.21
+
+
 ## Tests
 docker~=7.1.0
 pytest~=8.4.1
 pytest-docker~=3.1.1
 
-googleapis-common-protos==1.70.0
-google-cloud-storage==2.19.0
-
-azure-identity==1.25.0
-azure-storage-blob==12.24.1
-
-
 ## LDAP
 ldap3==2.9.1
 
 ## Firecrawl
-firecrawl-py==1.12.0
-
-# Sougou API SDK(Tencentcloud SDK)
-tencentcloud-sdk-python==3.0.1336
+firecrawl-py==4.5.0
 
 ## Trace
 opentelemetry-api==1.37.0

docker-compose.yaml

@@ -11,8 +11,6 @@ services:
   open-webui:
     build:
       context: .
-      args:
-        OLLAMA_BASE_URL: '/ollama'
       dockerfile: Dockerfile
     image: ghcr.io/open-webui/open-webui:${WEBUI_DOCKER_TAG-main}
     container_name: open-webui

docs/CONTRIBUTING.md

@@ -24,6 +24,10 @@ Noticed something off? Have an idea? Check our [Issues tab](https://github.com/o
 > - **Template Compliance:** Please be aware that failure to follow the provided issue template, or not providing the requested information at all, will likely result in your issue being closed without further consideration. This approach is critical for maintaining the manageability and integrity of issue tracking.
 > - **Detail is Key:** To ensure your issue is understood and can be effectively addressed, it's imperative to include comprehensive details. Descriptions should be clear, including steps to reproduce, expected outcomes, and actual results. Lack of sufficient detail may hinder our ability to resolve your issue.
 
+> [!WARNING]
+> Reporting vulnerabilities is not wanted through Issues!
+> Instead, [use the security reporting functionality](https://github.com/open-webui/open-webui/security) and ensure you comply with the outlined requirements.
+
 ### 🧭 Scope of Support
 
 We've noticed an uptick in issues not directly related to Open WebUI but rather to the environment it's run in, especially Docker setups. While we strive to support Docker deployment, understanding Docker fundamentals is crucial for a smooth experience.
@@ -32,6 +36,8 @@ We've noticed an uptick in issues not directly related to Open WebUI but rather
 
 - **Advanced Configurations**: Setting up reverse proxies for HTTPS and managing Docker deployments requires foundational knowledge. There are numerous online resources available to learn these skills. Ensuring you have this knowledge will greatly enhance your experience with Open WebUI and similar projects.
 
+- **Check the documentation and help improve it**: [Our documentation](https://docs.openwebui.com) has ever growing troubleshooting guides and detailed installation tutorials. Please verify if it is of help to your issue and help expand it by submitting issues and PRs on our [Docs Repository](https://github.com/open-webui/docs).
+
 ## 💡 Contributing
 
 Looking to contribute? Great! Here's how you can help:
@@ -46,9 +52,15 @@ We welcome pull requests. Before submitting one, please:
 4. Write clear, descriptive commit messages.
 5. It's essential to complete your pull request in a timely manner. We move fast, and having PRs hang around too long is not feasible. If you can't get it done within a reasonable time frame, we may have to close it to keep the project moving forward.
 
+> [!NOTE]
+> The Pull Request Template has various requirements outlined. Go through the PR-checklist one by one and ensure you completed all steps before submitting your PR for review (you can open it as draft otherwise!).
+
 ### 📚 Documentation & Tutorials
 
-Help us make Open WebUI more accessible by improving documentation, writing tutorials, or creating guides on setting up and optimizing the web UI.
+Help us make Open WebUI more accessible by improving the documentation, writing tutorials, or creating guides on setting up and optimizing the Web UI.
+
+Help expand our documentation by submitting issues and PRs on our [Docs Repository](https://github.com/open-webui/docs).
+We welcome tutorials, guides and other documentation improvements!
 
 ### 🌐 Translations and Internationalization
 
@@ -62,9 +74,12 @@ To add a new language:
 - Copy the American English translation file(s) (from `en-US` directory in `src/lib/i18n/locale`) to this new directory and update the string values in JSON format according to your language. Make sure to preserve the structure of the JSON object.
 - Add the language code and its respective title to languages file at `src/lib/i18n/locales/languages.json`.
 
+> [!NOTE]
+> When adding new translations, do so in a standalone PR! Feature PRs or PRs fixing a bug should not contain translation updates. Always keep the scope of a PR narrow.
+
 ### 🤔 Questions & Feedback
 
-Got questions or feedback? Join our [Discord community](https://discord.gg/5rJgQTnV4s) or open an issue. We're here to help!
+Got questions or feedback? Join our [Discord community](https://discord.gg/5rJgQTnV4s) or open an issue or discussion. We're here to help!
 
 ## 🙏 Thank You!
 

docs/SECURITY.md

@@ -4,9 +4,10 @@ Our primary goal is to ensure the protection and confidentiality of sensitive da
 
 ## Supported Versions
 
-| Version | Supported          |
-| ------- | ------------------ |
+| Version (Branch) | Supported          |
+| ---------------- | ------------------ |
 | main             | :white_check_mark: |
+| dev              | :x:                |
 | others           | :x:                |
 
 ## Zero Tolerance for External Platforms
@@ -17,28 +18,113 @@ Any reports or solicitations arriving from sources other than our designated Git
 
 ## Reporting a Vulnerability
 
-We appreciate the community's interest in identifying potential vulnerabilities. However, effective immediately, we will **not** accept low-effort vulnerability reports. To ensure that submissions are constructive and actionable, please adhere to the following guidelines:
-
 Reports not submitted through our designated GitHub repository will be disregarded, and we will categorically reject invitations to collaborate on external platforms. Our aggressive stance on this matter underscores our commitment to a secure, transparent, and open community where all operations are visible and contributors are accountable.
 
-1. **No Vague Reports**: Submissions such as "I found a vulnerability" without any details will be treated as spam and will not be accepted.
+We appreciate the community's interest in identifying potential vulnerabilities. However, effective immediately, we will **not** accept low-effort vulnerability reports. Ensure that **submissions are constructive, actionable, reproducible, well documented and adhere to the following guidelines**:
 
-2. **In-Depth Understanding Required**: Reports must reflect a clear understanding of the codebase and provide specific details about the vulnerability, including the affected components and potential impacts.
+1. **Report MUST be a vulnerability:** A security vulnerability is an exploitable weakness where the system behaves in an unintended way, allowing attackers to bypass security controls, gain unauthorized access, execute arbitrary code, or escalate privileges. Configuration options, missing features, and expected protocol behavior are **not vulnerabilities**.
 
-3. **Proof of Concept (PoC) is Mandatory**: Each submission must include a well-documented proof of concept (PoC) that demonstrates the vulnerability. If confidentiality is a concern, reporters are encouraged to create a private fork of the repository and share access with the maintainers. Reports lacking valid evidence will be disregarded.
+2. **No Vague Reports**: Submissions such as "I found a vulnerability" without any details will be treated as spam and will not be accepted.
 
-4. **Required Patch Submission**: Along with the PoC, reporters must provide a patch or actionable steps to remediate the identified vulnerability. This helps us evaluate and implement fixes rapidly.
+3. **In-Depth Understanding Required**: Reports must reflect a clear understanding of the codebase and provide specific details about the vulnerability, including the affected components and potential impacts.
 
-5. **Streamlined Merging Process**: When vulnerability reports meet the above criteria, we can consider them for immediate merging, similar to regular pull requests. Well-structured and thorough submissions will expedite the process of enhancing our security.
+4. **Proof of Concept (PoC) is Mandatory**: Each submission must include a well-documented proof of concept (PoC) that demonstrates the vulnerability. If confidentiality is a concern, reporters are encouraged to create a private fork of the repository and share access with the maintainers. Reports lacking valid evidence may be disregarded.
 
-**Non-compliant submissions will be closed, and repeat violators may be banned.** Our goal is to foster a constructive reporting environment where quality submissions promote better security for all users.
+> [!NOTE]
+> A PoC (Proof of Concept) is a **demonstration of exploitation of a vulnerability**. Your PoC must show:
+>
+> 1. What security boundary was crossed (Confidentiality, Integrity, Availability, Authenticity, Non-repudiation)
+> 2. How this vulnerability was abused
+> 3. What actions the attacker can now perform
+>
+> **Examples of valid PoCs:**
+>
+> - Step-by-step reproduction instructions with exact commands
+> - Complete exploit code with detailed execution instructions
+> - Screenshots/videos demonstrating the exploit (supplementary to written steps)
+>
+> **Failure to provide a reproducible PoC may lead to closure of the report**
+>
+> We will notify you, if we struggle to reproduce the exploit using your PoC to allow you to improve your PoC.
+> However, if we repeatedly cannot reproduce the exploit using the PoC, the report may be closed.
 
-## Product Security
+5. **Required Patch or Actionable Remediation Plan Submission**: Along with the PoC, reporters must provide a patch or some actionable steps to remediate the identified vulnerability. This helps us evaluate and implement fixes rapidly.
+
+6. **Streamlined Merging Process**: When vulnerability reports meet the above criteria, we can consider provided patches for immediate merging, similar to regular pull requests. Well-structured and thorough submissions will expedite the process of enhancing our security.
+
+7. **Default Configuration Testing**: All vulnerability reports MUST be tested and reproducible using Open WebUI's out-of-the-box default configuration. Claims of vulnerabilities that only manifest with explicitly weakened security settings may be discarded, unless they are covered by the following exception:
+
+> [!NOTE]  
+> **Note**: If you believe you have found a security issue that
+>
+> 1. affects default configurations, **or**
+> 2. represents a genuine bypass of intended security controls, **or**
+> 3. works only with non-default configurations, **but the configuration in question is likely to be used by production deployments**, **then we absolutely want to hear about it.** This policy is intended to filter configuration issues and deployment problems, not to discourage legitimate security research.
+
+8. **Threat Model Understanding Required**: Reports must demonstrate understanding of Open WebUI's self-hosted, authenticated, role-based access control architecture. Comparing Open WebUI to services with fundamentally different security models without acknowledging the architectural differences may result in report rejection.
+
+9. **CVSS Scoring Accuracy:** If you include a CVSS score with your report, it must accurately reflect the vulnerability according to CVSS methodology. Common errors include 1) rating PR:N (None) when authentication is required, 2) scoring hypothetical attack chains instead of the actual vulnerability, or 3) inflating severity without evidence. **We will adjust inaccurate CVSS scores.** Intentionally inflated scores may result in report rejection.
+
+> [!WARNING]
+>
+> **Using CVE Precedents:** If you cite other CVEs to support your report, ensure they are **genuinely comparable** in vulnerability type, threat model, and attack vector. Citing CVEs from different product categories, different vulnerability classes or different deployment models will lead us to suspect the use of AI in your report.
+
+10. **Admin Actions Are Out of Scope:** Vulnerabilities that require an administrator to actively perform unsafe actions are **not considered valid vulnerabilities**. Admins have full system control and are expected to understand the security implications of their actions and configurations. This includes but is not limited to: adding malicious external servers (models, tools, webhooks), pasting untrusted code into Functions/Tools, or intentionally weakening security settings. **Reports requiring admin negligence or social engineering of admins may be rejected.**
+
+> [!NOTE]
+> Similar to rule "Default Configuration Testing": If you believe you have found a vulnerability that affects admins and is NOT caused by admin negligence or intentionally malicious actions,
+> **then we absolutely want to hear about it.** This policy is intended to filter social engineering attacks on admins, malicious plugins being deployed by admins and similar malicious actions, not to discourage legitimate security research.
+
+11. **AI report transparency:** Due to an extreme spike in AI-aided vulnerability reports **YOU MUST DISCLOSE if AI was used in any capacity** - whether for writing the report, generating the PoC, or identifying the vulnerability. If AI helped you in any way shape or form in the creation of the report, PoC or finding the vulnerability, you MUST disclose it.
+
+> [!NOTE]
+> AI-aided vulnerability reports **will not be rejected by us by default**. But:
+>
+> - If we suspect you used AI (but you did not disclose it to us), we will be asking tough follow-up questions to validate your understanding of the reported vulnerability and Open WebUI itself.
+> - If we suspect you used AI (but you did not disclose it to us) **and** your report ends up being invalid/not a vulnerability/not reproducible, then you **may be banned** from reporting future vulnerabilities.
+>
+> This measure was necessary due to the extreme rise in clearly AI written vulnerability reports, where the vast majority of them
+>
+> - were not a vulnerability
+> - were faulty configurations rather than a real vulnerability
+> - did not provide a PoC
+> - violated any of the rules outlined here
+> - had a clear lack of understanding of Open WebUI
+> - wrote comments with conflicting information
+> - used illogical arguments
+
+**Non-compliant submissions will be closed, and repeat extreme violators may be banned.** Our goal is to foster a constructive reporting environment where quality submissions promote better security for all users.
+
+## Where to report the vulnerability
+
+If you want to report a vulnerability and can meet the outlined requirements, [open a vulnerability report here](https://github.com/open-webui/open-webui/security/advisories/new).
+If you feel like you are not able to follow ALL outlined requirements for vulnerability-specific reasons, still do report it, we will check every report either way.
+
+## Product Security And For Non-Vulnerability Related Security Concerns:
+
+If your concern does not meet the vulnerability requirements outlined above, is not a vulnerability, **but is still related to security concerns**, then use the following channels instead:
+
+- **Documentation issues/improvement ideas:** Open an issue on our [Documentation Repository](https://github.com/open-webui/docs)
+- **Feature requests:** Create a discussion in [GitHub Discussions - Ideas](https://github.com/open-webui/open-webui/discussions/) to discuss with the community if this feature request is wanted by multiple people
+- **Configuration help:** Ask the community for help and guidance on our [Discord Server](https://discord.gg/5rJgQTnV4s) or on [Reddit](https://www.reddit.com/r/OpenWebUI/)
+- **General issues:** Use our [Issue Tracker](https://github.com/open-webui/open-webui/issues)
+
+**Examples of non-vulnerability, still security related concerns:**
+
+- Suggestions for better default configuration values
+- Security hardening recommendations
+- Deployment best practices guidance
+- Unclear configuration instructions
+- Need for additional security documentation
+- Feature requests for optional security enhancements (2FA, audit logging, etc.)
+- General security questions about production deployment
+
+Please use the adequate channel for your specific issue - e.g. best-practice guidance or additional documentation needs into the Documentation Repository, and feature requests into the Main Repository as an issue or discussion.
 
 We regularly audit our internal processes and system architecture for vulnerabilities using a combination of automated and manual testing techniques. We are also planning to implement SAST and SCA scans in our project soon.
 
-For immediate concerns or detailed reports that meet our guidelines, please create an issue in our [issue tracker](/open-webui/open-webui/issues) or contact us on [Discord](https://discord.gg/5rJgQTnV4s).
+For any other immediate concerns, please create an issue in our [issue tracker](https://github.com/open-webui/open-webui/issues) or contact our team on [Discord](https://discord.gg/5rJgQTnV4s).
 
 ---
 
-_Last updated on **2024-08-19**._
+_Last updated on **2025-11-06**._

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "open-webui",
-	"version": "0.6.32",
+	"version": "0.6.40",
 	"private": true,
 	"scripts": {
 		"dev": "npm run pyodide:fetch && vite dev --host",
@@ -24,8 +24,8 @@
 	"devDependencies": {
 		"@sveltejs/adapter-auto": "3.2.2",
 		"@sveltejs/adapter-static": "^3.0.2",
-		"@sveltejs/kit": "^2.5.20",
-		"@sveltejs/vite-plugin-svelte": "^3.1.1",
+		"@sveltejs/kit": "^2.5.27",
+		"@sveltejs/vite-plugin-svelte": "^4.0.0",
 		"@tailwindcss/container-queries": "^0.1.1",
 		"@tailwindcss/postcss": "^4.0.0",
 		"@tailwindcss/typography": "^0.5.13",
@@ -35,14 +35,14 @@
 		"eslint": "^8.56.0",
 		"eslint-config-prettier": "^9.1.0",
 		"eslint-plugin-cypress": "^3.4.0",
-		"eslint-plugin-svelte": "^2.43.0",
+		"eslint-plugin-svelte": "^2.45.1",
 		"i18next-parser": "^9.0.1",
 		"postcss": "^8.4.31",
 		"prettier": "^3.3.3",
 		"prettier-plugin-svelte": "^3.2.6",
 		"sass-embedded": "^1.81.0",
-		"svelte": "^4.2.18",
-		"svelte-check": "^3.8.5",
+		"svelte": "^5.0.0",
+		"svelte-check": "^4.0.0",
 		"svelte-confetti": "^1.3.2",
 		"tailwindcss": "^4.0.0",
 		"tslib": "^2.4.1",
@@ -137,6 +137,8 @@
 		"turndown-plugin-gfm": "^1.0.2",
 		"undici": "^7.3.0",
 		"uuid": "^9.0.1",
+		"vega": "^6.2.0",
+		"vega-lite": "^6.4.1",
 		"vite-plugin-static-copy": "^2.2.0",
 		"y-prosemirror": "^1.3.7",
 		"yaml": "^2.7.1",

pyproject.toml

@@ -6,19 +6,19 @@ authors = [
 ]
 license = { file = "LICENSE" }
 dependencies = [
-    "fastapi==0.115.7",
-    "uvicorn[standard]==0.35.0",
-    "pydantic==2.11.7",
+    "fastapi==0.118.0",
+    "uvicorn[standard]==0.37.0",
+    "pydantic==2.11.9",
     "python-multipart==0.0.20",
     "itsdangerous==2.2.0",
 
-    "python-socketio==5.13.0",
-    "python-jose==3.4.0",
+    "python-socketio==5.15.0",
+    "python-jose==3.5.0",
     "cryptography",
     "bcrypt==5.0.0",
     "argon2-cffi==25.1.0",
     "PyJWT[crypto]==2.10.1",
-    "authlib==1.6.3",
+    "authlib==1.6.5",
 
     "requests==2.32.5",
     "aiohttp==3.12.15",
@@ -37,9 +37,6 @@ dependencies = [
     "pycrdt==0.12.25",
     "redis",
 
-    "PyMySQL==1.1.1",
-    "boto3==1.40.5",
-
     "APScheduler==3.10.4",
     "RestrictedPython==8.0",
 
@@ -47,11 +44,11 @@ dependencies = [
     "asgiref==3.8.1",
 
     "tiktoken",
-    "mcp==1.14.1",
+    "mcp==1.21.2",
 
     "openai",
     "anthropic",
-    "google-genai==1.38.0",
+    "google-genai==1.52.0",
     "google-generativeai==0.8.5",
 
     "langchain==0.3.27",
@@ -60,22 +57,26 @@ dependencies = [
     "fake-useragent==2.2.0",
     "chromadb==1.0.20",
     "opensearch-py==2.8.0",
+    "PyMySQL==1.1.1",
+    "boto3==1.40.5",
     
-    "transformers",
-    "sentence-transformers==5.1.1",
+    "transformers==4.57.3",
+    "sentence-transformers==5.1.2",
     "accelerate",
     "pyarrow==20.0.0",
     "einops==0.8.1",
 
     "ftfy==6.2.3",
-    "pypdf==6.0.0",
+    "chardet==5.2.0",
+    "pypdf==6.4.0",
     "fpdf2==2.8.2",
     "pymdown-extensions==10.14.2",
     "docx2txt==0.8",
     "python-pptx==1.0.2",
-    "unstructured==0.16.17",
+    "unstructured==0.18.18",
+    "msoffcrypto-tool==5.4.2",
     "nltk==3.9.1",
-    "Markdown==3.8.2",
+    "Markdown==3.9",
     "pypandoc==1.15",
     "pandas==2.2.3",
     "openpyxl==3.1.5",
@@ -95,8 +96,8 @@ dependencies = [
     "onnxruntime==1.20.1",
     "faster-whisper==1.1.1",
 
-    "black==25.1.0",
-    "youtube-transcript-api==1.1.0",
+    "black==25.9.0",
+    "youtube-transcript-api==1.2.2",
     "pytube==15.0.0",
 
     "pydub",
@@ -106,8 +107,6 @@ dependencies = [
     "google-auth-httplib2",
     "google-auth-oauthlib",
 
-    
-
     "googleapis-common-protos==1.70.0",
     "google-cloud-storage==2.19.0",
 
@@ -115,12 +114,6 @@ dependencies = [
     "azure-storage-blob==12.24.1",
 
     "ldap3==2.9.1",
-
-    "firecrawl-py==1.12.0",
-    "tencentcloud-sdk-python==3.0.1336",
-    
-    "oracledb>=3.2.0",
-
 ]
 readme = "README.md"
 requires-python = ">= 3.11, < 3.13.0a1"
@@ -154,11 +147,14 @@ all = [
     "elasticsearch==9.1.0",
 
     "qdrant-client==1.14.3",
-    "pymilvus==2.5.0",
+    "weaviate-client==4.17.0",
+    "pymilvus==2.6.4",
     "pinecone==6.0.2",
     "oracledb==3.2.0",
-
     "colbert-ai==0.2.21",
+
+    "firecrawl-py==4.5.0",
+    "azure-search-documents==11.6.0",
 ]
 
 [project.scripts]

src/app.css

@@ -30,8 +30,33 @@
 	font-display: swap;
 }
 
+/* --app-text-scale is updated via the UI Scale slider (Interface.svelte) */
+:root {
+	--app-text-scale: 1;
+}
+
 html {
 	word-break: break-word;
+	/* font-size scales the entire document via the same UI control */
+	font-size: calc(1rem * var(--app-text-scale, 1));
+}
+
+#sidebar-chat-item {
+	/* sidebar item sizing scales for the chat list entries */
+	min-height: calc(32px * var(--app-text-scale, 1));
+	padding-inline: calc(11px * var(--app-text-scale, 1));
+	padding-block: calc(6px * var(--app-text-scale, 1));
+}
+
+#sidebar-chat-item div[dir='auto'] {
+	/* chat title line height follows the text scale */
+	height: calc(20px * var(--app-text-scale, 1));
+	line-height: calc(20px * var(--app-text-scale, 1));
+}
+
+#sidebar-chat-item input {
+	/* editing state input height is kept in sync */
+	min-height: calc(20px * var(--app-text-scale, 1));
 }
 
 code {
@@ -129,8 +154,8 @@ li p {
 }
 
 ::-webkit-scrollbar {
-	height: 0.4rem;
-	width: 0.4rem;
+	height: 0.45rem;
+	width: 0.45rem;
 }
 
 ::-webkit-scrollbar-track {
@@ -152,6 +177,14 @@ select {
 	-webkit-appearance: none;
 }
 
+.dark select:not([class*='bg-transparent']) {
+	@apply bg-gray-900 text-gray-300;
+}
+
+.dark select option {
+	@apply bg-gray-850 text-white;
+}
+
 @keyframes shimmer {
 	0% {
 		background-position: 200% 0;

src/app.html

@@ -23,12 +23,7 @@
 			href="/static/apple-touch-icon.png"
 			crossorigin="use-credentials"
 		/>
-		<link
-			rel="manifest"
-			href="/manifest.json"
-			crossorigin="use-credentials"
-			crossorigin="use-credentials"
-		/>
+		<link rel="manifest" href="/manifest.json" crossorigin="use-credentials" />
 		<meta
 			name="viewport"
 			content="width=device-width, initial-scale=1, maximum-scale=1, viewport-fit=cover"
@@ -179,9 +174,8 @@
 			</span> -->
 		</div>
 	</body>
-</html>
 
-<style type="text/css" nonce="">
+	<style type="text/css" nonce="">
 		html {
 			overflow-y: hidden !important;
 			overscroll-behavior-y: none;
@@ -247,4 +241,5 @@
 		.animate-pulse-fast {
 			animation: pulse 1.5s cubic-bezier(0.4, 0, 0.6, 1) infinite;
 		}
-</style>
+	</style>
+</html>

src/lib/apis/channels/index.ts

@@ -1,10 +1,12 @@
 import { WEBUI_API_BASE_URL } from '$lib/constants';
 
 type ChannelForm = {
+	type?: string;
 	name: string;
 	data?: object;
 	meta?: object;
 	access_control?: object;
+	user_ids?: string[];
 };
 
 export const createNewChannel = async (token: string = '', channel: ChannelForm) => {
@@ -101,6 +103,96 @@ export const getChannelById = async (token: string = '', channel_id: string) =>
 	return res;
 };
 
+export const getChannelMembersById = async (
+	token: string,
+	channel_id: string,
+	query?: string,
+	orderBy?: string,
+	direction?: string,
+	page = 1
+) => {
+	let error = null;
+	let res = null;
+
+	const searchParams = new URLSearchParams();
+
+	searchParams.set('page', `${page}`);
+
+	if (query) {
+		searchParams.set('query', query);
+	}
+
+	if (orderBy) {
+		searchParams.set('order_by', orderBy);
+	}
+
+	if (direction) {
+		searchParams.set('direction', direction);
+	}
+
+	res = await fetch(
+		`${WEBUI_API_BASE_URL}/channels/${channel_id}/members?${searchParams.toString()}`,
+		{
+			method: 'GET',
+			headers: {
+				'Content-Type': 'application/json',
+				Authorization: `Bearer ${token}`
+			}
+		}
+	)
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			console.error(err);
+			error = err.detail;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const updateChannelMemberActiveStatusById = async (
+	token: string = '',
+	channel_id: string,
+	is_active: boolean
+) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/channels/${channel_id}/members/active`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({ is_active })
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.error(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
 export const updateChannelById = async (
 	token: string = '',
 	channel_id: string,
@@ -207,6 +299,44 @@ export const getChannelMessages = async (
 	return res;
 };
 
+export const getChannelPinnedMessages = async (
+	token: string = '',
+	channel_id: string,
+	page: number = 1
+) => {
+	let error = null;
+
+	const res = await fetch(
+		`${WEBUI_API_BASE_URL}/channels/${channel_id}/messages/pinned?page=${page}`,
+		{
+			method: 'GET',
+			headers: {
+				Accept: 'application/json',
+				'Content-Type': 'application/json',
+				authorization: `Bearer ${token}`
+			}
+		}
+	)
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.error(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
 export const getChannelThreadMessages = async (
 	token: string = '',
 	channel_id: string,
@@ -248,6 +378,7 @@ export const getChannelThreadMessages = async (
 };
 
 type MessageForm = {
+	temp_id?: string;
 	reply_to_id?: string;
 	parent_id?: string;
 	content: string;
@@ -287,6 +418,46 @@ export const sendMessage = async (token: string = '', channel_id: string, messag
 	return res;
 };
 
+export const pinMessage = async (
+	token: string = '',
+	channel_id: string,
+	message_id: string,
+	is_pinned: boolean
+) => {
+	let error = null;
+
+	const res = await fetch(
+		`${WEBUI_API_BASE_URL}/channels/${channel_id}/messages/${message_id}/pin`,
+		{
+			method: 'POST',
+			headers: {
+				Accept: 'application/json',
+				'Content-Type': 'application/json',
+				authorization: `Bearer ${token}`
+			},
+			body: JSON.stringify({ is_pinned })
+		}
+	)
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.error(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
 export const updateMessage = async (
 	token: string = '',
 	channel_id: string,