From d454e6a03359155a10fd6e8305f1a640945206ea Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Sun, 10 Aug 2025 23:40:01 +0200 Subject: [PATCH 01/43] Feat/prune orphaned data (#16) * feat: Add prune orphaned data functionality * feat: Add prune orphaned data functionality * feat: Add prune orphaned data functionality * fix: Restyle PruneDataDialog modal * feat: Add comprehensive prune orphaned data functionality and fix circular import * feat: Add comprehensive prune orphaned data functionality and fix circular import * feat: Add comprehensive prune orphaned data functionality and fix database size issues * feat: Add comprehensive prune orphaned data functionality and fix database size issues * feat: Add comprehensive prune orphaned data functionality and fix database size issues * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update folders.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Update prune.py * Delete backend/open_webui/test/test_prune.py * Update prune.ts * Update PruneDataDialog.svelte * Update prune.py * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update prune.py * Update PruneDataDialog.svelte * Update prune.ts * Update Database.svelte * Update prune.py * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update prune.py * Update prune.py * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update Database.svelte * Update prune.py * Update prune.ts * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte * Update prune.py * Update prune.ts * Update PruneDataDialog.svelte * Update files.py * Update prompts.py * Update notes.py * Update models.py * Update access_control.py * Update PruneDataDialog.svelte * Update PruneDataDialog.svelte --------- Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> --- backend/open_webui/main.py | 2 + backend/open_webui/models/folders.py | 4 + backend/open_webui/routers/prune.py | 684 ++++++++++++++++++ src/lib/apis/prune.ts | 54 ++ .../components/admin/Settings/Database.svelte | 93 ++- .../components/common/PruneDataDialog.svelte | 589 +++++++++++++++ 6 files changed, 1402 insertions(+), 24 deletions(-) create mode 100644 backend/open_webui/routers/prune.py create mode 100644 src/lib/apis/prune.ts create mode 100644 src/lib/components/common/PruneDataDialog.svelte diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 618640486d..f6398b23fa 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -81,6 +81,7 @@ from open_webui.routers import ( models, knowledge, prompts, + prune, evaluations, tools, users, @@ -1234,6 +1235,7 @@ app.include_router( evaluations.router, prefix="/api/v1/evaluations", tags=["evaluations"] ) app.include_router(utils.router, prefix="/api/v1/utils", tags=["utils"]) +app.include_router(prune.router, prefix="/api/v1/prune", tags=["prune"]) # SCIM 2.0 API for identity management if SCIM_ENABLED: diff --git a/backend/open_webui/models/folders.py b/backend/open_webui/models/folders.py index 15deecbf42..8b631f88de 100644 --- a/backend/open_webui/models/folders.py +++ b/backend/open_webui/models/folders.py @@ -135,6 +135,10 @@ class FolderTable: for folder in db.query(Folder).filter_by(user_id=user_id).all() ] + def get_all_folders(self) -> list[FolderModel]: + with get_db() as db: + return [FolderModel.model_validate(folder) for folder in db.query(Folder).all()] + def get_folder_by_parent_id_and_user_id_and_name( self, parent_id: Optional[str], user_id: str, name: str ) -> Optional[FolderModel]: diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py new file mode 100644 index 0000000000..78c333e538 --- /dev/null +++ b/backend/open_webui/routers/prune.py @@ -0,0 +1,684 @@ +import logging +import time +import os +import shutil +import json +import re +from typing import Optional, Set +from pathlib import Path + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel +from sqlalchemy import text + +from open_webui.utils.auth import get_admin_user +from open_webui.models.users import Users +from open_webui.models.chats import Chats +from open_webui.models.files import Files +from open_webui.models.notes import Notes +from open_webui.models.prompts import Prompts +from open_webui.models.models import Models +from open_webui.models.knowledge import Knowledges +from open_webui.models.functions import Functions +from open_webui.models.tools import Tools +from open_webui.models.folders import Folders +from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT, VECTOR_DB +from open_webui.constants import ERROR_MESSAGES +from open_webui.env import SRC_LOG_LEVELS +from open_webui.config import CACHE_DIR +from open_webui.internal.db import get_db + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["MODELS"]) + +router = APIRouter() + + +class PruneDataForm(BaseModel): + days: Optional[int] = None + exempt_archived_chats: bool = False + exempt_chats_in_folders: bool = False + # Orphaned resource deletion toggles (for deleted users) + delete_orphaned_chats: bool = True + delete_orphaned_tools: bool = False + delete_orphaned_functions: bool = False + delete_orphaned_prompts: bool = True + delete_orphaned_knowledge_bases: bool = True + delete_orphaned_models: bool = True + delete_orphaned_notes: bool = True + delete_orphaned_folders: bool = True + + +def get_active_file_ids() -> Set[str]: + """ + Get all file IDs that are actively referenced by knowledge bases, chats, folders, and messages. + This is the ground truth for what files should be preserved. + """ + active_file_ids = set() + + try: + # 1. Get files referenced by knowledge bases (original logic) + knowledge_bases = Knowledges.get_knowledge_bases() + log.debug(f"Found {len(knowledge_bases)} knowledge bases") + + for kb in knowledge_bases: + if not kb.data: + continue + + # Handle different possible data structures for file references + file_ids = [] + + # Check for file_ids array + if isinstance(kb.data, dict) and "file_ids" in kb.data: + if isinstance(kb.data["file_ids"], list): + file_ids.extend(kb.data["file_ids"]) + + # Check for files array with id field + if isinstance(kb.data, dict) and "files" in kb.data: + if isinstance(kb.data["files"], list): + for file_ref in kb.data["files"]: + if isinstance(file_ref, dict) and "id" in file_ref: + file_ids.append(file_ref["id"]) + elif isinstance(file_ref, str): + file_ids.append(file_ref) + + # Add all found file IDs + for file_id in file_ids: + if isinstance(file_id, str) and file_id.strip(): + active_file_ids.add(file_id.strip()) + log.debug(f"KB {kb.id} references file {file_id}") + + # 2. Get files referenced in chats (NEW: scan chat JSON for file references) + chats = Chats.get_chats() + log.debug(f"Found {len(chats)} chats to scan for file references") + + for chat in chats: + if not chat.chat or not isinstance(chat.chat, dict): + continue + + try: + # Convert entire chat JSON to string and extract all file IDs + chat_json_str = json.dumps(chat.chat) + + # Find all file ID patterns in the JSON + # Pattern 1: "id": "uuid" where uuid looks like a file ID + file_id_pattern = re.compile(r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"') + potential_file_ids = file_id_pattern.findall(chat_json_str) + + # Pattern 2: URLs containing /api/v1/files/uuid + url_pattern = re.compile(r'/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})') + url_file_ids = url_pattern.findall(chat_json_str) + + # Combine and validate against actual file records + all_potential_ids = set(potential_file_ids + url_file_ids) + for file_id in all_potential_ids: + # Verify this ID exists in the file table to avoid false positives + if Files.get_file_by_id(file_id): + active_file_ids.add(file_id) + log.debug(f"Chat {chat.id}: Found active file {file_id}") + + except Exception as e: + log.debug(f"Error processing chat {chat.id} for file references: {e}") + + # 3. Get files referenced in folders (scan folder.items, folder.data, folder.meta) + try: + folders = Folders.get_all_folders() + log.debug(f"Found {len(folders)} folders to scan for file references") + + for folder in folders: + # Check folder.items JSON + if folder.items: + try: + items_str = json.dumps(folder.items) + # Look for file ID patterns in the JSON + file_id_pattern = re.compile(r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"') + url_pattern = re.compile(r'/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})') + + potential_ids = file_id_pattern.findall(items_str) + url_pattern.findall(items_str) + for file_id in potential_ids: + if Files.get_file_by_id(file_id): + active_file_ids.add(file_id) + log.debug(f"Folder {folder.id}: Found file {file_id} in items") + except Exception as e: + log.debug(f"Error processing folder {folder.id} items: {e}") + + # Check folder.data JSON + if hasattr(folder, 'data') and folder.data: + try: + data_str = json.dumps(folder.data) + file_id_pattern = re.compile(r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"') + url_pattern = re.compile(r'/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})') + + potential_ids = file_id_pattern.findall(data_str) + url_pattern.findall(data_str) + for file_id in potential_ids: + if Files.get_file_by_id(file_id): + active_file_ids.add(file_id) + log.debug(f"Folder {folder.id}: Found file {file_id} in data") + except Exception as e: + log.debug(f"Error processing folder {folder.id} data: {e}") + + except Exception as e: + log.debug(f"Error scanning folders for file references: {e}") + + # 4. Get files referenced in standalone messages (message table) + try: + # Query message table directly since we may not have a Messages model + with get_db() as db: + message_results = db.execute(text("SELECT id, data FROM message WHERE data IS NOT NULL")).fetchall() + log.debug(f"Found {len(message_results)} messages with data to scan") + + for message_id, message_data_json in message_results: + if message_data_json: + try: + # Convert JSON to string and scan for file patterns + data_str = json.dumps(message_data_json) if isinstance(message_data_json, dict) else str(message_data_json) + + file_id_pattern = re.compile(r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"') + url_pattern = re.compile(r'/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})') + + potential_ids = file_id_pattern.findall(data_str) + url_pattern.findall(data_str) + for file_id in potential_ids: + if Files.get_file_by_id(file_id): + active_file_ids.add(file_id) + log.debug(f"Message {message_id}: Found file {file_id}") + except Exception as e: + log.debug(f"Error processing message {message_id} data: {e}") + except Exception as e: + log.debug(f"Error scanning messages for file references: {e}") + + except Exception as e: + log.error(f"Error determining active file IDs: {e}") + # Fail safe: return empty set, which will prevent deletion + return set() + + log.info(f"Found {len(active_file_ids)} active file IDs") + return active_file_ids + + +def safe_delete_vector_collection(collection_name: str) -> bool: + """ + Safely delete a vector collection, handling both logical and physical cleanup. + """ + try: + # First, try to delete the collection through the client + try: + VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name) + log.debug(f"Deleted collection from vector DB: {collection_name}") + except Exception as e: + log.debug(f"Collection {collection_name} may not exist in DB: {e}") + + # Then, handle physical cleanup for ChromaDB + if "chroma" in VECTOR_DB.lower(): + vector_dir = Path(CACHE_DIR).parent / "vector_db" / collection_name + if vector_dir.exists() and vector_dir.is_dir(): + shutil.rmtree(vector_dir) + log.debug(f"Deleted physical vector directory: {vector_dir}") + return True + + return True + + except Exception as e: + log.error(f"Error deleting vector collection {collection_name}: {e}") + return False + + +def safe_delete_file_by_id(file_id: str) -> bool: + """ + Safely delete a file record and its associated vector collection. + """ + try: + # Get file info before deletion + file_record = Files.get_file_by_id(file_id) + if not file_record: + log.debug(f"File {file_id} not found in database") + return True # Already gone + + # Delete vector collection first + collection_name = f"file-{file_id}" + safe_delete_vector_collection(collection_name) + + # Delete database record + Files.delete_file_by_id(file_id) + log.debug(f"Deleted file record: {file_id}") + + return True + + except Exception as e: + log.error(f"Error deleting file {file_id}: {e}") + return False + + +def cleanup_orphaned_uploads(active_file_ids: Set[str]) -> None: + """ + Clean up orphaned files in the uploads directory. + """ + upload_dir = Path(CACHE_DIR).parent / "uploads" + if not upload_dir.exists(): + log.debug("Uploads directory does not exist") + return + + deleted_count = 0 + + try: + for file_path in upload_dir.iterdir(): + if not file_path.is_file(): + continue + + filename = file_path.name + + # Extract file ID from filename (common patterns) + file_id = None + + # Pattern 1: UUID_filename or UUID-filename + if len(filename) > 36: + potential_id = filename[:36] + if potential_id.count('-') == 4: # UUID format + file_id = potential_id + + # Pattern 2: filename might be the file ID itself + if not file_id and filename.count('-') == 4 and len(filename) == 36: + file_id = filename + + # Pattern 3: Check if any part of filename matches active IDs + if not file_id: + for active_id in active_file_ids: + if active_id in filename: + file_id = active_id + break + + # If we found a potential file ID and it's not active, delete it + if file_id and file_id not in active_file_ids: + try: + file_path.unlink() + deleted_count += 1 + log.debug(f"Deleted orphaned upload file: {filename}") + except Exception as e: + log.error(f"Failed to delete upload file {filename}: {e}") + + except Exception as e: + log.error(f"Error cleaning uploads directory: {e}") + + if deleted_count > 0: + log.info(f"Deleted {deleted_count} orphaned upload files") + + +def cleanup_orphaned_vector_collections(active_file_ids: Set[str], active_kb_ids: Set[str]) -> None: + """ + Clean up orphaned vector collections by querying ChromaDB metadata. + """ + if "chroma" not in VECTOR_DB.lower(): + return + + vector_dir = Path(CACHE_DIR).parent / "vector_db" + if not vector_dir.exists(): + log.debug("Vector DB directory does not exist") + return + + chroma_db_path = vector_dir / "chroma.sqlite3" + if not chroma_db_path.exists(): + log.debug("ChromaDB metadata file does not exist") + return + + # Build expected collection names + expected_collections = set() + + # File collections: file-{file_id} + for file_id in active_file_ids: + expected_collections.add(f"file-{file_id}") + + # Knowledge base collections: {kb_id} + for kb_id in active_kb_ids: + expected_collections.add(kb_id) + + log.debug(f"Expected collections to preserve: {expected_collections}") + + # Query ChromaDB metadata to get the complete mapping chain: + # Directory UUID -> Collection ID -> Collection Name + uuid_to_collection = {} + try: + import sqlite3 + log.debug(f"Attempting to connect to ChromaDB at: {chroma_db_path}") + + with sqlite3.connect(str(chroma_db_path)) as conn: + # First, check what tables exist + tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() + log.debug(f"ChromaDB tables: {tables}") + + # Check the schema of collections table + schema = conn.execute("PRAGMA table_info(collections)").fetchall() + log.debug(f"Collections table schema: {schema}") + + # Get Collection ID -> Collection Name mapping + collection_id_to_name = {} + cursor = conn.execute("SELECT id, name FROM collections") + rows = cursor.fetchall() + log.debug(f"Raw ChromaDB collections query results: {rows}") + + for row in rows: + collection_id, collection_name = row + collection_id_to_name[collection_id] = collection_name + log.debug(f"Mapped collection ID {collection_id} -> name {collection_name}") + + # Get Directory UUID -> Collection ID mapping from segments table + # Only interested in VECTOR segments as those are the actual data directories + cursor = conn.execute("SELECT id, collection FROM segments WHERE scope = 'VECTOR'") + segment_rows = cursor.fetchall() + log.debug(f"Raw ChromaDB segments query results: {segment_rows}") + + for row in segment_rows: + segment_id, collection_id = row + if collection_id in collection_id_to_name: + collection_name = collection_id_to_name[collection_id] + uuid_to_collection[segment_id] = collection_name + log.debug(f"Mapped directory UUID {segment_id} -> collection {collection_name}") + + log.debug(f"Final uuid_to_collection mapping: {uuid_to_collection}") + log.info(f"Found {len(uuid_to_collection)} vector segments in ChromaDB metadata") + + except Exception as e: + log.error(f"Error reading ChromaDB metadata: {e}") + # Fail safe: don't delete anything if we can't read metadata + return + + deleted_count = 0 + + try: + for collection_dir in vector_dir.iterdir(): + if not collection_dir.is_dir(): + continue + + dir_uuid = collection_dir.name + + # Skip system/metadata files + if dir_uuid.startswith('.'): + continue + + # Get the actual collection name from metadata + collection_name = uuid_to_collection.get(dir_uuid) + + if collection_name is None: + # Directory exists but no metadata entry - it's orphaned + log.debug(f"Directory {dir_uuid} has no metadata entry, deleting") + try: + shutil.rmtree(collection_dir) + deleted_count += 1 + except Exception as e: + log.error(f"Failed to delete orphaned directory {dir_uuid}: {e}") + + elif collection_name not in expected_collections: + # Collection exists but should be deleted + log.debug(f"Collection {collection_name} (UUID: {dir_uuid}) is orphaned, deleting") + try: + shutil.rmtree(collection_dir) + deleted_count += 1 + except Exception as e: + log.error(f"Failed to delete collection directory {dir_uuid}: {e}") + + else: + # Collection should be preserved + log.debug(f"Preserving collection {collection_name} (UUID: {dir_uuid})") + + except Exception as e: + log.error(f"Error cleaning vector collections: {e}") + + if deleted_count > 0: + log.info(f"Deleted {deleted_count} orphaned vector collections") + + +@router.post("/", response_model=bool) +async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): + """ + Prunes old and orphaned data using a safe, multi-stage process. + + Parameters: + - days: Optional[int] = None + - If None: Skip chat deletion entirely + - If 0: Delete all chats (older than 0 days = all chats) + - If >= 1: Delete chats older than specified number of days + - exempt_archived_chats: bool = False + - If True: Exempt archived chats from deletion (only applies when days is not None) + - exempt_chats_in_folders: bool = False + - If True: Exempt chats that are in folders OR pinned chats from deletion (only applies when days is not None) + Note: Pinned chats behave the same as chats in folders + - delete_orphaned_chats: bool = True + - If True: Delete chats from deleted users + - delete_orphaned_tools: bool = True + - If True: Delete tools from deleted users + - delete_orphaned_functions: bool = True + - If True: Delete functions from deleted users + - delete_orphaned_prompts: bool = True + - If True: Delete prompts from deleted users + - delete_orphaned_knowledge_bases: bool = True + - If True: Delete knowledge bases from deleted users + - delete_orphaned_models: bool = True + - If True: Delete models from deleted users + - delete_orphaned_notes: bool = True + - If True: Delete notes from deleted users + - delete_orphaned_folders: bool = True + - If True: Delete folders from deleted users + """ + try: + log.info("Starting data pruning process") + + # Stage 1: Delete old chats based on user criteria (optional) + if form_data.days is not None: + cutoff_time = int(time.time()) - (form_data.days * 86400) + chats_to_delete = [] + + for chat in Chats.get_chats(): + if chat.updated_at < cutoff_time: + # Check exemption conditions + if form_data.exempt_archived_chats and chat.archived: + log.debug(f"Exempting archived chat: {chat.id}") + continue + if form_data.exempt_chats_in_folders and (getattr(chat, 'folder_id', None) is not None or getattr(chat, 'pinned', False)): + folder_status = f"folder_id: {getattr(chat, 'folder_id', None)}" if getattr(chat, 'folder_id', None) else "not in folder" + pinned_status = f"pinned: {getattr(chat, 'pinned', False)}" + log.debug(f"Exempting chat in folder or pinned: {chat.id} ({folder_status}, {pinned_status})") + continue + log.debug(f"Chat {chat.id} will be deleted - archived: {getattr(chat, 'archived', False)}, folder_id: {getattr(chat, 'folder_id', None)}, pinned: {getattr(chat, 'pinned', False)}") + chats_to_delete.append(chat) + + if chats_to_delete: + log.info(f"Deleting {len(chats_to_delete)} old chats (older than {form_data.days} days)") + for chat in chats_to_delete: + Chats.delete_chat_by_id(chat.id) + else: + log.info(f"No chats found older than {form_data.days} days") + else: + log.info("Skipping chat deletion (days parameter is None)") + + # Stage 2: Build ground truth of what should be preserved + log.info("Building preservation set") + + # Get all active users + active_user_ids = {user.id for user in Users.get_users()["users"]} + log.info(f"Found {len(active_user_ids)} active users") + + # Get all active knowledge bases and their file references + active_kb_ids = set() + knowledge_bases = Knowledges.get_knowledge_bases() + + for kb in knowledge_bases: + if kb.user_id in active_user_ids: + active_kb_ids.add(kb.id) + + log.info(f"Found {len(active_kb_ids)} active knowledge bases") + + # Get all files that should be preserved (NOW COMPREHENSIVE!) + active_file_ids = get_active_file_ids() + + # Stage 3: Delete orphaned database records + log.info("Deleting orphaned database records") + + # Delete files not referenced by any knowledge base or belonging to deleted users + deleted_files = 0 + for file_record in Files.get_files(): + should_delete = ( + file_record.id not in active_file_ids or + file_record.user_id not in active_user_ids + ) + + if should_delete: + if safe_delete_file_by_id(file_record.id): + deleted_files += 1 + + if deleted_files > 0: + log.info(f"Deleted {deleted_files} orphaned files") + + # Delete knowledge bases from deleted users (if enabled) + deleted_kbs = 0 + if form_data.delete_orphaned_knowledge_bases: + for kb in knowledge_bases: + if kb.user_id not in active_user_ids: + if safe_delete_vector_collection(kb.id): + Knowledges.delete_knowledge_by_id(kb.id) + deleted_kbs += 1 + + if deleted_kbs > 0: + log.info(f"Deleted {deleted_kbs} orphaned knowledge bases") + else: + log.info("Skipping knowledge base deletion (disabled)") + + # Delete other user-owned resources from deleted users (conditional) + deleted_others = 0 + + # Delete orphaned chats of deleted users (conditional) + if form_data.delete_orphaned_chats: + chats_deleted = 0 + for chat in Chats.get_chats(): + if chat.user_id not in active_user_ids: + Chats.delete_chat_by_id(chat.id) + chats_deleted += 1 + deleted_others += 1 + if chats_deleted > 0: + log.info(f"Deleted {chats_deleted} orphaned chats") + else: + log.info("Skipping orphaned chat deletion (disabled)") + + # Delete orphaned tools of deleted users (conditional) + if form_data.delete_orphaned_tools: + tools_deleted = 0 + for tool in Tools.get_tools(): + if tool.user_id not in active_user_ids: + Tools.delete_tool_by_id(tool.id) + tools_deleted += 1 + deleted_others += 1 + if tools_deleted > 0: + log.info(f"Deleted {tools_deleted} orphaned tools") + else: + log.info("Skipping tool deletion (disabled)") + + # Delete orphaned functions of deleted users (conditional) + if form_data.delete_orphaned_functions: + functions_deleted = 0 + for function in Functions.get_functions(): + if function.user_id not in active_user_ids: + Functions.delete_function_by_id(function.id) + functions_deleted += 1 + deleted_others += 1 + if functions_deleted > 0: + log.info(f"Deleted {functions_deleted} orphaned functions") + else: + log.info("Skipping function deletion (disabled)") + + # Delete orphaned notes of deleted users (conditional) + if form_data.delete_orphaned_notes: + notes_deleted = 0 + for note in Notes.get_notes(): + if note.user_id not in active_user_ids: + Notes.delete_note_by_id(note.id) + notes_deleted += 1 + deleted_others += 1 + if notes_deleted > 0: + log.info(f"Deleted {notes_deleted} orphaned notes") + else: + log.info("Skipping note deletion (disabled)") + + # Delete orphaned prompts of deleted users (conditional) + if form_data.delete_orphaned_prompts: + prompts_deleted = 0 + for prompt in Prompts.get_prompts(): + if prompt.user_id not in active_user_ids: + Prompts.delete_prompt_by_command(prompt.command) + prompts_deleted += 1 + deleted_others += 1 + if prompts_deleted > 0: + log.info(f"Deleted {prompts_deleted} orphaned prompts") + else: + log.info("Skipping prompt deletion (disabled)") + + # Delete orphaned models of deleted users (conditional) + if form_data.delete_orphaned_models: + models_deleted = 0 + for model in Models.get_all_models(): + if model.user_id not in active_user_ids: + Models.delete_model_by_id(model.id) + models_deleted += 1 + deleted_others += 1 + if models_deleted > 0: + log.info(f"Deleted {models_deleted} orphaned models") + else: + log.info("Skipping model deletion (disabled)") + + # Delete orphaned folders of deleted users (conditional) + if form_data.delete_orphaned_folders: + folders_deleted = 0 + for folder in Folders.get_all_folders(): + if folder.user_id not in active_user_ids: + Folders.delete_folder_by_id_and_user_id(folder.id, folder.user_id, delete_chats=False) + folders_deleted += 1 + deleted_others += 1 + if folders_deleted > 0: + log.info(f"Deleted {folders_deleted} orphaned folders") + else: + log.info("Skipping folder deletion (disabled)") + + if deleted_others > 0: + log.info(f"Total other orphaned records deleted: {deleted_others}") + + # Stage 4: Clean up orphaned physical files + log.info("Cleaning up orphaned physical files") + + # Rebuild active sets after database cleanup + final_active_file_ids = get_active_file_ids() + final_active_kb_ids = {kb.id for kb in Knowledges.get_knowledge_bases()} + + # Clean uploads directory + cleanup_orphaned_uploads(final_active_file_ids) + + # Clean vector collections + cleanup_orphaned_vector_collections(final_active_file_ids, final_active_kb_ids) + + # Stage 5: Database optimization + log.info("Optimizing database") + + # Vacuum main database + try: + with get_db() as db: + db.execute(text("VACUUM")) + log.debug("Vacuumed main database") + except Exception as e: + log.error(f"Failed to vacuum main database: {e}") + + # Vacuum ChromaDB database if it exists + if "chroma" in VECTOR_DB.lower(): + chroma_db_path = Path(CACHE_DIR).parent / "vector_db" / "chroma.sqlite3" + if chroma_db_path.exists(): + try: + import sqlite3 + with sqlite3.connect(str(chroma_db_path)) as conn: + conn.execute("VACUUM") + log.debug("Vacuumed ChromaDB database") + except Exception as e: + log.error(f"Failed to vacuum ChromaDB database: {e}") + + log.info("Data pruning completed successfully") + return True + + except Exception as e: + log.exception(f"Error during data pruning: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=ERROR_MESSAGES.DEFAULT("Data pruning failed"), + ) diff --git a/src/lib/apis/prune.ts b/src/lib/apis/prune.ts new file mode 100644 index 0000000000..d95d662438 --- /dev/null +++ b/src/lib/apis/prune.ts @@ -0,0 +1,54 @@ +import { WEBUI_API_BASE_URL } from '$lib/constants'; + +export const pruneData = async ( + token: string, + days: number | null, + exempt_archived_chats: boolean, + exempt_chats_in_folders: boolean, + delete_orphaned_chats: boolean = true, + delete_orphaned_tools: boolean = false, + delete_orphaned_functions: boolean = false, + delete_orphaned_prompts: boolean = true, + delete_orphaned_knowledge_bases: boolean = true, + delete_orphaned_models: boolean = true, + delete_orphaned_notes: boolean = true, + delete_orphaned_folders: boolean = true +) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/prune/`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + days, + exempt_archived_chats, + exempt_chats_in_folders, + delete_orphaned_chats, + delete_orphaned_tools, + delete_orphaned_functions, + delete_orphaned_prompts, + delete_orphaned_knowledge_bases, + delete_orphaned_models, + delete_orphaned_notes, + delete_orphaned_folders + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; diff --git a/src/lib/components/admin/Settings/Database.svelte b/src/lib/components/admin/Settings/Database.svelte index b2ac5553de..19ec874746 100644 --- a/src/lib/components/admin/Settings/Database.svelte +++ b/src/lib/components/admin/Settings/Database.svelte @@ -1,7 +1,6 @@ - +
{ @@ -58,7 +91,6 @@
{$i18n.t('Database')}
- { const file = e.target.files[0]; const reader = new FileReader(); - reader.onload = async (e) => { const res = await importConfig(localStorage.token, JSON.parse(e.target.result)).catch( (error) => { toast.error(`${error}`); } ); - if (res) { toast.success('Config imported successfully'); } e.target.value = null; }; - reader.readAsText(file); }} /> - - -
- {#if $config?.features.enable_admin_export ?? true}
-
-
-
{/if} +
+ - -
+ \ No newline at end of file diff --git a/src/lib/components/common/PruneDataDialog.svelte b/src/lib/components/common/PruneDataDialog.svelte new file mode 100644 index 0000000000..10a29d2594 --- /dev/null +++ b/src/lib/components/common/PruneDataDialog.svelte @@ -0,0 +1,589 @@ + + + +
+
+
+ {$i18n.t('Prune Orphaned Data')} +
+ +
+ +
+
+ +
+
+
+ + + +
+
+

+ {$i18n.t('Destructive Operation - Backup Recommended')} +

+
+

{$i18n.t('This action will permanently delete data from your database. Only orphaned or old data, based on your configuration settings, will be deleted. All active, referenced data remains completely safe.')}

+

{$i18n.t('This operation cannot be undone. Create a complete backup of your database and files before proceeding. This operation is performed entirely at your own risk - having a backup ensures you can restore any data if something unexpected occurs.')}

+ + +
+ + + {#if showDetailsExpanded} +
+

{$i18n.t('Note:')} {$i18n.t('This list provides an overview of what will be deleted during the pruning process and may not be complete or fully up-to-date.')}

+ + +
+ + + + + +
+ + +
+ {#if activeDetailsTab === 'chats'} +
+

{$i18n.t('Age-Based Chat Deletion:')}

+

• {$i18n.t('Removes conversations older than specified days based on when they were last modified or updated (not when they were created)')}

+

• {$i18n.t('Supports exemptions for:')}

+

◦ {$i18n.t('Archived chats')}

+

◦ {$i18n.t('Chats organized in folders and pinned chats')}

+ +

{$i18n.t('Orphaned Content Cleanup:')}

+

• {$i18n.t('Delete orphaned chats from deleted users')}

+

• {$i18n.t('Delete orphaned folders from deleted users')}

+
+ {:else if activeDetailsTab === 'workspace'} +
+

{$i18n.t('Orphaned Workspace Items from Deleted Users:')}

+

• {$i18n.t('Delete orphaned knowledge bases')}

+

• {$i18n.t('Delete orphaned custom tools')}

+

• {$i18n.t('Delete orphaned custom functions (Actions, Pipes, Filters)')}

+

• {$i18n.t('Delete orphaned custom prompts and templates')}

+

• {$i18n.t('Delete orphaned custom models and configurations')}

+

• {$i18n.t('Delete orphaned notes')}

+
+ {:else if activeDetailsTab === 'datavector'} +
+

{$i18n.t('Files & Vector Storage:')}

+

• {$i18n.t('Orphaned files and attachments from deleted content')}

+

• {$i18n.t('Vector embeddings and collections for removed data')}

+

• {$i18n.t('Uploaded files that lost their database references')}

+

• {$i18n.t('Vector storage directories without corresponding data')}

+
+ {:else if activeDetailsTab === 'imagesaudio'} +
+

{$i18n.t('Images & Audio Content Cleanup:')}

+

• {$i18n.t('TBD - Image cleanup functionality')}

+

• {$i18n.t('TBD - Audio cleanup functionality')}

+

• {$i18n.t('TBD - Orphaned images and audio files')}

+

• {$i18n.t('TBD - Media processing cache cleanup')}

+
+ {:else if activeDetailsTab === 'system'} +
+

{$i18n.t('Database & System Cleanup:')}

+

• {$i18n.t('Removal of broken database references and stale entries')}

+

• {$i18n.t('Disk space reclamation by database cleanup')}

+

• {$i18n.t('Synchronization of database records with actual file storage')}

+

• {$i18n.t('Fix inconsistencies between storage systems')}

+

• {$i18n.t('Database performance optimization')}

+
+ {/if} +
+
+ {/if} +
+
+
+
+
+ + +
+
+
+ + + +
+
+

+ {$i18n.t('Performance Warning: This operation may take a very long time to complete, especially if you have never cleaned your database before or if your instance stores large amounts of data. The process could take anywhere from seconds, to minutes, to half an hour and beyond depending on your data size.')} +

+
+
+
+ + +
+
+ + + +

+ {$i18n.t('Pruning Configuration')} +

+
+

+ {$i18n.t('Configure what data should be cleaned up during the pruning process.')} +

+ + +
+ + +
+ + +
+ {#if activeSettingsTab === 'chats'} + +
+
+
+
+ +
+
+
+ {$i18n.t('Delete chats by age')} +
+
+ {$i18n.t('Optionally remove old chats based on last update time')} +
+
+
+
+ + + {#if deleteChatsByAge} +
+
+ +
+ + {$i18n.t('days')} +
+

+ {$i18n.t('Set to 0 to delete all chats, or specify number of days')} +

+
+ +
+
+
+ +
+
+
+ {$i18n.t('Exempt archived chats')} +
+
+ {$i18n.t('Keep archived chats even if they are old')} +
+
+
+
+ +
+
+
+ +
+
+
+ {$i18n.t('Exempt chats in folders')} +
+
+ {$i18n.t('Keep chats that are organized in folders or pinned')} +
+
+
+
+
+ {/if} + + +
+
+
+
+ +
+
+
+ {$i18n.t('Delete orphaned chats')} +
+
+ {$i18n.t('Delete orphaned chats from deleted users')} +
+
+
+
+ +
+
+
+ +
+
+
+ {$i18n.t('Delete orphaned folders')} +
+
+ {$i18n.t('Delete orphaned folders from deleted users')} +
+
+
+
+
+
+ + {:else if activeSettingsTab === 'workspace'} +
+ +
+
+
+ +
+
+
+ {$i18n.t('Delete orphaned knowledge bases')} +
+
+ {$i18n.t('Delete orphaned knowledge bases from deleted users')} +
+
+
+
+ + +
+
+
+ +
+
+
+ {$i18n.t('Delete orphaned tools')} +
+
+ {$i18n.t('Delete orphaned custom tools from deleted users')} +
+
+
+
+ + +
+
+
+ +
+
+
+ {$i18n.t('Delete orphaned functions')} +
+ + + +
+
{$i18n.t('Admin panel functions - all functions, including:')}
+
+
• {$i18n.t('Actions')}
+
• {$i18n.t('Pipes')}
+
• {$i18n.t('Filters')}
+
+
+
+
+
+
+ {$i18n.t('Delete orphaned custom functions from deleted users')} +
+
+
+
+ + +
+
+
+ +
+
+
+ {$i18n.t('Delete orphaned prompts')} +
+
+ {$i18n.t('Delete orphaned custom prompts from deleted users')} +
+
+
+
+ + +
+
+
+ +
+
+
+ {$i18n.t('Delete orphaned models')} +
+
+ {$i18n.t('Delete orphaned custom models from deleted users')} +
+
+
+
+ + +
+
+
+ +
+
+
+ {$i18n.t('Delete orphaned notes')} +
+
+ {$i18n.t('Delete orphaned notes from deleted users')} +
+
+
+
+
+ {/if} +
+
+ + +
+
+
+ + + +
+
+

+ {$i18n.t('API Automation Helper')} +

+ + + + {#if showApiPreview} +
+

+ {$i18n.t('Use this API call configuration to automate pruning operations in your own maintenance scripts.')} +

+
+ + +
+
+ {/if} +
+
+
+
+ + +
+ + +
+
+
+ From 028a2e598497f4f28d0b583a309911af0f17dc8f Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:47:19 +0200 Subject: [PATCH 02/43] Update prune.py --- backend/open_webui/routers/prune.py | 62 ++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 78c333e538..d8b221e87d 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -47,6 +47,8 @@ class PruneDataForm(BaseModel): delete_orphaned_models: bool = True delete_orphaned_notes: bool = True delete_orphaned_folders: bool = True + # Audio cache cleanup + audio_cache_max_age_days: Optional[int] = 30 def get_active_file_ids() -> Set[str]: @@ -425,6 +427,57 @@ def cleanup_orphaned_vector_collections(active_file_ids: Set[str], active_kb_ids log.info(f"Deleted {deleted_count} orphaned vector collections") +def cleanup_audio_cache(max_age_days: Optional[int] = 30) -> None: + """ + Clean up audio cache files older than specified days. + + Args: + max_age_days: Delete audio files older than this many days. If None, skip audio cleanup. + """ + if max_age_days is None: + log.info("Skipping audio cache cleanup (max_age_days is None)") + return + + cutoff_time = time.time() - (max_age_days * 86400) + deleted_count = 0 + total_size_deleted = 0 + + # Audio cache directories + audio_dirs = [ + Path(CACHE_DIR) / "audio" / "speech", + Path(CACHE_DIR) / "audio" / "transcriptions" + ] + + for audio_dir in audio_dirs: + if not audio_dir.exists(): + log.debug(f"Audio directory does not exist: {audio_dir}") + continue + + try: + for file_path in audio_dir.iterdir(): + if not file_path.is_file(): + continue + + # Check file age + file_mtime = file_path.stat().st_mtime + if file_mtime < cutoff_time: + try: + file_size = file_path.stat().st_size + file_path.unlink() + deleted_count += 1 + total_size_deleted += file_size + log.debug(f"Deleted old audio file: {file_path}") + except Exception as e: + log.error(f"Failed to delete audio file {file_path}: {e}") + + except Exception as e: + log.error(f"Error cleaning audio directory {audio_dir}: {e}") + + if deleted_count > 0: + size_mb = total_size_deleted / (1024 * 1024) + log.info(f"Deleted {deleted_count} audio cache files ({size_mb:.1f} MB), older than {max_age_days} days") + + @router.post("/", response_model=bool) async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): """ @@ -456,6 +509,9 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): - If True: Delete notes from deleted users - delete_orphaned_folders: bool = True - If True: Delete folders from deleted users + - audio_cache_max_age_days: Optional[int] = 30 + - If None: Skip audio cache cleanup + - If >= 0: Delete audio cache files (TTS, STT) older than specified days """ try: log.info("Starting data pruning process") @@ -650,7 +706,11 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): # Clean vector collections cleanup_orphaned_vector_collections(final_active_file_ids, final_active_kb_ids) - # Stage 5: Database optimization + # Stage 5: Audio cache cleanup + log.info("Cleaning audio cache") + cleanup_audio_cache(form_data.audio_cache_max_age_days) + + # Stage 6: Database optimization log.info("Optimizing database") # Vacuum main database From 0bd42e5c6d93d2bea2930041636124148a8b47d0 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:47:34 +0200 Subject: [PATCH 03/43] Update Database.svelte --- .../components/admin/Settings/Database.svelte | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/lib/components/admin/Settings/Database.svelte b/src/lib/components/admin/Settings/Database.svelte index 19ec874746..736f201931 100644 --- a/src/lib/components/admin/Settings/Database.svelte +++ b/src/lib/components/admin/Settings/Database.svelte @@ -32,7 +32,8 @@ delete_orphaned_knowledge_bases, delete_orphaned_models, delete_orphaned_notes, - delete_orphaned_folders + delete_orphaned_folders, + audio_cache_max_age_days } = event.detail; const res = await pruneData( @@ -47,7 +48,8 @@ delete_orphaned_knowledge_bases, delete_orphaned_models, delete_orphaned_notes, - delete_orphaned_folders + delete_orphaned_folders, + audio_cache_max_age_days ).catch((error) => { toast.error(`${error}`); return null; @@ -243,15 +245,15 @@ - - -
- {$i18n.t('Export Users')} -
- - {/if} + clip-rule="evenodd" + /> + + +
+ {$i18n.t('Export Users')} +
+ + {/if}
+ - +
{#if activeSettingsTab === 'chats'} @@ -508,6 +520,61 @@ Authorization: Bearer
+ + {:else if activeSettingsTab === 'audio'} + +
+
+
+
+ +
+
+
+ {$i18n.t('Clean audio cache')} +
+
+ {$i18n.t('Remove old audio cache files (TTS and STT recordings)')} +
+
+
+
+ + + {#if cleanupAudioCache} +
+
+ +
+ + {$i18n.t('days')} +
+

+ {$i18n.t('Remove cached TTS (text-to-speech) and STT (speech-to-text) files older than specified days')} +

+
+ +
+
+ {$i18n.t('Audio Cache Types:')} +
+
+

• {$i18n.t('TTS Files:')} {$i18n.t('Generated audio files when AI speaks text to you')}

+

• {$i18n.t('STT Files:')} {$i18n.t('Uploaded audio files for transcription (voice messages)')}

+

• {$i18n.t('Metadata:')} {$i18n.t('Associated JSON files with transcription data')}

+
+
+
+ {/if} +
{/if} From 8d7273afaeb64e144b3cf91a26d2553df4db405a Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:48:05 +0200 Subject: [PATCH 05/43] Update prune.ts --- src/lib/apis/prune.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/apis/prune.ts b/src/lib/apis/prune.ts index d95d662438..8413ca24c0 100644 --- a/src/lib/apis/prune.ts +++ b/src/lib/apis/prune.ts @@ -12,7 +12,8 @@ export const pruneData = async ( delete_orphaned_knowledge_bases: boolean = true, delete_orphaned_models: boolean = true, delete_orphaned_notes: boolean = true, - delete_orphaned_folders: boolean = true + delete_orphaned_folders: boolean = true, + audio_cache_max_age_days: number | null = 30 ) => { let error = null; @@ -33,7 +34,8 @@ export const pruneData = async ( delete_orphaned_knowledge_bases, delete_orphaned_models, delete_orphaned_notes, - delete_orphaned_folders + delete_orphaned_folders, + audio_cache_max_age_days }) }) .then(async (res) => { From e4a0bd86405d9eb7ba613e3401c221d9733ab35b Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 12 Aug 2025 13:15:38 +0200 Subject: [PATCH 06/43] Update Database.svelte --- .../components/admin/Settings/Database.svelte | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/lib/components/admin/Settings/Database.svelte b/src/lib/components/admin/Settings/Database.svelte index 736f201931..1ee2d79325 100644 --- a/src/lib/components/admin/Settings/Database.svelte +++ b/src/lib/components/admin/Settings/Database.svelte @@ -1,6 +1,7 @@ +
{ const file = e.target.files[0]; const reader = new FileReader(); + reader.onload = async (e) => { const res = await importConfig(localStorage.token, JSON.parse(e.target.result)).catch( (error) => { toast.error(`${error}`); } ); + if (res) { toast.success('Config imported successfully'); } e.target.value = null; }; + reader.readAsText(file); }} /> + + +
+ {#if $config?.features.enable_admin_export ?? true}
+
+ +
- {/if} + clip-rule="evenodd" + /> + + +
+ {$i18n.t('Export Users')} +
+ + {/if}
From 7abcc7bc590cbac7839d4554034963feeb516828 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:16:31 +0200 Subject: [PATCH 20/43] Update Database.svelte --- .../components/admin/Settings/Database.svelte | 238 +++++++++++++++--- 1 file changed, 204 insertions(+), 34 deletions(-) diff --git a/src/lib/components/admin/Settings/Database.svelte b/src/lib/components/admin/Settings/Database.svelte index 2a8f221aa5..ea2de29e4d 100644 --- a/src/lib/components/admin/Settings/Database.svelte +++ b/src/lib/components/admin/Settings/Database.svelte @@ -17,6 +17,10 @@ export let saveHandler: Function; let showPruneDataDialog = false; + let showPreviewResults = false; + let previewResults = null; + let lastPruneSettings = null; + const exportAllUserChats = async () => { let blob = new Blob([JSON.stringify(await getAllUserChats(localStorage.token))], { type: 'application/json' @@ -24,48 +28,70 @@ saveAs(blob, `all-chats-export-${Date.now()}.json`); }; - const handlePruneDataConfirm = async (event) => { - const { - days, - exempt_archived_chats, - exempt_chats_in_folders, - delete_orphaned_chats, - delete_orphaned_tools, - delete_orphaned_functions, - delete_orphaned_prompts, - delete_orphaned_knowledge_bases, - delete_orphaned_models, - delete_orphaned_notes, - delete_orphaned_folders, - audio_cache_max_age_days, - delete_inactive_users_days, - exempt_admin_users, - exempt_pending_users - } = event.detail; + const handlePruneDataPreview = async (event) => { + const settings = event.detail; + lastPruneSettings = settings; const res = await pruneData( localStorage.token, - days, - exempt_archived_chats, - exempt_chats_in_folders, - delete_orphaned_chats, - delete_orphaned_tools, - delete_orphaned_functions, - delete_orphaned_prompts, - delete_orphaned_knowledge_bases, - delete_orphaned_models, - delete_orphaned_notes, - delete_orphaned_folders, - audio_cache_max_age_days, - delete_inactive_users_days, - exempt_admin_users, - exempt_pending_users + settings.days, + settings.exempt_archived_chats, + settings.exempt_chats_in_folders, + settings.delete_orphaned_chats, + settings.delete_orphaned_tools, + settings.delete_orphaned_functions, + settings.delete_orphaned_prompts, + settings.delete_orphaned_knowledge_bases, + settings.delete_orphaned_models, + settings.delete_orphaned_notes, + settings.delete_orphaned_folders, + settings.audio_cache_max_age_days, + settings.delete_inactive_users_days, + settings.exempt_admin_users, + settings.exempt_pending_users, + true // dry_run = true for preview ).catch((error) => { toast.error(`${error}`); return null; }); + + if (res) { + previewResults = res; + showPreviewResults = true; + } + }; + + const handleConfirmPrune = async () => { + if (!lastPruneSettings) return; + + const res = await pruneData( + localStorage.token, + lastPruneSettings.days, + lastPruneSettings.exempt_archived_chats, + lastPruneSettings.exempt_chats_in_folders, + lastPruneSettings.delete_orphaned_chats, + lastPruneSettings.delete_orphaned_tools, + lastPruneSettings.delete_orphaned_functions, + lastPruneSettings.delete_orphaned_prompts, + lastPruneSettings.delete_orphaned_knowledge_bases, + lastPruneSettings.delete_orphaned_models, + lastPruneSettings.delete_orphaned_notes, + lastPruneSettings.delete_orphaned_folders, + lastPruneSettings.audio_cache_max_age_days, + lastPruneSettings.delete_inactive_users_days, + lastPruneSettings.exempt_admin_users, + lastPruneSettings.exempt_pending_users, + false // dry_run = false for actual pruning + ).catch((error) => { + toast.error(`${error}`); + return null; + }); + if (res) { toast.success('Data pruned successfully'); + showPreviewResults = false; + previewResults = null; + lastPruneSettings = null; } }; @@ -97,7 +123,151 @@ }); - + +{#if showPreviewResults && previewResults} +
+
+
+

+ {$i18n.t('Pruning Preview Results')} +

+ +
+ +
+
+

+ {$i18n.t('The following items would be deleted:')} +

+
+ {#if previewResults.inactive_users > 0} +
+ {$i18n.t('Inactive users')}: + {previewResults.inactive_users} +
+ {/if} + {#if previewResults.old_chats > 0} +
+ {$i18n.t('Old chats')}: + {previewResults.old_chats} +
+ {/if} + {#if previewResults.orphaned_chats > 0} +
+ {$i18n.t('Orphaned chats')}: + {previewResults.orphaned_chats} +
+ {/if} + {#if previewResults.orphaned_files > 0} +
+ {$i18n.t('Orphaned files')}: + {previewResults.orphaned_files} +
+ {/if} + {#if previewResults.orphaned_tools > 0} +
+ {$i18n.t('Orphaned tools')}: + {previewResults.orphaned_tools} +
+ {/if} + {#if previewResults.orphaned_functions > 0} +
+ {$i18n.t('Orphaned functions')}: + {previewResults.orphaned_functions} +
+ {/if} + {#if previewResults.orphaned_prompts > 0} +
+ {$i18n.t('Orphaned prompts')}: + {previewResults.orphaned_prompts} +
+ {/if} + {#if previewResults.orphaned_knowledge_bases > 0} +
+ {$i18n.t('Orphaned knowledge bases')}: + {previewResults.orphaned_knowledge_bases} +
+ {/if} + {#if previewResults.orphaned_models > 0} +
+ {$i18n.t('Orphaned models')}: + {previewResults.orphaned_models} +
+ {/if} + {#if previewResults.orphaned_notes > 0} +
+ {$i18n.t('Orphaned notes')}: + {previewResults.orphaned_notes} +
+ {/if} + {#if previewResults.orphaned_folders > 0} +
+ {$i18n.t('Orphaned folders')}: + {previewResults.orphaned_folders} +
+ {/if} + {#if previewResults.orphaned_uploads > 0} +
+ {$i18n.t('Orphaned upload files')}: + {previewResults.orphaned_uploads} +
+ {/if} + {#if previewResults.orphaned_vector_collections > 0} +
+ {$i18n.t('Orphaned vector collections')}: + {previewResults.orphaned_vector_collections} +
+ {/if} + {#if previewResults.audio_cache_files > 0} +
+ {$i18n.t('Audio cache files')}: + {previewResults.audio_cache_files} +
+ {/if} +
+ + {#if Object.values(previewResults).every(count => count === 0)} +
+
+ {$i18n.t('No items would be deleted with current settings')} +
+
+ {$i18n.t('Your system is already clean or no cleanup options are enabled')} +
+
+ {/if} +
+ + +
+ + {#if !Object.values(previewResults).every(count => count === 0)} + + {/if} +
+
+
+
+{/if} + +
{ From 808fd0324de32ab99b4529447a385aa9b5889bd2 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:16:42 +0200 Subject: [PATCH 21/43] Update prune.ts --- src/lib/apis/prune.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/apis/prune.ts b/src/lib/apis/prune.ts index 63c251b801..c7abb4152c 100644 --- a/src/lib/apis/prune.ts +++ b/src/lib/apis/prune.ts @@ -16,7 +16,8 @@ export const pruneData = async ( audio_cache_max_age_days: number | null = 30, delete_inactive_users_days: number | null = null, exempt_admin_users: boolean = true, - exempt_pending_users: boolean = true + exempt_pending_users: boolean = true, + dry_run: boolean = true ) => { let error = null; @@ -41,7 +42,8 @@ export const pruneData = async ( audio_cache_max_age_days, delete_inactive_users_days, exempt_admin_users, - exempt_pending_users + exempt_pending_users, + dry_run }) }) .then(async (res) => { From 28f0079193fdad53c76a523aac9e5670f36940e3 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:16:59 +0200 Subject: [PATCH 22/43] Update prune.py --- backend/open_webui/routers/prune.py | 342 ++++++++++++++++++++++++---- 1 file changed, 302 insertions(+), 40 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 7cb0498523..3b47d6767b 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -5,7 +5,7 @@ import shutil import json import re import sqlite3 -from typing import Optional, Set +from typing import Optional, Set, Union from pathlib import Path from fastapi import APIRouter, Depends, HTTPException, status @@ -53,6 +53,263 @@ class PruneDataForm(BaseModel): exempt_pending_users: bool = True +class PrunePreviewResult(BaseModel): + inactive_users: int = 0 + old_chats: int = 0 + orphaned_chats: int = 0 + orphaned_files: int = 0 + orphaned_tools: int = 0 + orphaned_functions: int = 0 + orphaned_prompts: int = 0 + orphaned_knowledge_bases: int = 0 + orphaned_models: int = 0 + orphaned_notes: int = 0 + orphaned_folders: int = 0 + orphaned_uploads: int = 0 + orphaned_vector_collections: int = 0 + audio_cache_files: int = 0 + + +# Counting helper functions for dry-run preview +def count_inactive_users(inactive_days: Optional[int], exempt_admin: bool, exempt_pending: bool) -> int: + """Count users that would be deleted for inactivity.""" + if inactive_days is None: + return 0 + + cutoff_time = int(time.time()) - (inactive_days * 86400) + count = 0 + + try: + all_users = Users.get_users()["users"] + for user in all_users: + if exempt_admin and user.role == "admin": + continue + if exempt_pending and user.role == "pending": + continue + if user.last_active_at < cutoff_time: + count += 1 + except Exception as e: + log.debug(f"Error counting inactive users: {e}") + + return count + + +def count_old_chats(days: Optional[int], exempt_archived: bool, exempt_in_folders: bool) -> int: + """Count chats that would be deleted by age.""" + if days is None: + return 0 + + cutoff_time = int(time.time()) - (days * 86400) + count = 0 + + try: + for chat in Chats.get_chats(): + if chat.updated_at < cutoff_time: + if exempt_archived and chat.archived: + continue + if exempt_in_folders and ( + getattr(chat, "folder_id", None) is not None + or getattr(chat, "pinned", False) + ): + continue + count += 1 + except Exception as e: + log.debug(f"Error counting old chats: {e}") + + return count + + +def count_orphaned_records(form_data: PruneDataForm) -> dict: + """Count orphaned database records that would be deleted.""" + counts = { + "chats": 0, + "files": 0, + "tools": 0, + "functions": 0, + "prompts": 0, + "knowledge_bases": 0, + "models": 0, + "notes": 0, + "folders": 0 + } + + try: + # Get active user IDs + active_user_ids = {user.id for user in Users.get_users()["users"]} + + # Get active file IDs for file orphan detection + active_file_ids = get_active_file_ids() + + # Count orphaned files + for file_record in Files.get_files(): + should_delete = ( + file_record.id not in active_file_ids + or file_record.user_id not in active_user_ids + ) + if should_delete: + counts["files"] += 1 + + # Count other orphaned records + if form_data.delete_orphaned_chats: + for chat in Chats.get_chats(): + if chat.user_id not in active_user_ids: + counts["chats"] += 1 + + if form_data.delete_orphaned_tools: + for tool in Tools.get_tools(): + if tool.user_id not in active_user_ids: + counts["tools"] += 1 + + if form_data.delete_orphaned_functions: + for function in Functions.get_functions(): + if function.user_id not in active_user_ids: + counts["functions"] += 1 + + if form_data.delete_orphaned_prompts: + for prompt in Prompts.get_prompts(): + if prompt.user_id not in active_user_ids: + counts["prompts"] += 1 + + if form_data.delete_orphaned_knowledge_bases: + for kb in Knowledges.get_knowledge_bases(): + if kb.user_id not in active_user_ids: + counts["knowledge_bases"] += 1 + + if form_data.delete_orphaned_models: + for model in Models.get_all_models(): + if model.user_id not in active_user_ids: + counts["models"] += 1 + + if form_data.delete_orphaned_notes: + for note in Notes.get_notes(): + if note.user_id not in active_user_ids: + counts["notes"] += 1 + + if form_data.delete_orphaned_folders: + for folder in Folders.get_all_folders(): + if folder.user_id not in active_user_ids: + counts["folders"] += 1 + + except Exception as e: + log.debug(f"Error counting orphaned records: {e}") + + return counts + + +def count_orphaned_uploads(active_file_ids: Set[str]) -> int: + """Count orphaned files in uploads directory.""" + upload_dir = Path(CACHE_DIR).parent / "uploads" + if not upload_dir.exists(): + return 0 + + count = 0 + try: + for file_path in upload_dir.iterdir(): + if not file_path.is_file(): + continue + + filename = file_path.name + file_id = None + + # Extract file ID from filename patterns + if len(filename) > 36: + potential_id = filename[:36] + if potential_id.count("-") == 4: + file_id = potential_id + + if not file_id and filename.count("-") == 4 and len(filename) == 36: + file_id = filename + + if not file_id: + for active_id in active_file_ids: + if active_id in filename: + file_id = active_id + break + + if file_id and file_id not in active_file_ids: + count += 1 + except Exception as e: + log.debug(f"Error counting orphaned uploads: {e}") + + return count + + +def count_orphaned_vector_collections(active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + """Count orphaned vector collections.""" + if "chroma" not in VECTOR_DB.lower(): + return 0 + + vector_dir = Path(CACHE_DIR).parent / "vector_db" + if not vector_dir.exists(): + return 0 + + chroma_db_path = vector_dir / "chroma.sqlite3" + if not chroma_db_path.exists(): + return 0 + + expected_collections = set() + for file_id in active_file_ids: + expected_collections.add(f"file-{file_id}") + for kb_id in active_kb_ids: + expected_collections.add(kb_id) + + count = 0 + try: + uuid_to_collection = {} + with sqlite3.connect(str(chroma_db_path)) as conn: + collection_id_to_name = {} + cursor = conn.execute("SELECT id, name FROM collections") + for collection_id, collection_name in cursor.fetchall(): + collection_id_to_name[collection_id] = collection_name + + cursor = conn.execute("SELECT id, collection FROM segments WHERE scope = 'VECTOR'") + for segment_id, collection_id in cursor.fetchall(): + if collection_id in collection_id_to_name: + collection_name = collection_id_to_name[collection_id] + uuid_to_collection[segment_id] = collection_name + + for collection_dir in vector_dir.iterdir(): + if not collection_dir.is_dir() or collection_dir.name.startswith("."): + continue + + dir_uuid = collection_dir.name + collection_name = uuid_to_collection.get(dir_uuid) + + if collection_name is None or collection_name not in expected_collections: + count += 1 + except Exception as e: + log.debug(f"Error counting orphaned vector collections: {e}") + + return count + + +def count_audio_cache_files(max_age_days: Optional[int]) -> int: + """Count audio cache files that would be deleted.""" + if max_age_days is None: + return 0 + + cutoff_time = time.time() - (max_age_days * 86400) + count = 0 + + audio_dirs = [ + Path(CACHE_DIR) / "audio" / "speech", + Path(CACHE_DIR) / "audio" / "transcriptions", + ] + + for audio_dir in audio_dirs: + if not audio_dir.exists(): + continue + + try: + for file_path in audio_dir.iterdir(): + if file_path.is_file() and file_path.stat().st_mtime < cutoff_time: + count += 1 + except Exception as e: + log.debug(f"Error counting audio files in {audio_dir}: {e}") + + return count + + def get_active_file_ids() -> Set[str]: """ Get all file IDs that are actively referenced by knowledge bases, chats, folders, and messages. @@ -483,49 +740,54 @@ def cleanup_audio_cache(max_age_days: Optional[int] = 30) -> None: ) -@router.post("/", response_model=bool) -async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): +@router.post("/", response_model=Union[bool, PrunePreviewResult]) +async def prune_data(form_data: PruneDataForm, dry_run: bool = True, user=Depends(get_admin_user)): """ Prunes old and orphaned data using a safe, multi-stage process. - - Parameters: - - days: Optional[int] = None - - If None: Skip chat deletion entirely - - If 0: Delete all chats (older than 0 days = all chats) - - If >= 1: Delete chats older than specified number of days - - exempt_archived_chats: bool = False - - If True: Exempt archived chats from deletion (only applies when days is not None) - - exempt_chats_in_folders: bool = False - - If True: Exempt chats that are in folders OR pinned chats from deletion (only applies when days is not None) - Note: Pinned chats behave the same as chats in folders - - delete_orphaned_chats: bool = True - - If True: Delete chats from deleted users - - delete_orphaned_tools: bool = True - - If True: Delete tools from deleted users - - delete_orphaned_functions: bool = True - - If True: Delete functions from deleted users - - delete_orphaned_prompts: bool = True - - If True: Delete prompts from deleted users - - delete_orphaned_knowledge_bases: bool = True - - If True: Delete knowledge bases from deleted users - - delete_orphaned_models: bool = True - - If True: Delete models from deleted users - - delete_orphaned_notes: bool = True - - If True: Delete notes from deleted users - - delete_orphaned_folders: bool = True - - If True: Delete folders from deleted users - - audio_cache_max_age_days: Optional[int] = 30 - - If None: Skip audio cache cleanup - - If >= 0: Delete audio cache files (TTS, STT) older than specified days - - delete_inactive_users_days: Optional[int] = None - - If None: Skip inactive user deletion - - If >= 1: Delete users inactive for more than specified days - - exempt_admin_users: bool = True - - If True: Exempt admin users from deletion (recommended for safety) - - exempt_pending_users: bool = True - - If True: Exempt pending users from deletion (recommended for safety) + + If dry_run=True (default), returns preview counts without deleting anything. + If dry_run=False, performs actual deletion and returns True on success. """ try: + if dry_run: + log.info("Starting data pruning preview (dry run)") + + # Get counts for all enabled operations + active_file_ids = get_active_file_ids() + active_user_ids = {user.id for user in Users.get_users()["users"]} + active_kb_ids = {kb.id for kb in Knowledges.get_knowledge_bases() if kb.user_id in active_user_ids} + + orphaned_counts = count_orphaned_records(form_data) + + result = PrunePreviewResult( + inactive_users=count_inactive_users( + form_data.delete_inactive_users_days, + form_data.exempt_admin_users, + form_data.exempt_pending_users + ), + old_chats=count_old_chats( + form_data.days, + form_data.exempt_archived_chats, + form_data.exempt_chats_in_folders + ), + orphaned_chats=orphaned_counts["chats"], + orphaned_files=orphaned_counts["files"], + orphaned_tools=orphaned_counts["tools"], + orphaned_functions=orphaned_counts["functions"], + orphaned_prompts=orphaned_counts["prompts"], + orphaned_knowledge_bases=orphaned_counts["knowledge_bases"], + orphaned_models=orphaned_counts["models"], + orphaned_notes=orphaned_counts["notes"], + orphaned_folders=orphaned_counts["folders"], + orphaned_uploads=count_orphaned_uploads(active_file_ids), + orphaned_vector_collections=count_orphaned_vector_collections(active_file_ids, active_kb_ids), + audio_cache_files=count_audio_cache_files(form_data.audio_cache_max_age_days) + ) + + log.info("Data pruning preview completed") + return result + + # Actual deletion logic (dry_run=False) log.info("Starting data pruning process") # Stage 0: Delete inactive users (if enabled) From bc19b515279d0ea97c2c933c8c3e7b854a5f91ce Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:29:11 +0200 Subject: [PATCH 23/43] Update prune.py From 0230a1208b3f3bce6c8f42afbeecbdad749205f9 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:29:31 +0200 Subject: [PATCH 24/43] Update Database.svelte From f6c7c145a88f1bf44594ad0ffe6ac95479602052 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:29:43 +0200 Subject: [PATCH 25/43] Update PruneDataDialog.svelte From 98650bd7d9f9e82f344be4d5d4af5f08e8a82e26 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:29:51 +0200 Subject: [PATCH 26/43] Update prune.ts --- src/lib/apis/prune.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/apis/prune.ts b/src/lib/apis/prune.ts index c7abb4152c..5dda128836 100644 --- a/src/lib/apis/prune.ts +++ b/src/lib/apis/prune.ts @@ -17,7 +17,7 @@ export const pruneData = async ( delete_inactive_users_days: number | null = null, exempt_admin_users: boolean = true, exempt_pending_users: boolean = true, - dry_run: boolean = true + dry_run: boolean // Removed default value to ensure explicit passing ) => { let error = null; From 2681fd268bdd25584f4a5fd79c04957b07178700 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:33:02 +0200 Subject: [PATCH 27/43] Update prune.py --- backend/open_webui/routers/prune.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 3b47d6767b..2ab89d985a 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -51,6 +51,7 @@ class PruneDataForm(BaseModel): delete_inactive_users_days: Optional[int] = None exempt_admin_users: bool = True exempt_pending_users: bool = True + dry_run: bool = True class PrunePreviewResult(BaseModel): From 13100ab9b362e7eafb7dd316cf963d72d4ab6887 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:33:04 +0200 Subject: [PATCH 28/43] Update Database.svelte --- src/lib/components/admin/Settings/Database.svelte | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/components/admin/Settings/Database.svelte b/src/lib/components/admin/Settings/Database.svelte index ea2de29e4d..7b2072ed29 100644 --- a/src/lib/components/admin/Settings/Database.svelte +++ b/src/lib/components/admin/Settings/Database.svelte @@ -32,6 +32,7 @@ const settings = event.detail; lastPruneSettings = settings; + console.log('Preview call - dry_run should be TRUE'); const res = await pruneData( localStorage.token, settings.days, @@ -64,6 +65,7 @@ const handleConfirmPrune = async () => { if (!lastPruneSettings) return; + console.log('Confirm call - dry_run should be FALSE'); const res = await pruneData( localStorage.token, lastPruneSettings.days, From 262848d647cf3f9584519254ed9baf7106f60e71 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:39:47 +0200 Subject: [PATCH 29/43] Update prune.py --- backend/open_webui/routers/prune.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 2ab89d985a..4c6db6b60b 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -749,6 +749,8 @@ async def prune_data(form_data: PruneDataForm, dry_run: bool = True, user=Depend If dry_run=True (default), returns preview counts without deleting anything. If dry_run=False, performs actual deletion and returns True on success. """ + log.info(f"DEBUG: dry_run parameter = {dry_run}") + log.info(f"DEBUG: form_data.dry_run = {form_data.dry_run}") try: if dry_run: log.info("Starting data pruning preview (dry run)") From 4c7e6bd752f58f3f9b785a75a3cd5f34835a0902 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 16:43:06 +0200 Subject: [PATCH 30/43] Update prune.py --- backend/open_webui/routers/prune.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 4c6db6b60b..ccc9950d62 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -742,17 +742,15 @@ def cleanup_audio_cache(max_age_days: Optional[int] = 30) -> None: @router.post("/", response_model=Union[bool, PrunePreviewResult]) -async def prune_data(form_data: PruneDataForm, dry_run: bool = True, user=Depends(get_admin_user)): +async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): """ Prunes old and orphaned data using a safe, multi-stage process. If dry_run=True (default), returns preview counts without deleting anything. If dry_run=False, performs actual deletion and returns True on success. """ - log.info(f"DEBUG: dry_run parameter = {dry_run}") - log.info(f"DEBUG: form_data.dry_run = {form_data.dry_run}") try: - if dry_run: + if form_data.dry_run: log.info("Starting data pruning preview (dry run)") # Get counts for all enabled operations From b5d93ae3db96167b207c09e539f08dc6f97cac75 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 17:02:36 +0200 Subject: [PATCH 31/43] Update prune.py --- backend/open_webui/routers/prune.py | 117 +++++++++++++++------------- 1 file changed, 61 insertions(+), 56 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index ccc9950d62..530cb754d0 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -35,6 +35,55 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"]) router = APIRouter() +class JSONFileIDExtractor: + """ + Utility for extracting and validating file IDs from JSON content. + + Replaces duplicated regex compilation and validation logic used throughout + the file scanning functions. Compiles patterns once for better performance. + """ + + # Compile patterns once at class level for performance + _FILE_ID_PATTERN = re.compile( + r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"' + ) + _URL_PATTERN = re.compile( + r"/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})" + ) + + @classmethod + def extract_and_validate_file_ids(cls, json_string: str) -> Set[str]: + """ + Extract file IDs from JSON string and validate they exist in database. + + Args: + json_string: JSON content as string (or any string to scan) + + Returns: + Set of validated file IDs that exist in the Files table + + Note: + This method replaces the repeated pattern of: + 1. Compiling the same regex patterns + 2. Extracting potential IDs + 3. Validating each ID exists via Files.get_file_by_id() + 4. Building a set of validated IDs + """ + validated_ids = set() + + # Extract potential IDs using both patterns + potential_ids = [] + potential_ids.extend(cls._FILE_ID_PATTERN.findall(json_string)) + potential_ids.extend(cls._URL_PATTERN.findall(json_string)) + + # Validate each ID exists in database + for file_id in potential_ids: + if Files.get_file_by_id(file_id): + validated_ids.add(file_id) + + return validated_ids + + class PruneDataForm(BaseModel): days: Optional[int] = None exempt_archived_chats: bool = False @@ -354,22 +403,9 @@ def get_active_file_ids() -> Set[str]: try: chat_json_str = json.dumps(chat.chat) - - # Extract file IDs using regex patterns - file_id_pattern = re.compile( - r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"' - ) - url_pattern = re.compile( - r"/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})" - ) - - potential_file_ids = file_id_pattern.findall(chat_json_str) - url_file_ids = url_pattern.findall(chat_json_str) - - all_potential_ids = set(potential_file_ids + url_file_ids) - for file_id in all_potential_ids: - if Files.get_file_by_id(file_id): - active_file_ids.add(file_id) + # Use utility to extract and validate file IDs + validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids(chat_json_str) + active_file_ids.update(validated_ids) except Exception as e: log.debug(f"Error processing chat {chat.id} for file references: {e}") @@ -382,38 +418,18 @@ def get_active_file_ids() -> Set[str]: if folder.items: try: items_str = json.dumps(folder.items) - file_id_pattern = re.compile( - r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"' - ) - url_pattern = re.compile( - r"/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})" - ) - - potential_ids = file_id_pattern.findall( - items_str - ) + url_pattern.findall(items_str) - for file_id in potential_ids: - if Files.get_file_by_id(file_id): - active_file_ids.add(file_id) + # Use utility to extract and validate file IDs + validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids(items_str) + active_file_ids.update(validated_ids) except Exception as e: log.debug(f"Error processing folder {folder.id} items: {e}") if hasattr(folder, "data") and folder.data: try: data_str = json.dumps(folder.data) - file_id_pattern = re.compile( - r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"' - ) - url_pattern = re.compile( - r"/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})" - ) - - potential_ids = file_id_pattern.findall( - data_str - ) + url_pattern.findall(data_str) - for file_id in potential_ids: - if Files.get_file_by_id(file_id): - active_file_ids.add(file_id) + # Use utility to extract and validate file IDs + validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids(data_str) + active_file_ids.update(validated_ids) except Exception as e: log.debug(f"Error processing folder {folder.id} data: {e}") @@ -435,20 +451,9 @@ def get_active_file_ids() -> Set[str]: if isinstance(message_data_json, dict) else str(message_data_json) ) - - file_id_pattern = re.compile( - r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"' - ) - url_pattern = re.compile( - r"/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})" - ) - - potential_ids = file_id_pattern.findall( - data_str - ) + url_pattern.findall(data_str) - for file_id in potential_ids: - if Files.get_file_by_id(file_id): - active_file_ids.add(file_id) + # Use utility to extract and validate file IDs + validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids(data_str) + active_file_ids.update(validated_ids) except Exception as e: log.debug( f"Error processing message {message_id} data: {e}" From bfa2eb631d050ac9e58f4ea0e9bbc579d243ae17 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 17:37:02 +0200 Subject: [PATCH 32/43] Update prune.py --- backend/open_webui/routers/prune.py | 467 +++++++++++++++++----------- 1 file changed, 290 insertions(+), 177 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 530cb754d0..a4d6fc588f 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -7,6 +7,7 @@ import re import sqlite3 from typing import Optional, Set, Union from pathlib import Path +from abc import ABC, abstractmethod from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel @@ -84,6 +85,276 @@ class JSONFileIDExtractor: return validated_ids +class VectorDatabaseCleaner(ABC): + """ + Abstract base class for vector database cleanup operations. + + This interface defines the contract that all vector database implementations + must follow. Community contributors can implement support for new vector + databases by extending this class. + + Supported operations: + - Count orphaned collections (for dry-run preview) + - Cleanup orphaned collections (actual deletion) + - Delete individual collections by name + """ + + @abstractmethod + def count_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + """ + Count how many orphaned vector collections would be deleted. + + Args: + active_file_ids: Set of file IDs that are still referenced + active_kb_ids: Set of knowledge base IDs that are still active + + Returns: + Number of orphaned collections that would be deleted + """ + pass + + @abstractmethod + def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + """ + Actually delete orphaned vector collections. + + Args: + active_file_ids: Set of file IDs that are still referenced + active_kb_ids: Set of knowledge base IDs that are still active + + Returns: + Number of collections that were actually deleted + """ + pass + + @abstractmethod + def delete_collection(self, collection_name: str) -> bool: + """ + Delete a specific vector collection by name. + + Args: + collection_name: Name of the collection to delete + + Returns: + True if deletion was successful, False otherwise + """ + pass + + +class ChromaDatabaseCleaner(VectorDatabaseCleaner): + """ + ChromaDB-specific implementation of vector database cleanup. + + Handles ChromaDB's specific storage structure including: + - SQLite metadata database (chroma.sqlite3) + - Physical vector storage directories + - Collection name to UUID mapping + - Segment-based storage architecture + """ + + def __init__(self): + self.vector_dir = Path(CACHE_DIR).parent / "vector_db" + self.chroma_db_path = self.vector_dir / "chroma.sqlite3" + + def count_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + """Count orphaned ChromaDB collections for preview.""" + if not self.chroma_db_path.exists(): + return 0 + + expected_collections = self._build_expected_collections(active_file_ids, active_kb_ids) + uuid_to_collection = self._get_collection_mappings() + + count = 0 + try: + for collection_dir in self.vector_dir.iterdir(): + if not collection_dir.is_dir() or collection_dir.name.startswith("."): + continue + + dir_uuid = collection_dir.name + collection_name = uuid_to_collection.get(dir_uuid) + + if collection_name is None or collection_name not in expected_collections: + count += 1 + except Exception as e: + log.debug(f"Error counting orphaned ChromaDB collections: {e}") + + return count + + def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + """Actually delete orphaned ChromaDB collections.""" + if not self.chroma_db_path.exists(): + return 0 + + expected_collections = self._build_expected_collections(active_file_ids, active_kb_ids) + uuid_to_collection = self._get_collection_mappings() + + deleted_count = 0 + + try: + for collection_dir in self.vector_dir.iterdir(): + if not collection_dir.is_dir() or collection_dir.name.startswith("."): + continue + + dir_uuid = collection_dir.name + collection_name = uuid_to_collection.get(dir_uuid) + + # Delete if no corresponding collection name or collection is not expected + if collection_name is None: + try: + shutil.rmtree(collection_dir) + deleted_count += 1 + log.debug(f"Deleted orphaned ChromaDB directory: {dir_uuid}") + except Exception as e: + log.error(f"Failed to delete orphaned directory {dir_uuid}: {e}") + + elif collection_name not in expected_collections: + try: + shutil.rmtree(collection_dir) + deleted_count += 1 + log.debug(f"Deleted orphaned ChromaDB collection: {collection_name}") + except Exception as e: + log.error(f"Failed to delete collection directory {dir_uuid}: {e}") + + except Exception as e: + log.error(f"Error cleaning ChromaDB collections: {e}") + + if deleted_count > 0: + log.info(f"Deleted {deleted_count} orphaned ChromaDB collections") + + return deleted_count + + def delete_collection(self, collection_name: str) -> bool: + """Delete a specific ChromaDB collection by name.""" + try: + # Attempt to delete via ChromaDB client first + try: + VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name) + log.debug(f"Deleted ChromaDB collection via client: {collection_name}") + except Exception as e: + log.debug(f"Collection {collection_name} may not exist in ChromaDB: {e}") + + # Also clean up physical directory if it exists + # Note: ChromaDB uses UUID directories, so we'd need to map collection name to UUID + # For now, let the cleanup_orphaned_collections method handle physical cleanup + return True + + except Exception as e: + log.error(f"Error deleting ChromaDB collection {collection_name}: {e}") + return False + + def _build_expected_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> Set[str]: + """Build set of collection names that should exist.""" + expected_collections = set() + + # File collections use "file-{id}" pattern + for file_id in active_file_ids: + expected_collections.add(f"file-{file_id}") + + # Knowledge base collections use the KB ID directly + for kb_id in active_kb_ids: + expected_collections.add(kb_id) + + return expected_collections + + def _get_collection_mappings(self) -> dict: + """Get mapping from ChromaDB directory UUID to collection name.""" + uuid_to_collection = {} + + try: + with sqlite3.connect(str(self.chroma_db_path)) as conn: + # First, get collection ID to name mapping + collection_id_to_name = {} + cursor = conn.execute("SELECT id, name FROM collections") + for collection_id, collection_name in cursor.fetchall(): + collection_id_to_name[collection_id] = collection_name + + # Then, get segment ID to collection mapping (segments are the directory UUIDs) + cursor = conn.execute("SELECT id, collection FROM segments WHERE scope = 'VECTOR'") + for segment_id, collection_id in cursor.fetchall(): + if collection_id in collection_id_to_name: + collection_name = collection_id_to_name[collection_id] + uuid_to_collection[segment_id] = collection_name + + log.debug(f"Found {len(uuid_to_collection)} ChromaDB vector segments") + + except Exception as e: + log.error(f"Error reading ChromaDB metadata: {e}") + + return uuid_to_collection + + +class PGVectorDatabaseCleaner(VectorDatabaseCleaner): + """ + Placeholder implementation for PGVector database cleanup. + + This is a stub implementation that can be extended by the community + to support PGVector-specific cleanup operations. + + According to PR feedback, PGVector stores data in document_chunk table + and cleanup should involve finding rows with matching file IDs. + """ + + def count_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + """Count orphaned PGVector collections - to be implemented by community.""" + log.debug("PGVector collection counting not yet implemented") + return 0 + + def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + """Cleanup orphaned PGVector collections - to be implemented by community.""" + log.debug("PGVector collection cleanup not yet implemented") + return 0 + + def delete_collection(self, collection_name: str) -> bool: + """Delete PGVector collection - to be implemented by community.""" + log.debug(f"PGVector collection deletion not yet implemented: {collection_name}") + return True + + +class NoOpVectorDatabaseCleaner(VectorDatabaseCleaner): + """ + No-operation implementation for unsupported vector databases. + + This implementation does nothing and is used when the configured + vector database is not supported by the cleanup system. + """ + + def count_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + """No orphaned collections to count for unsupported databases.""" + return 0 + + def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + """No collections to cleanup for unsupported databases.""" + return 0 + + def delete_collection(self, collection_name: str) -> bool: + """No collection to delete for unsupported databases.""" + return True + + +def get_vector_database_cleaner() -> VectorDatabaseCleaner: + """ + Factory function to get the appropriate vector database cleaner. + + This function detects the configured vector database type and returns + the appropriate cleaner implementation. Community contributors can + extend this function to support additional vector databases. + + Returns: + VectorDatabaseCleaner: Appropriate implementation for the configured database + """ + vector_db_type = VECTOR_DB.lower() + + if "chroma" in vector_db_type: + log.debug("Using ChromaDB cleaner") + return ChromaDatabaseCleaner() + elif "pgvector" in vector_db_type: + log.debug("Using PGVector cleaner (placeholder implementation)") + return PGVectorDatabaseCleaner() + else: + log.debug(f"No specific cleaner for vector database type: {VECTOR_DB}, using no-op cleaner") + return NoOpVectorDatabaseCleaner() + + class PruneDataForm(BaseModel): days: Optional[int] = None exempt_archived_chats: bool = False @@ -284,55 +555,6 @@ def count_orphaned_uploads(active_file_ids: Set[str]) -> int: return count -def count_orphaned_vector_collections(active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: - """Count orphaned vector collections.""" - if "chroma" not in VECTOR_DB.lower(): - return 0 - - vector_dir = Path(CACHE_DIR).parent / "vector_db" - if not vector_dir.exists(): - return 0 - - chroma_db_path = vector_dir / "chroma.sqlite3" - if not chroma_db_path.exists(): - return 0 - - expected_collections = set() - for file_id in active_file_ids: - expected_collections.add(f"file-{file_id}") - for kb_id in active_kb_ids: - expected_collections.add(kb_id) - - count = 0 - try: - uuid_to_collection = {} - with sqlite3.connect(str(chroma_db_path)) as conn: - collection_id_to_name = {} - cursor = conn.execute("SELECT id, name FROM collections") - for collection_id, collection_name in cursor.fetchall(): - collection_id_to_name[collection_id] = collection_name - - cursor = conn.execute("SELECT id, collection FROM segments WHERE scope = 'VECTOR'") - for segment_id, collection_id in cursor.fetchall(): - if collection_id in collection_id_to_name: - collection_name = collection_id_to_name[collection_id] - uuid_to_collection[segment_id] = collection_name - - for collection_dir in vector_dir.iterdir(): - if not collection_dir.is_dir() or collection_dir.name.startswith("."): - continue - - dir_uuid = collection_dir.name - collection_name = uuid_to_collection.get(dir_uuid) - - if collection_name is None or collection_name not in expected_collections: - count += 1 - except Exception as e: - log.debug(f"Error counting orphaned vector collections: {e}") - - return count - - def count_audio_cache_files(max_age_days: Optional[int]) -> int: """Count audio cache files that would be deleted.""" if max_age_days is None: @@ -469,29 +691,6 @@ def get_active_file_ids() -> Set[str]: return active_file_ids -def safe_delete_vector_collection(collection_name: str) -> bool: - """ - Safely delete a vector collection, handling both logical and physical cleanup. - """ - try: - try: - VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name) - except Exception as e: - log.debug(f"Collection {collection_name} may not exist in DB: {e}") - - if "chroma" in VECTOR_DB.lower(): - vector_dir = Path(CACHE_DIR).parent / "vector_db" / collection_name - if vector_dir.exists() and vector_dir.is_dir(): - shutil.rmtree(vector_dir) - return True - - return True - - except Exception as e: - log.error(f"Error deleting vector collection {collection_name}: {e}") - return False - - def safe_delete_file_by_id(file_id: str) -> bool: """ Safely delete a file record and its associated vector collection. @@ -501,11 +700,12 @@ def safe_delete_file_by_id(file_id: str) -> bool: if not file_record: return True + # Use modular vector database cleaner + vector_cleaner = get_vector_database_cleaner() collection_name = f"file-{file_id}" - safe_delete_vector_collection(collection_name) + vector_cleaner.delete_collection(collection_name) Files.delete_file_by_id(file_id) - return True except Exception as e: @@ -560,97 +760,6 @@ def cleanup_orphaned_uploads(active_file_ids: Set[str]) -> None: log.info(f"Deleted {deleted_count} orphaned upload files") -def cleanup_orphaned_vector_collections( - active_file_ids: Set[str], active_kb_ids: Set[str] -) -> None: - """ - Clean up orphaned vector collections by querying ChromaDB metadata. - """ - if "chroma" not in VECTOR_DB.lower(): - return - - vector_dir = Path(CACHE_DIR).parent / "vector_db" - if not vector_dir.exists(): - return - - chroma_db_path = vector_dir / "chroma.sqlite3" - if not chroma_db_path.exists(): - return - - expected_collections = set() - - for file_id in active_file_ids: - expected_collections.add(f"file-{file_id}") - - for kb_id in active_kb_ids: - expected_collections.add(kb_id) - - uuid_to_collection = {} - try: - - with sqlite3.connect(str(chroma_db_path)) as conn: - collection_id_to_name = {} - cursor = conn.execute("SELECT id, name FROM collections") - rows = cursor.fetchall() - - for row in rows: - collection_id, collection_name = row - collection_id_to_name[collection_id] = collection_name - - cursor = conn.execute( - "SELECT id, collection FROM segments WHERE scope = 'VECTOR'" - ) - segment_rows = cursor.fetchall() - - for row in segment_rows: - segment_id, collection_id = row - if collection_id in collection_id_to_name: - collection_name = collection_id_to_name[collection_id] - uuid_to_collection[segment_id] = collection_name - - log.info( - f"Found {len(uuid_to_collection)} vector segments in ChromaDB metadata" - ) - - except Exception as e: - log.error(f"Error reading ChromaDB metadata: {e}") - return - - deleted_count = 0 - - try: - for collection_dir in vector_dir.iterdir(): - if not collection_dir.is_dir(): - continue - - dir_uuid = collection_dir.name - - if dir_uuid.startswith("."): - continue - - collection_name = uuid_to_collection.get(dir_uuid) - - if collection_name is None: - try: - shutil.rmtree(collection_dir) - deleted_count += 1 - except Exception as e: - log.error(f"Failed to delete orphaned directory {dir_uuid}: {e}") - - elif collection_name not in expected_collections: - try: - shutil.rmtree(collection_dir) - deleted_count += 1 - except Exception as e: - log.error(f"Failed to delete collection directory {dir_uuid}: {e}") - - except Exception as e: - log.error(f"Error cleaning vector collections: {e}") - - if deleted_count > 0: - log.info(f"Deleted {deleted_count} orphaned vector collections") - - def delete_inactive_users( inactive_days: int, exempt_admin: bool = True, @@ -755,6 +864,9 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): If dry_run=False, performs actual deletion and returns True on success. """ try: + # Get vector database cleaner based on configuration + vector_cleaner = get_vector_database_cleaner() + if form_data.dry_run: log.info("Starting data pruning preview (dry run)") @@ -786,7 +898,7 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): orphaned_notes=orphaned_counts["notes"], orphaned_folders=orphaned_counts["folders"], orphaned_uploads=count_orphaned_uploads(active_file_ids), - orphaned_vector_collections=count_orphaned_vector_collections(active_file_ids, active_kb_ids), + orphaned_vector_collections=vector_cleaner.count_orphaned_collections(active_file_ids, active_kb_ids), audio_cache_files=count_audio_cache_files(form_data.audio_cache_max_age_days) ) @@ -877,7 +989,7 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): if form_data.delete_orphaned_knowledge_bases: for kb in knowledge_bases: if kb.user_id not in active_user_ids: - if safe_delete_vector_collection(kb.id): + if vector_cleaner.delete_collection(kb.id): Knowledges.delete_knowledge_by_id(kb.id) deleted_kbs += 1 @@ -984,7 +1096,9 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): final_active_kb_ids = {kb.id for kb in Knowledges.get_knowledge_bases()} cleanup_orphaned_uploads(final_active_file_ids) - cleanup_orphaned_vector_collections(final_active_file_ids, final_active_kb_ids) + + # Use modular vector database cleanup + vector_cleaner.cleanup_orphaned_collections(final_active_file_ids, final_active_kb_ids) # Stage 5: Audio cache cleanup log.info("Cleaning audio cache") @@ -999,15 +1113,14 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): except Exception as e: log.error(f"Failed to vacuum main database: {e}") - if "chroma" in VECTOR_DB.lower(): - chroma_db_path = Path(CACHE_DIR).parent / "vector_db" / "chroma.sqlite3" - if chroma_db_path.exists(): - try: - - with sqlite3.connect(str(chroma_db_path)) as conn: - conn.execute("VACUUM") - except Exception as e: - log.error(f"Failed to vacuum ChromaDB database: {e}") + # Vector database-specific optimization + if isinstance(vector_cleaner, ChromaDatabaseCleaner): + try: + with sqlite3.connect(str(vector_cleaner.chroma_db_path)) as conn: + conn.execute("VACUUM") + log.info("Vacuumed ChromaDB database") + except Exception as e: + log.error(f"Failed to vacuum ChromaDB database: {e}") log.info("Data pruning completed successfully") return True From 155f53b867fa9a8f6481f715979d9a0b87b54483 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 18:17:24 +0200 Subject: [PATCH 33/43] Update prune.py --- backend/open_webui/routers/prune.py | 184 +++++++++++++++++++++++++++- 1 file changed, 183 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index a4d6fc588f..c5377d79e9 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -181,7 +181,7 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): return count def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: - """Actually delete orphaned ChromaDB collections.""" + """Actually delete orphaned ChromaDB collections and database records.""" if not self.chroma_db_path.exists(): return 0 @@ -190,6 +190,13 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): deleted_count = 0 + # First, clean up orphaned database records + try: + deleted_count += self._cleanup_orphaned_database_records() + except Exception as e: + log.error(f"Error cleaning orphaned database records: {e}") + + # Then clean up physical directories try: for collection_dir in self.vector_dir.iterdir(): if not collection_dir.is_dir() or collection_dir.name.startswith("."): @@ -281,6 +288,181 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): log.error(f"Error reading ChromaDB metadata: {e}") return uuid_to_collection + + def _cleanup_orphaned_database_records(self) -> int: + """ + Clean up orphaned database records that ChromaDB's delete_collection() method leaves behind. + + This is the key fix for the file size issue - ChromaDB doesn't properly cascade + deletions, leaving orphaned embeddings, metadata, and FTS data that prevent + VACUUM from reclaiming space. + + Returns: + Number of orphaned records cleaned up + """ + cleaned_records = 0 + + try: + with sqlite3.connect(str(self.chroma_db_path)) as conn: + # Count orphaned records before cleanup + cursor = conn.execute(""" + SELECT COUNT(*) FROM embeddings + WHERE segment_id NOT IN (SELECT id FROM segments) + """) + orphaned_embeddings = cursor.fetchone()[0] + + if orphaned_embeddings == 0: + log.debug("No orphaned ChromaDB embeddings found") + return 0 + + log.info(f"Cleaning up {orphaned_embeddings} orphaned ChromaDB embeddings and related data") + + # Delete orphaned embedding_metadata first (child records) + cursor = conn.execute(""" + DELETE FROM embedding_metadata + WHERE id IN ( + SELECT id FROM embeddings + WHERE segment_id NOT IN (SELECT id FROM segments) + ) + """) + metadata_deleted = cursor.rowcount + cleaned_records += metadata_deleted + + # Delete orphaned embeddings + cursor = conn.execute(""" + DELETE FROM embeddings + WHERE segment_id NOT IN (SELECT id FROM segments) + """) + embeddings_deleted = cursor.rowcount + cleaned_records += embeddings_deleted + + # Selectively clean FTS while preserving active content + fts_cleaned = self._cleanup_fts_selectively(conn) + log.info(f"FTS cleanup: preserved {fts_cleaned} valid text entries") + + # Clean up orphaned collection and segment metadata + cursor = conn.execute(""" + DELETE FROM collection_metadata + WHERE collection_id NOT IN (SELECT id FROM collections) + """) + collection_meta_deleted = cursor.rowcount + cleaned_records += collection_meta_deleted + + cursor = conn.execute(""" + DELETE FROM segment_metadata + WHERE segment_id NOT IN (SELECT id FROM segments) + """) + segment_meta_deleted = cursor.rowcount + cleaned_records += segment_meta_deleted + + # Clean up orphaned max_seq_id records + cursor = conn.execute(""" + DELETE FROM max_seq_id + WHERE segment_id NOT IN (SELECT id FROM segments) + """) + seq_id_deleted = cursor.rowcount + cleaned_records += seq_id_deleted + + # Force FTS index rebuild - this is crucial for VACUUM to work properly + conn.execute("INSERT INTO embedding_fulltext_search(embedding_fulltext_search) VALUES('rebuild')") + + # Commit changes + conn.commit() + + log.info(f"ChromaDB cleanup: {embeddings_deleted} embeddings, {metadata_deleted} metadata, " + f"{collection_meta_deleted} collection metadata, {segment_meta_deleted} segment metadata, " + f"{seq_id_deleted} sequence IDs") + + except Exception as e: + log.error(f"Error cleaning orphaned ChromaDB database records: {e}") + raise + + return cleaned_records + + def _cleanup_fts_selectively(self, conn) -> int: + """ + Selectively clean FTS content with atomic operations, preserving only data from active embeddings. + + This method prevents destroying valid search data by: + 1. Creating and validating temporary table with valid content + 2. Using atomic transactions for DELETE/INSERT operations + 3. Rolling back on failure to preserve existing data + 4. Conservative fallback: skip FTS cleanup if validation fails + + Returns: + Number of valid FTS entries preserved, or -1 if FTS cleanup was skipped + """ + try: + # Step 1: Create temporary table with valid content + conn.execute(""" + CREATE TEMPORARY TABLE temp_valid_fts AS + SELECT DISTINCT em.string_value + FROM embedding_metadata em + JOIN embeddings e ON em.id = e.id + JOIN segments s ON e.segment_id = s.id + WHERE em.string_value IS NOT NULL + AND em.string_value != '' + """) + + # Step 2: Validate temp table creation and count records + cursor = conn.execute("SELECT COUNT(*) FROM temp_valid_fts") + valid_count = cursor.fetchone()[0] + + # Step 3: Validate temp table is accessible + try: + conn.execute("SELECT 1 FROM temp_valid_fts LIMIT 1") + temp_table_ok = True + except Exception: + temp_table_ok = False + + # Step 4: Only proceed if validation passed + if not temp_table_ok: + log.warning("FTS temp table validation failed, skipping FTS cleanup for safety") + conn.execute("DROP TABLE IF EXISTS temp_valid_fts") + return -1 # Signal FTS cleanup was skipped + + # Step 5: Atomic FTS cleanup operation + conn.execute("BEGIN IMMEDIATE") + try: + # Delete all FTS content + conn.execute("DELETE FROM embedding_fulltext_search") + + # Re-insert only valid content if any exists + if valid_count > 0: + conn.execute(""" + INSERT INTO embedding_fulltext_search(string_value) + SELECT string_value FROM temp_valid_fts + """) + log.debug(f"Preserved {valid_count} valid FTS entries") + else: + log.debug("No valid FTS content found, cleared all entries") + + # Rebuild FTS index + conn.execute("INSERT INTO embedding_fulltext_search(embedding_fulltext_search) VALUES('rebuild')") + + # Commit the atomic operation + conn.execute("COMMIT") + + except Exception as e: + # Rollback on any failure to preserve existing FTS data + conn.execute("ROLLBACK") + log.error(f"FTS cleanup failed, rolled back changes: {e}") + conn.execute("DROP TABLE IF EXISTS temp_valid_fts") + return -1 # Signal FTS cleanup failed + + # Step 6: Clean up temporary table + conn.execute("DROP TABLE IF EXISTS temp_valid_fts") + + return valid_count + + except Exception as e: + log.error(f"FTS cleanup validation failed, leaving FTS untouched: {e}") + # Conservative approach: don't touch FTS if anything goes wrong + try: + conn.execute("DROP TABLE IF EXISTS temp_valid_fts") + except: + pass + return -1 # Signal FTS cleanup was skipped class PGVectorDatabaseCleaner(VectorDatabaseCleaner): From 46288924a2ad8444201c16a1905192005b3b60b8 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 18:29:26 +0200 Subject: [PATCH 34/43] Update prune.py --- backend/open_webui/routers/prune.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index c5377d79e9..3d60e19a61 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -421,8 +421,7 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): conn.execute("DROP TABLE IF EXISTS temp_valid_fts") return -1 # Signal FTS cleanup was skipped - # Step 5: Atomic FTS cleanup operation - conn.execute("BEGIN IMMEDIATE") + # Step 5: FTS cleanup operation (already in transaction) try: # Delete all FTS content conn.execute("DELETE FROM embedding_fulltext_search") @@ -440,13 +439,8 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): # Rebuild FTS index conn.execute("INSERT INTO embedding_fulltext_search(embedding_fulltext_search) VALUES('rebuild')") - # Commit the atomic operation - conn.execute("COMMIT") - except Exception as e: - # Rollback on any failure to preserve existing FTS data - conn.execute("ROLLBACK") - log.error(f"FTS cleanup failed, rolled back changes: {e}") + log.error(f"FTS cleanup failed: {e}") conn.execute("DROP TABLE IF EXISTS temp_valid_fts") return -1 # Signal FTS cleanup failed From 8231588eb4b103074ddeb9ebe4736a855f8a9f1c Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 19:17:52 +0200 Subject: [PATCH 35/43] pgvector --- backend/open_webui/routers/prune.py | 166 +++++++++++++++++++++++++--- 1 file changed, 150 insertions(+), 16 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 3d60e19a61..f36e0c9c53 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -461,29 +461,156 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): class PGVectorDatabaseCleaner(VectorDatabaseCleaner): """ - Placeholder implementation for PGVector database cleanup. + PGVector database cleanup implementation. - This is a stub implementation that can be extended by the community - to support PGVector-specific cleanup operations. - - According to PR feedback, PGVector stores data in document_chunk table - and cleanup should involve finding rows with matching file IDs. + Leverages the existing PGVector client's delete() method for simple, + reliable collection cleanup while maintaining comprehensive error handling + and safety features. """ + def __init__(self): + # Validate that we can access the PGVector client + try: + if VECTOR_DB_CLIENT is None: + raise Exception("VECTOR_DB_CLIENT is not available") + # Test if we can access the session + if hasattr(VECTOR_DB_CLIENT, 'session') and VECTOR_DB_CLIENT.session: + self.session = VECTOR_DB_CLIENT.session + log.debug("PGVector cleaner initialized successfully") + else: + raise Exception("PGVector client session not available") + except Exception as e: + log.error(f"Failed to initialize PGVector client for cleanup: {e}") + self.session = None + def count_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: - """Count orphaned PGVector collections - to be implemented by community.""" - log.debug("PGVector collection counting not yet implemented") - return 0 + """Count orphaned PGVector collections for preview.""" + if not self.session: + log.warning("PGVector session not available for counting orphaned collections") + return 0 + + try: + orphaned_collections = self._get_orphaned_collections(active_file_ids, active_kb_ids) + self.session.rollback() # Read-only transaction + return len(orphaned_collections) + + except Exception as e: + if self.session: + self.session.rollback() + log.error(f"Error counting orphaned PGVector collections: {e}") + return 0 def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: - """Cleanup orphaned PGVector collections - to be implemented by community.""" - log.debug("PGVector collection cleanup not yet implemented") - return 0 + """ + Delete orphaned PGVector collections using the existing client's delete method. + + This is the "super easy" approach suggested by @recrudesce - just use the + existing PGVector client's delete() method for each orphaned collection. + """ + if not self.session: + log.warning("PGVector session not available for cleanup") + return 0 + + try: + orphaned_collections = self._get_orphaned_collections(active_file_ids, active_kb_ids) + + if not orphaned_collections: + log.debug("No orphaned PGVector collections found") + return 0 + + deleted_count = 0 + log.info(f"Deleting {len(orphaned_collections)} orphaned PGVector collections") + + # SIMPLIFIED DELETION: Use existing PGVector client delete method + for collection_name in orphaned_collections: + try: + # This is @recrudesce's "super easy" approach: + # Just call the existing delete method! + VECTOR_DB_CLIENT.delete(collection_name) + deleted_count += 1 + log.debug(f"Deleted PGVector collection: {collection_name}") + + except Exception as e: + log.error(f"Failed to delete PGVector collection '{collection_name}': {e}") + # Continue with other collections even if one fails + continue + + # PostgreSQL-specific optimization (if we have access to session) + try: + if self.session: + self.session.execute(text("VACUUM ANALYZE document_chunk")) + self.session.commit() + log.debug("Executed VACUUM ANALYZE on document_chunk table") + except Exception as e: + log.warning(f"Failed to VACUUM PGVector table: {e}") + + if deleted_count > 0: + log.info(f"Successfully deleted {deleted_count} orphaned PGVector collections") + + return deleted_count + + except Exception as e: + if self.session: + self.session.rollback() + log.error(f"Error cleaning orphaned PGVector collections: {e}") + return 0 def delete_collection(self, collection_name: str) -> bool: - """Delete PGVector collection - to be implemented by community.""" - log.debug(f"PGVector collection deletion not yet implemented: {collection_name}") - return True + """ + Delete a specific PGVector collection using the existing client method. + + Super simple - just call the existing delete method! + """ + try: + # @recrudesce's "super easy" approach: use existing client! + VECTOR_DB_CLIENT.delete(collection_name) + log.debug(f"Deleted PGVector collection: {collection_name}") + return True + + except Exception as e: + log.error(f"Error deleting PGVector collection '{collection_name}': {e}") + return False + + def _get_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> Set[str]: + """ + Find collections that exist in PGVector but are no longer referenced. + + This is the only "complex" part - discovery. The actual deletion is simple! + """ + try: + expected_collections = self._build_expected_collections(active_file_ids, active_kb_ids) + + # Query distinct collection names from document_chunk table + result = self.session.execute( + text("SELECT DISTINCT collection_name FROM document_chunk") + ).fetchall() + + existing_collections = {row[0] for row in result} + orphaned_collections = existing_collections - expected_collections + + log.debug(f"Found {len(existing_collections)} existing collections, " + f"{len(expected_collections)} expected, " + f"{len(orphaned_collections)} orphaned") + + return orphaned_collections + + except Exception as e: + log.error(f"Error finding orphaned PGVector collections: {e}") + return set() + + def _build_expected_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> Set[str]: + """Build set of collection names that should exist.""" + expected_collections = set() + + # File collections use "file-{id}" pattern (same as ChromaDB) + for file_id in active_file_ids: + expected_collections.add(f"file-{file_id}") + + # Knowledge base collections use the KB ID directly (same as ChromaDB) + for kb_id in active_kb_ids: + expected_collections.add(kb_id) + + return expected_collections class NoOpVectorDatabaseCleaner(VectorDatabaseCleaner): @@ -524,7 +651,7 @@ def get_vector_database_cleaner() -> VectorDatabaseCleaner: log.debug("Using ChromaDB cleaner") return ChromaDatabaseCleaner() elif "pgvector" in vector_db_type: - log.debug("Using PGVector cleaner (placeholder implementation)") + log.debug("Using PGVector cleaner") return PGVectorDatabaseCleaner() else: log.debug(f"No specific cleaner for vector database type: {VECTOR_DB}, using no-op cleaner") @@ -1297,6 +1424,13 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): log.info("Vacuumed ChromaDB database") except Exception as e: log.error(f"Failed to vacuum ChromaDB database: {e}") + elif isinstance(vector_cleaner, PGVectorDatabaseCleaner) and vector_cleaner.session: + try: + vector_cleaner.session.execute(text("VACUUM ANALYZE")) + vector_cleaner.session.commit() + log.info("Executed VACUUM ANALYZE on PostgreSQL database") + except Exception as e: + log.error(f"Failed to vacuum PostgreSQL database: {e}") log.info("Data pruning completed successfully") return True From 8156d0a30ea82b7af2582abc9565725f0c1b2136 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Fri, 22 Aug 2025 19:33:17 +0200 Subject: [PATCH 36/43] Update prune.py --- backend/open_webui/routers/prune.py | 598 +++++++++++++++++----------- 1 file changed, 355 insertions(+), 243 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index f36e0c9c53..e43f8061d7 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -39,11 +39,11 @@ router = APIRouter() class JSONFileIDExtractor: """ Utility for extracting and validating file IDs from JSON content. - + Replaces duplicated regex compilation and validation logic used throughout the file scanning functions. Compiles patterns once for better performance. """ - + # Compile patterns once at class level for performance _FILE_ID_PATTERN = re.compile( r'"id":\s*"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"' @@ -51,90 +51,94 @@ class JSONFileIDExtractor: _URL_PATTERN = re.compile( r"/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})" ) - + @classmethod def extract_and_validate_file_ids(cls, json_string: str) -> Set[str]: """ Extract file IDs from JSON string and validate they exist in database. - + Args: json_string: JSON content as string (or any string to scan) - + Returns: Set of validated file IDs that exist in the Files table - + Note: This method replaces the repeated pattern of: 1. Compiling the same regex patterns - 2. Extracting potential IDs + 2. Extracting potential IDs 3. Validating each ID exists via Files.get_file_by_id() 4. Building a set of validated IDs """ validated_ids = set() - + # Extract potential IDs using both patterns potential_ids = [] potential_ids.extend(cls._FILE_ID_PATTERN.findall(json_string)) potential_ids.extend(cls._URL_PATTERN.findall(json_string)) - + # Validate each ID exists in database for file_id in potential_ids: if Files.get_file_by_id(file_id): validated_ids.add(file_id) - + return validated_ids class VectorDatabaseCleaner(ABC): """ Abstract base class for vector database cleanup operations. - + This interface defines the contract that all vector database implementations must follow. Community contributors can implement support for new vector databases by extending this class. - + Supported operations: - Count orphaned collections (for dry-run preview) - - Cleanup orphaned collections (actual deletion) + - Cleanup orphaned collections (actual deletion) - Delete individual collections by name """ - + @abstractmethod - def count_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + def count_orphaned_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> int: """ Count how many orphaned vector collections would be deleted. - + Args: active_file_ids: Set of file IDs that are still referenced active_kb_ids: Set of knowledge base IDs that are still active - + Returns: Number of orphaned collections that would be deleted """ pass - + @abstractmethod - def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + def cleanup_orphaned_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> int: """ Actually delete orphaned vector collections. - + Args: - active_file_ids: Set of file IDs that are still referenced + active_file_ids: Set of file IDs that are still referenced active_kb_ids: Set of knowledge base IDs that are still active - + Returns: Number of collections that were actually deleted """ pass - + @abstractmethod def delete_collection(self, collection_name: str) -> bool: """ Delete a specific vector collection by name. - + Args: collection_name: Name of the collection to delete - + Returns: True if deletion was successful, False otherwise """ @@ -144,67 +148,78 @@ class VectorDatabaseCleaner(ABC): class ChromaDatabaseCleaner(VectorDatabaseCleaner): """ ChromaDB-specific implementation of vector database cleanup. - + Handles ChromaDB's specific storage structure including: - SQLite metadata database (chroma.sqlite3) - Physical vector storage directories - Collection name to UUID mapping - Segment-based storage architecture """ - + def __init__(self): self.vector_dir = Path(CACHE_DIR).parent / "vector_db" self.chroma_db_path = self.vector_dir / "chroma.sqlite3" - - def count_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + + def count_orphaned_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> int: """Count orphaned ChromaDB collections for preview.""" if not self.chroma_db_path.exists(): return 0 - - expected_collections = self._build_expected_collections(active_file_ids, active_kb_ids) + + expected_collections = self._build_expected_collections( + active_file_ids, active_kb_ids + ) uuid_to_collection = self._get_collection_mappings() - + count = 0 try: for collection_dir in self.vector_dir.iterdir(): if not collection_dir.is_dir() or collection_dir.name.startswith("."): continue - + dir_uuid = collection_dir.name collection_name = uuid_to_collection.get(dir_uuid) - - if collection_name is None or collection_name not in expected_collections: + + if ( + collection_name is None + or collection_name not in expected_collections + ): count += 1 except Exception as e: log.debug(f"Error counting orphaned ChromaDB collections: {e}") - + return count - - def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + + def cleanup_orphaned_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> int: """Actually delete orphaned ChromaDB collections and database records.""" if not self.chroma_db_path.exists(): return 0 - - expected_collections = self._build_expected_collections(active_file_ids, active_kb_ids) + + expected_collections = self._build_expected_collections( + active_file_ids, active_kb_ids + ) uuid_to_collection = self._get_collection_mappings() - + deleted_count = 0 - + # First, clean up orphaned database records try: deleted_count += self._cleanup_orphaned_database_records() except Exception as e: log.error(f"Error cleaning orphaned database records: {e}") - + # Then clean up physical directories try: for collection_dir in self.vector_dir.iterdir(): if not collection_dir.is_dir() or collection_dir.name.startswith("."): continue - + dir_uuid = collection_dir.name collection_name = uuid_to_collection.get(dir_uuid) - + # Delete if no corresponding collection name or collection is not expected if collection_name is None: try: @@ -212,24 +227,30 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): deleted_count += 1 log.debug(f"Deleted orphaned ChromaDB directory: {dir_uuid}") except Exception as e: - log.error(f"Failed to delete orphaned directory {dir_uuid}: {e}") - + log.error( + f"Failed to delete orphaned directory {dir_uuid}: {e}" + ) + elif collection_name not in expected_collections: try: shutil.rmtree(collection_dir) deleted_count += 1 - log.debug(f"Deleted orphaned ChromaDB collection: {collection_name}") + log.debug( + f"Deleted orphaned ChromaDB collection: {collection_name}" + ) except Exception as e: - log.error(f"Failed to delete collection directory {dir_uuid}: {e}") - + log.error( + f"Failed to delete collection directory {dir_uuid}: {e}" + ) + except Exception as e: log.error(f"Error cleaning ChromaDB collections: {e}") - + if deleted_count > 0: log.info(f"Deleted {deleted_count} orphaned ChromaDB collections") - + return deleted_count - + def delete_collection(self, collection_name: str) -> bool: """Delete a specific ChromaDB collection by name.""" try: @@ -238,35 +259,39 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name) log.debug(f"Deleted ChromaDB collection via client: {collection_name}") except Exception as e: - log.debug(f"Collection {collection_name} may not exist in ChromaDB: {e}") - + log.debug( + f"Collection {collection_name} may not exist in ChromaDB: {e}" + ) + # Also clean up physical directory if it exists # Note: ChromaDB uses UUID directories, so we'd need to map collection name to UUID # For now, let the cleanup_orphaned_collections method handle physical cleanup return True - + except Exception as e: log.error(f"Error deleting ChromaDB collection {collection_name}: {e}") return False - - def _build_expected_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> Set[str]: + + def _build_expected_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> Set[str]: """Build set of collection names that should exist.""" expected_collections = set() - + # File collections use "file-{id}" pattern for file_id in active_file_ids: expected_collections.add(f"file-{file_id}") - + # Knowledge base collections use the KB ID directly for kb_id in active_kb_ids: expected_collections.add(kb_id) - + return expected_collections - + def _get_collection_mappings(self) -> dict: """Get mapping from ChromaDB directory UUID to collection name.""" uuid_to_collection = {} - + try: with sqlite3.connect(str(self.chroma_db_path)) as conn: # First, get collection ID to name mapping @@ -274,127 +299,148 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): cursor = conn.execute("SELECT id, name FROM collections") for collection_id, collection_name in cursor.fetchall(): collection_id_to_name[collection_id] = collection_name - + # Then, get segment ID to collection mapping (segments are the directory UUIDs) - cursor = conn.execute("SELECT id, collection FROM segments WHERE scope = 'VECTOR'") + cursor = conn.execute( + "SELECT id, collection FROM segments WHERE scope = 'VECTOR'" + ) for segment_id, collection_id in cursor.fetchall(): if collection_id in collection_id_to_name: collection_name = collection_id_to_name[collection_id] uuid_to_collection[segment_id] = collection_name - + log.debug(f"Found {len(uuid_to_collection)} ChromaDB vector segments") - + except Exception as e: log.error(f"Error reading ChromaDB metadata: {e}") - + return uuid_to_collection - + def _cleanup_orphaned_database_records(self) -> int: """ Clean up orphaned database records that ChromaDB's delete_collection() method leaves behind. - + This is the key fix for the file size issue - ChromaDB doesn't properly cascade deletions, leaving orphaned embeddings, metadata, and FTS data that prevent VACUUM from reclaiming space. - + Returns: Number of orphaned records cleaned up """ cleaned_records = 0 - + try: with sqlite3.connect(str(self.chroma_db_path)) as conn: # Count orphaned records before cleanup - cursor = conn.execute(""" + cursor = conn.execute( + """ SELECT COUNT(*) FROM embeddings WHERE segment_id NOT IN (SELECT id FROM segments) - """) + """ + ) orphaned_embeddings = cursor.fetchone()[0] - + if orphaned_embeddings == 0: log.debug("No orphaned ChromaDB embeddings found") return 0 - - log.info(f"Cleaning up {orphaned_embeddings} orphaned ChromaDB embeddings and related data") - + + log.info( + f"Cleaning up {orphaned_embeddings} orphaned ChromaDB embeddings and related data" + ) + # Delete orphaned embedding_metadata first (child records) - cursor = conn.execute(""" + cursor = conn.execute( + """ DELETE FROM embedding_metadata WHERE id IN ( SELECT id FROM embeddings WHERE segment_id NOT IN (SELECT id FROM segments) ) - """) + """ + ) metadata_deleted = cursor.rowcount cleaned_records += metadata_deleted - + # Delete orphaned embeddings - cursor = conn.execute(""" + cursor = conn.execute( + """ DELETE FROM embeddings WHERE segment_id NOT IN (SELECT id FROM segments) - """) + """ + ) embeddings_deleted = cursor.rowcount cleaned_records += embeddings_deleted - + # Selectively clean FTS while preserving active content fts_cleaned = self._cleanup_fts_selectively(conn) log.info(f"FTS cleanup: preserved {fts_cleaned} valid text entries") - + # Clean up orphaned collection and segment metadata - cursor = conn.execute(""" + cursor = conn.execute( + """ DELETE FROM collection_metadata WHERE collection_id NOT IN (SELECT id FROM collections) - """) + """ + ) collection_meta_deleted = cursor.rowcount cleaned_records += collection_meta_deleted - - cursor = conn.execute(""" + + cursor = conn.execute( + """ DELETE FROM segment_metadata WHERE segment_id NOT IN (SELECT id FROM segments) - """) + """ + ) segment_meta_deleted = cursor.rowcount cleaned_records += segment_meta_deleted - + # Clean up orphaned max_seq_id records - cursor = conn.execute(""" + cursor = conn.execute( + """ DELETE FROM max_seq_id WHERE segment_id NOT IN (SELECT id FROM segments) - """) + """ + ) seq_id_deleted = cursor.rowcount cleaned_records += seq_id_deleted - + # Force FTS index rebuild - this is crucial for VACUUM to work properly - conn.execute("INSERT INTO embedding_fulltext_search(embedding_fulltext_search) VALUES('rebuild')") - + conn.execute( + "INSERT INTO embedding_fulltext_search(embedding_fulltext_search) VALUES('rebuild')" + ) + # Commit changes conn.commit() - - log.info(f"ChromaDB cleanup: {embeddings_deleted} embeddings, {metadata_deleted} metadata, " - f"{collection_meta_deleted} collection metadata, {segment_meta_deleted} segment metadata, " - f"{seq_id_deleted} sequence IDs") - + + log.info( + f"ChromaDB cleanup: {embeddings_deleted} embeddings, {metadata_deleted} metadata, " + f"{collection_meta_deleted} collection metadata, {segment_meta_deleted} segment metadata, " + f"{seq_id_deleted} sequence IDs" + ) + except Exception as e: log.error(f"Error cleaning orphaned ChromaDB database records: {e}") raise - + return cleaned_records - + def _cleanup_fts_selectively(self, conn) -> int: """ Selectively clean FTS content with atomic operations, preserving only data from active embeddings. - + This method prevents destroying valid search data by: 1. Creating and validating temporary table with valid content 2. Using atomic transactions for DELETE/INSERT operations 3. Rolling back on failure to preserve existing data 4. Conservative fallback: skip FTS cleanup if validation fails - + Returns: Number of valid FTS entries preserved, or -1 if FTS cleanup was skipped """ try: # Step 1: Create temporary table with valid content - conn.execute(""" + conn.execute( + """ CREATE TEMPORARY TABLE temp_valid_fts AS SELECT DISTINCT em.string_value FROM embedding_metadata em @@ -402,53 +448,60 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): JOIN segments s ON e.segment_id = s.id WHERE em.string_value IS NOT NULL AND em.string_value != '' - """) - + """ + ) + # Step 2: Validate temp table creation and count records cursor = conn.execute("SELECT COUNT(*) FROM temp_valid_fts") valid_count = cursor.fetchone()[0] - + # Step 3: Validate temp table is accessible try: conn.execute("SELECT 1 FROM temp_valid_fts LIMIT 1") temp_table_ok = True except Exception: temp_table_ok = False - + # Step 4: Only proceed if validation passed if not temp_table_ok: - log.warning("FTS temp table validation failed, skipping FTS cleanup for safety") + log.warning( + "FTS temp table validation failed, skipping FTS cleanup for safety" + ) conn.execute("DROP TABLE IF EXISTS temp_valid_fts") return -1 # Signal FTS cleanup was skipped - + # Step 5: FTS cleanup operation (already in transaction) try: # Delete all FTS content conn.execute("DELETE FROM embedding_fulltext_search") - + # Re-insert only valid content if any exists if valid_count > 0: - conn.execute(""" + conn.execute( + """ INSERT INTO embedding_fulltext_search(string_value) SELECT string_value FROM temp_valid_fts - """) + """ + ) log.debug(f"Preserved {valid_count} valid FTS entries") else: log.debug("No valid FTS content found, cleared all entries") - + # Rebuild FTS index - conn.execute("INSERT INTO embedding_fulltext_search(embedding_fulltext_search) VALUES('rebuild')") - + conn.execute( + "INSERT INTO embedding_fulltext_search(embedding_fulltext_search) VALUES('rebuild')" + ) + except Exception as e: log.error(f"FTS cleanup failed: {e}") conn.execute("DROP TABLE IF EXISTS temp_valid_fts") return -1 # Signal FTS cleanup failed - + # Step 6: Clean up temporary table conn.execute("DROP TABLE IF EXISTS temp_valid_fts") - + return valid_count - + except Exception as e: log.error(f"FTS cleanup validation failed, leaving FTS untouched: {e}") # Conservative approach: don't touch FTS if anything goes wrong @@ -462,19 +515,19 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): class PGVectorDatabaseCleaner(VectorDatabaseCleaner): """ PGVector database cleanup implementation. - + Leverages the existing PGVector client's delete() method for simple, reliable collection cleanup while maintaining comprehensive error handling and safety features. """ - + def __init__(self): # Validate that we can access the PGVector client try: if VECTOR_DB_CLIENT is None: raise Exception("VECTOR_DB_CLIENT is not available") # Test if we can access the session - if hasattr(VECTOR_DB_CLIENT, 'session') and VECTOR_DB_CLIENT.session: + if hasattr(VECTOR_DB_CLIENT, "session") and VECTOR_DB_CLIENT.session: self.session = VECTOR_DB_CLIENT.session log.debug("PGVector cleaner initialized successfully") else: @@ -482,45 +535,57 @@ class PGVectorDatabaseCleaner(VectorDatabaseCleaner): except Exception as e: log.error(f"Failed to initialize PGVector client for cleanup: {e}") self.session = None - - def count_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + + def count_orphaned_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> int: """Count orphaned PGVector collections for preview.""" if not self.session: - log.warning("PGVector session not available for counting orphaned collections") + log.warning( + "PGVector session not available for counting orphaned collections" + ) return 0 - + try: - orphaned_collections = self._get_orphaned_collections(active_file_ids, active_kb_ids) + orphaned_collections = self._get_orphaned_collections( + active_file_ids, active_kb_ids + ) self.session.rollback() # Read-only transaction return len(orphaned_collections) - + except Exception as e: if self.session: self.session.rollback() log.error(f"Error counting orphaned PGVector collections: {e}") return 0 - - def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + + def cleanup_orphaned_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> int: """ Delete orphaned PGVector collections using the existing client's delete method. - + This is the "super easy" approach suggested by @recrudesce - just use the existing PGVector client's delete() method for each orphaned collection. """ if not self.session: log.warning("PGVector session not available for cleanup") return 0 - + try: - orphaned_collections = self._get_orphaned_collections(active_file_ids, active_kb_ids) - + orphaned_collections = self._get_orphaned_collections( + active_file_ids, active_kb_ids + ) + if not orphaned_collections: log.debug("No orphaned PGVector collections found") return 0 - + deleted_count = 0 - log.info(f"Deleting {len(orphaned_collections)} orphaned PGVector collections") - + log.info( + f"Deleting {len(orphaned_collections)} orphaned PGVector collections" + ) + # SIMPLIFIED DELETION: Use existing PGVector client delete method for collection_name in orphaned_collections: try: @@ -529,12 +594,14 @@ class PGVectorDatabaseCleaner(VectorDatabaseCleaner): VECTOR_DB_CLIENT.delete(collection_name) deleted_count += 1 log.debug(f"Deleted PGVector collection: {collection_name}") - + except Exception as e: - log.error(f"Failed to delete PGVector collection '{collection_name}': {e}") + log.error( + f"Failed to delete PGVector collection '{collection_name}': {e}" + ) # Continue with other collections even if one fails continue - + # PostgreSQL-specific optimization (if we have access to session) try: if self.session: @@ -543,22 +610,24 @@ class PGVectorDatabaseCleaner(VectorDatabaseCleaner): log.debug("Executed VACUUM ANALYZE on document_chunk table") except Exception as e: log.warning(f"Failed to VACUUM PGVector table: {e}") - + if deleted_count > 0: - log.info(f"Successfully deleted {deleted_count} orphaned PGVector collections") - + log.info( + f"Successfully deleted {deleted_count} orphaned PGVector collections" + ) + return deleted_count - + except Exception as e: if self.session: self.session.rollback() log.error(f"Error cleaning orphaned PGVector collections: {e}") return 0 - + def delete_collection(self, collection_name: str) -> bool: """ Delete a specific PGVector collection using the existing client method. - + Super simple - just call the existing delete method! """ try: @@ -566,69 +635,81 @@ class PGVectorDatabaseCleaner(VectorDatabaseCleaner): VECTOR_DB_CLIENT.delete(collection_name) log.debug(f"Deleted PGVector collection: {collection_name}") return True - + except Exception as e: log.error(f"Error deleting PGVector collection '{collection_name}': {e}") return False - - def _get_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> Set[str]: + + def _get_orphaned_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> Set[str]: """ Find collections that exist in PGVector but are no longer referenced. - + This is the only "complex" part - discovery. The actual deletion is simple! """ try: - expected_collections = self._build_expected_collections(active_file_ids, active_kb_ids) - + expected_collections = self._build_expected_collections( + active_file_ids, active_kb_ids + ) + # Query distinct collection names from document_chunk table result = self.session.execute( text("SELECT DISTINCT collection_name FROM document_chunk") ).fetchall() - + existing_collections = {row[0] for row in result} orphaned_collections = existing_collections - expected_collections - - log.debug(f"Found {len(existing_collections)} existing collections, " - f"{len(expected_collections)} expected, " - f"{len(orphaned_collections)} orphaned") - + + log.debug( + f"Found {len(existing_collections)} existing collections, " + f"{len(expected_collections)} expected, " + f"{len(orphaned_collections)} orphaned" + ) + return orphaned_collections - + except Exception as e: log.error(f"Error finding orphaned PGVector collections: {e}") return set() - - def _build_expected_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> Set[str]: + + def _build_expected_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> Set[str]: """Build set of collection names that should exist.""" expected_collections = set() - + # File collections use "file-{id}" pattern (same as ChromaDB) for file_id in active_file_ids: expected_collections.add(f"file-{file_id}") - + # Knowledge base collections use the KB ID directly (same as ChromaDB) for kb_id in active_kb_ids: expected_collections.add(kb_id) - + return expected_collections class NoOpVectorDatabaseCleaner(VectorDatabaseCleaner): """ No-operation implementation for unsupported vector databases. - + This implementation does nothing and is used when the configured vector database is not supported by the cleanup system. """ - - def count_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + + def count_orphaned_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> int: """No orphaned collections to count for unsupported databases.""" return 0 - - def cleanup_orphaned_collections(self, active_file_ids: Set[str], active_kb_ids: Set[str]) -> int: + + def cleanup_orphaned_collections( + self, active_file_ids: Set[str], active_kb_ids: Set[str] + ) -> int: """No collections to cleanup for unsupported databases.""" return 0 - + def delete_collection(self, collection_name: str) -> bool: """No collection to delete for unsupported databases.""" return True @@ -637,16 +718,16 @@ class NoOpVectorDatabaseCleaner(VectorDatabaseCleaner): def get_vector_database_cleaner() -> VectorDatabaseCleaner: """ Factory function to get the appropriate vector database cleaner. - + This function detects the configured vector database type and returns the appropriate cleaner implementation. Community contributors can extend this function to support additional vector databases. - + Returns: VectorDatabaseCleaner: Appropriate implementation for the configured database """ vector_db_type = VECTOR_DB.lower() - + if "chroma" in vector_db_type: log.debug("Using ChromaDB cleaner") return ChromaDatabaseCleaner() @@ -654,7 +735,9 @@ def get_vector_database_cleaner() -> VectorDatabaseCleaner: log.debug("Using PGVector cleaner") return PGVectorDatabaseCleaner() else: - log.debug(f"No specific cleaner for vector database type: {VECTOR_DB}, using no-op cleaner") + log.debug( + f"No specific cleaner for vector database type: {VECTOR_DB}, using no-op cleaner" + ) return NoOpVectorDatabaseCleaner() @@ -695,14 +778,16 @@ class PrunePreviewResult(BaseModel): # Counting helper functions for dry-run preview -def count_inactive_users(inactive_days: Optional[int], exempt_admin: bool, exempt_pending: bool) -> int: +def count_inactive_users( + inactive_days: Optional[int], exempt_admin: bool, exempt_pending: bool +) -> int: """Count users that would be deleted for inactivity.""" if inactive_days is None: return 0 - + cutoff_time = int(time.time()) - (inactive_days * 86400) count = 0 - + try: all_users = Users.get_users()["users"] for user in all_users: @@ -714,18 +799,20 @@ def count_inactive_users(inactive_days: Optional[int], exempt_admin: bool, exemp count += 1 except Exception as e: log.debug(f"Error counting inactive users: {e}") - + return count -def count_old_chats(days: Optional[int], exempt_archived: bool, exempt_in_folders: bool) -> int: +def count_old_chats( + days: Optional[int], exempt_archived: bool, exempt_in_folders: bool +) -> int: """Count chats that would be deleted by age.""" if days is None: return 0 - + cutoff_time = int(time.time()) - (days * 86400) count = 0 - + try: for chat in Chats.get_chats(): if chat.updated_at < cutoff_time: @@ -739,7 +826,7 @@ def count_old_chats(days: Optional[int], exempt_archived: bool, exempt_in_folder count += 1 except Exception as e: log.debug(f"Error counting old chats: {e}") - + return count @@ -754,16 +841,16 @@ def count_orphaned_records(form_data: PruneDataForm) -> dict: "knowledge_bases": 0, "models": 0, "notes": 0, - "folders": 0 + "folders": 0, } - + try: # Get active user IDs active_user_ids = {user.id for user in Users.get_users()["users"]} - + # Get active file IDs for file orphan detection active_file_ids = get_active_file_ids() - + # Count orphaned files for file_record in Files.get_files(): should_delete = ( @@ -772,51 +859,51 @@ def count_orphaned_records(form_data: PruneDataForm) -> dict: ) if should_delete: counts["files"] += 1 - + # Count other orphaned records if form_data.delete_orphaned_chats: for chat in Chats.get_chats(): if chat.user_id not in active_user_ids: counts["chats"] += 1 - + if form_data.delete_orphaned_tools: for tool in Tools.get_tools(): if tool.user_id not in active_user_ids: counts["tools"] += 1 - + if form_data.delete_orphaned_functions: for function in Functions.get_functions(): if function.user_id not in active_user_ids: counts["functions"] += 1 - + if form_data.delete_orphaned_prompts: for prompt in Prompts.get_prompts(): if prompt.user_id not in active_user_ids: counts["prompts"] += 1 - + if form_data.delete_orphaned_knowledge_bases: for kb in Knowledges.get_knowledge_bases(): if kb.user_id not in active_user_ids: counts["knowledge_bases"] += 1 - + if form_data.delete_orphaned_models: for model in Models.get_all_models(): if model.user_id not in active_user_ids: counts["models"] += 1 - + if form_data.delete_orphaned_notes: for note in Notes.get_notes(): if note.user_id not in active_user_ids: counts["notes"] += 1 - + if form_data.delete_orphaned_folders: for folder in Folders.get_all_folders(): if folder.user_id not in active_user_ids: counts["folders"] += 1 - + except Exception as e: log.debug(f"Error counting orphaned records: {e}") - + return counts @@ -825,36 +912,36 @@ def count_orphaned_uploads(active_file_ids: Set[str]) -> int: upload_dir = Path(CACHE_DIR).parent / "uploads" if not upload_dir.exists(): return 0 - + count = 0 try: for file_path in upload_dir.iterdir(): if not file_path.is_file(): continue - + filename = file_path.name file_id = None - + # Extract file ID from filename patterns if len(filename) > 36: potential_id = filename[:36] if potential_id.count("-") == 4: file_id = potential_id - + if not file_id and filename.count("-") == 4 and len(filename) == 36: file_id = filename - + if not file_id: for active_id in active_file_ids: if active_id in filename: file_id = active_id break - + if file_id and file_id not in active_file_ids: count += 1 except Exception as e: log.debug(f"Error counting orphaned uploads: {e}") - + return count @@ -862,26 +949,26 @@ def count_audio_cache_files(max_age_days: Optional[int]) -> int: """Count audio cache files that would be deleted.""" if max_age_days is None: return 0 - + cutoff_time = time.time() - (max_age_days * 86400) count = 0 - + audio_dirs = [ Path(CACHE_DIR) / "audio" / "speech", Path(CACHE_DIR) / "audio" / "transcriptions", ] - + for audio_dir in audio_dirs: if not audio_dir.exists(): continue - + try: for file_path in audio_dir.iterdir(): if file_path.is_file() and file_path.stat().st_mtime < cutoff_time: count += 1 except Exception as e: log.debug(f"Error counting audio files in {audio_dir}: {e}") - + return count @@ -929,7 +1016,9 @@ def get_active_file_ids() -> Set[str]: try: chat_json_str = json.dumps(chat.chat) # Use utility to extract and validate file IDs - validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids(chat_json_str) + validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids( + chat_json_str + ) active_file_ids.update(validated_ids) except Exception as e: @@ -944,7 +1033,9 @@ def get_active_file_ids() -> Set[str]: try: items_str = json.dumps(folder.items) # Use utility to extract and validate file IDs - validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids(items_str) + validated_ids = ( + JSONFileIDExtractor.extract_and_validate_file_ids(items_str) + ) active_file_ids.update(validated_ids) except Exception as e: log.debug(f"Error processing folder {folder.id} items: {e}") @@ -953,7 +1044,9 @@ def get_active_file_ids() -> Set[str]: try: data_str = json.dumps(folder.data) # Use utility to extract and validate file IDs - validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids(data_str) + validated_ids = ( + JSONFileIDExtractor.extract_and_validate_file_ids(data_str) + ) active_file_ids.update(validated_ids) except Exception as e: log.debug(f"Error processing folder {folder.id} data: {e}") @@ -977,7 +1070,11 @@ def get_active_file_ids() -> Set[str]: else str(message_data_json) ) # Use utility to extract and validate file IDs - validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids(data_str) + validated_ids = ( + JSONFileIDExtractor.extract_and_validate_file_ids( + data_str + ) + ) active_file_ids.update(validated_ids) except Exception as e: log.debug( @@ -1064,51 +1161,51 @@ def cleanup_orphaned_uploads(active_file_ids: Set[str]) -> None: def delete_inactive_users( - inactive_days: int, - exempt_admin: bool = True, - exempt_pending: bool = True + inactive_days: int, exempt_admin: bool = True, exempt_pending: bool = True ) -> int: """ Delete users who have been inactive for the specified number of days. - + Returns the number of users deleted. """ if inactive_days is None: return 0 - + cutoff_time = int(time.time()) - (inactive_days * 86400) deleted_count = 0 - + try: users_to_delete = [] - + # Get all users and check activity all_users = Users.get_users()["users"] - + for user in all_users: # Skip if user is exempt if exempt_admin and user.role == "admin": continue if exempt_pending and user.role == "pending": continue - + # Check if user is inactive based on last_active_at if user.last_active_at < cutoff_time: users_to_delete.append(user) - + # Delete inactive users for user in users_to_delete: try: # Delete the user - this will cascade to all their data Users.delete_user_by_id(user.id) deleted_count += 1 - log.info(f"Deleted inactive user: {user.email} (last active: {user.last_active_at})") + log.info( + f"Deleted inactive user: {user.email} (last active: {user.last_active_at})" + ) except Exception as e: log.error(f"Failed to delete user {user.id}: {e}") - + except Exception as e: log.error(f"Error during inactive user deletion: {e}") - + return deleted_count @@ -1162,34 +1259,38 @@ def cleanup_audio_cache(max_age_days: Optional[int] = 30) -> None: async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): """ Prunes old and orphaned data using a safe, multi-stage process. - + If dry_run=True (default), returns preview counts without deleting anything. If dry_run=False, performs actual deletion and returns True on success. """ try: # Get vector database cleaner based on configuration vector_cleaner = get_vector_database_cleaner() - + if form_data.dry_run: log.info("Starting data pruning preview (dry run)") - + # Get counts for all enabled operations active_file_ids = get_active_file_ids() active_user_ids = {user.id for user in Users.get_users()["users"]} - active_kb_ids = {kb.id for kb in Knowledges.get_knowledge_bases() if kb.user_id in active_user_ids} - + active_kb_ids = { + kb.id + for kb in Knowledges.get_knowledge_bases() + if kb.user_id in active_user_ids + } + orphaned_counts = count_orphaned_records(form_data) - + result = PrunePreviewResult( inactive_users=count_inactive_users( form_data.delete_inactive_users_days, form_data.exempt_admin_users, - form_data.exempt_pending_users + form_data.exempt_pending_users, ), old_chats=count_old_chats( form_data.days, form_data.exempt_archived_chats, - form_data.exempt_chats_in_folders + form_data.exempt_chats_in_folders, ), orphaned_chats=orphaned_counts["chats"], orphaned_files=orphaned_counts["files"], @@ -1201,10 +1302,14 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): orphaned_notes=orphaned_counts["notes"], orphaned_folders=orphaned_counts["folders"], orphaned_uploads=count_orphaned_uploads(active_file_ids), - orphaned_vector_collections=vector_cleaner.count_orphaned_collections(active_file_ids, active_kb_ids), - audio_cache_files=count_audio_cache_files(form_data.audio_cache_max_age_days) + orphaned_vector_collections=vector_cleaner.count_orphaned_collections( + active_file_ids, active_kb_ids + ), + audio_cache_files=count_audio_cache_files( + form_data.audio_cache_max_age_days + ), ) - + log.info("Data pruning preview completed") return result @@ -1214,11 +1319,13 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): # Stage 0: Delete inactive users (if enabled) deleted_users = 0 if form_data.delete_inactive_users_days is not None: - log.info(f"Deleting users inactive for more than {form_data.delete_inactive_users_days} days") + log.info( + f"Deleting users inactive for more than {form_data.delete_inactive_users_days} days" + ) deleted_users = delete_inactive_users( form_data.delete_inactive_users_days, form_data.exempt_admin_users, - form_data.exempt_pending_users + form_data.exempt_pending_users, ) if deleted_users > 0: log.info(f"Deleted {deleted_users} inactive users") @@ -1399,9 +1506,11 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): final_active_kb_ids = {kb.id for kb in Knowledges.get_knowledge_bases()} cleanup_orphaned_uploads(final_active_file_ids) - + # Use modular vector database cleanup - vector_cleaner.cleanup_orphaned_collections(final_active_file_ids, final_active_kb_ids) + vector_cleaner.cleanup_orphaned_collections( + final_active_file_ids, final_active_kb_ids + ) # Stage 5: Audio cache cleanup log.info("Cleaning audio cache") @@ -1424,7 +1533,10 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): log.info("Vacuumed ChromaDB database") except Exception as e: log.error(f"Failed to vacuum ChromaDB database: {e}") - elif isinstance(vector_cleaner, PGVectorDatabaseCleaner) and vector_cleaner.session: + elif ( + isinstance(vector_cleaner, PGVectorDatabaseCleaner) + and vector_cleaner.session + ): try: vector_cleaner.session.execute(text("VACUUM ANALYZE")) vector_cleaner.session.commit() From 195c3a57ae7ca4164073f8ade9648b343c0760c0 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:33:17 +0100 Subject: [PATCH 37/43] Remove redundant parameter from delete_folder call --- backend/open_webui/routers/prune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index e43f8061d7..cc9a198f97 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -1487,7 +1487,7 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): for folder in Folders.get_all_folders(): if folder.user_id not in active_user_ids: Folders.delete_folder_by_id_and_user_id( - folder.id, folder.user_id, delete_chats=False + folder.id, folder.user_id ) folders_deleted += 1 deleted_others += 1 From 60d7ad22ee7ed74cd65b62d27fe5862e160d8eea Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Mon, 10 Nov 2025 17:14:27 +0100 Subject: [PATCH 38/43] Claude/vacuum optional 011 c uw61vf5 s rym bh cw u1 ls w (#28) PruneLock class Vector cleanup error reporting Lock acquisition/release Optional VACUUM Fixed folder deletion --- backend/open_webui/routers/prune.py | 599 +++++++++++------- src/lib/apis/prune.ts | 2 + .../components/admin/Settings/Database.svelte | 18 +- .../components/common/PruneDataDialog.svelte | 86 ++- 4 files changed, 448 insertions(+), 257 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index cc9a198f97..2968764d07 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -5,6 +5,8 @@ import shutil import json import re import sqlite3 +import uuid +from datetime import datetime, timedelta from typing import Optional, Set, Union from pathlib import Path from abc import ABC, abstractmethod @@ -36,6 +38,80 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"]) router = APIRouter() +class PruneLock: + """ + Simple file-based locking mechanism to prevent concurrent prune operations. + + This uses a lock file with timestamp to prevent multiple admins from running + prune simultaneously, which could cause race conditions and data corruption. + """ + + LOCK_FILE = Path(CACHE_DIR) / ".prune.lock" + LOCK_TIMEOUT = timedelta(hours=2) # Safety timeout + + @classmethod + def acquire(cls) -> bool: + """ + Try to acquire the lock. Returns True if acquired, False if already locked. + + If lock file exists but is stale (older than timeout), automatically + removes it and acquires a new lock. + """ + try: + # Check if lock file exists + if cls.LOCK_FILE.exists(): + # Read lock file to check if it's stale + try: + with open(cls.LOCK_FILE, 'r') as f: + lock_data = json.load(f) + lock_time = datetime.fromisoformat(lock_data['timestamp']) + operation_id = lock_data.get('operation_id', 'unknown') + + # Check if lock is stale + if datetime.utcnow() - lock_time > cls.LOCK_TIMEOUT: + log.warning(f"Found stale lock from {lock_time} (operation {operation_id}), removing") + cls.LOCK_FILE.unlink() + else: + # Lock is still valid + log.warning(f"Prune operation already in progress (started {lock_time}, operation {operation_id})") + return False + except (json.JSONDecodeError, KeyError, ValueError) as e: + # Corrupt lock file, remove it + log.warning(f"Found corrupt lock file, removing: {e}") + cls.LOCK_FILE.unlink() + + # Create lock file + operation_id = str(uuid.uuid4())[:8] + lock_data = { + 'timestamp': datetime.utcnow().isoformat(), + 'operation_id': operation_id, + 'pid': os.getpid() + } + + # Ensure parent directory exists + cls.LOCK_FILE.parent.mkdir(parents=True, exist_ok=True) + + with open(cls.LOCK_FILE, 'w') as f: + json.dump(lock_data, f) + + log.info(f"Acquired prune lock (operation {operation_id})") + return True + + except Exception as e: + log.error(f"Error acquiring prune lock: {e}") + return False + + @classmethod + def release(cls) -> None: + """Release the lock by removing the lock file.""" + try: + if cls.LOCK_FILE.exists(): + cls.LOCK_FILE.unlink() + log.info("Released prune lock") + except Exception as e: + log.error(f"Error releasing prune lock: {e}") + + class JSONFileIDExtractor: """ Utility for extracting and validating file IDs from JSON content. @@ -118,7 +194,7 @@ class VectorDatabaseCleaner(ABC): @abstractmethod def cleanup_orphaned_collections( self, active_file_ids: Set[str], active_kb_ids: Set[str] - ) -> int: + ) -> tuple[int, Optional[str]]: """ Actually delete orphaned vector collections. @@ -127,7 +203,9 @@ class VectorDatabaseCleaner(ABC): active_kb_ids: Set of knowledge base IDs that are still active Returns: - Number of collections that were actually deleted + Tuple of (deleted_count, error_message) + - deleted_count: Number of collections that were deleted + - error_message: None on success, error description on failure """ pass @@ -193,10 +271,10 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): def cleanup_orphaned_collections( self, active_file_ids: Set[str], active_kb_ids: Set[str] - ) -> int: + ) -> tuple[int, Optional[str]]: """Actually delete orphaned ChromaDB collections and database records.""" if not self.chroma_db_path.exists(): - return 0 + return (0, None) expected_collections = self._build_expected_collections( active_file_ids, active_kb_ids @@ -204,12 +282,15 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): uuid_to_collection = self._get_collection_mappings() deleted_count = 0 + errors = [] # First, clean up orphaned database records try: deleted_count += self._cleanup_orphaned_database_records() except Exception as e: - log.error(f"Error cleaning orphaned database records: {e}") + error_msg = f"ChromaDB database cleanup failed: {e}" + log.error(error_msg) + errors.append(error_msg) # Then clean up physical directories try: @@ -244,12 +325,17 @@ class ChromaDatabaseCleaner(VectorDatabaseCleaner): ) except Exception as e: - log.error(f"Error cleaning ChromaDB collections: {e}") + error_msg = f"ChromaDB directory cleanup failed: {e}" + log.error(error_msg) + errors.append(error_msg) if deleted_count > 0: log.info(f"Deleted {deleted_count} orphaned ChromaDB collections") - return deleted_count + # Return error if any critical failures occurred + if errors: + return (deleted_count, "; ".join(errors)) + return (deleted_count, None) def delete_collection(self, collection_name: str) -> bool: """Delete a specific ChromaDB collection by name.""" @@ -561,7 +647,7 @@ class PGVectorDatabaseCleaner(VectorDatabaseCleaner): def cleanup_orphaned_collections( self, active_file_ids: Set[str], active_kb_ids: Set[str] - ) -> int: + ) -> tuple[int, Optional[str]]: """ Delete orphaned PGVector collections using the existing client's delete method. @@ -569,8 +655,9 @@ class PGVectorDatabaseCleaner(VectorDatabaseCleaner): existing PGVector client's delete() method for each orphaned collection. """ if not self.session: - log.warning("PGVector session not available for cleanup") - return 0 + error_msg = "PGVector session not available for cleanup" + log.warning(error_msg) + return (0, error_msg) try: orphaned_collections = self._get_orphaned_collections( @@ -579,7 +666,7 @@ class PGVectorDatabaseCleaner(VectorDatabaseCleaner): if not orphaned_collections: log.debug("No orphaned PGVector collections found") - return 0 + return (0, None) deleted_count = 0 log.info( @@ -616,13 +703,14 @@ class PGVectorDatabaseCleaner(VectorDatabaseCleaner): f"Successfully deleted {deleted_count} orphaned PGVector collections" ) - return deleted_count + return (deleted_count, None) except Exception as e: if self.session: self.session.rollback() - log.error(f"Error cleaning orphaned PGVector collections: {e}") - return 0 + error_msg = f"PGVector cleanup failed: {e}" + log.error(error_msg) + return (0, error_msg) def delete_collection(self, collection_name: str) -> bool: """ @@ -706,9 +794,9 @@ class NoOpVectorDatabaseCleaner(VectorDatabaseCleaner): def cleanup_orphaned_collections( self, active_file_ids: Set[str], active_kb_ids: Set[str] - ) -> int: + ) -> tuple[int, Optional[str]]: """No collections to cleanup for unsupported databases.""" - return 0 + return (0, None) def delete_collection(self, collection_name: str) -> bool: """No collection to delete for unsupported databases.""" @@ -757,6 +845,7 @@ class PruneDataForm(BaseModel): delete_inactive_users_days: Optional[int] = None exempt_admin_users: bool = True exempt_pending_users: bool = True + run_vacuum: bool = False dry_run: bool = True @@ -1314,238 +1403,262 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): return result # Actual deletion logic (dry_run=False) - log.info("Starting data pruning process") - - # Stage 0: Delete inactive users (if enabled) - deleted_users = 0 - if form_data.delete_inactive_users_days is not None: - log.info( - f"Deleting users inactive for more than {form_data.delete_inactive_users_days} days" + # Acquire lock to prevent concurrent operations + if not PruneLock.acquire(): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="A prune operation is already in progress. Please wait for it to complete." ) - deleted_users = delete_inactive_users( - form_data.delete_inactive_users_days, - form_data.exempt_admin_users, - form_data.exempt_pending_users, - ) - if deleted_users > 0: - log.info(f"Deleted {deleted_users} inactive users") - else: - log.info("No inactive users found to delete") - else: - log.info("Skipping inactive user deletion (disabled)") - - # Stage 1: Delete old chats based on user criteria - if form_data.days is not None: - cutoff_time = int(time.time()) - (form_data.days * 86400) - chats_to_delete = [] - - for chat in Chats.get_chats(): - if chat.updated_at < cutoff_time: - if form_data.exempt_archived_chats and chat.archived: - continue - if form_data.exempt_chats_in_folders and ( - getattr(chat, "folder_id", None) is not None - or getattr(chat, "pinned", False) - ): - continue - chats_to_delete.append(chat) - - if chats_to_delete: - log.info( - f"Deleting {len(chats_to_delete)} old chats (older than {form_data.days} days)" - ) - for chat in chats_to_delete: - Chats.delete_chat_by_id(chat.id) - else: - log.info(f"No chats found older than {form_data.days} days") - else: - log.info("Skipping chat deletion (days parameter is None)") - - # Stage 2: Build preservation set - log.info("Building preservation set") - - active_user_ids = {user.id for user in Users.get_users()["users"]} - log.info(f"Found {len(active_user_ids)} active users") - - active_kb_ids = set() - knowledge_bases = Knowledges.get_knowledge_bases() - - for kb in knowledge_bases: - if kb.user_id in active_user_ids: - active_kb_ids.add(kb.id) - - log.info(f"Found {len(active_kb_ids)} active knowledge bases") - - active_file_ids = get_active_file_ids() - - # Stage 3: Delete orphaned database records - log.info("Deleting orphaned database records") - - deleted_files = 0 - for file_record in Files.get_files(): - should_delete = ( - file_record.id not in active_file_ids - or file_record.user_id not in active_user_ids - ) - - if should_delete: - if safe_delete_file_by_id(file_record.id): - deleted_files += 1 - - if deleted_files > 0: - log.info(f"Deleted {deleted_files} orphaned files") - - deleted_kbs = 0 - if form_data.delete_orphaned_knowledge_bases: - for kb in knowledge_bases: - if kb.user_id not in active_user_ids: - if vector_cleaner.delete_collection(kb.id): - Knowledges.delete_knowledge_by_id(kb.id) - deleted_kbs += 1 - - if deleted_kbs > 0: - log.info(f"Deleted {deleted_kbs} orphaned knowledge bases") - else: - log.info("Skipping knowledge base deletion (disabled)") - - deleted_others = 0 - - if form_data.delete_orphaned_chats: - chats_deleted = 0 - for chat in Chats.get_chats(): - if chat.user_id not in active_user_ids: - Chats.delete_chat_by_id(chat.id) - chats_deleted += 1 - deleted_others += 1 - if chats_deleted > 0: - log.info(f"Deleted {chats_deleted} orphaned chats") - else: - log.info("Skipping orphaned chat deletion (disabled)") - - if form_data.delete_orphaned_tools: - tools_deleted = 0 - for tool in Tools.get_tools(): - if tool.user_id not in active_user_ids: - Tools.delete_tool_by_id(tool.id) - tools_deleted += 1 - deleted_others += 1 - if tools_deleted > 0: - log.info(f"Deleted {tools_deleted} orphaned tools") - else: - log.info("Skipping tool deletion (disabled)") - - if form_data.delete_orphaned_functions: - functions_deleted = 0 - for function in Functions.get_functions(): - if function.user_id not in active_user_ids: - Functions.delete_function_by_id(function.id) - functions_deleted += 1 - deleted_others += 1 - if functions_deleted > 0: - log.info(f"Deleted {functions_deleted} orphaned functions") - else: - log.info("Skipping function deletion (disabled)") - - if form_data.delete_orphaned_notes: - notes_deleted = 0 - for note in Notes.get_notes(): - if note.user_id not in active_user_ids: - Notes.delete_note_by_id(note.id) - notes_deleted += 1 - deleted_others += 1 - if notes_deleted > 0: - log.info(f"Deleted {notes_deleted} orphaned notes") - else: - log.info("Skipping note deletion (disabled)") - - if form_data.delete_orphaned_prompts: - prompts_deleted = 0 - for prompt in Prompts.get_prompts(): - if prompt.user_id not in active_user_ids: - Prompts.delete_prompt_by_command(prompt.command) - prompts_deleted += 1 - deleted_others += 1 - if prompts_deleted > 0: - log.info(f"Deleted {prompts_deleted} orphaned prompts") - else: - log.info("Skipping prompt deletion (disabled)") - - if form_data.delete_orphaned_models: - models_deleted = 0 - for model in Models.get_all_models(): - if model.user_id not in active_user_ids: - Models.delete_model_by_id(model.id) - models_deleted += 1 - deleted_others += 1 - if models_deleted > 0: - log.info(f"Deleted {models_deleted} orphaned models") - else: - log.info("Skipping model deletion (disabled)") - - if form_data.delete_orphaned_folders: - folders_deleted = 0 - for folder in Folders.get_all_folders(): - if folder.user_id not in active_user_ids: - Folders.delete_folder_by_id_and_user_id( - folder.id, folder.user_id - ) - folders_deleted += 1 - deleted_others += 1 - if folders_deleted > 0: - log.info(f"Deleted {folders_deleted} orphaned folders") - else: - log.info("Skipping folder deletion (disabled)") - - if deleted_others > 0: - log.info(f"Total other orphaned records deleted: {deleted_others}") - - # Stage 4: Clean up orphaned physical files - log.info("Cleaning up orphaned physical files") - - final_active_file_ids = get_active_file_ids() - final_active_kb_ids = {kb.id for kb in Knowledges.get_knowledge_bases()} - - cleanup_orphaned_uploads(final_active_file_ids) - - # Use modular vector database cleanup - vector_cleaner.cleanup_orphaned_collections( - final_active_file_ids, final_active_kb_ids - ) - - # Stage 5: Audio cache cleanup - log.info("Cleaning audio cache") - cleanup_audio_cache(form_data.audio_cache_max_age_days) - - # Stage 6: Database optimization - log.info("Optimizing database") try: - with get_db() as db: - db.execute(text("VACUUM")) - except Exception as e: - log.error(f"Failed to vacuum main database: {e}") + log.info("Starting data pruning process") - # Vector database-specific optimization - if isinstance(vector_cleaner, ChromaDatabaseCleaner): - try: - with sqlite3.connect(str(vector_cleaner.chroma_db_path)) as conn: - conn.execute("VACUUM") - log.info("Vacuumed ChromaDB database") - except Exception as e: - log.error(f"Failed to vacuum ChromaDB database: {e}") - elif ( - isinstance(vector_cleaner, PGVectorDatabaseCleaner) - and vector_cleaner.session - ): - try: - vector_cleaner.session.execute(text("VACUUM ANALYZE")) - vector_cleaner.session.commit() - log.info("Executed VACUUM ANALYZE on PostgreSQL database") - except Exception as e: - log.error(f"Failed to vacuum PostgreSQL database: {e}") + # Stage 0: Delete inactive users (if enabled) + deleted_users = 0 + if form_data.delete_inactive_users_days is not None: + log.info( + f"Deleting users inactive for more than {form_data.delete_inactive_users_days} days" + ) + deleted_users = delete_inactive_users( + form_data.delete_inactive_users_days, + form_data.exempt_admin_users, + form_data.exempt_pending_users, + ) + if deleted_users > 0: + log.info(f"Deleted {deleted_users} inactive users") + else: + log.info("No inactive users found to delete") + else: + log.info("Skipping inactive user deletion (disabled)") - log.info("Data pruning completed successfully") - return True + # Stage 1: Delete old chats based on user criteria + if form_data.days is not None: + cutoff_time = int(time.time()) - (form_data.days * 86400) + chats_to_delete = [] + + for chat in Chats.get_chats(): + if chat.updated_at < cutoff_time: + if form_data.exempt_archived_chats and chat.archived: + continue + if form_data.exempt_chats_in_folders and ( + getattr(chat, "folder_id", None) is not None + or getattr(chat, "pinned", False) + ): + continue + chats_to_delete.append(chat) + + if chats_to_delete: + log.info( + f"Deleting {len(chats_to_delete)} old chats (older than {form_data.days} days)" + ) + for chat in chats_to_delete: + Chats.delete_chat_by_id(chat.id) + else: + log.info(f"No chats found older than {form_data.days} days") + else: + log.info("Skipping chat deletion (days parameter is None)") + + # Stage 2: Build preservation set + log.info("Building preservation set") + + active_user_ids = {user.id for user in Users.get_users()["users"]} + log.info(f"Found {len(active_user_ids)} active users") + + active_kb_ids = set() + knowledge_bases = Knowledges.get_knowledge_bases() + + for kb in knowledge_bases: + if kb.user_id in active_user_ids: + active_kb_ids.add(kb.id) + + log.info(f"Found {len(active_kb_ids)} active knowledge bases") + + active_file_ids = get_active_file_ids() + + # Stage 3: Delete orphaned database records + log.info("Deleting orphaned database records") + + deleted_files = 0 + for file_record in Files.get_files(): + should_delete = ( + file_record.id not in active_file_ids + or file_record.user_id not in active_user_ids + ) + + if should_delete: + if safe_delete_file_by_id(file_record.id): + deleted_files += 1 + + if deleted_files > 0: + log.info(f"Deleted {deleted_files} orphaned files") + + deleted_kbs = 0 + if form_data.delete_orphaned_knowledge_bases: + for kb in knowledge_bases: + if kb.user_id not in active_user_ids: + if vector_cleaner.delete_collection(kb.id): + Knowledges.delete_knowledge_by_id(kb.id) + deleted_kbs += 1 + + if deleted_kbs > 0: + log.info(f"Deleted {deleted_kbs} orphaned knowledge bases") + else: + log.info("Skipping knowledge base deletion (disabled)") + + deleted_others = 0 + + if form_data.delete_orphaned_chats: + chats_deleted = 0 + for chat in Chats.get_chats(): + if chat.user_id not in active_user_ids: + Chats.delete_chat_by_id(chat.id) + chats_deleted += 1 + deleted_others += 1 + if chats_deleted > 0: + log.info(f"Deleted {chats_deleted} orphaned chats") + else: + log.info("Skipping orphaned chat deletion (disabled)") + + if form_data.delete_orphaned_tools: + tools_deleted = 0 + for tool in Tools.get_tools(): + if tool.user_id not in active_user_ids: + Tools.delete_tool_by_id(tool.id) + tools_deleted += 1 + deleted_others += 1 + if tools_deleted > 0: + log.info(f"Deleted {tools_deleted} orphaned tools") + else: + log.info("Skipping tool deletion (disabled)") + + if form_data.delete_orphaned_functions: + functions_deleted = 0 + for function in Functions.get_functions(): + if function.user_id not in active_user_ids: + Functions.delete_function_by_id(function.id) + functions_deleted += 1 + deleted_others += 1 + if functions_deleted > 0: + log.info(f"Deleted {functions_deleted} orphaned functions") + else: + log.info("Skipping function deletion (disabled)") + + if form_data.delete_orphaned_notes: + notes_deleted = 0 + for note in Notes.get_notes(): + if note.user_id not in active_user_ids: + Notes.delete_note_by_id(note.id) + notes_deleted += 1 + deleted_others += 1 + if notes_deleted > 0: + log.info(f"Deleted {notes_deleted} orphaned notes") + else: + log.info("Skipping note deletion (disabled)") + + if form_data.delete_orphaned_prompts: + prompts_deleted = 0 + for prompt in Prompts.get_prompts(): + if prompt.user_id not in active_user_ids: + Prompts.delete_prompt_by_command(prompt.command) + prompts_deleted += 1 + deleted_others += 1 + if prompts_deleted > 0: + log.info(f"Deleted {prompts_deleted} orphaned prompts") + else: + log.info("Skipping prompt deletion (disabled)") + + if form_data.delete_orphaned_models: + models_deleted = 0 + for model in Models.get_all_models(): + if model.user_id not in active_user_ids: + Models.delete_model_by_id(model.id) + models_deleted += 1 + deleted_others += 1 + if models_deleted > 0: + log.info(f"Deleted {models_deleted} orphaned models") + else: + log.info("Skipping model deletion (disabled)") + + if form_data.delete_orphaned_folders: + folders_deleted = 0 + for folder in Folders.get_all_folders(): + if folder.user_id not in active_user_ids: + Folders.delete_folder_by_id_and_user_id( + folder.id, folder.user_id + ) + folders_deleted += 1 + deleted_others += 1 + if folders_deleted > 0: + log.info(f"Deleted {folders_deleted} orphaned folders") + else: + log.info("Skipping folder deletion (disabled)") + + if deleted_others > 0: + log.info(f"Total other orphaned records deleted: {deleted_others}") + + # Stage 4: Clean up orphaned physical files + log.info("Cleaning up orphaned physical files") + + final_active_file_ids = get_active_file_ids() + final_active_kb_ids = {kb.id for kb in Knowledges.get_knowledge_bases()} + + cleanup_orphaned_uploads(final_active_file_ids) + + # Use modular vector database cleanup + warnings = [] + deleted_vector_count, vector_error = vector_cleaner.cleanup_orphaned_collections( + final_active_file_ids, final_active_kb_ids + ) + if vector_error: + warnings.append(f"Vector cleanup warning: {vector_error}") + log.warning(f"Vector cleanup completed with errors: {vector_error}") + + # Stage 5: Audio cache cleanup + log.info("Cleaning audio cache") + cleanup_audio_cache(form_data.audio_cache_max_age_days) + + # Stage 6: Database optimization (optional) + if form_data.run_vacuum: + log.info("Optimizing database with VACUUM (this may take a while and lock the database)") + + try: + with get_db() as db: + db.execute(text("VACUUM")) + log.info("Vacuumed main database") + except Exception as e: + log.error(f"Failed to vacuum main database: {e}") + + # Vector database-specific optimization + if isinstance(vector_cleaner, ChromaDatabaseCleaner): + try: + with sqlite3.connect(str(vector_cleaner.chroma_db_path)) as conn: + conn.execute("VACUUM") + log.info("Vacuumed ChromaDB database") + except Exception as e: + log.error(f"Failed to vacuum ChromaDB database: {e}") + elif ( + isinstance(vector_cleaner, PGVectorDatabaseCleaner) + and vector_cleaner.session + ): + try: + vector_cleaner.session.execute(text("VACUUM ANALYZE")) + vector_cleaner.session.commit() + log.info("Executed VACUUM ANALYZE on PostgreSQL database") + except Exception as e: + log.error(f"Failed to vacuum PostgreSQL database: {e}") + else: + log.info("Skipping VACUUM optimization (not enabled)") + + # Log any warnings collected during pruning + if warnings: + log.warning(f"Data pruning completed with warnings: {'; '.join(warnings)}") + + log.info("Data pruning completed successfully") + return True + + finally: + # Always release lock, even if operation fails + PruneLock.release() except Exception as e: log.exception(f"Error during data pruning: {e}") diff --git a/src/lib/apis/prune.ts b/src/lib/apis/prune.ts index 5dda128836..f5e555aebf 100644 --- a/src/lib/apis/prune.ts +++ b/src/lib/apis/prune.ts @@ -17,6 +17,7 @@ export const pruneData = async ( delete_inactive_users_days: number | null = null, exempt_admin_users: boolean = true, exempt_pending_users: boolean = true, + run_vacuum: boolean = false, dry_run: boolean // Removed default value to ensure explicit passing ) => { let error = null; @@ -43,6 +44,7 @@ export const pruneData = async ( delete_inactive_users_days, exempt_admin_users, exempt_pending_users, + run_vacuum, dry_run }) }) diff --git a/src/lib/components/admin/Settings/Database.svelte b/src/lib/components/admin/Settings/Database.svelte index bee4f0c01d..1b56ca57e0 100644 --- a/src/lib/components/admin/Settings/Database.svelte +++ b/src/lib/components/admin/Settings/Database.svelte @@ -31,12 +31,12 @@ const handlePruneDataPreview = async (event) => { const settings = event.detail; lastPruneSettings = settings; - + console.log('Preview call - dry_run should be TRUE'); const res = await pruneData( - localStorage.token, - settings.days, - settings.exempt_archived_chats, + localStorage.token, + settings.days, + settings.exempt_archived_chats, settings.exempt_chats_in_folders, settings.delete_orphaned_chats, settings.delete_orphaned_tools, @@ -50,6 +50,7 @@ settings.delete_inactive_users_days, settings.exempt_admin_users, settings.exempt_pending_users, + settings.run_vacuum, true // dry_run = true for preview ).catch((error) => { toast.error(`${error}`); @@ -64,12 +65,12 @@ const handleConfirmPrune = async () => { if (!lastPruneSettings) return; - + console.log('Confirm call - dry_run should be FALSE'); const res = await pruneData( - localStorage.token, - lastPruneSettings.days, - lastPruneSettings.exempt_archived_chats, + localStorage.token, + lastPruneSettings.days, + lastPruneSettings.exempt_archived_chats, lastPruneSettings.exempt_chats_in_folders, lastPruneSettings.delete_orphaned_chats, lastPruneSettings.delete_orphaned_tools, @@ -83,6 +84,7 @@ lastPruneSettings.delete_inactive_users_days, lastPruneSettings.exempt_admin_users, lastPruneSettings.exempt_pending_users, + lastPruneSettings.run_vacuum, false // dry_run = false for actual pruning ).catch((error) => { toast.error(`${error}`); diff --git a/src/lib/components/common/PruneDataDialog.svelte b/src/lib/components/common/PruneDataDialog.svelte index 8d5d910422..22e31eeb3d 100644 --- a/src/lib/components/common/PruneDataDialog.svelte +++ b/src/lib/components/common/PruneDataDialog.svelte @@ -31,7 +31,10 @@ // Audio cache cleanup let cleanupAudioCache = true; let audio_cache_max_age_days = 30; - + + // System/Database optimization + let run_vacuum = false; + let showDetailsExpanded = false; let activeDetailsTab = 'users'; let activeSettingsTab = 'users'; @@ -40,8 +43,8 @@ const dispatch = createEventDispatcher(); const preview = () => { - dispatch('preview', { - days: deleteChatsByAge ? days : null, + dispatch('preview', { + days: deleteChatsByAge ? days : null, exempt_archived_chats, exempt_chats_in_folders, delete_orphaned_chats, @@ -55,7 +58,8 @@ audio_cache_max_age_days: cleanupAudioCache ? audio_cache_max_age_days : null, delete_inactive_users_days: deleteInactiveUsers ? delete_inactive_users_days : null, exempt_admin_users, - exempt_pending_users + exempt_pending_users, + run_vacuum }); show = false; }; @@ -94,9 +98,12 @@ curl -X POST "${window.location.origin}/api/v1/prune/" \\ "delete_orphaned_models": ${delete_orphaned_models}, "delete_orphaned_notes": ${delete_orphaned_notes}, "delete_orphaned_folders": ${delete_orphaned_folders}, - + // AUDIO CACHE CLEANUP (null = disabled) - "audio_cache_max_age_days": ${cleanupAudioCache ? audio_cache_max_age_days : null} // TTS/STT files + "audio_cache_max_age_days": ${cleanupAudioCache ? audio_cache_max_age_days : null}, // TTS/STT files + + // DATABASE OPTIMIZATION (WARNING: Locks database during execution!) + "run_vacuum": ${run_vacuum} // Reclaim disk space - only enable during maintenance windows }' # API KEY vs JWT TOKEN: @@ -359,6 +366,12 @@ curl -X POST "${window.location.origin}/api/v1/prune/" \\ > {$i18n.t('Audio Cache')} + @@ -744,6 +757,67 @@ curl -X POST "${window.location.origin}/api/v1/prune/" \\ {/if} + + {:else if activeSettingsTab === 'system'} + +
+
+
+
+ +
+
+
+ {$i18n.t('Run VACUUM optimization')} +
+ + + +
+
{$i18n.t('Database Optimization Warning:')}
+
+

{$i18n.t('VACUUM reclaims disk space by rebuilding the database file.')}

+

{$i18n.t('⚠️ This may take a very long time on large databases and will LOCK the entire database during execution.')}

+

{$i18n.t('It is strongly recommended to NOT run this while users are actively using the platform.')}

+

{$i18n.t('💡 Best practice: Run during scheduled maintenance windows.')}

+
+
+
+
+
+
+ {$i18n.t('Reclaim disk space after cleanup (locks database during operation)')} +
+
+
+
+ + + {#if run_vacuum} +
+
+
+
+ + + +
+
+

+ {$i18n.t('VACUUM Enabled - Important Considerations:')} +

+
+

• {$i18n.t('Database will be locked during VACUUM - all users will experience errors')}

+

• {$i18n.t('Operation duration depends on database size (can be 5-30+ minutes)')}

+

• {$i18n.t('Recommended only during scheduled maintenance windows')}

+

• {$i18n.t('Not required for routine cleanups - only when reclaiming disk space is critical')}

+
+
+
+
+
+ {/if} +
{/if} From 873b73e66873e5dd6fb44fed9520dadfec69e53f Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 11 Nov 2025 19:39:20 +0100 Subject: [PATCH 39/43] feat: Make VACUUM database optimization optional (#30) Co-authored-by: Claude --- backend/open_webui/routers/prune.py | 97 +++++++++++++++++++---------- 1 file changed, 63 insertions(+), 34 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 2968764d07..112901118d 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -17,7 +17,7 @@ from sqlalchemy import text from open_webui.utils.auth import get_admin_user from open_webui.models.users import Users -from open_webui.models.chats import Chats +from open_webui.models.chats import Chat, ChatModel, Chats from open_webui.models.files import Files from open_webui.models.notes import Notes from open_webui.models.prompts import Prompts @@ -128,6 +128,26 @@ class JSONFileIDExtractor: r"/api/v1/files/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})" ) + @classmethod + def extract_file_ids(cls, json_string: str) -> Set[str]: + """ + Extract file IDs from JSON string WITHOUT database validation. + + Args: + json_string: JSON content as string (or any string to scan) + + Returns: + Set of extracted file IDs (not validated against database) + + Note: + Use this method when you have a preloaded set of valid file IDs + to validate against, avoiding N database queries. + """ + potential_ids = [] + potential_ids.extend(cls._FILE_ID_PATTERN.findall(json_string)) + potential_ids.extend(cls._URL_PATTERN.findall(json_string)) + return set(potential_ids) + @classmethod def extract_and_validate_file_ids(cls, json_string: str) -> Set[str]: """ @@ -1068,6 +1088,10 @@ def get_active_file_ids() -> Set[str]: active_file_ids = set() try: + # Preload all valid file IDs to avoid N database queries during validation + # This is O(1) set lookup instead of O(n) DB queries + all_file_ids = {f.id for f in Files.get_files()} + log.debug(f"Preloaded {len(all_file_ids)} file IDs for validation") # Scan knowledge bases for file references knowledge_bases = Knowledges.get_knowledge_bases() log.debug(f"Found {len(knowledge_bases)} knowledge bases") @@ -1092,26 +1116,34 @@ def get_active_file_ids() -> Set[str]: for file_id in file_ids: if isinstance(file_id, str) and file_id.strip(): - active_file_ids.add(file_id.strip()) + stripped_id = file_id.strip() + # Validate against preloaded set (O(1) lookup) + if stripped_id in all_file_ids: + active_file_ids.add(stripped_id) # Scan chats for file references - chats = Chats.get_chats() - log.debug(f"Found {len(chats)} chats to scan for file references") + # Stream chats to avoid loading all into memory + chat_count = 0 + with get_db() as db: + for chat_orm in db.query(Chat).yield_per(1000): + chat_count += 1 + chat = ChatModel.model_validate(chat_orm) - for chat in chats: - if not chat.chat or not isinstance(chat.chat, dict): - continue + if not chat.chat or not isinstance(chat.chat, dict): + continue - try: - chat_json_str = json.dumps(chat.chat) - # Use utility to extract and validate file IDs - validated_ids = JSONFileIDExtractor.extract_and_validate_file_ids( - chat_json_str - ) - active_file_ids.update(validated_ids) + try: + chat_json_str = json.dumps(chat.chat) + # Extract file IDs without DB queries + extracted_ids = JSONFileIDExtractor.extract_file_ids(chat_json_str) + # Validate against preloaded set (O(1) per ID) + validated_ids = extracted_ids & all_file_ids + active_file_ids.update(validated_ids) - except Exception as e: - log.debug(f"Error processing chat {chat.id} for file references: {e}") + except Exception as e: + log.debug(f"Error processing chat {chat.id} for file references: {e}") + + log.debug(f"Scanned {chat_count} chats for file references") # Scan folders for file references try: @@ -1121,10 +1153,10 @@ def get_active_file_ids() -> Set[str]: if folder.items: try: items_str = json.dumps(folder.items) - # Use utility to extract and validate file IDs - validated_ids = ( - JSONFileIDExtractor.extract_and_validate_file_ids(items_str) - ) + # Extract file IDs without DB queries + extracted_ids = JSONFileIDExtractor.extract_file_ids(items_str) + # Validate against preloaded set (O(1) per ID) + validated_ids = extracted_ids & all_file_ids active_file_ids.update(validated_ids) except Exception as e: log.debug(f"Error processing folder {folder.id} items: {e}") @@ -1132,10 +1164,10 @@ def get_active_file_ids() -> Set[str]: if hasattr(folder, "data") and folder.data: try: data_str = json.dumps(folder.data) - # Use utility to extract and validate file IDs - validated_ids = ( - JSONFileIDExtractor.extract_and_validate_file_ids(data_str) - ) + # Extract file IDs without DB queries + extracted_ids = JSONFileIDExtractor.extract_file_ids(data_str) + # Validate against preloaded set (O(1) per ID) + validated_ids = extracted_ids & all_file_ids active_file_ids.update(validated_ids) except Exception as e: log.debug(f"Error processing folder {folder.id} data: {e}") @@ -1146,11 +1178,10 @@ def get_active_file_ids() -> Set[str]: # Scan standalone messages for file references try: with get_db() as db: - message_results = db.execute( - text("SELECT id, data FROM message WHERE data IS NOT NULL") - ).fetchall() + stmt = text("SELECT id, data FROM message WHERE data IS NOT NULL") - for message_id, message_data_json in message_results: + for row in db.execute(stmt).yield_per(1000): + message_id, message_data_json = row if message_data_json: try: data_str = ( @@ -1158,12 +1189,10 @@ def get_active_file_ids() -> Set[str]: if isinstance(message_data_json, dict) else str(message_data_json) ) - # Use utility to extract and validate file IDs - validated_ids = ( - JSONFileIDExtractor.extract_and_validate_file_ids( - data_str - ) - ) + # Extract file IDs without DB queries + extracted_ids = JSONFileIDExtractor.extract_file_ids(data_str) + # Validate against preloaded set (O(1) per ID) + validated_ids = extracted_ids & all_file_ids active_file_ids.update(validated_ids) except Exception as e: log.debug( From 20187f9a2dd64633f8e713745929439a60688bc2 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Thu, 13 Nov 2025 18:01:25 +0100 Subject: [PATCH 40/43] fix file lock (#33) Co-authored-by: Claude --- backend/open_webui/routers/prune.py | 81 ++++++++++++++++++----------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 112901118d..cd7053e7fa 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -1381,6 +1381,13 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): If dry_run=True (default), returns preview counts without deleting anything. If dry_run=False, performs actual deletion and returns True on success. """ + # Acquire lock to prevent concurrent operations (including previews) + if not PruneLock.acquire(): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="A prune operation is already in progress. Please wait for it to complete." + ) + try: # Get vector database cleaner based on configuration vector_cleaner = get_vector_database_cleaner() @@ -1642,45 +1649,54 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): warnings.append(f"Vector cleanup warning: {vector_error}") log.warning(f"Vector cleanup completed with errors: {vector_error}") - # Stage 5: Audio cache cleanup - log.info("Cleaning audio cache") - cleanup_audio_cache(form_data.audio_cache_max_age_days) + # Use modular vector database cleanup + warnings = [] + deleted_vector_count, vector_error = vector_cleaner.cleanup_orphaned_collections( + final_active_file_ids, final_active_kb_ids + ) + if vector_error: + warnings.append(f"Vector cleanup warning: {vector_error}") + log.warning(f"Vector cleanup completed with errors: {vector_error}") # Stage 6: Database optimization (optional) if form_data.run_vacuum: log.info("Optimizing database with VACUUM (this may take a while and lock the database)") + # Stage 6: Database optimization (optional) + if form_data.run_vacuum: + log.info("Optimizing database with VACUUM (this may take a while and lock the database)") + + try: + with get_db() as db: + db.execute(text("VACUUM")) + log.info("Vacuumed main database") + except Exception as e: + log.error(f"Failed to vacuum main database: {e}") + + # Vector database-specific optimization + if isinstance(vector_cleaner, ChromaDatabaseCleaner): try: - with get_db() as db: - db.execute(text("VACUUM")) - log.info("Vacuumed main database") + with sqlite3.connect(str(vector_cleaner.chroma_db_path)) as conn: + conn.execute("VACUUM") + log.info("Vacuumed ChromaDB database") except Exception as e: - log.error(f"Failed to vacuum main database: {e}") + log.error(f"Failed to vacuum ChromaDB database: {e}") + elif ( + isinstance(vector_cleaner, PGVectorDatabaseCleaner) + and vector_cleaner.session + ): + try: + vector_cleaner.session.execute(text("VACUUM ANALYZE")) + vector_cleaner.session.commit() + log.info("Executed VACUUM ANALYZE on PostgreSQL database") + except Exception as e: + log.error(f"Failed to vacuum PostgreSQL database: {e}") + else: + log.info("Skipping VACUUM optimization (not enabled)") - # Vector database-specific optimization - if isinstance(vector_cleaner, ChromaDatabaseCleaner): - try: - with sqlite3.connect(str(vector_cleaner.chroma_db_path)) as conn: - conn.execute("VACUUM") - log.info("Vacuumed ChromaDB database") - except Exception as e: - log.error(f"Failed to vacuum ChromaDB database: {e}") - elif ( - isinstance(vector_cleaner, PGVectorDatabaseCleaner) - and vector_cleaner.session - ): - try: - vector_cleaner.session.execute(text("VACUUM ANALYZE")) - vector_cleaner.session.commit() - log.info("Executed VACUUM ANALYZE on PostgreSQL database") - except Exception as e: - log.error(f"Failed to vacuum PostgreSQL database: {e}") - else: - log.info("Skipping VACUUM optimization (not enabled)") - - # Log any warnings collected during pruning - if warnings: - log.warning(f"Data pruning completed with warnings: {'; '.join(warnings)}") + # Log any warnings collected during pruning + if warnings: + log.warning(f"Data pruning completed with warnings: {'; '.join(warnings)}") log.info("Data pruning completed successfully") return True @@ -1695,3 +1711,6 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=ERROR_MESSAGES.DEFAULT("Data pruning failed"), ) + finally: + # Always release lock, even if operation fails + PruneLock.release() From c307d872629bfa4cc7f49077b2a2276bdc33d774 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Thu, 13 Nov 2025 19:13:21 +0100 Subject: [PATCH 41/43] sync (#34) Co-authored-by: Claude --- backend/open_webui/routers/prune.py | 178 +++++++++++++++++++--------- 1 file changed, 120 insertions(+), 58 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index cd7053e7fa..fc83cd6a9c 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -13,11 +13,12 @@ from abc import ABC, abstractmethod from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel -from sqlalchemy import text +from sqlalchemy import select, text from open_webui.utils.auth import get_admin_user from open_webui.models.users import Users from open_webui.models.chats import Chat, ChatModel, Chats +from open_webui.models.messages import Message from open_webui.models.files import Files from open_webui.models.notes import Notes from open_webui.models.prompts import Prompts @@ -25,7 +26,7 @@ from open_webui.models.models import Models from open_webui.models.knowledge import Knowledges from open_webui.models.functions import Functions from open_webui.models.tools import Tools -from open_webui.models.folders import Folders +from open_webui.models.folders import Folder, Folders from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT, VECTOR_DB from open_webui.constants import ERROR_MESSAGES from open_webui.env import SRC_LOG_LEVELS @@ -181,6 +182,65 @@ class JSONFileIDExtractor: return validated_ids +# UUID pattern for direct dict traversal (Phase 1.5 optimization) +UUID_PATTERN = re.compile( + r'^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$' +) + + +def collect_file_ids_from_dict(obj, out: Set[str], valid_ids: Set[str], _depth: int = 0) -> None: + """ + Recursively traverse dict/list structures and collect file IDs. + + This function replaces json.dumps() + regex approach with direct dict traversal, + reducing memory usage by ~75% on large chat databases. + + Args: + obj: Dict, list, or any value to traverse + out: Set to accumulate found file IDs into + valid_ids: Set of known valid file IDs (for O(1) validation) + _depth: Current recursion depth (safety limit) + + Patterns detected: + - {"id": "uuid"} + - {"file_id": "uuid"} + - {"fileId": "uuid"} + - {"file_ids": ["uuid1", "uuid2"]} + - {"fileIds": ["uuid1", "uuid2"]} + """ + # Safety: Prevent excessive recursion + if _depth > 100: + return + + if isinstance(obj, dict): + # Check individual file ID fields + for field_name in ['id', 'file_id', 'fileId']: + fid = obj.get(field_name) + if isinstance(fid, str) and UUID_PATTERN.fullmatch(fid): + if fid in valid_ids: + out.add(fid) + + # Check file ID array fields + for field_name in ['file_ids', 'fileIds']: + fid_array = obj.get(field_name) + if isinstance(fid_array, list): + for fid in fid_array: + if isinstance(fid, str) and UUID_PATTERN.fullmatch(fid): + if fid in valid_ids: + out.add(fid) + + # Recurse into all dict values + for value in obj.values(): + collect_file_ids_from_dict(value, out, valid_ids, _depth + 1) + + elif isinstance(obj, list): + # Recurse into all list items + for item in obj: + collect_file_ids_from_dict(item, out, valid_ids, _depth + 1) + + # Primitives (str, int, None, etc.) - do nothing + + class VectorDatabaseCleaner(ABC): """ Abstract base class for vector database cleanup operations. @@ -1122,82 +1182,84 @@ def get_active_file_ids() -> Set[str]: active_file_ids.add(stripped_id) # Scan chats for file references - # Stream chats to avoid loading all into memory + # Stream chats using Core SELECT to avoid ORM overhead chat_count = 0 with get_db() as db: - for chat_orm in db.query(Chat).yield_per(1000): - chat_count += 1 - chat = ChatModel.model_validate(chat_orm) + stmt = select(Chat.id, Chat.chat) + result = db.execution_options(stream_results=True).execute(stmt) - if not chat.chat or not isinstance(chat.chat, dict): - continue + while True: + rows = result.fetchmany(1000) + if not rows: + break - try: - chat_json_str = json.dumps(chat.chat) - # Extract file IDs without DB queries - extracted_ids = JSONFileIDExtractor.extract_file_ids(chat_json_str) - # Validate against preloaded set (O(1) per ID) - validated_ids = extracted_ids & all_file_ids - active_file_ids.update(validated_ids) + for chat_id, chat_dict in rows: + chat_count += 1 - except Exception as e: - log.debug(f"Error processing chat {chat.id} for file references: {e}") + # Skip if no chat data or not a dict + if not chat_dict or not isinstance(chat_dict, dict): + continue + + try: + # Direct dict traversal (no json.dumps needed) + collect_file_ids_from_dict(chat_dict, active_file_ids, all_file_ids) + except Exception as e: + log.debug(f"Error processing chat {chat_id} for file references: {e}") log.debug(f"Scanned {chat_count} chats for file references") # Scan folders for file references + # Stream folders using Core SELECT to avoid ORM overhead try: - folders = Folders.get_all_folders() + with get_db() as db: + stmt = select(Folder.id, Folder.items, Folder.data) + result = db.execution_options(stream_results=True).execute(stmt) - for folder in folders: - if folder.items: - try: - items_str = json.dumps(folder.items) - # Extract file IDs without DB queries - extracted_ids = JSONFileIDExtractor.extract_file_ids(items_str) - # Validate against preloaded set (O(1) per ID) - validated_ids = extracted_ids & all_file_ids - active_file_ids.update(validated_ids) - except Exception as e: - log.debug(f"Error processing folder {folder.id} items: {e}") + while True: + rows = result.fetchmany(100) + if not rows: + break - if hasattr(folder, "data") and folder.data: - try: - data_str = json.dumps(folder.data) - # Extract file IDs without DB queries - extracted_ids = JSONFileIDExtractor.extract_file_ids(data_str) - # Validate against preloaded set (O(1) per ID) - validated_ids = extracted_ids & all_file_ids - active_file_ids.update(validated_ids) - except Exception as e: - log.debug(f"Error processing folder {folder.id} data: {e}") + for folder_id, items_dict, data_dict in rows: + # Process folder.items + if items_dict: + try: + # Direct dict traversal (no json.dumps needed) + collect_file_ids_from_dict(items_dict, active_file_ids, all_file_ids) + except Exception as e: + log.debug(f"Error processing folder {folder_id} items: {e}") + + # Process folder.data + if data_dict: + try: + # Direct dict traversal (no json.dumps needed) + collect_file_ids_from_dict(data_dict, active_file_ids, all_file_ids) + except Exception as e: + log.debug(f"Error processing folder {folder_id} data: {e}") except Exception as e: log.debug(f"Error scanning folders for file references: {e}") # Scan standalone messages for file references + # Stream messages using Core SELECT to avoid text() and yield_per issues try: with get_db() as db: - stmt = text("SELECT id, data FROM message WHERE data IS NOT NULL") + stmt = select(Message.id, Message.data).where(Message.data.isnot(None)) + result = db.execution_options(stream_results=True).execute(stmt) + + while True: + rows = result.fetchmany(1000) + if not rows: + break + + for message_id, message_data_dict in rows: + if message_data_dict: + try: + # Direct dict traversal (no json.dumps needed) + collect_file_ids_from_dict(message_data_dict, active_file_ids, all_file_ids) + except Exception as e: + log.debug(f"Error processing message {message_id} data: {e}") - for row in db.execute(stmt).yield_per(1000): - message_id, message_data_json = row - if message_data_json: - try: - data_str = ( - json.dumps(message_data_json) - if isinstance(message_data_json, dict) - else str(message_data_json) - ) - # Extract file IDs without DB queries - extracted_ids = JSONFileIDExtractor.extract_file_ids(data_str) - # Validate against preloaded set (O(1) per ID) - validated_ids = extracted_ids & all_file_ids - active_file_ids.update(validated_ids) - except Exception as e: - log.debug( - f"Error processing message {message_id} data: {e}" - ) except Exception as e: log.debug(f"Error scanning messages for file references: {e}") From a4ddb4b15be7d8cce09daf648df66a41b7469a9f Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Thu, 13 Nov 2025 19:29:02 +0100 Subject: [PATCH 42/43] fix (#35) Co-authored-by: Claude Fix #1: Remove duplicate scan in preview mode Fix #2: Cache stat() result in audio cleanup --- backend/open_webui/routers/prune.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index fc83cd6a9c..857832883f 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -999,7 +999,11 @@ def count_old_chats( return count -def count_orphaned_records(form_data: PruneDataForm) -> dict: +def count_orphaned_records( + form_data: PruneDataForm, + active_file_ids: Set[str], + active_user_ids: Set[str] +) -> dict: """Count orphaned database records that would be deleted.""" counts = { "chats": 0, @@ -1014,12 +1018,6 @@ def count_orphaned_records(form_data: PruneDataForm) -> dict: } try: - # Get active user IDs - active_user_ids = {user.id for user in Users.get_users()["users"]} - - # Get active file IDs for file orphan detection - active_file_ids = get_active_file_ids() - # Count orphaned files for file_record in Files.get_files(): should_delete = ( @@ -1415,10 +1413,11 @@ def cleanup_audio_cache(max_age_days: Optional[int] = 30) -> None: if not file_path.is_file(): continue - file_mtime = file_path.stat().st_mtime + stat_info = file_path.stat() + file_mtime = stat_info.st_mtime if file_mtime < cutoff_time: try: - file_size = file_path.stat().st_size + file_size = stat_info.st_size file_path.unlink() deleted_count += 1 total_size_deleted += file_size @@ -1466,7 +1465,7 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): if kb.user_id in active_user_ids } - orphaned_counts = count_orphaned_records(form_data) + orphaned_counts = count_orphaned_records(form_data, active_file_ids, active_user_ids) result = PrunePreviewResult( inactive_users=count_inactive_users( From 81c7617508101cfe16d551b07e94716d930e9bde Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Thu, 13 Nov 2025 20:45:47 +0100 Subject: [PATCH 43/43] feat: Make VACUUM database optimization optional (#36) Co-authored-by: Claude Fix #1: Remove duplicate scan in preview mode Fix #2: Cache stat() result in audio cleanup --- backend/open_webui/routers/prune.py | 34 +++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/backend/open_webui/routers/prune.py b/backend/open_webui/routers/prune.py index 857832883f..c90cf8d785 100644 --- a/backend/open_webui/routers/prune.py +++ b/backend/open_webui/routers/prune.py @@ -948,9 +948,16 @@ class PrunePreviewResult(BaseModel): # Counting helper functions for dry-run preview def count_inactive_users( - inactive_days: Optional[int], exempt_admin: bool, exempt_pending: bool + inactive_days: Optional[int], exempt_admin: bool, exempt_pending: bool, all_users=None ) -> int: - """Count users that would be deleted for inactivity.""" + """Count users that would be deleted for inactivity. + + Args: + inactive_days: Number of days of inactivity before deletion + exempt_admin: Whether to exempt admin users + exempt_pending: Whether to exempt pending users + all_users: Optional pre-fetched list of users to avoid duplicate queries + """ if inactive_days is None: return 0 @@ -958,7 +965,8 @@ def count_inactive_users( count = 0 try: - all_users = Users.get_users()["users"] + if all_users is None: + all_users = Users.get_users()["users"] for user in all_users: if exempt_admin and user.role == "admin": continue @@ -1139,9 +1147,12 @@ def count_audio_cache_files(max_age_days: Optional[int]) -> int: return count -def get_active_file_ids() -> Set[str]: +def get_active_file_ids(knowledge_bases=None) -> Set[str]: """ Get all file IDs that are actively referenced by knowledge bases, chats, folders, and messages. + + Args: + knowledge_bases: Optional pre-fetched list of knowledge bases to avoid duplicate queries """ active_file_ids = set() @@ -1151,7 +1162,8 @@ def get_active_file_ids() -> Set[str]: all_file_ids = {f.id for f in Files.get_files()} log.debug(f"Preloaded {len(all_file_ids)} file IDs for validation") # Scan knowledge bases for file references - knowledge_bases = Knowledges.get_knowledge_bases() + if knowledge_bases is None: + knowledge_bases = Knowledges.get_knowledge_bases() log.debug(f"Found {len(knowledge_bases)} knowledge bases") for kb in knowledge_bases: @@ -1457,13 +1469,16 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): log.info("Starting data pruning preview (dry run)") # Get counts for all enabled operations - active_file_ids = get_active_file_ids() - active_user_ids = {user.id for user in Users.get_users()["users"]} + # Fetch knowledge bases and users once to avoid duplicate queries + knowledge_bases = Knowledges.get_knowledge_bases() + all_users = Users.get_users()["users"] + active_user_ids = {user.id for user in all_users} active_kb_ids = { kb.id - for kb in Knowledges.get_knowledge_bases() + for kb in knowledge_bases if kb.user_id in active_user_ids } + active_file_ids = get_active_file_ids(knowledge_bases) orphaned_counts = count_orphaned_records(form_data, active_file_ids, active_user_ids) @@ -1472,6 +1487,7 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): form_data.delete_inactive_users_days, form_data.exempt_admin_users, form_data.exempt_pending_users, + all_users, ), old_chats=count_old_chats( form_data.days, @@ -1570,7 +1586,7 @@ async def prune_data(form_data: PruneDataForm, user=Depends(get_admin_user)): log.info(f"Found {len(active_kb_ids)} active knowledge bases") - active_file_ids = get_active_file_ids() + active_file_ids = get_active_file_ids(knowledge_bases) # Stage 3: Delete orphaned database records log.info("Deleting orphaned database records")