From b23abcbfe55f0357674ec139cbf781b31db0a9a0 Mon Sep 17 00:00:00 2001 From: Adam Tao Date: Wed, 18 Jun 2025 19:29:37 +0800 Subject: [PATCH 1/3] feat(db): Add DATABASE_ENABLE_SQLITE_WAL to enable SQLite WAL Signed-off-by: Adam Tao --- backend/open_webui/env.py | 4 ++++ backend/open_webui/internal/db.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index e561036408..391da6a53b 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -336,6 +336,10 @@ else: except Exception: DATABASE_POOL_RECYCLE = 3600 +DATABASE_ENABLE_SQLITE_WAL = ( + os.environ.get("DATABASE_ENABLE_SQLITE_WAL", "False").lower() == "true" +) + RESET_CONFIG_ON_START = ( os.environ.get("RESET_CONFIG_ON_START", "False").lower() == "true" ) diff --git a/backend/open_webui/internal/db.py b/backend/open_webui/internal/db.py index d7a200ff20..ccc3995ded 100644 --- a/backend/open_webui/internal/db.py +++ b/backend/open_webui/internal/db.py @@ -14,9 +14,10 @@ from open_webui.env import ( DATABASE_POOL_RECYCLE, DATABASE_POOL_SIZE, DATABASE_POOL_TIMEOUT, + DATABASE_ENABLE_SQLITE_WAL, ) from peewee_migrate import Router -from sqlalchemy import Dialect, create_engine, MetaData, types +from sqlalchemy import Dialect, create_engine, MetaData, event, types from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.pool import QueuePool, NullPool @@ -114,6 +115,14 @@ elif "sqlite" in SQLALCHEMY_DATABASE_URL: engine = create_engine( SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False} ) + def on_connect(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + if DATABASE_ENABLE_SQLITE_WAL: + cursor.execute("PRAGMA journal_mode=WAL") + else: + cursor.execute("PRAGMA journal_mode=DELETE") + cursor.close() + event.listen(engine, "connect", on_connect) else: if isinstance(DATABASE_POOL_SIZE, int): if DATABASE_POOL_SIZE > 0: From 635cb8e3ff0208724ff122b37128ae4e42f7ff05 Mon Sep 17 00:00:00 2001 From: Adam Tao Date: Mon, 7 Jul 2025 22:16:19 +0800 Subject: [PATCH 2/3] perf(db): deduplicate update_user_last_active_by_id to reduce conflicts Signed-off-by: Adam Tao --- backend/open_webui/env.py | 11 +++++++++ backend/open_webui/models/users.py | 3 +++ backend/open_webui/utils/misc.py | 39 ++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 391da6a53b..7ce17f57fc 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -340,6 +340,17 @@ DATABASE_ENABLE_SQLITE_WAL = ( os.environ.get("DATABASE_ENABLE_SQLITE_WAL", "False").lower() == "true" ) +DATABASE_DEDUPLICATE_INTERVAL = ( + os.environ.get("DATABASE_DEDUPLICATE_INTERVAL", 0.) +) +if DATABASE_DEDUPLICATE_INTERVAL == "": + DATABASE_DEDUPLICATE_INTERVAL = 0.0 +else: + try: + DATABASE_DEDUPLICATE_INTERVAL = float(DATABASE_DEDUPLICATE_INTERVAL) + except Exception: + DATABASE_DEDUPLICATE_INTERVAL = 0.0 + RESET_CONFIG_ON_START = ( os.environ.get("RESET_CONFIG_ON_START", "False").lower() == "true" ) diff --git a/backend/open_webui/models/users.py b/backend/open_webui/models/users.py index 60b6ad0c10..47cd7b0eb0 100644 --- a/backend/open_webui/models/users.py +++ b/backend/open_webui/models/users.py @@ -4,8 +4,10 @@ from typing import Optional from open_webui.internal.db import Base, JSONField, get_db +from open_webui.env import DATABASE_DEDUPLICATE_INTERVAL from open_webui.models.chats import Chats from open_webui.models.groups import Groups +from open_webui.utils.misc import deduplicate from pydantic import BaseModel, ConfigDict @@ -311,6 +313,7 @@ class UsersTable: except Exception: return None + @deduplicate(DATABASE_DEDUPLICATE_INTERVAL) def update_user_last_active_by_id(self, id: str) -> Optional[UserModel]: try: with get_db() as db: diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index 2a780209a7..8c4ae9d9a2 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -1,5 +1,6 @@ import hashlib import re +import threading import time import uuid import logging @@ -478,3 +479,41 @@ def convert_logit_bias_input_to_json(user_input): bias = 100 if bias > 100 else -100 if bias < -100 else bias logit_bias_json[token] = bias return json.dumps(logit_bias_json) + + +def freeze(value): + """ + Freeze a value to make it hashable. + """ + if isinstance(value, dict): + return frozenset((k, freeze(v)) for k, v in value.items()) + elif isinstance(value, list): + return tuple(freeze(v) for v in value) + return value + + +def deduplicate(interval: float = 10.0): + """ + Decorator to prevent a function from being called more than once within a specified duration. + If the function is called again within the duration, it returns None. To avoid returning + different types, the return type of the function should be Optional[T]. + + :param interval: Duration in seconds to wait before allowing the function to be called again. + """ + def decorator(func): + last_calls = {} + lock = threading.Lock() + + def wrapper(*args, **kwargs): + key = (args, freeze(kwargs)) + now = time.time() + if now - last_calls.get(key, 0) < interval: + return None + with lock: + if now - last_calls.get(key, 0) < interval: + return None + last_calls[key] = now + return func(*args, **kwargs) + return wrapper + + return decorator From 7bd7559bfe490c71d9601e35ae119d788a90c495 Mon Sep 17 00:00:00 2001 From: Adam Tao Date: Sat, 19 Jul 2025 19:20:53 +0800 Subject: [PATCH 3/3] refactor: format Signed-off-by: Adam Tao --- backend/open_webui/env.py | 4 +--- backend/open_webui/internal/db.py | 2 ++ backend/open_webui/utils/misc.py | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 7ce17f57fc..5f5f1be8e4 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -336,9 +336,7 @@ else: except Exception: DATABASE_POOL_RECYCLE = 3600 -DATABASE_ENABLE_SQLITE_WAL = ( - os.environ.get("DATABASE_ENABLE_SQLITE_WAL", "False").lower() == "true" -) +DATABASE_ENABLE_SQLITE_WAL = (os.environ.get("DATABASE_ENABLE_SQLITE_WAL", "False").lower() == "true") DATABASE_DEDUPLICATE_INTERVAL = ( os.environ.get("DATABASE_DEDUPLICATE_INTERVAL", 0.) diff --git a/backend/open_webui/internal/db.py b/backend/open_webui/internal/db.py index ccc3995ded..b6913d87b0 100644 --- a/backend/open_webui/internal/db.py +++ b/backend/open_webui/internal/db.py @@ -115,6 +115,7 @@ elif "sqlite" in SQLALCHEMY_DATABASE_URL: engine = create_engine( SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False} ) + def on_connect(dbapi_connection, connection_record): cursor = dbapi_connection.cursor() if DATABASE_ENABLE_SQLITE_WAL: @@ -122,6 +123,7 @@ elif "sqlite" in SQLALCHEMY_DATABASE_URL: else: cursor.execute("PRAGMA journal_mode=DELETE") cursor.close() + event.listen(engine, "connect", on_connect) else: if isinstance(DATABASE_POOL_SIZE, int): diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index 8c4ae9d9a2..e7a007df38 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -500,6 +500,7 @@ def deduplicate(interval: float = 10.0): :param interval: Duration in seconds to wait before allowing the function to be called again. """ + def decorator(func): last_calls = {} lock = threading.Lock() @@ -514,6 +515,7 @@ def deduplicate(interval: float = 10.0): return None last_calls[key] = now return func(*args, **kwargs) + return wrapper return decorator