From 789e6a0db3b5e2f05ed8281b5649756053998c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=88=EC=B0=AC=EC=9A=B0=5BAI=20Product=5D?= Date: Thu, 26 Jun 2025 22:24:57 +0900 Subject: [PATCH] feat : Retry acquiring usage cleanup lock to handle potential stale locks --- backend/open_webui/socket/main.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/backend/open_webui/socket/main.py b/backend/open_webui/socket/main.py index 35e40dccb2..96bcbcf1b5 100644 --- a/backend/open_webui/socket/main.py +++ b/backend/open_webui/socket/main.py @@ -1,4 +1,6 @@ import asyncio +import random + import socketio import logging import sys @@ -105,10 +107,26 @@ else: async def periodic_usage_pool_cleanup(): - if not aquire_func(): - log.debug("Usage pool cleanup lock already exists. Not running it.") - return - log.debug("Running periodic_usage_pool_cleanup") + max_retries = 2 + retry_delay = random.uniform( + WEBSOCKET_REDIS_LOCK_TIMEOUT / 2, WEBSOCKET_REDIS_LOCK_TIMEOUT + ) + for attempt in range(max_retries + 1): + if aquire_func(): + break + else: + if attempt < max_retries: + log.debug( + f"Cleanup lock already exists. Retry {attempt + 1} after {retry_delay}s..." + ) + await asyncio.sleep(retry_delay) + else: + log.warning( + "Failed to acquire cleanup lock after retries. Skipping cleanup." + ) + return + + log.debug("Running periodic_cleanup") try: while True: if not renew_func():