Merge pull request #15328 from acwoo97/feat/lock

feat : Retry acquiring usage cleanup lock to handle potential stale locks
This commit is contained in:
Tim Jaeryang Baek 2025-06-27 11:52:13 +04:00 committed by GitHub
commit 767c4426cd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -1,4 +1,6 @@
import asyncio import asyncio
import random
import socketio import socketio
import logging import logging
import sys import sys
@ -105,10 +107,26 @@ else:
async def periodic_usage_pool_cleanup(): async def periodic_usage_pool_cleanup():
if not aquire_func(): max_retries = 2
log.debug("Usage pool cleanup lock already exists. Not running it.") retry_delay = random.uniform(
WEBSOCKET_REDIS_LOCK_TIMEOUT / 2, WEBSOCKET_REDIS_LOCK_TIMEOUT
)
for attempt in range(max_retries + 1):
if aquire_func():
break
else:
if attempt < max_retries:
log.debug(
f"Cleanup lock already exists. Retry {attempt + 1} after {retry_delay}s..."
)
await asyncio.sleep(retry_delay)
else:
log.warning(
"Failed to acquire cleanup lock after retries. Skipping cleanup."
)
return return
log.debug("Running periodic_usage_pool_cleanup")
log.debug("Running periodic_cleanup")
try: try:
while True: while True:
if not renew_func(): if not renew_func():