ti.sil/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2025-12-12 04:15:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

feat: WEB_LOADER_TIMEOUT

Browse source
This commit is contained in:
Timothy Jaeryang Baek 2025-12-08 11:49:27 -05:00
parent bcd50ed8f1
commit b02397e460
5 changed files with 42 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
backend/open_webui/config.py
View file

@ -635,6 +635,7 @@ OAUTH_AUDIENCE = PersistentConfig(
os.environ.get("OAUTH_AUDIENCE", ""),
)
def load_oauth_providers():
OAUTH_PROVIDERS.clear()
if GOOGLE_CLIENT_ID.value and GOOGLE_CLIENT_SECRET.value:
@ -2999,6 +3000,12 @@ WEB_LOADER_CONCURRENT_REQUESTS = PersistentConfig(
int(os.getenv("WEB_LOADER_CONCURRENT_REQUESTS", "10")),
)
WEB_LOADER_TIMEOUT = PersistentConfig(
"WEB_LOADER_TIMEOUT",
"rag.web.loader.timeout",
os.getenv("WEB_LOADER_TIMEOUT", ""),
)
ENABLE_WEB_LOADER_SSL_VERIFICATION = PersistentConfig(
"ENABLE_WEB_LOADER_SSL_VERIFICATION",

2
backend/open_webui/main.py
View file

@ -208,6 +208,7 @@ from open_webui.config import (
FIRECRAWL_API_KEY,
WEB_LOADER_ENGINE,
WEB_LOADER_CONCURRENT_REQUESTS,
WEB_LOADER_TIMEOUT,
WHISPER_MODEL,
WHISPER_VAD_FILTER,
WHISPER_LANGUAGE,
@ -922,6 +923,7 @@ app.state.config.WEB_SEARCH_CONCURRENT_REQUESTS = WEB_SEARCH_CONCURRENT_REQUESTS
app.state.config.WEB_LOADER_ENGINE = WEB_LOADER_ENGINE
app.state.config.WEB_LOADER_CONCURRENT_REQUESTS = WEB_LOADER_CONCURRENT_REQUESTS
app.state.config.WEB_LOADER_TIMEOUT = WEB_LOADER_TIMEOUT
app.state.config.WEB_SEARCH_TRUST_ENV = WEB_SEARCH_TRUST_ENV
app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL = (

15
backend/open_webui/retrieval/web/utils.py
View file

@ -33,6 +33,7 @@ from open_webui.config import (
PLAYWRIGHT_WS_URL,
PLAYWRIGHT_TIMEOUT,
WEB_LOADER_ENGINE,
WEB_LOADER_TIMEOUT,
FIRECRAWL_API_BASE_URL,
FIRECRAWL_API_KEY,
TAVILY_API_KEY,
@ -674,6 +675,20 @@ def get_web_loader(
if WEB_LOADER_ENGINE.value == "" or WEB_LOADER_ENGINE.value == "safe_web":
WebLoaderClass = SafeWebBaseLoader
request_kwargs = {}
if WEB_LOADER_TIMEOUT.value:
try:
timeout_value = float(WEB_LOADER_TIMEOUT.value)
except ValueError:
timeout_value = None
if timeout_value:
request_kwargs["timeout"] = timeout_value
if request_kwargs:
web_loader_args["requests_kwargs"] = request_kwargs
if WEB_LOADER_ENGINE.value == "playwright":
WebLoaderClass = SafePlaywrightURLLoader
web_loader_args["playwright_timeout"] = PLAYWRIGHT_TIMEOUT.value

5
backend/open_webui/routers/retrieval.py
View file

@ -536,6 +536,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
"SOUGOU_API_SID": request.app.state.config.SOUGOU_API_SID,
"SOUGOU_API_SK": request.app.state.config.SOUGOU_API_SK,
"WEB_LOADER_ENGINE": request.app.state.config.WEB_LOADER_ENGINE,
"WEB_LOADER_TIMEOUT": request.app.state.config.WEB_LOADER_TIMEOUT,
"ENABLE_WEB_LOADER_SSL_VERIFICATION": request.app.state.config.ENABLE_WEB_LOADER_SSL_VERIFICATION,
"PLAYWRIGHT_WS_URL": request.app.state.config.PLAYWRIGHT_WS_URL,
"PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT,
@ -594,6 +595,7 @@ class WebConfig(BaseModel):
SOUGOU_API_SID: Optional[str] = None
SOUGOU_API_SK: Optional[str] = None
WEB_LOADER_ENGINE: Optional[str] = None
WEB_LOADER_TIMEOUT: Optional[str] = None
ENABLE_WEB_LOADER_SSL_VERIFICATION: Optional[bool] = None
PLAYWRIGHT_WS_URL: Optional[str] = None
PLAYWRIGHT_TIMEOUT: Optional[int] = None
@ -1071,6 +1073,8 @@ async def update_rag_config(
# Web loader settings
request.app.state.config.WEB_LOADER_ENGINE = form_data.web.WEB_LOADER_ENGINE
request.app.state.config.WEB_LOADER_TIMEOUT = form_data.web.WEB_LOADER_TIMEOUT
request.app.state.config.ENABLE_WEB_LOADER_SSL_VERIFICATION = (
form_data.web.ENABLE_WEB_LOADER_SSL_VERIFICATION
)
@ -1206,6 +1210,7 @@ async def update_rag_config(
"SOUGOU_API_SID": request.app.state.config.SOUGOU_API_SID,
"SOUGOU_API_SK": request.app.state.config.SOUGOU_API_SK,
"WEB_LOADER_ENGINE": request.app.state.config.WEB_LOADER_ENGINE,
"WEB_LOADER_TIMEOUT": request.app.state.config.WEB_LOADER_TIMEOUT,
"ENABLE_WEB_LOADER_SSL_VERIFICATION": request.app.state.config.ENABLE_WEB_LOADER_SSL_VERIFICATION,
"PLAYWRIGHT_WS_URL": request.app.state.config.PLAYWRIGHT_WS_URL,
"PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT,

13
src/lib/components/admin/Settings/WebSearch.svelte
View file

@ -767,6 +767,19 @@
</div>
{#if webConfig.WEB_LOADER_ENGINE === '' || webConfig.WEB_LOADER_ENGINE === 'safe_web'}
<div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
{$i18n.t('Timeout')}
</div>
<div class="flex items-center relative">
<input
class="flex-1 w-full text-sm bg-transparent outline-hidden"
placeholder={$i18n.t('Timeout')}
bind:value={webConfig.WEB_LOADER_TIMEOUT}
/>
</div>
</div>
<div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
{$i18n.t('Verify SSL Certificate')}