From dfbc12594702bf601c4c391cfe080f8e0844d01e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BA=D0=BB=D0=B5=D0=BC=D0=B8=D1=88=D0=B5?= =?UTF-8?q?=D0=B2=20=D0=9F=D0=B5=D1=82=D1=80=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B5=D0=B5=D0=B2=D0=B8=D1=87?= Date: Thu, 30 May 2024 18:55:58 +0700 Subject: [PATCH 001/155] Reconnect to postgresql & mysql external databases when getting disconnected --- backend/apps/webui/internal/db.py | 9 ++++ backend/apps/webui/internal/wrappers.py | 59 +++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 backend/apps/webui/internal/wrappers.py diff --git a/backend/apps/webui/internal/db.py b/backend/apps/webui/internal/db.py index 0e7b1f95d1..dda58a4e1c 100644 --- a/backend/apps/webui/internal/db.py +++ b/backend/apps/webui/internal/db.py @@ -7,6 +7,12 @@ from config import SRC_LOG_LEVELS, DATA_DIR, DATABASE_URL, BACKEND_DIR import os import logging +from peewee_migrate import Router +from playhouse.db_url import connect + +from apps.webui.internal.wrappers import PeeweeConnectionState, register_peewee_databases +from config import SRC_LOG_LEVELS, DATA_DIR, DATABASE_URL + log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["DB"]) @@ -20,6 +26,8 @@ class JSONField(TextField): return json.loads(value) +register_peewee_databases() + # Check if the file exists if os.path.exists(f"{DATA_DIR}/ollama.db"): # Rename the file @@ -29,6 +37,7 @@ else: pass DB = connect(DATABASE_URL) +DB._state = PeeweeConnectionState() log.info(f"Connected to a {DB.__class__.__name__} database.") router = Router( DB, diff --git a/backend/apps/webui/internal/wrappers.py b/backend/apps/webui/internal/wrappers.py new file mode 100644 index 0000000000..406599b5aa --- /dev/null +++ b/backend/apps/webui/internal/wrappers.py @@ -0,0 +1,59 @@ +from contextvars import ContextVar + +from peewee import PostgresqlDatabase, InterfaceError as PeeWeeInterfaceError, MySQLDatabase, _ConnectionState +from playhouse.db_url import register_database +from playhouse.pool import PooledPostgresqlDatabase, PooledMySQLDatabase +from playhouse.shortcuts import ReconnectMixin +from psycopg2 import OperationalError +from psycopg2.errors import InterfaceError + + +db_state_default = {"closed": None, "conn": None, "ctx": None, "transactions": None} +db_state = ContextVar("db_state", default=db_state_default.copy()) + + +class PeeweeConnectionState(_ConnectionState): + def __init__(self, **kwargs): + super().__setattr__("_state", db_state) + super().__init__(**kwargs) + + def __setattr__(self, name, value): + self._state.get()[name] = value + + def __getattr__(self, name): + return self._state.get()[name] + + +class CustomReconnectMixin(ReconnectMixin): + reconnect_errors = ( + # default ReconnectMixin exceptions (MySQL specific) + *ReconnectMixin.reconnect_errors, + # psycopg2 + (OperationalError, 'termin'), + (InterfaceError, 'closed'), + # peewee + (PeeWeeInterfaceError, 'closed'), + ) + + +class ReconnectingPostgresqlDatabase(CustomReconnectMixin, PostgresqlDatabase): + pass + + +class ReconnectingPooledPostgresqlDatabase(CustomReconnectMixin, PooledPostgresqlDatabase): + pass + + +class ReconnectingMySQLDatabase(CustomReconnectMixin, MySQLDatabase): + pass + + +class ReconnectingPooledMySQLDatabase(CustomReconnectMixin, PooledMySQLDatabase): + pass + + +def register_peewee_databases(): + register_database(MySQLDatabase, 'mysql') + register_database(PooledMySQLDatabase, 'mysql+pool') + register_database(ReconnectingPostgresqlDatabase, 'postgres', 'postgresql') + register_database(ReconnectingPooledPostgresqlDatabase, 'postgres+pool', 'postgresql+pool') From e1fa453edae689c5eea6af0b23e3dc1e158b302f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BA=D0=BB=D0=B5=D0=BC=D0=B8=D1=88=D0=B5?= =?UTF-8?q?=D0=B2=20=D0=9F=D0=B5=D1=82=D1=80=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B5=D0=B5=D0=B2=D0=B8=D1=87?= Date: Thu, 30 May 2024 20:20:09 +0700 Subject: [PATCH 002/155] Test reconnection to postgres in gh actions --- .github/workflows/integration-test.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 2426aff27a..e64f93bc1b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -170,6 +170,26 @@ jobs: echo "Server has stopped" exit 1 fi + + # Check that service will reconnect to postgres when connection will be closed + status_code=$(curl --write-out %{http_code} -s --output /dev/null http://localhost:8081/api/tags) + if [[ "$status_code" -ne 200 ]] ; then + echo "Server has failed before postgres reconnect check" + exit 1 + fi + + echo "Terminating all connections to postgres..." + python -c "import os, psycopg2 as pg2; \ + conn = pg2.connect(dsn=os.environ['DATABASE_URL'].replace('+pool', '')); \ + cur = conn.cursor(); \ + cur.execute('SELECT pg_terminate_backend(psa.pid) FROM pg_stat_activity psa WHERE datname = current_database() AND pid <> pg_backend_pid();')" + + status_code=$(curl --write-out %{http_code} -s --output /dev/null http://localhost:8081/api/tags) + if [[ "$status_code" -ne 200 ]] ; then + echo "Server has not reconnected to postgres after connection was closed: returned status $status_code" + exit 1 + fi + # - name: Test backend with MySQL # if: success() || steps.sqlite.conclusion == 'failure' || steps.postgres.conclusion == 'failure' From e59e1f5049c1b3d9b5ff7f61d7f6051cd2950b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BA=D0=BB=D0=B5=D0=BC=D0=B8=D1=88=D0=B5?= =?UTF-8?q?=D0=B2=20=D0=9F=D0=B5=D1=82=D1=80=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B5=D0=B5=D0=B2=D0=B8=D1=87?= Date: Thu, 30 May 2024 20:44:13 +0700 Subject: [PATCH 003/155] Fix rebase artifacts --- backend/apps/webui/internal/db.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/backend/apps/webui/internal/db.py b/backend/apps/webui/internal/db.py index dda58a4e1c..7420bd019d 100644 --- a/backend/apps/webui/internal/db.py +++ b/backend/apps/webui/internal/db.py @@ -1,17 +1,13 @@ +import os +import logging import json from peewee import * -from peewee_migrate import Router -from playhouse.db_url import connect -from config import SRC_LOG_LEVELS, DATA_DIR, DATABASE_URL, BACKEND_DIR -import os -import logging - from peewee_migrate import Router from playhouse.db_url import connect from apps.webui.internal.wrappers import PeeweeConnectionState, register_peewee_databases -from config import SRC_LOG_LEVELS, DATA_DIR, DATABASE_URL +from config import SRC_LOG_LEVELS, DATA_DIR, DATABASE_URL, BACKEND_DIR log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["DB"]) From ad32a2ef3cc508d6540b16a762dec31b773818c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BA=D0=BB=D0=B5=D0=BC=D0=B8=D1=88=D0=B5?= =?UTF-8?q?=D0=B2=20=D0=9F=D0=B5=D1=82=D1=80=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B5=D0=B5=D0=B2=D0=B8=D1=87?= Date: Fri, 31 May 2024 13:26:23 +0700 Subject: [PATCH 004/155] Drop mysql restarts --- backend/apps/webui/internal/wrappers.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/backend/apps/webui/internal/wrappers.py b/backend/apps/webui/internal/wrappers.py index 406599b5aa..53869b9160 100644 --- a/backend/apps/webui/internal/wrappers.py +++ b/backend/apps/webui/internal/wrappers.py @@ -1,8 +1,8 @@ from contextvars import ContextVar -from peewee import PostgresqlDatabase, InterfaceError as PeeWeeInterfaceError, MySQLDatabase, _ConnectionState +from peewee import PostgresqlDatabase, InterfaceError as PeeWeeInterfaceError, _ConnectionState from playhouse.db_url import register_database -from playhouse.pool import PooledPostgresqlDatabase, PooledMySQLDatabase +from playhouse.pool import PooledPostgresqlDatabase from playhouse.shortcuts import ReconnectMixin from psycopg2 import OperationalError from psycopg2.errors import InterfaceError @@ -26,7 +26,7 @@ class PeeweeConnectionState(_ConnectionState): class CustomReconnectMixin(ReconnectMixin): reconnect_errors = ( - # default ReconnectMixin exceptions (MySQL specific) + # default ReconnectMixin exceptions *ReconnectMixin.reconnect_errors, # psycopg2 (OperationalError, 'termin'), @@ -44,16 +44,6 @@ class ReconnectingPooledPostgresqlDatabase(CustomReconnectMixin, PooledPostgresq pass -class ReconnectingMySQLDatabase(CustomReconnectMixin, MySQLDatabase): - pass - - -class ReconnectingPooledMySQLDatabase(CustomReconnectMixin, PooledMySQLDatabase): - pass - - def register_peewee_databases(): - register_database(MySQLDatabase, 'mysql') - register_database(PooledMySQLDatabase, 'mysql+pool') register_database(ReconnectingPostgresqlDatabase, 'postgres', 'postgresql') register_database(ReconnectingPooledPostgresqlDatabase, 'postgres+pool', 'postgresql+pool') From 7b5f434a079b335be8452af7a05982a779973fcd Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Thu, 13 Jun 2024 07:14:48 +0700 Subject: [PATCH 005/155] Implement domain whitelisting for web search results --- backend/apps/rag/main.py | 10 +++++++++- backend/apps/rag/search/brave.py | 9 +++++---- backend/apps/rag/search/duckduckgo.py | 11 ++++++----- backend/apps/rag/search/google_pse.py | 9 +++++---- backend/apps/rag/search/main.py | 12 +++++++++++- backend/apps/rag/search/searxng.py | 5 +++-- backend/apps/rag/search/serper.py | 9 +++++---- backend/apps/rag/search/serply.py | 9 +++++---- backend/apps/rag/search/serpstack.py | 9 +++++---- backend/config.py | 9 +++++++++ 10 files changed, 63 insertions(+), 29 deletions(-) diff --git a/backend/apps/rag/main.py b/backend/apps/rag/main.py index 0e493eaaa3..37da4db5a1 100644 --- a/backend/apps/rag/main.py +++ b/backend/apps/rag/main.py @@ -111,6 +111,7 @@ from config import ( YOUTUBE_LOADER_LANGUAGE, ENABLE_RAG_WEB_SEARCH, RAG_WEB_SEARCH_ENGINE, + RAG_WEB_SEARCH_WHITE_LIST_DOMAINS, SEARXNG_QUERY_URL, GOOGLE_PSE_API_KEY, GOOGLE_PSE_ENGINE_ID, @@ -163,6 +164,7 @@ app.state.YOUTUBE_LOADER_TRANSLATION = None app.state.config.ENABLE_RAG_WEB_SEARCH = ENABLE_RAG_WEB_SEARCH app.state.config.RAG_WEB_SEARCH_ENGINE = RAG_WEB_SEARCH_ENGINE +app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS = RAG_WEB_SEARCH_WHITE_LIST_DOMAINS app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY @@ -768,6 +770,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.SEARXNG_QUERY_URL, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS ) else: raise Exception("No SEARXNG_QUERY_URL found in environment variables") @@ -781,6 +784,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.GOOGLE_PSE_ENGINE_ID, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS ) else: raise Exception( @@ -792,6 +796,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.BRAVE_SEARCH_API_KEY, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS ) else: raise Exception("No BRAVE_SEARCH_API_KEY found in environment variables") @@ -801,6 +806,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.SERPSTACK_API_KEY, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS, https_enabled=app.state.config.SERPSTACK_HTTPS, ) else: @@ -811,6 +817,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.SERPER_API_KEY, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS ) else: raise Exception("No SERPER_API_KEY found in environment variables") @@ -820,11 +827,12 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.SERPLY_API_KEY, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS ) else: raise Exception("No SERPLY_API_KEY found in environment variables") elif engine == "duckduckgo": - return search_duckduckgo(query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT) + return search_duckduckgo(query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS) else: raise Exception("No search engine API key found in environment variables") diff --git a/backend/apps/rag/search/brave.py b/backend/apps/rag/search/brave.py index 4e0f56807f..04cd184964 100644 --- a/backend/apps/rag/search/brave.py +++ b/backend/apps/rag/search/brave.py @@ -1,15 +1,15 @@ import logging - +from typing import List import requests -from apps.rag.search.main import SearchResult +from apps.rag.search.main import SearchResult, filter_by_whitelist from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["RAG"]) -def search_brave(api_key: str, query: str, count: int) -> list[SearchResult]: +def search_brave(api_key: str, query: str, whitelist:List[str], count: int) -> list[SearchResult]: """Search using Brave's Search API and return the results as a list of SearchResult objects. Args: @@ -29,9 +29,10 @@ def search_brave(api_key: str, query: str, count: int) -> list[SearchResult]: json_response = response.json() results = json_response.get("web", {}).get("results", []) + filtered_results = filter_by_whitelist(results, whitelist) return [ SearchResult( link=result["url"], title=result.get("title"), snippet=result.get("snippet") ) - for result in results[:count] + for result in filtered_results[:count] ] diff --git a/backend/apps/rag/search/duckduckgo.py b/backend/apps/rag/search/duckduckgo.py index 188ae2bea4..9342e53e43 100644 --- a/backend/apps/rag/search/duckduckgo.py +++ b/backend/apps/rag/search/duckduckgo.py @@ -1,6 +1,6 @@ import logging - -from apps.rag.search.main import SearchResult +from typing import List +from apps.rag.search.main import SearchResult, filter_by_whitelist from duckduckgo_search import DDGS from config import SRC_LOG_LEVELS @@ -8,7 +8,7 @@ log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["RAG"]) -def search_duckduckgo(query: str, count: int) -> list[SearchResult]: +def search_duckduckgo(query: str, count: int, whitelist:List[str]) -> list[SearchResult]: """ Search using DuckDuckGo's Search API and return the results as a list of SearchResult objects. Args: @@ -41,6 +41,7 @@ def search_duckduckgo(query: str, count: int) -> list[SearchResult]: snippet=result.get("body"), ) ) - print(results) + # print(results) + filtered_results = filter_by_whitelist(results, whitelist) # Return the list of search results - return results + return filtered_results diff --git a/backend/apps/rag/search/google_pse.py b/backend/apps/rag/search/google_pse.py index 7ff54c7850..bc89b2f3a6 100644 --- a/backend/apps/rag/search/google_pse.py +++ b/backend/apps/rag/search/google_pse.py @@ -1,9 +1,9 @@ import json import logging - +from typing import List import requests -from apps.rag.search.main import SearchResult +from apps.rag.search.main import SearchResult, filter_by_whitelist from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) @@ -11,7 +11,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"]) def search_google_pse( - api_key: str, search_engine_id: str, query: str, count: int + api_key: str, search_engine_id: str, query: str, count: int, whitelist:List[str] ) -> list[SearchResult]: """Search using Google's Programmable Search Engine API and return the results as a list of SearchResult objects. @@ -35,11 +35,12 @@ def search_google_pse( json_response = response.json() results = json_response.get("items", []) + filtered_results = filter_by_whitelist(results, whitelist) return [ SearchResult( link=result["link"], title=result.get("title"), snippet=result.get("snippet"), ) - for result in results + for result in filtered_results ] diff --git a/backend/apps/rag/search/main.py b/backend/apps/rag/search/main.py index b5478f9496..612177581a 100644 --- a/backend/apps/rag/search/main.py +++ b/backend/apps/rag/search/main.py @@ -1,8 +1,18 @@ from typing import Optional - +from urllib.parse import urlparse from pydantic import BaseModel +def filter_by_whitelist(results, whitelist): + if not whitelist: + return results + filtered_results = [] + for result in results: + domain = urlparse(result["url"]).netloc + if any(domain.endswith(whitelisted_domain) for whitelisted_domain in whitelist): + filtered_results.append(result) + return filtered_results + class SearchResult(BaseModel): link: str title: Optional[str] diff --git a/backend/apps/rag/search/searxng.py b/backend/apps/rag/search/searxng.py index c8ad888133..954aaf0724 100644 --- a/backend/apps/rag/search/searxng.py +++ b/backend/apps/rag/search/searxng.py @@ -11,7 +11,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"]) def search_searxng( - query_url: str, query: str, count: int, **kwargs + query_url: str, query: str, count: int, whitelist:List[str], **kwargs ) -> List[SearchResult]: """ Search a SearXNG instance for a given query and return the results as a list of SearchResult objects. @@ -78,9 +78,10 @@ def search_searxng( json_response = response.json() results = json_response.get("results", []) sorted_results = sorted(results, key=lambda x: x.get("score", 0), reverse=True) + filtered_results = filter_by_whitelist(sorted_results, whitelist) return [ SearchResult( link=result["url"], title=result.get("title"), snippet=result.get("content") ) - for result in sorted_results[:count] + for result in filtered_results[:count] ] diff --git a/backend/apps/rag/search/serper.py b/backend/apps/rag/search/serper.py index 150da6e074..e12126a358 100644 --- a/backend/apps/rag/search/serper.py +++ b/backend/apps/rag/search/serper.py @@ -1,16 +1,16 @@ import json import logging - +from typing import List import requests -from apps.rag.search.main import SearchResult +from apps.rag.search.main import SearchResult, filter_by_whitelist from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["RAG"]) -def search_serper(api_key: str, query: str, count: int) -> list[SearchResult]: +def search_serper(api_key: str, query: str, count: int, whitelist:List[str]) -> list[SearchResult]: """Search using serper.dev's API and return the results as a list of SearchResult objects. Args: @@ -29,11 +29,12 @@ def search_serper(api_key: str, query: str, count: int) -> list[SearchResult]: results = sorted( json_response.get("organic", []), key=lambda x: x.get("position", 0) ) + filtered_results = filter_by_whitelist(results, whitelist) return [ SearchResult( link=result["link"], title=result.get("title"), snippet=result.get("description"), ) - for result in results[:count] + for result in filtered_results[:count] ] diff --git a/backend/apps/rag/search/serply.py b/backend/apps/rag/search/serply.py index fccf70ecd8..e4040a8485 100644 --- a/backend/apps/rag/search/serply.py +++ b/backend/apps/rag/search/serply.py @@ -1,10 +1,10 @@ import json import logging - +from typing import List import requests from urllib.parse import urlencode -from apps.rag.search.main import SearchResult +from apps.rag.search.main import SearchResult, filter_by_whitelist from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) @@ -15,6 +15,7 @@ def search_serply( api_key: str, query: str, count: int, + whitelist:List[str], hl: str = "us", limit: int = 10, device_type: str = "desktop", @@ -57,12 +58,12 @@ def search_serply( results = sorted( json_response.get("results", []), key=lambda x: x.get("realPosition", 0) ) - + filtered_results = filter_by_whitelist(results, whitelist) return [ SearchResult( link=result["link"], title=result.get("title"), snippet=result.get("description"), ) - for result in results[:count] + for result in filtered_results[:count] ] diff --git a/backend/apps/rag/search/serpstack.py b/backend/apps/rag/search/serpstack.py index 0d247d1ab0..344e250733 100644 --- a/backend/apps/rag/search/serpstack.py +++ b/backend/apps/rag/search/serpstack.py @@ -1,9 +1,9 @@ import json import logging - +from typing import List import requests -from apps.rag.search.main import SearchResult +from apps.rag.search.main import SearchResult, filter_by_whitelist from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) @@ -11,7 +11,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"]) def search_serpstack( - api_key: str, query: str, count: int, https_enabled: bool = True + api_key: str, query: str, count: int, whitelist:List[str], https_enabled: bool = True ) -> list[SearchResult]: """Search using serpstack.com's and return the results as a list of SearchResult objects. @@ -35,9 +35,10 @@ def search_serpstack( results = sorted( json_response.get("organic_results", []), key=lambda x: x.get("position", 0) ) + filtered_results = filter_by_whitelist(results, whitelist) return [ SearchResult( link=result["url"], title=result.get("title"), snippet=result.get("snippet") ) - for result in results[:count] + for result in filtered_results[:count] ] diff --git a/backend/config.py b/backend/config.py index 30a23f29ee..6d145465ae 100644 --- a/backend/config.py +++ b/backend/config.py @@ -894,6 +894,15 @@ RAG_WEB_SEARCH_ENGINE = PersistentConfig( os.getenv("RAG_WEB_SEARCH_ENGINE", ""), ) +RAG_WEB_SEARCH_WHITE_LIST_DOMAINS = PersistentConfig( + "RAG_WEB_SEARCH_WHITE_LIST_DOMAINS", + "rag.rag_web_search_white_list_domains", + [ + # "example.com", + # "anotherdomain.com", + ], +) + SEARXNG_QUERY_URL = PersistentConfig( "SEARXNG_QUERY_URL", "rag.web.search.searxng_query_url", From 5844d0525ae48fc46e9af6daec908193634fbddb Mon Sep 17 00:00:00 2001 From: rdavis Date: Thu, 13 Jun 2024 19:01:13 +0000 Subject: [PATCH 006/155] Added the ability to sort users in the admin panel --- src/routes/(app)/admin/+page.svelte | 61 ++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/src/routes/(app)/admin/+page.svelte b/src/routes/(app)/admin/+page.svelte index 00a2c5fb56..858f5f82d6 100644 --- a/src/routes/(app)/admin/+page.svelte +++ b/src/routes/(app)/admin/+page.svelte @@ -75,6 +75,17 @@ } loaded = true; }); + let sortKey = 'created_at'; // default sort key + let sortOrder = 'asc'; // default sort order + + function setSortKey(key) { + if (sortKey === key) { + sortOrder = sortOrder === 'asc' ? 'desc' : 'asc'; + } else { + sortKey = key; + sortOrder = 'asc'; + } + } {#key selectedUser} @@ -139,12 +150,46 @@ - - - - - - + + + + + @@ -159,6 +204,10 @@ const query = search.toLowerCase(); return name.includes(query); } + }).sort((a, b) => { + if (a[sortKey] < b[sortKey]) return sortOrder === 'asc' ? -1 : 1; + if (a[sortKey] > b[sortKey]) return sortOrder === 'asc' ? 1 : -1; + return 0; }) .slice((page - 1) * 20, page * 20) as user} From c07e7221e58582c2dfadddc72b14d33cd00099e2 Mon Sep 17 00:00:00 2001 From: rdavis Date: Thu, 13 Jun 2024 19:01:13 +0000 Subject: [PATCH 007/155] Added the ability to sort users in the admin panel --- src/routes/(app)/admin/+page.svelte | 61 ++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/src/routes/(app)/admin/+page.svelte b/src/routes/(app)/admin/+page.svelte index 00a2c5fb56..858f5f82d6 100644 --- a/src/routes/(app)/admin/+page.svelte +++ b/src/routes/(app)/admin/+page.svelte @@ -75,6 +75,17 @@ } loaded = true; }); + let sortKey = 'created_at'; // default sort key + let sortOrder = 'asc'; // default sort order + + function setSortKey(key) { + if (sortKey === key) { + sortOrder = sortOrder === 'asc' ? 'desc' : 'asc'; + } else { + sortKey = key; + sortOrder = 'asc'; + } + } {#key selectedUser} @@ -139,12 +150,46 @@
{$i18n.t('Role')} {$i18n.t('Name')} {$i18n.t('Email')} {$i18n.t('Last Active')} {$i18n.t('Created at')} setSortKey('role')}> + {$i18n.t('Role')} + {#if sortKey === 'role'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} + setSortKey('name')}> + {$i18n.t('Name')} + {#if sortKey === 'name'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} + setSortKey('email')}> + {$i18n.t('Email')} + {#if sortKey === 'email'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} + setSortKey('last_active_at')}> + {$i18n.t('Last Active')} + {#if sortKey === 'last_active_at'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} + setSortKey('created_at')}> + {$i18n.t('Created at')} + {#if sortKey === 'created_at'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} +
- - - - - - + + + + + @@ -159,6 +204,10 @@ const query = search.toLowerCase(); return name.includes(query); } + }).sort((a, b) => { + if (a[sortKey] < b[sortKey]) return sortOrder === 'asc' ? -1 : 1; + if (a[sortKey] > b[sortKey]) return sortOrder === 'asc' ? 1 : -1; + return 0; }) .slice((page - 1) * 20, page * 20) as user} From 91d53530e6e9dd3e5ed410cbd4839a38de04361d Mon Sep 17 00:00:00 2001 From: rdavis Date: Thu, 13 Jun 2024 23:24:52 +0000 Subject: [PATCH 008/155] Added the ability to sort chats in the admin panel chats modal Added "Updated at" column to the admin panel chats modal. --- .../components/admin/UserChatsModal.svelte | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/src/lib/components/admin/UserChatsModal.svelte b/src/lib/components/admin/UserChatsModal.svelte index 67fa367cd1..a36b0af695 100644 --- a/src/lib/components/admin/UserChatsModal.svelte +++ b/src/lib/components/admin/UserChatsModal.svelte @@ -31,6 +31,20 @@ } })(); } + + let sortKey = 'updated_at'; // default sort key + let sortOrder = 'desc'; // default sort order + function setSortKey(key) { + if (sortKey === key) { + sortOrder = sortOrder === 'asc' ? 'desc' : 'asc'; + } else { + sortKey = key; + sortOrder = 'asc'; + } + } + $: { + console.log(chats); + } @@ -69,18 +83,45 @@ class="text-xs text-gray-700 uppercase bg-transparent dark:text-gray-200 border-b-2 dark:border-gray-800" > - - + + + - {#each chats as chat, idx} + {#each chats + .sort((a, b) => { + if (a[sortKey] < b[sortKey]) return sortOrder === 'asc' ? -1 : 1; + if (a[sortKey] > b[sortKey]) return sortOrder === 'asc' ? 1 : -1; + return 0; + }) as chat, idx} - - + From dbfb6d5993736721ae0557bab9e5a148cc4b4a13 Mon Sep 17 00:00:00 2001 From: John Karabudak Date: Sat, 15 Jun 2024 22:18:18 -0230 Subject: [PATCH 012/155] fixed GitHub actions on usernames with uppercase characters --- .github/workflows/docker-build.yaml | 56 +++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index da40f56ffe..60575cebe4 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -11,8 +11,6 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - FULL_IMAGE_NAME: ghcr.io/${{ github.repository }} jobs: build-main-image: @@ -28,6 +26,15 @@ jobs: - linux/arm64 steps: + # GitHub Packages requires the entire repository name to be in lowercase + # although the repository owner has a lowercase username, this prevents some people from running actions after forking + - name: Set repository and image name to lowercase + run: | + echo "IMAGE_NAME=${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + echo "FULL_IMAGE_NAME=ghcr.io/${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + env: + IMAGE_NAME_MIXED_CASE: '${{ github.repository }}' + - name: Prepare run: | platform=${{ matrix.platform }} @@ -116,6 +123,15 @@ jobs: - linux/arm64 steps: + # GitHub Packages requires the entire repository name to be in lowercase + # although the repository owner has a lowercase username, this prevents some people from running actions after forking + - name: Set repository and image name to lowercase + run: | + echo "IMAGE_NAME=${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + echo "FULL_IMAGE_NAME=ghcr.io/${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + env: + IMAGE_NAME_MIXED_CASE: '${{ github.repository }}' + - name: Prepare run: | platform=${{ matrix.platform }} @@ -207,6 +223,15 @@ jobs: - linux/arm64 steps: + # GitHub Packages requires the entire repository name to be in lowercase + # although the repository owner has a lowercase username, this prevents some people from running actions after forking + - name: Set repository and image name to lowercase + run: | + echo "IMAGE_NAME=${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + echo "FULL_IMAGE_NAME=ghcr.io/${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + env: + IMAGE_NAME_MIXED_CASE: '${{ github.repository }}' + - name: Prepare run: | platform=${{ matrix.platform }} @@ -289,6 +314,15 @@ jobs: runs-on: ubuntu-latest needs: [ build-main-image ] steps: + # GitHub Packages requires the entire repository name to be in lowercase + # although the repository owner has a lowercase username, this prevents some people from running actions after forking + - name: Set repository and image name to lowercase + run: | + echo "IMAGE_NAME=${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + echo "FULL_IMAGE_NAME=ghcr.io/${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + env: + IMAGE_NAME_MIXED_CASE: '${{ github.repository }}' + - name: Download digests uses: actions/download-artifact@v4 with: @@ -335,6 +369,15 @@ jobs: runs-on: ubuntu-latest needs: [ build-cuda-image ] steps: + # GitHub Packages requires the entire repository name to be in lowercase + # although the repository owner has a lowercase username, this prevents some people from running actions after forking + - name: Set repository and image name to lowercase + run: | + echo "IMAGE_NAME=${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + echo "FULL_IMAGE_NAME=ghcr.io/${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + env: + IMAGE_NAME_MIXED_CASE: '${{ github.repository }}' + - name: Download digests uses: actions/download-artifact@v4 with: @@ -382,6 +425,15 @@ jobs: runs-on: ubuntu-latest needs: [ build-ollama-image ] steps: + # GitHub Packages requires the entire repository name to be in lowercase + # although the repository owner has a lowercase username, this prevents some people from running actions after forking + - name: Set repository and image name to lowercase + run: | + echo "IMAGE_NAME=${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + echo "FULL_IMAGE_NAME=ghcr.io/${IMAGE_NAME_MIXED_CASE,,}" >>${GITHUB_ENV} + env: + IMAGE_NAME_MIXED_CASE: '${{ github.repository }}' + - name: Download digests uses: actions/download-artifact@v4 with: From 78a145f1813ff911f84afb9871d7c3f9365bb93f Mon Sep 17 00:00:00 2001 From: Robbie Date: Sun, 16 Jun 2024 21:16:07 +1000 Subject: [PATCH 013/155] Add host.docker.internal overide --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 444002fbc6..8f67a86895 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ Check our Migration Guide available in our [Open WebUI Documentation](https://do If you want to try out the latest bleeding-edge features and are okay with occasional instability, you can use the `:dev` tag like this: ```bash -docker run -d -p 3000:8080 -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:dev +docker run -d -p 3000:8080 -v open-webui:/app/backend/data --name open-webui --add-host=host.docker.internal:host-gateway --restart always ghcr.io/open-webui/open-webui:dev ``` ## What's Next? 🌟 From 3eba963d030d98826e782ff9876d111c095d5bf8 Mon Sep 17 00:00:00 2001 From: Andrew Phillips Date: Sun, 16 Jun 2024 12:38:20 -0300 Subject: [PATCH 014/155] Remove redundant logging --- backend/apps/openai/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/apps/openai/main.py b/backend/apps/openai/main.py index 93f913dea2..6acb082377 100644 --- a/backend/apps/openai/main.py +++ b/backend/apps/openai/main.py @@ -435,8 +435,6 @@ async def generate_chat_completion( url = app.state.config.OPENAI_API_BASE_URLS[idx] key = app.state.config.OPENAI_API_KEYS[idx] - print(payload) - headers = {} headers["Authorization"] = f"Bearer {key}" headers["Content-Type"] = "application/json" From c0c875eae28642441c9e7eed177a88b5c9973fb3 Mon Sep 17 00:00:00 2001 From: Andrew Phillips Date: Sun, 16 Jun 2024 12:40:16 -0300 Subject: [PATCH 015/155] Use log.debug() for logging request bodies for the backend API --- backend/apps/ollama/main.py | 3 +-- backend/apps/openai/main.py | 2 +- backend/main.py | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/apps/ollama/main.py b/backend/apps/ollama/main.py index 1447554188..c8ea6fd373 100644 --- a/backend/apps/ollama/main.py +++ b/backend/apps/ollama/main.py @@ -839,8 +839,7 @@ async def generate_chat_completion( url = app.state.config.OLLAMA_BASE_URLS[url_idx] log.info(f"url: {url}") - - print(payload) + log.debug(payload) return await post_streaming_url(f"{url}/api/chat", json.dumps(payload)) diff --git a/backend/apps/openai/main.py b/backend/apps/openai/main.py index 6acb082377..c09c030d2d 100644 --- a/backend/apps/openai/main.py +++ b/backend/apps/openai/main.py @@ -430,7 +430,7 @@ async def generate_chat_completion( # Convert the modified body back to JSON payload = json.dumps(payload) - print(payload) + log.debug(payload) url = app.state.config.OPENAI_API_BASE_URLS[idx] key = app.state.config.OPENAI_API_KEYS[idx] diff --git a/backend/main.py b/backend/main.py index de8827d12d..6a589528e1 100644 --- a/backend/main.py +++ b/backend/main.py @@ -773,7 +773,7 @@ async def generate_title(form_data: dict, user=Depends(get_verified_user)): "title": True, } - print(payload) + log.debug(payload) try: payload = filter_pipeline(payload, user) @@ -837,7 +837,7 @@ async def generate_search_query(form_data: dict, user=Depends(get_verified_user) "max_tokens": 30, } - print(payload) + log.debug(payload) try: payload = filter_pipeline(payload, user) From a4810a5e42ab6589860dc63e3a4e8ffcca47c4f7 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sun, 16 Jun 2024 10:24:16 -0600 Subject: [PATCH 016/155] refac --- .../chat/MessageInput/Documents.svelte | 16 ++++++++++------ .../components/chat/MessageInput/Models.svelte | 12 +++++++----- .../chat/MessageInput/PromptCommands.svelte | 14 ++++++++------ 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/lib/components/chat/MessageInput/Documents.svelte b/src/lib/components/chat/MessageInput/Documents.svelte index fed9b82573..d563b719a9 100644 --- a/src/lib/components/chat/MessageInput/Documents.svelte +++ b/src/lib/components/chat/MessageInput/Documents.svelte @@ -102,17 +102,19 @@ {#if filteredItems.length > 0 || prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
-
-
+
+
#
-
-
+
+
{#each filteredItems as doc, docIdx}
-
{$i18n.t('Collection')}
+
+ {$i18n.t('Collection')} +
{:else}
#{doc.name} ({doc.filename}) diff --git a/src/lib/components/chat/MessageInput/Models.svelte b/src/lib/components/chat/MessageInput/Models.svelte index b4bf97df93..8312123827 100644 --- a/src/lib/components/chat/MessageInput/Models.svelte +++ b/src/lib/components/chat/MessageInput/Models.svelte @@ -134,17 +134,19 @@ {#if prompt.charAt(0) === '@'} {#if filteredModels.length > 0}
-
-
+
+
@
-
-
+
+
{#each filteredModels as model, modelIdx}
Date: Sun, 16 Jun 2024 10:27:34 -0600 Subject: [PATCH 017/155] fix: shift delete --- src/lib/components/layout/Sidebar.svelte | 10 +++++++--- src/lib/components/layout/Sidebar/ChatItem.svelte | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib/components/layout/Sidebar.svelte b/src/lib/components/layout/Sidebar.svelte index e844d4f39a..8b4c5664e9 100644 --- a/src/lib/components/layout/Sidebar.svelte +++ b/src/lib/components/layout/Sidebar.svelte @@ -464,9 +464,13 @@ on:select={() => { selectedChatId = chat.id; }} - on:delete={() => { - deleteChat = chat; - showDeleteConfirm = true; + on:delete={(e) => { + if ((e?.detail ?? '') === 'shift') { + deleteChatHandler(chat.id); + } else { + deleteChat = chat; + showDeleteConfirm = true; + } }} /> {/each} diff --git a/src/lib/components/layout/Sidebar/ChatItem.svelte b/src/lib/components/layout/Sidebar/ChatItem.svelte index 1504906909..8ed0d674fc 100644 --- a/src/lib/components/layout/Sidebar/ChatItem.svelte +++ b/src/lib/components/layout/Sidebar/ChatItem.svelte @@ -201,7 +201,7 @@
+
+
+
{$i18n.t('Widescreen Mode')}
+ + +
+
+
{$i18n.t('Title Auto-Generation')}
@@ -186,16 +230,16 @@
-
{$i18n.t('Widescreen Mode')}
+
{$i18n.t('Allow User Location')}
{/if}
{#if !camera} - {#if emoji} -
- {emoji} -
- {:else if loading} -
+ {emoji} +
+ {:else if loading} + - {:else} -
- {/if} + @keyframes spinner_8HQG { + 0%, + 57.14% { + animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1); + transform: translate(0); + } + 28.57% { + animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33); + transform: translateY(-6px); + } + 100% { + transform: translate(0); + } + } +
- - {#each chats - .sort((a, b) => { - if (a[sortKey] < b[sortKey]) return sortOrder === 'asc' ? -1 : 1; - if (a[sortKey] > b[sortKey]) return sortOrder === 'asc' ? 1 : -1; - return 0; - }) as chat, idx} + {#each chats.sort((a, b) => { + if (a[sortKey] < b[sortKey]) return sortOrder === 'asc' ? -1 : 1; + if (a[sortKey] > b[sortKey]) return sortOrder === 'asc' ? 1 : -1; + return 0; + }) as chat, idx} - - - - -
{$i18n.t('Role')} {$i18n.t('Name')} {$i18n.t('Email')} {$i18n.t('Last Active')} {$i18n.t('Created at')} setSortKey('role')}> + {$i18n.t('Role')} + {#if sortKey === 'role'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} + setSortKey('name')}> + {$i18n.t('Name')} + {#if sortKey === 'name'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} + setSortKey('email')}> + {$i18n.t('Email')} + {#if sortKey === 'email'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} + setSortKey('last_active_at')}> + {$i18n.t('Last Active')} + {#if sortKey === 'last_active_at'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} + setSortKey('created_at')}> + {$i18n.t('Created at')} + {#if sortKey === 'created_at'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} +
{$i18n.t('Name')} setSortKey('title')}> + {$i18n.t('Name')} + {#if sortKey === 'title'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} + setSortKey('created_at')}> + {$i18n.t('Created at')} + {#if sortKey === 'created_at'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + ▲ + {/if} +
+
{chat.title} @@ -88,11 +129,16 @@
{dayjs(chat.created_at * 1000).format($i18n.t('MMMM DD, YYYY HH:mm'))}
From 26575c508669489d1515a67e7370e4f22de10afb Mon Sep 17 00:00:00 2001 From: rdavis Date: Thu, 13 Jun 2024 23:59:15 +0000 Subject: [PATCH 009/155] Changed column header text to match property. Removed debugging code. --- src/lib/components/admin/UserChatsModal.svelte | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/lib/components/admin/UserChatsModal.svelte b/src/lib/components/admin/UserChatsModal.svelte index a36b0af695..7a2dea0486 100644 --- a/src/lib/components/admin/UserChatsModal.svelte +++ b/src/lib/components/admin/UserChatsModal.svelte @@ -42,9 +42,6 @@ sortOrder = 'asc'; } } - $: { - console.log(chats); - } @@ -84,7 +81,7 @@ >
setSortKey('title')}> - {$i18n.t('Name')} + {$i18n.t('Title')} {#if sortKey === 'title'} {sortOrder === 'asc' ? '▲' : '▼'} {:else} From b0d9aa38d2fb0c76790bd72081e969357d29f49b Mon Sep 17 00:00:00 2001 From: rdavis Date: Fri, 14 Jun 2024 00:07:21 +0000 Subject: [PATCH 010/155] Swapped from inline style to using tailwind class. --- src/lib/components/admin/UserChatsModal.svelte | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/components/admin/UserChatsModal.svelte b/src/lib/components/admin/UserChatsModal.svelte index 7a2dea0486..e1590f2684 100644 --- a/src/lib/components/admin/UserChatsModal.svelte +++ b/src/lib/components/admin/UserChatsModal.svelte @@ -85,7 +85,7 @@ {#if sortKey === 'title'} {sortOrder === 'asc' ? '▲' : '▼'} {:else} - ▲ + {/if} setSortKey('created_at')}> @@ -93,7 +93,7 @@ {#if sortKey === 'created_at'} {sortOrder === 'asc' ? '▲' : '▼'} {:else} - ▲ + {/if} From 385fcfe8d0a607ff3be69d6a07ea04f43555a575 Mon Sep 17 00:00:00 2001 From: rdavis Date: Fri, 14 Jun 2024 00:29:31 +0000 Subject: [PATCH 011/155] Swapped from inline style to using tailwind class. --- src/routes/(app)/admin/+page.svelte | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/routes/(app)/admin/+page.svelte b/src/routes/(app)/admin/+page.svelte index 858f5f82d6..3ea36c6896 100644 --- a/src/routes/(app)/admin/+page.svelte +++ b/src/routes/(app)/admin/+page.svelte @@ -155,7 +155,7 @@ {#if sortKey === 'role'} {sortOrder === 'asc' ? '▲' : '▼'} {:else} - ▲ + {/if} setSortKey('name')}> @@ -163,7 +163,7 @@ {#if sortKey === 'name'} {sortOrder === 'asc' ? '▲' : '▼'} {:else} - ▲ + {/if} setSortKey('email')}> @@ -171,7 +171,7 @@ {#if sortKey === 'email'} {sortOrder === 'asc' ? '▲' : '▼'} {:else} - ▲ + {/if} setSortKey('last_active_at')}> @@ -179,7 +179,7 @@ {#if sortKey === 'last_active_at'} {sortOrder === 'asc' ? '▲' : '▼'} {:else} - ▲ + {/if} setSortKey('created_at')}> @@ -187,7 +187,7 @@ {#if sortKey === 'created_at'} {sortOrder === 'asc' ? '▲' : '▼'} {:else} - ▲ + {/if} setSortKey('created_at')}> + setSortKey('created_at')} + > {$i18n.t('Created at')} {#if sortKey === 'created_at'} {sortOrder === 'asc' ? '▲' : '▼'} @@ -96,7 +104,11 @@ {/if}
setSortKey('role')}> - {$i18n.t('Role')} - {#if sortKey === 'role'} - {sortOrder === 'asc' ? '▲' : '▼'} - {:else} - - {/if} + setSortKey('role')} + > + {$i18n.t('Role')} + {#if sortKey === 'role'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + + {/if} setSortKey('name')}> - {$i18n.t('Name')} - {#if sortKey === 'name'} - {sortOrder === 'asc' ? '▲' : '▼'} - {:else} - - {/if} + setSortKey('name')} + > + {$i18n.t('Name')} + {#if sortKey === 'name'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + + {/if} setSortKey('email')}> - {$i18n.t('Email')} - {#if sortKey === 'email'} - {sortOrder === 'asc' ? '▲' : '▼'} - {:else} - - {/if} + setSortKey('email')} + > + {$i18n.t('Email')} + {#if sortKey === 'email'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + + {/if} setSortKey('last_active_at')}> - {$i18n.t('Last Active')} - {#if sortKey === 'last_active_at'} - {sortOrder === 'asc' ? '▲' : '▼'} - {:else} - - {/if} + setSortKey('last_active_at')} + > + {$i18n.t('Last Active')} + {#if sortKey === 'last_active_at'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + + {/if} setSortKey('created_at')}> - {$i18n.t('Created at')} - {#if sortKey === 'created_at'} - {sortOrder === 'asc' ? '▲' : '▼'} - {:else} - - {/if} + setSortKey('created_at')} + > + {$i18n.t('Created at')} + {#if sortKey === 'created_at'} + {sortOrder === 'asc' ? '▲' : '▼'} + {:else} + + {/if} @@ -213,7 +233,8 @@ const query = search.toLowerCase(); return name.includes(query); } - }).sort((a, b) => { + }) + .sort((a, b) => { if (a[sortKey] < b[sortKey]) return sortOrder === 'asc' ? -1 : 1; if (a[sortKey] > b[sortKey]) return sortOrder === 'asc' ? 1 : -1; return 0; From 4a67ae119502f61b933ff14eade9f9743808c119 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sun, 16 Jun 2024 22:28:26 -0700 Subject: [PATCH 032/155] fix: message delete issue --- src/lib/components/chat/Messages.svelte | 59 +++++---- .../chat/Messages/ResponseMessage.svelte | 116 +++++++++--------- 2 files changed, 94 insertions(+), 81 deletions(-) diff --git a/src/lib/components/chat/Messages.svelte b/src/lib/components/chat/Messages.svelte index f633b57744..3cab9a584e 100644 --- a/src/lib/components/chat/Messages.svelte +++ b/src/lib/components/chat/Messages.svelte @@ -202,38 +202,51 @@ }, 100); }; - const messageDeleteHandler = async (messageId) => { + const deleteMessageHandler = async (messageId) => { const messageToDelete = history.messages[messageId]; - const messageParentId = messageToDelete.parentId; - const messageChildrenIds = messageToDelete.childrenIds ?? []; - const hasSibling = messageChildrenIds.some( + + const parentMessageId = messageToDelete.parentId; + const childMessageIds = messageToDelete.childrenIds ?? []; + + const hasDescendantMessages = childMessageIds.some( (childId) => history.messages[childId]?.childrenIds?.length > 0 ); - messageChildrenIds.forEach((childId) => { - const child = history.messages[childId]; - if (child && child.childrenIds) { - if (child.childrenIds.length === 0 && !hasSibling) { - // if last prompt/response pair - history.messages[messageParentId].childrenIds = []; - history.currentId = messageParentId; + + history.currentId = parentMessageId; + await tick(); + + // Remove the message itself from the parent message's children array + history.messages[parentMessageId].childrenIds = history.messages[ + parentMessageId + ].childrenIds.filter((id) => id !== messageId); + + await tick(); + + childMessageIds.forEach((childId) => { + const childMessage = history.messages[childId]; + + if (childMessage && childMessage.childrenIds) { + if (childMessage.childrenIds.length === 0 && !hasDescendantMessages) { + // If there are no other responses/prompts + history.messages[parentMessageId].childrenIds = []; } else { - child.childrenIds.forEach((grandChildId) => { + childMessage.childrenIds.forEach((grandChildId) => { if (history.messages[grandChildId]) { - history.messages[grandChildId].parentId = messageParentId; - history.messages[messageParentId].childrenIds.push(grandChildId); + history.messages[grandChildId].parentId = parentMessageId; + history.messages[parentMessageId].childrenIds.push(grandChildId); } }); } } - // remove response - history.messages[messageParentId].childrenIds = history.messages[ - messageParentId + + // Remove child message id from the parent message's children array + history.messages[parentMessageId].childrenIds = history.messages[ + parentMessageId ].childrenIds.filter((id) => id !== childId); }); - // remove prompt - history.messages[messageParentId].childrenIds = history.messages[ - messageParentId - ].childrenIds.filter((id) => id !== messageId); + + await tick(); + await updateChatById(localStorage.token, chatId, { messages: messages, history: history @@ -292,7 +305,7 @@ > {#if message.role === 'user'} messageDeleteHandler(message.id)} + on:delete={() => deleteMessageHandler(message.id)} {user} {readOnly} {message} @@ -308,7 +321,7 @@ copyToClipboard={copyToClipboardWithToast} /> {:else if $mobile || (history.messages[message.parentId]?.models?.length ?? 1) === 1} - {#key message.id} + {#key message.id && history.currentId} - {/if} - {#if isLastMessage && !readOnly} - - - + + + + + + - - - + + + + + + {/if} {/if} {/if} From 4559c8af74cea3b3136a58ca645fdb2e5078caf3 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sun, 16 Jun 2024 22:52:27 -0700 Subject: [PATCH 033/155] doc: changelog --- CHANGELOG.md | 24 ++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfff72eed2..6756d105b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.5] - 2024-06-16 + +### Added + +- **📞 Enhanced Voice Call**: Text-to-speech (TTS) callback now operates in real-time for each sentence, reducing latency by not waiting for full completion. +- **👆 Tap to Interrupt**: During a call, you can now stop the assistant from speaking by simply tapping, instead of using voice. This resolves the issue of the speaker's voice being mistakenly registered as input. +- **😊 Emoji Call**: Toggle this feature on from the Settings > Interface, allowing LLMs to express emotions using emojis during voice calls for a more dynamic interaction. +- **🖱️ Quick Archive/Delete**: Use the Shift key + mouseover on the chat list to swiftly archive or delete items. +- **📝 Markdown Support in Model Descriptions**: You can now format model descriptions with markdown, enabling bold text, links, etc. +- **🧠 Editable Memories**: Adds the capability to modify memories. +- **📋 Admin Panel Sorting**: Introduces the ability to sort users/chats within the admin panel. +- **🌑 Dark Mode for Quick Selectors**: Dark mode now available for chat quick selectors (prompts, models, documents). +- **🔧 Advanced Parameters**: Adds 'num_keep' and 'num_batch' to advanced parameters for customization. +- **📅 Dynamic System Prompts**: New variables '{{CURRENT_DATETIME}}', '{{CURRENT_TIME}}', '{{USER_LOCATION}}' added for system prompts. Ensure '{{USER_LOCATION}}' is toggled on from Settings > Interface. +- **🌐 Tavily Web Search**: Includes Tavily as a web search provider option. +- **🖊️ Federated Auth Usernames**: Ability to set user names for federated authentication. +- **🔗 Auto Clean URLs**: When adding connection URLs, trailing slashes are now automatically removed. +- **🌐 Enhanced Translations**: Improved Chinese and Swedish translations. + +### Fixed + +- **⏳ AIOHTTP_CLIENT_TIMEOUT**: Introduced a new environment variable 'AIOHTTP_CLIENT_TIMEOUT' for requests to Ollama lasting longer than 5 minutes. Default is 300 seconds; set to blank ('') for no timeout. +- **❌ Message Delete Freeze**: Resolved an issue where message deletion would sometimes cause the web UI to freeze. + ## [0.3.4] - 2024-06-12 ### Fixed diff --git a/package-lock.json b/package-lock.json index f5b9d6a788..513993c74d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "open-webui", - "version": "0.3.4", + "version": "0.3.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "open-webui", - "version": "0.3.4", + "version": "0.3.5", "dependencies": { "@codemirror/lang-javascript": "^6.2.2", "@codemirror/lang-python": "^6.1.6", diff --git a/package.json b/package.json index bf353ef7f4..46aeb14f77 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-webui", - "version": "0.3.4", + "version": "0.3.5", "private": true, "scripts": { "dev": "npm run pyodide:fetch && vite dev --host", From a28ad06bf00153979a790d26595cb8ff8f3d18e2 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sun, 16 Jun 2024 23:36:21 -0700 Subject: [PATCH 034/155] fix --- backend/apps/ollama/main.py | 149 ++---------------------------------- 1 file changed, 6 insertions(+), 143 deletions(-) diff --git a/backend/apps/ollama/main.py b/backend/apps/ollama/main.py index 81a3b2a0e8..e82046e136 100644 --- a/backend/apps/ollama/main.py +++ b/backend/apps/ollama/main.py @@ -199,9 +199,6 @@ def merge_models_lists(model_lists): return list(merged_models.values()) -# user=Depends(get_current_user) - - async def get_all_models(): log.info("get_all_models()") @@ -1094,17 +1091,13 @@ async def download_file_stream( raise "Ollama: Could not create blob, Please try again." -# def number_generator(): -# for i in range(1, 101): -# yield f"data: {i}\n" - - # url = "https://huggingface.co/TheBloke/stablelm-zephyr-3b-GGUF/resolve/main/stablelm-zephyr-3b.Q2_K.gguf" @app.post("/models/download") @app.post("/models/download/{url_idx}") async def download_model( form_data: UrlForm, url_idx: Optional[int] = None, + user=Depends(get_admin_user), ): allowed_hosts = ["https://huggingface.co/", "https://github.com/"] @@ -1133,7 +1126,11 @@ async def download_model( @app.post("/models/upload") @app.post("/models/upload/{url_idx}") -def upload_model(file: UploadFile = File(...), url_idx: Optional[int] = None): +def upload_model( + file: UploadFile = File(...), + url_idx: Optional[int] = None, + user=Depends(get_admin_user), +): if url_idx == None: url_idx = 0 ollama_url = app.state.config.OLLAMA_BASE_URLS[url_idx] @@ -1196,137 +1193,3 @@ def upload_model(file: UploadFile = File(...), url_idx: Optional[int] = None): yield f"data: {json.dumps(res)}\n\n" return StreamingResponse(file_process_stream(), media_type="text/event-stream") - - -# async def upload_model(file: UploadFile = File(), url_idx: Optional[int] = None): -# if url_idx == None: -# url_idx = 0 -# url = app.state.config.OLLAMA_BASE_URLS[url_idx] - -# file_location = os.path.join(UPLOAD_DIR, file.filename) -# total_size = file.size - -# async def file_upload_generator(file): -# print(file) -# try: -# async with aiofiles.open(file_location, "wb") as f: -# completed_size = 0 -# while True: -# chunk = await file.read(1024*1024) -# if not chunk: -# break -# await f.write(chunk) -# completed_size += len(chunk) -# progress = (completed_size / total_size) * 100 - -# print(progress) -# yield f'data: {json.dumps({"status": "uploading", "percentage": progress, "total": total_size, "completed": completed_size, "done": False})}\n' -# except Exception as e: -# print(e) -# yield f"data: {json.dumps({'status': 'error', 'message': str(e)})}\n" -# finally: -# await file.close() -# print("done") -# yield f'data: {json.dumps({"status": "completed", "percentage": 100, "total": total_size, "completed": completed_size, "done": True})}\n' - -# return StreamingResponse( -# file_upload_generator(copy.deepcopy(file)), media_type="text/event-stream" -# ) - - -@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE"]) -async def deprecated_proxy( - path: str, request: Request, user=Depends(get_verified_user) -): - url = app.state.config.OLLAMA_BASE_URLS[0] - target_url = f"{url}/{path}" - - body = await request.body() - headers = dict(request.headers) - - if user.role in ["user", "admin"]: - if path in ["pull", "delete", "push", "copy", "create"]: - if user.role != "admin": - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail=ERROR_MESSAGES.ACCESS_PROHIBITED, - ) - else: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail=ERROR_MESSAGES.ACCESS_PROHIBITED, - ) - - headers.pop("host", None) - headers.pop("authorization", None) - headers.pop("origin", None) - headers.pop("referer", None) - - r = None - - def get_request(): - nonlocal r - - request_id = str(uuid.uuid4()) - try: - REQUEST_POOL.append(request_id) - - def stream_content(): - try: - if path == "generate": - data = json.loads(body.decode("utf-8")) - - if data.get("stream", True): - yield json.dumps({"id": request_id, "done": False}) + "\n" - - elif path == "chat": - yield json.dumps({"id": request_id, "done": False}) + "\n" - - for chunk in r.iter_content(chunk_size=8192): - if request_id in REQUEST_POOL: - yield chunk - else: - log.warning("User: canceled request") - break - finally: - if hasattr(r, "close"): - r.close() - if request_id in REQUEST_POOL: - REQUEST_POOL.remove(request_id) - - r = requests.request( - method=request.method, - url=target_url, - data=body, - headers=headers, - stream=True, - ) - - r.raise_for_status() - - # r.close() - - return StreamingResponse( - stream_content(), - status_code=r.status_code, - headers=dict(r.headers), - ) - except Exception as e: - raise e - - try: - return await run_in_threadpool(get_request) - except Exception as e: - error_detail = "Open WebUI: Server Connection Error" - if r is not None: - try: - res = r.json() - if "error" in res: - error_detail = f"Ollama: {res['error']}" - except: - error_detail = f"Ollama: {e}" - - raise HTTPException( - status_code=r.status_code if r else 500, - detail=error_detail, - ) From a3ac9ee774f0cb9dc511970f596c462d678e5fb7 Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:31:44 +0700 Subject: [PATCH 035/155] Refactor main.py Rename RAG_WEB_SEARCH_WHITE_LIST_DOMAINS to RAG_WEB_SEARCH_DOMAIN_FILTER_LIST --- backend/apps/rag/main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/apps/rag/main.py b/backend/apps/rag/main.py index 62be56aa41..a47874f22d 100644 --- a/backend/apps/rag/main.py +++ b/backend/apps/rag/main.py @@ -112,7 +112,7 @@ from config import ( YOUTUBE_LOADER_LANGUAGE, ENABLE_RAG_WEB_SEARCH, RAG_WEB_SEARCH_ENGINE, - RAG_WEB_SEARCH_WHITE_LIST_DOMAINS, + RAG_WEB_SEARCH_DOMAIN_FILTER_LIST, SEARXNG_QUERY_URL, GOOGLE_PSE_API_KEY, GOOGLE_PSE_ENGINE_ID, @@ -166,7 +166,7 @@ app.state.YOUTUBE_LOADER_TRANSLATION = None app.state.config.ENABLE_RAG_WEB_SEARCH = ENABLE_RAG_WEB_SEARCH app.state.config.RAG_WEB_SEARCH_ENGINE = RAG_WEB_SEARCH_ENGINE -app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS = RAG_WEB_SEARCH_WHITE_LIST_DOMAINS +app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST = RAG_WEB_SEARCH_DOMAIN_FILTER_LIST app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY @@ -777,7 +777,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.SEARXNG_QUERY_URL, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, - app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS + app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST ) else: raise Exception("No SEARXNG_QUERY_URL found in environment variables") @@ -791,7 +791,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.GOOGLE_PSE_ENGINE_ID, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, - app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS + app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST ) else: raise Exception( @@ -803,7 +803,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.BRAVE_SEARCH_API_KEY, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, - app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS + app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST ) else: raise Exception("No BRAVE_SEARCH_API_KEY found in environment variables") @@ -813,7 +813,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.SERPSTACK_API_KEY, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, - app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS, + app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST, https_enabled=app.state.config.SERPSTACK_HTTPS, ) else: @@ -824,7 +824,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.SERPER_API_KEY, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, - app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS + app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST ) else: raise Exception("No SERPER_API_KEY found in environment variables") @@ -834,12 +834,12 @@ def search_web(engine: str, query: str) -> list[SearchResult]: app.state.config.SERPLY_API_KEY, query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, - app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS + app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST ) else: raise Exception("No SERPLY_API_KEY found in environment variables") elif engine == "duckduckgo": - return search_duckduckgo(query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, app.state.config.RAG_WEB_SEARCH_WHITE_LIST_DOMAINS) + return search_duckduckgo(query, app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST) elif engine == "tavily": if app.state.config.TAVILY_API_KEY: return search_tavily( From b3d136b3b33781a4607ebdce952c3ec7daad204c Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:33:23 +0700 Subject: [PATCH 036/155] Refactored config.py Renamed RAG_WEB_SEARCH_WHITE_LIST_DOMAINS to RAG_WEB_SEARCH_DOMAIN_FILTER_LIST --- backend/config.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/config.py b/backend/config.py index d19a81fce1..5981debb02 100644 --- a/backend/config.py +++ b/backend/config.py @@ -903,12 +903,15 @@ RAG_WEB_SEARCH_ENGINE = PersistentConfig( os.getenv("RAG_WEB_SEARCH_ENGINE", ""), ) -RAG_WEB_SEARCH_WHITE_LIST_DOMAINS = PersistentConfig( - "RAG_WEB_SEARCH_WHITE_LIST_DOMAINS", - "rag.rag_web_search_white_list_domains", +# You can provide a list of your own websites to filter after performing a web search. +# This ensures the highest level of safety and reliability of the information sources. +RAG_WEB_SEARCH_DOMAIN_FILTER_LIST = PersistentConfig( + "RAG_WEB_SEARCH_DOMAIN_FILTER_LIST", + "rag.rag.web.search.domain.filter_list", [ - # "example.com", - # "anotherdomain.com", + # "wikipedia.com", + # "wikimedia.org", + # "wikidata.org", ], ) From a02139ba9df0513696c3cb89aecf037e19aee4d2 Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:34:17 +0700 Subject: [PATCH 037/155] Set filter_list as optional param in brave.py --- backend/apps/rag/search/brave.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/apps/rag/search/brave.py b/backend/apps/rag/search/brave.py index 04cd184964..a20a2cde80 100644 --- a/backend/apps/rag/search/brave.py +++ b/backend/apps/rag/search/brave.py @@ -1,15 +1,15 @@ import logging -from typing import List +from typing import List, Optional import requests -from apps.rag.search.main import SearchResult, filter_by_whitelist +from apps.rag.search.main import SearchResult, get_filtered_results from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["RAG"]) -def search_brave(api_key: str, query: str, whitelist:List[str], count: int) -> list[SearchResult]: +def search_brave(api_key: str, query: str, count: int, filter_list: Optional[List[str]] = None) -> list[SearchResult]: """Search using Brave's Search API and return the results as a list of SearchResult objects. Args: @@ -29,10 +29,12 @@ def search_brave(api_key: str, query: str, whitelist:List[str], count: int) -> l json_response = response.json() results = json_response.get("web", {}).get("results", []) - filtered_results = filter_by_whitelist(results, whitelist) + if filter_list: + results = get_filtered_results(results, filter_list) + return [ SearchResult( link=result["url"], title=result.get("title"), snippet=result.get("snippet") ) - for result in filtered_results[:count] + for result in results[:count] ] From 7d2ad8c4bf44e2c5b180310a462e9fc90d2ad2ec Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:34:59 +0700 Subject: [PATCH 038/155] Set filter_list as optional param in duckduckgo.py --- backend/apps/rag/search/duckduckgo.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/apps/rag/search/duckduckgo.py b/backend/apps/rag/search/duckduckgo.py index 9342e53e43..8bcb18cc9d 100644 --- a/backend/apps/rag/search/duckduckgo.py +++ b/backend/apps/rag/search/duckduckgo.py @@ -1,6 +1,6 @@ import logging -from typing import List -from apps.rag.search.main import SearchResult, filter_by_whitelist +from typing import List, Optional +from apps.rag.search.main import SearchResult, get_filtered_results from duckduckgo_search import DDGS from config import SRC_LOG_LEVELS @@ -8,7 +8,7 @@ log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["RAG"]) -def search_duckduckgo(query: str, count: int, whitelist:List[str]) -> list[SearchResult]: +def search_duckduckgo(query: str, count: int, filter_list: Optional[List[str]] = None) -> list[SearchResult]: """ Search using DuckDuckGo's Search API and return the results as a list of SearchResult objects. Args: @@ -41,7 +41,7 @@ def search_duckduckgo(query: str, count: int, whitelist:List[str]) -> list[Searc snippet=result.get("body"), ) ) - # print(results) - filtered_results = filter_by_whitelist(results, whitelist) + if filter_list: + results = get_filtered_results(results, filter_list) # Return the list of search results - return filtered_results + return results From d8beed13b4a19281d114f6740a718a418a59e8c3 Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:35:27 +0700 Subject: [PATCH 039/155] Set filter_list as optional param in google_pse.py --- backend/apps/rag/search/google_pse.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/apps/rag/search/google_pse.py b/backend/apps/rag/search/google_pse.py index bc89b2f3a6..84ecafa81e 100644 --- a/backend/apps/rag/search/google_pse.py +++ b/backend/apps/rag/search/google_pse.py @@ -1,9 +1,9 @@ import json import logging -from typing import List +from typing import List, Optional import requests -from apps.rag.search.main import SearchResult, filter_by_whitelist +from apps.rag.search.main import SearchResult, get_filtered_results from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) @@ -11,7 +11,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"]) def search_google_pse( - api_key: str, search_engine_id: str, query: str, count: int, whitelist:List[str] + api_key: str, search_engine_id: str, query: str, count: int, filter_list: Optional[List[str]] = None ) -> list[SearchResult]: """Search using Google's Programmable Search Engine API and return the results as a list of SearchResult objects. @@ -35,12 +35,13 @@ def search_google_pse( json_response = response.json() results = json_response.get("items", []) - filtered_results = filter_by_whitelist(results, whitelist) + if filter_list: + results = get_filtered_results(results, filter_list) return [ SearchResult( link=result["link"], title=result.get("title"), snippet=result.get("snippet"), ) - for result in filtered_results + for result in results ] From 3cc0e3ecb6f7ff6b4cd9ae9bf244fb237122fc99 Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:36:26 +0700 Subject: [PATCH 040/155] Refactor rag/main.py Renamed function get_filtered_results --- backend/apps/rag/search/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/apps/rag/search/main.py b/backend/apps/rag/search/main.py index 612177581a..4041372f3b 100644 --- a/backend/apps/rag/search/main.py +++ b/backend/apps/rag/search/main.py @@ -3,13 +3,13 @@ from urllib.parse import urlparse from pydantic import BaseModel -def filter_by_whitelist(results, whitelist): - if not whitelist: +def get_filtered_results(results, filter_list): + if not filter_list: return results filtered_results = [] for result in results: domain = urlparse(result["url"]).netloc - if any(domain.endswith(whitelisted_domain) for whitelisted_domain in whitelist): + if any(domain.endswith(filtered_domain) for filtered_domain in filter_list): filtered_results.append(result) return filtered_results From 9c446d9fb4961cf39cb9750c8fec18909a6e0aa8 Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:36:56 +0700 Subject: [PATCH 041/155] Set filter_list as optional param in searxng.py --- backend/apps/rag/search/searxng.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/apps/rag/search/searxng.py b/backend/apps/rag/search/searxng.py index 954aaf0724..850513ee1c 100644 --- a/backend/apps/rag/search/searxng.py +++ b/backend/apps/rag/search/searxng.py @@ -1,9 +1,9 @@ import logging import requests -from typing import List +from typing import List, Optional -from apps.rag.search.main import SearchResult +from apps.rag.search.main import SearchResult, get_filtered_results from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) @@ -11,7 +11,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"]) def search_searxng( - query_url: str, query: str, count: int, whitelist:List[str], **kwargs + query_url: str, query: str, count: int, filter_list: Optional[List[str]] = None, **kwargs ) -> List[SearchResult]: """ Search a SearXNG instance for a given query and return the results as a list of SearchResult objects. @@ -78,10 +78,11 @@ def search_searxng( json_response = response.json() results = json_response.get("results", []) sorted_results = sorted(results, key=lambda x: x.get("score", 0), reverse=True) - filtered_results = filter_by_whitelist(sorted_results, whitelist) + if filter_list: + sorted_results = get_filtered_results(sorted_results, whitelist) return [ SearchResult( link=result["url"], title=result.get("title"), snippet=result.get("content") ) - for result in filtered_results[:count] + for result in sorted_results[:count] ] From 6b8290fa6dffd5b92fbc2b270d84ed0a90593738 Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:37:26 +0700 Subject: [PATCH 042/155] Set filter_list as optional param in serper.py --- backend/apps/rag/search/serper.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/apps/rag/search/serper.py b/backend/apps/rag/search/serper.py index e12126a358..1e39184467 100644 --- a/backend/apps/rag/search/serper.py +++ b/backend/apps/rag/search/serper.py @@ -1,16 +1,16 @@ import json import logging -from typing import List +from typing import List, Optional import requests -from apps.rag.search.main import SearchResult, filter_by_whitelist +from apps.rag.search.main import SearchResult, get_filtered_results from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["RAG"]) -def search_serper(api_key: str, query: str, count: int, whitelist:List[str]) -> list[SearchResult]: +def search_serper(api_key: str, query: str, count: int, filter_list: Optional[List[str]] = None) -> list[SearchResult]: """Search using serper.dev's API and return the results as a list of SearchResult objects. Args: @@ -29,12 +29,13 @@ def search_serper(api_key: str, query: str, count: int, whitelist:List[str]) -> results = sorted( json_response.get("organic", []), key=lambda x: x.get("position", 0) ) - filtered_results = filter_by_whitelist(results, whitelist) + if filter_list: + results = get_filtered_results(results, filter_list) return [ SearchResult( link=result["link"], title=result.get("title"), snippet=result.get("description"), ) - for result in filtered_results[:count] + for result in results[:count] ] From bcb84235b17007f6d176074ee1ee327fbcdc4844 Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:37:52 +0700 Subject: [PATCH 043/155] Set filter_list as optional param in serply.py --- backend/apps/rag/search/serply.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/apps/rag/search/serply.py b/backend/apps/rag/search/serply.py index e4040a8485..24b249b739 100644 --- a/backend/apps/rag/search/serply.py +++ b/backend/apps/rag/search/serply.py @@ -1,10 +1,10 @@ import json import logging -from typing import List +from typing import List, Optional import requests from urllib.parse import urlencode -from apps.rag.search.main import SearchResult, filter_by_whitelist +from apps.rag.search.main import SearchResult, get_filtered_results from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) @@ -15,11 +15,11 @@ def search_serply( api_key: str, query: str, count: int, - whitelist:List[str], hl: str = "us", limit: int = 10, device_type: str = "desktop", proxy_location: str = "US", + filter_list: Optional[List[str]] = None, ) -> list[SearchResult]: """Search using serper.dev's API and return the results as a list of SearchResult objects. @@ -58,12 +58,13 @@ def search_serply( results = sorted( json_response.get("results", []), key=lambda x: x.get("realPosition", 0) ) - filtered_results = filter_by_whitelist(results, whitelist) + if filter_list: + results = get_filtered_results(results, filter_list) return [ SearchResult( link=result["link"], title=result.get("title"), snippet=result.get("description"), ) - for result in filtered_results[:count] + for result in results[:count] ] From c4873859802303c37eeeac1ead3c44fae3aa182e Mon Sep 17 00:00:00 2001 From: Que Nguyen Date: Mon, 17 Jun 2024 14:38:11 +0700 Subject: [PATCH 044/155] Set filter_list as optional param in serpstack.py --- backend/apps/rag/search/serpstack.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/apps/rag/search/serpstack.py b/backend/apps/rag/search/serpstack.py index 344e250733..f69baaf924 100644 --- a/backend/apps/rag/search/serpstack.py +++ b/backend/apps/rag/search/serpstack.py @@ -1,9 +1,9 @@ import json import logging -from typing import List +from typing import List, Optional import requests -from apps.rag.search.main import SearchResult, filter_by_whitelist +from apps.rag.search.main import SearchResult, get_filtered_results from config import SRC_LOG_LEVELS log = logging.getLogger(__name__) @@ -11,7 +11,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"]) def search_serpstack( - api_key: str, query: str, count: int, whitelist:List[str], https_enabled: bool = True + api_key: str, query: str, count: int, filter_list: Optional[List[str]] = None, https_enabled: bool = True ) -> list[SearchResult]: """Search using serpstack.com's and return the results as a list of SearchResult objects. @@ -35,10 +35,11 @@ def search_serpstack( results = sorted( json_response.get("organic_results", []), key=lambda x: x.get("position", 0) ) - filtered_results = filter_by_whitelist(results, whitelist) + if filter_list: + results = get_filtered_results(results, filter_list) return [ SearchResult( link=result["url"], title=result.get("title"), snippet=result.get("snippet") ) - for result in filtered_results[:count] + for result in results[:count] ] From 5fa355e1ae3c509ccef63bc79dcd8794e4ec186d Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Mon, 17 Jun 2024 01:31:22 -0700 Subject: [PATCH 045/155] feat: background image --- src/lib/components/chat/Chat.svelte | 22 +- .../components/chat/Settings/Interface.svelte | 348 +++++++++++------- src/lib/utils/characters/index.ts | 0 src/routes/(app)/c/[id]/+page.svelte | 5 +- 4 files changed, 229 insertions(+), 146 deletions(-) create mode 100644 src/lib/utils/characters/index.ts diff --git a/src/lib/components/chat/Chat.svelte b/src/lib/components/chat/Chat.svelte index 8819a0428a..a5609685e6 100644 --- a/src/lib/components/chat/Chat.svelte +++ b/src/lib/components/chat/Chat.svelte @@ -273,6 +273,7 @@ id: m.id, role: m.role, content: m.content, + info: m.info ? m.info : undefined, timestamp: m.timestamp })), chat_id: $chatId @@ -1322,6 +1323,19 @@ ? 'md:max-w-[calc(100%-260px)]' : ''} w-full max-w-full flex flex-col" > + {#if $settings?.backgroundImageUrl ?? null} +
+ +
+ {/if} + 0 && messages.length === 0 && !$chatId && selectedModels.length <= 1}
{#each $banners.filter( (b) => (b.dismissible ? !JSON.parse(localStorage.getItem('dismissedBannerIds') ?? '[]').includes(b.id) : true) ) as banner} @@ -1358,9 +1374,9 @@
{/if} -
+