diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 0a36356f3e..2a45c2c16e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,7 +11,7 @@ - [ ] **Dependencies:** Are there any new dependencies? Have you updated the dependency versions in the documentation? - [ ] **Testing:** Have you written and run sufficient tests for validating the changes? - [ ] **Code review:** Have you performed a self-review of your code, addressing any coding standard issues and ensuring adherence to the project's coding standards? -- [ ] **Label:** To cleary categorize this pull request, assign a relevant label to the pull request title, using one of the following: +- [ ] **Prefix:** To cleary categorize this pull request, prefix the pull request title, using one of the following: - **BREAKING CHANGE**: Significant changes that may affect compatibility - **build**: Changes that affect the build system or external dependencies - **ci**: Changes to our continuous integration processes or workflows diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 86d27f4dc7..da40f56ffe 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -70,8 +70,10 @@ jobs: images: ${{ env.FULL_IMAGE_NAME }} tags: | type=ref,event=branch + ${{ github.ref_type == 'tag' && 'type=raw,value=main' || '' }} flavor: | prefix=cache-${{ matrix.platform }}- + latest=false - name: Build Docker image (latest) uses: docker/build-push-action@v5 @@ -158,8 +160,10 @@ jobs: images: ${{ env.FULL_IMAGE_NAME }} tags: | type=ref,event=branch + ${{ github.ref_type == 'tag' && 'type=raw,value=main' || '' }} flavor: | prefix=cache-cuda-${{ matrix.platform }}- + latest=false - name: Build Docker image (cuda) uses: docker/build-push-action@v5 @@ -247,8 +251,10 @@ jobs: images: ${{ env.FULL_IMAGE_NAME }} tags: | type=ref,event=branch + ${{ github.ref_type == 'tag' && 'type=raw,value=main' || '' }} flavor: | prefix=cache-ollama-${{ matrix.platform }}- + latest=false - name: Build Docker image (ollama) uses: docker/build-push-action@v5 diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index b786329c25..70a19c64a6 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -4,7 +4,6 @@ on: push: branches: - main # or whatever branch you want to use - - dev jobs: release: diff --git a/CHANGELOG.md b/CHANGELOG.md index 98656a309e..d19e82c392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,115 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.2.5] - 2024-06-05 + +### Added + +- **👥 Active Users Indicator**: Now you can see how many people are currently active and what they are running. This helps you gauge when performance might slow down due to a high number of users. +- **🗂️ Create Ollama Modelfile**: The option to create a modelfile for Ollama has been reintroduced in the Settings > Models section, making it easier to manage your models. +- **⚙️ Default Model Setting**: Added an option to set the default model from Settings > Interface. This feature is now easily accessible, especially convenient for mobile users as it was previously hidden. +- **🌐 Enhanced Translations**: We've improved the Chinese translations and added support for Turkmen and Norwegian languages to make the interface more accessible globally. + +### Fixed + +- **📱 Mobile View Improvements**: The UI now uses dvh (dynamic viewport height) instead of vh (viewport height), providing a better and more responsive experience for mobile users. + +## [0.2.4] - 2024-06-03 + +### Added + +- **👤 Improved Account Pending Page**: The account pending page now displays admin details by default to avoid confusion. You can disable this feature in the admin settings if needed. +- **🌐 HTTP Proxy Support**: We have enabled the use of the 'http_proxy' environment variable in OpenAI and Ollama API calls, making it easier to configure network settings. +- **❓ Quick Access to Documentation**: You can now easily access Open WebUI documents via a question mark button located at the bottom right corner of the screen (available on larger screens like PCs). +- **🌍 Enhanced Translation**: Improvements have been made to translations. + +### Fixed + +- **🔍 SearxNG Web Search**: Fixed the issue where the SearxNG web search functionality was not working properly. + +## [0.2.3] - 2024-06-03 + +### Added + +- **📁 Export Chat as JSON**: You can now export individual chats as JSON files from the navbar menu by navigating to 'Download > Export Chat'. This makes sharing specific conversations easier. +- **✏️ Edit Titles with Double Click**: Double-click on titles to rename them quickly and efficiently. +- **🧩 Batch Multiple Embeddings**: Introduced 'RAG_EMBEDDING_OPENAI_BATCH_SIZE' to process multiple embeddings in a batch, enhancing performance for large datasets. +- **🌍 Improved Translations**: Enhanced the translation quality across various languages for a better user experience. + +### Fixed + +- **🛠️ Modelfile Migration Script**: Fixed an issue where the modelfile migration script would fail if an invalid modelfile was encountered. +- **💬 Zhuyin Input Method on Mac**: Resolved an issue where using the Zhuyin input method in the Web UI on a Mac caused text to send immediately upon pressing the enter key, leading to incorrect input. +- **🔊 Local TTS Voice Selection**: Fixed the issue where the selected local Text-to-Speech (TTS) voice was not being displayed in settings. + +## [0.2.2] - 2024-06-02 + +### Added + +- **🌊 Mermaid Rendering Support**: We've included support for Mermaid rendering. This allows you to create beautiful diagrams and flowcharts directly within Open WebUI. +- **🔄 New Environment Variable 'RESET_CONFIG_ON_START'**: Introducing a new environment variable: 'RESET_CONFIG_ON_START'. Set this variable to reset your configuration settings upon starting the application, making it easier to revert to default settings. + +### Fixed + +- **🔧 Pipelines Filter Issue**: We've addressed an issue with the pipelines where filters were not functioning as expected. + +## [0.2.1] - 2024-06-02 + +### Added + +- **🖱️ Single Model Export Button**: Easily export models with just one click using the new single model export button. +- **🖥️ Advanced Parameters Support**: Added support for 'num_thread', 'use_mmap', and 'use_mlock' parameters for Ollama. +- **🌐 Improved Vietnamese Translation**: Enhanced Vietnamese language support for a better user experience for our Vietnamese-speaking community. + +### Fixed + +- **🔧 OpenAI URL API Save Issue**: Corrected a problem preventing the saving of OpenAI URL API settings. +- **🚫 Display Issue with Disabled Ollama API**: Fixed the display bug causing models to appear in settings when the Ollama API was disabled. + +### Changed + +- **💡 Versioning Update**: As a reminder from our previous update, version 0.2.y will focus primarily on bug fixes, while major updates will be designated as 0.x from now on for better version tracking. + +## [0.2.0] - 2024-06-01 + +### Added + +- **🔧 Pipelines Support**: Open WebUI now includes a plugin framework for enhanced customization and functionality (https://github.com/open-webui/pipelines). Easily add custom logic and integrate Python libraries, from AI agents to home automation APIs. +- **🔗 Function Calling via Pipelines**: Integrate function calling seamlessly through Pipelines. +- **⚖️ User Rate Limiting via Pipelines**: Implement user-specific rate limits to manage API usage efficiently. +- **📊 Usage Monitoring with Langfuse**: Track and analyze usage statistics with Langfuse integration through Pipelines. +- **🕒 Conversation Turn Limits**: Set limits on conversation turns to manage interactions better through Pipelines. +- **🛡️ Toxic Message Filtering**: Automatically filter out toxic messages to maintain a safe environment using Pipelines. +- **🔍 Web Search Support**: Introducing built-in web search capabilities via RAG API, allowing users to search using SearXNG, Google Programmatic Search Engine, Brave Search, serpstack, and serper. Activate it effortlessly by adding necessary variables from Document settings > Web Params. +- **🗂️ Models Workspace**: Create and manage model presets for both Ollama/OpenAI API. Note: The old Modelfiles workspace is deprecated. +- **🛠️ Model Builder Feature**: Build and edit all models with persistent builder mode. +- **🏷️ Model Tagging Support**: Organize models with tagging features in the models workspace. +- **📋 Model Ordering Support**: Effortlessly organize models by dragging and dropping them into the desired positions within the models workspace. +- **📈 OpenAI Generation Stats**: Access detailed generation statistics for OpenAI models. +- **📅 System Prompt Variables**: New variables added: '{{CURRENT_DATE}}' and '{{USER_NAME}}' for dynamic prompts. +- **📢 Global Banner Support**: Manage global banners from admin settings > banners. +- **🗃️ Enhanced Archived Chats Modal**: Search and export archived chats easily. +- **📂 Archive All Button**: Quickly archive all chats from settings > chats. +- **🌐 Improved Translations**: Added and improved translations for French, Croatian, Cebuano, and Vietnamese. + +### Fixed + +- **🔍 Archived Chats Visibility**: Resolved issue with archived chats not showing in the admin panel. +- **💬 Message Styling**: Fixed styling issues affecting message appearance. +- **🔗 Shared Chat Responses**: Corrected the issue where shared chat response messages were not readonly. +- **🖥️ UI Enhancement**: Fixed the scrollbar overlapping issue with the message box in the user interface. + +### Changed + +- **💾 User Settings Storage**: User settings are now saved on the backend, ensuring consistency across all devices. +- **📡 Unified API Requests**: The API request for getting models is now unified to '/api/models' for easier usage. +- **🔄 Versioning Update**: Our versioning will now follow the format 0.x for major updates and 0.x.y for patches. +- **📦 Export All Chats (All Users)**: Moved this functionality to the Admin Panel settings for better organization and accessibility. + +### Removed + +- **🚫 Bundled LiteLLM Support Deprecated**: Migrate your LiteLLM config.yaml to a self-hosted LiteLLM instance. LiteLLM can still be added via OpenAI Connections. Download the LiteLLM config.yaml from admin settings > database > export LiteLLM config.yaml. + ## [0.1.125] - 2024-05-19 ### Added diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..37ac5263cb --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,77 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contribute to a positive environment for our community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, without their explicit permission +- **Spamming of any kind** +- Aggressive sales tactics targeting our community members are strictly prohibited. You can mention your product if it's relevant to the discussion, but under no circumstances should you push it forcefully +- Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all community spaces and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, spamming, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at hello@openwebui.com. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Temporary Ban + +**Community Impact**: Any violation of community standards, including but not limited to inappropriate language, unprofessional behavior, harassment, or spamming. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +### 2. Permanent Ban + +**Community Impact**: Repeated or severe violations of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/Dockerfile b/Dockerfile index be5c1da411..2bd05e6baa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,6 @@ ARG USE_OLLAMA ARG USE_CUDA_VER ARG USE_EMBEDDING_MODEL ARG USE_RERANKING_MODEL -ARG BUILD_HASH ARG UID ARG GID @@ -154,6 +153,7 @@ HEALTHCHECK CMD curl --silent --fail http://localhost:8080/health | jq -e '.stat USER $UID:$GID +ARG BUILD_HASH ENV WEBUI_BUILD_VERSION=${BUILD_HASH} CMD [ "bash", "start.sh"] diff --git a/README.md b/README.md index a40018f0ae..a8d79bd5cd 100644 --- a/README.md +++ b/README.md @@ -11,97 +11,43 @@ [![Discord](https://img.shields.io/badge/Discord-Open_WebUI-blue?logo=discord&logoColor=white)](https://discord.gg/5rJgQTnV4s) [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/tjbck) -Open WebUI is an extensible, feature-rich, and user-friendly self-hosted WebUI designed to operate entirely offline. It supports various LLM runners, including Ollama and OpenAI-compatible APIs. For more information, be sure to check out our [Open WebUI Documentation](https://docs.openwebui.com/). +Open WebUI is an [extensible](https://github.com/open-webui/pipelines), feature-rich, and user-friendly self-hosted WebUI designed to operate entirely offline. It supports various LLM runners, including Ollama and OpenAI-compatible APIs. For more information, be sure to check out our [Open WebUI Documentation](https://docs.openwebui.com/). ![Open WebUI Demo](./demo.gif) -## Features ⭐ +## Key Features of Open WebUI ⭐ -- 🖥️ **Intuitive Interface**: Our chat interface takes inspiration from ChatGPT, ensuring a user-friendly experience. +- 🚀 **Effortless Setup**: Install seamlessly using Docker or Kubernetes (kubectl, kustomize or helm) for a hassle-free experience with support for both `:ollama` and `:cuda` tagged images. -- 📱 **Responsive Design**: Enjoy a seamless experience on both desktop and mobile devices. +- 🤝 **Ollama/OpenAI API Integration**: Effortlessly integrate OpenAI-compatible APIs for versatile conversations alongside Ollama models. Customize the OpenAI API URL to link with **LMStudio, GroqCloud, Mistral, OpenRouter, and more**. -- ⚡ **Swift Responsiveness**: Enjoy fast and responsive performance. +- 🧩 **Pipelines, Open WebUI Plugin Support**: Seamlessly integrate custom logic and Python libraries into Open WebUI using [Pipelines Plugin Framework](https://github.com/open-webui/pipelines). Launch your Pipelines instance, set the OpenAI URL to the Pipelines URL, and explore endless possibilities. [Examples](https://github.com/open-webui/pipelines/tree/main/examples) include **Function Calling**, User **Rate Limiting** to control access, **Usage Monitoring** with tools like Langfuse, **Live Translation with LibreTranslate** for multilingual support, **Toxic Message Filtering** and much more. -- 🚀 **Effortless Setup**: Install seamlessly using Docker or Kubernetes (kubectl, kustomize or helm) for a hassle-free experience. +- 📱 **Responsive Design**: Enjoy a seamless experience across Desktop PC, Laptop, and Mobile devices. -- 🌈 **Theme Customization**: Choose from a variety of themes to personalize your Open WebUI experience. - -- 💻 **Code Syntax Highlighting**: Enjoy enhanced code readability with our syntax highlighting feature. +- 📱 **Progressive Web App (PWA) for Mobile**: Enjoy a native app-like experience on your mobile device with our PWA, providing offline access on localhost and a seamless user interface. - ✒️🔢 **Full Markdown and LaTeX Support**: Elevate your LLM experience with comprehensive Markdown and LaTeX capabilities for enriched interaction. -- 📚 **Local RAG Integration**: Dive into the future of chat interactions with the groundbreaking Retrieval Augmented Generation (RAG) support. This feature seamlessly integrates document interactions into your chat experience. You can load documents directly into the chat or add files to your document library, effortlessly accessing them using `#` command in the prompt. In its alpha phase, occasional issues may arise as we actively refine and enhance this feature to ensure optimal performance and reliability. +- 🛠️ **Model Builder**: Easily create Ollama models via the Web UI. Create and add custom characters/agents, customize chat elements, and import models effortlessly through [Open WebUI Community](https://openwebui.com/) integration. -- 🔍 **RAG Embedding Support**: Change the RAG embedding model directly in document settings, enhancing document processing. This feature supports Ollama and OpenAI models. +- 📚 **Local RAG Integration**: Dive into the future of chat interactions with groundbreaking Retrieval Augmented Generation (RAG) support. This feature seamlessly integrates document interactions into your chat experience. You can load documents directly into the chat or add files to your document library, effortlessly accessing them using the `#` command before a query. -- 🌐 **Web Browsing Capability**: Seamlessly integrate websites into your chat experience using the `#` command followed by the URL. This feature allows you to incorporate web content directly into your conversations, enhancing the richness and depth of your interactions. +- 🔍 **Web Search for RAG**: Perform web searches using providers like `SearXNG`, `Google PSE`, `Brave Search`, `serpstack`, and `serper`, and inject the results directly into your chat experience. -- 📜 **Prompt Preset Support**: Instantly access preset prompts using the `/` command in the chat input. Load predefined conversation starters effortlessly and expedite your interactions. Effortlessly import prompts through [Open WebUI Community](https://openwebui.com/) integration. +- 🌐 **Web Browsing Capability**: Seamlessly integrate websites into your chat experience using the `#` command followed by a URL. This feature allows you to incorporate web content directly into your conversations, enhancing the richness and depth of your interactions. -- 👍👎 **RLHF Annotation**: Empower your messages by rating them with thumbs up and thumbs down, followed by the option to provide textual feedback, facilitating the creation of datasets for Reinforcement Learning from Human Feedback (RLHF). Utilize your messages to train or fine-tune models, all while ensuring the confidentiality of locally saved data. - -- 🏷️ **Conversation Tagging**: Effortlessly categorize and locate specific chats for quick reference and streamlined data collection. - -- 📥🗑️ **Download/Delete Models**: Easily download or remove models directly from the web UI. - -- 🔄 **Update All Ollama Models**: Easily update locally installed models all at once with a convenient button, streamlining model management. - -- ⬆️ **GGUF File Model Creation**: Effortlessly create Ollama models by uploading GGUF files directly from the web UI. Streamlined process with options to upload from your machine or download GGUF files from Hugging Face. - -- 🤖 **Multiple Model Support**: Seamlessly switch between different chat models for diverse interactions. - -- 🔄 **Multi-Modal Support**: Seamlessly engage with models that support multimodal interactions, including images (e.g., LLava). - -- 🧩 **Modelfile Builder**: Easily create Ollama modelfiles via the web UI. Create and add characters/agents, customize chat elements, and import modelfiles effortlessly through [Open WebUI Community](https://openwebui.com/) integration. +- 🎨 **Image Generation Integration**: Seamlessly incorporate image generation capabilities using options such as AUTOMATIC1111 API or ComfyUI (local), and OpenAI's DALL-E (external), enriching your chat experience with dynamic visual content. - ⚙️ **Many Models Conversations**: Effortlessly engage with various models simultaneously, harnessing their unique strengths for optimal responses. Enhance your experience by leveraging a diverse set of models in parallel. -- 💬 **Collaborative Chat**: Harness the collective intelligence of multiple models by seamlessly orchestrating group conversations. Use the `@` command to specify the model, enabling dynamic and diverse dialogues within your chat interface. Immerse yourself in the collective intelligence woven into your chat environment. - -- 🗨️ **Local Chat Sharing**: Generate and share chat links seamlessly between users, enhancing collaboration and communication. - -- 🔄 **Regeneration History Access**: Easily revisit and explore your entire regeneration history. - -- 📜 **Chat History**: Effortlessly access and manage your conversation history. - -- 📬 **Archive Chats**: Effortlessly store away completed conversations with LLMs for future reference, maintaining a tidy and clutter-free chat interface while allowing for easy retrieval and reference. - -- 📤📥 **Import/Export Chat History**: Seamlessly move your chat data in and out of the platform. - -- 🗣️ **Voice Input Support**: Engage with your model through voice interactions; enjoy the convenience of talking to your model directly. Additionally, explore the option for sending voice input automatically after 3 seconds of silence for a streamlined experience. - -- 🔊 **Configurable Text-to-Speech Endpoint**: Customize your Text-to-Speech experience with configurable OpenAI endpoints. - -- ⚙️ **Fine-Tuned Control with Advanced Parameters**: Gain a deeper level of control by adjusting parameters such as temperature and defining your system prompts to tailor the conversation to your specific preferences and needs. - -- 🎨🤖 **Image Generation Integration**: Seamlessly incorporate image generation capabilities using options such as AUTOMATIC1111 API (local), ComfyUI (local), and DALL-E, enriching your chat experience with dynamic visual content. - -- 🤝 **OpenAI API Integration**: Effortlessly integrate OpenAI-compatible API for versatile conversations alongside Ollama models. Customize the API Base URL to link with **LMStudio, Mistral, OpenRouter, and more**. - -- ✨ **Multiple OpenAI-Compatible API Support**: Seamlessly integrate and customize various OpenAI-compatible APIs, enhancing the versatility of your chat interactions. - -- 🔑 **API Key Generation Support**: Generate secret keys to leverage Open WebUI with OpenAI libraries, simplifying integration and development. - -- 🔗 **External Ollama Server Connection**: Seamlessly link to an external Ollama server hosted on a different address by configuring the environment variable. - -- 🔀 **Multiple Ollama Instance Load Balancing**: Effortlessly distribute chat requests across multiple Ollama instances for enhanced performance and reliability. - -- 👥 **Multi-User Management**: Easily oversee and administer users via our intuitive admin panel, streamlining user management processes. - -- 🔗 **Webhook Integration**: Subscribe to new user sign-up events via webhook (compatible with Google Chat and Microsoft Teams), providing real-time notifications and automation capabilities. - -- 🛡️ **Model Whitelisting**: Admins can whitelist models for users with the 'user' role, enhancing security and access control. - -- 📧 **Trusted Email Authentication**: Authenticate using a trusted email header, adding an additional layer of security and authentication. - - 🔐 **Role-Based Access Control (RBAC)**: Ensure secure access with restricted permissions; only authorized individuals can access your Ollama, and exclusive model creation/pulling rights are reserved for administrators. -- 🔒 **Backend Reverse Proxy Support**: Bolster security through direct communication between Open WebUI backend and Ollama. This key feature eliminates the need to expose Ollama over LAN. Requests made to the '/ollama/api' route from the web UI are seamlessly redirected to Ollama from the backend, enhancing overall system security. - - 🌐🌍 **Multilingual Support**: Experience Open WebUI in your preferred language with our internationalization (i18n) support. Join us in expanding our supported languages! We're actively seeking contributors! -- 🌟 **Continuous Updates**: We are committed to improving Open WebUI with regular updates and new features. +- 🌟 **Continuous Updates**: We are committed to improving Open WebUI with regular updates, fixes, and new features. + +Want to learn more about Open WebUI's features? Check out our [Open WebUI documentation](https://docs.openwebui.com/features) for a comprehensive overview! ## 🔗 Also Check Out Open WebUI Community! diff --git a/backend/apps/ollama/main.py b/backend/apps/ollama/main.py index 01e127074b..82cd8d383b 100644 --- a/backend/apps/ollama/main.py +++ b/backend/apps/ollama/main.py @@ -29,6 +29,8 @@ import time from urllib.parse import urlparse from typing import Optional, List, Union +from starlette.background import BackgroundTask + from apps.webui.models.models import Models from apps.webui.models.users import Users from constants import ERROR_MESSAGES @@ -75,9 +77,6 @@ app.state.config.OLLAMA_BASE_URLS = OLLAMA_BASE_URLS app.state.MODELS = {} -REQUEST_POOL = [] - - # TODO: Implement a more intelligent load balancing mechanism for distributing requests among multiple backend instances. # Current implementation uses a simple round-robin approach (random.choice). Consider incorporating algorithms like weighted round-robin, # least connections, or least response time for better resource utilization and performance optimization. @@ -132,20 +131,10 @@ async def update_ollama_api_url(form_data: UrlUpdateForm, user=Depends(get_admin return {"OLLAMA_BASE_URLS": app.state.config.OLLAMA_BASE_URLS} -@app.get("/cancel/{request_id}") -async def cancel_ollama_request(request_id: str, user=Depends(get_current_user)): - if user: - if request_id in REQUEST_POOL: - REQUEST_POOL.remove(request_id) - return True - else: - raise HTTPException(status_code=401, detail=ERROR_MESSAGES.ACCESS_PROHIBITED) - - async def fetch_url(url): timeout = aiohttp.ClientTimeout(total=5) try: - async with aiohttp.ClientSession(timeout=timeout) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: async with session.get(url) as response: return await response.json() except Exception as e: @@ -154,6 +143,45 @@ async def fetch_url(url): return None +async def cleanup_response( + response: Optional[aiohttp.ClientResponse], + session: Optional[aiohttp.ClientSession], +): + if response: + response.close() + if session: + await session.close() + + +async def post_streaming_url(url: str, payload: str): + r = None + try: + session = aiohttp.ClientSession(trust_env=True) + r = await session.post(url, data=payload) + r.raise_for_status() + + return StreamingResponse( + r.content, + status_code=r.status, + headers=dict(r.headers), + background=BackgroundTask(cleanup_response, response=r, session=session), + ) + except Exception as e: + error_detail = "Open WebUI: Server Connection Error" + if r is not None: + try: + res = await r.json() + if "error" in res: + error_detail = f"Ollama: {res['error']}" + except: + error_detail = f"Ollama: {e}" + + raise HTTPException( + status_code=r.status if r else 500, + detail=error_detail, + ) + + def merge_models_lists(model_lists): merged_models = {} @@ -219,6 +247,8 @@ async def get_ollama_tags( return models else: url = app.state.config.OLLAMA_BASE_URLS[url_idx] + + r = None try: r = requests.request(method="GET", url=f"{url}/api/tags") r.raise_for_status() @@ -244,52 +274,57 @@ async def get_ollama_tags( @app.get("/api/version") @app.get("/api/version/{url_idx}") async def get_ollama_versions(url_idx: Optional[int] = None): + if app.state.config.ENABLE_OLLAMA_API: + if url_idx == None: - if url_idx == None: + # returns lowest version + tasks = [ + fetch_url(f"{url}/api/version") + for url in app.state.config.OLLAMA_BASE_URLS + ] + responses = await asyncio.gather(*tasks) + responses = list(filter(lambda x: x is not None, responses)) - # returns lowest version - tasks = [ - fetch_url(f"{url}/api/version") for url in app.state.config.OLLAMA_BASE_URLS - ] - responses = await asyncio.gather(*tasks) - responses = list(filter(lambda x: x is not None, responses)) + if len(responses) > 0: + lowest_version = min( + responses, + key=lambda x: tuple( + map(int, re.sub(r"^v|-.*", "", x["version"]).split(".")) + ), + ) - if len(responses) > 0: - lowest_version = min( - responses, - key=lambda x: tuple( - map(int, re.sub(r"^v|-.*", "", x["version"]).split(".")) - ), - ) - - return {"version": lowest_version["version"]} + return {"version": lowest_version["version"]} + else: + raise HTTPException( + status_code=500, + detail=ERROR_MESSAGES.OLLAMA_NOT_FOUND, + ) else: - raise HTTPException( - status_code=500, - detail=ERROR_MESSAGES.OLLAMA_NOT_FOUND, - ) + url = app.state.config.OLLAMA_BASE_URLS[url_idx] + + r = None + try: + r = requests.request(method="GET", url=f"{url}/api/version") + r.raise_for_status() + + return r.json() + except Exception as e: + log.exception(e) + error_detail = "Open WebUI: Server Connection Error" + if r is not None: + try: + res = r.json() + if "error" in res: + error_detail = f"Ollama: {res['error']}" + except: + error_detail = f"Ollama: {e}" + + raise HTTPException( + status_code=r.status_code if r else 500, + detail=error_detail, + ) else: - url = app.state.config.OLLAMA_BASE_URLS[url_idx] - try: - r = requests.request(method="GET", url=f"{url}/api/version") - r.raise_for_status() - - return r.json() - except Exception as e: - log.exception(e) - error_detail = "Open WebUI: Server Connection Error" - if r is not None: - try: - res = r.json() - if "error" in res: - error_detail = f"Ollama: {res['error']}" - except: - error_detail = f"Ollama: {e}" - - raise HTTPException( - status_code=r.status_code if r else 500, - detail=error_detail, - ) + return {"version": False} class ModelNameForm(BaseModel): @@ -309,65 +344,7 @@ async def pull_model( # Admin should be able to pull models from any source payload = {**form_data.model_dump(exclude_none=True), "insecure": True} - def get_request(): - nonlocal url - nonlocal r - - request_id = str(uuid.uuid4()) - try: - REQUEST_POOL.append(request_id) - - def stream_content(): - try: - yield json.dumps({"id": request_id, "done": False}) + "\n" - - for chunk in r.iter_content(chunk_size=8192): - if request_id in REQUEST_POOL: - yield chunk - else: - log.warning("User: canceled request") - break - finally: - if hasattr(r, "close"): - r.close() - if request_id in REQUEST_POOL: - REQUEST_POOL.remove(request_id) - - r = requests.request( - method="POST", - url=f"{url}/api/pull", - data=json.dumps(payload), - stream=True, - ) - - r.raise_for_status() - - return StreamingResponse( - stream_content(), - status_code=r.status_code, - headers=dict(r.headers), - ) - except Exception as e: - raise e - - try: - return await run_in_threadpool(get_request) - - except Exception as e: - log.exception(e) - error_detail = "Open WebUI: Server Connection Error" - if r is not None: - try: - res = r.json() - if "error" in res: - error_detail = f"Ollama: {res['error']}" - except: - error_detail = f"Ollama: {e}" - - raise HTTPException( - status_code=r.status_code if r else 500, - detail=error_detail, - ) + return await post_streaming_url(f"{url}/api/pull", json.dumps(payload)) class PushModelForm(BaseModel): @@ -395,50 +372,9 @@ async def push_model( url = app.state.config.OLLAMA_BASE_URLS[url_idx] log.debug(f"url: {url}") - r = None - - def get_request(): - nonlocal url - nonlocal r - try: - - def stream_content(): - for chunk in r.iter_content(chunk_size=8192): - yield chunk - - r = requests.request( - method="POST", - url=f"{url}/api/push", - data=form_data.model_dump_json(exclude_none=True).encode(), - ) - - r.raise_for_status() - - return StreamingResponse( - stream_content(), - status_code=r.status_code, - headers=dict(r.headers), - ) - except Exception as e: - raise e - - try: - return await run_in_threadpool(get_request) - except Exception as e: - log.exception(e) - error_detail = "Open WebUI: Server Connection Error" - if r is not None: - try: - res = r.json() - if "error" in res: - error_detail = f"Ollama: {res['error']}" - except: - error_detail = f"Ollama: {e}" - - raise HTTPException( - status_code=r.status_code if r else 500, - detail=error_detail, - ) + return await post_streaming_url( + f"{url}/api/push", form_data.model_dump_json(exclude_none=True).encode() + ) class CreateModelForm(BaseModel): @@ -457,53 +393,9 @@ async def create_model( url = app.state.config.OLLAMA_BASE_URLS[url_idx] log.info(f"url: {url}") - r = None - - def get_request(): - nonlocal url - nonlocal r - try: - - def stream_content(): - for chunk in r.iter_content(chunk_size=8192): - yield chunk - - r = requests.request( - method="POST", - url=f"{url}/api/create", - data=form_data.model_dump_json(exclude_none=True).encode(), - stream=True, - ) - - r.raise_for_status() - - log.debug(f"r: {r}") - - return StreamingResponse( - stream_content(), - status_code=r.status_code, - headers=dict(r.headers), - ) - except Exception as e: - raise e - - try: - return await run_in_threadpool(get_request) - except Exception as e: - log.exception(e) - error_detail = "Open WebUI: Server Connection Error" - if r is not None: - try: - res = r.json() - if "error" in res: - error_detail = f"Ollama: {res['error']}" - except: - error_detail = f"Ollama: {e}" - - raise HTTPException( - status_code=r.status_code if r else 500, - detail=error_detail, - ) + return await post_streaming_url( + f"{url}/api/create", form_data.model_dump_json(exclude_none=True).encode() + ) class CopyModelForm(BaseModel): @@ -793,66 +685,9 @@ async def generate_completion( url = app.state.config.OLLAMA_BASE_URLS[url_idx] log.info(f"url: {url}") - r = None - - def get_request(): - nonlocal form_data - nonlocal r - - request_id = str(uuid.uuid4()) - try: - REQUEST_POOL.append(request_id) - - def stream_content(): - try: - if form_data.stream: - yield json.dumps({"id": request_id, "done": False}) + "\n" - - for chunk in r.iter_content(chunk_size=8192): - if request_id in REQUEST_POOL: - yield chunk - else: - log.warning("User: canceled request") - break - finally: - if hasattr(r, "close"): - r.close() - if request_id in REQUEST_POOL: - REQUEST_POOL.remove(request_id) - - r = requests.request( - method="POST", - url=f"{url}/api/generate", - data=form_data.model_dump_json(exclude_none=True).encode(), - stream=True, - ) - - r.raise_for_status() - - return StreamingResponse( - stream_content(), - status_code=r.status_code, - headers=dict(r.headers), - ) - except Exception as e: - raise e - - try: - return await run_in_threadpool(get_request) - except Exception as e: - error_detail = "Open WebUI: Server Connection Error" - if r is not None: - try: - res = r.json() - if "error" in res: - error_detail = f"Ollama: {res['error']}" - except: - error_detail = f"Ollama: {e}" - - raise HTTPException( - status_code=r.status_code if r else 500, - detail=error_detail, - ) + return await post_streaming_url( + f"{url}/api/generate", form_data.model_dump_json(exclude_none=True).encode() + ) class ChatMessage(BaseModel): @@ -902,44 +737,77 @@ async def generate_chat_completion( if model_info.params: payload["options"] = {} - payload["options"]["mirostat"] = model_info.params.get("mirostat", None) - payload["options"]["mirostat_eta"] = model_info.params.get( - "mirostat_eta", None - ) - payload["options"]["mirostat_tau"] = model_info.params.get( - "mirostat_tau", None - ) - payload["options"]["num_ctx"] = model_info.params.get("num_ctx", None) + if model_info.params.get("mirostat", None): + payload["options"]["mirostat"] = model_info.params.get("mirostat", None) - payload["options"]["repeat_last_n"] = model_info.params.get( - "repeat_last_n", None - ) - payload["options"]["repeat_penalty"] = model_info.params.get( - "frequency_penalty", None - ) + if model_info.params.get("mirostat_eta", None): + payload["options"]["mirostat_eta"] = model_info.params.get( + "mirostat_eta", None + ) - payload["options"]["temperature"] = model_info.params.get( - "temperature", None - ) - payload["options"]["seed"] = model_info.params.get("seed", None) + if model_info.params.get("mirostat_tau", None): - payload["options"]["stop"] = ( - [ - bytes(stop, "utf-8").decode("unicode_escape") - for stop in model_info.params["stop"] - ] - if model_info.params.get("stop", None) - else None - ) + payload["options"]["mirostat_tau"] = model_info.params.get( + "mirostat_tau", None + ) - payload["options"]["tfs_z"] = model_info.params.get("tfs_z", None) + if model_info.params.get("num_ctx", None): + payload["options"]["num_ctx"] = model_info.params.get("num_ctx", None) - payload["options"]["num_predict"] = model_info.params.get( - "max_tokens", None - ) - payload["options"]["top_k"] = model_info.params.get("top_k", None) + if model_info.params.get("repeat_last_n", None): + payload["options"]["repeat_last_n"] = model_info.params.get( + "repeat_last_n", None + ) - payload["options"]["top_p"] = model_info.params.get("top_p", None) + if model_info.params.get("frequency_penalty", None): + payload["options"]["repeat_penalty"] = model_info.params.get( + "frequency_penalty", None + ) + + if model_info.params.get("temperature", None): + payload["options"]["temperature"] = model_info.params.get( + "temperature", None + ) + + if model_info.params.get("seed", None): + payload["options"]["seed"] = model_info.params.get("seed", None) + + if model_info.params.get("stop", None): + payload["options"]["stop"] = ( + [ + bytes(stop, "utf-8").decode("unicode_escape") + for stop in model_info.params["stop"] + ] + if model_info.params.get("stop", None) + else None + ) + + if model_info.params.get("tfs_z", None): + payload["options"]["tfs_z"] = model_info.params.get("tfs_z", None) + + if model_info.params.get("max_tokens", None): + payload["options"]["num_predict"] = model_info.params.get( + "max_tokens", None + ) + + if model_info.params.get("top_k", None): + payload["options"]["top_k"] = model_info.params.get("top_k", None) + + if model_info.params.get("top_p", None): + payload["options"]["top_p"] = model_info.params.get("top_p", None) + + if model_info.params.get("use_mmap", None): + payload["options"]["use_mmap"] = model_info.params.get("use_mmap", None) + + if model_info.params.get("use_mlock", None): + payload["options"]["use_mlock"] = model_info.params.get( + "use_mlock", None + ) + + if model_info.params.get("num_thread", None): + payload["options"]["num_thread"] = model_info.params.get( + "num_thread", None + ) if model_info.params.get("system", None): # Check if the payload already has a system message @@ -977,67 +845,7 @@ async def generate_chat_completion( print(payload) - r = None - - def get_request(): - nonlocal payload - nonlocal r - - request_id = str(uuid.uuid4()) - try: - REQUEST_POOL.append(request_id) - - def stream_content(): - try: - if payload.get("stream", None): - yield json.dumps({"id": request_id, "done": False}) + "\n" - - for chunk in r.iter_content(chunk_size=8192): - if request_id in REQUEST_POOL: - yield chunk - else: - log.warning("User: canceled request") - break - finally: - if hasattr(r, "close"): - r.close() - if request_id in REQUEST_POOL: - REQUEST_POOL.remove(request_id) - - r = requests.request( - method="POST", - url=f"{url}/api/chat", - data=json.dumps(payload), - stream=True, - ) - - r.raise_for_status() - - return StreamingResponse( - stream_content(), - status_code=r.status_code, - headers=dict(r.headers), - ) - except Exception as e: - log.exception(e) - raise e - - try: - return await run_in_threadpool(get_request) - except Exception as e: - error_detail = "Open WebUI: Server Connection Error" - if r is not None: - try: - res = r.json() - if "error" in res: - error_detail = f"Ollama: {res['error']}" - except: - error_detail = f"Ollama: {e}" - - raise HTTPException( - status_code=r.status_code if r else 500, - detail=error_detail, - ) + return await post_streaming_url(f"{url}/api/chat", json.dumps(payload)) # TODO: we should update this part once Ollama supports other types @@ -1128,68 +936,7 @@ async def generate_openai_chat_completion( url = app.state.config.OLLAMA_BASE_URLS[url_idx] log.info(f"url: {url}") - r = None - - def get_request(): - nonlocal payload - nonlocal r - - request_id = str(uuid.uuid4()) - try: - REQUEST_POOL.append(request_id) - - def stream_content(): - try: - if payload.get("stream"): - yield json.dumps( - {"request_id": request_id, "done": False} - ) + "\n" - - for chunk in r.iter_content(chunk_size=8192): - if request_id in REQUEST_POOL: - yield chunk - else: - log.warning("User: canceled request") - break - finally: - if hasattr(r, "close"): - r.close() - if request_id in REQUEST_POOL: - REQUEST_POOL.remove(request_id) - - r = requests.request( - method="POST", - url=f"{url}/v1/chat/completions", - data=json.dumps(payload), - stream=True, - ) - - r.raise_for_status() - - return StreamingResponse( - stream_content(), - status_code=r.status_code, - headers=dict(r.headers), - ) - except Exception as e: - raise e - - try: - return await run_in_threadpool(get_request) - except Exception as e: - error_detail = "Open WebUI: Server Connection Error" - if r is not None: - try: - res = r.json() - if "error" in res: - error_detail = f"Ollama: {res['error']}" - except: - error_detail = f"Ollama: {e}" - - raise HTTPException( - status_code=r.status_code if r else 500, - detail=error_detail, - ) + return await post_streaming_url(f"{url}/v1/chat/completions", json.dumps(payload)) @app.get("/v1/models") @@ -1301,7 +1048,7 @@ async def download_file_stream( timeout = aiohttp.ClientTimeout(total=600) # Set the timeout - async with aiohttp.ClientSession(timeout=timeout) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: async with session.get(file_url, headers=headers) as response: total_size = int(response.headers.get("content-length", 0)) + current_size @@ -1518,7 +1265,7 @@ async def deprecated_proxy( if path == "generate": data = json.loads(body.decode("utf-8")) - if not ("stream" in data and data["stream"] == False): + if data.get("stream", True): yield json.dumps({"id": request_id, "done": False}) + "\n" elif path == "chat": diff --git a/backend/apps/openai/main.py b/backend/apps/openai/main.py index e19e0bbcf8..472699f1d2 100644 --- a/backend/apps/openai/main.py +++ b/backend/apps/openai/main.py @@ -9,6 +9,7 @@ import json import logging from pydantic import BaseModel +from starlette.background import BackgroundTask from apps.webui.models.models import Models from apps.webui.models.users import Users @@ -184,19 +185,26 @@ async def speech(request: Request, user=Depends(get_verified_user)): async def fetch_url(url, key): timeout = aiohttp.ClientTimeout(total=5) try: - if key != "": - headers = {"Authorization": f"Bearer {key}"} - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.get(url, headers=headers) as response: - return await response.json() - else: - return None + headers = {"Authorization": f"Bearer {key}"} + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with session.get(url, headers=headers) as response: + return await response.json() except Exception as e: # Handle connection error here log.error(f"Connection error: {e}") return None +async def cleanup_response( + response: Optional[aiohttp.ClientResponse], + session: Optional[aiohttp.ClientSession], +): + if response: + response.close() + if session: + await session.close() + + def merge_models_lists(model_lists): log.debug(f"merge_models_lists {model_lists}") merged_list = [] @@ -222,7 +230,7 @@ def merge_models_lists(model_lists): return merged_list -async def get_all_models(): +async def get_all_models(raw: bool = False): log.info("get_all_models()") if ( @@ -231,6 +239,27 @@ async def get_all_models(): ) or not app.state.config.ENABLE_OPENAI_API: models = {"data": []} else: + # Check if API KEYS length is same than API URLS length + if len(app.state.config.OPENAI_API_KEYS) != len( + app.state.config.OPENAI_API_BASE_URLS + ): + # if there are more keys than urls, remove the extra keys + if len(app.state.config.OPENAI_API_KEYS) > len( + app.state.config.OPENAI_API_BASE_URLS + ): + app.state.config.OPENAI_API_KEYS = app.state.config.OPENAI_API_KEYS[ + : len(app.state.config.OPENAI_API_BASE_URLS) + ] + # if there are more urls than keys, add empty keys + else: + app.state.config.OPENAI_API_KEYS += [ + "" + for _ in range( + len(app.state.config.OPENAI_API_BASE_URLS) + - len(app.state.config.OPENAI_API_KEYS) + ) + ] + tasks = [ fetch_url(f"{url}/models", app.state.config.OPENAI_API_KEYS[idx]) for idx, url in enumerate(app.state.config.OPENAI_API_BASE_URLS) @@ -239,6 +268,9 @@ async def get_all_models(): responses = await asyncio.gather(*tasks) log.debug(f"get_all_models:responses() {responses}") + if raw: + return responses + models = { "data": merge_models_lists( list( @@ -277,11 +309,16 @@ async def get_models(url_idx: Optional[int] = None, user=Depends(get_current_use return models else: url = app.state.config.OPENAI_API_BASE_URLS[url_idx] + key = app.state.config.OPENAI_API_KEYS[url_idx] + + headers = {} + headers["Authorization"] = f"Bearer {key}" + headers["Content-Type"] = "application/json" r = None try: - r = requests.request(method="GET", url=f"{url}/models") + r = requests.request(method="GET", url=f"{url}/models", headers=headers) r.raise_for_status() response_data = r.json() @@ -336,21 +373,36 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)): model_info.params = model_info.params.model_dump() if model_info.params: - payload["temperature"] = model_info.params.get("temperature", None) - payload["top_p"] = model_info.params.get("top_p", None) - payload["max_tokens"] = model_info.params.get("max_tokens", None) - payload["frequency_penalty"] = model_info.params.get( - "frequency_penalty", None - ) - payload["seed"] = model_info.params.get("seed", None) - payload["stop"] = ( - [ - bytes(stop, "utf-8").decode("unicode_escape") - for stop in model_info.params["stop"] - ] - if model_info.params.get("stop", None) - else None - ) + if model_info.params.get("temperature", None): + payload["temperature"] = int( + model_info.params.get("temperature") + ) + + if model_info.params.get("top_p", None): + payload["top_p"] = int(model_info.params.get("top_p", None)) + + if model_info.params.get("max_tokens", None): + payload["max_tokens"] = int( + model_info.params.get("max_tokens", None) + ) + + if model_info.params.get("frequency_penalty", None): + payload["frequency_penalty"] = int( + model_info.params.get("frequency_penalty", None) + ) + + if model_info.params.get("seed", None): + payload["seed"] = model_info.params.get("seed", None) + + if model_info.params.get("stop", None): + payload["stop"] = ( + [ + bytes(stop, "utf-8").decode("unicode_escape") + for stop in model_info.params["stop"] + ] + if model_info.params.get("stop", None) + else None + ) if model_info.params.get("system", None): # Check if the payload already has a system message @@ -374,18 +426,12 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)): else: pass - print(app.state.MODELS) model = app.state.MODELS[payload.get("model")] idx = model["urlIdx"] if "pipeline" in model and model.get("pipeline"): payload["user"] = {"name": user.name, "id": user.id} - payload["title"] = ( - True - if payload["stream"] == False and payload["max_tokens"] == 50 - else False - ) # Check if the model is "gpt-4-vision-preview" and set "max_tokens" to 4000 # This is a workaround until OpenAI fixes the issue with this model @@ -407,47 +453,53 @@ async def proxy(path: str, request: Request, user=Depends(get_verified_user)): target_url = f"{url}/{path}" - if key == "": - raise HTTPException(status_code=401, detail=ERROR_MESSAGES.API_KEY_NOT_FOUND) - headers = {} headers["Authorization"] = f"Bearer {key}" headers["Content-Type"] = "application/json" r = None + session = None + streaming = False try: - r = requests.request( + session = aiohttp.ClientSession(trust_env=True) + r = await session.request( method=request.method, url=target_url, data=payload if payload else body, headers=headers, - stream=True, ) r.raise_for_status() # Check if response is SSE if "text/event-stream" in r.headers.get("Content-Type", ""): + streaming = True return StreamingResponse( - r.iter_content(chunk_size=8192), - status_code=r.status_code, + r.content, + status_code=r.status, headers=dict(r.headers), + background=BackgroundTask( + cleanup_response, response=r, session=session + ), ) else: - response_data = r.json() + response_data = await r.json() return response_data except Exception as e: log.exception(e) error_detail = "Open WebUI: Server Connection Error" if r is not None: try: - res = r.json() + res = await r.json() + print(res) if "error" in res: error_detail = f"External: {res['error']['message'] if 'message' in res['error'] else res['error']}" except: error_detail = f"External: {e}" - - raise HTTPException( - status_code=r.status_code if r else 500, detail=error_detail - ) + raise HTTPException(status_code=r.status if r else 500, detail=error_detail) + finally: + if not streaming and session: + if r: + r.close() + await session.close() diff --git a/backend/apps/rag/main.py b/backend/apps/rag/main.py index d04c256d78..d405ef0b4e 100644 --- a/backend/apps/rag/main.py +++ b/backend/apps/rag/main.py @@ -11,7 +11,7 @@ from fastapi.middleware.cors import CORSMiddleware import os, shutil, logging, re from pathlib import Path -from typing import List +from typing import List, Union, Sequence from chromadb.utils.batch_utils import create_batches @@ -61,6 +61,14 @@ from apps.rag.utils import ( query_collection_with_hybrid_search, ) +from apps.rag.search.brave import search_brave +from apps.rag.search.google_pse import search_google_pse +from apps.rag.search.main import SearchResult +from apps.rag.search.searxng import search_searxng +from apps.rag.search.serper import search_serper +from apps.rag.search.serpstack import search_serpstack + + from utils.misc import ( calculate_sha256, calculate_sha256_string, @@ -70,6 +78,7 @@ from utils.misc import ( from utils.utils import get_current_user, get_admin_user from config import ( + AppConfig, ENV, SRC_LOG_LEVELS, UPLOAD_DIR, @@ -95,7 +104,18 @@ from config import ( RAG_TEMPLATE, ENABLE_RAG_LOCAL_WEB_FETCH, YOUTUBE_LOADER_LANGUAGE, - AppConfig, + ENABLE_RAG_WEB_SEARCH, + RAG_WEB_SEARCH_ENGINE, + SEARXNG_QUERY_URL, + GOOGLE_PSE_API_KEY, + GOOGLE_PSE_ENGINE_ID, + BRAVE_SEARCH_API_KEY, + SERPSTACK_API_KEY, + SERPSTACK_HTTPS, + SERPER_API_KEY, + RAG_WEB_SEARCH_RESULT_COUNT, + RAG_WEB_SEARCH_CONCURRENT_REQUESTS, + RAG_EMBEDDING_OPENAI_BATCH_SIZE, ) from constants import ERROR_MESSAGES @@ -120,6 +140,7 @@ app.state.config.CHUNK_OVERLAP = CHUNK_OVERLAP app.state.config.RAG_EMBEDDING_ENGINE = RAG_EMBEDDING_ENGINE app.state.config.RAG_EMBEDDING_MODEL = RAG_EMBEDDING_MODEL +app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE = RAG_EMBEDDING_OPENAI_BATCH_SIZE app.state.config.RAG_RERANKING_MODEL = RAG_RERANKING_MODEL app.state.config.RAG_TEMPLATE = RAG_TEMPLATE @@ -134,6 +155,20 @@ app.state.config.YOUTUBE_LOADER_LANGUAGE = YOUTUBE_LOADER_LANGUAGE app.state.YOUTUBE_LOADER_TRANSLATION = None +app.state.config.ENABLE_RAG_WEB_SEARCH = ENABLE_RAG_WEB_SEARCH +app.state.config.RAG_WEB_SEARCH_ENGINE = RAG_WEB_SEARCH_ENGINE + +app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL +app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY +app.state.config.GOOGLE_PSE_ENGINE_ID = GOOGLE_PSE_ENGINE_ID +app.state.config.BRAVE_SEARCH_API_KEY = BRAVE_SEARCH_API_KEY +app.state.config.SERPSTACK_API_KEY = SERPSTACK_API_KEY +app.state.config.SERPSTACK_HTTPS = SERPSTACK_HTTPS +app.state.config.SERPER_API_KEY = SERPER_API_KEY +app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT +app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS + + def update_embedding_model( embedding_model: str, update_model: bool = False, @@ -179,6 +214,7 @@ app.state.EMBEDDING_FUNCTION = get_embedding_function( app.state.sentence_transformer_ef, app.state.config.OPENAI_API_KEY, app.state.config.OPENAI_API_BASE_URL, + app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, ) origins = ["*"] @@ -201,6 +237,10 @@ class UrlForm(CollectionNameForm): url: str +class SearchForm(CollectionNameForm): + query: str + + @app.get("/") async def get_status(): return { @@ -211,6 +251,7 @@ async def get_status(): "embedding_engine": app.state.config.RAG_EMBEDDING_ENGINE, "embedding_model": app.state.config.RAG_EMBEDDING_MODEL, "reranking_model": app.state.config.RAG_RERANKING_MODEL, + "openai_batch_size": app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, } @@ -223,6 +264,7 @@ async def get_embedding_config(user=Depends(get_admin_user)): "openai_config": { "url": app.state.config.OPENAI_API_BASE_URL, "key": app.state.config.OPENAI_API_KEY, + "batch_size": app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, }, } @@ -238,6 +280,7 @@ async def get_reraanking_config(user=Depends(get_admin_user)): class OpenAIConfigForm(BaseModel): url: str key: str + batch_size: Optional[int] = None class EmbeddingModelUpdateForm(BaseModel): @@ -258,9 +301,14 @@ async def update_embedding_config( app.state.config.RAG_EMBEDDING_MODEL = form_data.embedding_model if app.state.config.RAG_EMBEDDING_ENGINE in ["ollama", "openai"]: - if form_data.openai_config != None: + if form_data.openai_config is not None: app.state.config.OPENAI_API_BASE_URL = form_data.openai_config.url app.state.config.OPENAI_API_KEY = form_data.openai_config.key + app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE = ( + form_data.openai_config.batch_size + if form_data.openai_config.batch_size + else 1 + ) update_embedding_model(app.state.config.RAG_EMBEDDING_MODEL) @@ -270,6 +318,7 @@ async def update_embedding_config( app.state.sentence_transformer_ef, app.state.config.OPENAI_API_KEY, app.state.config.OPENAI_API_BASE_URL, + app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, ) return { @@ -279,6 +328,7 @@ async def update_embedding_config( "openai_config": { "url": app.state.config.OPENAI_API_BASE_URL, "key": app.state.config.OPENAI_API_KEY, + "batch_size": app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, }, } except Exception as e: @@ -326,11 +376,26 @@ async def get_rag_config(user=Depends(get_admin_user)): "chunk_size": app.state.config.CHUNK_SIZE, "chunk_overlap": app.state.config.CHUNK_OVERLAP, }, - "web_loader_ssl_verification": app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION, "youtube": { "language": app.state.config.YOUTUBE_LOADER_LANGUAGE, "translation": app.state.YOUTUBE_LOADER_TRANSLATION, }, + "web": { + "ssl_verification": app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION, + "search": { + "enabled": app.state.config.ENABLE_RAG_WEB_SEARCH, + "engine": app.state.config.RAG_WEB_SEARCH_ENGINE, + "searxng_query_url": app.state.config.SEARXNG_QUERY_URL, + "google_pse_api_key": app.state.config.GOOGLE_PSE_API_KEY, + "google_pse_engine_id": app.state.config.GOOGLE_PSE_ENGINE_ID, + "brave_search_api_key": app.state.config.BRAVE_SEARCH_API_KEY, + "serpstack_api_key": app.state.config.SERPSTACK_API_KEY, + "serpstack_https": app.state.config.SERPSTACK_HTTPS, + "serper_api_key": app.state.config.SERPER_API_KEY, + "result_count": app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + "concurrent_requests": app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS, + }, + }, } @@ -344,11 +409,30 @@ class YoutubeLoaderConfig(BaseModel): translation: Optional[str] = None +class WebSearchConfig(BaseModel): + enabled: bool + engine: Optional[str] = None + searxng_query_url: Optional[str] = None + google_pse_api_key: Optional[str] = None + google_pse_engine_id: Optional[str] = None + brave_search_api_key: Optional[str] = None + serpstack_api_key: Optional[str] = None + serpstack_https: Optional[bool] = None + serper_api_key: Optional[str] = None + result_count: Optional[int] = None + concurrent_requests: Optional[int] = None + + +class WebConfig(BaseModel): + search: WebSearchConfig + web_loader_ssl_verification: Optional[bool] = None + + class ConfigUpdateForm(BaseModel): pdf_extract_images: Optional[bool] = None chunk: Optional[ChunkParamUpdateForm] = None - web_loader_ssl_verification: Optional[bool] = None youtube: Optional[YoutubeLoaderConfig] = None + web: Optional[WebConfig] = None @app.post("/config/update") @@ -359,35 +443,36 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_ else app.state.config.PDF_EXTRACT_IMAGES ) - app.state.config.CHUNK_SIZE = ( - form_data.chunk.chunk_size - if form_data.chunk is not None - else app.state.config.CHUNK_SIZE - ) + if form_data.chunk is not None: + app.state.config.CHUNK_SIZE = form_data.chunk.chunk_size + app.state.config.CHUNK_OVERLAP = form_data.chunk.chunk_overlap - app.state.config.CHUNK_OVERLAP = ( - form_data.chunk.chunk_overlap - if form_data.chunk is not None - else app.state.config.CHUNK_OVERLAP - ) + if form_data.youtube is not None: + app.state.config.YOUTUBE_LOADER_LANGUAGE = form_data.youtube.language + app.state.YOUTUBE_LOADER_TRANSLATION = form_data.youtube.translation - app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = ( - form_data.web_loader_ssl_verification - if form_data.web_loader_ssl_verification != None - else app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION - ) + if form_data.web is not None: + app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = ( + form_data.web.web_loader_ssl_verification + ) - app.state.config.YOUTUBE_LOADER_LANGUAGE = ( - form_data.youtube.language - if form_data.youtube is not None - else app.state.config.YOUTUBE_LOADER_LANGUAGE - ) - - app.state.YOUTUBE_LOADER_TRANSLATION = ( - form_data.youtube.translation - if form_data.youtube is not None - else app.state.YOUTUBE_LOADER_TRANSLATION - ) + app.state.config.ENABLE_RAG_WEB_SEARCH = form_data.web.search.enabled + app.state.config.RAG_WEB_SEARCH_ENGINE = form_data.web.search.engine + app.state.config.SEARXNG_QUERY_URL = form_data.web.search.searxng_query_url + app.state.config.GOOGLE_PSE_API_KEY = form_data.web.search.google_pse_api_key + app.state.config.GOOGLE_PSE_ENGINE_ID = ( + form_data.web.search.google_pse_engine_id + ) + app.state.config.BRAVE_SEARCH_API_KEY = ( + form_data.web.search.brave_search_api_key + ) + app.state.config.SERPSTACK_API_KEY = form_data.web.search.serpstack_api_key + app.state.config.SERPSTACK_HTTPS = form_data.web.search.serpstack_https + app.state.config.SERPER_API_KEY = form_data.web.search.serper_api_key + app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = form_data.web.search.result_count + app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = ( + form_data.web.search.concurrent_requests + ) return { "status": True, @@ -396,11 +481,26 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_ "chunk_size": app.state.config.CHUNK_SIZE, "chunk_overlap": app.state.config.CHUNK_OVERLAP, }, - "web_loader_ssl_verification": app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION, "youtube": { "language": app.state.config.YOUTUBE_LOADER_LANGUAGE, "translation": app.state.YOUTUBE_LOADER_TRANSLATION, }, + "web": { + "ssl_verification": app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION, + "search": { + "enabled": app.state.config.ENABLE_RAG_WEB_SEARCH, + "engine": app.state.config.RAG_WEB_SEARCH_ENGINE, + "searxng_query_url": app.state.config.SEARXNG_QUERY_URL, + "google_pse_api_key": app.state.config.GOOGLE_PSE_API_KEY, + "google_pse_engine_id": app.state.config.GOOGLE_PSE_ENGINE_ID, + "brave_search_api_key": app.state.config.BRAVE_SEARCH_API_KEY, + "serpstack_api_key": app.state.config.SERPSTACK_API_KEY, + "serpstack_https": app.state.config.SERPSTACK_HTTPS, + "serper_api_key": app.state.config.SERPER_API_KEY, + "result_count": app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + "concurrent_requests": app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS, + }, + }, } @@ -589,24 +689,40 @@ def store_web(form_data: UrlForm, user=Depends(get_current_user)): ) -def get_web_loader(url: str, verify_ssl: bool = True): +def get_web_loader(url: Union[str, Sequence[str]], verify_ssl: bool = True): # Check if the URL is valid - if isinstance(validators.url(url), validators.ValidationError): + if not validate_url(url): raise ValueError(ERROR_MESSAGES.INVALID_URL) - if not ENABLE_RAG_LOCAL_WEB_FETCH: - # Local web fetch is disabled, filter out any URLs that resolve to private IP addresses - parsed_url = urllib.parse.urlparse(url) - # Get IPv4 and IPv6 addresses - ipv4_addresses, ipv6_addresses = resolve_hostname(parsed_url.hostname) - # Check if any of the resolved addresses are private - # This is technically still vulnerable to DNS rebinding attacks, as we don't control WebBaseLoader - for ip in ipv4_addresses: - if validators.ipv4(ip, private=True): - raise ValueError(ERROR_MESSAGES.INVALID_URL) - for ip in ipv6_addresses: - if validators.ipv6(ip, private=True): - raise ValueError(ERROR_MESSAGES.INVALID_URL) - return WebBaseLoader(url, verify_ssl=verify_ssl) + return WebBaseLoader( + url, + verify_ssl=verify_ssl, + requests_per_second=RAG_WEB_SEARCH_CONCURRENT_REQUESTS, + continue_on_failure=True, + ) + + +def validate_url(url: Union[str, Sequence[str]]): + if isinstance(url, str): + if isinstance(validators.url(url), validators.ValidationError): + raise ValueError(ERROR_MESSAGES.INVALID_URL) + if not ENABLE_RAG_LOCAL_WEB_FETCH: + # Local web fetch is disabled, filter out any URLs that resolve to private IP addresses + parsed_url = urllib.parse.urlparse(url) + # Get IPv4 and IPv6 addresses + ipv4_addresses, ipv6_addresses = resolve_hostname(parsed_url.hostname) + # Check if any of the resolved addresses are private + # This is technically still vulnerable to DNS rebinding attacks, as we don't control WebBaseLoader + for ip in ipv4_addresses: + if validators.ipv4(ip, private=True): + raise ValueError(ERROR_MESSAGES.INVALID_URL) + for ip in ipv6_addresses: + if validators.ipv6(ip, private=True): + raise ValueError(ERROR_MESSAGES.INVALID_URL) + return True + elif isinstance(url, Sequence): + return all(validate_url(u) for u in url) + else: + return False def resolve_hostname(hostname): @@ -620,6 +736,114 @@ def resolve_hostname(hostname): return ipv4_addresses, ipv6_addresses +def search_web(engine: str, query: str) -> list[SearchResult]: + """Search the web using a search engine and return the results as a list of SearchResult objects. + Will look for a search engine API key in environment variables in the following order: + - SEARXNG_QUERY_URL + - GOOGLE_PSE_API_KEY + GOOGLE_PSE_ENGINE_ID + - BRAVE_SEARCH_API_KEY + - SERPSTACK_API_KEY + - SERPER_API_KEY + + Args: + query (str): The query to search for + """ + + # TODO: add playwright to search the web + if engine == "searxng": + if app.state.config.SEARXNG_QUERY_URL: + return search_searxng( + app.state.config.SEARXNG_QUERY_URL, + query, + app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + ) + else: + raise Exception("No SEARXNG_QUERY_URL found in environment variables") + elif engine == "google_pse": + if ( + app.state.config.GOOGLE_PSE_API_KEY + and app.state.config.GOOGLE_PSE_ENGINE_ID + ): + return search_google_pse( + app.state.config.GOOGLE_PSE_API_KEY, + app.state.config.GOOGLE_PSE_ENGINE_ID, + query, + app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + ) + else: + raise Exception( + "No GOOGLE_PSE_API_KEY or GOOGLE_PSE_ENGINE_ID found in environment variables" + ) + elif engine == "brave": + if app.state.config.BRAVE_SEARCH_API_KEY: + return search_brave( + app.state.config.BRAVE_SEARCH_API_KEY, + query, + app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + ) + else: + raise Exception("No BRAVE_SEARCH_API_KEY found in environment variables") + elif engine == "serpstack": + if app.state.config.SERPSTACK_API_KEY: + return search_serpstack( + app.state.config.SERPSTACK_API_KEY, + query, + app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + https_enabled=app.state.config.SERPSTACK_HTTPS, + ) + else: + raise Exception("No SERPSTACK_API_KEY found in environment variables") + elif engine == "serper": + if app.state.config.SERPER_API_KEY: + return search_serper( + app.state.config.SERPER_API_KEY, + query, + app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + ) + else: + raise Exception("No SERPER_API_KEY found in environment variables") + else: + raise Exception("No search engine API key found in environment variables") + + +@app.post("/web/search") +def store_web_search(form_data: SearchForm, user=Depends(get_current_user)): + try: + web_results = search_web( + app.state.config.RAG_WEB_SEARCH_ENGINE, form_data.query + ) + except Exception as e: + log.exception(e) + + print(e) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.WEB_SEARCH_ERROR(e), + ) + + try: + urls = [result.link for result in web_results] + loader = get_web_loader(urls) + data = loader.load() + + collection_name = form_data.collection_name + if collection_name == "": + collection_name = calculate_sha256_string(form_data.query)[:63] + + store_data_in_vector_db(data, collection_name, overwrite=True) + return { + "status": True, + "collection_name": collection_name, + "filenames": urls, + } + except Exception as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.DEFAULT(e), + ) + + def store_data_in_vector_db(data, collection_name, overwrite: bool = False) -> bool: text_splitter = RecursiveCharacterTextSplitter( @@ -670,6 +894,7 @@ def store_docs_in_vector_db(docs, collection_name, overwrite: bool = False) -> b app.state.sentence_transformer_ef, app.state.config.OPENAI_API_KEY, app.state.config.OPENAI_API_BASE_URL, + app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, ) embedding_texts = list(map(lambda x: x.replace("\n", " "), texts)) @@ -939,6 +1164,30 @@ def reset_vector_db(user=Depends(get_admin_user)): CHROMA_CLIENT.reset() +@app.get("/reset/uploads") +def reset_upload_dir(user=Depends(get_admin_user)) -> bool: + folder = f"{UPLOAD_DIR}" + try: + # Check if the directory exists + if os.path.exists(folder): + # Iterate over all the files and directories in the specified directory + for filename in os.listdir(folder): + file_path = os.path.join(folder, filename) + try: + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) # Remove the file or link + elif os.path.isdir(file_path): + shutil.rmtree(file_path) # Remove the directory + except Exception as e: + print(f"Failed to delete {file_path}. Reason: {e}") + else: + print(f"The directory {folder} does not exist") + except Exception as e: + print(f"Failed to process the directory {folder}. Reason: {e}") + + return True + + @app.get("/reset") def reset(user=Depends(get_admin_user)) -> bool: folder = f"{UPLOAD_DIR}" diff --git a/backend/apps/rag/search/brave.py b/backend/apps/rag/search/brave.py new file mode 100644 index 0000000000..4e0f56807f --- /dev/null +++ b/backend/apps/rag/search/brave.py @@ -0,0 +1,37 @@ +import logging + +import requests + +from apps.rag.search.main import SearchResult +from config import SRC_LOG_LEVELS + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["RAG"]) + + +def search_brave(api_key: str, query: str, count: int) -> list[SearchResult]: + """Search using Brave's Search API and return the results as a list of SearchResult objects. + + Args: + api_key (str): A Brave Search API key + query (str): The query to search for + """ + url = "https://api.search.brave.com/res/v1/web/search" + headers = { + "Accept": "application/json", + "Accept-Encoding": "gzip", + "X-Subscription-Token": api_key, + } + params = {"q": query, "count": count} + + response = requests.get(url, headers=headers, params=params) + response.raise_for_status() + + json_response = response.json() + results = json_response.get("web", {}).get("results", []) + return [ + SearchResult( + link=result["url"], title=result.get("title"), snippet=result.get("snippet") + ) + for result in results[:count] + ] diff --git a/backend/apps/rag/search/google_pse.py b/backend/apps/rag/search/google_pse.py new file mode 100644 index 0000000000..7ff54c7850 --- /dev/null +++ b/backend/apps/rag/search/google_pse.py @@ -0,0 +1,45 @@ +import json +import logging + +import requests + +from apps.rag.search.main import SearchResult +from config import SRC_LOG_LEVELS + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["RAG"]) + + +def search_google_pse( + api_key: str, search_engine_id: str, query: str, count: int +) -> list[SearchResult]: + """Search using Google's Programmable Search Engine API and return the results as a list of SearchResult objects. + + Args: + api_key (str): A Programmable Search Engine API key + search_engine_id (str): A Programmable Search Engine ID + query (str): The query to search for + """ + url = "https://www.googleapis.com/customsearch/v1" + + headers = {"Content-Type": "application/json"} + params = { + "cx": search_engine_id, + "q": query, + "key": api_key, + "num": count, + } + + response = requests.request("GET", url, headers=headers, params=params) + response.raise_for_status() + + json_response = response.json() + results = json_response.get("items", []) + return [ + SearchResult( + link=result["link"], + title=result.get("title"), + snippet=result.get("snippet"), + ) + for result in results + ] diff --git a/backend/apps/rag/search/main.py b/backend/apps/rag/search/main.py new file mode 100644 index 0000000000..b5478f9496 --- /dev/null +++ b/backend/apps/rag/search/main.py @@ -0,0 +1,9 @@ +from typing import Optional + +from pydantic import BaseModel + + +class SearchResult(BaseModel): + link: str + title: Optional[str] + snippet: Optional[str] diff --git a/backend/apps/rag/search/searxng.py b/backend/apps/rag/search/searxng.py new file mode 100644 index 0000000000..a62ab5089c --- /dev/null +++ b/backend/apps/rag/search/searxng.py @@ -0,0 +1,83 @@ +import logging +import requests + +from typing import List + +from apps.rag.search.main import SearchResult +from config import SRC_LOG_LEVELS + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["RAG"]) + + +def search_searxng( + query_url: str, query: str, count: int, **kwargs +) -> List[SearchResult]: + """ + Search a SearXNG instance for a given query and return the results as a list of SearchResult objects. + + The function allows passing additional parameters such as language or time_range to tailor the search result. + + Args: + query_url (str): The base URL of the SearXNG server. + query (str): The search term or question to find in the SearXNG database. + count (int): The maximum number of results to retrieve from the search. + + Keyword Args: + language (str): Language filter for the search results; e.g., "en-US". Defaults to an empty string. + time_range (str): Time range for filtering results by date; e.g., "2023-04-05..today" or "all-time". Defaults to ''. + categories: (Optional[List[str]]): Specific categories within which the search should be performed, defaulting to an empty string if not provided. + + Returns: + List[SearchResult]: A list of SearchResults sorted by relevance score in descending order. + + Raise: + requests.exceptions.RequestException: If a request error occurs during the search process. + """ + + # Default values for optional parameters are provided as empty strings or None when not specified. + language = kwargs.get("language", "en-US") + time_range = kwargs.get("time_range", "") + categories = "".join(kwargs.get("categories", [])) + + params = { + "q": query, + "format": "json", + "pageno": 1, + "language": language, + "time_range": time_range, + "categories": categories, + "theme": "simple", + "image_proxy": 0, + } + + # Legacy query format + if "" in query_url: + # Strip all query parameters from the URL + query_url = query_url.split("?")[0] + + log.debug(f"searching {query_url}") + + response = requests.get( + query_url, + headers={ + "User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot", + "Accept": "text/html", + "Accept-Encoding": "gzip, deflate", + "Accept-Language": "en-US,en;q=0.5", + "Connection": "keep-alive", + }, + params=params, + ) + + response.raise_for_status() # Raise an exception for HTTP errors. + + json_response = response.json() + results = json_response.get("results", []) + sorted_results = sorted(results, key=lambda x: x.get("score", 0), reverse=True) + return [ + SearchResult( + link=result["url"], title=result.get("title"), snippet=result.get("content") + ) + for result in sorted_results[:count] + ] diff --git a/backend/apps/rag/search/serper.py b/backend/apps/rag/search/serper.py new file mode 100644 index 0000000000..150da6e074 --- /dev/null +++ b/backend/apps/rag/search/serper.py @@ -0,0 +1,39 @@ +import json +import logging + +import requests + +from apps.rag.search.main import SearchResult +from config import SRC_LOG_LEVELS + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["RAG"]) + + +def search_serper(api_key: str, query: str, count: int) -> list[SearchResult]: + """Search using serper.dev's API and return the results as a list of SearchResult objects. + + Args: + api_key (str): A serper.dev API key + query (str): The query to search for + """ + url = "https://google.serper.dev/search" + + payload = json.dumps({"q": query}) + headers = {"X-API-KEY": api_key, "Content-Type": "application/json"} + + response = requests.request("POST", url, headers=headers, data=payload) + response.raise_for_status() + + json_response = response.json() + results = sorted( + json_response.get("organic", []), key=lambda x: x.get("position", 0) + ) + return [ + SearchResult( + link=result["link"], + title=result.get("title"), + snippet=result.get("description"), + ) + for result in results[:count] + ] diff --git a/backend/apps/rag/search/serpstack.py b/backend/apps/rag/search/serpstack.py new file mode 100644 index 0000000000..0d247d1ab0 --- /dev/null +++ b/backend/apps/rag/search/serpstack.py @@ -0,0 +1,43 @@ +import json +import logging + +import requests + +from apps.rag.search.main import SearchResult +from config import SRC_LOG_LEVELS + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["RAG"]) + + +def search_serpstack( + api_key: str, query: str, count: int, https_enabled: bool = True +) -> list[SearchResult]: + """Search using serpstack.com's and return the results as a list of SearchResult objects. + + Args: + api_key (str): A serpstack.com API key + query (str): The query to search for + https_enabled (bool): Whether to use HTTPS or HTTP for the API request + """ + url = f"{'https' if https_enabled else 'http'}://api.serpstack.com/search" + + headers = {"Content-Type": "application/json"} + params = { + "access_key": api_key, + "query": query, + } + + response = requests.request("POST", url, headers=headers, params=params) + response.raise_for_status() + + json_response = response.json() + results = sorted( + json_response.get("organic_results", []), key=lambda x: x.get("position", 0) + ) + return [ + SearchResult( + link=result["url"], title=result.get("title"), snippet=result.get("snippet") + ) + for result in results[:count] + ] diff --git a/backend/apps/rag/search/testdata/brave.json b/backend/apps/rag/search/testdata/brave.json new file mode 100644 index 0000000000..38487390d9 --- /dev/null +++ b/backend/apps/rag/search/testdata/brave.json @@ -0,0 +1,998 @@ +{ + "query": { + "original": "python", + "show_strict_warning": false, + "is_navigational": true, + "is_news_breaking": false, + "spellcheck_off": true, + "country": "us", + "bad_results": false, + "should_fallback": false, + "postal_code": "", + "city": "", + "header_country": "", + "more_results_available": true, + "state": "" + }, + "mixed": { + "type": "mixed", + "main": [ + { + "type": "web", + "index": 0, + "all": false + }, + { + "type": "web", + "index": 1, + "all": false + }, + { + "type": "news", + "all": true + }, + { + "type": "web", + "index": 2, + "all": false + }, + { + "type": "videos", + "all": true + }, + { + "type": "web", + "index": 3, + "all": false + }, + { + "type": "web", + "index": 4, + "all": false + }, + { + "type": "web", + "index": 5, + "all": false + }, + { + "type": "web", + "index": 6, + "all": false + }, + { + "type": "web", + "index": 7, + "all": false + }, + { + "type": "web", + "index": 8, + "all": false + }, + { + "type": "web", + "index": 9, + "all": false + }, + { + "type": "web", + "index": 10, + "all": false + }, + { + "type": "web", + "index": 11, + "all": false + }, + { + "type": "web", + "index": 12, + "all": false + }, + { + "type": "web", + "index": 13, + "all": false + }, + { + "type": "web", + "index": 14, + "all": false + }, + { + "type": "web", + "index": 15, + "all": false + }, + { + "type": "web", + "index": 16, + "all": false + }, + { + "type": "web", + "index": 17, + "all": false + }, + { + "type": "web", + "index": 18, + "all": false + }, + { + "type": "web", + "index": 19, + "all": false + } + ], + "top": [], + "side": [] + }, + "news": { + "type": "news", + "results": [ + { + "title": "Google lays off staff from Flutter, Dart and Python teams weeks before its developer conference | TechCrunch", + "url": "https://techcrunch.com/2024/05/01/google-lays-off-staff-from-flutter-dart-python-weeks-before-its-developer-conference/", + "is_source_local": false, + "is_source_both": false, + "description": "Google told TechCrunch that Flutter will have new updates to share at I/O this year.", + "page_age": "2024-05-02T17:40:05", + "family_friendly": true, + "meta_url": { + "scheme": "https", + "netloc": "techcrunch.com", + "hostname": "techcrunch.com", + "favicon": "https://imgs.search.brave.com/N6VSEVahheQOb7lqfb47dhUOB4XD-6sfQOP94sCe3Oo/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvZGI5Njk0Yzlk/YWM3ZWMwZjg1MTM1/NmIyMWEyNzBjZDZj/ZDQyNmFlNGU0NDRi/MDgyYjQwOGU0Y2Qy/ZWMwNWQ2ZC90ZWNo/Y3J1bmNoLmNvbS8", + "path": "› 2024 › 05 › 01 › google-lays-off-staff-from-flutter-dart-python-weeks-before-its-developer-conference" + }, + "breaking": false, + "thumbnail": { + "src": "https://imgs.search.brave.com/gCI5UG8muOEOZDAx9vpu6L6r6R00mD7jOF08-biFoyQ/rs:fit:200:200:1/g:ce/aHR0cHM6Ly90ZWNo/Y3J1bmNoLmNvbS93/cC1jb250ZW50L3Vw/bG9hZHMvMjAxOC8x/MS9HZXR0eUltYWdl/cy0xMDAyNDg0NzQ2/LmpwZz9yZXNpemU9/MTIwMCw4MDA" + }, + "age": "3 days ago", + "extra_snippets": [ + "Ahead of Google’s annual I/O developer conference in May, the tech giant has laid off staff across key teams like Flutter, Dart, Python and others, according to reports from affected employees shared on social media. Google confirmed the layoffs to TechCrunch, but not the specific teams, roles or how many people were let go.", + "In a separate post on Reddit, another commenter noted the Python team affected by the layoffs were those who managed the internal Python runtimes and toolchains and worked with OSS Python. Included in this group were “multiple current and former core devs and steering council members,” they said.", + "Meanwhile, others shared on Y Combinator’s Hacker News, where a Python team member detailed their specific duties on the technical front and noted that, for years, much of the work was done with fewer than 10 people. Another Hacker News commenter said their early years on the Python team were spent paying down internal technical debt accumulated from not having a strong Python strategy.", + "CNBC reports that a total of 200 people were let go across Google’s “Core” teams, which included those working on Python, app platforms, and other engineering roles. Some jobs were being shifted to India and Mexico, it said, citing internal documents." + ] + } + ], + "mutated_by_goggles": false + }, + "type": "search", + "videos": { + "type": "videos", + "results": [ + { + "type": "video_result", + "url": "https://www.youtube.com/watch?v=b093aqAZiPU", + "title": "👩‍💻 Python for Beginners Tutorial - YouTube", + "description": "In this step-by-step Python for beginner's tutorial, learn how you can get started programming in Python. In this video, I assume that you are completely new...", + "age": "March 25, 2021", + "page_age": "2021-03-25T10:00:08", + "video": {}, + "meta_url": { + "scheme": "https", + "netloc": "youtube.com", + "hostname": "www.youtube.com", + "favicon": "https://imgs.search.brave.com/Ux4Hee4evZhvjuTKwtapBycOGjGDci2Gvn2pbSzvbC0/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTkyZTZiMWU3/YzU3Nzc5YjExYzUy/N2VhZTIxOWNlYjM5/ZGVjN2MyZDY4Nzdh/ZDYzMTYxNmI5N2Rk/Y2Q3N2FkNy93d3cu/eW91dHViZS5jb20v", + "path": "› watch" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/tZI4Do4_EYcTCsD_MvE3Jx8FzjIXwIJ5ZuKhwiWTyZs/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9pLnl0/aW1nLmNvbS92aS9i/MDkzYXFBWmlQVS9t/YXhyZXNkZWZhdWx0/LmpwZw" + } + }, + { + "type": "video_result", + "url": "https://www.youtube.com/watch?v=rfscVS0vtbw", + "title": "Learn Python - Full Course for Beginners [Tutorial] - YouTube", + "description": "This course will give you a full introduction into all of the core concepts in python. Follow along with the videos and you'll be a python programmer in no t...", + "age": "July 11, 2018", + "page_age": "2018-07-11T18:00:42", + "video": {}, + "meta_url": { + "scheme": "https", + "netloc": "youtube.com", + "hostname": "www.youtube.com", + "favicon": "https://imgs.search.brave.com/Ux4Hee4evZhvjuTKwtapBycOGjGDci2Gvn2pbSzvbC0/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTkyZTZiMWU3/YzU3Nzc5YjExYzUy/N2VhZTIxOWNlYjM5/ZGVjN2MyZDY4Nzdh/ZDYzMTYxNmI5N2Rk/Y2Q3N2FkNy93d3cu/eW91dHViZS5jb20v", + "path": "› watch" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/65zkx_kPU_zJb-4nmvvY-q5-ZZwzceChz-N00V8cqvk/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9pLnl0/aW1nLmNvbS92aS9y/ZnNjVlMwdnRidy9t/YXhyZXNkZWZhdWx0/LmpwZw" + } + }, + { + "type": "video_result", + "url": "https://www.youtube.com/watch?v=_uQrJ0TkZlc", + "title": "Python Tutorial - Python Full Course for Beginners - YouTube", + "description": "Become a Python pro! 🚀 This comprehensive tutorial takes you from beginner to hero, covering the basics, machine learning, and web development projects.🚀 W...", + "age": "February 18, 2019", + "page_age": "2019-02-18T15:00:08", + "video": {}, + "meta_url": { + "scheme": "https", + "netloc": "youtube.com", + "hostname": "www.youtube.com", + "favicon": "https://imgs.search.brave.com/Ux4Hee4evZhvjuTKwtapBycOGjGDci2Gvn2pbSzvbC0/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTkyZTZiMWU3/YzU3Nzc5YjExYzUy/N2VhZTIxOWNlYjM5/ZGVjN2MyZDY4Nzdh/ZDYzMTYxNmI5N2Rk/Y2Q3N2FkNy93d3cu/eW91dHViZS5jb20v", + "path": "› watch" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/Djiv1pXLq1ClqBSE_86jQnEYR8bW8UJP6Cs7LrgyQzQ/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9pLnl0/aW1nLmNvbS92aS9f/dVFySjBUa1psYy9t/YXhyZXNkZWZhdWx0/LmpwZw" + } + }, + { + "type": "video_result", + "url": "https://www.youtube.com/watch?v=wRKgzC-MhIc", + "title": "[] and {} vs list() and dict(), which is better?", + "description": "Enjoy the videos and music you love, upload original content, and share it all with friends, family, and the world on YouTube.", + "video": {}, + "meta_url": { + "scheme": "https", + "netloc": "youtube.com", + "hostname": "www.youtube.com", + "favicon": "https://imgs.search.brave.com/Ux4Hee4evZhvjuTKwtapBycOGjGDci2Gvn2pbSzvbC0/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTkyZTZiMWU3/YzU3Nzc5YjExYzUy/N2VhZTIxOWNlYjM5/ZGVjN2MyZDY4Nzdh/ZDYzMTYxNmI5N2Rk/Y2Q3N2FkNy93d3cu/eW91dHViZS5jb20v", + "path": "› watch" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/Hw9ep2Pio13X1VZjRw_h9R2VH_XvZFOuGlQJVnVkeq0/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9pLnl0/aW1nLmNvbS92aS93/UktnekMtTWhJYy9o/cWRlZmF1bHQuanBn" + } + }, + { + "type": "video_result", + "url": "https://www.youtube.com/watch?v=LWdsF79H1Pg", + "title": "print() vs. return in Python Functions - YouTube", + "description": "In this video, you will learn the differences between the return statement and the print function when they are used inside Python functions. We will see an ...", + "age": "June 11, 2022", + "page_age": "2022-06-11T21:33:26", + "video": {}, + "meta_url": { + "scheme": "https", + "netloc": "youtube.com", + "hostname": "www.youtube.com", + "favicon": "https://imgs.search.brave.com/Ux4Hee4evZhvjuTKwtapBycOGjGDci2Gvn2pbSzvbC0/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTkyZTZiMWU3/YzU3Nzc5YjExYzUy/N2VhZTIxOWNlYjM5/ZGVjN2MyZDY4Nzdh/ZDYzMTYxNmI5N2Rk/Y2Q3N2FkNy93d3cu/eW91dHViZS5jb20v", + "path": "› watch" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/ebglnr5_jwHHpvon3WU-5hzt0eHdTZSVGg3Ts6R38xY/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9pLnl0/aW1nLmNvbS92aS9M/V2RzRjc5SDFQZy9t/YXhyZXNkZWZhdWx0/LmpwZw" + } + }, + { + "type": "video_result", + "url": "https://www.youtube.com/watch?v=AovxLr8jUH4", + "title": "Python Tutorial for Beginners 5 - Python print() and input() Function ...", + "description": "In this Video I am going to show How to use print() Function and input() Function in Python. In python The print() function is used to print the specified ...", + "age": "August 28, 2018", + "page_age": "2018-08-28T20:11:09", + "video": {}, + "meta_url": { + "scheme": "https", + "netloc": "youtube.com", + "hostname": "www.youtube.com", + "favicon": "https://imgs.search.brave.com/Ux4Hee4evZhvjuTKwtapBycOGjGDci2Gvn2pbSzvbC0/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTkyZTZiMWU3/YzU3Nzc5YjExYzUy/N2VhZTIxOWNlYjM5/ZGVjN2MyZDY4Nzdh/ZDYzMTYxNmI5N2Rk/Y2Q3N2FkNy93d3cu/eW91dHViZS5jb20v", + "path": "› watch" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/nCoLEcWkKtiecprWbS6nufwGCaSbPH7o0-sMeIkFmjI/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9pLnl0/aW1nLmNvbS92aS9B/b3Z4THI4alVINC9o/cWRlZmF1bHQuanBn" + } + } + ], + "mutated_by_goggles": false + }, + "web": { + "type": "search", + "results": [ + { + "title": "Welcome to Python.org", + "url": "https://www.python.org", + "is_source_local": false, + "is_source_both": false, + "description": "The official home of the Python Programming Language", + "page_age": "2023-09-09T15:55:05", + "profile": { + "name": "Python", + "url": "https://www.python.org", + "long_name": "python.org", + "img": "https://imgs.search.brave.com/vBaRH-v6oPS4csO4cdvuKhZ7-xDVvydin3oe3zXYxAI/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvNTJjMzZjNDBj/MmIzODgwMGUyOTRj/Y2E5MjM3YjRkYTZj/YWI1Yzk1NTlmYTgw/ZDBjNzM0MGMxZjQz/YWFjNTczYy93d3cu/cHl0aG9uLm9yZy8" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "python.org", + "hostname": "www.python.org", + "favicon": "https://imgs.search.brave.com/vBaRH-v6oPS4csO4cdvuKhZ7-xDVvydin3oe3zXYxAI/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvNTJjMzZjNDBj/MmIzODgwMGUyOTRj/Y2E5MjM3YjRkYTZj/YWI1Yzk1NTlmYTgw/ZDBjNzM0MGMxZjQz/YWFjNTczYy93d3cu/cHl0aG9uLm9yZy8", + "path": "" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/GGfNfe5rxJ8QWEoxXniSLc0-POLU3qPyTIpuqPdbmXk/rs:fit:200:200:1/g:ce/aHR0cHM6Ly93d3cu/cHl0aG9uLm9yZy9z/dGF0aWMvb3Blbmdy/YXBoLWljb24tMjAw/eDIwMC5wbmc", + "original": "https://www.python.org/static/opengraph-icon-200x200.png", + "logo": false + }, + "age": "September 9, 2023", + "cluster_type": "generic", + "cluster": [ + { + "title": "Downloads", + "url": "https://www.python.org/downloads/", + "is_source_local": false, + "is_source_both": false, + "description": "The official home of the Python Programming Language", + "family_friendly": true + }, + { + "title": "Macos", + "url": "https://www.python.org/downloads/macos/", + "is_source_local": false, + "is_source_both": false, + "description": "The official home of the Python Programming Language", + "family_friendly": true + }, + { + "title": "Windows", + "url": "https://www.python.org/downloads/windows/", + "is_source_local": false, + "is_source_both": false, + "description": "The official home of the Python Programming Language", + "family_friendly": true + }, + { + "title": "Getting Started", + "url": "https://www.python.org/about/gettingstarted/", + "is_source_local": false, + "is_source_both": false, + "description": "The official home of the Python Programming Language", + "family_friendly": true + } + ], + "extra_snippets": [ + "Calculations are simple with Python, and expression syntax is straightforward: the operators +, -, * and / work as expected; parentheses () can be used for grouping. More about simple math functions in Python 3.", + "The core of extensible programming is defining functions. Python allows mandatory and optional arguments, keyword arguments, and even arbitrary argument lists. More about defining functions in Python 3", + "Lists (known as arrays in other languages) are one of the compound data types that Python understands. Lists can be indexed, sliced and manipulated with other built-in functions. More about lists in Python 3", + "# Python 3: Simple output (with Unicode) >>> print(\"Hello, I'm Python!\") Hello, I'm Python! # Input, assignment >>> name = input('What is your name?\\n') >>> print('Hi, %s.' % name) What is your name? Python Hi, Python." + ] + }, + { + "title": "Python (programming language) - Wikipedia", + "url": "https://en.wikipedia.org/wiki/Python_(programming_language)", + "is_source_local": false, + "is_source_both": false, + "description": "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected. It supports multiple programming paradigms, including structured (particularly procedural), ...", + "page_age": "2024-05-01T12:54:03", + "profile": { + "name": "Wikipedia", + "url": "https://en.wikipedia.org/wiki/Python_(programming_language)", + "long_name": "en.wikipedia.org", + "img": "https://imgs.search.brave.com/0kxnVOiqv-faZvOJc7zpym4Zin1CTs1f1svfNZSzmfU/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvNjQwNGZhZWY0/ZTQ1YWUzYzQ3MDUw/MmMzMGY3NTQ0ZjNj/NDUwMDk5ZTI3MWRk/NWYyNTM4N2UwOTE0/NTI3ZDQzNy9lbi53/aWtpcGVkaWEub3Jn/Lw" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "en.wikipedia.org", + "hostname": "en.wikipedia.org", + "favicon": "https://imgs.search.brave.com/0kxnVOiqv-faZvOJc7zpym4Zin1CTs1f1svfNZSzmfU/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvNjQwNGZhZWY0/ZTQ1YWUzYzQ3MDUw/MmMzMGY3NTQ0ZjNj/NDUwMDk5ZTI3MWRk/NWYyNTM4N2UwOTE0/NTI3ZDQzNy9lbi53/aWtpcGVkaWEub3Jn/Lw", + "path": "› wiki › Python_(programming_language)" + }, + "age": "4 days ago", + "extra_snippets": [ + "Python is dynamically typed and garbage-collected. It supports multiple programming paradigms, including structured (particularly procedural), object-oriented and functional programming. It is often described as a \"batteries included\" language due to its comprehensive standard library.", + "Guido van Rossum began working on Python in the late 1980s as a successor to the ABC programming language and first released it in 1991 as Python 0.9.0. Python 2.0 was released in 2000. Python 3.0, released in 2008, was a major revision not completely backward-compatible with earlier versions. Python 2.7.18, released in 2020, was the last release of Python 2.", + "Python was invented in the late 1980s by Guido van Rossum at Centrum Wiskunde & Informatica (CWI) in the Netherlands as a successor to the ABC programming language, which was inspired by SETL, capable of exception handling and interfacing with the Amoeba operating system.", + "Python consistently ranks as one of the most popular programming languages, and has gained widespread use in the machine learning community." + ] + }, + { + "title": "Python Tutorial", + "url": "https://www.w3schools.com/python/", + "is_source_local": false, + "is_source_both": false, + "description": "W3Schools offers free online tutorials, references and exercises in all the major languages of the web. Covering popular subjects like HTML, CSS, JavaScript, Python, SQL, Java, and many, many more.", + "page_age": "2017-12-07T00:00:00", + "profile": { + "name": "W3Schools", + "url": "https://www.w3schools.com/python/", + "long_name": "w3schools.com", + "img": "https://imgs.search.brave.com/JwO5r7z3HTBkU29vgNH_4rrSWLf2M4-8FMWNvbxrKX8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYjVlMGVkZDVj/ZGMyZWRmMzAwODRi/ZDAwZGE4NWI3NmU4/MjRhNjEzOGFhZWY3/ZGViMjY1OWY2ZDYw/YTZiOGUyZS93d3cu/dzNzY2hvb2xzLmNv/bS8" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "w3schools.com", + "hostname": "www.w3schools.com", + "favicon": "https://imgs.search.brave.com/JwO5r7z3HTBkU29vgNH_4rrSWLf2M4-8FMWNvbxrKX8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYjVlMGVkZDVj/ZGMyZWRmMzAwODRi/ZDAwZGE4NWI3NmU4/MjRhNjEzOGFhZWY3/ZGViMjY1OWY2ZDYw/YTZiOGUyZS93d3cu/dzNzY2hvb2xzLmNv/bS8", + "path": "› python" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/EMfp8dodbJehmj0yCJh8317RHuaumsddnHI4bujvFcg/rs:fit:200:200:1/g:ce/aHR0cHM6Ly93d3cu/dzNzY2hvb2xzLmNv/bS9pbWFnZXMvdzNz/Y2hvb2xzX2xvZ29f/NDM2XzIucG5n", + "original": "https://www.w3schools.com/images/w3schools_logo_436_2.png", + "logo": true + }, + "age": "December 7, 2017", + "extra_snippets": [ + "Well organized and easy to understand Web building tutorials with lots of examples of how to use HTML, CSS, JavaScript, SQL, Python, PHP, Bootstrap, Java, XML and more.", + "HTML CSS JAVASCRIPT SQL PYTHON JAVA PHP HOW TO W3.CSS C C++ C# BOOTSTRAP REACT MYSQL JQUERY EXCEL XML DJANGO NUMPY PANDAS NODEJS R TYPESCRIPT ANGULAR GIT POSTGRESQL MONGODB ASP AI GO KOTLIN SASS VUE DSA GEN AI SCIPY AWS CYBERSECURITY DATA SCIENCE", + "Python Variables Variable Names Assign Multiple Values Output Variables Global Variables Variable Exercises Python Data Types Python Numbers Python Casting Python Strings", + "Python Strings Slicing Strings Modify Strings Concatenate Strings Format Strings Escape Characters String Methods String Exercises Python Booleans Python Operators Python Lists" + ] + }, + { + "title": "Online Python - IDE, Editor, Compiler, Interpreter", + "url": "https://www.online-python.com/", + "is_source_local": false, + "is_source_both": false, + "description": "Build and Run your Python code instantly. Online-Python is a quick and easy tool that helps you to build, compile, test your python programs.", + "profile": { + "name": "Online-python", + "url": "https://www.online-python.com/", + "long_name": "online-python.com", + "img": "https://imgs.search.brave.com/kfaEvapwHxSsRObO52-I-otYFPHpG1h7UXJyUqDM2Ec/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvZGYxODdjNWQ0/NjZjZTNiMjk5NDY1/MWI5MTgyYjU3Y2Q3/MTI3NGM5MjUzY2Fi/OGQ3MTQ4MmIxMTQx/ZTcxNWFhMC93d3cu/b25saW5lLXB5dGhv/bi5jb20v" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "online-python.com", + "hostname": "www.online-python.com", + "favicon": "https://imgs.search.brave.com/kfaEvapwHxSsRObO52-I-otYFPHpG1h7UXJyUqDM2Ec/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvZGYxODdjNWQ0/NjZjZTNiMjk5NDY1/MWI5MTgyYjU3Y2Q3/MTI3NGM5MjUzY2Fi/OGQ3MTQ4MmIxMTQx/ZTcxNWFhMC93d3cu/b25saW5lLXB5dGhv/bi5jb20v", + "path": "" + }, + "extra_snippets": [ + "Build, run, and share Python code online for free with the help of online-integrated python's development environment (IDE). It is one of the most efficient, dependable, and potent online compilers for the Python programming language. It is not necessary for you to bother about establishing a Python environment in your local.", + "It is one of the most efficient, dependable, and potent online compilers for the Python programming language. It is not necessary for you to bother about establishing a Python environment in your local. Now You can immediately execute the Python code in the web browser of your choice.", + "It is not necessary for you to bother about establishing a Python environment in your local. Now You can immediately execute the Python code in the web browser of your choice. Using this Python editor is simple and quick to get up and running with. Simply type in the programme, and then press the RUN button!", + "Now You can immediately execute the Python code in the web browser of your choice. Using this Python editor is simple and quick to get up and running with. Simply type in the programme, and then press the RUN button! The code can be saved online by choosing the SHARE option, which also gives you the ability to access your code from any location providing you have internet access." + ] + }, + { + "title": "Python · GitHub", + "url": "https://github.com/python", + "is_source_local": false, + "is_source_both": false, + "description": "Repositories related to the Python Programming language - Python", + "page_age": "2023-03-06T00:00:00", + "profile": { + "name": "GitHub", + "url": "https://github.com/python", + "long_name": "github.com", + "img": "https://imgs.search.brave.com/v8685zI4XInM0zxlNI2s7oE_2Sb-EL7lAy81WXbkQD8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYWQyNWM1NjA5/ZjZmZjNlYzI2MDNk/N2VkNmJhYjE2MzZl/MDY5ZTMxMDUzZmY1/NmU3NWIzNWVmMjk0/NTBjMjJjZi9naXRo/dWIuY29tLw" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "github.com", + "hostname": "github.com", + "favicon": "https://imgs.search.brave.com/v8685zI4XInM0zxlNI2s7oE_2Sb-EL7lAy81WXbkQD8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYWQyNWM1NjA5/ZjZmZjNlYzI2MDNk/N2VkNmJhYjE2MzZl/MDY5ZTMxMDUzZmY1/NmU3NWIzNWVmMjk0/NTBjMjJjZi9naXRo/dWIuY29tLw", + "path": "› python" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/POoaRfu_7gfp-D_O3qMNJrwDqJNbiDu1HuBpNJ_MpVQ/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9hdmF0/YXJzLmdpdGh1YnVz/ZXJjb250ZW50LmNv/bS91LzE1MjU5ODE_/cz0yMDAmYW1wO3Y9/NA", + "original": "https://avatars.githubusercontent.com/u/1525981?s=200&v=4", + "logo": false + }, + "age": "March 6, 2023", + "extra_snippets": ["Configuration for Python planets (e.g. http://planetpython.org)"] + }, + { + "title": "Online Python Compiler (Interpreter)", + "url": "https://www.programiz.com/python-programming/online-compiler/", + "is_source_local": false, + "is_source_both": false, + "description": "Write and run Python code using our online compiler (interpreter). You can use Python Shell like IDLE, and take inputs from the user in our Python compiler.", + "page_age": "2020-06-02T00:00:00", + "profile": { + "name": "Programiz", + "url": "https://www.programiz.com/python-programming/online-compiler/", + "long_name": "programiz.com", + "img": "https://imgs.search.brave.com/ozj4JFayZ3Fs5c9eTp7M5g12azQ_Hblgu4dpTuHRz6U/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvMGJlN2U1YjVi/Y2M3ZDU5OGMwMWNi/M2Q3YjhjOTM1ZTFk/Y2NkZjE4NGQwOGIx/MTQ4NjI2YmNhODVj/MzFkMmJhYy93d3cu/cHJvZ3JhbWl6LmNv/bS8" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "programiz.com", + "hostname": "www.programiz.com", + "favicon": "https://imgs.search.brave.com/ozj4JFayZ3Fs5c9eTp7M5g12azQ_Hblgu4dpTuHRz6U/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvMGJlN2U1YjVi/Y2M3ZDU5OGMwMWNi/M2Q3YjhjOTM1ZTFk/Y2NkZjE4NGQwOGIx/MTQ4NjI2YmNhODVj/MzFkMmJhYy93d3cu/cHJvZ3JhbWl6LmNv/bS8", + "path": "› python-programming › online-compiler" + }, + "age": "June 2, 2020", + "extra_snippets": [ + "Python Online Compiler Online R Compiler SQL Online Editor Online HTML/CSS Editor Online Java Compiler C Online Compiler C++ Online Compiler C# Online Compiler JavaScript Online Compiler Online GoLang Compiler Online PHP Compiler Online Swift Compiler Online Rust Compiler", + "# Online Python compiler (interpreter) to run Python online. # Write Python 3 code in this online editor and run it. print(\"Try programiz.pro\")" + ] + }, + { + "title": "Python Developer", + "url": "https://twitter.com/Python_Dv/status/1786763460992544791", + "is_source_local": false, + "is_source_both": false, + "description": "Python Developer", + "page_age": "2024-05-04T14:30:03", + "profile": { + "name": "X", + "url": "https://twitter.com/Python_Dv/status/1786763460992544791", + "long_name": "twitter.com", + "img": "https://imgs.search.brave.com/Zq483bGX0GnSgym-1P7iyOyEDX3PkDZSNT8m56F862A/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvN2MxOTUxNzhj/OTY1ZTQ3N2I0MjJk/MTY5NGM0MTRlYWVi/MjU1YWE2NDUwYmQ2/YTA2MDFhMDlkZDEx/NTAzZGNiNi90d2l0/dGVyLmNvbS8" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "twitter.com", + "hostname": "twitter.com", + "favicon": "https://imgs.search.brave.com/Zq483bGX0GnSgym-1P7iyOyEDX3PkDZSNT8m56F862A/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvN2MxOTUxNzhj/OTY1ZTQ3N2I0MjJk/MTY5NGM0MTRlYWVi/MjU1YWE2NDUwYmQ2/YTA2MDFhMDlkZDEx/NTAzZGNiNi90d2l0/dGVyLmNvbS8", + "path": "› Python_Dv › status › 1786763460992544791" + }, + "age": "20 hours ago" + }, + { + "title": "input table name? - python script - KNIME Extensions - KNIME Community Forum", + "url": "https://forum.knime.com/t/input-table-name-python-script/78978", + "is_source_local": false, + "is_source_both": false, + "description": "Hi, when running a python script node, I get the error seen on the screenshot Same happens with this code too: The script input is output from the csv reader node. How can I get the right name for that table? Best wishes, Dario", + "page_age": "2024-05-04T09:20:44", + "profile": { + "name": "Knime", + "url": "https://forum.knime.com/t/input-table-name-python-script/78978", + "long_name": "forum.knime.com", + "img": "https://imgs.search.brave.com/WQoOhAD5i6uEhJ-qXvlWMJwbGA52f2Ycc_ns36EK698/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTAxNzMxNjFl/MzJjNzU5NzRkOTMz/Mjg4NDU2OWUxM2Rj/YzVkOGM3MzIwNzI2/YTY1NzYxNzA1MDE5/NzQzOWU3NC9mb3J1/bS5rbmltZS5jb20v" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "article", + "meta_url": { + "scheme": "https", + "netloc": "forum.knime.com", + "hostname": "forum.knime.com", + "favicon": "https://imgs.search.brave.com/WQoOhAD5i6uEhJ-qXvlWMJwbGA52f2Ycc_ns36EK698/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTAxNzMxNjFl/MzJjNzU5NzRkOTMz/Mjg4NDU2OWUxM2Rj/YzVkOGM3MzIwNzI2/YTY1NzYxNzA1MDE5/NzQzOWU3NC9mb3J1/bS5rbmltZS5jb20v", + "path": " › knime extensions" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/DtEl38dcvuM1kGfhN0T5HfOrsMJcztWNyriLvtDJmKI/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9mb3J1/bS1jZG4ua25pbWUu/Y29tL3VwbG9hZHMv/ZGVmYXVsdC9vcmln/aW5hbC8zWC9lLzYv/ZTY0M2M2NzFlNzAz/MDg2MjkwMWY2YzJh/OWFjOWI5ZmEwM2M3/ZjMwZi5wbmc", + "original": "https://forum-cdn.knime.com/uploads/default/original/3X/e/6/e643c671e7030862901f6c2a9ac9b9fa03c7f30f.png", + "logo": false + }, + "age": "1 day ago", + "extra_snippets": [ + "Hi, when running a python script node, I get the error seen on the screenshot Same happens with this code too: The script input is output from the csv reader node. How can I get the right name for that table? …" + ] + }, + { + "title": "What does the Double Star operator mean in Python? - GeeksforGeeks", + "url": "https://www.geeksforgeeks.org/what-does-the-double-star-operator-mean-in-python/", + "is_source_local": false, + "is_source_both": false, + "description": "A Computer Science portal for geeks. It contains well written, well thought and well explained computer science and programming articles, quizzes and practice/competitive programming/company interview Questions.", + "page_age": "2023-03-14T17:15:04", + "profile": { + "name": "GeeksforGeeks", + "url": "https://www.geeksforgeeks.org/what-does-the-double-star-operator-mean-in-python/", + "long_name": "geeksforgeeks.org", + "img": "https://imgs.search.brave.com/fhzcfv5xltx6-YBvJI9RZgS7xZo0dPNaASsrB8YOsCs/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYjBhOGQ3MmNi/ZWE5N2EwMmZjYzA1/ZTI0ZTFhMGUyMTE0/MGM0ZTBmMWZlM2Y2/Yzk2ODMxZTRhYTBi/NDdjYTE0OS93d3cu/Z2Vla3Nmb3JnZWVr/cy5vcmcv" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "article", + "meta_url": { + "scheme": "https", + "netloc": "geeksforgeeks.org", + "hostname": "www.geeksforgeeks.org", + "favicon": "https://imgs.search.brave.com/fhzcfv5xltx6-YBvJI9RZgS7xZo0dPNaASsrB8YOsCs/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYjBhOGQ3MmNi/ZWE5N2EwMmZjYzA1/ZTI0ZTFhMGUyMTE0/MGM0ZTBmMWZlM2Y2/Yzk2ODMxZTRhYTBi/NDdjYTE0OS93d3cu/Z2Vla3Nmb3JnZWVr/cy5vcmcv", + "path": "› what-does-the-double-star-operator-mean-in-python" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/GcR-j_dLbyHkbHEI3ffLMi6xpXGhF_2Z8POIoqtokhM/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9tZWRp/YS5nZWVrc2Zvcmdl/ZWtzLm9yZy93cC1j/b250ZW50L3VwbG9h/ZHMvZ2ZnXzIwMFgy/MDAtMTAweDEwMC5w/bmc", + "original": "https://media.geeksforgeeks.org/wp-content/uploads/gfg_200X200-100x100.png", + "logo": false + }, + "age": "March 14, 2023", + "extra_snippets": [ + "Difference between / vs. // operator in Python", + "Double Star or (**) is one of the Arithmetic Operator (Like +, -, *, **, /, //, %) in Python Language. It is also known as Power Operator.", + "The time complexity of the given Python program is O(n), where n is the number of key-value pairs in the input dictionary.", + "Inplace Operators in Python | Set 2 (ixor(), iand(), ipow(),…)" + ] + }, + { + "title": "r/Python", + "url": "https://www.reddit.com/r/Python/", + "is_source_local": false, + "is_source_both": false, + "description": "The official Python community for Reddit! Stay up to date with the latest news, packages, and meta information relating to the Python programming language. --- If you have questions or are new to Python use r/LearnPython", + "page_age": "2022-12-30T16:25:02", + "profile": { + "name": "Reddit", + "url": "https://www.reddit.com/r/Python/", + "long_name": "reddit.com", + "img": "https://imgs.search.brave.com/mAZYEK9Wi13WLDUge7XZ8YuDTwm6DP6gBjvz1GdYZVY/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvN2ZiNTU0M2Nj/MTFhZjRiYWViZDlk/MjJiMjBjMzFjMDRk/Y2IzYWI0MGI0MjVk/OGY5NzQzOGQ5NzQ5/NWJhMWI0NC93d3cu/cmVkZGl0LmNvbS8" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "reddit.com", + "hostname": "www.reddit.com", + "favicon": "https://imgs.search.brave.com/mAZYEK9Wi13WLDUge7XZ8YuDTwm6DP6gBjvz1GdYZVY/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvN2ZiNTU0M2Nj/MTFhZjRiYWViZDlk/MjJiMjBjMzFjMDRk/Y2IzYWI0MGI0MjVk/OGY5NzQzOGQ5NzQ5/NWJhMWI0NC93d3cu/cmVkZGl0LmNvbS8", + "path": "› r › Python" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/zWd10t3zg34ciHiAB-K5WWK3h_H4LedeDot9BVX7Ydo/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9zdHls/ZXMucmVkZGl0bWVk/aWEuY29tL3Q1XzJx/aDB5L3N0eWxlcy9j/b21tdW5pdHlJY29u/X2NpZmVobDR4dDdu/YzEucG5n", + "original": "https://styles.redditmedia.com/t5_2qh0y/styles/communityIcon_cifehl4xt7nc1.png", + "logo": false + }, + "age": "December 30, 2022", + "extra_snippets": [ + "r/Python: The official Python community for Reddit! Stay up to date with the latest news, packages, and meta information relating to the Python…", + "By default, Python allows you to import and use anything, anywhere. Over time, this results in modules that were intended to be separate getting tightly coupled together, and domain boundaries breaking down. We experienced this first-hand at a unicorn startup, where the eng team paused development for over a year in an attempt to split up packages into independent services.", + "Hello r/Python! It's time to share what you've been working on! Whether it's a work-in-progress, a completed masterpiece, or just a rough idea, let us know what you're up to!", + "Whether it's your job, your hobby, or your passion project, all Python-related work is welcome here." + ] + }, + { + "title": "GitHub - python/cpython: The Python programming language", + "url": "https://github.com/python/cpython", + "is_source_local": false, + "is_source_both": false, + "description": "The Python programming language. Contribute to python/cpython development by creating an account on GitHub.", + "page_age": "2022-10-29T00:00:00", + "profile": { + "name": "GitHub", + "url": "https://github.com/python/cpython", + "long_name": "github.com", + "img": "https://imgs.search.brave.com/v8685zI4XInM0zxlNI2s7oE_2Sb-EL7lAy81WXbkQD8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYWQyNWM1NjA5/ZjZmZjNlYzI2MDNk/N2VkNmJhYjE2MzZl/MDY5ZTMxMDUzZmY1/NmU3NWIzNWVmMjk0/NTBjMjJjZi9naXRo/dWIuY29tLw" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "software", + "meta_url": { + "scheme": "https", + "netloc": "github.com", + "hostname": "github.com", + "favicon": "https://imgs.search.brave.com/v8685zI4XInM0zxlNI2s7oE_2Sb-EL7lAy81WXbkQD8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYWQyNWM1NjA5/ZjZmZjNlYzI2MDNk/N2VkNmJhYjE2MzZl/MDY5ZTMxMDUzZmY1/NmU3NWIzNWVmMjk0/NTBjMjJjZi9naXRo/dWIuY29tLw", + "path": "› python › cpython" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/BJbWFRUqgP-tKIyGK9ByXjuYjHO2mtYigUOEFNz_gXk/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9vcGVu/Z3JhcGguZ2l0aHVi/YXNzZXRzLmNvbS82/MTY5YmJkNTQ0YzAy/NDg0MGU4NDdjYTU1/YTU3ZGZmMDA2ZDAw/YWQ1NDIzOTFmYTQ3/YmJjODg3OWM0NWYw/MTZhL3B5dGhvbi9j/cHl0aG9u", + "original": "https://opengraph.githubassets.com/6169bbd544c024840e847ca55a57dff006d00ad542391fa47bbc8879c45f016a/python/cpython", + "logo": false + }, + "age": "October 29, 2022", + "extra_snippets": [ + "You can pass many options to the configure script; run ./configure --help to find out more. On macOS case-insensitive file systems and on Cygwin, the executable is called python.exe; elsewhere it's just python.", + "Building a complete Python installation requires the use of various additional third-party libraries, depending on your build platform and configure options. Not all standard library modules are buildable or useable on all platforms. Refer to the Install dependencies section of the Developer Guide for current detailed information on dependencies for various Linux distributions and macOS.", + "To get an optimized build of Python, configure --enable-optimizations before you run make. This sets the default make targets up to enable Profile Guided Optimization (PGO) and may be used to auto-enable Link Time Optimization (LTO) on some platforms. For more details, see the sections below.", + "Copyright © 2001-2024 Python Software Foundation. All rights reserved." + ] + }, + { + "title": "5. Data Structures — Python 3.12.3 documentation", + "url": "https://docs.python.org/3/tutorial/datastructures.html", + "is_source_local": false, + "is_source_both": false, + "description": "This chapter describes some things you’ve learned about already in more detail, and adds some new things as well. More on Lists: The list data type has some more methods. Here are all of the method...", + "page_age": "2023-07-04T00:00:00", + "profile": { + "name": "Python documentation", + "url": "https://docs.python.org/3/tutorial/datastructures.html", + "long_name": "docs.python.org", + "img": "https://imgs.search.brave.com/F5Ym7eSElhGdGUFKLRxDj9Z_tc180ldpeMvQ2Q6ARbA/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvMTUzOTFjOGVi/YTcyOTVmODA3ODIy/YjE2NzFjY2ViMjhl/NzRlY2JhYTc5YjNm/ZjhmODAyZWI2OGUw/ZjU4NDVlNy9kb2Nz/LnB5dGhvbi5vcmcv" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "docs.python.org", + "hostname": "docs.python.org", + "favicon": "https://imgs.search.brave.com/F5Ym7eSElhGdGUFKLRxDj9Z_tc180ldpeMvQ2Q6ARbA/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvMTUzOTFjOGVi/YTcyOTVmODA3ODIy/YjE2NzFjY2ViMjhl/NzRlY2JhYTc5YjNm/ZjhmODAyZWI2OGUw/ZjU4NDVlNy9kb2Nz/LnB5dGhvbi5vcmcv", + "path": "› 3 › tutorial › datastructures.html" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/Y7GrMRF8WorDIMLuOl97XC8ltYpoOCqNwWF2pQIIKls/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9kb2Nz/LnB5dGhvbi5vcmcv/My9fc3RhdGljL29n/LWltYWdlLnBuZw", + "original": "https://docs.python.org/3/_static/og-image.png", + "logo": false + }, + "age": "July 4, 2023", + "extra_snippets": [ + "You might have noticed that methods like insert, remove or sort that only modify the list have no return value printed – they return the default None. [1] This is a design principle for all mutable data structures in Python.", + "We saw that lists and strings have many common properties, such as indexing and slicing operations. They are two examples of sequence data types (see Sequence Types — list, tuple, range). Since Python is an evolving language, other sequence data types may be added. There is also another standard sequence data type: the tuple.", + "Python also includes a data type for sets. A set is an unordered collection with no duplicate elements. Basic uses include membership testing and eliminating duplicate entries. Set objects also support mathematical operations like union, intersection, difference, and symmetric difference.", + "Another useful data type built into Python is the dictionary (see Mapping Types — dict). Dictionaries are sometimes found in other languages as “associative memories” or “associative arrays”. Unlike sequences, which are indexed by a range of numbers, dictionaries are indexed by keys, which can be any immutable type; strings and numbers can always be keys." + ] + }, + { + "title": "Something wrong with python packages / AUR Issues, Discussion & PKGBUILD Requests / Arch Linux Forums", + "url": "https://bbs.archlinux.org/viewtopic.php?id=295466", + "is_source_local": false, + "is_source_both": false, + "description": "Big Python updates require Python packages to be rebuild. For some reason they didn't think a bump that made it necessary to rebuild half the official repo was a news post.", + "page_age": "2024-05-04T08:30:02", + "profile": { + "name": "Archlinux", + "url": "https://bbs.archlinux.org/viewtopic.php?id=295466", + "long_name": "bbs.archlinux.org", + "img": "https://imgs.search.brave.com/3au9oqkzSri_aLEec3jo-0bFgLuICkydrWfjFcC8lkI/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvNWNkODM1MWJl/ZmJhMzkzNzYzMDkz/NmEyMWMxNjI5MjNk/NGJmZjFhNTBlZDNl/Mzk5MzJjOGZkYjZl/MjNmY2IzNS9iYnMu/YXJjaGxpbnV4Lm9y/Zy8" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "bbs.archlinux.org", + "hostname": "bbs.archlinux.org", + "favicon": "https://imgs.search.brave.com/3au9oqkzSri_aLEec3jo-0bFgLuICkydrWfjFcC8lkI/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvNWNkODM1MWJl/ZmJhMzkzNzYzMDkz/NmEyMWMxNjI5MjNk/NGJmZjFhNTBlZDNl/Mzk5MzJjOGZkYjZl/MjNmY2IzNS9iYnMu/YXJjaGxpbnV4Lm9y/Zy8", + "path": "› viewtopic.php" + }, + "age": "1 day ago", + "extra_snippets": [ + "Traceback (most recent call last): File \"/usr/lib/python3.12/importlib/metadata/__init__.py\", line 397, in from_name return next(cls.discover(name=name)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ StopIteration During handling of the above exception, another exception occurred: Traceback (most recent call last): File \"/usr/bin/informant\", line 33, in sys.exit(load_entry_point('informant==0.5.0', 'console_scripts', 'informant')()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File \"/usr/bin/informant\", line 22, in importlib_load_entry_point for entry_point in distribution(dis" + ] + }, + { + "title": "Introduction to Python", + "url": "https://www.w3schools.com/python/python_intro.asp", + "is_source_local": false, + "is_source_both": false, + "description": "W3Schools offers free online tutorials, references and exercises in all the major languages of the web. Covering popular subjects like HTML, CSS, JavaScript, Python, SQL, Java, and many, many more.", + "profile": { + "name": "W3Schools", + "url": "https://www.w3schools.com/python/python_intro.asp", + "long_name": "w3schools.com", + "img": "https://imgs.search.brave.com/JwO5r7z3HTBkU29vgNH_4rrSWLf2M4-8FMWNvbxrKX8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYjVlMGVkZDVj/ZGMyZWRmMzAwODRi/ZDAwZGE4NWI3NmU4/MjRhNjEzOGFhZWY3/ZGViMjY1OWY2ZDYw/YTZiOGUyZS93d3cu/dzNzY2hvb2xzLmNv/bS8" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "w3schools.com", + "hostname": "www.w3schools.com", + "favicon": "https://imgs.search.brave.com/JwO5r7z3HTBkU29vgNH_4rrSWLf2M4-8FMWNvbxrKX8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYjVlMGVkZDVj/ZGMyZWRmMzAwODRi/ZDAwZGE4NWI3NmU4/MjRhNjEzOGFhZWY3/ZGViMjY1OWY2ZDYw/YTZiOGUyZS93d3cu/dzNzY2hvb2xzLmNv/bS8", + "path": "› python › python_intro.asp" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/EMfp8dodbJehmj0yCJh8317RHuaumsddnHI4bujvFcg/rs:fit:200:200:1/g:ce/aHR0cHM6Ly93d3cu/dzNzY2hvb2xzLmNv/bS9pbWFnZXMvdzNz/Y2hvb2xzX2xvZ29f/NDM2XzIucG5n", + "original": "https://www.w3schools.com/images/w3schools_logo_436_2.png", + "logo": true + }, + "extra_snippets": [ + "Well organized and easy to understand Web building tutorials with lots of examples of how to use HTML, CSS, JavaScript, SQL, Python, PHP, Bootstrap, Java, XML and more.", + "HTML CSS JAVASCRIPT SQL PYTHON JAVA PHP HOW TO W3.CSS C C++ C# BOOTSTRAP REACT MYSQL JQUERY EXCEL XML DJANGO NUMPY PANDAS NODEJS R TYPESCRIPT ANGULAR GIT POSTGRESQL MONGODB ASP AI GO KOTLIN SASS VUE DSA GEN AI SCIPY AWS CYBERSECURITY DATA SCIENCE", + "Python Variables Variable Names Assign Multiple Values Output Variables Global Variables Variable Exercises Python Data Types Python Numbers Python Casting Python Strings", + "Python Strings Slicing Strings Modify Strings Concatenate Strings Format Strings Escape Characters String Methods String Exercises Python Booleans Python Operators Python Lists" + ] + }, + { + "title": "bug: AUR package wants to use python but does not find any preset version · Issue #1740 · asdf-vm/asdf", + "url": "https://github.com/asdf-vm/asdf/issues/1740", + "is_source_local": false, + "is_source_both": false, + "description": "Describe the Bug I am not sure why this is happening, I am trying to install tlpui from AUR and it fails, here are some logs to help: ==> Making package: tlpui 2:1.6.5-1 (Mi 10 apr 2024 23:19:15 +0...", + "page_age": "2024-05-04T06:45:04", + "profile": { + "name": "GitHub", + "url": "https://github.com/asdf-vm/asdf/issues/1740", + "long_name": "github.com", + "img": "https://imgs.search.brave.com/v8685zI4XInM0zxlNI2s7oE_2Sb-EL7lAy81WXbkQD8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYWQyNWM1NjA5/ZjZmZjNlYzI2MDNk/N2VkNmJhYjE2MzZl/MDY5ZTMxMDUzZmY1/NmU3NWIzNWVmMjk0/NTBjMjJjZi9naXRo/dWIuY29tLw" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "software", + "meta_url": { + "scheme": "https", + "netloc": "github.com", + "hostname": "github.com", + "favicon": "https://imgs.search.brave.com/v8685zI4XInM0zxlNI2s7oE_2Sb-EL7lAy81WXbkQD8/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYWQyNWM1NjA5/ZjZmZjNlYzI2MDNk/N2VkNmJhYjE2MzZl/MDY5ZTMxMDUzZmY1/NmU3NWIzNWVmMjk0/NTBjMjJjZi9naXRo/dWIuY29tLw", + "path": "› asdf-vm › asdf › issues › 1740" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/KrLW5s_2n4jyP8XLbc3ZPVBaLD963tQgWzG9EWPZlQs/rs:fit:200:200:1/g:ce/aHR0cHM6Ly9vcGVu/Z3JhcGguZ2l0aHVi/YXNzZXRzLmNvbS81/MTE0ZTdkOGIwODM2/YmQ2MTY3NzQ1ZGI4/MmZjMGE3OGUyMjcw/MGFlY2ZjMWZkODBl/MDYzZTNiN2ZjOWNj/NzYyL2FzZGYtdm0v/YXNkZi9pc3N1ZXMv/MTc0MA", + "original": "https://opengraph.githubassets.com/5114e7d8b0836bd6167745db82fc0a78e22700aecfc1fd80e063e3b7fc9cc762/asdf-vm/asdf/issues/1740", + "logo": false + }, + "age": "1 day ago", + "extra_snippets": [ + "==> Starting build()... No preset version installed for command python Please install a version by running one of the following: asdf install python 3.8 or add one of the following versions in your config file at /home/ferret/.tool-versions python 3.11.0 python 3.12.1 python 3.12.3 ==> ERROR: A failure occurred in build(). Aborting...", + "-> error making: tlpui-exit status 4 -> Failed to install the following packages. Manual intervention is required: tlpui - exit status 4 ferret@FX505DT in ~ $ cat /home/ferret/.tool-versions nodejs 21.6.0 python 3.12.3 ferret@FX505DT in ~ $ python -V Python 3.12.3 ferret@FX505DT in ~ $ which python /home/ferret/.asdf/shims/python", + "Describe the Bug I am not sure why this is happening, I am trying to install tlpui from AUR and it fails, here are some logs to help: ==> Making package: tlpui 2:1.6.5-1 (Mi 10 apr 2024 23:19:15 +0300) ==> Retrieving sources... -> Found ..." + ] + }, + { + "title": "What are python.exe and python3.exe, and why do they appear to point to App Installer? | Windows 11 Forum", + "url": "https://www.elevenforum.com/t/what-are-python-exe-and-python3-exe-and-why-do-they-appear-to-point-to-app-installer.24886/", + "is_source_local": false, + "is_source_both": false, + "description": "I was looking at App execution aliases (Settings > Apps > Advanced app settings > App execution aliases) on my new computer -- my first Windows 11 computer. Why are python.exe and python3.exe listed as App Installer? I assume that App Installer refers to installation of Microsoft Store / UWP...", + "page_age": "2024-05-03T17:30:04", + "profile": { + "name": "Windows 11 Forum", + "url": "https://www.elevenforum.com/t/what-are-python-exe-and-python3-exe-and-why-do-they-appear-to-point-to-app-installer.24886/", + "long_name": "elevenforum.com", + "img": "https://imgs.search.brave.com/XVRAYMEj6Im8i7jV5RxeTwpiRPtY9IWg4wRIuh-WhEw/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvZjk5MDZkMDIw/M2U1OWIwNjM5Y2U1/M2U2NzNiNzVkNTA5/NzA5OTI1ZTFmOTc4/MzU3OTlhYzU5OTVi/ZGNjNTY4MS93d3cu/ZWxldmVuZm9ydW0u/Y29tLw" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "elevenforum.com", + "hostname": "www.elevenforum.com", + "favicon": "https://imgs.search.brave.com/XVRAYMEj6Im8i7jV5RxeTwpiRPtY9IWg4wRIuh-WhEw/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvZjk5MDZkMDIw/M2U1OWIwNjM5Y2U1/M2U2NzNiNzVkNTA5/NzA5OTI1ZTFmOTc4/MzU3OTlhYzU5OTVi/ZGNjNTY4MS93d3cu/ZWxldmVuZm9ydW0u/Y29tLw", + "path": " › windows support forums › apps and software" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/DVoFcE6d_-lx3BVGNS-RZK_lZzxQ8VhwZVf3AVqEJFA/rs:fit:200:200:1/g:ce/aHR0cHM6Ly93d3cu/ZWxldmVuZm9ydW0u/Y29tL2RhdGEvYXNz/ZXRzL2xvZ28vbWV0/YTEtMjAxLnBuZw", + "original": "https://www.elevenforum.com/data/assets/logo/meta1-201.png", + "logo": true + }, + "age": "2 days ago", + "extra_snippets": [ + "Why are python.exe and python3.exe listed as App Installer? I assume that App Installer refers to installation of Microsoft Store / UWP apps, but if that's the case, then why are they called python.exe and python3.exe? Or are python.exe and python3.exe simply serving as aliases / pointers pointing to App Installer, which is itself a Microsoft Store App?", + "Or are python.exe and python3.exe simply serving as aliases / pointers pointing to App Installer, which is itself a Microsoft Store App? I wish to soon install Python, along with an integrated development editor (IDE), on my machine, so that I can code in Python.", + "I wish to soon install Python, along with an integrated development editor (IDE), on my machine, so that I can code in Python. But is a Python interpreter already on my computer as suggested, if obliquely, by the presence of python.exe and python3.exe? I kind of doubt it." + ] + }, + { + "title": "How to Watermark Your Images Using Python OpenCV in ...", + "url": "https://medium.com/@daily_data_prep/how-to-watermark-your-images-using-python-opencv-in-bulk-e472085389a1", + "is_source_local": false, + "is_source_both": false, + "description": "Medium is an open platform where readers find dynamic thinking, and where expert and undiscovered voices can share their writing on any topic.", + "page_age": "2024-05-03T14:05:06", + "profile": { + "name": "Medium", + "url": "https://medium.com/@daily_data_prep/how-to-watermark-your-images-using-python-opencv-in-bulk-e472085389a1", + "long_name": "medium.com", + "img": "https://imgs.search.brave.com/qvE2kIQCiAsnPv2C6P9xM5J2VVWdm55g-A-2Q_yIJ0g/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTZhYmQ1N2Q4/NDg4ZDcyODIyMDZi/MzFmOWNhNjE3Y2E4/Y2YzMThjNjljNDIx/ZjllZmNhYTcwODhl/YTcwNDEzYy9tZWRp/dW0uY29tLw" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "medium.com", + "hostname": "medium.com", + "favicon": "https://imgs.search.brave.com/qvE2kIQCiAsnPv2C6P9xM5J2VVWdm55g-A-2Q_yIJ0g/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvOTZhYmQ1N2Q4/NDg4ZDcyODIyMDZi/MzFmOWNhNjE3Y2E4/Y2YzMThjNjljNDIx/ZjllZmNhYTcwODhl/YTcwNDEzYy9tZWRp/dW0uY29tLw", + "path": "› @daily_data_prep › how-to-watermark-your-images-using-python-opencv-in-bulk-e472085389a1" + }, + "age": "2 days ago" + }, + { + "title": "Increment and Decrement Operators in Python?", + "url": "https://www.tutorialspoint.com/increment-and-decrement-operators-in-python", + "is_source_local": false, + "is_source_both": false, + "description": "Increment and Decrement Operators in Python - Python does not have unary increment/decrement operator (++/--). Instead to increment a value, usea += 1to decrement a value, use −a -= 1Example>>> a = 0 >>> >>> #Increment >>> a +=1 >>> >>> #Decrement >>> a -= 1 >>> >>> #value of a >>> a 0Python ...", + "page_age": "2023-08-23T00:00:00", + "profile": { + "name": "Tutorialspoint", + "url": "https://www.tutorialspoint.com/increment-and-decrement-operators-in-python", + "long_name": "tutorialspoint.com", + "img": "https://imgs.search.brave.com/Wt8BSkivPlFwcU5yBtf7YzuvTuRExyd_502cdABCS5c/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYjcyYjAzYmVl/ODU4MzZiMjJiYTFh/MjJhZDNmNWE4YzA5/MDgyYTZhMDg3NTYw/M2NiY2NiZTUxN2I5/MjU1MWFmMS93d3cu/dHV0b3JpYWxzcG9p/bnQuY29tLw" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "tutorialspoint.com", + "hostname": "www.tutorialspoint.com", + "favicon": "https://imgs.search.brave.com/Wt8BSkivPlFwcU5yBtf7YzuvTuRExyd_502cdABCS5c/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYjcyYjAzYmVl/ODU4MzZiMjJiYTFh/MjJhZDNmNWE4YzA5/MDgyYTZhMDg3NTYw/M2NiY2NiZTUxN2I5/MjU1MWFmMS93d3cu/dHV0b3JpYWxzcG9p/bnQuY29tLw", + "path": "› increment-and-decrement-operators-in-python" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/ddG5vyZGLVudvecEbQJPeG8tGuaZ7g3Xz6Gyjdl5WA8/rs:fit:200:200:1/g:ce/aHR0cHM6Ly93d3cu/dHV0b3JpYWxzcG9p/bnQuY29tL2ltYWdl/cy90cF9sb2dvXzQz/Ni5wbmc", + "original": "https://www.tutorialspoint.com/images/tp_logo_436.png", + "logo": true + }, + "age": "August 23, 2023", + "extra_snippets": [ + "Increment and Decrement Operators in Python - Python does not have unary increment/decrement operator (++/--). Instead to increment a value, usea += 1to decrement a value, use −a -= 1Example>>> a = 0 >>> >>> #Increment >>> a +=1 >>> >>> #Decrement >>> a -= 1 >>> >>> #value of a >>> a 0Python does not provide multiple ways to do the same thing", + "So what above statement means in python is: create an object of type int having value 1 and give the name a to it. The object is an instance of int having value 1 and the name a refers to it. The assigned name a and the object to which it refers are distinct.", + "Python does not provide multiple ways to do the same thing .", + "However, be careful if you are coming from a language like C, Python doesn’t have \"variables\" in the sense that C does, instead python uses names and objects and in python integers (int’s) are immutable." + ] + }, + { + "title": "Gumroad – How not to suck at Python / SideFX Houdini | CG Persia", + "url": "https://cgpersia.com/2024/05/gumroad-how-not-to-suck-at-python-sidefx-houdini-195370.html", + "is_source_local": false, + "is_source_both": false, + "description": "Info: This course is made for artists or TD (technical director) willing to learn Python to improve their workflows inside SideFX Houdini, get faster in production and develop all the tools you always wished you had.", + "page_age": "2024-05-03T08:35:03", + "profile": { + "name": "Cgpersia", + "url": "https://cgpersia.com/2024/05/gumroad-how-not-to-suck-at-python-sidefx-houdini-195370.html", + "long_name": "cgpersia.com", + "img": "https://imgs.search.brave.com/VjyaopAm-M9sWvM7n-KnGZ3T5swIOwwE80iF5QVqQPg/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYmE0MzQ4NmI2/NjFhMTA1ZDBiN2Iw/ZWNiNDUxNjUwYjdh/MGE5ZjQ0ZjIxNzll/NmVkZDE2YzYyMDBh/NDNiMDgwMy9jZ3Bl/cnNpYS5jb20v" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "cgpersia.com", + "hostname": "cgpersia.com", + "favicon": "https://imgs.search.brave.com/VjyaopAm-M9sWvM7n-KnGZ3T5swIOwwE80iF5QVqQPg/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvYmE0MzQ4NmI2/NjFhMTA1ZDBiN2Iw/ZWNiNDUxNjUwYjdh/MGE5ZjQ0ZjIxNzll/NmVkZDE2YzYyMDBh/NDNiMDgwMy9jZ3Bl/cnNpYS5jb20v", + "path": "› 2024 › 05 › gumroad-how-not-to-suck-at-python-sidefx-houdini-195370.html" + }, + "age": "2 days ago", + "extra_snippets": [ + "Posted in: 2D, CG Releases, Downloads, Learning, Tutorials, Videos. Tagged: Gumroad, Python, Sidefx. Leave a Comment", + "01 – Python – Fundamentals Get the Fundamentals of python before starting the fun stuff ! 02 – Python Construction Part02 digging further into python concepts 03 – Houdini – Python Basics Applying some basic python in Houdini and starting to make tools !", + "02 – Python Construction Part02 digging further into python concepts 03 – Houdini – Python Basics Applying some basic python in Houdini and starting to make tools ! 04 – Houdini – Python Intermediate Applying some more advanced python in Houdini to make tools ! 05 – Houdini – Python Expert Using QtDesigner in combinaison with Houdini Python/Pyside to create advanced tools." + ] + }, + { + "title": "How to install Python: The complete Python programmer’s guide", + "url": "https://www.pluralsight.com/resources/blog/software-development/python-installation-guide", + "is_source_local": false, + "is_source_both": false, + "description": "An easy guide on how set up your operating system so you can program in Python, and how to update or uninstall it. For Linux, Windows, and macOS.", + "page_age": "2024-05-02T07:30:02", + "profile": { + "name": "Pluralsight", + "url": "https://www.pluralsight.com/resources/blog/software-development/python-installation-guide", + "long_name": "pluralsight.com", + "img": "https://imgs.search.brave.com/zvwQNSVu9-jR2CRlNcsTzxjaXKPlXNuh-Jo9-0yA1OE/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvMTNkNWQyNjk3/M2Q0NzYyMmUyNDc3/ZjYwMWFlZDI5YTI4/ODhmYzc2MDkzMjAy/MjNkMWY1MDE3NTQw/MzI5NWVkZS93d3cu/cGx1cmFsc2lnaHQu/Y29tLw" + }, + "language": "en", + "family_friendly": true, + "type": "search_result", + "subtype": "generic", + "meta_url": { + "scheme": "https", + "netloc": "pluralsight.com", + "hostname": "www.pluralsight.com", + "favicon": "https://imgs.search.brave.com/zvwQNSVu9-jR2CRlNcsTzxjaXKPlXNuh-Jo9-0yA1OE/rs:fit:32:32:1/g:ce/aHR0cDovL2Zhdmlj/b25zLnNlYXJjaC5i/cmF2ZS5jb20vaWNv/bnMvMTNkNWQyNjk3/M2Q0NzYyMmUyNDc3/ZjYwMWFlZDI5YTI4/ODhmYzc2MDkzMjAy/MjNkMWY1MDE3NTQw/MzI5NWVkZS93d3cu/cGx1cmFsc2lnaHQu/Y29tLw", + "path": " › blog › blog" + }, + "thumbnail": { + "src": "https://imgs.search.brave.com/xrv5PHH2Bzmq2rcIYzk__8h5RqCj6kS3I6SGCNw5dZM/rs:fit:200:200:1/g:ce/aHR0cHM6Ly93d3cu/cGx1cmFsc2lnaHQu/Y29tL2NvbnRlbnQv/ZGFtL3BzL2ltYWdl/cy9yZXNvdXJjZS1j/ZW50ZXIvYmxvZy9o/ZWFkZXItaGVyby1p/bWFnZXMvUHl0aG9u/LndlYnA", + "original": "https://www.pluralsight.com/content/dam/ps/images/resource-center/blog/header-hero-images/Python.webp", + "logo": false + }, + "age": "3 days ago", + "extra_snippets": [ + "Whether it’s your first time programming or you’re a seasoned programmer, you’ll have to install or update Python every now and then --- or if necessary, uninstall it. In this article, you'll learn how to do just that.", + "Some systems come with Python, so to start off, we’ll first check to see if it’s installed on your system before we proceed. To do that, we’ll need to open a terminal. Since you might be new to programming, let’s go over how to open a terminal for Linux, Windows, and macOS.", + "Before we dive into setting up your system so you can program in Python, let’s talk terminal basics and benefits.", + "However, let’s focus on why we need it for working with Python. We use a terminal, or command line, to:" + ] + } + ], + "family_friendly": true + } +} diff --git a/backend/apps/rag/search/testdata/google_pse.json b/backend/apps/rag/search/testdata/google_pse.json new file mode 100644 index 0000000000..15da9729cd --- /dev/null +++ b/backend/apps/rag/search/testdata/google_pse.json @@ -0,0 +1,442 @@ +{ + "kind": "customsearch#search", + "url": { + "type": "application/json", + "template": "https://www.googleapis.com/customsearch/v1?q={searchTerms}&num={count?}&start={startIndex?}&lr={language?}&safe={safe?}&cx={cx?}&sort={sort?}&filter={filter?}&gl={gl?}&cr={cr?}&googlehost={googleHost?}&c2coff={disableCnTwTranslation?}&hq={hq?}&hl={hl?}&siteSearch={siteSearch?}&siteSearchFilter={siteSearchFilter?}&exactTerms={exactTerms?}&excludeTerms={excludeTerms?}&linkSite={linkSite?}&orTerms={orTerms?}&dateRestrict={dateRestrict?}&lowRange={lowRange?}&highRange={highRange?}&searchType={searchType}&fileType={fileType?}&rights={rights?}&imgSize={imgSize?}&imgType={imgType?}&imgColorType={imgColorType?}&imgDominantColor={imgDominantColor?}&alt=json" + }, + "queries": { + "request": [ + { + "title": "Google Custom Search - lectures", + "totalResults": "2450000000", + "searchTerms": "lectures", + "count": 10, + "startIndex": 1, + "inputEncoding": "utf8", + "outputEncoding": "utf8", + "safe": "off", + "cx": "0473ef98502d44e18" + } + ], + "nextPage": [ + { + "title": "Google Custom Search - lectures", + "totalResults": "2450000000", + "searchTerms": "lectures", + "count": 10, + "startIndex": 11, + "inputEncoding": "utf8", + "outputEncoding": "utf8", + "safe": "off", + "cx": "0473ef98502d44e18" + } + ] + }, + "context": { + "title": "LLM Search" + }, + "searchInformation": { + "searchTime": 0.445959, + "formattedSearchTime": "0.45", + "totalResults": "2450000000", + "formattedTotalResults": "2,450,000,000" + }, + "items": [ + { + "kind": "customsearch#result", + "title": "The Feynman Lectures on Physics", + "htmlTitle": "The Feynman \u003cb\u003eLectures\u003c/b\u003e on Physics", + "link": "https://www.feynmanlectures.caltech.edu/", + "displayLink": "www.feynmanlectures.caltech.edu", + "snippet": "This edition has been designed for ease of reading on devices of any size or shape; text, figures and equations can all be zoomed without degradation.", + "htmlSnippet": "This edition has been designed for ease of reading on devices of any size or shape; text, figures and equations can all be zoomed without degradation.", + "cacheId": "CyXMWYWs9UEJ", + "formattedUrl": "https://www.feynmanlectures.caltech.edu/", + "htmlFormattedUrl": "https://www.feynman\u003cb\u003electures\u003c/b\u003e.caltech.edu/", + "pagemap": { + "metatags": [ + { + "viewport": "width=device-width, initial-scale=1.0" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Video Lectures", + "htmlTitle": "Video \u003cb\u003eLectures\u003c/b\u003e", + "link": "https://www.reddit.com/r/lectures/", + "displayLink": "www.reddit.com", + "snippet": "r/lectures: This subreddit is all about video lectures, talks and interesting public speeches. The topics include mathematics, physics, computer…", + "htmlSnippet": "r/\u003cb\u003electures\u003c/b\u003e: This subreddit is all about video \u003cb\u003electures\u003c/b\u003e, talks and interesting public speeches. The topics include mathematics, physics, computer…", + "formattedUrl": "https://www.reddit.com/r/lectures/", + "htmlFormattedUrl": "https://www.reddit.com/r/\u003cb\u003electures\u003c/b\u003e/", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTZtOjhfkgUKQbL3DZxe5F6OVsgeDNffleObjJ7n9RllKQTSsimax7VIaY&s", + "width": "192", + "height": "192" + } + ], + "metatags": [ + { + "og:image": "https://www.redditstatic.com/shreddit/assets/favicon/192x192.png", + "theme-color": "#000000", + "og:image:width": "256", + "og:type": "website", + "twitter:card": "summary", + "twitter:title": "r/lectures", + "og:site_name": "Reddit", + "og:title": "r/lectures", + "og:image:height": "256", + "bingbot": "noarchive", + "msapplication-navbutton-color": "#000000", + "og:description": "This subreddit is all about video lectures, talks and interesting public speeches.\n\nThe topics include mathematics, physics, computer science, programming, engineering, biology, medicine, economics, politics, social sciences, and any other subjects!", + "twitter:image": "https://www.redditstatic.com/shreddit/assets/favicon/192x192.png", + "apple-mobile-web-app-status-bar-style": "black", + "twitter:site": "@reddit", + "viewport": "width=device-width, initial-scale=1, viewport-fit=cover", + "apple-mobile-web-app-capable": "yes", + "og:ttl": "600", + "og:url": "https://www.reddit.com/r/lectures/" + } + ], + "cse_image": [ + { + "src": "https://www.redditstatic.com/shreddit/assets/favicon/192x192.png" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Lectures & Discussions | Flint Institute of Arts", + "htmlTitle": "\u003cb\u003eLectures\u003c/b\u003e & Discussions | Flint Institute of Arts", + "link": "https://flintarts.org/events/lectures", + "displayLink": "flintarts.org", + "snippet": "It will trace the intricate relationship between jewelry, attire, and the expression of personal identity, social hierarchy, and spiritual belief systems that ...", + "htmlSnippet": "It will trace the intricate relationship between jewelry, attire, and the expression of personal identity, social hierarchy, and spiritual belief systems that ...", + "cacheId": "jvpb9DxrfxoJ", + "formattedUrl": "https://flintarts.org/events/lectures", + "htmlFormattedUrl": "https://flintarts.org/events/\u003cb\u003electures\u003c/b\u003e", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcS23tMtAeNhJbOWdGxShYsmnyzFdzOC9Hb7lRykA9Pw72z1IlKTkjTdZw&s", + "width": "447", + "height": "113" + } + ], + "metatags": [ + { + "og:image": "https://flintarts.org/uploads/images/page-headers/_headerImage/nightshot.jpg", + "og:type": "website", + "viewport": "width=device-width, initial-scale=1", + "og:title": "Lectures & Discussions | Flint Institute of Arts", + "og:description": "The Flint Institute of Arts is the second largest art museum in Michigan and one of the largest museum art schools in the nation." + } + ], + "cse_image": [ + { + "src": "https://flintarts.org/uploads/images/page-headers/_headerImage/nightshot.jpg" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Mandel Lectures | Mandel Center for the Humanities ... - Waltham", + "htmlTitle": "Mandel \u003cb\u003eLectures\u003c/b\u003e | Mandel Center for the Humanities ... - Waltham", + "link": "https://www.brandeis.edu/mandel-center-humanities/mandel-lectures.html", + "displayLink": "www.brandeis.edu", + "snippet": "Past Lectures · Lecture 1: \"Invisible Music: The Sonic Idea of Black Revolution From Captivity to Reconstruction\" · Lecture 2: \"Solidarity in Sound: Grassroots ...", + "htmlSnippet": "Past \u003cb\u003eLectures\u003c/b\u003e · \u003cb\u003eLecture\u003c/b\u003e 1: "Invisible Music: The Sonic Idea of Black Revolution From Captivity to Reconstruction" · \u003cb\u003eLecture\u003c/b\u003e 2: "Solidarity in Sound: Grassroots ...", + "cacheId": "cQLOZr0kgEEJ", + "formattedUrl": "https://www.brandeis.edu/mandel-center-humanities/mandel-lectures.html", + "htmlFormattedUrl": "https://www.brandeis.edu/mandel-center-humanities/mandel-\u003cb\u003electures\u003c/b\u003e.html", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQWlU7bcJ5pIHk7RBCk2QKE-48ejF7hyPV0pr-20_cBt2BGdfKtiYXBuyw&s", + "width": "275", + "height": "183" + } + ], + "metatags": [ + { + "og:image": "https://www.brandeis.edu/mandel-center-humanities/events/events-images/mlhzumba", + "twitter:card": "summary_large_image", + "viewport": "width=device-width,initial-scale=1,minimum-scale=1", + "og:title": "Mandel Lectures in the Humanities", + "og:url": "https://www.brandeis.edu/mandel-center-humanities/mandel-lectures.html", + "og:description": "Annual Lecture Series", + "twitter:image": "https://www.brandeis.edu/mandel-center-humanities/events/events-images/mlhzumba" + } + ], + "cse_image": [ + { + "src": "https://www.brandeis.edu/mandel-center-humanities/events/events-images/mlhzumba" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Brian Douglas - YouTube", + "htmlTitle": "Brian Douglas - YouTube", + "link": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg", + "displayLink": "www.youtube.com", + "snippet": "Welcome to Control Systems Lectures! This collection of videos is intended to supplement a first year controls class, not replace it.", + "htmlSnippet": "Welcome to Control Systems \u003cb\u003eLectures\u003c/b\u003e! This collection of videos is intended to supplement a first year controls class, not replace it.", + "cacheId": "NEROyBHolL0J", + "formattedUrl": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg", + "htmlFormattedUrl": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg", + "pagemap": { + "hcard": [ + { + "fn": "Brian Douglas", + "url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg" + } + ], + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcR7G0CeCBz_wVTZgjnhEr2QbiKP7f3uYzKitZYn74Mi32cDmVxvsegJoLI&s", + "width": "225", + "height": "225" + } + ], + "imageobject": [ + { + "width": "900", + "url": "https://yt3.googleusercontent.com/ytc/AIdro_nLo68wetImbwGUYP3stve_iKmAEccjhqB-q4o79xdInN4=s900-c-k-c0x00ffffff-no-rj", + "height": "900" + } + ], + "person": [ + { + "name": "Brian Douglas", + "url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg" + } + ], + "metatags": [ + { + "apple-itunes-app": "app-id=544007664, app-argument=https://m.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg?referring_app=com.apple.mobilesafari-smartbanner, affiliate-data=ct=smart_app_banner_polymer&pt=9008", + "og:image": "https://yt3.googleusercontent.com/ytc/AIdro_nLo68wetImbwGUYP3stve_iKmAEccjhqB-q4o79xdInN4=s900-c-k-c0x00ffffff-no-rj", + "twitter:app:url:iphone": "vnd.youtube://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg", + "twitter:app:id:googleplay": "com.google.android.youtube", + "theme-color": "rgb(255, 255, 255)", + "og:image:width": "900", + "twitter:card": "summary", + "og:site_name": "YouTube", + "twitter:url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg", + "twitter:app:url:ipad": "vnd.youtube://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg", + "al:android:package": "com.google.android.youtube", + "twitter:app:name:googleplay": "YouTube", + "al:ios:url": "vnd.youtube://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg", + "twitter:app:id:iphone": "544007664", + "og:description": "Welcome to Control Systems Lectures! This collection of videos is intended to supplement a first year controls class, not replace it. My goal is to take specific concepts in controls and expand on them in order to provide an intuitive understanding which will ultimately make you a better controls engineer. \n\nI'm glad you made it to my channel and I hope you find it useful.\n\nShoot me a message at controlsystemlectures@gmail.com, leave a comment or question and I'll get back to you if I can. Don't forget to subscribe!\n \nTwitter: @BrianBDouglas for engineering tweets and announcement of new videos.\nWebpage: http://engineeringmedia.com\n\nHere is the hardware/software I use: http://www.youtube.com/watch?v=m-M5_mIyHe4\n\nHere's a list of my favorite references: http://bit.ly/2skvmWd\n\n--Brian", + "al:ios:app_store_id": "544007664", + "twitter:image": "https://yt3.googleusercontent.com/ytc/AIdro_nLo68wetImbwGUYP3stve_iKmAEccjhqB-q4o79xdInN4=s900-c-k-c0x00ffffff-no-rj", + "twitter:site": "@youtube", + "og:type": "profile", + "twitter:title": "Brian Douglas", + "al:ios:app_name": "YouTube", + "og:title": "Brian Douglas", + "og:image:height": "900", + "twitter:app:id:ipad": "544007664", + "al:web:url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg?feature=applinks", + "al:android:url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg?feature=applinks", + "fb:app_id": "87741124305", + "twitter:app:url:googleplay": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg", + "twitter:app:name:ipad": "YouTube", + "viewport": "width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no,", + "twitter:description": "Welcome to Control Systems Lectures! This collection of videos is intended to supplement a first year controls class, not replace it. My goal is to take specific concepts in controls and expand on them in order to provide an intuitive understanding which will ultimately make you a better controls engineer. \n\nI'm glad you made it to my channel and I hope you find it useful.\n\nShoot me a message at controlsystemlectures@gmail.com, leave a comment or question and I'll get back to you if I can. Don't forget to subscribe!\n \nTwitter: @BrianBDouglas for engineering tweets and announcement of new videos.\nWebpage: http://engineeringmedia.com\n\nHere is the hardware/software I use: http://www.youtube.com/watch?v=m-M5_mIyHe4\n\nHere's a list of my favorite references: http://bit.ly/2skvmWd\n\n--Brian", + "og:url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg", + "al:android:app_name": "YouTube", + "twitter:app:name:iphone": "YouTube" + } + ], + "cse_image": [ + { + "src": "https://yt3.googleusercontent.com/ytc/AIdro_nLo68wetImbwGUYP3stve_iKmAEccjhqB-q4o79xdInN4=s900-c-k-c0x00ffffff-no-rj" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Lecture - Wikipedia", + "htmlTitle": "\u003cb\u003eLecture\u003c/b\u003e - Wikipedia", + "link": "https://en.wikipedia.org/wiki/Lecture", + "displayLink": "en.wikipedia.org", + "snippet": "Lecture ... For the academic rank, see Lecturer. A lecture (from Latin: lēctūra 'reading') is an oral presentation intended to present information or teach people ...", + "htmlSnippet": "\u003cb\u003eLecture\u003c/b\u003e ... For the academic rank, see \u003cb\u003eLecturer\u003c/b\u003e. A \u003cb\u003electure\u003c/b\u003e (from Latin: lēctūra 'reading') is an oral presentation intended to present information or teach people ...", + "cacheId": "d9Pjta02fmgJ", + "formattedUrl": "https://en.wikipedia.org/wiki/Lecture", + "htmlFormattedUrl": "https://en.wikipedia.org/wiki/Lecture", + "pagemap": { + "metatags": [ + { + "referrer": "origin", + "og:image": "https://upload.wikimedia.org/wikipedia/commons/thumb/2/26/ADFA_Lecture_Theatres.jpg/1200px-ADFA_Lecture_Theatres.jpg", + "theme-color": "#eaecf0", + "og:image:width": "1200", + "og:type": "website", + "viewport": "width=device-width, initial-scale=1.0, user-scalable=yes, minimum-scale=0.25, maximum-scale=5.0", + "og:title": "Lecture - Wikipedia", + "og:image:height": "799", + "format-detection": "telephone=no" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Mount Wilson Observatory | Lectures", + "htmlTitle": "Mount Wilson Observatory | \u003cb\u003eLectures\u003c/b\u003e", + "link": "https://www.mtwilson.edu/lectures/", + "displayLink": "www.mtwilson.edu", + "snippet": "Talks & Telescopes: August 24, 2024 – Panel: The Triumph of Hubble ... Compelling talks followed by picnicking and convivial stargazing through both the big ...", + "htmlSnippet": "Talks & Telescopes: August 24, 2024 – Panel: The Triumph of Hubble ... Compelling talks followed by picnicking and convivial stargazing through both the big ...", + "cacheId": "wdXI0azqx5UJ", + "formattedUrl": "https://www.mtwilson.edu/lectures/", + "htmlFormattedUrl": "https://www.mtwilson.edu/\u003cb\u003electures\u003c/b\u003e/", + "pagemap": { + "metatags": [ + { + "viewport": "width=device-width,initial-scale=1,user-scalable=no" + } + ], + "webpage": [ + { + "image": "http://www.mtwilson.edu/wp-content/uploads/2016/09/Logo.jpg", + "url": "https://www.facebook.com/WilsonObs" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Lectures | NBER", + "htmlTitle": "\u003cb\u003eLectures\u003c/b\u003e | NBER", + "link": "https://www.nber.org/research/lectures", + "displayLink": "www.nber.org", + "snippet": "Results 1 - 50 of 354 ... Among featured events at the NBER Summer Institute are the Martin Feldstein Lecture, which examines a current issue involving economic ...", + "htmlSnippet": "Results 1 - 50 of 354 \u003cb\u003e...\u003c/b\u003e Among featured events at the NBER Summer Institute are the Martin Feldstein \u003cb\u003eLecture\u003c/b\u003e, which examines a current issue involving economic ...", + "cacheId": "CvvP3U3nb44J", + "formattedUrl": "https://www.nber.org/research/lectures", + "htmlFormattedUrl": "https://www.nber.org/research/\u003cb\u003electures\u003c/b\u003e", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTmeViEZyV1YmFEFLhcA6WdgAG3v3RV6tB93ncyxSJ5JPst_p2aWrL7D1k&s", + "width": "310", + "height": "163" + } + ], + "metatags": [ + { + "og:image": "https://www.nber.org/sites/default/files/2022-06/NBER-FB-Share-Tile-1200.jpg", + "og:site_name": "NBER", + "handheldfriendly": "true", + "viewport": "width=device-width, initial-scale=1.0", + "og:title": "Lectures", + "mobileoptimized": "width", + "og:url": "https://www.nber.org/research/lectures" + } + ], + "cse_image": [ + { + "src": "https://www.nber.org/sites/default/files/2022-06/NBER-FB-Share-Tile-1200.jpg" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "STUDENTS CANNOT ACCESS RECORDED LECTURES ... - Solved", + "htmlTitle": "STUDENTS CANNOT ACCESS RECORDED LECTURES ... - Solved", + "link": "https://community.canvaslms.com/t5/Canvas-Question-Forum/STUDENTS-CANNOT-ACCESS-RECORDED-LECTURES/td-p/190358", + "displayLink": "community.canvaslms.com", + "snippet": "Mar 19, 2020 ... I believe the issue is that students were not invited. Are you trying to capture your screen? If not, there is an option to just record your web ...", + "htmlSnippet": "Mar 19, 2020 \u003cb\u003e...\u003c/b\u003e I believe the issue is that students were not invited. Are you trying to capture your screen? If not, there is an option to just record your web ...", + "cacheId": "wqrynQXX61sJ", + "formattedUrl": "https://community.canvaslms.com/t5/Canvas...LECTURES/td-p/190358", + "htmlFormattedUrl": "https://community.canvaslms.com/t5/Canvas...\u003cb\u003eLECTURES\u003c/b\u003e/td-p/190358", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRUqXau3N8LfKgSD7OJOvV7xzGarLKRU-ckWXy1ZQ1p4CLPsedvLKmLMhk&s", + "width": "310", + "height": "163" + } + ], + "metatags": [ + { + "og:image": "https://community.canvaslms.com/html/@6A1FDD4D5FF35E4BBB4083A1022FA0DB/assets/CommunityPreview23.png", + "og:type": "article", + "article:section": "Canvas Question Forum", + "article:published_time": "2020-03-19T15:50:03.409Z", + "og:site_name": "Instructure Community", + "article:modified_time": "2020-03-19T13:55:53-07:00", + "viewport": "width=device-width, initial-scale=1.0, user-scalable=yes", + "og:title": "STUDENTS CANNOT ACCESS RECORDED LECTURES", + "og:url": "https://community.canvaslms.com/t5/Canvas-Question-Forum/STUDENTS-CANNOT-ACCESS-RECORDED-LECTURES/m-p/190358#M93667", + "og:description": "I can access and see my recorded lectures but my students can't. They have an error message when they try to open the recorded presentation or notes.", + "article:author": "https://community.canvaslms.com/t5/user/viewprofilepage/user-id/794287", + "twitter:image": "https://community.canvaslms.com/html/@6A1FDD4D5FF35E4BBB4083A1022FA0DB/assets/CommunityPreview23.png" + } + ], + "cse_image": [ + { + "src": "https://community.canvaslms.com/html/@6A1FDD4D5FF35E4BBB4083A1022FA0DB/assets/CommunityPreview23.png" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Public Lecture Series - Sam Fox School of Design & Visual Arts", + "htmlTitle": "Public \u003cb\u003eLecture\u003c/b\u003e Series - Sam Fox School of Design & Visual Arts", + "link": "https://samfoxschool.wustl.edu/calendar/series/2-public-lecture-series", + "displayLink": "samfoxschool.wustl.edu", + "snippet": "The Sam Fox School's Spring 2024 Public Lecture Series highlights design and art as catalysts for change. Renowned speakers will delve into themes like ...", + "htmlSnippet": "The Sam Fox School's Spring 2024 Public \u003cb\u003eLecture\u003c/b\u003e Series highlights design and art as catalysts for change. Renowned speakers will delve into themes like ...", + "cacheId": "B-cgQG0j6tUJ", + "formattedUrl": "https://samfoxschool.wustl.edu/calendar/series/2-public-lecture-series", + "htmlFormattedUrl": "https://samfoxschool.wustl.edu/calendar/series/2-public-lecture-series", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQSmHaGianm-64m-qauYjkPK_Q0JKWe-7yom4m1ogFYTmpWArA7k6dmk0sR&s", + "width": "307", + "height": "164" + } + ], + "website": [ + { + "name": "Public Lecture Series - Sam Fox School of Design & Visual Arts — Washington University in St. Louis" + } + ], + "metatags": [ + { + "og:image": "https://dvsp0hlm0xrn3.cloudfront.net/assets/default_og_image-44e73dee4b9d1e2c6a6295901371270c8ec5899eaed48ee8167a9b12f1b0f8b3.jpg", + "og:type": "website", + "og:site_name": "Sam Fox School of Design & Visual Arts — Washington University in St. Louis", + "viewport": "width=device-width, initial-scale=1.0", + "og:title": "Public Lecture Series - Sam Fox School of Design & Visual Arts — Washington University in St. Louis", + "csrf-token": "jBQsfZGY3RH8NVs0-KVDBYB-2N2kib4UYZHYdrShfTdLkvzfSvGeOaMrRKTRdYBPRKzdcGIuP7zwm9etqX_uvg", + "csrf-param": "authenticity_token", + "og:description": "The Sam Fox School's Spring 2024 Public Lecture Series highlights design and art as catalysts for change. Renowned speakers will delve into themes like social equity, resilient cities, and the impact of emerging technologies on contemporary life. Speakers include artists, architects, designers, and critics of the highest caliber, widely recognized for their research-based practices and multidisciplinary approaches to their fields." + } + ], + "cse_image": [ + { + "src": "https://dvsp0hlm0xrn3.cloudfront.net/assets/default_og_image-44e73dee4b9d1e2c6a6295901371270c8ec5899eaed48ee8167a9b12f1b0f8b3.jpg" + } + ] + } + } + ] +} diff --git a/backend/apps/rag/search/testdata/searxng.json b/backend/apps/rag/search/testdata/searxng.json new file mode 100644 index 0000000000..0e6952baa8 --- /dev/null +++ b/backend/apps/rag/search/testdata/searxng.json @@ -0,0 +1,476 @@ +{ + "query": "python", + "number_of_results": 116000000, + "results": [ + { + "url": "https://www.python.org/", + "title": "Welcome to Python.org", + "content": "Python is a versatile and powerful language that lets you work quickly and integrate systems more effectively. Learn how to get started, download the latest version, access documentation, find jobs, and join the Python community.", + "engine": "bing", + "parsed_url": ["https", "www.python.org", "/", "", "", ""], + "template": "default.html", + "engines": ["bing", "qwant", "duckduckgo"], + "positions": [1, 1, 1], + "score": 9.0, + "category": "general" + }, + { + "url": "https://wiki.nerdvpn.de/wiki/Python_(programming_language)", + "title": "Python (programming language) - Wikipedia", + "content": "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected. It supports multiple programming paradigms, including structured (particularly procedural), object-oriented and functional programming.", + "engine": "bing", + "parsed_url": ["https", "wiki.nerdvpn.de", "/wiki/Python_(programming_language)", "", "", ""], + "template": "default.html", + "engines": ["bing", "qwant", "duckduckgo"], + "positions": [4, 3, 2], + "score": 3.25, + "category": "general" + }, + { + "url": "https://docs.python.org/3/tutorial/index.html", + "title": "The Python Tutorial \u2014 Python 3.12.3 documentation", + "content": "3 days ago \u00b7 Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python\u2019s elegant syntax and dynamic typing, together with its interpreted nature, make it an ideal language for scripting and rapid application development in many \u2026", + "engine": "bing", + "parsed_url": ["https", "docs.python.org", "/3/tutorial/index.html", "", "", ""], + "template": "default.html", + "engines": ["bing", "qwant", "duckduckgo"], + "positions": [5, 5, 3], + "score": 2.2, + "category": "general" + }, + { + "url": "https://www.python.org/downloads/", + "title": "Download Python | Python.org", + "content": "Python is a popular programming language for various purposes. Find the latest version of Python for different operating systems, download release notes, and learn about the development process.", + "engine": "bing", + "parsed_url": ["https", "www.python.org", "/downloads/", "", "", ""], + "template": "default.html", + "engines": ["bing", "duckduckgo"], + "positions": [2, 2], + "score": 2.0, + "category": "general" + }, + { + "url": "https://www.python.org/about/gettingstarted/", + "title": "Python For Beginners | Python.org", + "content": "Learn the basics of Python, a popular and easy-to-use programming language, from installing it to using it for various purposes. Find out how to access online documentation, tutorials, books, code samples, and more resources to help you get started with Python.", + "engine": "bing", + "parsed_url": ["https", "www.python.org", "/about/gettingstarted/", "", "", ""], + "template": "default.html", + "engines": ["bing", "qwant", "duckduckgo"], + "positions": [9, 4, 4], + "score": 1.8333333333333333, + "category": "general" + }, + { + "url": "https://www.python.org/shell/", + "title": "Welcome to Python.org", + "content": "Python is a versatile and easy-to-use programming language that lets you work quickly. Learn more about Python, download the latest version, access documentation, find jobs, and join the community.", + "engine": "bing", + "parsed_url": ["https", "www.python.org", "/shell/", "", "", ""], + "template": "default.html", + "engines": ["bing", "qwant", "duckduckgo"], + "positions": [3, 10, 8], + "score": 1.675, + "category": "general" + }, + { + "url": "https://realpython.com/", + "title": "Python Tutorials \u2013 Real Python", + "content": "Real Python offers comprehensive and up-to-date tutorials, books, and courses for Python developers of all skill levels. Whether you want to learn Python basics, web development, data science, machine learning, or more, you can find clear and practical guides and code examples here.", + "engine": "bing", + "parsed_url": ["https", "realpython.com", "/", "", "", ""], + "template": "default.html", + "engines": ["bing", "qwant", "duckduckgo"], + "positions": [6, 6, 5], + "score": 1.6, + "category": "general" + }, + { + "url": "https://wiki.nerdvpn.de/wiki/Python", + "title": "Python", + "content": "Topics referred to by the same term", + "engine": "wikipedia", + "parsed_url": ["https", "wiki.nerdvpn.de", "/wiki/Python", "", "", ""], + "template": "default.html", + "engines": ["wikipedia"], + "positions": [1], + "score": 1.0, + "category": "general" + }, + { + "title": "Online Python - IDE, Editor, Compiler, Interpreter", + "content": "Online Python IDE is a free online tool that lets you write, execute, and share Python code in the web browser. Learn about Python, its features, and its popularity as a general-purpose programming language for web development, data science, and more.", + "url": "https://www.online-python.com/", + "engine": "duckduckgo", + "parsed_url": ["https", "www.online-python.com", "/", "", "", ""], + "template": "default.html", + "engines": ["qwant", "duckduckgo"], + "positions": [8, 6], + "score": 0.5833333333333333, + "category": "general" + }, + { + "url": "https://micropython.org/", + "title": "MicroPython - Python for microcontrollers", + "content": "MicroPython is a full Python compiler and runtime that runs on the bare-metal. You get an interactive prompt (the REPL) to execute commands immediately, along ...", + "img_src": null, + "engine": "google", + "parsed_url": ["https", "micropython.org", "/", "", "", ""], + "template": "default.html", + "engines": ["google"], + "positions": [1], + "score": 1.0, + "category": "general" + }, + { + "url": "https://dictionary.cambridge.org/uk/dictionary/english/python", + "title": "PYTHON | \u0417\u043d\u0430\u0447\u0435\u043d\u043d\u044f \u0432 \u0430\u043d\u0433\u043b\u0456\u0439\u0441\u044c\u043a\u0456\u0439 \u043c\u043e\u0432\u0456 - Cambridge Dictionary", + "content": "Apr 17, 2024 \u2014 \u0412\u0438\u0437\u043d\u0430\u0447\u0435\u043d\u043d\u044f PYTHON: 1. a very large snake that kills animals for food by wrapping itself around them and crushing them\u2026. \u0414\u0456\u0437\u043d\u0430\u0439\u0442\u0435\u0441\u044f \u0431\u0456\u043b\u044c\u0448\u0435.", + "img_src": null, + "engine": "google", + "parsed_url": [ + "https", + "dictionary.cambridge.org", + "/uk/dictionary/english/python", + "", + "", + "" + ], + "template": "default.html", + "engines": ["google"], + "positions": [2], + "score": 0.5, + "category": "general" + }, + { + "url": "https://www.codetoday.co.uk/code", + "title": "Web-based Python Editor (with Turtle graphics)", + "content": "Quick way of starting to write Python code, including drawing with Turtle, provided by CodeToday using Trinket.io Ideal for young children to start ...", + "img_src": null, + "engine": "google", + "parsed_url": ["https", "www.codetoday.co.uk", "/code", "", "", ""], + "template": "default.html", + "engines": ["google"], + "positions": [3], + "score": 0.3333333333333333, + "category": "general" + }, + { + "url": "https://snapcraft.io/docs/python-plugin", + "title": "The python plugin | Snapcraft documentation", + "content": "The python plugin can be used by either Python 2 or Python 3 based parts using a setup.py script for building the project, or using a package published to ...", + "img_src": null, + "engine": "google", + "parsed_url": ["https", "snapcraft.io", "/docs/python-plugin", "", "", ""], + "template": "default.html", + "engines": ["google"], + "positions": [4], + "score": 0.25, + "category": "general" + }, + { + "url": "https://www.developer-tech.com/categories/developer-languages/developer-languages-python/", + "title": "Latest Python Developer News", + "content": "Python's status as the primary language for AI and machine learning projects, from its extensive data-handling capabilities to its flexibility and ...", + "img_src": null, + "engine": "google", + "parsed_url": [ + "https", + "www.developer-tech.com", + "/categories/developer-languages/developer-languages-python/", + "", + "", + "" + ], + "template": "default.html", + "engines": ["google"], + "positions": [5], + "score": 0.2, + "category": "general" + }, + { + "url": "https://subjectguides.york.ac.uk/coding/python", + "title": "Coding: a Practical Guide - Python - Subject Guides", + "content": "Python is a coding language used for a wide range of things, including working with data, building systems and software, and even creating games.", + "img_src": null, + "engine": "google", + "parsed_url": ["https", "subjectguides.york.ac.uk", "/coding/python", "", "", ""], + "template": "default.html", + "engines": ["google"], + "positions": [6], + "score": 0.16666666666666666, + "category": "general" + }, + { + "url": "https://hub.salford.ac.uk/psytech/python/getting-started-python/", + "title": "Getting Started - Python - Salford PsyTech Home - The Hub", + "content": "Python in itself is a very friendly programming language, when we get to grips with writing code, once you grasp the logic, it will become very intuitive.", + "img_src": null, + "engine": "google", + "parsed_url": [ + "https", + "hub.salford.ac.uk", + "/psytech/python/getting-started-python/", + "", + "", + "" + ], + "template": "default.html", + "engines": ["google"], + "positions": [7], + "score": 0.14285714285714285, + "category": "general" + }, + { + "url": "https://snapcraft.io/docs/python-apps", + "title": "Python apps | Snapcraft documentation", + "content": "Snapcraft can be used to package and distribute Python applications in a way that enables convenient installation by users. The process of creating a snap ...", + "img_src": null, + "engine": "google", + "parsed_url": ["https", "snapcraft.io", "/docs/python-apps", "", "", ""], + "template": "default.html", + "engines": ["google"], + "positions": [8], + "score": 0.125, + "category": "general" + }, + { + "url": "https://anvil.works/", + "title": "Anvil | Build Web Apps with Nothing but Python", + "content": "Anvil is a free Python-based drag-and-drop web app builder.\u200eSign Up \u00b7 \u200eSign in \u00b7 \u200ePricing \u00b7 \u200eForum", + "img_src": null, + "engine": "google", + "parsed_url": ["https", "anvil.works", "/", "", "", ""], + "template": "default.html", + "engines": ["google"], + "positions": [9], + "score": 0.1111111111111111, + "category": "general" + }, + { + "url": "https://docs.python.org/", + "title": "Python 3.12.3 documentation", + "content": "3 days ago \u00b7 This is the official documentation for Python 3.12.3. Documentation sections: What's new in Python 3.12? Or all \"What's new\" documents since Python 2.0. Tutorial. Start here: a tour of Python's syntax and features. Library reference. Standard library and builtins. Language reference.", + "engine": "bing", + "parsed_url": ["https", "docs.python.org", "/", "", "", ""], + "template": "default.html", + "engines": ["bing", "duckduckgo"], + "positions": [7, 13], + "score": 0.43956043956043955, + "category": "general" + }, + { + "title": "How to Use Python: Your First Steps - Real Python", + "content": "Learn the basics of Python syntax, installation, error handling, and more in this tutorial. You'll also code your first Python program and test your knowledge with a quiz.", + "url": "https://realpython.com/python-first-steps/", + "engine": "duckduckgo", + "parsed_url": ["https", "realpython.com", "/python-first-steps/", "", "", ""], + "template": "default.html", + "engines": ["qwant", "duckduckgo"], + "positions": [14, 7], + "score": 0.42857142857142855, + "category": "general" + }, + { + "title": "The Python Tutorial \u2014 Python 3.11.8 documentation", + "content": "This tutorial introduces the reader informally to the basic concepts and features of the Python language and system. It helps to have a Python interpreter handy for hands-on experience, but all examples are self-contained, so the tutorial can be read off-line as well. For a description of standard objects and modules, see The Python Standard ...", + "url": "https://docs.python.org/3.11/tutorial/", + "engine": "duckduckgo", + "parsed_url": ["https", "docs.python.org", "/3.11/tutorial/", "", "", ""], + "template": "default.html", + "engines": ["duckduckgo"], + "positions": [7], + "score": 0.14285714285714285, + "category": "general" + }, + { + "url": "https://realpython.com/python-introduction/", + "title": "Introduction to Python 3 \u2013 Real Python", + "content": "Python programming language, including a brief history of the development of Python and reasons why you might select Python as your language of choice.", + "engine": "bing", + "parsed_url": ["https", "realpython.com", "/python-introduction/", "", "", ""], + "template": "default.html", + "engines": ["bing"], + "positions": [8], + "score": 0.125, + "category": "general" + }, + { + "title": "Our Documentation | Python.org", + "content": "Find online or download Python's documentation, tutorials, and guides for beginners and advanced users. Learn how to port from Python 2 to Python 3, contribute to Python, and access Python videos and books.", + "url": "https://www.python.org/doc/", + "engine": "duckduckgo", + "parsed_url": ["https", "www.python.org", "/doc/", "", "", ""], + "template": "default.html", + "engines": ["duckduckgo"], + "positions": [9], + "score": 0.1111111111111111, + "category": "general" + }, + { + "title": "Welcome to Python.org", + "url": "http://www.get-python.org/shell/", + "content": "The mission of the Python Software Foundation is to promote, protect, and advance the Python programming language, and to support and facilitate the growth of a diverse and international community of Python programmers. Learn more. Become a Member Donate to the PSF.", + "engine": "qwant", + "parsed_url": ["http", "www.get-python.org", "/shell/", "", "", ""], + "template": "default.html", + "engines": ["qwant"], + "positions": [9], + "score": 0.1111111111111111, + "category": "general" + }, + { + "title": "About Python\u2122 | Python.org", + "content": "Python is a powerful, fast, and versatile programming language that runs on various platforms and is easy to learn. Learn how to get started, explore the applications, and join the community of Python programmers and users.", + "url": "https://www.python.org/about/", + "engine": "duckduckgo", + "parsed_url": ["https", "www.python.org", "/about/", "", "", ""], + "template": "default.html", + "engines": ["duckduckgo"], + "positions": [11], + "score": 0.09090909090909091, + "category": "general" + }, + { + "title": "Online Python Compiler (Interpreter) - Programiz", + "content": "Write and run Python code using this online tool. You can use Python Shell like IDLE, and take inputs from the user in our Python compiler.", + "url": "https://www.programiz.com/python-programming/online-compiler/", + "engine": "duckduckgo", + "parsed_url": [ + "https", + "www.programiz.com", + "/python-programming/online-compiler/", + "", + "", + "" + ], + "template": "default.html", + "engines": ["duckduckgo"], + "positions": [12], + "score": 0.08333333333333333, + "category": "general" + }, + { + "title": "Welcome to Python.org", + "content": "Python is a versatile and powerful language that lets you work quickly and integrate systems more effectively. Download the latest version, read the documentation, find jobs, events, success stories, and more on Python.org.", + "url": "https://www.python.org/?downloads", + "engine": "duckduckgo", + "parsed_url": ["https", "www.python.org", "/", "", "downloads", ""], + "template": "default.html", + "engines": ["duckduckgo"], + "positions": [15], + "score": 0.06666666666666667, + "category": "general" + }, + { + "url": "https://www.matillion.com/blog/the-importance-of-python-and-its-growing-influence-on-data-productivty-a-matillion-perspective", + "title": "The Importance of Python and its Growing Influence on ...", + "content": "Jan 30, 2024 \u2014 The synergy of low-code functionality with Python's versatility empowers data professionals to orchestrate complex transformations seamlessly.", + "img_src": null, + "engine": "google", + "parsed_url": [ + "https", + "www.matillion.com", + "/blog/the-importance-of-python-and-its-growing-influence-on-data-productivty-a-matillion-perspective", + "", + "", + "" + ], + "template": "default.html", + "engines": ["google"], + "positions": [10], + "score": 0.1, + "category": "general" + }, + { + "title": "BeginnersGuide - Python Wiki", + "content": "This is the program that reads Python programs and carries out their instructions; you need it before you can do any Python programming. Mac and Linux distributions may include an outdated version of Python (Python 2), but you should install an updated one (Python 3). See BeginnersGuide/Download for instructions to download the correct version ...", + "url": "https://wiki.python.org/moin/BeginnersGuide", + "engine": "duckduckgo", + "parsed_url": ["https", "wiki.python.org", "/moin/BeginnersGuide", "", "", ""], + "template": "default.html", + "engines": ["duckduckgo"], + "positions": [16], + "score": 0.0625, + "category": "general" + }, + { + "title": "Learn Python - Free Interactive Python Tutorial", + "content": "Learn Python from scratch or improve your skills with this website that offers tutorials, exercises, tests and certification. Explore topics such as basics, data science, advanced features and more with DataCamp.", + "url": "https://www.learnpython.org/", + "engine": "duckduckgo", + "parsed_url": ["https", "www.learnpython.org", "/", "", "", ""], + "template": "default.html", + "engines": ["duckduckgo"], + "positions": [17], + "score": 0.058823529411764705, + "category": "general" + } + ], + "answers": [], + "corrections": [], + "infoboxes": [ + { + "infobox": "Python", + "id": "https://en.wikipedia.org/wiki/Python_(programming_language)", + "content": "general-purpose programming language", + "img_src": "https://upload.wikimedia.org/wikipedia/commons/thumb/6/6f/.PY_file_recreation.png/500px-.PY_file_recreation.png", + "urls": [ + { + "title": "Official website", + "url": "https://www.python.org/", + "official": true + }, + { + "title": "Wikipedia (en)", + "url": "https://en.wikipedia.org/wiki/Python_(programming_language)" + }, + { + "title": "Wikidata", + "url": "http://www.wikidata.org/entity/Q28865" + } + ], + "attributes": [ + { + "label": "Inception", + "value": "Wednesday, February 20, 1991", + "entity": "P571" + }, + { + "label": "Developer", + "value": "Python Software Foundation, Guido van Rossum", + "entity": "P178" + }, + { + "label": "Copyright license", + "value": "Python Software Foundation License", + "entity": "P275" + }, + { + "label": "Programmed in", + "value": "C, Python", + "entity": "P277" + }, + { + "label": "Software version identifier", + "value": "3.12.3, 3.13.0a6", + "entity": "P348" + } + ], + "engine": "wikidata", + "engines": ["wikidata"] + } + ], + "suggestions": [ + "python turtle", + "micro python tutorial", + "python docs", + "python compiler", + "snapcraft python", + "micropython vs python", + "python online", + "python download" + ], + "unresponsive_engines": [] +} diff --git a/backend/apps/rag/search/testdata/serper.json b/backend/apps/rag/search/testdata/serper.json new file mode 100644 index 0000000000..b269eaf5b3 --- /dev/null +++ b/backend/apps/rag/search/testdata/serper.json @@ -0,0 +1,190 @@ +{ + "searchParameters": { + "q": "apple inc", + "gl": "us", + "hl": "en", + "autocorrect": true, + "page": 1, + "type": "search" + }, + "knowledgeGraph": { + "title": "Apple", + "type": "Technology company", + "website": "http://www.apple.com/", + "imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQwGQRv5TjjkycpctY66mOg_e2-npacrmjAb6_jAWhzlzkFE3OTjxyzbA&s=0", + "description": "Apple Inc. is an American multinational technology company specializing in consumer electronics, software and online services headquartered in Cupertino, California, United States.", + "descriptionSource": "Wikipedia", + "descriptionLink": "https://en.wikipedia.org/wiki/Apple_Inc.", + "attributes": { + "Headquarters": "Cupertino, CA", + "CEO": "Tim Cook (Aug 24, 2011–)", + "Founded": "April 1, 1976, Los Altos, CA", + "Sales": "1 (800) 692-7753", + "Products": "iPhone, Apple Watch, iPad, and more", + "Founders": "Steve Jobs, Steve Wozniak, and Ronald Wayne", + "Subsidiaries": "Apple Store, Beats Electronics, Beddit, and more" + } + }, + "organic": [ + { + "title": "Apple", + "link": "https://www.apple.com/", + "snippet": "Discover the innovative world of Apple and shop everything iPhone, iPad, Apple Watch, Mac, and Apple TV, plus explore accessories, entertainment, ...", + "sitelinks": [ + { + "title": "Support", + "link": "https://support.apple.com/" + }, + { + "title": "iPhone", + "link": "https://www.apple.com/iphone/" + }, + { + "title": "Apple makes business better.", + "link": "https://www.apple.com/business/" + }, + { + "title": "Mac", + "link": "https://www.apple.com/mac/" + } + ], + "position": 1 + }, + { + "title": "Apple Inc. - Wikipedia", + "link": "https://en.wikipedia.org/wiki/Apple_Inc.", + "snippet": "Apple Inc. is an American multinational technology company specializing in consumer electronics, software and online services headquartered in Cupertino, ...", + "attributes": { + "Products": "AirPods; Apple Watch; iPad; iPhone; Mac", + "Founders": "Steve Jobs; Steve Wozniak; Ronald Wayne", + "Founded": "April 1, 1976; 46 years ago in Los Altos, California, U.S", + "Industry": "Consumer electronics; Software services; Online services" + }, + "sitelinks": [ + { + "title": "History", + "link": "https://en.wikipedia.org/wiki/History_of_Apple_Inc." + }, + { + "title": "Timeline of Apple Inc. products", + "link": "https://en.wikipedia.org/wiki/Timeline_of_Apple_Inc._products" + }, + { + "title": "List of software by Apple Inc.", + "link": "https://en.wikipedia.org/wiki/List_of_software_by_Apple_Inc." + }, + { + "title": "Apple Store", + "link": "https://en.wikipedia.org/wiki/Apple_Store" + } + ], + "position": 2 + }, + { + "title": "Apple Inc. | History, Products, Headquarters, & Facts | Britannica", + "link": "https://www.britannica.com/topic/Apple-Inc", + "snippet": "Apple Inc., formerly Apple Computer, Inc., American manufacturer of personal computers, smartphones, tablet computers, computer peripherals, ...", + "date": "Aug 31, 2022", + "attributes": { + "Related People": "Steve Jobs Steve Wozniak Jony Ive Tim Cook Angela Ahrendts", + "Date": "1976 - present", + "Areas Of Involvement": "peripheral device" + }, + "position": 3 + }, + { + "title": "AAPL: Apple Inc Stock Price Quote - NASDAQ GS - Bloomberg.com", + "link": "https://www.bloomberg.com/quote/AAPL:US", + "snippet": "Stock analysis for Apple Inc (AAPL:NASDAQ GS) including stock price, stock chart, company news, key statistics, fundamentals and company profile.", + "position": 4 + }, + { + "title": "Apple Inc. (AAPL) Company Profile & Facts - Yahoo Finance", + "link": "https://finance.yahoo.com/quote/AAPL/profile/", + "snippet": "Apple Inc. designs, manufactures, and markets smartphones, personal computers, tablets, wearables, and accessories worldwide. It also sells various related ...", + "position": 5 + }, + { + "title": "AAPL | Apple Inc. Stock Price & News - WSJ", + "link": "https://www.wsj.com/market-data/quotes/AAPL", + "snippet": "Apple, Inc. engages in the design, manufacture, and sale of smartphones, personal computers, tablets, wearables and accessories, and other varieties of ...", + "position": 6 + }, + { + "title": "Apple Inc Company Profile - Apple Inc Overview - GlobalData", + "link": "https://www.globaldata.com/company-profile/apple-inc/", + "snippet": "Apple Inc (Apple) designs, manufactures, and markets smartphones, tablets, personal computers (PCs), portable and wearable devices. The company also offers ...", + "position": 7 + }, + { + "title": "Apple Inc (AAPL) Stock Price & News - Google Finance", + "link": "https://www.google.com/finance/quote/AAPL:NASDAQ?hl=en", + "snippet": "Get the latest Apple Inc (AAPL) real-time quote, historical performance, charts, and other financial information to help you make more informed trading and ...", + "position": 8 + } + ], + "peopleAlsoAsk": [ + { + "question": "What does Apple Inc mean?", + "snippet": "Apple Inc., formerly Apple Computer, Inc., American manufacturer of personal\ncomputers, smartphones, tablet computers, computer peripherals, and computer\nsoftware. It was the first successful personal computer company and the\npopularizer of the graphical user interface.\nAug 31, 2022", + "title": "Apple Inc. | History, Products, Headquarters, & Facts | Britannica", + "link": "https://www.britannica.com/topic/Apple-Inc" + }, + { + "question": "Is Apple and Apple Inc same?", + "snippet": "Apple was founded as Apple Computer Company on April 1, 1976, by Steve Jobs,\nSteve Wozniak and Ronald Wayne to develop and sell Wozniak's Apple I personal\ncomputer. It was incorporated by Jobs and Wozniak as Apple Computer, Inc.", + "title": "Apple Inc. - Wikipedia", + "link": "https://en.wikipedia.org/wiki/Apple_Inc." + }, + { + "question": "Who owns Apple Inc?", + "snippet": "Apple Inc. is owned by two main institutional investors (Vanguard Group and\nBlackRock, Inc). While its major individual shareholders comprise people like\nArt Levinson, Tim Cook, Bruce Sewell, Al Gore, Johny Sroujli, and others.", + "title": "Who Owns Apple In 2022? - FourWeekMBA", + "link": "https://fourweekmba.com/who-owns-apple/" + }, + { + "question": "What products does Apple Inc offer?", + "snippet": "APPLE FOOTER\nStore.\nMac.\niPad.\niPhone.\nWatch.\nAirPods.\nTV & Home.\nAirTag.", + "title": "More items...", + "link": "https://www.apple.com/business/" + } + ], + "relatedSearches": [ + { + "query": "Who invented the iPhone" + }, + { + "query": "Apple Inc competitors" + }, + { + "query": "Apple iPad" + }, + { + "query": "iPhones" + }, + { + "query": "Apple Inc us" + }, + { + "query": "Apple company history" + }, + { + "query": "Apple Store" + }, + { + "query": "Apple customer service" + }, + { + "query": "Apple Watch" + }, + { + "query": "Apple Inc Industry" + }, + { + "query": "Apple Inc registered address" + }, + { + "query": "Apple Inc Bloomberg" + } + ] +} diff --git a/backend/apps/rag/search/testdata/serpstack.json b/backend/apps/rag/search/testdata/serpstack.json new file mode 100644 index 0000000000..a82f689d8b --- /dev/null +++ b/backend/apps/rag/search/testdata/serpstack.json @@ -0,0 +1,276 @@ +{ + "request": { + "success": true, + "total_time_taken": 3.4, + "processed_timestamp": 1714968442, + "search_url": "http://www.google.com/search?q=mcdonalds\u0026gl=us\u0026hl=en\u0026safe=0\u0026num=10" + }, + "search_parameters": { + "engine": "google", + "type": "web", + "device": "desktop", + "auto_location": "1", + "google_domain": "google.com", + "gl": "us", + "hl": "en", + "safe": "0", + "news_type": "all", + "exclude_autocorrected_results": "0", + "images_color": "any", + "page": "1", + "num": "10", + "output": "json", + "csv_fields": "search_parameters.query,organic_results.position,organic_results.title,organic_results.url,organic_results.domain", + "query": "mcdonalds", + "action": "search", + "access_key": "aac48e007e15c532bb94ffb34532a4b2", + "error": {} + }, + "search_information": { + "total_results": 1170000000, + "time_taken_displayed": 0.49, + "detected_location": {}, + "did_you_mean": {}, + "no_results_for_original_query": false, + "showing_results_for": {} + }, + "organic_results": [ + { + "position": 1, + "title": "Our Full McDonald\u0027s Food Menu", + "snippet": "", + "prerender": false, + "cached_page_url": {}, + "related_pages_url": {}, + "url": "https://www.mcdonalds.com/us/en-us/full-menu.html", + "domain": "www.mcdonalds.com", + "displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a full-menu" + }, + { + "position": 2, + "title": "McDonald\u0027s", + "snippet": "McDonald\u0027s is the world\u0027s largest fast food restaurant chain, serving over 69 million customers daily in over 100 countries in more than 40,000 outlets as of\u00a0...", + "prerender": false, + "cached_page_url": {}, + "related_pages_url": {}, + "url": "https://en.wikipedia.org/wiki/McDonald%27s", + "domain": "en.wikipedia.org", + "displayed_url": "https://en.wikipedia.org \u203a wiki \u203a McDonald\u0027s" + }, + { + "position": 3, + "title": "Restaurants Near Me: Nearby McDonald\u0027s Locations", + "snippet": "", + "prerender": false, + "cached_page_url": {}, + "related_pages_url": {}, + "url": "https://www.mcdonalds.com/us/en-us/restaurant-locator.html", + "domain": "www.mcdonalds.com", + "displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a restaurant-locator" + }, + { + "position": 4, + "title": "Download the McDonald\u0027s App: Deals, Promotions \u0026 ...", + "snippet": "Download the McDonald\u0027s app for Mobile Order \u0026 Pay, exclusive deals and coupons, menu information and special promotions.", + "prerender": false, + "cached_page_url": {}, + "related_pages_url": {}, + "url": "https://www.mcdonalds.com/us/en-us/download-app.html", + "domain": "www.mcdonalds.com", + "displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a download-app" + }, + { + "position": 5, + "title": "McDonald\u0027s Restaurant Careers in the US", + "snippet": "McDonald\u0027s restaurant jobs are one-of-a-kind \u2013 just like you. Restaurants are hiring across all levels, from Crew team to Management. Apply today!", + "prerender": false, + "cached_page_url": {}, + "related_pages_url": {}, + "url": "https://jobs.mchire.com/", + "domain": "jobs.mchire.com", + "displayed_url": "https://jobs.mchire.com" + } + ], + "inline_images": [ + { + "image_url": "https://serpstack-assets.apilayer.net/2418910010831954152.png", + "title": "" + } + ], + "local_results": [ + { + "position": 1, + "title": "McDonald\u0027s", + "coordinates": { + "latitude": 0, + "longitude": 0 + }, + "address": "", + "rating": 0, + "reviews": 0, + "type": "", + "price": {}, + "url": 0 + }, + { + "position": 2, + "title": "McDonald\u0027s", + "coordinates": { + "latitude": 0, + "longitude": 0 + }, + "address": "", + "rating": 0, + "reviews": 0, + "type": "", + "price": {}, + "url": 0 + }, + { + "position": 3, + "title": "McDonald\u0027s", + "coordinates": { + "latitude": 0, + "longitude": 0 + }, + "address": "", + "rating": 0, + "reviews": 0, + "type": "", + "price": {}, + "url": 0 + } + ], + "top_stories": [ + { + "block_position": 1, + "title": "Menu nutrition", + "url": "/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=mcdonald%27s+double+quarter+pounder+with+cheese\u0026stick=H4sIAAAAAAAAAONgFuLUz9U3ME-vLDBX4tVP1zc0TCsuNE0ytjTTUs5OttJPy89P0c9NzSuNLyjKL8tMSS2yAvNS80qKMlOLF7Hq5ian5Ocl5qSoFyuk5Jcm5aQqFJYmFpWkFikU5JfmATUolGeWZCgkZ6SmFqcCAM4ilJtxAAAA\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4Qri56BAh0EAM", + "source": "", + "uploaded": "", + "uploaded_utc": "2024-05-06T04:07:22.082Z" + }, + { + "block_position": 2, + "title": "Profiles", + "url": "https://www.instagram.com/McDonalds", + "source": "", + "uploaded": "", + "uploaded_utc": "2024-05-06T04:07:22.082Z" + }, + { + "block_position": 3, + "title": "People also search for", + "url": "/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-L5Wg8IL2sxPFxxcDEhVbocy-LJPZIvZySijw0ho2hfZ-KtV-sSEEJ9lw7JuEkXHDnRK5y4Dm8aqbiLwugbLbslwjG3hO_gpDTFZK2VoUGZPy2nrmOBCy0G3PoOfoiEtct2GSZlUz0uufG-xP8emtNzQKQpvjkAm5Zmi57iVZueiD62upz7-x2N3dAbwtm6FkInAPRw1yR91zuT7F3lEaPblTW3LaRwCDC0bvaRCh9x4N9zHgY1OOQa_rzts2jf5WpXcuw4Y%3D\u0026q=Burger+King\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4Qs9oBKAB6BAhzEAI", + "source": "", + "uploaded": "", + "uploaded_utc": "2024-05-06T04:07:22.082Z" + } + ], + "related_questions": [ + { + "question": "What\u0027s a number 7 at McDonald\u0027s?What\u0027s a number 7 at McDonald\u0027s?What\u0027s a number 7 at McDonald\u0027s?", + "answer": "", + "title": "", + "displayed_url": "" + }, + { + "question": "Why is McDonald\u0027s changing their name?Why is McDonald\u0027s changing their name?Why is McDonald\u0027s changing their name?", + "answer": "", + "title": "", + "displayed_url": "" + }, + { + "question": "What is the oldest still running Mcdonalds?What is the oldest still running Mcdonalds?What is the oldest still running Mcdonalds?", + "answer": "", + "title": "", + "displayed_url": "" + }, + { + "question": "Why is McDonald\u0027s now WcDonald\u0027s?Why is McDonald\u0027s now WcDonald\u0027s?Why is McDonald\u0027s now WcDonald\u0027s?", + "answer": "", + "title": "", + "displayed_url": "" + } + ], + "knowledge_graph": { + "title": "", + "type": "Fast-food restaurant company", + "image_urls": ["https://serpstack-assets.apilayer.net/2418910010831954152.png"], + "description": "McDonald\u0027s Corporation is an American multinational fast food chain, founded in 1940 as a restaurant operated by Richard and Maurice McDonald, in San Bernardino, California, United States.", + "source": { + "name": "Wikipedia", + "url": "https://en.wikipedia.org/wiki/McDonald\u0027s" + }, + "people_also_search_for": [], + "known_attributes": [ + { + "attribute": "kc:/business/business_operation:founder", + "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Ray+Kroc\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-LxARWRdbk5SkoY2sDn5Qq7yOmqYGei6qZ7sfJhsjZXBPgjMlLbS7824rpJOm69GzqVWMdoNIZiFX2T4A2td14sZOn4a1BexZLtZXHU7NZdF6VsWbGMVuiSYtXdev7uaUjEJKumiwlqTAATTebOriYTEBuSzC\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHgQAg", + "name": "Founder: ", + "value": "Ray Kroc" + }, + { + "attribute": "kc:/organization/organization:ceo", + "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chris+Kempczinski\u0026si=ACC90nwLLwns5sISZcdzuISy7t-NHozt8Cbt6G3WNQfC9ekAgKFbjdEFCDgxLbt57EDZGosYDGiZuq1AcBhA6IhTOSZxfVSySuGQ3VDwmmTA7Z93n3K3596jAuZH9VVv5h8PyvKJSuGuSsQWviJTl3eKj2UL1ZIWuDgkjyVMnC47rN7j0G9PlHRCCLdQF7VDQ1gubTiC4onXqLRBTbwAj6a--PD6Jv_NoA%3D%3D\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHUQAg", + "name": "CEO: ", + "value": "Chris Kempczinski (Nov 1, 2019\u2013)" + }, + { + "attribute": "kc:/business/employer:revenue", + "link": "", + "name": "Revenue: ", + "value": "25.49\u00a0billion USD (2023)" + }, + { + "attribute": "kc:/organization/organization:founded", + "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Des+Plaines\u0026si=ACC90nyvvWro6QmnyY1IfSdgk5wwjB1r8BGd_IWRjXqmKPQqm_yqLtI_DBi5PXGOtg_Z3qrzzEP6mcih1nN7h5A7v6OefnEJiC7a8dBR-v9LxlRubfyR6vlMr3fZ3TmVKWwz9FRpvZb1eYNt-RM7KIDKQlwGEIgINvzhxjUrv6uxSmceduzxd8W7Pkz71XGwxF0F8OlSzHlx\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECG4QAg", + "name": "Founded: ", + "value": "April 15, 1955, Des Plaines, IL" + }, + { + "attribute": "kc:/organization/organization:headquarters", + "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chicago\u0026si=ACC90nyvvWro6QmnyY1IfSdgk5wwjB1r8BGd_IWRjXqmKPQqm-46AEJ_kJbUIEvsvEEZqteiYJvXVXs2ScRNDvFFpjfeAaW3dxtpTGCgcsf5RMdi6IdzOdtjJMN3ZaFwqZOmdi7tC6r0Mh1O9bnP3HrVDB9hH02m7aA6f70dCAfTdpOFnGxDU6wVMAI5MxWBE3wTugtUDOK-\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHYQAg", + "name": "Headquarters: ", + "value": "Chicago, IL" + }, + { + "attribute": "kc:/organization/organization:president", + "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chris+Kempczinski\u0026si=ACC90nwLLwns5sISZcdzuISy7t-NHozt8Cbt6G3WNQfC9ekAgKFbjdEFCDgxLbt57EDZGosYDGiZuq1AcBhA6IhTOSZxfVSySuGQ3VDwmmTA7Z93n3K3596jAuZH9VVv5h8PyvKJSuGuSsQWviJTl3eKj2UL1ZIWuDgkjyVMnC47rN7j0G9PlHRCCLdQF7VDQ1gubTiC4onXqLRBTbwAj6a--PD6Jv_NoA%3D%3D\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHEQAg", + "name": "President: ", + "value": "Chris Kempczinski" + } + ], + "website": "https://www.mcdonalds.com/us/en-us.html", + "profiles": [ + { + "name": "Instagram", + "url": "https://www.instagram.com/McDonalds" + }, + { + "name": "X (Twitter)", + "url": "https://twitter.com/McDonalds" + }, + { + "name": "Facebook", + "url": "https://www.facebook.com/McDonaldsUS" + }, + { + "name": "YouTube", + "url": "https://www.youtube.com/user/McDonaldsUS" + }, + { + "name": "Pinterest", + "url": "https://www.pinterest.com/mcdonalds" + } + ], + "founded": "April 15, 1955, Des Plaines, IL", + "headquarters": "Chicago, IL", + "founders": [ + { + "name": "Ray Kroc", + "link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Ray+Kroc\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-LxARWRdbk5SkoY2sDn5Qq7yOmqYGei6qZ7sfJhsjZXBPgjMlLbS7824rpJOm69GzqVWMdoNIZiFX2T4A2td14sZOn4a1BexZLtZXHU7NZdF6VsWbGMVuiSYtXdev7uaUjEJKumiwlqTAATTebOriYTEBuSzC\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHgQAg" + } + ] + } +} diff --git a/backend/apps/rag/utils.py b/backend/apps/rag/utils.py index 1b3694fdbb..ac52dc3d84 100644 --- a/backend/apps/rag/utils.py +++ b/backend/apps/rag/utils.py @@ -2,7 +2,7 @@ import os import logging import requests -from typing import List +from typing import List, Union from apps.ollama.main import ( generate_ollama_embeddings, @@ -19,9 +19,10 @@ from langchain.retrievers import ( ) from typing import Optional -from config import SRC_LOG_LEVELS, CHROMA_CLIENT +from config import SRC_LOG_LEVELS, CHROMA_CLIENT + log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["RAG"]) @@ -198,6 +199,7 @@ def get_embedding_function( embedding_function, openai_key, openai_url, + batch_size, ): if embedding_engine == "": return lambda query: embedding_function.encode(query).tolist() @@ -221,7 +223,13 @@ def get_embedding_function( def generate_multiple(query, f): if isinstance(query, list): - return [f(q) for q in query] + if embedding_engine == "openai": + embeddings = [] + for i in range(0, len(query), batch_size): + embeddings.extend(f(query[i : i + batch_size])) + return embeddings + else: + return [f(q) for q in query] else: return f(query) @@ -402,8 +410,22 @@ def get_model_path(model: str, update_model: bool = False): def generate_openai_embeddings( - model: str, text: str, key: str, url: str = "https://api.openai.com/v1" + model: str, + text: Union[str, list[str]], + key: str, + url: str = "https://api.openai.com/v1", ): + if isinstance(text, list): + embeddings = generate_openai_batch_embeddings(model, text, key, url) + else: + embeddings = generate_openai_batch_embeddings(model, [text], key, url) + + return embeddings[0] if isinstance(text, str) else embeddings + + +def generate_openai_batch_embeddings( + model: str, texts: list[str], key: str, url: str = "https://api.openai.com/v1" +) -> Optional[list[list[float]]]: try: r = requests.post( f"{url}/embeddings", @@ -411,12 +433,12 @@ def generate_openai_embeddings( "Content-Type": "application/json", "Authorization": f"Bearer {key}", }, - json={"input": text, "model": model}, + json={"input": texts, "model": model}, ) r.raise_for_status() data = r.json() if "data" in data: - return data["data"][0]["embedding"] + return [elem["embedding"] for elem in data["data"]] else: raise "Something went wrong :/" except Exception as e: diff --git a/backend/apps/socket/main.py b/backend/apps/socket/main.py new file mode 100644 index 0000000000..0bc45287a6 --- /dev/null +++ b/backend/apps/socket/main.py @@ -0,0 +1,132 @@ +import socketio +import asyncio + + +from apps.webui.models.users import Users +from utils.utils import decode_token + +sio = socketio.AsyncServer(cors_allowed_origins=[], async_mode="asgi") +app = socketio.ASGIApp(sio, socketio_path="/ws/socket.io") + +# Dictionary to maintain the user pool + + +USER_POOL = {} +USAGE_POOL = {} +# Timeout duration in seconds +TIMEOUT_DURATION = 3 + + +@sio.event +async def connect(sid, environ, auth): + print("connect ", sid) + + user = None + if auth and "token" in auth: + data = decode_token(auth["token"]) + + if data is not None and "id" in data: + user = Users.get_user_by_id(data["id"]) + + if user: + USER_POOL[sid] = user.id + print(f"user {user.name}({user.id}) connected with session ID {sid}") + + print(len(set(USER_POOL))) + await sio.emit("user-count", {"count": len(set(USER_POOL))}) + await sio.emit("usage", {"models": get_models_in_use()}) + + +@sio.on("user-join") +async def user_join(sid, data): + print("user-join", sid, data) + + auth = data["auth"] if "auth" in data else None + + if auth and "token" in auth: + data = decode_token(auth["token"]) + + if data is not None and "id" in data: + user = Users.get_user_by_id(data["id"]) + + if user: + USER_POOL[sid] = user.id + print(f"user {user.name}({user.id}) connected with session ID {sid}") + + print(len(set(USER_POOL))) + await sio.emit("user-count", {"count": len(set(USER_POOL))}) + + +@sio.on("user-count") +async def user_count(sid): + print("user-count", sid) + await sio.emit("user-count", {"count": len(set(USER_POOL))}) + + +def get_models_in_use(): + # Aggregate all models in use + models_in_use = [] + for model_id, data in USAGE_POOL.items(): + models_in_use.append(model_id) + print(f"Models in use: {models_in_use}") + + return models_in_use + + +@sio.on("usage") +async def usage(sid, data): + print(f'Received "usage" event from {sid}: {data}') + + model_id = data["model"] + + # Cancel previous callback if there is one + if model_id in USAGE_POOL: + USAGE_POOL[model_id]["callback"].cancel() + + # Store the new usage data and task + + if model_id in USAGE_POOL: + USAGE_POOL[model_id]["sids"].append(sid) + USAGE_POOL[model_id]["sids"] = list(set(USAGE_POOL[model_id]["sids"])) + + else: + USAGE_POOL[model_id] = {"sids": [sid]} + + # Schedule a task to remove the usage data after TIMEOUT_DURATION + USAGE_POOL[model_id]["callback"] = asyncio.create_task( + remove_after_timeout(sid, model_id) + ) + + # Broadcast the usage data to all clients + await sio.emit("usage", {"models": get_models_in_use()}) + + +async def remove_after_timeout(sid, model_id): + try: + print("remove_after_timeout", sid, model_id) + await asyncio.sleep(TIMEOUT_DURATION) + if model_id in USAGE_POOL: + print(USAGE_POOL[model_id]["sids"]) + USAGE_POOL[model_id]["sids"].remove(sid) + USAGE_POOL[model_id]["sids"] = list(set(USAGE_POOL[model_id]["sids"])) + + if len(USAGE_POOL[model_id]["sids"]) == 0: + del USAGE_POOL[model_id] + + print(f"Removed usage data for {model_id} due to timeout") + # Broadcast the usage data to all clients + await sio.emit("usage", {"models": get_models_in_use()}) + except asyncio.CancelledError: + # Task was cancelled due to new 'usage' event + pass + + +@sio.event +async def disconnect(sid): + if sid in USER_POOL: + disconnected_user = USER_POOL.pop(sid) + print(f"user {disconnected_user} disconnected with session ID {sid}") + + await sio.emit("user-count", {"count": len(USER_POOL)}) + else: + print(f"Unknown session ID {sid} disconnected") diff --git a/backend/apps/webui/main.py b/backend/apps/webui/main.py index c33f476e10..5b238b12bb 100644 --- a/backend/apps/webui/main.py +++ b/backend/apps/webui/main.py @@ -16,6 +16,8 @@ from apps.webui.routers import ( ) from config import ( WEBUI_BUILD_HASH, + SHOW_ADMIN_DETAILS, + ADMIN_EMAIL, WEBUI_AUTH, DEFAULT_MODELS, DEFAULT_PROMPT_SUGGESTIONS, @@ -39,6 +41,11 @@ app.state.config = AppConfig() app.state.config.ENABLE_SIGNUP = ENABLE_SIGNUP app.state.config.JWT_EXPIRES_IN = JWT_EXPIRES_IN + +app.state.config.SHOW_ADMIN_DETAILS = SHOW_ADMIN_DETAILS +app.state.config.ADMIN_EMAIL = ADMIN_EMAIL + + app.state.config.DEFAULT_MODELS = DEFAULT_MODELS app.state.config.DEFAULT_PROMPT_SUGGESTIONS = DEFAULT_PROMPT_SUGGESTIONS app.state.config.DEFAULT_USER_ROLE = DEFAULT_USER_ROLE diff --git a/backend/apps/webui/models/chats.py b/backend/apps/webui/models/chats.py index d4597f16db..a6f1ae9233 100644 --- a/backend/apps/webui/models/chats.py +++ b/backend/apps/webui/models/chats.py @@ -298,6 +298,15 @@ class ChatTable: # .limit(limit).offset(skip) ] + def get_archived_chats_by_user_id(self, user_id: str) -> List[ChatModel]: + return [ + ChatModel(**model_to_dict(chat)) + for chat in Chat.select() + .where(Chat.archived == True) + .where(Chat.user_id == user_id) + .order_by(Chat.updated_at.desc()) + ] + def delete_chat_by_id(self, id: str) -> bool: try: query = Chat.delete().where((Chat.id == id)) diff --git a/backend/apps/webui/routers/auths.py b/backend/apps/webui/routers/auths.py index 3c2b25b0ee..eb611d5148 100644 --- a/backend/apps/webui/routers/auths.py +++ b/backend/apps/webui/routers/auths.py @@ -271,73 +271,88 @@ async def add_user(form_data: AddUserForm, user=Depends(get_admin_user)): raise HTTPException(500, detail=ERROR_MESSAGES.DEFAULT(err)) +############################ +# GetAdminDetails +############################ + + +@router.get("/admin/details") +async def get_admin_details(request: Request, user=Depends(get_current_user)): + if request.app.state.config.SHOW_ADMIN_DETAILS: + admin_email = request.app.state.config.ADMIN_EMAIL + admin_name = None + + print(admin_email, admin_name) + + if admin_email: + admin = Users.get_user_by_email(admin_email) + if admin: + admin_name = admin.name + else: + admin = Users.get_first_user() + if admin: + admin_email = admin.email + admin_name = admin.name + + return { + "name": admin_name, + "email": admin_email, + } + else: + raise HTTPException(400, detail=ERROR_MESSAGES.ACTION_PROHIBITED) + + ############################ # ToggleSignUp ############################ -@router.get("/signup/enabled", response_model=bool) -async def get_sign_up_status(request: Request, user=Depends(get_admin_user)): - return request.app.state.config.ENABLE_SIGNUP +@router.get("/admin/config") +async def get_admin_config(request: Request, user=Depends(get_admin_user)): + return { + "SHOW_ADMIN_DETAILS": request.app.state.config.SHOW_ADMIN_DETAILS, + "ENABLE_SIGNUP": request.app.state.config.ENABLE_SIGNUP, + "DEFAULT_USER_ROLE": request.app.state.config.DEFAULT_USER_ROLE, + "JWT_EXPIRES_IN": request.app.state.config.JWT_EXPIRES_IN, + "ENABLE_COMMUNITY_SHARING": request.app.state.config.ENABLE_COMMUNITY_SHARING, + } -@router.get("/signup/enabled/toggle", response_model=bool) -async def toggle_sign_up(request: Request, user=Depends(get_admin_user)): - request.app.state.config.ENABLE_SIGNUP = not request.app.state.config.ENABLE_SIGNUP - return request.app.state.config.ENABLE_SIGNUP +class AdminConfig(BaseModel): + SHOW_ADMIN_DETAILS: bool + ENABLE_SIGNUP: bool + DEFAULT_USER_ROLE: str + JWT_EXPIRES_IN: str + ENABLE_COMMUNITY_SHARING: bool -############################ -# Default User Role -############################ - - -@router.get("/signup/user/role") -async def get_default_user_role(request: Request, user=Depends(get_admin_user)): - return request.app.state.config.DEFAULT_USER_ROLE - - -class UpdateRoleForm(BaseModel): - role: str - - -@router.post("/signup/user/role") -async def update_default_user_role( - request: Request, form_data: UpdateRoleForm, user=Depends(get_admin_user) +@router.post("/admin/config") +async def update_admin_config( + request: Request, form_data: AdminConfig, user=Depends(get_admin_user) ): - if form_data.role in ["pending", "user", "admin"]: - request.app.state.config.DEFAULT_USER_ROLE = form_data.role - return request.app.state.config.DEFAULT_USER_ROLE + request.app.state.config.SHOW_ADMIN_DETAILS = form_data.SHOW_ADMIN_DETAILS + request.app.state.config.ENABLE_SIGNUP = form_data.ENABLE_SIGNUP + if form_data.DEFAULT_USER_ROLE in ["pending", "user", "admin"]: + request.app.state.config.DEFAULT_USER_ROLE = form_data.DEFAULT_USER_ROLE -############################ -# JWT Expiration -############################ - - -@router.get("/token/expires") -async def get_token_expires_duration(request: Request, user=Depends(get_admin_user)): - return request.app.state.config.JWT_EXPIRES_IN - - -class UpdateJWTExpiresDurationForm(BaseModel): - duration: str - - -@router.post("/token/expires/update") -async def update_token_expires_duration( - request: Request, - form_data: UpdateJWTExpiresDurationForm, - user=Depends(get_admin_user), -): pattern = r"^(-1|0|(-?\d+(\.\d+)?)(ms|s|m|h|d|w))$" # Check if the input string matches the pattern - if re.match(pattern, form_data.duration): - request.app.state.config.JWT_EXPIRES_IN = form_data.duration - return request.app.state.config.JWT_EXPIRES_IN - else: - return request.app.state.config.JWT_EXPIRES_IN + if re.match(pattern, form_data.JWT_EXPIRES_IN): + request.app.state.config.JWT_EXPIRES_IN = form_data.JWT_EXPIRES_IN + + request.app.state.config.ENABLE_COMMUNITY_SHARING = ( + form_data.ENABLE_COMMUNITY_SHARING + ) + + return { + "SHOW_ADMIN_DETAILS": request.app.state.config.SHOW_ADMIN_DETAILS, + "ENABLE_SIGNUP": request.app.state.config.ENABLE_SIGNUP, + "DEFAULT_USER_ROLE": request.app.state.config.DEFAULT_USER_ROLE, + "JWT_EXPIRES_IN": request.app.state.config.JWT_EXPIRES_IN, + "ENABLE_COMMUNITY_SHARING": request.app.state.config.ENABLE_COMMUNITY_SHARING, + } ############################ diff --git a/backend/apps/webui/routers/chats.py b/backend/apps/webui/routers/chats.py index 5d52f40c96..49df3d2841 100644 --- a/backend/apps/webui/routers/chats.py +++ b/backend/apps/webui/routers/chats.py @@ -113,6 +113,19 @@ async def get_user_chats(user=Depends(get_current_user)): ] +############################ +# GetArchivedChats +############################ + + +@router.get("/all/archived", response_model=List[ChatResponse]) +async def get_user_chats(user=Depends(get_current_user)): + return [ + ChatResponse(**{**chat.model_dump(), "chat": json.loads(chat.chat)}) + for chat in Chats.get_archived_chats_by_user_id(user.id) + ] + + ############################ # GetAllChatsInDB ############################ @@ -288,6 +301,32 @@ async def delete_chat_by_id(request: Request, id: str, user=Depends(get_current_ return result +############################ +# CloneChat +############################ + + +@router.get("/{id}/clone", response_model=Optional[ChatResponse]) +async def clone_chat_by_id(id: str, user=Depends(get_current_user)): + chat = Chats.get_chat_by_id_and_user_id(id, user.id) + if chat: + + chat_body = json.loads(chat.chat) + updated_chat = { + **chat_body, + "originalChatId": chat.id, + "branchPointMessageId": chat_body["history"]["currentId"], + "title": f"Clone of {chat.title}", + } + + chat = Chats.insert_new_chat(user.id, ChatForm(**{"chat": updated_chat})) + return ChatResponse(**{**chat.model_dump(), "chat": json.loads(chat.chat)}) + else: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, detail=ERROR_MESSAGES.DEFAULT() + ) + + ############################ # ArchiveChat ############################ diff --git a/backend/apps/webui/routers/models.py b/backend/apps/webui/routers/models.py index 363737e259..acc1c6b47d 100644 --- a/backend/apps/webui/routers/models.py +++ b/backend/apps/webui/routers/models.py @@ -82,7 +82,6 @@ async def update_model_by_id( else: if form_data.id in request.app.state.MODELS: model = Models.insert_new_model(form_data, user.id) - print(model) if model: return model else: diff --git a/backend/apps/webui/routers/users.py b/backend/apps/webui/routers/users.py index cd17e3a7c0..eccafde103 100644 --- a/backend/apps/webui/routers/users.py +++ b/backend/apps/webui/routers/users.py @@ -19,7 +19,12 @@ from apps.webui.models.users import ( from apps.webui.models.auths import Auths from apps.webui.models.chats import Chats -from utils.utils import get_verified_user, get_password_hash, get_admin_user +from utils.utils import ( + get_verified_user, + get_password_hash, + get_current_user, + get_admin_user, +) from constants import ERROR_MESSAGES from config import SRC_LOG_LEVELS diff --git a/backend/apps/webui/routers/utils.py b/backend/apps/webui/routers/utils.py index b95fe88347..18491130ac 100644 --- a/backend/apps/webui/routers/utils.py +++ b/backend/apps/webui/routers/utils.py @@ -107,3 +107,12 @@ async def download_db(user=Depends(get_admin_user)): media_type="application/octet-stream", filename="webui.db", ) + + +@router.get("/litellm/config") +async def download_litellm_config_yaml(user=Depends(get_admin_user)): + return FileResponse( + f"{DATA_DIR}/litellm/config.yaml", + media_type="application/octet-stream", + filename="config.yaml", + ) diff --git a/backend/config.py b/backend/config.py index bd343c06f1..db87d3e40a 100644 --- a/backend/config.py +++ b/backend/config.py @@ -180,6 +180,17 @@ WEBUI_BUILD_HASH = os.environ.get("WEBUI_BUILD_HASH", "dev-build") DATA_DIR = Path(os.getenv("DATA_DIR", BACKEND_DIR / "data")).resolve() FRONTEND_BUILD_DIR = Path(os.getenv("FRONTEND_BUILD_DIR", BASE_DIR / "build")).resolve() +RESET_CONFIG_ON_START = ( + os.environ.get("RESET_CONFIG_ON_START", "False").lower() == "true" +) +if RESET_CONFIG_ON_START: + try: + os.remove(f"{DATA_DIR}/config.json") + with open(f"{DATA_DIR}/config.json", "w") as f: + f.write("{}") + except: + pass + try: CONFIG_DATA = json.loads((DATA_DIR / "config.json").read_text()) except: @@ -703,6 +714,7 @@ ENABLE_COMMUNITY_SHARING = PersistentConfig( os.environ.get("ENABLE_COMMUNITY_SHARING", "True").lower() == "true", ) + class BannerModel(BaseModel): id: str type: str @@ -718,6 +730,20 @@ WEBUI_BANNERS = PersistentConfig( [BannerModel(**banner) for banner in json.loads("[]")], ) + +SHOW_ADMIN_DETAILS = PersistentConfig( + "SHOW_ADMIN_DETAILS", + "auth.admin.show", + os.environ.get("SHOW_ADMIN_DETAILS", "true").lower() == "true", +) + +ADMIN_EMAIL = PersistentConfig( + "ADMIN_EMAIL", + "auth.admin.email", + os.environ.get("ADMIN_EMAIL", None), +) + + #################################### # WEBUI_SECRET_KEY #################################### @@ -805,6 +831,12 @@ RAG_EMBEDDING_MODEL_TRUST_REMOTE_CODE = ( os.environ.get("RAG_EMBEDDING_MODEL_TRUST_REMOTE_CODE", "").lower() == "true" ) +RAG_EMBEDDING_OPENAI_BATCH_SIZE = PersistentConfig( + "RAG_EMBEDDING_OPENAI_BATCH_SIZE", + "rag.embedding_openai_batch_size", + os.environ.get("RAG_EMBEDDING_OPENAI_BATCH_SIZE", 1), +) + RAG_RERANKING_MODEL = PersistentConfig( "RAG_RERANKING_MODEL", "rag.reranking_model", @@ -899,6 +931,75 @@ YOUTUBE_LOADER_LANGUAGE = PersistentConfig( os.getenv("YOUTUBE_LOADER_LANGUAGE", "en").split(","), ) + +ENABLE_RAG_WEB_SEARCH = PersistentConfig( + "ENABLE_RAG_WEB_SEARCH", + "rag.web.search.enable", + os.getenv("ENABLE_RAG_WEB_SEARCH", "False").lower() == "true", +) + +RAG_WEB_SEARCH_ENGINE = PersistentConfig( + "RAG_WEB_SEARCH_ENGINE", + "rag.web.search.engine", + os.getenv("RAG_WEB_SEARCH_ENGINE", ""), +) + +SEARXNG_QUERY_URL = PersistentConfig( + "SEARXNG_QUERY_URL", + "rag.web.search.searxng_query_url", + os.getenv("SEARXNG_QUERY_URL", ""), +) + +GOOGLE_PSE_API_KEY = PersistentConfig( + "GOOGLE_PSE_API_KEY", + "rag.web.search.google_pse_api_key", + os.getenv("GOOGLE_PSE_API_KEY", ""), +) + +GOOGLE_PSE_ENGINE_ID = PersistentConfig( + "GOOGLE_PSE_ENGINE_ID", + "rag.web.search.google_pse_engine_id", + os.getenv("GOOGLE_PSE_ENGINE_ID", ""), +) + +BRAVE_SEARCH_API_KEY = PersistentConfig( + "BRAVE_SEARCH_API_KEY", + "rag.web.search.brave_search_api_key", + os.getenv("BRAVE_SEARCH_API_KEY", ""), +) + +SERPSTACK_API_KEY = PersistentConfig( + "SERPSTACK_API_KEY", + "rag.web.search.serpstack_api_key", + os.getenv("SERPSTACK_API_KEY", ""), +) + +SERPSTACK_HTTPS = PersistentConfig( + "SERPSTACK_HTTPS", + "rag.web.search.serpstack_https", + os.getenv("SERPSTACK_HTTPS", "True").lower() == "true", +) + +SERPER_API_KEY = PersistentConfig( + "SERPER_API_KEY", + "rag.web.search.serper_api_key", + os.getenv("SERPER_API_KEY", ""), +) + + +RAG_WEB_SEARCH_RESULT_COUNT = PersistentConfig( + "RAG_WEB_SEARCH_RESULT_COUNT", + "rag.web.search.result_count", + int(os.getenv("RAG_WEB_SEARCH_RESULT_COUNT", "3")), +) + +RAG_WEB_SEARCH_CONCURRENT_REQUESTS = PersistentConfig( + "RAG_WEB_SEARCH_CONCURRENT_REQUESTS", + "rag.web.search.concurrent_requests", + int(os.getenv("RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "10")), +) + + #################################### # Transcribe #################################### diff --git a/backend/constants.py b/backend/constants.py index 86875d2df1..0740fa49d8 100644 --- a/backend/constants.py +++ b/backend/constants.py @@ -80,3 +80,11 @@ class ERROR_MESSAGES(str, Enum): INVALID_URL = ( "Oops! The URL you provided is invalid. Please double-check and try again." ) + + WEB_SEARCH_ERROR = ( + lambda err="": f"{err if err else 'Oops! Something went wrong while searching the web.'}" + ) + + OLLAMA_API_DISABLED = ( + "The Ollama API is disabled. Please enable it to use this feature." + ) diff --git a/backend/main.py b/backend/main.py index b0de5a099f..56aea3124e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -16,6 +16,7 @@ import mimetypes from fastapi import FastAPI, Request, Depends, status from fastapi.staticfiles import StaticFiles +from fastapi.responses import JSONResponse from fastapi import HTTPException from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.middleware.cors import CORSMiddleware @@ -24,6 +25,8 @@ from starlette.middleware.base import BaseHTTPMiddleware from starlette.middleware.sessions import SessionMiddleware from starlette.responses import StreamingResponse, Response, RedirectResponse + +from apps.socket.main import app as socket_app from apps.ollama.main import app as ollama_app, get_all_models as get_ollama_models from apps.openai.main import app as openai_app, get_all_models as get_openai_models @@ -43,6 +46,8 @@ from utils.misc import parse_duration from utils.utils import ( get_admin_user, get_verified_user, + get_current_user, + get_http_authorization_cred, get_password_hash, create_token, ) @@ -136,7 +141,6 @@ app.state.MODELS = {} origins = ["*"] - # Custom middleware to add security headers # class SecurityHeadersMiddleware(BaseHTTPMiddleware): # async def dispatch(self, request: Request, call_next): @@ -154,7 +158,8 @@ class RAGMiddleware(BaseHTTPMiddleware): return_citations = False if request.method == "POST" and ( - "/api/chat" in request.url.path or "/chat/completions" in request.url.path + "/ollama/api/chat" in request.url.path + or "/chat/completions" in request.url.path ): log.debug(f"request.url.path: {request.url.path}") @@ -239,6 +244,124 @@ class RAGMiddleware(BaseHTTPMiddleware): app.add_middleware(RAGMiddleware) +class PipelineMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + if request.method == "POST" and ( + "/ollama/api/chat" in request.url.path + or "/chat/completions" in request.url.path + ): + log.debug(f"request.url.path: {request.url.path}") + + # Read the original request body + body = await request.body() + # Decode body to string + body_str = body.decode("utf-8") + # Parse string to JSON + data = json.loads(body_str) if body_str else {} + + model_id = data["model"] + filters = [ + model + for model in app.state.MODELS.values() + if "pipeline" in model + and "type" in model["pipeline"] + and model["pipeline"]["type"] == "filter" + and ( + model["pipeline"]["pipelines"] == ["*"] + or any( + model_id == target_model_id + for target_model_id in model["pipeline"]["pipelines"] + ) + ) + ] + sorted_filters = sorted(filters, key=lambda x: x["pipeline"]["priority"]) + + user = None + if len(sorted_filters) > 0: + try: + user = get_current_user( + get_http_authorization_cred( + request.headers.get("Authorization") + ) + ) + user = {"id": user.id, "name": user.name, "role": user.role} + except: + pass + + model = app.state.MODELS[model_id] + + if "pipeline" in model: + sorted_filters.append(model) + + for filter in sorted_filters: + r = None + try: + urlIdx = filter["urlIdx"] + + url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx] + key = openai_app.state.config.OPENAI_API_KEYS[urlIdx] + + if key != "": + headers = {"Authorization": f"Bearer {key}"} + r = requests.post( + f"{url}/{filter['id']}/filter/inlet", + headers=headers, + json={ + "user": user, + "body": data, + }, + ) + + r.raise_for_status() + data = r.json() + except Exception as e: + # Handle connection error here + print(f"Connection error: {e}") + + if r is not None: + try: + res = r.json() + if "detail" in res: + return JSONResponse( + status_code=r.status_code, + content=res, + ) + except: + pass + + else: + pass + + if "pipeline" not in app.state.MODELS[model_id]: + if "chat_id" in data: + del data["chat_id"] + + if "title" in data: + del data["title"] + + modified_body_bytes = json.dumps(data).encode("utf-8") + # Replace the request body with the modified one + request._body = modified_body_bytes + # Set custom header to ensure content-length matches new body length + request.headers.__dict__["_list"] = [ + (b"content-length", str(len(modified_body_bytes)).encode("utf-8")), + *[ + (k, v) + for k, v in request.headers.raw + if k.lower() != b"content-length" + ], + ] + + response = await call_next(request) + return response + + async def _receive(self, body: bytes): + return {"type": "http.request", "body": body, "more_body": False} + + +app.add_middleware(PipelineMiddleware) + + app.add_middleware( CORSMiddleware, allow_origins=origins, @@ -271,6 +394,9 @@ async def update_embedding_function(request: Request, call_next): return response +app.mount("/ws", socket_app) + + app.mount("/ollama", ollama_app) app.mount("/openai", openai_app) @@ -351,6 +477,14 @@ async def get_all_models(): @app.get("/api/models") async def get_models(user=Depends(get_verified_user)): models = await get_all_models() + + # Filter out filter pipelines + models = [ + model + for model in models + if "pipeline" not in model or model["pipeline"].get("type", None) != "filter" + ] + if app.state.config.ENABLE_MODEL_FILTER: if user.role == "user": models = list( @@ -364,6 +498,339 @@ async def get_models(user=Depends(get_verified_user)): return {"data": models} +@app.post("/api/chat/completed") +async def chat_completed(form_data: dict, user=Depends(get_verified_user)): + data = form_data + model_id = data["model"] + + filters = [ + model + for model in app.state.MODELS.values() + if "pipeline" in model + and "type" in model["pipeline"] + and model["pipeline"]["type"] == "filter" + and ( + model["pipeline"]["pipelines"] == ["*"] + or any( + model_id == target_model_id + for target_model_id in model["pipeline"]["pipelines"] + ) + ) + ] + sorted_filters = sorted(filters, key=lambda x: x["pipeline"]["priority"]) + + print(model_id) + + if model_id in app.state.MODELS: + model = app.state.MODELS[model_id] + if "pipeline" in model: + sorted_filters = [model] + sorted_filters + + for filter in sorted_filters: + r = None + try: + urlIdx = filter["urlIdx"] + + url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx] + key = openai_app.state.config.OPENAI_API_KEYS[urlIdx] + + if key != "": + headers = {"Authorization": f"Bearer {key}"} + r = requests.post( + f"{url}/{filter['id']}/filter/outlet", + headers=headers, + json={ + "user": {"id": user.id, "name": user.name, "role": user.role}, + "body": data, + }, + ) + + r.raise_for_status() + data = r.json() + except Exception as e: + # Handle connection error here + print(f"Connection error: {e}") + + if r is not None: + try: + res = r.json() + if "detail" in res: + return JSONResponse( + status_code=r.status_code, + content=res, + ) + except: + pass + + else: + pass + + return data + + +@app.get("/api/pipelines/list") +async def get_pipelines_list(user=Depends(get_admin_user)): + responses = await get_openai_models(raw=True) + + print(responses) + urlIdxs = [ + idx + for idx, response in enumerate(responses) + if response != None and "pipelines" in response + ] + + return { + "data": [ + { + "url": openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx], + "idx": urlIdx, + } + for urlIdx in urlIdxs + ] + } + + +class AddPipelineForm(BaseModel): + url: str + urlIdx: int + + +@app.post("/api/pipelines/add") +async def add_pipeline(form_data: AddPipelineForm, user=Depends(get_admin_user)): + + r = None + try: + urlIdx = form_data.urlIdx + + url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx] + key = openai_app.state.config.OPENAI_API_KEYS[urlIdx] + + headers = {"Authorization": f"Bearer {key}"} + r = requests.post( + f"{url}/pipelines/add", headers=headers, json={"url": form_data.url} + ) + + r.raise_for_status() + data = r.json() + + return {**data} + except Exception as e: + # Handle connection error here + print(f"Connection error: {e}") + + detail = "Pipeline not found" + if r is not None: + try: + res = r.json() + if "detail" in res: + detail = res["detail"] + except: + pass + + raise HTTPException( + status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND), + detail=detail, + ) + + +class DeletePipelineForm(BaseModel): + id: str + urlIdx: int + + +@app.delete("/api/pipelines/delete") +async def delete_pipeline(form_data: DeletePipelineForm, user=Depends(get_admin_user)): + + r = None + try: + urlIdx = form_data.urlIdx + + url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx] + key = openai_app.state.config.OPENAI_API_KEYS[urlIdx] + + headers = {"Authorization": f"Bearer {key}"} + r = requests.delete( + f"{url}/pipelines/delete", headers=headers, json={"id": form_data.id} + ) + + r.raise_for_status() + data = r.json() + + return {**data} + except Exception as e: + # Handle connection error here + print(f"Connection error: {e}") + + detail = "Pipeline not found" + if r is not None: + try: + res = r.json() + if "detail" in res: + detail = res["detail"] + except: + pass + + raise HTTPException( + status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND), + detail=detail, + ) + + +@app.get("/api/pipelines") +async def get_pipelines(urlIdx: Optional[int] = None, user=Depends(get_admin_user)): + r = None + try: + urlIdx + + url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx] + key = openai_app.state.config.OPENAI_API_KEYS[urlIdx] + + headers = {"Authorization": f"Bearer {key}"} + r = requests.get(f"{url}/pipelines", headers=headers) + + r.raise_for_status() + data = r.json() + + return {**data} + except Exception as e: + # Handle connection error here + print(f"Connection error: {e}") + + detail = "Pipeline not found" + if r is not None: + try: + res = r.json() + if "detail" in res: + detail = res["detail"] + except: + pass + + raise HTTPException( + status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND), + detail=detail, + ) + + +@app.get("/api/pipelines/{pipeline_id}/valves") +async def get_pipeline_valves( + urlIdx: Optional[int], pipeline_id: str, user=Depends(get_admin_user) +): + models = await get_all_models() + r = None + try: + + url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx] + key = openai_app.state.config.OPENAI_API_KEYS[urlIdx] + + headers = {"Authorization": f"Bearer {key}"} + r = requests.get(f"{url}/{pipeline_id}/valves", headers=headers) + + r.raise_for_status() + data = r.json() + + return {**data} + except Exception as e: + # Handle connection error here + print(f"Connection error: {e}") + + detail = "Pipeline not found" + + if r is not None: + try: + res = r.json() + if "detail" in res: + detail = res["detail"] + except: + pass + + raise HTTPException( + status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND), + detail=detail, + ) + + +@app.get("/api/pipelines/{pipeline_id}/valves/spec") +async def get_pipeline_valves_spec( + urlIdx: Optional[int], pipeline_id: str, user=Depends(get_admin_user) +): + models = await get_all_models() + + r = None + try: + url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx] + key = openai_app.state.config.OPENAI_API_KEYS[urlIdx] + + headers = {"Authorization": f"Bearer {key}"} + r = requests.get(f"{url}/{pipeline_id}/valves/spec", headers=headers) + + r.raise_for_status() + data = r.json() + + return {**data} + except Exception as e: + # Handle connection error here + print(f"Connection error: {e}") + + detail = "Pipeline not found" + if r is not None: + try: + res = r.json() + if "detail" in res: + detail = res["detail"] + except: + pass + + raise HTTPException( + status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND), + detail=detail, + ) + + +@app.post("/api/pipelines/{pipeline_id}/valves/update") +async def update_pipeline_valves( + urlIdx: Optional[int], + pipeline_id: str, + form_data: dict, + user=Depends(get_admin_user), +): + models = await get_all_models() + + r = None + try: + url = openai_app.state.config.OPENAI_API_BASE_URLS[urlIdx] + key = openai_app.state.config.OPENAI_API_KEYS[urlIdx] + + headers = {"Authorization": f"Bearer {key}"} + r = requests.post( + f"{url}/{pipeline_id}/valves/update", + headers=headers, + json={**form_data}, + ) + + r.raise_for_status() + data = r.json() + + return {**data} + except Exception as e: + # Handle connection error here + print(f"Connection error: {e}") + + detail = "Pipeline not found" + + if r is not None: + try: + res = r.json() + if "detail" in res: + detail = res["detail"] + except: + pass + + raise HTTPException( + status_code=(r.status_code if r is not None else status.HTTP_404_NOT_FOUND), + detail=detail, + ) + + @app.get("/api/config") async def get_app_config(): # Checking and Handling the Absence of 'ui' in CONFIG_DATA @@ -384,9 +851,10 @@ async def get_app_config(): "auth": WEBUI_AUTH, "auth_trusted_header": bool(webui_app.state.AUTH_TRUSTED_EMAIL_HEADER), "enable_signup": webui_app.state.config.ENABLE_SIGNUP, + "enable_web_search": rag_app.state.config.ENABLE_RAG_WEB_SEARCH, "enable_image_generation": images_app.state.config.ENABLED, - "enable_admin_export": ENABLE_ADMIN_EXPORT, "enable_community_sharing": webui_app.state.config.ENABLE_COMMUNITY_SHARING, + "enable_admin_export": ENABLE_ADMIN_EXPORT, }, "oauth": { "providers": { @@ -438,23 +906,7 @@ class UrlForm(BaseModel): async def update_webhook_url(form_data: UrlForm, user=Depends(get_admin_user)): app.state.config.WEBHOOK_URL = form_data.url webui_app.state.WEBHOOK_URL = app.state.config.WEBHOOK_URL - - return { - "url": app.state.config.WEBHOOK_URL, - } - - -@app.get("/api/community_sharing", response_model=bool) -async def get_community_sharing_status(request: Request, user=Depends(get_admin_user)): - return webui_app.state.config.ENABLE_COMMUNITY_SHARING - - -@app.get("/api/community_sharing/toggle", response_model=bool) -async def toggle_community_sharing(request: Request, user=Depends(get_admin_user)): - webui_app.state.config.ENABLE_COMMUNITY_SHARING = ( - not webui_app.state.config.ENABLE_COMMUNITY_SHARING - ) - return webui_app.state.config.ENABLE_COMMUNITY_SHARING + return {"url": app.state.config.WEBHOOK_URL} @app.get("/api/version") diff --git a/backend/utils/misc.py b/backend/utils/misc.py index fca9412637..9069857b7f 100644 --- a/backend/utils/misc.py +++ b/backend/utils/misc.py @@ -123,11 +123,25 @@ def parse_ollama_modelfile(model_text): "repeat_penalty": float, "temperature": float, "seed": int, - "stop": str, "tfs_z": float, "num_predict": int, "top_k": int, "top_p": float, + "num_keep": int, + "typical_p": float, + "presence_penalty": float, + "frequency_penalty": float, + "penalize_newline": bool, + "numa": bool, + "num_batch": int, + "num_gpu": int, + "main_gpu": int, + "low_vram": bool, + "f16_kv": bool, + "vocab_only": bool, + "use_mmap": bool, + "use_mlock": bool, + "num_thread": int, } data = {"base_model_id": None, "params": {}} @@ -156,10 +170,18 @@ def parse_ollama_modelfile(model_text): param_match = re.search(rf"PARAMETER {param} (.+)", model_text, re.IGNORECASE) if param_match: value = param_match.group(1) - if param_type == int: - value = int(value) - elif param_type == float: - value = float(value) + + try: + if param_type == int: + value = int(value) + elif param_type == float: + value = float(value) + elif param_type == bool: + value = value.lower() == "true" + except Exception as e: + print(e) + continue + data["params"][param] = value # Parse adapter diff --git a/demo.gif b/demo.gif index 5a3f36269d..6e56b74a0a 100644 Binary files a/demo.gif and b/demo.gif differ diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 3e82c979c8..92238d3078 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -45,6 +45,7 @@ We welcome pull requests. Before submitting one, please: 2. Follow the project's coding standards and include tests for new features. 3. Update documentation as necessary. 4. Write clear, descriptive commit messages. +5. It's essential to complete your pull request in a timely manner. We move fast, and having PRs hang around too long is not feasible. If you can't get it done within a reasonable time frame, we may have to close it to keep the project moving forward. ### 📚 Documentation & Tutorials diff --git a/package-lock.json b/package-lock.json index 5b1461939c..7d3b385e24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "open-webui", - "version": "0.2.0.dev2", + "version": "0.2.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "open-webui", - "version": "0.2.0.dev2", + "version": "0.2.5", "dependencies": { "@pyscript/core": "^0.4.32", "@sveltejs/adapter-node": "^1.3.1", @@ -23,7 +23,10 @@ "js-sha256": "^0.10.1", "katex": "^0.16.9", "marked": "^9.1.0", + "mermaid": "^10.9.1", "pyodide": "^0.26.0-alpha.4", + "socket.io-client": "^4.7.5", + "sortablejs": "^1.15.2", "svelte-sonner": "^0.3.19", "tippy.js": "^6.3.7", "uuid": "^9.0.1" @@ -100,6 +103,11 @@ "node": ">=6.9.0" } }, + "node_modules/@braintree/sanitize-url": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-6.0.4.tgz", + "integrity": "sha512-s3jaWicZd0pkP0jf5ysyHUI/RE7MHos6qlToFcGWXVp+ykHOy77OUMrfbgJ9it2C5bow7OIQwYYaHjk9XlBQ2A==" + }, "node_modules/@colors/colors": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", @@ -1207,6 +1215,11 @@ "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", "dev": true }, + "node_modules/@socket.io/component-emitter": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz", + "integrity": "sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==" + }, "node_modules/@sveltejs/adapter-auto": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@sveltejs/adapter-auto/-/adapter-auto-2.1.1.tgz", @@ -1347,6 +1360,32 @@ "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.5.4.tgz", "integrity": "sha512-7z/eR6O859gyWIAjuvBWFzNURmf2oPBmJlfVWkwehU5nzIyjwBsTh7WMmEEV4JFnHuQ3ex4oyTvfKzcyJVDBNA==" }, + "node_modules/@types/d3-scale": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.8.tgz", + "integrity": "sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ==", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.3.tgz", + "integrity": "sha512-laXM4+1o5ImZv3RpFAsTRn3TEkzqkytiOY0Dz0sq5cnd1dtNlk6sHLon4OvqaiJb28T0S/TdsBI3Sjsy+keJrw==" + }, + "node_modules/@types/d3-time": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.3.tgz", + "integrity": "sha512-2p6olUZ4w3s+07q3Tm2dbiMZy5pCDfYwtLXXHUnVzXgQlZ/OyPtUz6OL382BkOuGlLXqfT+wqv8Fw2v8/0geBw==" + }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", @@ -1358,12 +1397,25 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "node_modules/@types/mdast": { + "version": "3.0.15", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.15.tgz", + "integrity": "sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==", + "dependencies": { + "@types/unist": "^2" + } + }, "node_modules/@types/minimatch": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz", "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==", "dev": true }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==" + }, "node_modules/@types/node": { "version": "20.11.30", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.30.tgz", @@ -1408,6 +1460,11 @@ "integrity": "sha512-MQ1AnmTLOncwEf9IVU+B2e4Hchrku5N67NkgcAHW0p3sdzPe0FNMANxEm6OJUzPniEQGkeT3OROLlCwZJLWFZA==", "dev": true }, + "node_modules/@types/unist": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.10.tgz", + "integrity": "sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==" + }, "node_modules/@types/ws": { "version": "8.5.10", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz", @@ -2458,6 +2515,15 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/check-error": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.3.tgz", @@ -2817,6 +2883,14 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "dev": true }, + "node_modules/cose-base": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz", + "integrity": "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==", + "dependencies": { + "layout-base": "^1.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -3003,6 +3077,447 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/cytoscape": { + "version": "3.29.2", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.29.2.tgz", + "integrity": "sha512-2G1ycU28Nh7OHT9rkXRLpCDP30MKH1dXJORZuBhtEhEW7pKwgPi77ImqlCWinouyE1PNepIOGZBOrE84DG7LyQ==", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/cytoscape-cose-bilkent": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz", + "integrity": "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==", + "dependencies": { + "cose-base": "^1.0.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/d3": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "engines": { + "node": ">= 10" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "dependencies": { + "d3-dsv": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz", + "integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==" + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==" + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/dagre-d3-es": { + "version": "7.0.10", + "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.10.tgz", + "integrity": "sha512-qTCQmEhcynucuaZgY5/+ti3X/rnszKZhEQH/ZdWdtP1tA/y3VoHJzcVrO9pjjJCNpigfscAtoUB5ONcd2wNn0A==", + "dependencies": { + "d3": "^7.8.2", + "lodash-es": "^4.17.21" + } + }, "node_modules/dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -3042,6 +3557,18 @@ } } }, + "node_modules/decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/deep-eql": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz", @@ -3085,6 +3612,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delaunator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.0.1.tgz", + "integrity": "sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw==", + "dependencies": { + "robust-predicates": "^3.0.2" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -3122,6 +3657,14 @@ "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", "dev": true }, + "node_modules/diff": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", + "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/diff-sequences": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", @@ -3202,6 +3745,11 @@ "url": "https://github.com/fb55/domhandler?sponsor=1" } }, + "node_modules/dompurify": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.5.tgz", + "integrity": "sha512-lwG+n5h8QNpxtyrJW/gJWckL+1/DQiYMX8f7t8Z2AZTPw1esVrqjI63i7Zc2Gz0aKzLVMYC1V1PL/ky+aY/NgA==" + }, "node_modules/domutils": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", @@ -3238,6 +3786,11 @@ "integrity": "sha512-XzWNH4ZSa9BwVUQSDorPWAUQ5WGuYz7zJUNpNif40zFCiCl20t8zgylmreNmn26h5kiyw2lg7RfTmeMBsDklqg==", "dev": true }, + "node_modules/elkjs": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/elkjs/-/elkjs-0.9.3.tgz", + "integrity": "sha512-f/ZeWvW/BCXbhGEf1Ujp29EASo/lk1FDnETgNKwJrsVvGZhUWCZyg3xLJjAsxfOmt8KjswHmI5EwCQcPMpOYhQ==" + }, "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -3253,6 +3806,46 @@ "once": "^1.4.0" } }, + "node_modules/engine.io-client": { + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.3.tgz", + "integrity": "sha512-9Z0qLB0NIisTRt1DZ/8U2k12RJn8yls/nXMZLn+/N8hANT3TcYjKFKcwbw5zFQiN4NTde3TSY9zb79e1ij6j9Q==", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.1", + "engine.io-parser": "~5.2.1", + "ws": "~8.11.0", + "xmlhttprequest-ssl": "~2.0.0" + } + }, + "node_modules/engine.io-client/node_modules/ws": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", + "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/engine.io-parser": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.2.tgz", + "integrity": "sha512-RcyUFKA93/CXH20l4SoVvzZfrSDMOTUS3bWVpTt2FuFP+XYrL8i8oonHP7WInRyVHXh0n/ORtoeiE1os+8qkSw==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/enquirer": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", @@ -4512,7 +5105,6 @@ "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "dev": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, @@ -4621,6 +5213,14 @@ "node": ">=10" } }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "engines": { + "node": ">=12" + } + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -4970,6 +5570,11 @@ "json-buffer": "3.0.1" } }, + "node_modules/khroma": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/khroma/-/khroma-2.1.0.tgz", + "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" + }, "node_modules/kleur": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", @@ -4984,6 +5589,11 @@ "integrity": "sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==", "dev": true }, + "node_modules/layout-base": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", + "integrity": "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==" + }, "node_modules/lazy-ass": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/lazy-ass/-/lazy-ass-1.6.0.tgz", @@ -5139,6 +5749,11 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, + "node_modules/lodash-es": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", + "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" + }, "node_modules/lodash.castarray": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz", @@ -5326,6 +5941,41 @@ "node": "*" } }, + "node_modules/mdast-util-from-markdown": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-1.3.1.tgz", + "integrity": "sha512-4xTO/M8c82qBcnQc1tgpNtubGUW/Y1tBQ1B0i5CtSoelOLKFYlElIr3bvgREYYO5iRqbMY1YuqZng0GVOI8Qww==", + "dependencies": { + "@types/mdast": "^3.0.0", + "@types/unist": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "mdast-util-to-string": "^3.1.0", + "micromark": "^3.0.0", + "micromark-util-decode-numeric-character-reference": "^1.0.0", + "micromark-util-decode-string": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "unist-util-stringify-position": "^3.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-3.2.0.tgz", + "integrity": "sha512-V4Zn/ncyN1QNSqSBxTrMOLpjr+IKdHl2v3KVLoWmDPscP4r9GcCi71gjgvUV1SFSKh92AjAG4peFuBl2/YgCJg==", + "dependencies": { + "@types/mdast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/mdn-data": { "version": "2.0.30", "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz", @@ -5346,6 +5996,454 @@ "node": ">= 8" } }, + "node_modules/mermaid": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.9.1.tgz", + "integrity": "sha512-Mx45Obds5W1UkW1nv/7dHRsbfMM1aOKA2+Pxs/IGHNonygDHwmng8xTHyS9z4KWVi0rbko8gjiBmuwwXQ7tiNA==", + "dependencies": { + "@braintree/sanitize-url": "^6.0.1", + "@types/d3-scale": "^4.0.3", + "@types/d3-scale-chromatic": "^3.0.0", + "cytoscape": "^3.28.1", + "cytoscape-cose-bilkent": "^4.1.0", + "d3": "^7.4.0", + "d3-sankey": "^0.12.3", + "dagre-d3-es": "7.0.10", + "dayjs": "^1.11.7", + "dompurify": "^3.0.5", + "elkjs": "^0.9.0", + "katex": "^0.16.9", + "khroma": "^2.0.0", + "lodash-es": "^4.17.21", + "mdast-util-from-markdown": "^1.3.0", + "non-layered-tidy-tree-layout": "^2.0.2", + "stylis": "^4.1.3", + "ts-dedent": "^2.2.0", + "uuid": "^9.0.0", + "web-worker": "^1.2.0" + } + }, + "node_modules/micromark": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-3.2.0.tgz", + "integrity": "sha512-uD66tJj54JLYq0De10AhWycZWGQNUvDI55xPgk2sQM5kn1JYlhbCMTtEeT27+vAhW2FBQxLlOmS3pmA7/2z4aA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "micromark-core-commonmark": "^1.0.1", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-chunked": "^1.0.0", + "micromark-util-combine-extensions": "^1.0.0", + "micromark-util-decode-numeric-character-reference": "^1.0.0", + "micromark-util-encode": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-subtokenize": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.1", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-1.1.0.tgz", + "integrity": "sha512-BgHO1aRbolh2hcrzL2d1La37V0Aoz73ymF8rAcKnohLy93titmv62E0gP8Hrx9PKcKrqCZ1BbLGbP3bEhoXYlw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-factory-destination": "^1.0.0", + "micromark-factory-label": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-factory-title": "^1.0.0", + "micromark-factory-whitespace": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-chunked": "^1.0.0", + "micromark-util-classify-character": "^1.0.0", + "micromark-util-html-tag-name": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-subtokenize": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.1", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-factory-destination": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-1.1.0.tgz", + "integrity": "sha512-XaNDROBgx9SgSChd69pjiGKbV+nfHGDPVYFs5dOoDd7ZnMAE+Cuu91BCpsY8RT2NP9vo/B8pds2VQNCLiu0zhg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-1.1.0.tgz", + "integrity": "sha512-OLtyez4vZo/1NjxGhcpDSbHQ+m0IIGnT8BoPamh+7jVlzLJBH98zzuCoUeMxvM6WsNeh8wx8cKvqLiPHEACn0w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-1.1.0.tgz", + "integrity": "sha512-cRzEj7c0OL4Mw2v6nwzttyOZe8XY/Z8G0rzmWQZTBi/jjwyw/U4uqKtUORXQrR5bAZZnbTI/feRV/R7hc4jQYQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-1.1.0.tgz", + "integrity": "sha512-J7n9R3vMmgjDOCY8NPw55jiyaQnH5kBdV2/UXCtZIpnHH3P6nHUKaH7XXEYuWwx/xUJcawa8plLBEjMPU24HzQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-1.1.0.tgz", + "integrity": "sha512-v2WlmiymVSp5oMg+1Q0N1Lxmt6pMhIHD457whWM7/GUlEks1hI9xj5w3zbc4uuMKXGisksZk8DzP2UyGbGqNsQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-1.2.0.tgz", + "integrity": "sha512-lXraTwcX3yH/vMDaFWCQJP1uIszLVebzUa3ZHdrgxr7KEU/9mL4mVgCpGbyhvNLNlauROiNUq7WN5u7ndbY6xg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-1.1.0.tgz", + "integrity": "sha512-Ye01HXpkZPNcV6FiyoW2fGZDUw4Yc7vT0E9Sad83+bEDiCJ1uXu0S3mr8WLpsz3HaG3x2q0HM6CTuPdcZcluFQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-1.1.0.tgz", + "integrity": "sha512-SL0wLxtKSnklKSUplok1WQFoGhUdWYKggKUiqhX+Swala+BtptGCu5iPRc+xvzJ4PXE/hwM3FNXsfEVgoZsWbw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-1.1.0.tgz", + "integrity": "sha512-Q20sp4mfNf9yEqDL50WwuWZHUrCO4fEyeDCnMGmG5Pr0Cz15Uo7KBs6jq+dq0EgX4DPwwrh9m0X+zPV1ypFvUA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-1.1.0.tgz", + "integrity": "sha512-m9V0ExGv0jB1OT21mrWcuf4QhP46pH1KkfWy9ZEezqHKAxkj4mPCy3nIH1rkbdMlChLHX531eOrymlwyZIf2iw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-1.1.0.tgz", + "integrity": "sha512-YphLGCK8gM1tG1bd54azwyrQRjCFcmgj2S2GoJDNnh4vYtnL38JS8M4gpxzOPNyHdNEpheyWXCTnnTDY3N+NVQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-decode-numeric-character-reference": "^1.0.0", + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-1.1.0.tgz", + "integrity": "sha512-EuEzTWSTAj9PA5GOAs992GzNh2dGQO52UvAbtSOMvXTxv3Criqb6IOzJUBCmEqrrXSblJIJBbFFv6zPxpreiJw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-html-tag-name": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-1.2.0.tgz", + "integrity": "sha512-VTQzcuQgFUD7yYztuQFKXT49KghjtETQ+Wv/zUjGSGBioZnkA4P1XXZPT1FHeJA6RwRXSF47yvJ1tsJdoxwO+Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-1.1.0.tgz", + "integrity": "sha512-N+w5vhqrBihhjdpM8+5Xsxy71QWqGn7HYNUvch71iV2PM7+E3uWGox1Qp90loa1ephtCxG2ftRV/Conitc6P2Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-1.1.0.tgz", + "integrity": "sha512-b/G6BTMSg+bX+xVCshPTPyAu2tmA0E4X98NSR7eIbeC6ycCqCeE7wjfDIgzEbkzdEVJXRtOG4FbEm/uGbCRouA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-1.2.0.tgz", + "integrity": "sha512-QO4GXv0XZfWey4pYFndLUKEAktKkG5kZTdUNaTAkzbuJxn2tNBOr+QtxR2XpWaMhbImT2dPzyLrPXLlPhph34A==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-encode": "^1.0.0", + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-1.1.0.tgz", + "integrity": "sha512-kUQHyzRoxvZO2PuLzMt2P/dwVsTiivCK8icYTeR+3WgbuPqfHgPPy7nFKbeqRivBvn/3N3GBiNC+JRTMSxEC7A==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-1.1.0.tgz", + "integrity": "sha512-uEjpEYY6KMs1g7QfJ2eX1SQEV+ZT4rUD3UcF6l57acZvLNK7PBZL+ty82Z1qhK1/yXIY4bdx04FKMgR0g4IAag==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-types": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-1.1.0.tgz", + "integrity": "sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, "node_modules/micromatch": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", @@ -5525,6 +6623,11 @@ "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", "dev": true }, + "node_modules/non-layered-tidy-tree-layout": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/non-layered-tidy-tree-layout/-/non-layered-tidy-tree-layout-2.0.2.tgz", + "integrity": "sha512-gkXMxRzUH+PB0ax9dUN0yYF0S25BqeAYqhgMaLUFmpXLEk7Fcu8f4emJuOAY0V8kjDICxROIKsTAKsV/v355xw==" + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -6600,6 +7703,11 @@ "node": "*" } }, + "node_modules/robust-predicates": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", + "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==" + }, "node_modules/rollup": { "version": "3.29.4", "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.29.4.tgz", @@ -6647,6 +7755,11 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==" + }, "node_modules/rxjs": { "version": "7.8.1", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", @@ -6676,8 +7789,7 @@ "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/sander": { "version": "0.5.1", @@ -6883,6 +7995,32 @@ "node": ">=8" } }, + "node_modules/socket.io-client": { + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.7.5.tgz", + "integrity": "sha512-sJ/tqHOCe7Z50JCBCXrsY3I2k03iOiUe+tj1OmKeD2lXPiGH/RUCdTZFoqVyN7l1MnpIzPrGtLcijffmeouNlQ==", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.2", + "engine.io-client": "~6.5.2", + "socket.io-parser": "~4.2.4" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/socket.io-parser": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.4.tgz", + "integrity": "sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/sorcery": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/sorcery/-/sorcery-0.11.0.tgz", @@ -6913,6 +8051,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/sortablejs": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/sortablejs/-/sortablejs-1.15.2.tgz", + "integrity": "sha512-FJF5jgdfvoKn1MAKSdGs33bIqLi3LmsgVTliuX6iITj834F+JRQZN90Z93yql8h0K2t0RwDPBmxwlbZfDcxNZA==" + }, "node_modules/source-map-js": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", @@ -7135,6 +8278,11 @@ "url": "https://github.com/sponsors/antfu" } }, + "node_modules/stylis": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.3.2.tgz", + "integrity": "sha512-bhtUjWd/z6ltJiQwg0dUfxEJ+W+jdqQd8TbWLWyeIJHlnsqmGLRFFd8e5mA0AZi/zx90smXRlN66YMTcaSFifg==" + }, "node_modules/sucrase": { "version": "3.35.0", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz", @@ -7704,6 +8852,14 @@ "typescript": ">=4.2.0" } }, + "node_modules/ts-dedent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz", + "integrity": "sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==", + "engines": { + "node": ">=6.10" + } + }, "node_modules/ts-interface-checker": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", @@ -7820,6 +8976,18 @@ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "devOptional": true }, + "node_modules/unist-util-stringify-position": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-3.0.3.tgz", + "integrity": "sha512-k5GzIBZ/QatR8N5X2y+drfpWG8IDBzdnVj6OInRNWm1oXrzydiaAT2OQiA8DPRRZyAKb9b6I2a6PxYklZD0gKg==", + "dependencies": { + "@types/unist": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", @@ -7905,6 +9073,23 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/uvu": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/uvu/-/uvu-0.5.6.tgz", + "integrity": "sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==", + "dependencies": { + "dequal": "^2.0.0", + "diff": "^5.0.0", + "kleur": "^4.0.3", + "sade": "^1.7.3" + }, + "bin": { + "uvu": "bin.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/value-or-function": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/value-or-function/-/value-or-function-4.0.0.tgz", @@ -8874,6 +10059,11 @@ "node": "*" } }, + "node_modules/web-worker": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/web-worker/-/web-worker-1.3.0.tgz", + "integrity": "sha512-BSR9wyRsy/KOValMgd5kMyr3JzpdeoR9KVId8u5GVlTTAtNChlsE4yTxeY7zMdNSyOmoKBv8NH2qeRY9Tg+IaA==" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -9024,6 +10214,14 @@ } } }, + "node_modules/xmlhttprequest-ssl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz", + "integrity": "sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 8522cffe5b..7ea3bf3c78 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-webui", - "version": "0.2.0.dev2", + "version": "0.2.5", "private": true, "scripts": { "dev": "npm run pyodide:fetch && vite dev --host", @@ -63,7 +63,10 @@ "js-sha256": "^0.10.1", "katex": "^0.16.9", "marked": "^9.1.0", + "mermaid": "^10.9.1", "pyodide": "^0.26.0-alpha.4", + "socket.io-client": "^4.7.5", + "sortablejs": "^1.15.2", "svelte-sonner": "^0.3.19", "tippy.js": "^6.3.7", "uuid": "^9.0.1" diff --git a/src/lib/apis/auths/index.ts b/src/lib/apis/auths/index.ts index 26feb29b6e..e202115b4a 100644 --- a/src/lib/apis/auths/index.ts +++ b/src/lib/apis/auths/index.ts @@ -1,5 +1,87 @@ import { WEBUI_API_BASE_URL } from '$lib/constants'; +export const getAdminDetails = async (token: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/auths/admin/details`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = err.detail; + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const getAdminConfig = async (token: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/auths/admin/config`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = err.detail; + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const updateAdminConfig = async (token: string, body: object) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/auths/admin/config`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + }, + body: JSON.stringify(body) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = err.detail; + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + export const getSessionUser = async (token: string) => { let error = null; diff --git a/src/lib/apis/chats/index.ts b/src/lib/apis/chats/index.ts index 834e29d296..b046f1b10d 100644 --- a/src/lib/apis/chats/index.ts +++ b/src/lib/apis/chats/index.ts @@ -162,6 +162,37 @@ export const getAllChats = async (token: string) => { return res; }; +export const getAllArchivedChats = async (token: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/chats/all/archived`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + export const getAllUserChats = async (token: string) => { let error = null; @@ -325,6 +356,44 @@ export const getChatByShareId = async (token: string, share_id: string) => { return res; }; +export const cloneChatById = async (token: string, id: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/chats/${id}/clone`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err; + + if ('detail' in err) { + error = err.detail; + } else { + error = err; + } + + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + export const shareChatById = async (token: string, id: string) => { let error = null; diff --git a/src/lib/apis/index.ts b/src/lib/apis/index.ts index dc51abd522..f6b2de4d0d 100644 --- a/src/lib/apis/index.ts +++ b/src/lib/apis/index.ts @@ -29,8 +29,24 @@ export const getModels = async (token: string = '') => { models = models .filter((models) => models) + // Sort the models .sort((a, b) => { - // Compare case-insensitively + // Check if models have position property + const aHasPosition = a.info?.meta?.position !== undefined; + const bHasPosition = b.info?.meta?.position !== undefined; + + // If both a and b have the position property + if (aHasPosition && bHasPosition) { + return a.info.meta.position - b.info.meta.position; + } + + // If only a has the position property, it should come first + if (aHasPosition) return -1; + + // If only b has the position property, it should come first + if (bHasPosition) return 1; + + // Compare case-insensitively by name for models without position property const lowerA = a.name.toLowerCase(); const lowerB = b.name.toLowerCase(); @@ -39,8 +55,8 @@ export const getModels = async (token: string = '') => { // If same case-insensitively, sort by original strings, // lowercase will come before uppercase due to ASCII values - if (a < b) return -1; - if (a > b) return 1; + if (a.name < b.name) return -1; + if (a.name > b.name) return 1; return 0; // They are equal }); @@ -49,6 +65,299 @@ export const getModels = async (token: string = '') => { return models; }; +type ChatCompletedForm = { + model: string; + messages: string[]; + chat_id: string; +}; + +export const chatCompleted = async (token: string, body: ChatCompletedForm) => { + let error = null; + + const res = await fetch(`${WEBUI_BASE_URL}/api/chat/completed`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + }, + body: JSON.stringify(body) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + if ('detail' in err) { + error = err.detail; + } else { + error = err; + } + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const getPipelinesList = async (token: string = '') => { + let error = null; + + const res = await fetch(`${WEBUI_BASE_URL}/api/pipelines/list`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = err; + return null; + }); + + if (error) { + throw error; + } + + let pipelines = res?.data ?? []; + return pipelines; +}; + +export const downloadPipeline = async (token: string, url: string, urlIdx: string) => { + let error = null; + + const res = await fetch(`${WEBUI_BASE_URL}/api/pipelines/add`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + }, + body: JSON.stringify({ + url: url, + urlIdx: urlIdx + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + if ('detail' in err) { + error = err.detail; + } else { + error = err; + } + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const deletePipeline = async (token: string, id: string, urlIdx: string) => { + let error = null; + + const res = await fetch(`${WEBUI_BASE_URL}/api/pipelines/delete`, { + method: 'DELETE', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + }, + body: JSON.stringify({ + id: id, + urlIdx: urlIdx + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + if ('detail' in err) { + error = err.detail; + } else { + error = err; + } + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const getPipelines = async (token: string, urlIdx?: string) => { + let error = null; + + const searchParams = new URLSearchParams(); + if (urlIdx !== undefined) { + searchParams.append('urlIdx', urlIdx); + } + + const res = await fetch(`${WEBUI_BASE_URL}/api/pipelines?${searchParams.toString()}`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = err; + return null; + }); + + if (error) { + throw error; + } + + let pipelines = res?.data ?? []; + return pipelines; +}; + +export const getPipelineValves = async (token: string, pipeline_id: string, urlIdx: string) => { + let error = null; + + const searchParams = new URLSearchParams(); + if (urlIdx !== undefined) { + searchParams.append('urlIdx', urlIdx); + } + + const res = await fetch( + `${WEBUI_BASE_URL}/api/pipelines/${pipeline_id}/valves?${searchParams.toString()}`, + { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + } + ) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = err; + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const getPipelineValvesSpec = async (token: string, pipeline_id: string, urlIdx: string) => { + let error = null; + + const searchParams = new URLSearchParams(); + if (urlIdx !== undefined) { + searchParams.append('urlIdx', urlIdx); + } + + const res = await fetch( + `${WEBUI_BASE_URL}/api/pipelines/${pipeline_id}/valves/spec?${searchParams.toString()}`, + { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + } + ) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = err; + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const updatePipelineValves = async ( + token: string = '', + pipeline_id: string, + valves: object, + urlIdx: string +) => { + let error = null; + + const searchParams = new URLSearchParams(); + if (urlIdx !== undefined) { + searchParams.append('urlIdx', urlIdx); + } + + const res = await fetch( + `${WEBUI_BASE_URL}/api/pipelines/${pipeline_id}/valves/update?${searchParams.toString()}`, + { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + }, + body: JSON.stringify(valves) + } + ) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + + if ('detail' in err) { + error = err.detail; + } else { + error = err; + } + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + export const getBackendConfig = async () => { let error = null; diff --git a/src/lib/apis/ollama/index.ts b/src/lib/apis/ollama/index.ts index efc3f0d0f0..084d2d5f18 100644 --- a/src/lib/apis/ollama/index.ts +++ b/src/lib/apis/ollama/index.ts @@ -1,5 +1,5 @@ import { OLLAMA_API_BASE_URL } from '$lib/constants'; -import { promptTemplate } from '$lib/utils'; +import { titleGenerationTemplate } from '$lib/utils'; export const getOllamaConfig = async (token: string = '') => { let error = null; @@ -135,10 +135,10 @@ export const updateOllamaUrls = async (token: string = '', urls: string[]) => { return res.OLLAMA_BASE_URLS; }; -export const getOllamaVersion = async (token: string = '') => { +export const getOllamaVersion = async (token: string, urlIdx?: number) => { let error = null; - const res = await fetch(`${OLLAMA_API_BASE_URL}/api/version`, { + const res = await fetch(`${OLLAMA_API_BASE_URL}/api/version${urlIdx ? `/${urlIdx}` : ''}`, { method: 'GET', headers: { Accept: 'application/json', @@ -212,7 +212,7 @@ export const generateTitle = async ( ) => { let error = null; - template = promptTemplate(template, prompt); + template = titleGenerationTemplate(template, prompt); console.log(template); @@ -369,42 +369,29 @@ export const generateChatCompletion = async (token: string = '', body: object) = return [res, controller]; }; -export const cancelOllamaRequest = async (token: string = '', requestId: string) => { +export const createModel = async ( + token: string, + tagName: string, + content: string, + urlIdx: string | null = null +) => { let error = null; - const res = await fetch(`${OLLAMA_API_BASE_URL}/cancel/${requestId}`, { - method: 'GET', - headers: { - 'Content-Type': 'text/event-stream', - Authorization: `Bearer ${token}` + const res = await fetch( + `${OLLAMA_API_BASE_URL}/api/create${urlIdx !== null ? `/${urlIdx}` : ''}`, + { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + name: tagName, + modelfile: content + }) } - }).catch((err) => { - error = err; - return null; - }); - - if (error) { - throw error; - } - - return res; -}; - -export const createModel = async (token: string, tagName: string, content: string) => { - let error = null; - - const res = await fetch(`${OLLAMA_API_BASE_URL}/api/create`, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${token}` - }, - body: JSON.stringify({ - name: tagName, - modelfile: content - }) - }).catch((err) => { + ).catch((err) => { error = err; return null; }); @@ -461,8 +448,10 @@ export const deleteModel = async (token: string, tagName: string, urlIdx: string export const pullModel = async (token: string, tagName: string, urlIdx: string | null = null) => { let error = null; + const controller = new AbortController(); const res = await fetch(`${OLLAMA_API_BASE_URL}/api/pull${urlIdx !== null ? `/${urlIdx}` : ''}`, { + signal: controller.signal, method: 'POST', headers: { Accept: 'application/json', @@ -485,7 +474,7 @@ export const pullModel = async (token: string, tagName: string, urlIdx: string | if (error) { throw error; } - return res; + return [res, controller]; }; export const downloadModel = async ( diff --git a/src/lib/apis/openai/index.ts b/src/lib/apis/openai/index.ts index 8afcec018c..2a52ebb320 100644 --- a/src/lib/apis/openai/index.ts +++ b/src/lib/apis/openai/index.ts @@ -1,5 +1,6 @@ import { OPENAI_API_BASE_URL } from '$lib/constants'; -import { promptTemplate } from '$lib/utils'; +import { titleGenerationTemplate } from '$lib/utils'; +import { type Model, models, settings } from '$lib/stores'; export const getOpenAIConfig = async (token: string = '') => { let error = null; @@ -202,17 +203,20 @@ export const updateOpenAIKeys = async (token: string = '', keys: string[]) => { return res.OPENAI_API_KEYS; }; -export const getOpenAIModels = async (token: string = '') => { +export const getOpenAIModels = async (token: string, urlIdx?: number) => { let error = null; - const res = await fetch(`${OPENAI_API_BASE_URL}/models`, { - method: 'GET', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - ...(token && { authorization: `Bearer ${token}` }) + const res = await fetch( + `${OPENAI_API_BASE_URL}/models${typeof urlIdx === 'number' ? `/${urlIdx}` : ''}`, + { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } } - }) + ) .then(async (res) => { if (!res.ok) throw await res.json(); return res.json(); @@ -226,20 +230,7 @@ export const getOpenAIModels = async (token: string = '') => { throw error; } - const models = Array.isArray(res) ? res : res?.data ?? null; - - return models - ? models - .map((model) => ({ - id: model.id, - name: model.name ?? model.id, - external: true, - custom_info: model.custom_info - })) - .sort((a, b) => { - return a.name.localeCompare(b.name); - }) - : models; + return res; }; export const getOpenAIModelsDirect = async ( @@ -345,11 +336,12 @@ export const generateTitle = async ( template: string, model: string, prompt: string, + chat_id?: string, url: string = OPENAI_API_BASE_URL ) => { let error = null; - template = promptTemplate(template, prompt); + template = titleGenerationTemplate(template, prompt); console.log(template); @@ -370,7 +362,9 @@ export const generateTitle = async ( ], stream: false, // Restricting the max tokens to 50 to avoid long titles - max_tokens: 50 + max_tokens: 50, + ...(chat_id && { chat_id: chat_id }), + title: true }) }) .then(async (res) => { @@ -391,3 +385,71 @@ export const generateTitle = async ( return res?.choices[0]?.message?.content.replace(/["']/g, '') ?? 'New Chat'; }; + +export const generateSearchQuery = async ( + token: string = '', + model: string, + previousMessages: string[], + prompt: string, + url: string = OPENAI_API_BASE_URL +): Promise => { + let error = null; + + // TODO: Allow users to specify the prompt + // Get the current date in the format "January 20, 2024" + const currentDate = new Intl.DateTimeFormat('en-US', { + year: 'numeric', + month: 'long', + day: '2-digit' + }).format(new Date()); + + const res = await fetch(`${url}/chat/completions`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + model: model, + // Few shot prompting + messages: [ + { + role: 'assistant', + content: `You are tasked with generating web search queries. Give me an appropriate query to answer my question for google search. Answer with only the query. Today is ${currentDate}.` + }, + { + role: 'user', + content: prompt + } + // { + // role: 'user', + // content: + // (previousMessages.length > 0 + // ? `Previous Questions:\n${previousMessages.join('\n')}\n\n` + // : '') + `Current Question: ${prompt}` + // } + ], + stream: false, + // Restricting the max tokens to 30 to avoid long search queries + max_tokens: 30 + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + if ('detail' in err) { + error = err.detail; + } + return undefined; + }); + + if (error) { + throw error; + } + + return res?.choices[0]?.message?.content.replace(/["']/g, '') ?? undefined; +}; diff --git a/src/lib/apis/rag/index.ts b/src/lib/apis/rag/index.ts index b792b7f009..ca68827a3b 100644 --- a/src/lib/apis/rag/index.ts +++ b/src/lib/apis/rag/index.ts @@ -359,6 +359,32 @@ export const scanDocs = async (token: string) => { return res; }; +export const resetUploadDir = async (token: string) => { + let error = null; + + const res = await fetch(`${RAG_API_BASE_URL}/reset/uploads`, { + method: 'GET', + headers: { + Accept: 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + export const resetVectorDB = async (token: string) => { let error = null; @@ -415,6 +441,7 @@ export const getEmbeddingConfig = async (token: string) => { type OpenAIConfigForm = { key: string; url: string; + batch_size: number; }; type EmbeddingModelUpdateForm = { @@ -513,3 +540,44 @@ export const updateRerankingConfig = async (token: string, payload: RerankingMod return res; }; + +export const runWebSearch = async ( + token: string, + query: string, + collection_name?: string +): Promise => { + let error = null; + + const res = await fetch(`${RAG_API_BASE_URL}/web/search`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + query, + collection_name: collection_name ?? '' + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = err.detail; + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export interface SearchDocument { + status: boolean; + collection_name: string; + filenames: string[]; +} diff --git a/src/lib/apis/streaming/index.ts b/src/lib/apis/streaming/index.ts index bb1b55b1d0..f91edad83c 100644 --- a/src/lib/apis/streaming/index.ts +++ b/src/lib/apis/streaming/index.ts @@ -8,6 +8,16 @@ type TextStreamUpdate = { citations?: any; // eslint-disable-next-line @typescript-eslint/no-explicit-any error?: any; + usage?: ResponseUsage; +}; + +type ResponseUsage = { + /** Including images and tools if any */ + prompt_tokens: number; + /** The tokens generated */ + completion_tokens: number; + /** Sum of the above two fields */ + total_tokens: number; }; // createOpenAITextStream takes a responseBody with a SSE response, @@ -59,7 +69,11 @@ async function* openAIStreamToIterator( continue; } - yield { done: false, value: parsedData.choices?.[0]?.delta?.content ?? '' }; + yield { + done: false, + value: parsedData.choices?.[0]?.delta?.content ?? '', + usage: parsedData.usage + }; } catch (e) { console.error('Error extracting delta from SSE event:', e); } diff --git a/src/lib/apis/utils/index.ts b/src/lib/apis/utils/index.ts index 3f28d9444d..acb4a04aae 100644 --- a/src/lib/apis/utils/index.ts +++ b/src/lib/apis/utils/index.ts @@ -108,3 +108,39 @@ export const downloadDatabase = async (token: string) => { throw error; } }; + +export const downloadLiteLLMConfig = async (token: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/utils/litellm/config`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}` + } + }) + .then(async (response) => { + if (!response.ok) { + throw await response.json(); + } + return response.blob(); + }) + .then((blob) => { + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'config.yaml'; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + }) + .catch((err) => { + console.log(err); + error = err.detail; + return null; + }); + + if (error) { + throw error; + } +}; diff --git a/src/lib/components/admin/Settings/Database.svelte b/src/lib/components/admin/Settings/Database.svelte index 3ae01a668b..9ce8e9d8da 100644 --- a/src/lib/components/admin/Settings/Database.svelte +++ b/src/lib/components/admin/Settings/Database.svelte @@ -2,7 +2,7 @@ import fileSaver from 'file-saver'; const { saveAs } = fileSaver; - import { downloadDatabase } from '$lib/apis/utils'; + import { downloadDatabase, downloadLiteLLMConfig } from '$lib/apis/utils'; import { onMount, getContext } from 'svelte'; import { config, user } from '$lib/stores'; import { toast } from 'svelte-sonner'; @@ -68,10 +68,8 @@ -
- {/if} + +
+ +
+ + + +
diff --git a/src/lib/components/admin/Settings/General.svelte b/src/lib/components/admin/Settings/General.svelte index 572ff82add..a7ffdabfb3 100644 --- a/src/lib/components/admin/Settings/General.svelte +++ b/src/lib/components/admin/Settings/General.svelte @@ -6,61 +6,44 @@ updateWebhookUrl } from '$lib/apis'; import { + getAdminConfig, getDefaultUserRole, getJWTExpiresDuration, getSignUpEnabledStatus, toggleSignUpEnabledStatus, + updateAdminConfig, updateDefaultUserRole, updateJWTExpiresDuration } from '$lib/apis/auths'; + import Switch from '$lib/components/common/Switch.svelte'; import { onMount, getContext } from 'svelte'; const i18n = getContext('i18n'); export let saveHandler: Function; - let signUpEnabled = true; - let defaultUserRole = 'pending'; - let JWTExpiresIn = ''; + let adminConfig = null; let webhookUrl = ''; - let communitySharingEnabled = true; - const toggleSignUpEnabled = async () => { - signUpEnabled = await toggleSignUpEnabledStatus(localStorage.token); - }; - - const updateDefaultUserRoleHandler = async (role) => { - defaultUserRole = await updateDefaultUserRole(localStorage.token, role); - }; - - const updateJWTExpiresDurationHandler = async (duration) => { - JWTExpiresIn = await updateJWTExpiresDuration(localStorage.token, duration); - }; - - const updateWebhookUrlHandler = async () => { + const updateHandler = async () => { webhookUrl = await updateWebhookUrl(localStorage.token, webhookUrl); - }; + const res = await updateAdminConfig(localStorage.token, adminConfig); - const toggleCommunitySharingEnabled = async () => { - communitySharingEnabled = await toggleCommunitySharingEnabledStatus(localStorage.token); + if (res) { + toast.success(i18n.t('Settings updated successfully')); + } else { + toast.error(i18n.t('Failed to update settings')); + } }; onMount(async () => { await Promise.all([ (async () => { - signUpEnabled = await getSignUpEnabledStatus(localStorage.token); - })(), - (async () => { - defaultUserRole = await getDefaultUserRole(localStorage.token); - })(), - (async () => { - JWTExpiresIn = await getJWTExpiresDuration(localStorage.token); + adminConfig = await getAdminConfig(localStorage.token); })(), + (async () => { webhookUrl = await getWebhookUrl(localStorage.token); - })(), - (async () => { - communitySharingEnabled = await getCommunitySharingEnabledStatus(localStorage.token); })() ]); }); @@ -69,156 +52,94 @@
{ - updateJWTExpiresDurationHandler(JWTExpiresIn); - updateWebhookUrlHandler(); + updateHandler(); saveHandler(); }} > -
-
-
{$i18n.t('General Settings')}
+
+ {#if adminConfig !== null} +
+
{$i18n.t('General Settings')}
-
-
{$i18n.t('Enable New Sign Ups')}
+
+
{$i18n.t('Enable New Sign Ups')}
-
+ +
+
{$i18n.t('Default User Role')}
+
+ +
+
+ +
+ +
+
+ {$i18n.t('Show Admin Details in Account Pending Overlay')} +
+ + +
+ +
+
{$i18n.t('Enable Community Sharing')}
+ + +
+ +
+ +
+
+
{$i18n.t('JWT Expiration')}
+
+ +
+ +
+ +
+ {$i18n.t('Valid time units:')} + {$i18n.t("'s', 'm', 'h', 'd', 'w' or '-1' for no expiration.")} - - +
+
- {$i18n.t('Disabled')} - {/if} - -
+
-
-
{$i18n.t('Default User Role')}
-
- +
+
+
{$i18n.t('Webhook URL')}
+
+ +
+ +
- -
-
{$i18n.t('Enable Community Sharing')}
- - -
- -
- -
-
-
{$i18n.t('Webhook URL')}
-
- -
- -
-
- -
- -
-
-
{$i18n.t('JWT Expiration')}
-
- -
- -
- -
- {$i18n.t('Valid time units:')} - {$i18n.t("'s', 'm', 'h', 'd', 'w' or '-1' for no expiration.")} -
-
-
+ {/if}
diff --git a/src/lib/components/admin/Settings/Pipelines.svelte b/src/lib/components/admin/Settings/Pipelines.svelte new file mode 100644 index 0000000000..875e610519 --- /dev/null +++ b/src/lib/components/admin/Settings/Pipelines.svelte @@ -0,0 +1,405 @@ + + + { + updateHandler(); + }} +> +
+ {#if PIPELINES_LIST !== null} +
+
+ {$i18n.t('Manage Pipelines')} +
+
+ + {#if PIPELINES_LIST.length > 0} +
+
+
+ +
+
+
+ +
+
+ {$i18n.t('Install from Github URL')} +
+
+
+ +
+ +
+ +
+ Warning: Pipelines are a plugin + system with arbitrary code execution — + don't fetch random pipelines from sources you don't trust. +
+
+ +
+ + {#if pipelines !== null} + {#if pipelines.length > 0} +
+
+ {$i18n.t('Pipelines Valves')} +
+
+
+ {#if pipelines.length > 0} +
+
+ +
+ + +
+ {/if} + +
+ {#if pipelines[selectedPipelineIdx].valves} + {#if valves} + {#each Object.keys(valves_spec.properties) as property, idx} +
+
+
+ {valves_spec.properties[property].title} +
+ + +
+ + {#if (valves[property] ?? null) !== null} +
+
+ +
+
+ {/if} +
+ {/each} + {:else} + + {/if} + {:else} +
No valves
+ {/if} +
+
+ {:else if pipelines.length === 0} +
Pipelines Not Detected
+ {/if} + {:else} +
+
+ +
+
+ {/if} + {/if} + {:else} +
+
+ +
+
+ {/if} +
+ +
+ +
+ diff --git a/src/lib/components/admin/SettingsModal.svelte b/src/lib/components/admin/SettingsModal.svelte index 38a2602b60..78f48cdfcb 100644 --- a/src/lib/components/admin/SettingsModal.svelte +++ b/src/lib/components/admin/SettingsModal.svelte @@ -8,6 +8,7 @@ import Banners from '$lib/components/admin/Settings/Banners.svelte'; import { toast } from 'svelte-sonner'; + import Pipelines from './Settings/Pipelines.svelte'; const i18n = getContext('i18n'); @@ -149,33 +150,65 @@
{$i18n.t('Banners')}
+ +
{#if selectedTab === 'general'} { - show = false; toast.success($i18n.t('Settings saved successfully!')); }} /> {:else if selectedTab === 'users'} { - show = false; toast.success($i18n.t('Settings saved successfully!')); }} /> {:else if selectedTab === 'db'} { - show = false; toast.success($i18n.t('Settings saved successfully!')); }} /> {:else if selectedTab === 'banners'} { - show = false; + toast.success($i18n.t('Settings saved successfully!')); + }} + /> + {:else if selectedTab === 'pipelines'} + { toast.success($i18n.t('Settings saved successfully!')); }} /> diff --git a/src/lib/components/chat/Chat.svelte b/src/lib/components/chat/Chat.svelte index d2050dd05b..5d2f860a81 100644 --- a/src/lib/components/chat/Chat.svelte +++ b/src/lib/components/chat/Chat.svelte @@ -1,6 +1,7 @@ + + { + if (e.detail === false) { + onClose(); + } + }} +> + + + + +
+ + {#if $config?.features?.enable_web_search} +
+
+ +
{$i18n.t('Web Search')}
+
+ + +
+ +
+ {/if} + + { + uploadFilesHandler(); + }} + > + +
{$i18n.t('Upload Files')}
+ + +
+ diff --git a/src/lib/components/chat/Messages.svelte b/src/lib/components/chat/Messages.svelte index 4e2a383a18..f633b57744 100644 --- a/src/lib/components/chat/Messages.svelte +++ b/src/lib/components/chat/Messages.svelte @@ -1,8 +1,7 @@ -
+
{#if messages.length == 0} {#key chatId} {#each messages as message, messageIdx} -
+
@@ -340,6 +339,7 @@ {@html lang}
- {#if lang === 'python' || (lang === '' && checkPythonCode(code))} + {#if lang.toLowerCase() === 'python' || lang.toLowerCase() === 'py' || (lang === '' && checkPythonCode(code))} {#if executing}
Running
{:else} diff --git a/src/lib/components/chat/Messages/CompareMessages.svelte b/src/lib/components/chat/Messages/CompareMessages.svelte index f904a57abb..227b125ec2 100644 --- a/src/lib/components/chat/Messages/CompareMessages.svelte +++ b/src/lib/components/chat/Messages/CompareMessages.svelte @@ -13,6 +13,8 @@ export let parentMessage; + export let readOnly = false; + export let updateChatMessages: Function; export let confirmEditResponseMessage: Function; export let rateMessage: Function; @@ -134,6 +136,7 @@ {confirmEditResponseMessage} showPreviousMessage={() => showPreviousMessage(model)} showNextMessage={() => showNextMessage(model)} + {readOnly} {rateMessage} {copyToClipboard} {continueGeneration} diff --git a/src/lib/components/chat/Messages/Placeholder.svelte b/src/lib/components/chat/Messages/Placeholder.svelte index ed121dbe65..17beff50ed 100644 --- a/src/lib/components/chat/Messages/Placeholder.svelte +++ b/src/lib/components/chat/Messages/Placeholder.svelte @@ -64,7 +64,7 @@
- {#if models[selectedModelIdx]?.info} + {#if models[selectedModelIdx]?.info?.meta?.description ?? null}
{models[selectedModelIdx]?.info?.meta?.description}
diff --git a/src/lib/components/chat/Messages/ResponseMessage.svelte b/src/lib/components/chat/Messages/ResponseMessage.svelte index c5d753b51f..4a5d7fb0da 100644 --- a/src/lib/components/chat/Messages/ResponseMessage.svelte +++ b/src/lib/components/chat/Messages/ResponseMessage.svelte @@ -5,6 +5,7 @@ import tippy from 'tippy.js'; import auto_render from 'katex/dist/contrib/auto-render.mjs'; import 'katex/dist/katex.min.css'; + import mermaid from 'mermaid'; import { fade } from 'svelte/transition'; import { createEventDispatcher } from 'svelte'; @@ -33,6 +34,8 @@ import Tooltip from '$lib/components/common/Tooltip.svelte'; import RateComment from './RateComment.svelte'; import CitationsModal from '$lib/components/chat/Messages/CitationsModal.svelte'; + import Spinner from '$lib/components/common/Spinner.svelte'; + import WebSearchResults from './ResponseMessage/WebSearchResults.svelte'; export let message; export let siblings; @@ -106,8 +109,13 @@ renderLatex(); if (message.info) { - tooltipInstance = tippy(`#info-${message.id}`, { - content: `response_token/s: ${ + let tooltipContent = ''; + if (message.info.openai) { + tooltipContent = `prompt_tokens: ${message.info.prompt_tokens ?? 'N/A'}
+ completion_tokens: ${message.info.completion_tokens ?? 'N/A'}
+ total_tokens: ${message.info.total_tokens ?? 'N/A'}`; + } else { + tooltipContent = `response_token/s: ${ `${ Math.round( ((message.info.eval_count ?? 0) / (message.info.eval_duration / 1000000000)) * 100 @@ -137,9 +145,10 @@ eval_duration: ${ Math.round(((message.info.eval_duration ?? 0) / 1000000) * 100) / 100 ?? 'N/A' }ms
- approximate_total: ${approximateToHumanReadable( - message.info.total_duration - )}`, + approximate_total: ${approximateToHumanReadable(message.info.total_duration)}`; + } + tooltipInstance = tippy(`#info-${message.id}`, { + content: `${tooltipContent}`, allowHTML: true }); } @@ -332,9 +341,24 @@ generatingImage = false; }; + $: if (!edit) { + (async () => { + await tick(); + renderStyling(); + + await mermaid.run({ + querySelector: '.mermaid' + }); + })(); + } + onMount(async () => { await tick(); renderStyling(); + + await mermaid.run({ + querySelector: '.mermaid' + }); }); @@ -364,7 +388,7 @@ {/if} - {#if message.files} + {#if (message?.files ?? []).filter((f) => f.type === 'image').length > 0}
{#each message.files as file}
@@ -377,9 +401,35 @@ {/if}
+ {#if message?.status} +
+ {#if message?.status?.done === false} +
+ +
+ {/if} + + {#if message?.status?.action === 'web_search' && message?.status?.urls} + +
+
+ {message.status.description} +
+
+ + {:else} +
+
+ {message.status.description} +
+
+ {/if} +
+ {/if} + {#if edit === true}