enh/pref: convert markdown base64 images to urls

Co-Authored-By: Shirasawa <kaguyashirasawa@gmail.com>

backend/open_webui/env.py

@@ -544,6 +544,10 @@ else:
 # CHAT
 ####################################
 
+ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION = (
+    os.environ.get("REPLACE_IMAGE_URLS_IN_CHAT_RESPONSE", "False").lower() == "true"
+)
+
 CHAT_RESPONSE_STREAM_DELTA_CHUNK_SIZE = os.environ.get(
     "CHAT_RESPONSE_STREAM_DELTA_CHUNK_SIZE", "1"
 )

backend/open_webui/utils/files.py

@@ -16,10 +16,15 @@ from open_webui.routers.files import upload_file_handler
 import mimetypes
 import base64
 import io
+import re
+
+
+BASE64_IMAGE_URL_PREFIX = re.compile(r"data:image/\w+;base64,", re.IGNORECASE)
+MARKDOWN_IMAGE_URL_PATTERN = re.compile(r"!\[(.*?)\]\((.+?)\)", re.IGNORECASE)
 
 
 def get_image_url_from_base64(request, base64_image_string, metadata, user):
-    if "data:image/png;base64" in base64_image_string:
+    if BASE64_IMAGE_URL_PREFIX.match(base64_image_string):
         image_url = ""
         # Extract base64 image data from the line
         image_data, content_type = get_image_data(base64_image_string)
@@ -35,6 +40,19 @@ def get_image_url_from_base64(request, base64_image_string, metadata, user):
     return None
 
 
+def convert_markdown_base64_images(request, content: str, metadata, user):
+    def replace(match):
+        base64_string = match.group(2)
+        MIN_REPLACEMENT_URL_LENGTH = 1024
+        if len(base64_string) > MIN_REPLACEMENT_URL_LENGTH:
+            url = get_image_url_from_base64(request, base64_string, metadata, user)
+            if url:
+                return f"![{match.group(1)}]({url})"
+        return match.group(0)
+
+    return MARKDOWN_IMAGE_URL_PATTERN.sub(replace, content)
+
+
 def load_b64_audio_data(b64_str):
     try:
         if "," in b64_str:

backend/open_webui/utils/middleware.py

@@ -58,7 +58,7 @@ from open_webui.routers.memories import query_memory, QueryMemoryForm
 
 from open_webui.utils.webhook import post_webhook
 from open_webui.utils.files import (
-    get_audio_url_from_base64,
+    convert_markdown_base64_images,
     get_file_url_from_base64,
     get_image_url_from_base64,
 )
@@ -112,6 +112,7 @@ from open_webui.config import (
 from open_webui.env import (
     SRC_LOG_LEVELS,
     GLOBAL_LOG_LEVEL,
+    ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION,
     CHAT_RESPONSE_STREAM_DELTA_CHUNK_SIZE,
     CHAT_RESPONSE_MAX_TOOL_CALL_RETRIES,
     BYPASS_MODEL_ACCESS_CONTROL,
@@ -2680,6 +2681,11 @@ async def process_chat_response(
                                                 }
                                             )
 
+                                        if ENABLE_CHAT_RESPONSE_BASE64_IMAGE_URL_CONVERSION:
+                                            value = convert_markdown_base64_images(
+                                                request, value, metadata, user
+                                            )
+
                                         content = f"{content}{value}"
                                         if not content_blocks:
                                             content_blocks.append(