feat: tool ui element support

backend/open_webui/utils/middleware.py

@@ -1581,7 +1581,8 @@ async def process_chat_response(
                                         break
 
                                 if tool_result is not None:
-                                    tool_calls_display_content = f'{tool_calls_display_content}<details type="tool_calls" done="true" id="{tool_call_id}" name="{tool_name}" arguments="{html.escape(json.dumps(tool_arguments))}" result="{html.escape(json.dumps(tool_result, ensure_ascii=False))}" files="{html.escape(json.dumps(tool_result_files)) if tool_result_files else ""}">\n<summary>Tool Executed</summary>\n</details>\n'
+                                    tool_result_embeds = result.get("embeds", "")
+                                    tool_calls_display_content = f'{tool_calls_display_content}<details type="tool_calls" done="true" id="{tool_call_id}" name="{tool_name}" arguments="{html.escape(json.dumps(tool_arguments))}" result="{html.escape(json.dumps(tool_result, ensure_ascii=False))}" files="{html.escape(json.dumps(tool_result_files)) if tool_result_files else ""}" embeds="{html.escape(json.dumps(tool_result_embeds))}">\n<summary>Tool Executed</summary>\n</details>\n'
                                 else:
                                     tool_calls_display_content = f'{tool_calls_display_content}<details type="tool_calls" done="false" id="{tool_call_id}" name="{tool_name}" arguments="{html.escape(json.dumps(tool_arguments))}">\n<summary>Executing...</summary>\n</details>\n'
 
@@ -2402,6 +2403,38 @@ async def process_chat_response(
                             except Exception as e:
                                 tool_result = str(e)
 
+                        tool_result_embeds = []
+                        if tool.get("type") == "external" and isinstance(
+                            tool_result, tuple
+                        ):
+
+                            tool_result, tool_response_headers = tool_result
+
+                            if tool_response_headers:
+                                content_disposition = tool_response_headers.get(
+                                    "Content-Disposition", ""
+                                )
+
+                                if "inline" in content_disposition:
+                                    content_type = tool_response_headers.get(
+                                        "Content-Type", ""
+                                    )
+                                    location = tool_response_headers.get("Location", "")
+
+                                    if "text/html" in content_type:
+                                        # Display as iframe embed
+                                        tool_result_embeds.append(tool_result)
+                                        tool_result = {
+                                            "status": "success",
+                                            "message": "Displayed as embed",
+                                        }
+                                    elif location:
+                                        tool_result_embeds.append(location)
+                                        tool_result = {
+                                            "status": "success",
+                                            "message": "Displayed as embed",
+                                        }
+
                         tool_result_files = []
                         if isinstance(tool_result, list):
                             for item in tool_result:
@@ -2426,6 +2459,11 @@ async def process_chat_response(
                                     if tool_result_files
                                     else {}
                                 ),
+                                **(
+                                    {"embeds": tool_result_embeds}
+                                    if tool_result_embeds
+                                    else {}
+                                ),
                             }
                         )
 

backend/open_webui/utils/tools.py

@@ -171,6 +171,8 @@ async def get_tools(
                         "tool_id": tool_id,
                         "callable": callable,
                         "spec": spec,
+                        # Misc info
+                        "type": "external",
                     }
 
                     # Handle function name collisions
@@ -646,7 +648,7 @@ async def execute_tool_server(
     name: str,
     params: Dict[str, Any],
     server_data: Dict[str, Any],
-) -> Any:
+) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
     error = None
     try:
         openapi = server_data.get("openapi", {})
@@ -718,6 +720,7 @@ async def execute_tool_server(
                     headers=headers,
                     cookies=cookies,
                     ssl=AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL,
+                    allow_redirects=False,
                 ) as response:
                     if response.status >= 400:
                         text = await response.text()
@@ -728,13 +731,15 @@ async def execute_tool_server(
                     except Exception:
                         response_data = await response.text()
 
-                    return response_data
+                    response_headers = response.headers
+                    return (response_data, response_headers)
             else:
                 async with request_method(
                     final_url,
                     headers=headers,
                     cookies=cookies,
                     ssl=AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL,
+                    allow_redirects=False,
                 ) as response:
                     if response.status >= 400:
                         text = await response.text()
@@ -745,12 +750,13 @@ async def execute_tool_server(
                     except Exception:
                         response_data = await response.text()
 
-                    return response_data
+                    response_headers = response.headers
+                    return (response_data, response_headers)
 
     except Exception as err:
         error = str(err)
         log.exception(f"API Request Error: {error}")
-        return {"error": error}
+        return ({"error": error}, None)
 
 
 def get_tool_server_url(url: Optional[str], path: str) -> str:

src/lib/components/common/Collapsible.svelte

@@ -38,6 +38,8 @@
 	import CodeBlock from '../chat/Messages/CodeBlock.svelte';
 	import Markdown from '../chat/Messages/Markdown.svelte';
 	import Image from './Image.svelte';
+	import FullHeightIframe from './FullHeightIframe.svelte';
+	import { settings } from '$lib/stores';
 
 	export let open = false;
 
@@ -213,6 +215,21 @@
 		{@const args = decode(attributes?.arguments)}
 		{@const result = decode(attributes?.result ?? '')}
 		{@const files = parseJSONString(decode(attributes?.files ?? ''))}
+		{@const embeds = parseJSONString(decode(attributes?.embeds ?? ''))}
+
+		{#if embeds && Array.isArray(embeds) && embeds.length > 0}
+			{#each embeds as embed, idx}
+				<div class="my-2" id={`${collapsibleId}-tool-calls-${attributes?.id}-embed-${idx}`}>
+					<FullHeightIframe
+						src={embed}
+						allowScripts={true}
+						allowForms={$settings?.iframeSandboxAllowForms ?? false}
+						allowSameOrigin={$settings?.iframeSandboxAllowSameOrigin ?? false}
+						allowPopups={$settings?.iframeSandboxAllowPopups ?? false}
+					/>
+				</div>
+			{/each}
+		{/if}
 
 		{#if !grow}
 			{#if open && !hide}

src/lib/components/common/FullHeightIframe.svelte

@@ -0,0 +1,144 @@
+<script lang="ts">
+	import { onDestroy, onMount } from 'svelte';
+
+	// Props
+	export let src: string | null = null; // URL or raw HTML (auto-detected)
+	export let title = 'Embedded Content';
+	export let initialHeight = 400; // fallback height if we can't measure
+	export let allowScripts = true;
+	export let allowForms = false;
+
+	export let allowSameOrigin = false; // set to true only when you trust the content
+	export let allowPopups = false;
+	export let allowDownloads = true;
+
+	export let referrerPolicy: HTMLIFrameElement['referrerPolicy'] =
+		'strict-origin-when-cross-origin';
+	export let allowFullscreen = true;
+
+	let iframe: HTMLIFrameElement | null = null;
+
+	// Derived: build sandbox attribute from flags
+	$: sandbox =
+		[
+			allowScripts && 'allow-scripts',
+			allowForms && 'allow-forms',
+			allowSameOrigin && 'allow-same-origin',
+			allowPopups && 'allow-popups',
+			allowDownloads && 'allow-downloads'
+		]
+			.filter(Boolean)
+			.join(' ') || undefined;
+
+	// Detect URL vs raw HTML and prep src/srcdoc
+	$: isUrl = typeof src === 'string' && /^(https?:)?\/\//i.test(src);
+	$: iframeSrc = isUrl ? (src as string) : null;
+	$: iframeDoc = !isUrl && src ? ensureAutosizer(src) : null;
+
+	// Try to measure same-origin content safely
+	function resizeSameOrigin() {
+		if (!iframe) return;
+		try {
+			const doc = iframe.contentDocument || iframe.contentWindow?.document;
+			if (!doc) return;
+			const h = Math.max(doc.documentElement?.scrollHeight ?? 0, doc.body?.scrollHeight ?? 0);
+			if (h > 0) iframe.style.height = h + 20 + 'px';
+		} catch {
+			// Cross-origin → rely on postMessage from inside the iframe
+		}
+	}
+
+	// Handle height messages from the iframe (we also verify the sender)
+	function onMessage(e: MessageEvent) {
+		if (!iframe || e.source !== iframe.contentWindow) return;
+		const data = e.data as { type?: string; height?: number };
+		if (data?.type === 'iframe:height' && typeof data.height === 'number') {
+			iframe.style.height = Math.max(0, data.height) + 'px';
+		}
+	}
+
+	// Ensure event listener bound only while component lives
+	onMount(() => {
+		window.addEventListener('message', onMessage);
+	});
+
+	onDestroy(() => {
+		window.removeEventListener('message', onMessage);
+	});
+
+	// When the iframe loads, try same-origin resize (cross-origin will noop)
+	function onLoad() {
+		// schedule after layout
+		requestAnimationFrame(resizeSameOrigin);
+	}
+
+	/**
+	 * If user passes raw HTML, we inject a tiny autosizer that posts height.
+	 * This helps both same-origin and "about:srcdoc" cases.
+	 * (No effect if the caller already includes their own autosizer.)
+	 */
+	function ensureAutosizer(html: string): string {
+		const hasOurHook = /iframe:height/.test(html) || /postMessage\(.+height/i.test(html);
+		if (hasOurHook) return html;
+
+		// This script uses ResizeObserver to post the document height
+		const autosizer = `
+<script>
+(function () {
+  function send() {
+    try {
+      var h = Math.max(
+        document.documentElement.scrollHeight || 0,
+        document.body ? document.body.scrollHeight : 0
+      );
+      parent.postMessage({ type: 'iframe:height', height: h + 20 }, '*');
+    } catch (e) {}
+  }
+  var ro = new ResizeObserver(function(){ send(); });
+  ro.observe(document.documentElement);
+  window.addEventListener('load', send);
+  // Also observe body if present
+  if (document.body) ro.observe(document.body);
+  // Periodic guard in case of late content
+  setTimeout(send, 0);
+  setTimeout(send, 250);
+  setTimeout(send, 1000);
+})();
+<\/script>`;
+		// inject before </body> if present, else append
+		return (
+			html.replace(/<\/body\s*>/i, autosizer + '</body>') +
+			(/<\/body\s*>/i.test(html) ? '' : autosizer)
+		);
+	}
+</script>
+
+{#if iframeDoc}
+	<iframe
+		bind:this={iframe}
+		srcdoc={iframeDoc}
+		{title}
+		class="w-full rounded-xl"
+		style={`height:${initialHeight}px;`}
+		width="100%"
+		frameborder="0"
+		{sandbox}
+		referrerpolicy={referrerPolicy}
+		{allowFullscreen}
+		on:load={onLoad}
+	/>
+{:else if iframeSrc}
+	<iframe
+		bind:this={iframe}
+		src={iframeSrc}
+		{title}
+		class="w-full rounded-xl"
+		style={`height:${initialHeight}px;`}
+		width="100%"
+		frameborder="0"
+		{sandbox}
+		referrerpolicy={referrerPolicy}
+		{allowFullscreen}
+		on:load={onLoad}
+	/>
+{/if}