From 86caca495bd698e7d96d8f1bf7b142875903d068 Mon Sep 17 00:00:00 2001 From: Jannik Streidl Date: Mon, 7 Oct 2024 21:04:06 +0200 Subject: [PATCH 001/112] enh: show source documents vector distance + cleaner source view --- .../components/chat/Messages/Citations.svelte | 124 +++++++++++++++--- .../chat/Messages/CitationsModal.svelte | 48 ++++++- src/lib/i18n/locales/de-DE/translation.json | 4 +- 3 files changed, 151 insertions(+), 25 deletions(-) diff --git a/src/lib/components/chat/Messages/Citations.svelte b/src/lib/components/chat/Messages/Citations.svelte index 2c23e87a48..5f1b655f44 100644 --- a/src/lib/components/chat/Messages/Citations.svelte +++ b/src/lib/components/chat/Messages/Citations.svelte @@ -1,13 +1,24 @@ {#if _citations.length > 0}
- {#each _citations as citation, idx} -
- +
+ {/each} + {:else} + +
-
- {idx + 1} + {$i18n.t('References from')} + {#each _citations.slice(0, 2) as citation, idx} +
+ +
+ {#if idx === 0} + , + {/if} + {/each} + {$i18n.t('and')} +
+ {_citations.length - 2}
-
- {citation.source.name} + {$i18n.t('more')} + {#if isCollapsibleOpen} + + {:else} + + {/if} +
+
+
+ {#each _citations as citation, idx} +
+ +
+ {/each}
- -
- {/each} +
+ + {/if}
{/if} diff --git a/src/lib/components/chat/Messages/CitationsModal.svelte b/src/lib/components/chat/Messages/CitationsModal.svelte index 923fdea54b..d2510d924f 100644 --- a/src/lib/components/chat/Messages/CitationsModal.svelte +++ b/src/lib/components/chat/Messages/CitationsModal.svelte @@ -2,6 +2,7 @@ import { getContext, onMount, tick } from 'svelte'; import Modal from '$lib/components/common/Modal.svelte'; import Tooltip from '$lib/components/common/Tooltip.svelte'; + const i18n = getContext('i18n'); export let show = false; @@ -9,14 +10,32 @@ let mergedDocuments = []; + function calculatePercentage(distance) { + return Math.max(0, Math.min(100, (1 - distance / 2) * 100)); + } + + function getRelevanceColor(percentage) { + if (percentage >= 80) + return 'bg-green-200 dark:bg-green-800 text-green-800 dark:text-green-200'; + if (percentage >= 60) + return 'bg-yellow-200 dark:bg-yellow-800 text-yellow-800 dark:text-yellow-200'; + if (percentage >= 40) + return 'bg-orange-200 dark:bg-orange-800 text-orange-800 dark:text-orange-200'; + return 'bg-red-200 dark:bg-red-800 text-red-800 dark:text-red-200'; + } + $: if (citation) { mergedDocuments = citation.document?.map((c, i) => { return { source: citation.source, document: c, - metadata: citation.metadata?.[i] + metadata: citation.metadata?.[i], + distance: citation.distances?.[i] }; }); + if (mergedDocuments.every((doc) => doc.distance !== undefined)) { + mergedDocuments.sort((a, b) => (a.distance ?? Infinity) - (b.distance ?? Infinity)); + } } @@ -61,9 +80,9 @@ placement="left" tippyOptions={{ duration: [500, 0], animation: 'perspective' }} > -
+
{document?.metadata?.name ?? document.source.name} - {document?.metadata?.page - ? `(${$i18n.t('page')} ${document.metadata.page + 1})` - : ''} + {#if document?.metadata?.page} + + ({$i18n.t('page')} + {document.metadata.page + 1}) + + {/if}
+ {#if document.distance !== undefined} +
+ {$i18n.t('Relevance')} +
+ {@const percentage = calculatePercentage(document.distance)} +
+ + {percentage.toFixed(0)}% + + ({document.distance.toFixed(4)}) +
+ {/if} {:else}
{$i18n.t('No source available')} diff --git a/src/lib/i18n/locales/de-DE/translation.json b/src/lib/i18n/locales/de-DE/translation.json index c1ccdae6d6..7640a17a6a 100644 --- a/src/lib/i18n/locales/de-DE/translation.json +++ b/src/lib/i18n/locales/de-DE/translation.json @@ -443,6 +443,7 @@ "Modelfile Content": "Modelfile-Inhalt", "Models": "Modelle", "More": "Mehr", + "more": "mehr", "Move to Top": "", "Name": "Name", "Name Tag": "Namens-Tag", @@ -784,5 +785,6 @@ "Your account status is currently pending activation.": "Ihr Kontostatus ist derzeit ausstehend und wartet auf Aktivierung.", "Your entire contribution will go directly to the plugin developer; Open WebUI does not take any percentage. However, the chosen funding platform might have its own fees.": "", "Youtube": "YouTube", - "Youtube Loader Settings": "YouTube-Ladeeinstellungen" + "Youtube Loader Settings": "YouTube-Ladeeinstellungen", + "References from": "Referenzen aus" } From b105efa05f05f6bf6cd79ae80cb3b30e473e9a3b Mon Sep 17 00:00:00 2001 From: Jannik Streidl Date: Mon, 7 Oct 2024 21:11:04 +0200 Subject: [PATCH 002/112] enh: append citations with distance scores --- backend/open_webui/apps/retrieval/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/open_webui/apps/retrieval/utils.py b/backend/open_webui/apps/retrieval/utils.py index 0fe206c966..d9e82f3627 100644 --- a/backend/open_webui/apps/retrieval/utils.py +++ b/backend/open_webui/apps/retrieval/utils.py @@ -398,6 +398,7 @@ def get_rag_context( "source": context["file"], "document": context["documents"][0], "metadata": context["metadatas"][0], + "distances": context["distances"][0] } ) except Exception as e: From 9fc813cfa6e1967afc1ee758725b60b6539679b2 Mon Sep 17 00:00:00 2001 From: Jannik Streidl Date: Mon, 7 Oct 2024 21:13:13 +0200 Subject: [PATCH 003/112] fix: only append if distances are available --- backend/open_webui/apps/retrieval/utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/open_webui/apps/retrieval/utils.py b/backend/open_webui/apps/retrieval/utils.py index d9e82f3627..53961be2cb 100644 --- a/backend/open_webui/apps/retrieval/utils.py +++ b/backend/open_webui/apps/retrieval/utils.py @@ -393,14 +393,14 @@ def get_rag_context( ) if "metadatas" in context: - citations.append( - { - "source": context["file"], - "document": context["documents"][0], - "metadata": context["metadatas"][0], - "distances": context["distances"][0] - } - ) + citation = { + "source": context["file"], + "document": context["documents"][0], + "metadata": context["metadatas"][0], + } + if "distances" in context and context["distances"]: + citation["distances"] = context["distances"][0] + citations.append(citation) except Exception as e: log.exception(e) From 209948af6fd4bd2f0b0790fb1a6cbea0c6d5ef0e Mon Sep 17 00:00:00 2001 From: Jannik Streidl Date: Mon, 7 Oct 2024 21:15:10 +0200 Subject: [PATCH 004/112] styling --- src/lib/components/common/Collapsible.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/components/common/Collapsible.svelte b/src/lib/components/common/Collapsible.svelte index 1f6faf48b2..df032b7ef2 100644 --- a/src/lib/components/common/Collapsible.svelte +++ b/src/lib/components/common/Collapsible.svelte @@ -12,8 +12,8 @@
{#if title !== null} - {:else} - {/if} From 89c77f05a8d4f6aef320c37983fe051d41c94c97 Mon Sep 17 00:00:00 2001 From: Jannik Streidl Date: Thu, 10 Oct 2024 16:46:14 +0200 Subject: [PATCH 006/112] chromadb switch to cosine similarity --- backend/open_webui/apps/retrieval/vector/dbs/chroma.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/apps/retrieval/vector/dbs/chroma.py b/backend/open_webui/apps/retrieval/vector/dbs/chroma.py index 84f80b2531..da5382c323 100644 --- a/backend/open_webui/apps/retrieval/vector/dbs/chroma.py +++ b/backend/open_webui/apps/retrieval/vector/dbs/chroma.py @@ -109,7 +109,10 @@ class ChromaClient: def insert(self, collection_name: str, items: list[VectorItem]): # Insert the items into the collection, if the collection does not exist, it will be created. - collection = self.client.get_or_create_collection(name=collection_name) + collection = self.client.get_or_create_collection( + name=collection_name, + metadata={"hnsw:space": "cosine"} + ) ids = [item["id"] for item in items] documents = [item["text"] for item in items] From 741230bcdbf8ea6cc8d673e61cc712efd96e6bb8 Mon Sep 17 00:00:00 2001 From: Jannik Streidl Date: Thu, 10 Oct 2024 17:20:50 +0200 Subject: [PATCH 007/112] fix --- backend/open_webui/apps/retrieval/vector/dbs/chroma.py | 5 ++++- src/lib/components/chat/Messages/Citations.svelte | 9 --------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/backend/open_webui/apps/retrieval/vector/dbs/chroma.py b/backend/open_webui/apps/retrieval/vector/dbs/chroma.py index da5382c323..c6d95bd523 100644 --- a/backend/open_webui/apps/retrieval/vector/dbs/chroma.py +++ b/backend/open_webui/apps/retrieval/vector/dbs/chroma.py @@ -130,7 +130,10 @@ class ChromaClient: def upsert(self, collection_name: str, items: list[VectorItem]): # Update the items in the collection, if the items are not present, insert them. If the collection does not exist, it will be created. - collection = self.client.get_or_create_collection(name=collection_name) + collection = self.client.get_or_create_collection( + name=collection_name, + metadata={"hnsw:space": "cosine"} + ) ids = [item["id"] for item in items] documents = [item["text"] for item in items] diff --git a/src/lib/components/chat/Messages/Citations.svelte b/src/lib/components/chat/Messages/Citations.svelte index 162ad6683e..9ab3b31ae7 100644 --- a/src/lib/components/chat/Messages/Citations.svelte +++ b/src/lib/components/chat/Messages/Citations.svelte @@ -49,15 +49,6 @@ }); return acc; }, []); - - $: if (_citations.every((citation) => citation.distances !== undefined)) { - // Sort citations by distance (relevance) - _citations = _citations.sort((a, b) => { - const aMinDistance = Math.min(...(a.distances ?? [])); - const bMinDistance = Math.min(...(b.distances ?? [])); - return aMinDistance - bMinDistance; - }); - } From 9d4d96429f22ad943115df5c04907ee84e76c468 Mon Sep 17 00:00:00 2001 From: Jannik Streidl Date: Sat, 12 Oct 2024 13:51:27 +0200 Subject: [PATCH 008/112] only show relevance pertentage score if distances are in cosinus silimarity range --- .../chat/Messages/CitationsModal.svelte | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/lib/components/chat/Messages/CitationsModal.svelte b/src/lib/components/chat/Messages/CitationsModal.svelte index d2510d924f..c7a68d6fb8 100644 --- a/src/lib/components/chat/Messages/CitationsModal.svelte +++ b/src/lib/components/chat/Messages/CitationsModal.svelte @@ -11,19 +11,20 @@ let mergedDocuments = []; function calculatePercentage(distance) { - return Math.max(0, Math.min(100, (1 - distance / 2) * 100)); + if (distance < 0) return 100; + if (distance > 1) return 0; + return Math.round((1 - distance) * 100); } - function getRelevanceColor(percentage) { - if (percentage >= 80) - return 'bg-green-200 dark:bg-green-800 text-green-800 dark:text-green-200'; - if (percentage >= 60) - return 'bg-yellow-200 dark:bg-yellow-800 text-yellow-800 dark:text-yellow-200'; - if (percentage >= 40) - return 'bg-orange-200 dark:bg-orange-800 text-orange-800 dark:text-orange-200'; - return 'bg-red-200 dark:bg-red-800 text-red-800 dark:text-red-200'; + function shouldShowPercentage(documents) { + const validDistances = documents.filter( + (d) => d.distance !== undefined && d.distance >= 0 && d.distance <= 1 + ); + return validDistances.length >= 2; } + $: showPercentage = shouldShowPercentage(mergedDocuments); + $: if (citation) { mergedDocuments = citation.document?.map((c, i) => { return { @@ -104,15 +105,23 @@
{$i18n.t('Relevance')}
- {@const percentage = calculatePercentage(document.distance)} -
- - {percentage.toFixed(0)}% - - ({document.distance.toFixed(4)}) -
+ {#if showPercentage} + {@const percentage = calculatePercentage(document.distance)} +
+ + {percentage}% + + ({document.distance.toFixed(4)}) +
+ {:else} +
+ + {document.distance.toFixed(4)} + +
+ {/if} {/if} {:else}
From 0bebc898c84c95d00d9dc5873f56448f544e975a Mon Sep 17 00:00:00 2001 From: Jannik Streidl Date: Sat, 12 Oct 2024 15:18:56 +0200 Subject: [PATCH 009/112] finalizing fixes & additions --- .../components/chat/Messages/Citations.svelte | 149 ++++++++++-------- .../chat/Messages/CitationsModal.svelte | 60 ++++--- src/lib/components/common/Collapsible.svelte | 17 +- 3 files changed, 134 insertions(+), 92 deletions(-) diff --git a/src/lib/components/chat/Messages/Citations.svelte b/src/lib/components/chat/Messages/Citations.svelte index 9ab3b31ae7..a991e248bd 100644 --- a/src/lib/components/chat/Messages/Citations.svelte +++ b/src/lib/components/chat/Messages/Citations.svelte @@ -10,48 +10,61 @@ export let citations = []; let _citations = []; + let showPercentage = false; let showCitationModal = false; - let selectedCitation = null; + let selectedCitation: any = null; let isCollapsibleOpen = false; - $: _citations = citations.reduce((acc, citation) => { - citation.document.forEach((document, index) => { - const metadata = citation.metadata?.[index]; - const distance = citation.distances?.[index]; - const id = metadata?.source ?? 'N/A'; - let source = citation?.source; + function shouldShowPercentage(citations: any[]) { + return citations.every( + (citation) => + citation.distances && + citation.distances.length > 0 && + citation.distances.every((d: number) => d !== undefined && d >= -1 && d <= 1) + ); + } - if (metadata?.name) { - source = { ...source, name: metadata.name }; - } + $: { + _citations = citations.reduce((acc, citation) => { + citation.document.forEach((document, index) => { + const metadata = citation.metadata?.[index]; + const distance = citation.distances?.[index]; + const id = metadata?.source ?? 'N/A'; + let source = citation?.source; - // Check if ID looks like a URL - if (id.startsWith('http://') || id.startsWith('https://')) { - source = { name: id }; - } + if (metadata?.name) { + source = { ...source, name: metadata.name }; + } - const existingSource = acc.find((item) => item.id === id); + if (id.startsWith('http://') || id.startsWith('https://')) { + source = { name: id }; + } - if (existingSource) { - existingSource.document.push(document); - existingSource.metadata.push(metadata); - if (distance !== undefined) existingSource.distances.push(distance); - } else { - acc.push({ - id: id, - source: source, - document: [document], - metadata: metadata ? [metadata] : [], - distances: distance !== undefined ? [distance] : undefined - }); - } - }); - return acc; - }, []); + const existingSource = acc.find((item) => item.id === id); + + if (existingSource) { + existingSource.document.push(document); + existingSource.metadata.push(metadata); + if (distance !== undefined) existingSource.distances.push(distance); + } else { + acc.push({ + id: id, + source: source, + document: [document], + metadata: metadata ? [metadata] : [], + distances: distance !== undefined ? [distance] : undefined + }); + } + }); + return acc; + }, []); + + showPercentage = shouldShowPercentage(_citations); + } - + {#if _citations.length > 0}
@@ -79,42 +92,50 @@ {:else}
- {$i18n.t('References from')} - {#each _citations.slice(0, 2) as citation, idx} -
- + {#if idx === 0} + , + {/if}
- + {/each} +
+
+ + + {_citations.length - 2} + + {$i18n.t('more')}
- {#if idx === 0} - , - {/if} - {/each} - {$i18n.t('and')} -
- {_citations.length - 2}
- {$i18n.t('more')} - {#if isCollapsibleOpen} - - {:else} - - {/if} +
+ {#if isCollapsibleOpen} + + {:else} + + {/if} +
diff --git a/src/lib/components/chat/Messages/CitationsModal.svelte b/src/lib/components/chat/Messages/CitationsModal.svelte index c7a68d6fb8..4f4d6a4206 100644 --- a/src/lib/components/chat/Messages/CitationsModal.svelte +++ b/src/lib/components/chat/Messages/CitationsModal.svelte @@ -7,24 +7,26 @@ export let show = false; export let citation; + export let showPercentage = false; let mergedDocuments = []; - function calculatePercentage(distance) { + function calculatePercentage(distance: number) { if (distance < 0) return 100; if (distance > 1) return 0; - return Math.round((1 - distance) * 100); + return Math.round((1 - distance) * 10000) / 100; } - function shouldShowPercentage(documents) { - const validDistances = documents.filter( - (d) => d.distance !== undefined && d.distance >= 0 && d.distance <= 1 - ); - return validDistances.length >= 2; + function getRelevanceColor(percentage: number) { + if (percentage >= 80) + return 'bg-green-200 dark:bg-green-800 text-green-800 dark:text-green-200'; + if (percentage >= 60) + return 'bg-yellow-200 dark:bg-yellow-800 text-yellow-800 dark:text-yellow-200'; + if (percentage >= 40) + return 'bg-orange-200 dark:bg-orange-800 text-orange-800 dark:text-orange-200'; + return 'bg-red-200 dark:bg-red-800 text-red-800 dark:text-red-200'; } - $: showPercentage = shouldShowPercentage(mergedDocuments); - $: if (citation) { mergedDocuments = citation.document?.map((c, i) => { return { @@ -79,7 +81,7 @@
{$i18n.t('Relevance')}
- {#if showPercentage} - {@const percentage = calculatePercentage(document.distance)} +
- - {percentage}% - - ({document.distance.toFixed(4)}) + {#if showPercentage} + {@const percentage = calculatePercentage(document.distance)} + + {percentage.toFixed(2)}% + + + ({document.distance.toFixed(4)}) + + {:else} + + {document.distance.toFixed(4)} + + {/if}
- {:else} -
- - {document.distance.toFixed(4)} - -
- {/if} + + {:else} +
+ {$i18n.t('No source available')} +
{/if} {:else}
diff --git a/src/lib/components/common/Collapsible.svelte b/src/lib/components/common/Collapsible.svelte index d62e26c373..47f4ae44bf 100644 --- a/src/lib/components/common/Collapsible.svelte +++ b/src/lib/components/common/Collapsible.svelte @@ -9,11 +9,18 @@ export let className = ''; export let title = null; + let contentHeight = 0; + let contentElement: HTMLElement; + function handleClick(event) { if (!event.target.closest('.no-toggle')) { open = !open; } } + + $: if (contentElement) { + contentHeight = open ? contentElement.scrollHeight : 0; + }
@@ -43,9 +50,13 @@ {/if} - {#if open} -
+
+
- {/if} +
From 33c3dbd9fa144e9985b55485087f824cd77616a6 Mon Sep 17 00:00:00 2001 From: Jannik Streidl Date: Mon, 14 Oct 2024 10:37:54 +0200 Subject: [PATCH 010/112] fix --- .../components/chat/Messages/Citations.svelte | 80 +++++++++++++------ .../chat/Messages/CitationsModal.svelte | 10 +-- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/src/lib/components/chat/Messages/Citations.svelte b/src/lib/components/chat/Messages/Citations.svelte index a991e248bd..b9a8198aab 100644 --- a/src/lib/components/chat/Messages/Citations.svelte +++ b/src/lib/components/chat/Messages/Citations.svelte @@ -83,7 +83,7 @@ {idx + 1}
{/if} -
+
{citation.source.name}
@@ -94,37 +94,67 @@
-
+
-
- {#each _citations.slice(0, 2) as citation, idx} -
- + {#if idx === 0}, {/if} -
- {citation.source.name} -
- - {#if idx === 0} - , - {/if} -
- {/each} +
+ {/each} + {:else} + {#each _citations.slice(0, 1) as citation, idx} +
+ +
+ {/each} + {/if}
- {_citations.length - 2} + {_citations.length - + (_citations.length > 1 && + _citations + .slice(0, 2) + .reduce((acc, citation) => acc + citation.source.name.length, 0) <= 50 + ? 2 + : 1)} {$i18n.t('more')}
diff --git a/src/lib/components/chat/Messages/CitationsModal.svelte b/src/lib/components/chat/Messages/CitationsModal.svelte index 4f4d6a4206..6c9c96d9a6 100644 --- a/src/lib/components/chat/Messages/CitationsModal.svelte +++ b/src/lib/components/chat/Messages/CitationsModal.svelte @@ -103,10 +103,10 @@ {/if}
+
+ {$i18n.t('Relevance')} +
{#if document.distance !== undefined} -
- {$i18n.t('Relevance')} -
{:else}
- {$i18n.t('No source available')} + {$i18n.t('No distance available')}
{/if} {:else} @@ -140,7 +140,7 @@ {/if}
-
+
{$i18n.t('Content')} 

From 79c834d0e427f7d1ebe4c622105d3c6c9006fa68 Mon Sep 17 00:00:00 2001
From: Jannik Streidl 
Date: Mon, 14 Oct 2024 11:18:13 +0200
Subject: [PATCH 011/112] fix: do not show relevances it you have mixed
 collections (cosine + l2)

---
 .../components/chat/Messages/Citations.svelte | 36 ++++++++---
 .../chat/Messages/CitationsModal.svelte       | 59 ++++++++++---------
 2 files changed, 60 insertions(+), 35 deletions(-)

diff --git a/src/lib/components/chat/Messages/Citations.svelte b/src/lib/components/chat/Messages/Citations.svelte
index b9a8198aab..0a09b3b55e 100644
--- a/src/lib/components/chat/Messages/Citations.svelte
+++ b/src/lib/components/chat/Messages/Citations.svelte
@@ -11,18 +11,34 @@
 
 	let _citations = [];
 	let showPercentage = false;
+	let showRelevance = true;
 
 	let showCitationModal = false;
 	let selectedCitation: any = null;
 	let isCollapsibleOpen = false;
 
+	function calculateShowRelevance(citations: any[]) {
+		const distances = citations.flatMap((citation) => citation.distances ?? []);
+		const inRange = distances.filter((d) => d !== undefined && d >= -1 && d <= 1).length;
+		const outOfRange = distances.filter((d) => d !== undefined && (d < -1 || d > 1)).length;
+
+		if (distances.length === 0) {
+			return false;
+		}
+
+		if (
+			(inRange === distances.length - 1 && outOfRange === 1) ||
+			(outOfRange === distances.length - 1 && inRange === 1)
+		) {
+			return false;
+		}
+
+		return true;
+	}
+
 	function shouldShowPercentage(citations: any[]) {
-		return citations.every(
-			(citation) =>
-				citation.distances &&
-				citation.distances.length > 0 &&
-				citation.distances.every((d: number) => d !== undefined && d >= -1 && d <= 1)
-		);
+		const distances = citations.flatMap((citation) => citation.distances ?? []);
+		return distances.every((d) => d !== undefined && d >= -1 && d <= 1);
 	}
 
 	$: {
@@ -60,11 +76,17 @@
 			return acc;
 		}, []);
 
+		showRelevance = calculateShowRelevance(_citations);
 		showPercentage = shouldShowPercentage(_citations);
 	}
 
 
-
+
 
 {#if _citations.length > 0}
 	

diff --git a/src/lib/components/chat/Messages/CitationsModal.svelte b/src/lib/components/chat/Messages/CitationsModal.svelte
index 6c9c96d9a6..ad965e3502 100644
--- a/src/lib/components/chat/Messages/CitationsModal.svelte
+++ b/src/lib/components/chat/Messages/CitationsModal.svelte
@@ -8,6 +8,7 @@
 	export let show = false;
 	export let citation;
 	export let showPercentage = false;
+	export let showRelevance = true;
 
 	let mergedDocuments = [];
 
@@ -103,35 +104,37 @@
 									{/if}
 								

 							
-							

-								{$i18n.t('Relevance')}
-							

-							{#if document.distance !== undefined}
-								
-									

-										{#if showPercentage}
-											{@const percentage = calculatePercentage(document.distance)}
-											
-												{percentage.toFixed(2)}%
-											
-											
-												({document.distance.toFixed(4)})
-											
-										{:else}
-											
-												{document.distance.toFixed(4)}
-											
-										{/if}
-									

-								
-							{:else}
-								

-									{$i18n.t('No distance available')}
+							{#if showRelevance}
+								

+									{$i18n.t('Relevance')}
 								

+								{#if document.distance !== undefined}
+									
+										

+											{#if showPercentage}
+												{@const percentage = calculatePercentage(document.distance)}
+												
+													{percentage.toFixed(2)}%
+												
+												
+													({document.distance.toFixed(4)})
+												
+											{:else}
+												
+													{document.distance.toFixed(4)}
+												
+											{/if}
+										

+									
+								{:else}
+									

+										{$i18n.t('No distance available')}
+									

+								{/if}
 							{/if}
 						{:else}
 							


From dedb26fd5c60e089ff285fcaafa8b42f5db63b26 Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" 
Date: Wed, 16 Oct 2024 02:53:16 -0700
Subject: [PATCH 012/112] feat: folders db migration

---
 .../versions/c69f45358db4_add_folder_table.py | 41 +++++++++++++++++++
 src/lib/components/layout/Sidebar.svelte      | 22 ++++++----
 2 files changed, 54 insertions(+), 9 deletions(-)
 create mode 100644 backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py

diff --git a/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py b/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py
new file mode 100644
index 0000000000..016b0d419f
--- /dev/null
+++ b/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py
@@ -0,0 +1,41 @@
+"""Add folder table
+
+Revision ID: c69f45358db4
+Revises: 3ab32c4b8f59
+Create Date: 2024-10-16 02:02:35.241684
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "c69f45358db4"
+down_revision = "3ab32c4b8f59"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "folder",
+        sa.Column("id", sa.Text(), primary_key=True, nullable=False),
+        sa.Column("parent_id", sa.Text(), nullable=True),
+        sa.Column("user_id", sa.Text(), nullable=True),
+        sa.Column("name", sa.Text(), nullable=False),
+        sa.Column("items", sa.JSON(), nullable=True),
+        sa.Column("meta", sa.JSON(), nullable=True),
+        sa.Column(
+            "created_at", sa.DateTime(), server_default=sa.func.now(), nullable=False
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(),
+            nullable=False,
+            server_default=sa.func.now(),
+            onupdate=sa.func.now(),
+        ),
+    )
+
+
+def downgrade():
+    op.drop_table("folder")
diff --git a/src/lib/components/layout/Sidebar.svelte b/src/lib/components/layout/Sidebar.svelte
index 1376539f5e..ec54e0bc10 100644
--- a/src/lib/components/layout/Sidebar.svelte
+++ b/src/lib/components/layout/Sidebar.svelte
@@ -38,15 +38,11 @@
 	import DeleteConfirmDialog from '$lib/components/common/ConfirmDialog.svelte';
 	import Spinner from '../common/Spinner.svelte';
 	import Loader from '../common/Loader.svelte';
-	import FilesOverlay from '../chat/MessageInput/FilesOverlay.svelte';
 	import AddFilesPlaceholder from '../AddFilesPlaceholder.svelte';
-	import { select } from 'd3-selection';
 	import SearchInput from './Sidebar/SearchInput.svelte';
-	import ChevronDown from '../icons/ChevronDown.svelte';
-	import ChevronUp from '../icons/ChevronUp.svelte';
-	import ChevronRight from '../icons/ChevronRight.svelte';
-	import Collapsible from '../common/Collapsible.svelte';
 	import Folder from '../common/Folder.svelte';
+	import Plus from '../icons/Plus.svelte';
+	import Tooltip from '../common/Tooltip.svelte';
 
 	const BREAKPOINT = 768;
 
@@ -381,7 +377,7 @@
 		

 			 {
@@ -425,7 +421,7 @@
 			
 
 			
+				
+			

+
 			
Date: Wed, 16 Oct 2024 03:35:26 -0700
Subject: [PATCH 013/112] refac

---
 .../migrations/versions/c69f45358db4_add_folder_table.py     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py b/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py
index 016b0d419f..9bbec3a59e 100644
--- a/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py
+++ b/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py
@@ -18,9 +18,9 @@ depends_on = None
 def upgrade():
     op.create_table(
         "folder",
-        sa.Column("id", sa.Text(), primary_key=True, nullable=False),
+        sa.Column("id", sa.Text(), nullable=False),
         sa.Column("parent_id", sa.Text(), nullable=True),
-        sa.Column("user_id", sa.Text(), nullable=True),
+        sa.Column("user_id", sa.Text(), nullable=False),
         sa.Column("name", sa.Text(), nullable=False),
         sa.Column("items", sa.JSON(), nullable=True),
         sa.Column("meta", sa.JSON(), nullable=True),
@@ -34,6 +34,7 @@ def upgrade():
             server_default=sa.func.now(),
             onupdate=sa.func.now(),
         ),
+        sa.PrimaryKeyConstraint("id", "user_id")
     )
 
 

From ede71740d293c28fbd350b099cc9dd037bd4f761 Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" 
Date: Wed, 16 Oct 2024 15:13:38 -0700
Subject: [PATCH 014/112] feat: folders table

---
 .../open_webui/apps/webui/models/folders.py   | 132 ++++++++++++++++++
 1 file changed, 132 insertions(+)
 create mode 100644 backend/open_webui/apps/webui/models/folders.py

diff --git a/backend/open_webui/apps/webui/models/folders.py b/backend/open_webui/apps/webui/models/folders.py
new file mode 100644
index 0000000000..e8e6ffef16
--- /dev/null
+++ b/backend/open_webui/apps/webui/models/folders.py
@@ -0,0 +1,132 @@
+import logging
+import time
+import uuid
+from typing import Optional
+
+from open_webui.apps.webui.internal.db import Base, get_db
+
+
+from open_webui.env import SRC_LOG_LEVELS
+from pydantic import BaseModel, ConfigDict
+from sqlalchemy import BigInteger, Column, Text, JSON, PrimaryKeyConstraint
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["MODELS"])
+
+
+####################
+# Folder DB Schema
+####################
+
+
+class FolderItems(BaseModel):
+    chat_ids: Optional[list[str]] = None
+    file_ids: Optional[list[str]] = None
+    folder_ids: Optional[list[str]] = None
+
+    model_config = ConfigDict(extra="allow")
+
+
+class Folder(Base):
+    __tablename__ = "folder"
+    id = Column(Text, primary_key=True)
+    parent_id = Column(Text, nullable=True)
+    user_id = Column(Text)
+    name = Column(Text)
+    items = Column(JSON, nullable=True)
+    meta = Column(JSON, nullable=True)
+    created_at = Column(BigInteger)
+    updated_at = Column(BigInteger)
+
+
+class FolderModel(BaseModel):
+    id: str
+    parent_id: Optional[str] = None
+    user_id: str
+    name: str
+    items: Optional[FolderItems] = None
+    meta: Optional[dict] = None
+    created_at: int
+    updated_at: int
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class FolderTable:
+    def insert_new_folder(self, name: str, user_id: str) -> Optional[FolderModel]:
+        with get_db() as db:
+            id = name.lower()
+            folder = FolderModel(
+                **{
+                    "id": id,
+                    "user_id": user_id,
+                    "name": name,
+                    "created_at": int(time.time()),
+                    "updated_at": int(time.time()),
+                }
+            )
+            try:
+                result = Folder(**folder.model_dump())
+                db.add(result)
+                db.commit()
+                db.refresh(result)
+                if result:
+                    return FolderModel.model_validate(result)
+                else:
+                    return None
+            except Exception as e:
+                print(e)
+                return None
+
+    def get_folder_by_name_and_user_id(
+        self, name: str, user_id: str
+    ) -> Optional[FolderModel]:
+        try:
+            id = name.lower()
+            with get_db() as db:
+                folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
+                return FolderModel.model_validate(folder)
+        except Exception:
+            return None
+
+    def get_folders_by_user_id(self, user_id: str) -> list[FolderModel]:
+        with get_db() as db:
+            return [
+                FolderModel.model_validate(folder)
+                for folder in db.query(Folder).filter_by(user_id=user_id).all()
+            ]
+
+    def update_folder_by_name_and_user_id(
+        self, name: str, user_id: str, items: FolderItems
+    ) -> Optional[FolderModel]:
+        try:
+            id = name.lower()
+            with get_db() as db:
+                folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
+
+                folder.items = items.model_dump()
+                folder.updated_at = int(time.time())
+
+                db.commit()
+
+                return FolderModel.model_validate(folder)
+        except Exception as e:
+            log.error(f"update_folder: {e}")
+            return
+
+    def delete_folder_by_name_and_user_id(self, name: str, user_id: str) -> bool:
+        try:
+            with get_db() as db:
+                id = name.lower()
+
+                folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
+                db.delete(folder)
+
+                db.commit()
+                return True
+        except Exception as e:
+            log.error(f"delete_folder: {e}")
+            return False
+
+
+Folders = FolderTable()

From a942c30ca86c6d4fa6a577abc6b469f89afd6d18 Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" 
Date: Wed, 16 Oct 2024 21:05:03 -0700
Subject: [PATCH 015/112] feat: folder ui

---
 backend/open_webui/apps/webui/main.py         |   2 +
 backend/open_webui/apps/webui/models/chats.py |  25 ++
 .../open_webui/apps/webui/models/folders.py   |  70 +++++-
 .../open_webui/apps/webui/routers/folders.py  | 197 ++++++++++++++++
 .../versions/c69f45358db4_add_folder_table.py |   9 +-
 src/lib/apis/folders/index.ts                 | 198 ++++++++++++++++
 src/lib/components/common/DragGhost.svelte    |   2 +-
 src/lib/components/common/Folder.svelte       |  34 +--
 src/lib/components/icons/Document.svelte      |  19 ++
 src/lib/components/layout/Sidebar.svelte      | 119 ++++++++--
 .../components/layout/Sidebar/ChatItem.svelte |  95 ++++----
 .../components/layout/Sidebar/Folders.svelte  |  14 ++
 .../layout/Sidebar/RecursiveFolder.svelte     | 220 ++++++++++++++++++
 13 files changed, 917 insertions(+), 87 deletions(-)
 create mode 100644 backend/open_webui/apps/webui/routers/folders.py
 create mode 100644 src/lib/apis/folders/index.ts
 create mode 100644 src/lib/components/icons/Document.svelte
 create mode 100644 src/lib/components/layout/Sidebar/Folders.svelte
 create mode 100644 src/lib/components/layout/Sidebar/RecursiveFolder.svelte

diff --git a/backend/open_webui/apps/webui/main.py b/backend/open_webui/apps/webui/main.py
index 1d12d708eb..94e42f4a80 100644
--- a/backend/open_webui/apps/webui/main.py
+++ b/backend/open_webui/apps/webui/main.py
@@ -9,6 +9,7 @@ from open_webui.apps.webui.models.models import Models
 from open_webui.apps.webui.routers import (
     auths,
     chats,
+    folders,
     configs,
     files,
     functions,
@@ -110,6 +111,7 @@ app.include_router(configs.router, prefix="/configs", tags=["configs"])
 app.include_router(auths.router, prefix="/auths", tags=["auths"])
 app.include_router(users.router, prefix="/users", tags=["users"])
 app.include_router(chats.router, prefix="/chats", tags=["chats"])
+app.include_router(folders.router, prefix="/folders", tags=["folders"])
 
 app.include_router(models.router, prefix="/models", tags=["models"])
 app.include_router(knowledge.router, prefix="/knowledge", tags=["knowledge"])
diff --git a/backend/open_webui/apps/webui/models/chats.py b/backend/open_webui/apps/webui/models/chats.py
index 12bdd1c38c..2b0d297958 100644
--- a/backend/open_webui/apps/webui/models/chats.py
+++ b/backend/open_webui/apps/webui/models/chats.py
@@ -33,6 +33,7 @@ class Chat(Base):
     pinned = Column(Boolean, default=False, nullable=True)
 
     meta = Column(JSON, server_default="{}")
+    folder_id = Column(Text, nullable=True)
 
 
 class ChatModel(BaseModel):
@@ -51,6 +52,7 @@ class ChatModel(BaseModel):
     pinned: Optional[bool] = False
 
     meta: dict = {}
+    folder_id: Optional[str] = None
 
 
 ####################
@@ -512,6 +514,29 @@ class ChatTable:
             # Validate and return chats
             return [ChatModel.model_validate(chat) for chat in all_chats]
 
+    def get_chats_by_folder_id_and_user_id(
+        self, folder_id: str, user_id: str
+    ) -> list[ChatModel]:
+        with get_db() as db:
+            all_chats = (
+                db.query(Chat).filter_by(folder_id=folder_id, user_id=user_id).all()
+            )
+            return [ChatModel.model_validate(chat) for chat in all_chats]
+
+    def update_chat_folder_id_by_id_and_user_id(
+        self, id: str, user_id: str, folder_id: str
+    ) -> Optional[ChatModel]:
+        try:
+            with get_db() as db:
+                chat = db.get(Chat, id)
+                chat.folder_id = folder_id
+                chat.updated_at = int(time.time())
+                db.commit()
+                db.refresh(chat)
+                return ChatModel.model_validate(chat)
+        except Exception:
+            return None
+
     def get_chat_tags_by_id_and_user_id(self, id: str, user_id: str) -> list[TagModel]:
         with get_db() as db:
             chat = db.get(Chat, id)
diff --git a/backend/open_webui/apps/webui/models/folders.py b/backend/open_webui/apps/webui/models/folders.py
index e8e6ffef16..93a53a5905 100644
--- a/backend/open_webui/apps/webui/models/folders.py
+++ b/backend/open_webui/apps/webui/models/folders.py
@@ -22,7 +22,6 @@ log.setLevel(SRC_LOG_LEVELS["MODELS"])
 class FolderItems(BaseModel):
     chat_ids: Optional[list[str]] = None
     file_ids: Optional[list[str]] = None
-    folder_ids: Optional[list[str]] = None
 
     model_config = ConfigDict(extra="allow")
 
@@ -52,6 +51,21 @@ class FolderModel(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
 
+####################
+# Forms
+####################
+
+
+class FolderForm(BaseModel):
+    name: str
+    model_config = ConfigDict(extra="allow")
+
+
+class FolderItemsUpdateForm(BaseModel):
+    items: FolderItems
+    model_config = ConfigDict(extra="allow")
+
+
 class FolderTable:
     def insert_new_folder(self, name: str, user_id: str) -> Optional[FolderModel]:
         with get_db() as db:
@@ -96,7 +110,59 @@ class FolderTable:
                 for folder in db.query(Folder).filter_by(user_id=user_id).all()
             ]
 
-    def update_folder_by_name_and_user_id(
+    def get_folders_by_parent_id_and_user_id(self, parent_id: str, user_id: str):
+        with get_db() as db:
+            return [
+                FolderModel.model_validate(folder)
+                for folder in db.query(Folder)
+                .filter_by(parent_id=parent_id, user_id=user_id)
+                .all()
+            ]
+
+    def update_folder_parent_id_by_id_and_user_id(
+        self,
+        id: str,
+        user_id: str,
+        parent_id: str,
+    ) -> Optional[FolderModel]:
+        try:
+            with get_db() as db:
+                folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
+                folder.parent_id = parent_id
+                folder.updated_at = int(time.time())
+
+                db.commit()
+
+                return FolderModel.model_validate(folder)
+        except Exception as e:
+            log.error(f"update_folder: {e}")
+            return
+
+    def update_folder_name_by_name_and_user_id(
+        self, name: str, user_id: str, new_name: str
+    ) -> Optional[FolderModel]:
+        try:
+            id = name.lower()
+            new_id = new_name.lower()
+            with get_db() as db:
+                # Check if new folder name already exists
+                folder = db.query(Folder).filter_by(id=new_id, user_id=user_id).first()
+                if folder:
+                    return None
+
+                folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
+                folder.id = new_id
+                folder.name = new_name
+                folder.updated_at = int(time.time())
+
+                db.commit()
+
+                return FolderModel.model_validate(folder)
+        except Exception as e:
+            log.error(f"update_folder: {e}")
+            return
+
+    def update_folder_items_by_name_and_user_id(
         self, name: str, user_id: str, items: FolderItems
     ) -> Optional[FolderModel]:
         try:
diff --git a/backend/open_webui/apps/webui/routers/folders.py b/backend/open_webui/apps/webui/routers/folders.py
new file mode 100644
index 0000000000..b0c26b5957
--- /dev/null
+++ b/backend/open_webui/apps/webui/routers/folders.py
@@ -0,0 +1,197 @@
+import logging
+import os
+import shutil
+import uuid
+from pathlib import Path
+from typing import Optional
+from pydantic import BaseModel
+import mimetypes
+
+
+from open_webui.apps.webui.models.folders import (
+    FolderForm,
+    FolderItemsUpdateForm,
+    FolderModel,
+    Folders,
+)
+from open_webui.apps.webui.models.chats import Chats
+
+from open_webui.config import UPLOAD_DIR
+from open_webui.env import SRC_LOG_LEVELS
+from open_webui.constants import ERROR_MESSAGES
+
+
+from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status
+from fastapi.responses import FileResponse, StreamingResponse
+
+
+from open_webui.utils.utils import get_admin_user, get_verified_user
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["MODELS"])
+
+
+router = APIRouter()
+
+
+############################
+# Get Folders
+############################
+
+
+@router.get("/", response_model=list[FolderModel])
+async def get_folders(user=Depends(get_verified_user)):
+    folders = Folders.get_folders_by_user_id(user.id)
+    return folders
+
+
+############################
+# Create Folder
+############################
+
+
+@router.post("/")
+def create_folder(form_data: FolderForm, user=Depends(get_verified_user)):
+    folder = Folders.get_folder_by_name_and_user_id(form_data.name, user.id)
+    if folder:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.DEFAULT("Folder already exists"),
+        )
+
+    try:
+        folder = Folders.insert_new_folder(form_data.name, user.id)
+        return folder
+    except Exception as e:
+        log.exception(e)
+        log.error(f"Error creating folder: {form_data.name}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.DEFAULT("Error creating folder"),
+        )
+
+
+############################
+# Get Folders By Id
+############################
+
+
+@router.get("/{id}", response_model=Optional[FolderModel])
+async def get_folder_by_id(id: str, user=Depends(get_verified_user)):
+    folder = Folders.get_folder_by_name_and_user_id(id, user.id)
+    if folder:
+        return folder
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# Update Folder Name By Id
+############################
+
+
+@router.post("/{id}/update")
+async def update_folder_name_by_id(
+    id: str, form_data: FolderForm, user=Depends(get_verified_user)
+):
+    new_id = form_data.name.lower()
+    folder = Folders.get_folder_by_name_and_user_id(new_id, user.id)
+    if folder:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.DEFAULT("Folder already exists"),
+        )
+
+    folder = Folders.get_folder_by_name_and_user_id(id, user.id)
+    if folder:
+        try:
+            folder = Folders.update_folder_name_by_name_and_user_id(
+                id, user.id, form_data.name
+            )
+
+            # Update children folders parent_id
+            children_folders = Folders.get_folders_by_parent_id_and_user_id(id, user.id)
+            for child in children_folders:
+                Folders.update_folder_parent_id_by_id_and_user_id(
+                    child.id, user.id, folder.id
+                )
+
+            # Update children items parent_id
+            chats = Chats.get_chats_by_folder_id_and_user_id(id, user.id)
+            for chat in chats:
+                Chats.update_chat_folder_id_by_id_and_user_id(
+                    chat.id, user.id, folder.id
+                )
+
+            return folder
+        except Exception as e:
+            log.exception(e)
+            log.error(f"Error updating folder: {id}")
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("Error updating folder"),
+            )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# Update Folder Items By Id
+############################
+
+
+@router.post("/{id}/update/items")
+async def update_folder_items_by_id(
+    id: str, form_data: FolderItemsUpdateForm, user=Depends(get_verified_user)
+):
+    folder = Folders.get_folder_by_name_and_user_id(id, user.id)
+    if folder:
+        try:
+            folder = Folders.update_folder_by_name_and_user_id(
+                id, user.id, form_data.items
+            )
+            return folder
+        except Exception as e:
+            log.exception(e)
+            log.error(f"Error updating folder: {id}")
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("Error updating folder"),
+            )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# Delete Folder By Id
+############################
+
+
+@router.delete("/{id}")
+async def delete_folder_by_id(id: str, user=Depends(get_verified_user)):
+    folder = Folders.get_folder_by_name_and_user_id(id, user.id)
+    if folder:
+        try:
+            result = Folders.delete_folder_by_name_and_user_id(id, user.id)
+            return result
+        except Exception as e:
+            log.exception(e)
+            log.error(f"Error deleting folder: {id}")
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("Error deleting folder"),
+            )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
diff --git a/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py b/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py
index 9bbec3a59e..0341cad07d 100644
--- a/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py
+++ b/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py
@@ -34,9 +34,16 @@ def upgrade():
             server_default=sa.func.now(),
             onupdate=sa.func.now(),
         ),
-        sa.PrimaryKeyConstraint("id", "user_id")
+        sa.PrimaryKeyConstraint("id", "user_id"),
+    )
+
+    op.add_column(
+        "chat",
+        sa.Column("folder_id", sa.Text(), nullable=True),
     )
 
 
 def downgrade():
+    op.drop_column("chat", "folder_id")
+
     op.drop_table("folder")
diff --git a/src/lib/apis/folders/index.ts b/src/lib/apis/folders/index.ts
new file mode 100644
index 0000000000..2098a78101
--- /dev/null
+++ b/src/lib/apis/folders/index.ts
@@ -0,0 +1,198 @@
+import { WEBUI_API_BASE_URL } from '$lib/constants';
+
+export const createNewFolder = async (token: string, name: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/folders/`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			name: name
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const getFolders = async (token: string = '') => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/folders/`, {
+		method: 'GET',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const getFolderById = async (token: string, id: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/folders/${id}`, {
+		method: 'GET',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const updateFolderNameById = async (token: string, id: string, name: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/folders/${id}/update`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			name: name
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+type FolderItems = {
+	chat_ids: string[];
+	file_ids: string[];
+	folder_ids: string[];
+};
+
+export const updateFolderItemsById = async (token: string, id: string, items: FolderItems) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/folders/${id}/update/items`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			items: items
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const deleteFolderById = async (token: string, id: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/folders/${id}`, {
+		method: 'DELETE',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
diff --git a/src/lib/components/common/DragGhost.svelte b/src/lib/components/common/DragGhost.svelte
index 6d97a45174..7169d72f06 100644
--- a/src/lib/components/common/DragGhost.svelte
+++ b/src/lib/components/common/DragGhost.svelte
@@ -24,7 +24,7 @@
 	bind:this={popupElement}
 	class="fixed top-0 left-0 w-screen h-[100dvh] z-50 touch-none pointer-events-none"
 >
-	

+	

 		
 	

 

diff --git a/src/lib/components/common/Folder.svelte b/src/lib/components/common/Folder.svelte
index 2002b7e0be..42ca584fc6 100644
--- a/src/lib/components/common/Folder.svelte
+++ b/src/lib/components/common/Folder.svelte
@@ -14,13 +14,15 @@
 	export let name = '';
 	export let collapsible = true;
 
+	export let className = '';
+
 	let folderElement;
 
-	let dragged = false;
+	let draggedOver = false;
 
 	const onDragOver = (e) => {
 		e.preventDefault();
-		dragged = true;
+		draggedOver = true;
 	};
 
 	const onDrop = (e) => {
@@ -29,19 +31,23 @@
 		if (folderElement.contains(e.target)) {
 			console.log('Dropped on the Button');
 
-			// get data from the drag event
-			const dataTransfer = e.dataTransfer.getData('text/plain');
-			const data = JSON.parse(dataTransfer);
-			console.log(data);
-			dispatch('drop', data);
+			try {
+				// get data from the drag event
+				const dataTransfer = e.dataTransfer.getData('text/plain');
+				const data = JSON.parse(dataTransfer);
+				console.log(data);
+				dispatch('drop', data);
+			} catch (error) {
+				console.error(error);
+			}
 
-			dragged = false;
+			draggedOver = false;
 		}
 	};
 
 	const onDragLeave = (e) => {
 		e.preventDefault();
-		dragged = false;
+		draggedOver = false;
 	};
 
 	onMount(() => {
@@ -57,10 +63,10 @@
 	});
 
 
-

-	{#if dragged}
+

+	{#if draggedOver}
 		

 	{/if}
 
@@ -74,7 +80,7 @@
 			}}
 		>
 			
-			

+			

 				
 			

 
-			

+			

 				
 			

 		
diff --git a/src/lib/components/icons/Document.svelte b/src/lib/components/icons/Document.svelte
new file mode 100644
index 0000000000..9ae719725b
--- /dev/null
+++ b/src/lib/components/icons/Document.svelte
@@ -0,0 +1,19 @@
+
+
+
+	
+
diff --git a/src/lib/components/layout/Sidebar.svelte b/src/lib/components/layout/Sidebar.svelte
index ec54e0bc10..79f32851e4 100644
--- a/src/lib/components/layout/Sidebar.svelte
+++ b/src/lib/components/layout/Sidebar.svelte
@@ -43,6 +43,8 @@
 	import Folder from '../common/Folder.svelte';
 	import Plus from '../icons/Plus.svelte';
 	import Tooltip from '../common/Tooltip.svelte';
+	import { createNewFolder, getFolders } from '$lib/apis/folders';
+	import Folders from './Sidebar/Folders.svelte';
 
 	const BREAKPOINT = 768;
 
@@ -65,6 +67,55 @@
 	let chatListLoading = false;
 	let allChatsLoaded = false;
 
+	let folders = {};
+
+	const initFolders = async () => {
+		const folderList = await getFolders(localStorage.token).catch((error) => {
+			toast.error(error);
+			return [];
+		});
+
+		for (const folder of folderList) {
+			folders[folder.id] = { ...(folders[folder.id] ? folders[folder.id] : {}), ...folder };
+
+			if (folders[folder.id].parent_id) {
+				folders[folders[folder.id].parent_id].childrenIds = folders[folders[folder.id].parent_id]
+					.childrenIds
+					? [...folders[folders[folder.id].parent_id].childrenIds, folder.id]
+					: [folder.id];
+
+				folders[folders[folder.id].parent_id].childrenIds.sort((a, b) => {
+					return folders[b].updated_at - folders[a].updated_at;
+				});
+			}
+		}
+	};
+
+	const createFolder = async (name = 'Untitled') => {
+		if (name === '') {
+			toast.error($i18n.t('Folder name cannot be empty.'));
+			return;
+		}
+
+		if (name.toLowerCase() in folders) {
+			// If a folder with the same name already exists, append a number to the name
+			let i = 1;
+			while (name.toLowerCase() + ` ${i}` in folders) {
+				i++;
+			}
+			name = name + ` ${i}`;
+		}
+
+		const res = await createNewFolder(localStorage.token, name).catch((error) => {
+			toast.error(error);
+			return null;
+		});
+
+		if (res) {
+			await initFolders();
+		}
+	};
+
 	const initChatList = async () => {
 		// Reset pagination variables
 		tags.set(await getAllTags(localStorage.token));
@@ -280,6 +331,7 @@
 			localStorage.sidebar = value;
 		});
 
+		await initFolders();
 		await pinnedChats.set(await getPinnedChatList(localStorage.token));
 		await initChatList();
 
@@ -491,7 +543,12 @@
 
 			

 				
-					
 				
@@ -514,33 +571,40 @@
 			{/if}
 
 			{#if !search && $pinnedChats.length > 0}
-				

+				

 					 {
 							localStorage.setItem('showPinnedChat', e.detail);
 							console.log(e.detail);
 						}}
 						on:drop={async (e) => {
-							const { id } = e.detail;
+							const { type, id } = e.detail;
 
-							const status = await getChatPinnedStatusById(localStorage.token, id);
+							if (type === 'chat') {
+								const status = await getChatPinnedStatusById(localStorage.token, id);
 
-							if (!status) {
-								const res = await toggleChatPinnedStatusById(localStorage.token, id);
+								if (!status) {
+									const res = await toggleChatPinnedStatusById(localStorage.token, id);
 
-								if (res) {
-									await pinnedChats.set(await getPinnedChatList(localStorage.token));
-									initChatList();
+									if (res) {
+										await pinnedChats.set(await getPinnedChatList(localStorage.token));
+										initChatList();
+									}
 								}
 							}
 						}}
 						name={$i18n.t('Pinned')}
 					>
-						

+						

 							{#each $pinnedChats as chat, idx}
 								 {
@@ -557,6 +621,10 @@
 											showDeleteConfirm = true;
 										}
 									}}
+									on:change={async () => {
+										await pinnedChats.set(await getPinnedChatList(localStorage.token));
+										initChatList();
+									}}
 									on:tag={(e) => {
 										const { type, name } = e.detail;
 										tagEventHandler(type, name, chat.id);
@@ -568,20 +636,28 @@
 				

 			{/if}
 
+			{#if folders}
+				

+					
+				

+			{/if}
+
 			

 				 {
-						const { id } = e.detail;
+						const { type, id } = e.detail;
 
-						const status = await getChatPinnedStatusById(localStorage.token, id);
+						if (type === 'chat') {
+							const status = await getChatPinnedStatusById(localStorage.token, id);
 
-						if (status) {
-							const res = await toggleChatPinnedStatusById(localStorage.token, id);
+							if (status) {
+								const res = await toggleChatPinnedStatusById(localStorage.token, id);
 
-							if (res) {
-								await pinnedChats.set(await getPinnedChatList(localStorage.token));
-								initChatList();
+								if (res) {
+									await pinnedChats.set(await getPinnedChatList(localStorage.token));
+									initChatList();
+								}
 							}
 						}
 					}}
@@ -619,7 +695,8 @@
 								{/if}
 
 								 {
@@ -636,6 +713,10 @@
 											showDeleteConfirm = true;
 										}
 									}}
+									on:change={async () => {
+										await pinnedChats.set(await getPinnedChatList(localStorage.token));
+										initChatList();
+									}}
 									on:tag={(e) => {
 										const { type, name } = e.detail;
 										tagEventHandler(type, name, chat.id);
diff --git a/src/lib/components/layout/Sidebar/ChatItem.svelte b/src/lib/components/layout/Sidebar/ChatItem.svelte
index 4dbcb93311..03e7f10e1e 100644
--- a/src/lib/components/layout/Sidebar/ChatItem.svelte
+++ b/src/lib/components/layout/Sidebar/ChatItem.svelte
@@ -33,8 +33,15 @@
 	import Tooltip from '$lib/components/common/Tooltip.svelte';
 	import ArchiveBox from '$lib/components/icons/ArchiveBox.svelte';
 	import DragGhost from '$lib/components/common/DragGhost.svelte';
+	import Check from '$lib/components/icons/Check.svelte';
+	import XMark from '$lib/components/icons/XMark.svelte';
+	import Document from '$lib/components/icons/Document.svelte';
+
+	export let className = 'pr-2';
+
+	export let id;
+	export let title;
 
-	export let chat;
 	export let selected = false;
 	export let shiftKey = false;
 
@@ -43,7 +50,7 @@
 	let showShareChatModal = false;
 	let confirmEdit = false;
 
-	let chatTitle = chat.title;
+	let chatTitle = title;
 
 	const editChatTitle = async (id, title) => {
 		if (title === '') {
@@ -93,7 +100,7 @@
 
 	let itemElement;
 
-	let drag = false;
+	let dragged = false;
 	let x = 0;
 	let y = 0;
 
@@ -108,11 +115,12 @@
 		event.dataTransfer.setData(
 			'text/plain',
 			JSON.stringify({
-				id: chat.id
+				type: 'chat',
+				id: id
 			})
 		);
 
-		drag = true;
+		dragged = true;
 		itemElement.style.opacity = '0.5'; // Optional: Visual cue to show it's being dragged
 	};
 
@@ -123,7 +131,7 @@
 
 	const onDragEnd = (event) => {
 		itemElement.style.opacity = '1'; // Reset visual cue after drag
-		drag = false;
+		dragged = false;
 	};
 
 	onMount(() => {
@@ -146,24 +154,26 @@
 	});
 
 
-
+
 
-{#if drag && x && y}
+{#if dragged && x && y}
 	
-		

-			

+		

+			

+				
 				

-					{chat.title}
+					{title}
 				

 			

 		

 	
 {/if}
 
-

+

 	{#if confirmEdit}
 		

 	{:else}
 		 {
 				dispatch('select');
 
@@ -191,7 +202,7 @@
 				}
 			}}
 			on:dblclick={() => {
-				chatTitle = chat.title;
+				chatTitle = title;
 				confirmEdit = true;
 			}}
 			on:mouseenter={(e) => {
@@ -205,7 +216,7 @@
 		>
 			

 				

-					{chat.title}
+					{title}
 				

 			

 		
@@ -214,12 +225,14 @@
 	
 	
 {
@@ -230,28 +243,19 @@
 		}}
 	>
 		{#if confirmEdit}
-			

+			

 				
 					
 				
 
@@ -263,16 +267,7 @@
 							chatTitle = '';
 						}}
 					>
-						
-							
-						
+						
 					
 				
 			

@@ -282,7 +277,7 @@
 					
 				
 
-				{#if chat.id === $chatId}
+				{#if id === $chatId}
 					
 					
+		

+
+		

+			{#if folders[folderId].childrenIds || folders[folderId].items?.chat_ids}
+				

+					{#if folders[folderId]?.childrenIds}
+						{#each folders[folderId]?.childrenIds as folderId (folderId)}
+							
+						{/each}
+					{/if}
+
+					{#if folders[folderId].items?.chat_ids}
+						{#each folder.items.chat_ids as chatId (chatId)}
+							{chatId}
+						{/each}
+					{/if}
+				

+			{/if}
+		

+	
+


From dea12360f424dbbd9a011e16476db91eac49999d Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" 
Date: Wed, 16 Oct 2024 21:49:22 -0700
Subject: [PATCH 016/112] refac: folder id -> uuid

---
 .../open_webui/apps/webui/models/folders.py   | 63 +++++++-----
 .../open_webui/apps/webui/routers/folders.py  | 95 +++++++++++++------
 src/lib/components/layout/Sidebar.svelte      | 17 ++--
 .../components/layout/Sidebar/ChatItem.svelte |  4 +-
 .../components/layout/Sidebar/Folders.svelte  |  7 +-
 .../layout/Sidebar/Folders/FolderMenu.svelte  |  0
 .../layout/Sidebar/RecursiveFolder.svelte     | 27 +++++-
 7 files changed, 144 insertions(+), 69 deletions(-)
 create mode 100644 src/lib/components/layout/Sidebar/Folders/FolderMenu.svelte

diff --git a/backend/open_webui/apps/webui/models/folders.py b/backend/open_webui/apps/webui/models/folders.py
index 93a53a5905..6e28e37979 100644
--- a/backend/open_webui/apps/webui/models/folders.py
+++ b/backend/open_webui/apps/webui/models/folders.py
@@ -67,14 +67,17 @@ class FolderItemsUpdateForm(BaseModel):
 
 
 class FolderTable:
-    def insert_new_folder(self, name: str, user_id: str) -> Optional[FolderModel]:
+    def insert_new_folder(
+        self, user_id: str, name: str, parent_id: Optional[str] = None
+    ) -> Optional[FolderModel]:
         with get_db() as db:
-            id = name.lower()
+            id = str(uuid.uuid4())
             folder = FolderModel(
                 **{
                     "id": id,
                     "user_id": user_id,
                     "name": name,
+                    "parent_id": parent_id,
                     "created_at": int(time.time()),
                     "updated_at": int(time.time()),
                 }
@@ -92,11 +95,10 @@ class FolderTable:
                 print(e)
                 return None
 
-    def get_folder_by_name_and_user_id(
-        self, name: str, user_id: str
+    def get_folder_by_id_and_user_id(
+        self, id: str, user_id: str
     ) -> Optional[FolderModel]:
         try:
-            id = name.lower()
             with get_db() as db:
                 folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
                 return FolderModel.model_validate(folder)
@@ -110,7 +112,24 @@ class FolderTable:
                 for folder in db.query(Folder).filter_by(user_id=user_id).all()
             ]
 
-    def get_folders_by_parent_id_and_user_id(self, parent_id: str, user_id: str):
+    def get_folder_by_parent_id_and_user_id_and_name(
+        self, parent_id: Optional[str], user_id: str, name: str
+    ) -> Optional[FolderModel]:
+        try:
+            with get_db() as db:
+                folder = (
+                    db.query(Folder)
+                    .filter_by(parent_id=parent_id, user_id=user_id, name=name)
+                    .first()
+                )
+                return FolderModel.model_validate(folder)
+        except Exception as e:
+            log.error(f"get_folder_by_name_and_user_id: {e}")
+            return None
+
+    def get_folders_by_parent_id_and_user_id(
+        self, parent_id: Optional[str], user_id: str
+    ) -> list[FolderModel]:
         with get_db() as db:
             return [
                 FolderModel.model_validate(folder)
@@ -138,21 +157,23 @@ class FolderTable:
             log.error(f"update_folder: {e}")
             return
 
-    def update_folder_name_by_name_and_user_id(
-        self, name: str, user_id: str, new_name: str
+    def update_folder_name_by_id_and_user_id(
+        self, id: str, user_id: str, name: str
     ) -> Optional[FolderModel]:
         try:
-            id = name.lower()
-            new_id = new_name.lower()
             with get_db() as db:
-                # Check if new folder name already exists
-                folder = db.query(Folder).filter_by(id=new_id, user_id=user_id).first()
-                if folder:
+                folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
+
+                existing_folder = (
+                    db.query(Folder)
+                    .filter_by(name=name, parent_id=folder.parent_id, user_id=user_id)
+                    .first()
+                )
+
+                if existing_folder:
                     return None
 
-                folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
-                folder.id = new_id
-                folder.name = new_name
+                folder.name = name
                 folder.updated_at = int(time.time())
 
                 db.commit()
@@ -162,11 +183,10 @@ class FolderTable:
             log.error(f"update_folder: {e}")
             return
 
-    def update_folder_items_by_name_and_user_id(
-        self, name: str, user_id: str, items: FolderItems
+    def update_folder_items_by_id_and_user_id(
+        self, id: str, user_id: str, items: FolderItems
     ) -> Optional[FolderModel]:
         try:
-            id = name.lower()
             with get_db() as db:
                 folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
 
@@ -180,14 +200,11 @@ class FolderTable:
             log.error(f"update_folder: {e}")
             return
 
-    def delete_folder_by_name_and_user_id(self, name: str, user_id: str) -> bool:
+    def delete_folder_by_id_and_user_id(self, id: str, user_id: str) -> bool:
         try:
             with get_db() as db:
-                id = name.lower()
-
                 folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
                 db.delete(folder)
-
                 db.commit()
                 return True
         except Exception as e:
diff --git a/backend/open_webui/apps/webui/routers/folders.py b/backend/open_webui/apps/webui/routers/folders.py
index b0c26b5957..df38b2190d 100644
--- a/backend/open_webui/apps/webui/routers/folders.py
+++ b/backend/open_webui/apps/webui/routers/folders.py
@@ -52,7 +52,10 @@ async def get_folders(user=Depends(get_verified_user)):
 
 @router.post("/")
 def create_folder(form_data: FolderForm, user=Depends(get_verified_user)):
-    folder = Folders.get_folder_by_name_and_user_id(form_data.name, user.id)
+    folder = Folders.get_folder_by_parent_id_and_user_id_and_name(
+        None, user.id, form_data.name
+    )
+
     if folder:
         raise HTTPException(
             status_code=status.HTTP_400_BAD_REQUEST,
@@ -60,11 +63,11 @@ def create_folder(form_data: FolderForm, user=Depends(get_verified_user)):
         )
 
     try:
-        folder = Folders.insert_new_folder(form_data.name, user.id)
+        folder = Folders.insert_new_folder(user.id, form_data.name)
         return folder
     except Exception as e:
         log.exception(e)
-        log.error(f"Error creating folder: {form_data.name}")
+        log.error("Error creating folder")
         raise HTTPException(
             status_code=status.HTTP_400_BAD_REQUEST,
             detail=ERROR_MESSAGES.DEFAULT("Error creating folder"),
@@ -78,7 +81,7 @@ def create_folder(form_data: FolderForm, user=Depends(get_verified_user)):
 
 @router.get("/{id}", response_model=Optional[FolderModel])
 async def get_folder_by_id(id: str, user=Depends(get_verified_user)):
-    folder = Folders.get_folder_by_name_and_user_id(id, user.id)
+    folder = Folders.get_folder_by_id_and_user_id(id, user.id)
     if folder:
         return folder
     else:
@@ -97,35 +100,65 @@ async def get_folder_by_id(id: str, user=Depends(get_verified_user)):
 async def update_folder_name_by_id(
     id: str, form_data: FolderForm, user=Depends(get_verified_user)
 ):
-    new_id = form_data.name.lower()
-    folder = Folders.get_folder_by_name_and_user_id(new_id, user.id)
+    folder = Folders.get_folder_by_id_and_user_id(id, user.id)
     if folder:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail=ERROR_MESSAGES.DEFAULT("Folder already exists"),
+        existing_folder = Folders.get_folder_by_parent_id_and_user_id_and_name(
+            folder.parent_id, user.id, form_data.name
         )
-
-    folder = Folders.get_folder_by_name_and_user_id(id, user.id)
-    if folder:
-        try:
-            folder = Folders.update_folder_name_by_name_and_user_id(
-                id, user.id, form_data.name
+        if existing_folder:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("Folder already exists"),
             )
 
-            # Update children folders parent_id
-            children_folders = Folders.get_folders_by_parent_id_and_user_id(id, user.id)
-            for child in children_folders:
-                Folders.update_folder_parent_id_by_id_and_user_id(
-                    child.id, user.id, folder.id
-                )
+        try:
+            folder = Folders.update_folder_name_by_id_and_user_id(
+                id, user.id, form_data.name
+            )
+            return folder
+        except Exception as e:
+            log.exception(e)
+            log.error(f"Error updating folder: {id}")
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("Error updating folder"),
+            )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
 
-            # Update children items parent_id
-            chats = Chats.get_chats_by_folder_id_and_user_id(id, user.id)
-            for chat in chats:
-                Chats.update_chat_folder_id_by_id_and_user_id(
-                    chat.id, user.id, folder.id
-                )
 
+############################
+# Update Folder Name By Id
+############################
+
+
+class FolderParentIdForm(BaseModel):
+    parent_id: str
+
+
+@router.post("/{id}/update/parent")
+async def update_folder_parent_id_by_id(
+    id: str, form_data: FolderParentIdForm, user=Depends(get_verified_user)
+):
+    folder = Folders.get_folder_by_id_and_user_id(id, user.id)
+    if folder:
+        existing_folder = Folders.get_folder_by_parent_id_and_user_id_and_name(
+            form_data.parent_id, user.id, folder.name
+        )
+
+        if existing_folder:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("Folder already exists"),
+            )
+
+        try:
+            folder = Folders.update_folder_parent_id_by_id_and_user_id(
+                id, user.id, form_data.parent_id
+            )
             return folder
         except Exception as e:
             log.exception(e)
@@ -150,10 +183,10 @@ async def update_folder_name_by_id(
 async def update_folder_items_by_id(
     id: str, form_data: FolderItemsUpdateForm, user=Depends(get_verified_user)
 ):
-    folder = Folders.get_folder_by_name_and_user_id(id, user.id)
+    folder = Folders.get_folder_by_id_and_user_id(id, user.id)
     if folder:
         try:
-            folder = Folders.update_folder_by_name_and_user_id(
+            folder = Folders.update_folder_items_by_id_and_user_id(
                 id, user.id, form_data.items
             )
             return folder
@@ -178,10 +211,10 @@ async def update_folder_items_by_id(
 
 @router.delete("/{id}")
 async def delete_folder_by_id(id: str, user=Depends(get_verified_user)):
-    folder = Folders.get_folder_by_name_and_user_id(id, user.id)
+    folder = Folders.get_folder_by_id_and_user_id(id, user.id)
     if folder:
         try:
-            result = Folders.delete_folder_by_name_and_user_id(id, user.id)
+            result = Folders.delete_folder_by_id_and_user_id(id, user.id)
             return result
         except Exception as e:
             log.exception(e)
diff --git a/src/lib/components/layout/Sidebar.svelte b/src/lib/components/layout/Sidebar.svelte
index 79f32851e4..5043d39d96 100644
--- a/src/lib/components/layout/Sidebar.svelte
+++ b/src/lib/components/layout/Sidebar.svelte
@@ -75,6 +75,8 @@
 			return [];
 		});
 
+		folders = {};
+
 		for (const folder of folderList) {
 			folders[folder.id] = { ...(folders[folder.id] ? folders[folder.id] : {}), ...folder };
 
@@ -636,15 +638,15 @@
 				

 			{/if}
 
-			{#if folders}
-				

+			

+				{#if !search && folders}
 					
-				

-			{/if}
+				{/if}
 
-			

 				 {
 						const { type, id } = e.detail;
 
@@ -662,7 +664,7 @@
 						}
 					}}
 				>
-					

+					

 						{#if $chats}
 							{#each $chats as chat, idx}
 								{#if idx === 0 || (idx > 0 && chat.time_range !== $chats[idx - 1].time_range)}
@@ -695,6 +697,7 @@
 								{/if}
 
 								
-		

+		

 			

-				
+				
 				

 					{title}
 				

diff --git a/src/lib/components/layout/Sidebar/Folders.svelte b/src/lib/components/layout/Sidebar/Folders.svelte
index 102c256d44..e675c609e2 100644
--- a/src/lib/components/layout/Sidebar/Folders.svelte
+++ b/src/lib/components/layout/Sidebar/Folders.svelte
@@ -6,7 +6,12 @@
 	// Get the list of folders that have no parent, sorted by name alphabetically
 	$: folderList = Object.keys(folders)
 		.filter((key) => folders[key].parent_id === null)
-		.sort((a, b) => folders[a].name.localeCompare(folders[b].name));
+		.sort((a, b) =>
+			folders[a].name.localeCompare(folders[b].name, undefined, {
+				numeric: true,
+				sensitivity: 'base'
+			})
+		);
 
 
 {#each folderList as folderId (folderId)}
diff --git a/src/lib/components/layout/Sidebar/Folders/FolderMenu.svelte b/src/lib/components/layout/Sidebar/Folders/FolderMenu.svelte
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/src/lib/components/layout/Sidebar/RecursiveFolder.svelte b/src/lib/components/layout/Sidebar/RecursiveFolder.svelte
index 6c653769ac..68f4117a2c 100644
--- a/src/lib/components/layout/Sidebar/RecursiveFolder.svelte
+++ b/src/lib/components/layout/Sidebar/RecursiveFolder.svelte
@@ -10,13 +10,14 @@
 	import DragGhost from '$lib/components/common/DragGhost.svelte';
 
 	import FolderOpen from '$lib/components/icons/FolderOpen.svelte';
+	import EllipsisHorizontal from '$lib/components/icons/EllipsisHorizontal.svelte';
 
 	export let open = true;
 
 	export let folders;
 	export let folderId;
 
-	export let className;
+	export let className = '';
 
 	let folderElement;
 
@@ -67,6 +68,7 @@
 	let y;
 
 	const onDragStart = (event) => {
+		event.stopPropagation();
 		event.dataTransfer.setDragImage(dragImage, 0, 0);
 
 		// Set the data to be transferred
@@ -83,11 +85,15 @@
 	};
 
 	const onDrag = (event) => {
+		event.stopPropagation();
+
 		x = event.clientX;
 		y = event.clientY;
 	};
 
 	const onDragEnd = (event) => {
+		event.stopPropagation();
+
 		folderElement.style.opacity = '1'; // Reset visual cue after drag
 		dragged = false;
 	};
@@ -122,7 +128,7 @@
 
 {#if dragged && x && y}
 	
-		

+		

 			

 				
 				

@@ -149,7 +155,7 @@
 		}}
 	>
 		
-		

+		

 			

+
+				
 			
 		

 
@@ -203,8 +220,8 @@
 					class="ml-3 pl-1 mt-[1px] flex flex-col overflow-y-auto scrollbar-hidden border-s dark:border-gray-850"
 				>
 					{#if folders[folderId]?.childrenIds}
-						{#each folders[folderId]?.childrenIds as folderId (folderId)}
-							
+						{#each folders[folderId]?.childrenIds as childId (`${folderId}-${childId}`)}
+							
 						{/each}
 					{/if}
 

From 9df9f4a990a60ebf06462117ca7cfb7f5527291e Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" 
Date: Wed, 16 Oct 2024 21:55:25 -0700
Subject: [PATCH 017/112] refac

---
 backend/open_webui/constants.py          |  4 +++-
 src/lib/components/layout/Sidebar.svelte | 11 ++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/backend/open_webui/constants.py b/backend/open_webui/constants.py
index 704cdd0745..4e2ef008bc 100644
--- a/backend/open_webui/constants.py
+++ b/backend/open_webui/constants.py
@@ -20,7 +20,9 @@ class ERROR_MESSAGES(str, Enum):
     def __str__(self) -> str:
         return super().__str__()
 
-    DEFAULT = lambda err="": f"Something went wrong :/\n[ERROR: {err if err else ''}]"
+    DEFAULT = (
+        lambda err="": f'{"Something went wrong :/" if err == "" else "[ERROR: " + err + "]"}'
+    )
     ENV_VAR_NOT_FOUND = "Required environment variable not found. Terminating now."
     CREATE_USER_ERROR = "Oops! Something went wrong while creating your account. Please try again later. If the issue persists, contact support for assistance."
     DELETE_USER_ERROR = "Oops! Something went wrong. We encountered an issue while trying to delete the user. Please give it another shot."
diff --git a/src/lib/components/layout/Sidebar.svelte b/src/lib/components/layout/Sidebar.svelte
index 5043d39d96..f92da0c544 100644
--- a/src/lib/components/layout/Sidebar.svelte
+++ b/src/lib/components/layout/Sidebar.svelte
@@ -99,13 +99,18 @@
 			return;
 		}
 
-		if (name.toLowerCase() in folders) {
+		if (Object.values(folders).find((folder) => folder.name.toLowerCase() === name.toLowerCase())) {
 			// If a folder with the same name already exists, append a number to the name
 			let i = 1;
-			while (name.toLowerCase() + ` ${i}` in folders) {
+			while (
+				Object.values(folders).find(
+					(folder) => folder.name.toLowerCase() === `${name} ${i}`.toLowerCase()
+				)
+			) {
 				i++;
 			}
-			name = name + ` ${i}`;
+
+			name = `${name} ${i}`;
 		}
 
 		const res = await createNewFolder(localStorage.token, name).catch((error) => {

From 7b97d7a718c4562e82343d6028e70e3218d97c15 Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" 
Date: Wed, 16 Oct 2024 22:04:21 -0700
Subject: [PATCH 018/112] enh: disable drag event listener if parent dragged

---
 backend/open_webui/apps/webui/models/chats.py |  2 +-
 src/lib/components/layout/Sidebar.svelte      | 23 ++++++++++++++-----
 .../layout/Sidebar/RecursiveFolder.svelte     | 14 ++++++++++-
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/backend/open_webui/apps/webui/models/chats.py b/backend/open_webui/apps/webui/models/chats.py
index 2b0d297958..89e3b025f4 100644
--- a/backend/open_webui/apps/webui/models/chats.py
+++ b/backend/open_webui/apps/webui/models/chats.py
@@ -256,7 +256,7 @@ class ChatTable:
         limit: int = 50,
     ) -> list[ChatModel]:
         with get_db() as db:
-            query = db.query(Chat).filter_by(user_id=user_id)
+            query = db.query(Chat).filter_by(user_id=user_id).filter_by(parent_id=None)
             if not include_archived:
                 query = query.filter_by(archived=False)
 
diff --git a/src/lib/components/layout/Sidebar.svelte b/src/lib/components/layout/Sidebar.svelte
index f92da0c544..7df48ccbaf 100644
--- a/src/lib/components/layout/Sidebar.svelte
+++ b/src/lib/components/layout/Sidebar.svelte
@@ -77,16 +77,27 @@
 
 		folders = {};
 
+		// First pass: Initialize all folder entries
 		for (const folder of folderList) {
-			folders[folder.id] = { ...(folders[folder.id] ? folders[folder.id] : {}), ...folder };
+			// Ensure folder is added to folders with its data
+			folders[folder.id] = { ...(folders[folder.id] || {}), ...folder };
+		}
 
-			if (folders[folder.id].parent_id) {
-				folders[folders[folder.id].parent_id].childrenIds = folders[folders[folder.id].parent_id]
-					.childrenIds
-					? [...folders[folders[folder.id].parent_id].childrenIds, folder.id]
+		// Second pass: Tie child folders to their parents
+		for (const folder of folderList) {
+			if (folder.parent_id) {
+				// Ensure the parent folder is initialized if it doesn't exist
+				if (!folders[folder.parent_id]) {
+					folders[folder.parent_id] = {}; // Create a placeholder if not already present
+				}
+
+				// Initialize childrenIds array if it doesn't exist and add the current folder id
+				folders[folder.parent_id].childrenIds = folders[folder.parent_id].childrenIds
+					? [...folders[folder.parent_id].childrenIds, folder.id]
 					: [folder.id];
 
-				folders[folders[folder.id].parent_id].childrenIds.sort((a, b) => {
+				// Sort the children by updated_at field
+				folders[folder.parent_id].childrenIds.sort((a, b) => {
 					return folders[b].updated_at - folders[a].updated_at;
 				});
 			}
diff --git a/src/lib/components/layout/Sidebar/RecursiveFolder.svelte b/src/lib/components/layout/Sidebar/RecursiveFolder.svelte
index 68f4117a2c..7e4354b7d4 100644
--- a/src/lib/components/layout/Sidebar/RecursiveFolder.svelte
+++ b/src/lib/components/layout/Sidebar/RecursiveFolder.svelte
@@ -19,6 +19,8 @@
 
 	export let className = '';
 
+	export let parentDragged = false;
+
 	let folderElement;
 
 	let edit = false;
@@ -31,12 +33,18 @@
 	const onDragOver = (e) => {
 		e.preventDefault();
 		e.stopPropagation();
+		if (dragged || parentDragged) {
+			return;
+		}
 		draggedOver = true;
 	};
 
 	const onDrop = (e) => {
 		e.preventDefault();
 		e.stopPropagation();
+		if (dragged || parentDragged) {
+			return;
+		}
 
 		if (folderElement.contains(e.target)) {
 			console.log('Dropped on the Button');
@@ -57,6 +65,10 @@
 
 	const onDragLeave = (e) => {
 		e.preventDefault();
+		if (dragged || parentDragged) {
+			return;
+		}
+
 		draggedOver = false;
 	};
 
@@ -221,7 +233,7 @@
 				>
 					{#if folders[folderId]?.childrenIds}
 						{#each folders[folderId]?.childrenIds as childId (`${folderId}-${childId}`)}
-							
+							
 						{/each}
 					{/if}
 

From 29c39d44e181828862514264d4519cfc0b3df0de Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" 
Date: Wed, 16 Oct 2024 22:36:44 -0700
Subject: [PATCH 019/112] fix: collapsible space toggle issue

---
 .../open_webui/apps/webui/models/folders.py   | 22 ++++++++-
 .../open_webui/apps/webui/routers/folders.py  | 13 +++++-
 src/lib/components/common/Collapsible.svelte  | 12 +++--
 .../layout/Sidebar/RecursiveFolder.svelte     | 46 +++++++++++++++----
 4 files changed, 76 insertions(+), 17 deletions(-)

diff --git a/backend/open_webui/apps/webui/models/folders.py b/backend/open_webui/apps/webui/models/folders.py
index 6e28e37979..d1b6a0621b 100644
--- a/backend/open_webui/apps/webui/models/folders.py
+++ b/backend/open_webui/apps/webui/models/folders.py
@@ -101,6 +101,10 @@ class FolderTable:
         try:
             with get_db() as db:
                 folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
+
+                if not folder:
+                    return None
+
                 return FolderModel.model_validate(folder)
         except Exception:
             return None
@@ -119,12 +123,16 @@ class FolderTable:
             with get_db() as db:
                 folder = (
                     db.query(Folder)
-                    .filter_by(parent_id=parent_id, user_id=user_id, name=name)
+                    .filter_by(parent_id=parent_id, user_id=user_id, name=name.lower())
                     .first()
                 )
+
+                if not folder:
+                    return None
+
                 return FolderModel.model_validate(folder)
         except Exception as e:
-            log.error(f"get_folder_by_name_and_user_id: {e}")
+            log.error(f"get_folder_by_parent_id_and_user_id_and_name: {e}")
             return None
 
     def get_folders_by_parent_id_and_user_id(
@@ -147,6 +155,10 @@ class FolderTable:
         try:
             with get_db() as db:
                 folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
+
+                if not folder:
+                    return None
+
                 folder.parent_id = parent_id
                 folder.updated_at = int(time.time())
 
@@ -164,6 +176,9 @@ class FolderTable:
             with get_db() as db:
                 folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
 
+                if not folder:
+                    return None
+
                 existing_folder = (
                     db.query(Folder)
                     .filter_by(name=name, parent_id=folder.parent_id, user_id=user_id)
@@ -190,6 +205,9 @@ class FolderTable:
             with get_db() as db:
                 folder = db.query(Folder).filter_by(id=id, user_id=user_id).first()
 
+                if not folder:
+                    return None
+
                 folder.items = items.model_dump()
                 folder.updated_at = int(time.time())
 
diff --git a/backend/open_webui/apps/webui/routers/folders.py b/backend/open_webui/apps/webui/routers/folders.py
index df38b2190d..ffc6fdee36 100644
--- a/backend/open_webui/apps/webui/routers/folders.py
+++ b/backend/open_webui/apps/webui/routers/folders.py
@@ -215,7 +215,18 @@ async def delete_folder_by_id(id: str, user=Depends(get_verified_user)):
     if folder:
         try:
             result = Folders.delete_folder_by_id_and_user_id(id, user.id)
-            return result
+            if result:
+                # Delete all chats in the folder
+                chats = Chats.get_chats_by_folder_id_and_user_id(id, user.id)
+                for chat in chats:
+                    Chats.delete_chat_by_id(chat.id, user.id)
+
+                return result
+            else:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=ERROR_MESSAGES.DEFAULT("Error deleting folder"),
+                )
         except Exception as e:
             log.exception(e)
             log.error(f"Error deleting folder: {id}")
diff --git a/src/lib/components/common/Collapsible.svelte b/src/lib/components/common/Collapsible.svelte
index 6a9d1ff19a..72baec4cfe 100644
--- a/src/lib/components/common/Collapsible.svelte
+++ b/src/lib/components/common/Collapsible.svelte
@@ -18,7 +18,9 @@
 
 

 	{#if title !== null}
-		
+		

 	{:else}
-		
+		

 	{/if}
 
 	{#if open}
diff --git a/src/lib/components/layout/Sidebar/RecursiveFolder.svelte b/src/lib/components/layout/Sidebar/RecursiveFolder.svelte
index 7e4354b7d4..b097d51b43 100644
--- a/src/lib/components/layout/Sidebar/RecursiveFolder.svelte
+++ b/src/lib/components/layout/Sidebar/RecursiveFolder.svelte
@@ -11,6 +11,8 @@
 
 	import FolderOpen from '$lib/components/icons/FolderOpen.svelte';
 	import EllipsisHorizontal from '$lib/components/icons/EllipsisHorizontal.svelte';
+	import { updateFolderNameById } from '$lib/apis/folders';
+	import { toast } from 'svelte-sonner';
 
 	export let open = true;
 
@@ -136,6 +138,29 @@
 			folderElement.removeEventListener('dragend', onDragEnd);
 		}
 	});
+
+	const nameUpdateHandler = async () => {
+		name = name.trim();
+
+		if (name === '') {
+			toast.error("Folder name can't be empty");
+			return;
+		}
+
+		if (name === folders[folderId].name) {
+			edit = false;
+			return;
+		}
+
+		const res = await updateFolderNameById(localStorage.token, folderId, name).catch((error) => {
+			toast.error(error);
+			return null;
+		});
+
+		if (res) {
+			folders[folderId].name = name;
+		}
+	};
 
 
 {#if dragged && x && y}
@@ -194,17 +219,18 @@
 						 {
-								folders[folderId].name = e.target.value;
-							}}
+							bind:value={name}
 							on:blur={() => {
 								edit = false;
+								nameUpdateHandler();
 							}}
-							on:keydown={(e) => {
-								if (e.key === 'Enter') {
-									edit = false;
-								}
+							on:click={(e) => {
+								// Prevent accidental collapse toggling when clicking inside input
+								e.stopPropagation();
+							}}
+							on:mousedown={(e) => {
+								// Prevent accidental collapse toggling when clicking inside input
+								e.stopPropagation();
 							}}
 							class="w-full h-full bg-transparent text-gray-500 dark:text-gray-500 outline-none"
 						/>
@@ -213,14 +239,14 @@
 					{/if}
 				

 
-