open-webui/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte
Timothy Jaeryang Baek 4ecacda28c refac
2025-12-10 00:55:31 -05:00

1004 lines
26 KiB
Svelte

<script lang="ts">
import Fuse from 'fuse.js';
import { toast } from 'svelte-sonner';
import { v4 as uuidv4 } from 'uuid';
import { PaneGroup, Pane, PaneResizer } from 'paneforge';
import { onMount, getContext, onDestroy, tick } from 'svelte';
const i18n = getContext('i18n');
import { goto } from '$app/navigation';
import { page } from '$app/stores';
import {
mobile,
showSidebar,
knowledge as _knowledge,
config,
user,
settings
} from '$lib/stores';
import {
updateFileDataContentById,
uploadFile,
deleteFileById,
getFileById
} from '$lib/apis/files';
import {
addFileToKnowledgeById,
getKnowledgeById,
getKnowledgeBases,
removeFileFromKnowledgeById,
resetKnowledgeById,
updateFileFromKnowledgeById,
updateKnowledgeById,
searchKnowledgeFilesById
} from '$lib/apis/knowledge';
import { blobToFile } from '$lib/utils';
import Spinner from '$lib/components/common/Spinner.svelte';
import Files from './KnowledgeBase/Files.svelte';
import AddFilesPlaceholder from '$lib/components/AddFilesPlaceholder.svelte';
import AddContentMenu from './KnowledgeBase/AddContentMenu.svelte';
import AddTextContentModal from './KnowledgeBase/AddTextContentModal.svelte';
import SyncConfirmDialog from '../../common/ConfirmDialog.svelte';
import Drawer from '$lib/components/common/Drawer.svelte';
import ChevronLeft from '$lib/components/icons/ChevronLeft.svelte';
import LockClosed from '$lib/components/icons/LockClosed.svelte';
import AccessControlModal from '../common/AccessControlModal.svelte';
import Search from '$lib/components/icons/Search.svelte';
import FilesOverlay from '$lib/components/chat/MessageInput/FilesOverlay.svelte';
import DropdownOptions from '$lib/components/common/DropdownOptions.svelte';
import Pagination from '$lib/components/common/Pagination.svelte';
let largeScreen = true;
let pane;
let showSidepanel = true;
let showAddTextContentModal = false;
let showSyncConfirmModal = false;
let showAccessControlModal = false;
let minSize = 0;
type Knowledge = {
id: string;
name: string;
description: string;
data: {
file_ids: string[];
};
files: any[];
};
let id = null;
let knowledge: Knowledge | null = null;
let knowledgeId = null;
let selectedFileId = null;
let selectedFile = null;
let selectedFileContent = '';
let inputFiles = null;
let query = '';
let viewOption = null;
let sortKey = null;
let direction = null;
let currentPage = 1;
let fileItems = null;
let fileItemsTotal = null;
const reset = () => {
currentPage = 1;
};
const init = async () => {
reset();
await getItemsPage();
};
$: if (
knowledgeId !== null &&
query !== undefined &&
viewOption !== undefined &&
sortKey !== undefined &&
direction !== undefined &&
currentPage !== undefined
) {
getItemsPage();
}
$: if (
query !== undefined &&
viewOption !== undefined &&
sortKey !== undefined &&
direction !== undefined
) {
reset();
}
const getItemsPage = async () => {
if (knowledgeId === null) return;
fileItems = null;
fileItemsTotal = null;
if (sortKey === null) {
direction = null;
}
const res = await searchKnowledgeFilesById(
localStorage.token,
knowledge.id,
query,
viewOption,
sortKey,
direction,
currentPage
).catch(() => {
return null;
});
if (res) {
fileItems = res.items;
fileItemsTotal = res.total;
}
return res;
};
const fileSelectHandler = async (file) => {
try {
selectedFile = file;
selectedFileContent = selectedFile?.data?.content || '';
} catch (e) {
toast.error($i18n.t('Failed to load file content.'));
}
};
const createFileFromText = (name, content) => {
const blob = new Blob([content], { type: 'text/plain' });
const file = blobToFile(blob, `${name}.txt`);
console.log(file);
return file;
};
const uploadFileHandler = async (file) => {
console.log(file);
const tempItemId = uuidv4();
const fileItem = {
type: 'file',
file: '',
id: null,
url: '',
name: file.name,
size: file.size,
status: 'uploading',
error: '',
itemId: tempItemId
};
if (fileItem.size == 0) {
toast.error($i18n.t('You cannot upload an empty file.'));
return null;
}
if (
($config?.file?.max_size ?? null) !== null &&
file.size > ($config?.file?.max_size ?? 0) * 1024 * 1024
) {
console.log('File exceeds max size limit:', {
fileSize: file.size,
maxSize: ($config?.file?.max_size ?? 0) * 1024 * 1024
});
toast.error(
$i18n.t(`File size should not exceed {{maxSize}} MB.`, {
maxSize: $config?.file?.max_size
})
);
return;
}
fileItems = [...(fileItems ?? []), fileItem];
try {
// If the file is an audio file, provide the language for STT.
let metadata = null;
if (
(file.type.startsWith('audio/') || file.type.startsWith('video/')) &&
$settings?.audio?.stt?.language
) {
metadata = {
language: $settings?.audio?.stt?.language
};
}
const uploadedFile = await uploadFile(localStorage.token, file, metadata).catch((e) => {
toast.error(`${e}`);
return null;
});
if (uploadedFile) {
console.log(uploadedFile);
fileItems = fileItems.map((item) => {
if (item.itemId === tempItemId) {
item.id = uploadedFile.id;
}
// Remove temporary item id
delete item.itemId;
return item;
});
if (uploadedFile.error) {
console.warn('File upload warning:', uploadedFile.error);
toast.warning(uploadedFile.error);
fileItems = fileItems.filter((file) => file.id !== uploadedFile.id);
} else {
await addFileHandler(uploadedFile.id);
}
} else {
toast.error($i18n.t('Failed to upload file.'));
}
} catch (e) {
toast.error(`${e}`);
}
};
const uploadDirectoryHandler = async () => {
// Check if File System Access API is supported
const isFileSystemAccessSupported = 'showDirectoryPicker' in window;
try {
if (isFileSystemAccessSupported) {
// Modern browsers (Chrome, Edge) implementation
await handleModernBrowserUpload();
} else {
// Firefox fallback
await handleFirefoxUpload();
}
} catch (error) {
handleUploadError(error);
}
};
// Helper function to check if a path contains hidden folders
const hasHiddenFolder = (path) => {
return path.split('/').some((part) => part.startsWith('.'));
};
// Modern browsers implementation using File System Access API
const handleModernBrowserUpload = async () => {
const dirHandle = await window.showDirectoryPicker();
let totalFiles = 0;
let uploadedFiles = 0;
// Function to update the UI with the progress
const updateProgress = () => {
const percentage = (uploadedFiles / totalFiles) * 100;
toast.info(
$i18n.t('Upload Progress: {{uploadedFiles}}/{{totalFiles}} ({{percentage}}%)', {
uploadedFiles: uploadedFiles,
totalFiles: totalFiles,
percentage: percentage.toFixed(2)
})
);
};
// Recursive function to count all files excluding hidden ones
async function countFiles(dirHandle) {
for await (const entry of dirHandle.values()) {
// Skip hidden files and directories
if (entry.name.startsWith('.')) continue;
if (entry.kind === 'file') {
totalFiles++;
} else if (entry.kind === 'directory') {
// Only process non-hidden directories
if (!entry.name.startsWith('.')) {
await countFiles(entry);
}
}
}
}
// Recursive function to process directories excluding hidden files and folders
async function processDirectory(dirHandle, path = '') {
for await (const entry of dirHandle.values()) {
// Skip hidden files and directories
if (entry.name.startsWith('.')) continue;
const entryPath = path ? `${path}/${entry.name}` : entry.name;
// Skip if the path contains any hidden folders
if (hasHiddenFolder(entryPath)) continue;
if (entry.kind === 'file') {
const file = await entry.getFile();
const fileWithPath = new File([file], entryPath, { type: file.type });
await uploadFileHandler(fileWithPath);
uploadedFiles++;
updateProgress();
} else if (entry.kind === 'directory') {
// Only process non-hidden directories
if (!entry.name.startsWith('.')) {
await processDirectory(entry, entryPath);
}
}
}
}
await countFiles(dirHandle);
updateProgress();
if (totalFiles > 0) {
await processDirectory(dirHandle);
} else {
console.log('No files to upload.');
}
};
// Firefox fallback implementation using traditional file input
const handleFirefoxUpload = async () => {
return new Promise((resolve, reject) => {
// Create hidden file input
const input = document.createElement('input');
input.type = 'file';
input.webkitdirectory = true;
input.directory = true;
input.multiple = true;
input.style.display = 'none';
// Add input to DOM temporarily
document.body.appendChild(input);
input.onchange = async () => {
try {
const files = Array.from(input.files)
// Filter out files from hidden folders
.filter((file) => !hasHiddenFolder(file.webkitRelativePath));
let totalFiles = files.length;
let uploadedFiles = 0;
// Function to update the UI with the progress
const updateProgress = () => {
const percentage = (uploadedFiles / totalFiles) * 100;
toast.info(
$i18n.t('Upload Progress: {{uploadedFiles}}/{{totalFiles}} ({{percentage}}%)', {
uploadedFiles: uploadedFiles,
totalFiles: totalFiles,
percentage: percentage.toFixed(2)
})
);
};
updateProgress();
// Process all files
for (const file of files) {
// Skip hidden files (additional check)
if (!file.name.startsWith('.')) {
const relativePath = file.webkitRelativePath || file.name;
const fileWithPath = new File([file], relativePath, { type: file.type });
await uploadFileHandler(fileWithPath);
uploadedFiles++;
updateProgress();
}
}
// Clean up
document.body.removeChild(input);
resolve();
} catch (error) {
reject(error);
}
};
input.onerror = (error) => {
document.body.removeChild(input);
reject(error);
};
// Trigger file picker
input.click();
});
};
// Error handler
const handleUploadError = (error) => {
if (error.name === 'AbortError') {
toast.info($i18n.t('Directory selection was cancelled'));
} else {
toast.error($i18n.t('Error accessing directory'));
console.error('Directory access error:', error);
}
};
// Helper function to maintain file paths within zip
const syncDirectoryHandler = async () => {
if ((knowledge?.files ?? []).length > 0) {
const res = await resetKnowledgeById(localStorage.token, id).catch((e) => {
toast.error(`${e}`);
});
if (res) {
knowledge = res;
toast.success($i18n.t('Knowledge reset successfully.'));
// Upload directory
uploadDirectoryHandler();
}
} else {
uploadDirectoryHandler();
}
};
const addFileHandler = async (fileId) => {
const updatedKnowledge = await addFileToKnowledgeById(localStorage.token, id, fileId).catch(
(e) => {
toast.error(`${e}`);
return null;
}
);
if (updatedKnowledge) {
knowledge = updatedKnowledge;
toast.success($i18n.t('File added successfully.'));
} else {
toast.error($i18n.t('Failed to add file.'));
fileItems = fileItems.filter((file) => file.id !== fileId);
}
};
const deleteFileHandler = async (fileId) => {
try {
console.log('Starting file deletion process for:', fileId);
// Remove from knowledge base only
const updatedKnowledge = await removeFileFromKnowledgeById(localStorage.token, id, fileId);
console.log('Knowledge base updated:', updatedKnowledge);
if (updatedKnowledge) {
knowledge = updatedKnowledge;
toast.success($i18n.t('File removed successfully.'));
}
} catch (e) {
console.error('Error in deleteFileHandler:', e);
toast.error(`${e}`);
}
};
let debounceTimeout = null;
let mediaQuery;
let dragged = false;
let isSaving = false;
const updateFileContentHandler = async () => {
if (isSaving) {
console.log('Save operation already in progress, skipping...');
return;
}
isSaving = true;
try {
const res = await updateFileDataContentById(
localStorage.token,
selectedFile.id,
selectedFileContent
).catch((e) => {
toast.error(`${e}`);
return null;
});
if (res) {
toast.success($i18n.t('File content updated successfully.'));
selectedFileId = null;
selectedFile = null;
selectedFileContent = '';
await init();
}
} finally {
isSaving = false;
}
};
const changeDebounceHandler = () => {
console.log('debounce');
if (debounceTimeout) {
clearTimeout(debounceTimeout);
}
debounceTimeout = setTimeout(async () => {
if (knowledge.name.trim() === '' || knowledge.description.trim() === '') {
toast.error($i18n.t('Please fill in all fields.'));
return;
}
const res = await updateKnowledgeById(localStorage.token, id, {
...knowledge,
name: knowledge.name,
description: knowledge.description,
access_control: knowledge.access_control
}).catch((e) => {
toast.error(`${e}`);
});
if (res) {
toast.success($i18n.t('Knowledge updated successfully'));
_knowledge.set(await getKnowledgeBases(localStorage.token));
}
}, 1000);
};
const handleMediaQuery = async (e) => {
if (e.matches) {
largeScreen = true;
} else {
largeScreen = false;
}
};
const onDragOver = (e) => {
e.preventDefault();
// Check if a file is being draggedOver.
if (e.dataTransfer?.types?.includes('Files')) {
dragged = true;
} else {
dragged = false;
}
};
const onDragLeave = () => {
dragged = false;
};
const onDrop = async (e) => {
e.preventDefault();
dragged = false;
const handleUploadingFileFolder = (items) => {
for (const item of items) {
if (item.isFile) {
item.file((file) => {
uploadFileHandler(file);
});
continue;
}
// Not sure why you have to call webkitGetAsEntry and isDirectory seperate, but it won't work if you try item.webkitGetAsEntry().isDirectory
const wkentry = item.webkitGetAsEntry();
const isDirectory = wkentry.isDirectory;
if (isDirectory) {
// Read the directory
wkentry.createReader().readEntries(
(entries) => {
handleUploadingFileFolder(entries);
},
(error) => {
console.error('Error reading directory entries:', error);
}
);
} else {
toast.info($i18n.t('Uploading file...'));
uploadFileHandler(item.getAsFile());
toast.success($i18n.t('File uploaded!'));
}
}
};
if (e.dataTransfer?.types?.includes('Files')) {
if (e.dataTransfer?.files) {
const inputItems = e.dataTransfer?.items;
if (inputItems && inputItems.length > 0) {
handleUploadingFileFolder(inputItems);
} else {
toast.error($i18n.t(`File not found.`));
}
}
}
};
onMount(async () => {
// listen to resize 1024px
mediaQuery = window.matchMedia('(min-width: 1024px)');
mediaQuery.addEventListener('change', handleMediaQuery);
handleMediaQuery(mediaQuery);
// Select the container element you want to observe
const container = document.getElementById('collection-container');
// initialize the minSize based on the container width
minSize = !largeScreen ? 100 : Math.floor((300 / container.clientWidth) * 100);
// Create a new ResizeObserver instance
const resizeObserver = new ResizeObserver((entries) => {
for (let entry of entries) {
const width = entry.contentRect.width;
// calculate the percentage of 300
const percentage = (300 / width) * 100;
// set the minSize to the percentage, must be an integer
minSize = !largeScreen ? 100 : Math.floor(percentage);
if (showSidepanel) {
if (pane && pane.isExpanded() && pane.getSize() < minSize) {
pane.resize(minSize);
}
}
}
});
// Start observing the container's size changes
resizeObserver.observe(container);
if (pane) {
pane.expand();
}
id = $page.params.id;
const res = await getKnowledgeById(localStorage.token, id).catch((e) => {
toast.error(`${e}`);
return null;
});
if (res) {
knowledge = res;
knowledgeId = knowledge?.id;
} else {
goto('/workspace/knowledge');
}
const dropZone = document.querySelector('body');
dropZone?.addEventListener('dragover', onDragOver);
dropZone?.addEventListener('drop', onDrop);
dropZone?.addEventListener('dragleave', onDragLeave);
});
onDestroy(() => {
mediaQuery?.removeEventListener('change', handleMediaQuery);
const dropZone = document.querySelector('body');
dropZone?.removeEventListener('dragover', onDragOver);
dropZone?.removeEventListener('drop', onDrop);
dropZone?.removeEventListener('dragleave', onDragLeave);
});
const decodeString = (str: string) => {
try {
return decodeURIComponent(str);
} catch (e) {
return str;
}
};
</script>
<FilesOverlay show={dragged} />
<SyncConfirmDialog
bind:show={showSyncConfirmModal}
message={$i18n.t(
'This will reset the knowledge base and sync all files. Do you wish to continue?'
)}
on:confirm={() => {
syncDirectoryHandler();
}}
/>
<AddTextContentModal
bind:show={showAddTextContentModal}
on:submit={(e) => {
const file = createFileFromText(e.detail.name, e.detail.content);
uploadFileHandler(file);
}}
/>
<input
id="files-input"
bind:files={inputFiles}
type="file"
multiple
hidden
on:change={async () => {
if (inputFiles && inputFiles.length > 0) {
for (const file of inputFiles) {
await uploadFileHandler(file);
}
inputFiles = null;
const fileInputElement = document.getElementById('files-input');
if (fileInputElement) {
fileInputElement.value = '';
}
} else {
toast.error($i18n.t(`File not found.`));
}
}}
/>
<div class="flex flex-col w-full h-full min-h-full" id="collection-container">
{#if id && knowledge}
<AccessControlModal
bind:show={showAccessControlModal}
bind:accessControl={knowledge.access_control}
share={$user?.permissions?.sharing?.knowledge || $user?.role === 'admin'}
sharePublic={$user?.permissions?.sharing?.public_knowledge || $user?.role === 'admin'}
onChange={() => {
changeDebounceHandler();
}}
accessRoles={['read', 'write']}
/>
<div class="w-full px-2">
<div class=" flex w-full">
<div class="flex-1">
<div class="flex items-center justify-between w-full">
<div class="w-full flex justify-between items-center">
<input
type="text"
class="text-left w-full font-medium text-lg font-primary bg-transparent outline-hidden flex-1"
bind:value={knowledge.name}
placeholder={$i18n.t('Knowledge Name')}
on:input={() => {
changeDebounceHandler();
}}
/>
<div class="shrink-0 mr-2.5">
{#if (knowledge?.files ?? []).length}
<div class="text-xs text-gray-500">
{$i18n.t('{{count}} files', {
count: (knowledge?.files ?? []).length
})}
</div>
{/if}
</div>
</div>
<div class="self-center shrink-0">
<button
class="bg-gray-50 hover:bg-gray-100 text-black dark:bg-gray-850 dark:hover:bg-gray-800 dark:text-white transition px-2 py-1 rounded-full flex gap-1 items-center"
type="button"
on:click={() => {
showAccessControlModal = true;
}}
>
<LockClosed strokeWidth="2.5" className="size-3.5" />
<div class="text-sm font-medium shrink-0">
{$i18n.t('Access')}
</div>
</button>
</div>
</div>
<div class="flex w-full">
<input
type="text"
class="text-left text-xs w-full text-gray-500 bg-transparent outline-hidden"
bind:value={knowledge.description}
placeholder={$i18n.t('Knowledge Description')}
on:input={() => {
changeDebounceHandler();
}}
/>
</div>
</div>
</div>
</div>
<div
class="mt-2 mb-2.5 py-2 -mx-0 bg-white dark:bg-gray-900 rounded-3xl border border-gray-100/30 dark:border-gray-850/30 flex-1"
>
<div class="px-3.5 flex flex-1 items-center w-full space-x-2 py-0.5 pb-2">
<div class="flex flex-1 items-center">
<div class=" self-center ml-1 mr-3">
<Search className="size-3.5" />
</div>
<input
class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-hidden bg-transparent"
bind:value={query}
placeholder={`${$i18n.t('Search Collection')}`}
on:focus={() => {
selectedFileId = null;
}}
/>
<div>
<AddContentMenu
on:upload={(e) => {
if (e.detail.type === 'directory') {
uploadDirectoryHandler();
} else if (e.detail.type === 'text') {
showAddTextContentModal = true;
} else {
document.getElementById('files-input').click();
}
}}
on:sync={(e) => {
showSyncConfirmModal = true;
}}
/>
</div>
</div>
</div>
<div class="px-3 flex justify-between">
<div
class="flex w-full bg-transparent overflow-x-auto scrollbar-none"
on:wheel={(e) => {
if (e.deltaY !== 0) {
e.preventDefault();
e.currentTarget.scrollLeft += e.deltaY;
}
}}
>
<div
class="flex gap-3 w-fit text-center text-sm rounded-full bg-transparent px-0.5 whitespace-nowrap"
>
<DropdownOptions
align="start"
className="flex w-full items-center gap-2 truncate px-3 py-1.5 text-sm bg-gray-50 dark:bg-gray-850 rounded-xl placeholder-gray-400 outline-hidden focus:outline-hidden"
bind:value={viewOption}
items={[
{ value: null, label: $i18n.t('All') },
{ value: 'created', label: $i18n.t('Created by you') },
{ value: 'shared', label: $i18n.t('Shared with you') }
]}
onChange={(value) => {
if (value) {
localStorage.workspaceViewOption = value;
} else {
delete localStorage.workspaceViewOption;
}
}}
/>
<DropdownOptions
align="start"
bind:value={sortKey}
placeholder={$i18n.t('Sort')}
items={[
{ value: 'name', label: $i18n.t('Name') },
{ value: 'created_at', label: $i18n.t('Created At') },
{ value: 'updated_at', label: $i18n.t('Updated At') }
]}
/>
{#if sortKey}
<DropdownOptions
align="start"
bind:value={direction}
items={[
{ value: 'asc', label: $i18n.t('Asc') },
{ value: null, label: $i18n.t('Desc') }
]}
/>
{/if}
</div>
</div>
</div>
{#if fileItems !== null && fileItemsTotal !== null}
<div class="flex flex-row flex-1 gap-3 px-2.5 mt-2">
<div class="flex-1 flex">
<div class=" flex flex-col w-full space-x-2 rounded-lg h-full">
<div class="w-full h-full flex flex-col min-h-full">
{#if fileItems.length > 0}
<div class=" flex overflow-y-auto h-full w-full scrollbar-hidden text-xs">
<Files
files={fileItems}
{selectedFileId}
onClick={(fileId) => {
selectedFileId = fileId;
if (fileItems) {
const file = fileItems.find((file) => file.id === selectedFileId);
if (file) {
fileSelectHandler(file);
} else {
selectedFile = null;
}
}
}}
onDelete={(fileId) => {
selectedFileId = null;
selectedFile = null;
deleteFileHandler(fileId);
}}
/>
</div>
{#if fileItemsTotal > 30}
<Pagination bind:page={currentPage} count={fileItemsTotal} perPage={30} />
{/if}
{:else}
<div class="my-3 flex flex-col justify-center text-center text-gray-500 text-xs">
<div>
{$i18n.t('No content found')}
</div>
</div>
{/if}
</div>
</div>
</div>
{#if selectedFileId !== null}
<Drawer
className="h-full"
show={selectedFileId !== null}
onClose={() => {
selectedFileId = null;
selectedFile = null;
}}
>
<div class="flex flex-col justify-start h-full max-h-full">
<div class=" flex flex-col w-full h-full max-h-full">
<div class="shrink-0 flex items-center p-2">
<div class="mr-2">
<button
class="w-full text-left text-sm p-1.5 rounded-lg dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-gray-850"
on:click={() => {
selectedFileId = null;
selectedFile = null;
}}
>
<ChevronLeft strokeWidth="2.5" />
</button>
</div>
<div class=" flex-1 text-lg line-clamp-1">
{selectedFile?.meta?.name}
</div>
<div>
<button
class="flex self-center w-fit text-sm py-1 px-2.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed"
disabled={isSaving}
on:click={() => {
updateFileContentHandler();
}}
>
{$i18n.t('Save')}
{#if isSaving}
<div class="ml-2 self-center">
<Spinner />
</div>
{/if}
</button>
</div>
</div>
{#key selectedFile.id}
<textarea
class="w-full h-full text-sm outline-none resize-none px-3 py-2"
bind:value={selectedFileContent}
placeholder={$i18n.t('Add content here')}
/>
{/key}
</div>
</div>
</Drawer>
{/if}
</div>
{:else}
<div class="my-10">
<Spinner className="size-4" />
</div>
{/if}
</div>
{:else}
<Spinner className="size-5" />
{/if}
</div>