Merge 6e2333ccf7 into 1ea555a5ac

src/lib/components/chat/Chat.svelte

@@ -2538,7 +2538,13 @@
 										const { type, data } = e.detail;
 
 										if (type === 'web') {
-											await uploadWeb(data);
+											if (Array.isArray(data)) {
+												for (const url of data) {
+													await uploadWeb(url);
+												}
+											} else {
+												await uploadWeb(data);
+											}
 										} else if (type === 'youtube') {
 											await uploadYoutubeTranscription(data);
 										} else if (type === 'google-drive') {
@@ -2590,7 +2596,13 @@
 										const { type, data } = e.detail;
 
 										if (type === 'web') {
-											await uploadWeb(data);
+											if (Array.isArray(data)) {
+												for (const url of data) {
+													await uploadWeb(url);
+												}
+											} else {
+												await uploadWeb(data);
+											}
 										} else if (type === 'youtube') {
 											await uploadYoutubeTranscription(data);
 										}

src/lib/components/chat/MessageInput/AttachWebpageModal.svelte

@@ -1,85 +1,77 @@
 <script lang="ts">
 	import { toast } from 'svelte-sonner';
-
+	import { onMount, getContext } from 'svelte';
-	import { getContext } from 'svelte';
-	const i18n = getContext('i18n');
-	import { settings } from '$lib/stores';
-
 	import Modal from '$lib/components/common/Modal.svelte';
-	import XMark from '$lib/components/icons/XMark.svelte';
+	import { isValidHttpUrl } from '$lib/utils';
-	import { isValidHttpUrl, isYoutubeUrl } from '$lib/utils';
+
+	const i18n = getContext('i18n');
 
 	export let show = false;
-	export let onSubmit: (e) => void;
+	export let onSubmit: Function = () => {};
 
 	let url = '';
 
 	const submitHandler = () => {
-		if (isValidHttpUrl(url)) {
+		const urls = url
-			onSubmit({
+			.split('\n')
-				type: isYoutubeUrl(url) ? 'youtube' : 'web',
+			.map((u) => u.trim())
-				data: url
+			.filter((u) => u !== '');
-			});
 
-			show = false;
+		const validUrls = urls.filter((u) => isValidHttpUrl(u));
-			url = '';
+
-		} else {
+		if (validUrls.length === 0) {
-			toast.error($i18n.t('Please enter a valid URL.'));
+			toast.error($i18n.t('Please enter at least one valid URL.'));
+			return;
 		}
+
+		onSubmit({ type: 'web', data: validUrls });
+		show = false;
+		url = '';
 	};
 </script>
 
-<Modal bind:show size="sm">
+<Modal size="sm" bind:show>
-	<div class="flex flex-col h-full">
+	<div class="px-4 py-3">
-		<div class="flex justify-between items-center dark:text-gray-100 px-5 pt-4 pb-1.5">
+		<div class="flex justify-between items-center mb-3">
-			<h1 class="text-lg font-medium self-center font-primary">
+			<div class="text-lg font-bold">{$i18n.t('Attach Webpage')}</div>
-				{$i18n.t('Attach Webpage')}
-			</h1>
 			<button
 				class="self-center"
-				aria-label={$i18n.t('Close modal')}
 				on:click={() => {
 					show = false;
 				}}
 			>
-				<XMark className="size-5" />
+				<svg
+					xmlns="http://www.w3.org/2000/svg"
+					viewBox="0 0 20 20"
+					fill="currentColor"
+					class="w-5 h-5"
+				>
+					<path
+						d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
+					/>
+				</svg>
 			</button>
 		</div>
 
-		<div class="px-5 pb-4">
+		<div class="flex flex-col w-full">
-			<form
+			<div class="mb-2 text-xs text-gray-500">
-				on:submit={(e) => {
+				{$i18n.t('Enter one URL per line.')}
-					e.preventDefault();
+			</div>
-					submitHandler();
-				}}
-			>
-				<div class="flex justify-between mb-0.5">
-					<label
-						for="webpage-url"
-						class={`text-xs ${($settings?.highContrastMode ?? false) ? 'text-gray-800 dark:text-gray-100' : 'text-gray-500'}`}
-						>{$i18n.t('Webpage URL')}</label
-					>
-				</div>
 
-				<input
+			<textarea
-					id="webpage-url"
+				class="w-full h-32 p-2 rounded-lg bg-gray-50 dark:bg-gray-850 border border-gray-200 dark:border-gray-800 outline-hidden resize-none text-sm"
-					class={`w-full flex-1 text-sm bg-transparent ${($settings?.highContrastMode ?? false) ? 'placeholder:text-gray-700 dark:placeholder:text-gray-100' : 'outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700'}`}
+				placeholder={$i18n.t('Enter URLs here...')}
-					type="text"
+				bind:value={url}
-					bind:value={url}
+			/>
-					placeholder={'https://example.com'}
-					autocomplete="off"
-					required
-				/>
 
-				<div class="flex justify-end gap-2 pt-3 bg-gray-50 dark:bg-gray-900/50">
+			<div class="flex justify-end mt-3">
-					<button
+				<button
-						class="px-3.5 py-1.5 text-sm font-medium bg-black hover:bg-gray-800 text-white dark:bg-white dark:text-black dark:hover:bg-gray-200 transition rounded-full"
+					class="px-4 py-2 bg-black text-white dark:bg-white dark:text-black rounded-full font-medium text-sm transition hover:opacity-80"
-						type="submit"
+					on:click={submitHandler}
-					>
+				>
-						{$i18n.t('Add')}
+					{$i18n.t('Add')}
-					</button>
+				</button>
-				</div>
+			</div>
-			</form>
 		</div>
 	</div>
 </Modal>

src/lib/components/workspace/Knowledge/KnowledgeBase.svelte

@@ -33,6 +33,7 @@
 		updateFileFromKnowledgeById,
 		updateKnowledgeById
 	} from '$lib/apis/knowledge';
+	import { processWeb } from '$lib/apis/retrieval';
 	import { blobToFile } from '$lib/utils';
 
 	import Spinner from '$lib/components/common/Spinner.svelte';
@@ -209,6 +210,73 @@
 		}
 	};
 
+	const uploadWebBatchHandler = async (urls) => {
+		for (const url of urls) {
+			const tempItemId = uuidv4();
+			const fileItem = {
+				type: 'file',
+				file: '',
+				id: null,
+				url: url,
+				name: url,
+				collection_name: '',
+				status: 'uploading',
+				size: 0,
+				error: '',
+				itemId: tempItemId
+			};
+
+			knowledge.files = [...(knowledge.files ?? []), fileItem];
+
+			try {
+				const res = await processWeb(localStorage.token, '', url).catch((e) => {
+					toast.error(`${e}`);
+					return null;
+				});
+
+				if (res) {
+					console.log(res);
+
+					const content = res.file.data.content;
+					const filename = res.filename || url;
+
+					// Create a safe filename for the File object to avoid filesystem issues
+					const safeFilename = `web_content_${uuidv4()}.txt`;
+
+					const file = new File([content], safeFilename, { type: 'text/plain' });
+					const uploadedFile = await uploadFile(localStorage.token, file, { name: filename });
+
+					if (uploadedFile) {
+						knowledge.files = knowledge.files.map((item) => {
+							if (item.itemId === tempItemId) {
+								item.id = uploadedFile.id;
+								item.name = filename; // Use the URL/filename as the title
+								item.size = uploadedFile.size;
+								item.status = 'uploaded';
+							}
+							return item;
+						});
+
+						await addFileHandler(uploadedFile.id);
+					} else {
+						knowledge.files = knowledge.files.map((item) => {
+							if (item.itemId === tempItemId) {
+								item.status = 'failed';
+								item.error = 'Failed to upload file';
+							}
+							return item;
+						});
+					}
+				} else {
+					knowledge.files = knowledge.files.filter((item) => item.itemId !== tempItemId);
+				}
+			} catch (e) {
+				toast.error(`${e}`);
+				knowledge.files = knowledge.files.filter((item) => item.itemId !== tempItemId);
+			}
+		}
+	};
+
 	const uploadDirectoryHandler = async () => {
 		// Check if File System Access API is supported
 		const isFileSystemAccessSupported = 'showDirectoryPicker' in window;
@@ -920,16 +988,30 @@
 								<div>
 									<AddContentMenu
 										on:upload={(e) => {
-											if (e.detail.type === 'directory') {
+											if (e.detail.type === 'files') {
+												const input = document.createElement('input');
+												input.type = 'file';
+												input.multiple = true;
+												input.onchange = async (e) => {
+													if (e.target.files) {
+														for (const file of e.target.files) {
+															uploadFileHandler(file);
+														}
+													}
+												};
+												input.click();
+											} else if (e.detail.type === 'directory') {
 												uploadDirectoryHandler();
 											} else if (e.detail.type === 'text') {
 												showAddTextContentModal = true;
-											} else {
+											} else if (e.detail.type === 'web') {
-												document.getElementById('files-input').click();
+												uploadWebBatchHandler(e.detail.data);
 											}
 										}}
 										on:sync={(e) => {
-											showSyncConfirmModal = true;
+											if (e.detail.type === 'directory') {
+												showSyncConfirmModal = true;
+											}
 										}}
 									/>
 								</div>

src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte

@@ -10,14 +10,25 @@
 	import BarsArrowUp from '$lib/components/icons/BarsArrowUp.svelte';
 	import FolderOpen from '$lib/components/icons/FolderOpen.svelte';
 	import ArrowPath from '$lib/components/icons/ArrowPath.svelte';
+	import GlobeAlt from '$lib/components/icons/GlobeAlt.svelte';
+
+	import AttachWebpageModal from '$lib/components/chat/MessageInput/AttachWebpageModal.svelte';
 
 	const i18n = getContext('i18n');
 
 	export let onClose: Function = () => {};
 
 	let show = false;
+	let showAttachWebpageModal = false;
 </script>
 
+<AttachWebpageModal
+	bind:show={showAttachWebpageModal}
+	onSubmit={(e) => {
+		dispatch('upload', e);
+	}}
+/>
+
 <Dropdown
 	bind:show
 	on:change={(e) => {
@@ -93,6 +104,16 @@
 				</DropdownMenu.Item>
 			</Tooltip>
 
+			<DropdownMenu.Item
+				class="flex  gap-2  items-center px-3 py-2 text-sm  cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
+				on:click={() => {
+					showAttachWebpageModal = true;
+				}}
+			>
+				<GlobeAlt strokeWidth="2" />
+				<div class="flex items-center">{$i18n.t('Add from Website')}</div>
+			</DropdownMenu.Item>
+
 			<DropdownMenu.Item
 				class="flex  gap-2  items-center px-3 py-2 text-sm  cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
 				on:click={() => {