Add additional tools for repo searching and listing

packages/web/src/features/chat/agent.ts

@@ -6,7 +6,7 @@ import { ProviderOptions } from "@ai-sdk/provider-utils";
 import { createLogger } from "@sourcebot/logger";
 import { LanguageModel, ModelMessage, StopCondition, streamText } from "ai";
 import { ANSWER_TAG, FILE_REFERENCE_PREFIX, toolNames } from "./constants";
-import { createCodeSearchTool, findSymbolDefinitionsTool, findSymbolReferencesTool, readFilesTool } from "./tools";
+import { createCodeSearchTool, findSymbolDefinitionsTool, findSymbolReferencesTool, readFilesTool, searchReposTool, listAllReposTool } from "./tools";
 import { FileSource, Source } from "./types";
 import { addLineNumbers, fileReferenceToString } from "./utils";
 
@@ -54,6 +54,13 @@ export const createAgentStream = async ({
             [toolNames.readFiles]: readFilesTool,
             [toolNames.findSymbolReferences]: findSymbolReferencesTool,
             [toolNames.findSymbolDefinitions]: findSymbolDefinitionsTool,
+            // We only include these tools when there are no search scopes
+            // because the LLM will need to discover what repositories are
+            // available to it.
+            ...(searchScopeRepoNames.length === 0 ? {
+                [toolNames.searchRepos]: searchReposTool,
+                [toolNames.listAllRepos]: listAllReposTool,
+            } : {}),
         },
         prepareStep: async ({ stepNumber }) => {
             // The first step attaches any mentioned sources to the system prompt.
@@ -185,13 +192,7 @@ ${searchScopeRepoNames.map(repo => `- ${repo}`).join('\n')}
 </available_repositories>
 
 <research_phase_instructions>
-During the research phase, you have these tools available:
-- \`${toolNames.searchCode}\`: Search for code patterns, functions, or text across repositories
-- \`${toolNames.readFiles}\`: Read the contents of specific files
-- \`${toolNames.findSymbolReferences}\`: Find where symbols are referenced
-- \`${toolNames.findSymbolDefinitions}\`: Find where symbols are defined
-
-Use these tools to gather comprehensive context before answering. Always explain why you're using each tool.
+During the research phase, use the tools available to you to gather comprehensive context before answering. Always explain why you're using each tool. Depending on the user's question, you may need to use multiple tools. If the question is vague, ask the user for more information.
 </research_phase_instructions>
 
 ${answerInstructions}

packages/web/src/features/chat/components/chatThread/detailsCard.tsx

@@ -12,6 +12,8 @@ import { FindSymbolDefinitionsToolComponent } from './tools/findSymbolDefinition
 import { FindSymbolReferencesToolComponent } from './tools/findSymbolReferencesToolComponent';
 import { ReadFilesToolComponent } from './tools/readFilesToolComponent';
 import { SearchCodeToolComponent } from './tools/searchCodeToolComponent';
+import { SearchReposToolComponent } from './tools/searchReposToolComponent';
+import { ListAllReposToolComponent } from './tools/listAllReposToolComponent';
 import { SBChatMessageMetadata, SBChatMessagePart } from '../../types';
 import { SearchScopeIcon } from '../searchScopeIcon';
 
@@ -181,6 +183,20 @@ export const DetailsCard = ({
                                                         part={part}
                                                     />
                                                 )
+                                            case 'tool-searchRepos':
+                                                return (
+                                                    <SearchReposToolComponent
+                                                        key={index}
+                                                        part={part}
+                                                    />
+                                                )
+                                            case 'tool-listAllRepos':
+                                                return (
+                                                    <ListAllReposToolComponent
+                                                        key={index}
+                                                        part={part}
+                                                    />
+                                                )
                                             default:
                                                 return null;
                                         }

packages/web/src/features/chat/components/chatThread/referencedSourcesListView.tsx

@@ -221,7 +221,7 @@ export const ReferencedSourcesListView = ({
                                     <span className="text-sm font-medium">{fileName}</span>
                                 </div>
                                 <div className="p-4 text-sm text-destructive bg-destructive/10 rounded border">
-                                    Failed to load file: {isServiceError(query.data) ? query.data.message : 'Unknown error'}
+                                    Failed to load file: {isServiceError(query.data) ? query.data.message : query.error?.message ?? 'Unknown error'}
                                 </div>
                             </div>
                         );

packages/web/src/features/chat/components/chatThread/tools/listAllReposToolComponent.tsx

@@ -0,0 +1,66 @@
+'use client';
+
+import { ListAllReposToolUIPart } from "@/features/chat/tools";
+import { isServiceError } from "@/lib/utils";
+import { useMemo, useState } from "react";
+import { ToolHeader, TreeList } from "./shared";
+import { CodeSnippet } from "@/app/components/codeSnippet";
+import { Separator } from "@/components/ui/separator";
+import { FolderOpenIcon } from "lucide-react";
+
+export const ListAllReposToolComponent = ({ part }: { part: ListAllReposToolUIPart }) => {
+    const [isExpanded, setIsExpanded] = useState(false);
+
+    const label = useMemo(() => {
+        switch (part.state) {
+            case 'input-streaming':
+                return 'Loading all repositories...';
+            case 'output-error':
+                return '"List all repositories" tool call failed';
+            case 'input-available':
+            case 'output-available':
+                return 'Listed all repositories';
+        }
+    }, [part]);
+
+    return (
+        <div className="my-4">
+            <ToolHeader
+                isLoading={part.state !== 'output-available' && part.state !== 'output-error'}
+                isError={part.state === 'output-error' || (part.state === 'output-available' && isServiceError(part.output))}
+                isExpanded={isExpanded}
+                label={label}
+                Icon={FolderOpenIcon}
+                onExpand={setIsExpanded}
+            />
+            {part.state === 'output-available' && isExpanded && (
+                <>
+                    {isServiceError(part.output) ? (
+                        <TreeList>
+                            <span>Failed with the following error: <CodeSnippet className="text-sm text-destructive">{part.output.message}</CodeSnippet></span>
+                        </TreeList>
+                    ) : (
+                        <>
+                            {part.output.length === 0 ? (
+                                <span className="text-sm text-muted-foreground ml-[25px]">No repositories found</span>
+                            ) : (
+                                <TreeList>
+                                    <div className="text-sm text-muted-foreground mb-2">
+                                        Found {part.output.length} repositories:
+                                    </div>
+                                    {part.output.map((repoName, index) => (
+                                        <div key={index} className="flex items-center gap-2 text-sm">
+                                            <FolderOpenIcon className="h-4 w-4 text-muted-foreground" />
+                                            <span className="truncate">{repoName}</span>
+                                        </div>
+                                    ))}
+                                </TreeList>
+                            )}
+                        </>
+                    )}
+                    <Separator className='ml-[7px] my-2' />
+                </>
+            )}
+        </div>
+    )
+} 

packages/web/src/features/chat/components/chatThread/tools/searchCodeToolComponent.tsx

@@ -11,24 +11,38 @@ import { SearchIcon } from "lucide-react";
 import Link from "next/link";
 import { SearchQueryParams } from "@/lib/types";
 import { PlayIcon } from "@radix-ui/react-icons";
-
+import { buildSearchQuery } from "@/features/chat/utils";
 
 export const SearchCodeToolComponent = ({ part }: { part: SearchCodeToolUIPart }) => {
     const [isExpanded, setIsExpanded] = useState(false);
     const domain = useDomain();
 
+    const displayQuery = useMemo(() => {
+        if (part.state !== 'input-available' && part.state !== 'output-available') {
+            return '';
+        }
+
+        const query = buildSearchQuery({
+            query: part.input.queryRegexp,
+            repoNamesFilterRegexp: part.input.repoNamesFilterRegexp,
+            languageNamesFilter: part.input.languageNamesFilter,
+            fileNamesFilterRegexp: part.input.fileNamesFilterRegexp,
+        });
+
+        return query;
+    }, [part]);
+
     const label = useMemo(() => {
         switch (part.state) {
             case 'input-streaming':
                 return 'Searching...';
-            case 'input-available':
-                return <span>Searching for <CodeSnippet>{part.input.queryRegexp}</CodeSnippet></span>;
             case 'output-error':
                 return '"Search code" tool call failed';
+            case 'input-available':
             case 'output-available':
-                return <span>Searched for <CodeSnippet>{part.input.queryRegexp}</CodeSnippet></span>;
+                return <span>Searched for <CodeSnippet>{displayQuery}</CodeSnippet></span>;
         }
-    }, [part]);
+    }, [part, displayQuery]);
 
     return (
         <div className="my-4">

packages/web/src/features/chat/components/chatThread/tools/searchReposToolComponent.tsx

@@ -0,0 +1,63 @@
+'use client';
+
+import { SearchReposToolUIPart } from "@/features/chat/tools";
+import { isServiceError } from "@/lib/utils";
+import { useMemo, useState } from "react";
+import { ToolHeader, TreeList } from "./shared";
+import { CodeSnippet } from "@/app/components/codeSnippet";
+import { Separator } from "@/components/ui/separator";
+import { BookMarkedIcon } from "lucide-react";
+
+export const SearchReposToolComponent = ({ part }: { part: SearchReposToolUIPart }) => {
+    const [isExpanded, setIsExpanded] = useState(false);
+
+    const label = useMemo(() => {
+        switch (part.state) {
+            case 'input-streaming':
+                return 'Searching repositories...';
+            case 'output-error':
+                return '"Search repositories" tool call failed';
+            case 'input-available':
+            case 'output-available':
+                return <span>Searched for repositories: <CodeSnippet className="truncate">{part.input.query}</CodeSnippet></span>;
+        }
+    }, [part]);
+
+    return (
+        <div className="my-4">
+            <ToolHeader
+                isLoading={part.state !== 'output-available' && part.state !== 'output-error'}
+                isError={part.state === 'output-error' || (part.state === 'output-available' && isServiceError(part.output))}
+                isExpanded={isExpanded}
+                label={label}
+                Icon={BookMarkedIcon}
+                onExpand={setIsExpanded}
+            />
+            {part.state === 'output-available' && isExpanded && (
+                <>
+                    {isServiceError(part.output) ? (
+                        <TreeList>
+                            <span>Failed with the following error: <CodeSnippet className="text-sm text-destructive">{part.output.message}</CodeSnippet></span>
+                        </TreeList>
+                    ) : (
+                        <>
+                            {part.output.length === 0 ? (
+                                <span className="text-sm text-muted-foreground ml-[25px]">No repositories found</span>
+                            ) : (
+                                <TreeList>
+                                    {part.output.map((repoName, index) => (
+                                        <div key={index} className="flex items-center gap-2 text-sm">
+                                            <BookMarkedIcon className="h-4 w-4 text-muted-foreground" />
+                                            <span className="truncate">{repoName}</span>
+                                        </div>
+                                    ))}
+                                </TreeList>
+                            )}
+                        </>
+                    )}
+                    <Separator className='ml-[7px] my-2' />
+                </>
+            )}
+        </div>
+    )
+}

packages/web/src/features/chat/constants.ts

@@ -14,6 +14,8 @@ export const toolNames = {
     readFiles: 'readFiles',
     findSymbolReferences: 'findSymbolReferences',
     findSymbolDefinitions: 'findSymbolDefinitions',
+    searchRepos: 'searchRepos',
+    listAllRepos: 'listAllRepos',
 } as const;
 
 // These part types are visible in the UI.
@@ -24,4 +26,6 @@ export const uiVisiblePartTypes: SBChatMessagePart['type'][] = [
     'tool-readFiles',
     'tool-findSymbolDefinitions',
     'tool-findSymbolReferences',
+    'tool-searchRepos',
+    'tool-listAllRepos',
 ] as const;

packages/web/src/features/chat/tools.ts

@@ -8,6 +8,8 @@ import { findSearchBasedSymbolDefinitions, findSearchBasedSymbolReferences } fro
 import { FileSourceResponse } from "../search/types";
 import { addLineNumbers, buildSearchQuery } from "./utils";
 import { toolNames } from "./constants";
+import { getRepos } from "@/actions";
+import Fuse from "fuse.js";
 
 // @NOTE: When adding a new tool, follow these steps:
 // 1. Add the tool to the `toolNames` constant in `constants.ts`.
@@ -148,7 +150,7 @@ Queries consist of space-seperated regular expressions. Wrapping expressions in
 \`foo\` - Match files with regex /foo/
 \`foo bar\` - Match files with regex /foo/ and /bar/
 \`"foo bar"\` - Match files with regex /foo bar/
-\`console\.log\` - Match files with regex /console\.log/
+\`console.log\` - Match files with regex /console.log/
 
 Multiple expressions can be or'd together with or, negated with -, or grouped with (). Examples:
 \`foo or bar\` - Match files with regex /foo/ or /bar/
@@ -167,8 +169,9 @@ Multiple expressions can be or'd together with or, negated with -, or grouped wi
             .array(z.string())
             .describe(`Filter results from filepaths that match the regex. When this option is not specified, all files are searched.`)
             .optional(),
+        limit: z.number().default(10).describe("Maximum number of matches to return (default: 100)"),
     }),
-    execute: async ({ queryRegexp: _query, repoNamesFilterRegexp, languageNamesFilter, fileNamesFilterRegexp }) => {
+    execute: async ({ queryRegexp: _query, repoNamesFilterRegexp, languageNamesFilter, fileNamesFilterRegexp, limit }) => {
         const query = buildSearchQuery({
             query: _query,
             repoNamesFilter: selectedRepos,
@@ -179,7 +182,7 @@ Multiple expressions can be or'd together with or, negated with -, or grouped wi
 
         const response = await search({
             query,
-            matches: 100,
+            matches: limit ?? 100,
             // @todo: we can make this configurable.
             contextLines: 3,
             whole: false,
@@ -210,3 +213,59 @@ export type SearchCodeTool = InferUITool<ReturnType<typeof createCodeSearchTool>
 export type SearchCodeToolInput = InferToolInput<ReturnType<typeof createCodeSearchTool>>;
 export type SearchCodeToolOutput = InferToolOutput<ReturnType<typeof createCodeSearchTool>>;
 export type SearchCodeToolUIPart = ToolUIPart<{ [toolNames.searchCode]: SearchCodeTool }>;
+
+export const searchReposTool = tool({
+    description: `Search for repositories by name using fuzzy search. This helps find repositories in the codebase when you know part of their name.`,
+    inputSchema: z.object({
+        query: z.string().describe("The search query to find repositories by name (supports fuzzy matching)"),
+        limit: z.number().default(10).describe("Maximum number of repositories to return (default: 10)")
+    }),
+    execute: async ({ query, limit }) => {
+        const reposResponse = await getRepos(SINGLE_TENANT_ORG_DOMAIN);
+        
+        if (isServiceError(reposResponse)) {
+            return reposResponse;
+        }
+        
+        // Configure Fuse.js for fuzzy searching
+        const fuse = new Fuse(reposResponse, {
+            keys: [
+                { name: 'repoName', weight: 0.7 },
+                { name: 'repoDisplayName', weight: 0.3 }
+            ],
+            threshold: 0.4, // Lower threshold = more strict matching
+            includeScore: true,
+            minMatchCharLength: 1,
+        });
+        
+        const searchResults = fuse.search(query, { limit: limit ?? 10 });
+
+        searchResults.sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
+
+        return searchResults.map(({ item }) => item.repoName);
+    }
+});
+
+export type SearchReposTool = InferUITool<typeof searchReposTool>;
+export type SearchReposToolInput = InferToolInput<typeof searchReposTool>;
+export type SearchReposToolOutput = InferToolOutput<typeof searchReposTool>;
+export type SearchReposToolUIPart = ToolUIPart<{ [toolNames.searchRepos]: SearchReposTool }>;
+
+export const listAllReposTool = tool({
+    description: `Lists all repositories in the codebase. This provides a complete overview of all available repositories.`,
+    inputSchema: z.object({}),
+    execute: async () => {
+        const reposResponse = await getRepos(SINGLE_TENANT_ORG_DOMAIN);
+        
+        if (isServiceError(reposResponse)) {
+            return reposResponse;
+        }
+        
+        return reposResponse.map((repo) => repo.repoName);
+    }
+});
+
+export type ListAllReposTool = InferUITool<typeof listAllReposTool>;
+export type ListAllReposToolInput = InferToolInput<typeof listAllReposTool>;
+export type ListAllReposToolOutput = InferToolOutput<typeof listAllReposTool>;
+export type ListAllReposToolUIPart = ToolUIPart<{ [toolNames.listAllRepos]: ListAllReposTool }>;

packages/web/src/features/chat/types.ts

@@ -3,7 +3,7 @@ import { BaseEditor, Descendant } from "slate";
 import { HistoryEditor } from "slate-history";
 import { ReactEditor, RenderElementProps } from "slate-react";
 import { z } from "zod";
-import { FindSymbolDefinitionsTool, FindSymbolReferencesTool, ReadFilesTool, SearchCodeTool } from "./tools";
+import { FindSymbolDefinitionsTool, FindSymbolReferencesTool, ReadFilesTool, SearchCodeTool, SearchReposTool, ListAllReposTool } from "./tools";
 import { toolNames } from "./constants";
 import { LanguageModel } from "@sourcebot/schemas/v3/index.type";
 
@@ -83,6 +83,8 @@ export type SBChatMessageToolTypes = {
     [toolNames.readFiles]: ReadFilesTool,
     [toolNames.findSymbolReferences]: FindSymbolReferencesTool,
     [toolNames.findSymbolDefinitions]: FindSymbolDefinitionsTool,
+    [toolNames.searchRepos]: SearchReposTool,
+    [toolNames.listAllRepos]: ListAllReposTool,
 }
 
 export type SBChatMessageDataParts = {