ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-11 20:05:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add expanded context results + switch over to using zoekt's json apis

Browse source
This commit is contained in:
bkellam 2024-09-09 23:16:41 -07:00
parent e14a322c7f
commit 17bf94fc5f
22 changed files with 713 additions and 348 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/app/api/(client)/client.ts Normal file
View file

@ -0,0 +1,30 @@
'use client';
import { FileSourceResponse, fileSourceResponseSchema, SearchRequest, SearchResponse, searchResponseSchema } from "@/lib/schemas";
export const search = async (body: SearchRequest): Promise<SearchResponse> => {
const result = await fetch(`/api/search`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(body),
}).then(response => response.json());
return searchResponseSchema.parse(result);
}
export const fetchFileSource = async (fileName: string, repository: string): Promise<FileSourceResponse> => {
const result = await fetch(`/api/source`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
fileName,
repository,
}),
}).then(response => response.json());
return fileSourceResponseSchema.parse(result);
}

24
src/app/api/(server)/search/route.ts Normal file
View file

@ -0,0 +1,24 @@
'use server';
import { search } from "@/lib/server/searchService";
import { searchRequestSchema } from "@/lib/schemas";
import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError";
import { isServiceError } from "@/lib/utils";
import { NextRequest } from "next/server";
export const POST = async (request: NextRequest) => {
const body = await request.json();
const parsed = await searchRequestSchema.safeParseAsync(body);
if (!parsed.success) {
return serviceErrorResponse(
schemaValidationError(parsed.error)
);
}
const response = await search(parsed.data);
if (isServiceError(response)) {
return serviceErrorResponse(response);
}
return Response.json(response);
}

24
src/app/api/(server)/source/route.ts Normal file
View file

@ -0,0 +1,24 @@
'use server';
import { fileSourceRequestSchema } from "@/lib/schemas";
import { getFileSource } from "@/lib/server/searchService";
import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError";
import { isServiceError } from "@/lib/utils";
import { NextRequest } from "next/server";
export const POST = async (request: NextRequest) => {
const body = await request.json();
const parsed = await fileSourceRequestSchema.safeParseAsync(body);
if (!parsed.success) {
return serviceErrorResponse(
schemaValidationError(parsed.error)
);
}
const response = await getFileSource(parsed.data);
if (isServiceError(response)) {
return serviceErrorResponse(response);
}
return Response.json(response);
}

21
src/app/api/search/route.ts
View file

@ -1,21 +0,0 @@
import { ZOEKT_WEBSERVER_URL } from '@/lib/environment';
import { createPathWithQueryParams } from '@/lib/utils';
import { type NextRequest } from 'next/server'
export async function GET(request: NextRequest) {
// @todo: proper error handling
const searchParams = request.nextUrl.searchParams;
const query = searchParams.get('query');
const numResults = searchParams.get('numResults');
const url = createPathWithQueryParams(
`${ZOEKT_WEBSERVER_URL}/search`,
["q", query],
["num", numResults],
["format", "json"],
);
const res = await fetch(url);
const data = await res.json();
return Response.json({ ...data })
}

48
src/app/api/source/route.ts
View file

@ -1,48 +0,0 @@
"use server";
import { missingQueryParam } from "@/lib/serviceError";
import { StatusCodes } from "http-status-codes";
import { NextRequest } from "next/server";
import { GetSourceResponse, pathQueryParamName, repoQueryParamName, ZoektPrintResponse } from "@/lib/types";
import { ZOEKT_WEBSERVER_URL } from "@/lib/environment";
import { createPathWithQueryParams } from "@/lib/utils";
/**
* Returns the content of a source file at the given path.
*
* Usage:
* GET /api/source?path=<filepath>&repo=<repo>
*/
export async function GET(request: NextRequest) {
const searchParams = request.nextUrl.searchParams;
const filepath = searchParams.get(pathQueryParamName);
const repo = searchParams.get(repoQueryParamName);
if (!filepath) {
return missingQueryParam(pathQueryParamName);
}
if (!repo) {
return missingQueryParam(repoQueryParamName);
}
const url = createPathWithQueryParams(
`${ZOEKT_WEBSERVER_URL}/print`,
["f", filepath],
["r", repo],
["format", "json"],
);
const res = await fetch(url);
const data = await res.json() as ZoektPrintResponse;
return Response.json(
{
content: data.Content,
encoding: data.Encoding,
} satisfies GetSourceResponse,
{
status: StatusCodes.OK
}
);
}

50
src/app/search/codePreview.tsx → src/app/search/codePreviewPanel.tsx
View file

@ -4,9 +4,11 @@ import { Button } from "@/components/ui/button";
import { ScrollArea } from "@/components/ui/scroll-area";
import { useExtensionWithDependency } from "@/hooks/useExtensionWithDependency";
import { useKeymapType } from "@/hooks/useKeymapType";
import { useSyntaxHighlightingExtension } from "@/hooks/useSyntaxHighlightingExtension";
import { useThemeNormalized } from "@/hooks/useThemeNormalized";
import { gutterWidthExtension } from "@/lib/extensions/gutterWidthExtension";
import { markMatches, searchResultHighlightExtension } from "@/lib/extensions/searchResultHighlightExtension";
import { ZoektMatch } from "@/lib/types";
import { highlightRanges, searchResultHighlightExtension } from "@/lib/extensions/searchResultHighlightExtension";
import { SearchResultFileMatch } from "@/lib/schemas";
import { defaultKeymap } from "@codemirror/commands";
import { javascript } from "@codemirror/lang-javascript";
import { search } from "@codemirror/search";
@ -17,42 +19,33 @@ import { vim } from "@replit/codemirror-vim";
import CodeMirror, { ReactCodeMirrorRef } from '@uiw/react-codemirror';
import clsx from "clsx";
import { ArrowDown, ArrowUp } from "lucide-react";
import { useTheme } from "next-themes";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
export interface CodePreviewFile {
content: string;
filepath: string;
link?: string;
matches: ZoektMatch[];
matches: SearchResultFileMatch[];
language: string;
}
interface CodePreviewProps {
interface CodePreviewPanelProps {
file?: CodePreviewFile;
selectedMatchIndex: number;
onSelectedMatchIndexChange: (index: number) => void;
onClose: () => void;
}
export const CodePreview = ({
export const CodePreviewPanel = ({
file,
selectedMatchIndex,
onSelectedMatchIndexChange,
onClose,
}: CodePreviewProps) => {
}: CodePreviewPanelProps) => {
const editorRef = useRef<ReactCodeMirrorRef>(null);
const { theme: _theme, systemTheme } = useTheme();
const [ keymapType ] = useKeymapType();
const theme = useMemo(() => {
if (_theme === "system") {
return systemTheme ?? "light";
}
return _theme ?? "light";
}, [_theme, systemTheme]);
const { theme } = useThemeNormalized();
const [gutterWidth, setGutterWidth] = useState(0);
const keymapExtension = useExtensionWithDependency(
@ -68,11 +61,14 @@ export const CodePreview = ({
[keymapType]
);
const syntaxHighlighting = useSyntaxHighlightingExtension(file?.language ?? '', editorRef.current?.view);
const extensions = useMemo(() => {
return [
keymapExtension,
gutterWidthExtension,
javascript(),
syntaxHighlighting,
searchResultHighlightExtension(),
search({
top: true,
@ -84,15 +80,25 @@ export const CodePreview = ({
}
}),
];
}, [keymapExtension]);
}, [keymapExtension, syntaxHighlighting]);
const ranges = useMemo(() => {
if (!file || !file.matches.length) {
return [];
}
return file.matches.flatMap((match) => {
return match.Ranges;
})
}, [file]);
useEffect(() => {
if (!file || !editorRef.current?.view) {
return;
}
markMatches(selectedMatchIndex, file.matches, editorRef.current.view);
}, [file, file?.matches, selectedMatchIndex]);
highlightRanges(selectedMatchIndex, ranges, editorRef.current.view);
}, [ranges, selectedMatchIndex]);
const onUpClicked = useCallback(() => {
onSelectedMatchIndexChange(selectedMatchIndex - 1);
@ -126,7 +132,7 @@ export const CodePreview = ({
</span>
</div>
<div className="flex flex-row gap-1 items-center">
<p className="text-sm">{`${selectedMatchIndex + 1} of ${file?.matches.length}`}</p>
<p className="text-sm">{`${selectedMatchIndex + 1} of ${ranges.length}`}</p>
<Button
variant="ghost"
size="icon"
@ -141,7 +147,7 @@ export const CodePreview = ({
size="icon"
className="h-6 w-6"
onClick={onDownClicked}
disabled={file ? selectedMatchIndex === file?.matches.length - 1 : true}
disabled={file ? selectedMatchIndex === ranges.length - 1 : true}
>
<ArrowDown className="h-4 w-4" />
</Button>

61
src/app/search/page.tsx
View file

@ -7,8 +7,7 @@ import {
} from "@/components/ui/resizable";
import { Separator } from "@/components/ui/separator";
import { useNonEmptyQueryParam } from "@/hooks/useNonEmptyQueryParam";
import { GetSourceResponse, pathQueryParamName, repoQueryParamName, ZoektFileMatch, ZoektSearchResponse } from "@/lib/types";
import { createPathWithQueryParams, getCodeHostFilePreviewLink } from "@/lib/utils";
import { getCodeHostFilePreviewLink } from "@/lib/utils";
import { SymbolIcon } from "@radix-ui/react-icons";
import { useQuery } from "@tanstack/react-query";
import Image from "next/image";
@ -17,9 +16,11 @@ import logoDark from "../../../public/sb_logo_dark.png";
import logoLight from "../../../public/sb_logo_light.png";
import { SearchBar } from "../searchBar";
import { SettingsDropdown } from "../settingsDropdown";
import { CodePreview, CodePreviewFile } from "./codePreview";
import { SearchResults } from "./searchResults";
import { CodePreviewPanel, CodePreviewFile } from "./codePreviewPanel";
import { SearchResultsPanel } from "./searchResultsPanel";
import { useRouter } from "next/navigation";
import { fetchFileSource, search } from "../api/(client)/client";
import { SearchResultFile } from "@/lib/schemas";
export default function SearchPage() {
const router = useRouter();
@ -27,30 +28,28 @@ export default function SearchPage() {
const numResults = useNonEmptyQueryParam("numResults") ?? "100";
const [selectedMatchIndex, setSelectedMatchIndex] = useState(0);
const [selectedFile, setSelectedFile] = useState<ZoektFileMatch | undefined>(undefined);
const [selectedFile, setSelectedFile] = useState<SearchResultFile | undefined>(undefined);
const { data: searchResponse, isLoading } = useQuery({
queryKey: ["search", searchQuery, numResults],
queryFn: async (): Promise<ZoektSearchResponse> => {
console.log("Fetching search results");
const result = await fetch(`/api/search?query=${searchQuery}&numResults=${numResults}`)
.then(response => response.json());
console.log("Done");
return result;
},
queryFn: () => search({
query: searchQuery,
numResults: parseInt(numResults),
}),
enabled: searchQuery.length > 0,
});
const { fileMatches, searchDurationMs } = useMemo((): { fileMatches: ZoektFileMatch[], searchDurationMs: number } => {
const { fileMatches, searchDurationMs } = useMemo((): { fileMatches: SearchResultFile[], searchDurationMs: number } => {
if (!searchResponse) {
return {
fileMatches: [],
searchDurationMs: 0,
};
}
return {
fileMatches: searchResponse.result.FileMatches ?? [],
searchDurationMs: Math.round(searchResponse.result.Stats.Duration / 1000000),
fileMatches: searchResponse.Result.Files ?? [],
searchDurationMs: Math.round(searchResponse.Result.Duration / 1000000),
}
}, [searchResponse]);
@ -100,7 +99,7 @@ export default function SearchPage() {
{/* Search Results & Code Preview */}
<ResizablePanelGroup direction="horizontal">
<ResizablePanel minSize={20}>
<SearchResults
<SearchResultsPanel
fileMatches={fileMatches}
onOpenFileMatch={(fileMatch, matchIndex) => {
setSelectedFile(fileMatch);
@ -126,7 +125,7 @@ export default function SearchPage() {
}
interface CodePreviewWrapperProps {
fileMatch?: ZoektFileMatch;
fileMatch?: SearchResultFile;
onClose: () => void;
selectedMatchIndex: number;
onSelectedMatchIndexChange: (index: number) => void;
@ -140,33 +139,25 @@ const CodePreviewWrapper = ({
}: CodePreviewWrapperProps) => {
const { data: file } = useQuery({
queryKey: ["source", fileMatch?.FileName, fileMatch?.Repo],
queryKey: ["source", fileMatch?.FileName, fileMatch?.Repository],
queryFn: async (): Promise<CodePreviewFile | undefined> => {
if (!fileMatch) {
return undefined;
}
const url = createPathWithQueryParams(
`/api/source`,
[pathQueryParamName, fileMatch.FileName],
[repoQueryParamName, fileMatch.Repo]
);
return fetchFileSource(fileMatch.FileName, fileMatch.Repository)
.then(({ source }) => {
// @todo : refector this to use the templates provided by zoekt.
const link = getCodeHostFilePreviewLink(fileMatch.Repository, fileMatch.FileName)
return fetch(url)
.then(response => response.json())
.then((body: GetSourceResponse) => {
if (body.encoding !== "base64") {
throw new Error("Expected base64 encoding");
}
const content = atob(body.content);
const link = getCodeHostFilePreviewLink(fileMatch.Repo, fileMatch.FileName)
const decodedSource = atob(source);
return {
content,
content: decodedSource,
filepath: fileMatch.FileName,
matches: fileMatch.Matches,
matches: fileMatch.ChunkMatches,
link: link,
language: fileMatch.Language,
};
});
},
@ -174,7 +165,7 @@ const CodePreviewWrapper = ({
});
return (
<CodePreview
<CodePreviewPanel
file={file}
onClose={onClose}
selectedMatchIndex={selectedMatchIndex}

137
src/app/search/searchResults.tsx
View file

@ -1,137 +0,0 @@
'use client';
import { ScrollArea } from "@/components/ui/scroll-area";
import { Separator } from "@/components/ui/separator";
import { ZoektFileMatch } from "@/lib/types";
import { Scrollbar } from "@radix-ui/react-scroll-area";
import { useMemo, useState } from "react";
import { DoubleArrowDownIcon, DoubleArrowUpIcon, FileIcon } from "@radix-ui/react-icons";
import Image from "next/image";
import clsx from "clsx";
import { getRepoCodeHostInfo } from "@/lib/utils";
const MAX_MATCHES_TO_PREVIEW = 5;
interface SearchResultsProps {
fileMatches: ZoektFileMatch[];
onOpenFileMatch: (fileMatch: ZoektFileMatch, matchIndex: number) => void;
}
export const SearchResults = ({
fileMatches,
onOpenFileMatch,
}: SearchResultsProps) => {
return (
<ScrollArea className="h-full">
<div className="flex flex-col gap-2">
{fileMatches.map((fileMatch, index) => (
<FileMatch
key={index}
match={fileMatch}
onOpenFile={(matchIndex) => {
onOpenFileMatch(fileMatch, matchIndex);
}}
/>
))}
</div>
<Scrollbar orientation="vertical" />
</ScrollArea>
)
}
interface FileMatchProps {
match: ZoektFileMatch;
onOpenFile: (matchIndex: number) => void;
}
const FileMatch = ({
match,
onOpenFile,
}: FileMatchProps) => {
const [showAll, setShowAll] = useState(false);
const matchCount = useMemo(() => {
return match.Matches.length;
}, [match]);
const matches = useMemo(() => {
const sortedMatches = match.Matches.sort((a, b) => {
return a.LineNum - b.LineNum;
});
if (!showAll) {
return sortedMatches.slice(0, MAX_MATCHES_TO_PREVIEW);
}
return sortedMatches;
}, [match, showAll]);
const { repoIcon, repoName, repoLink } = useMemo(() => {
const info = getRepoCodeHostInfo(match.Repo);
if (info) {
return {
repoName: info.repoName,
repoLink: info.repoLink,
repoIcon: <Image
src={info.icon}
alt={info.costHostName}
className="w-4 h-4 dark:invert"
/>
}
}
return {
repoName: match.Repo,
repoLink: undefined,
repoIcon: <FileIcon className="w-4 h-4" />
}
}, [match]);
return (
<div>
<div className="bg-cyan-200 dark:bg-cyan-900 primary-foreground px-2 flex flex-row gap-2 items-center">
{repoIcon}
<span
className={clsx("font-medium", {
"cursor-pointer hover:underline": repoLink,
})}
onClick={() => {
if (repoLink) {
window.open(repoLink, "_blank");
}
}}
>
{repoName}
</span>
<span>· {match.FileName}</span>
</div>
{matches.map((match, index) => {
const fragment = match.Fragments[0];
return (
<div
key={index}
className="font-mono px-4 py-0.5 text-sm cursor-pointer"
onClick={() => {
onOpenFile(index);
}}
>
<p>{match.LineNum > 0 ? match.LineNum : "file match"}: {fragment.Pre}<span className="font-bold">{fragment.Match}</span>{fragment.Post}</p>
<Separator />
</div>
);
})}
{matchCount > MAX_MATCHES_TO_PREVIEW && (
<div className="px-4">
<p
onClick={() => setShowAll(!showAll)}
className="text-blue-500 cursor-pointer text-sm flex flex-row items-center gap-2"
>
{showAll ? <DoubleArrowUpIcon className="w-3 h-3" /> : <DoubleArrowDownIcon className="w-3 h-3" />}
{showAll ? `Show fewer matching lines` : `Show ${matchCount - MAX_MATCHES_TO_PREVIEW} more matching lines`}
</p>
</div>
)}
</div>
);
}

269
src/app/search/searchResultsPanel.tsx Normal file
View file

@ -0,0 +1,269 @@
'use client';
import { ScrollArea } from "@/components/ui/scroll-area";
import { Separator } from "@/components/ui/separator";
import { useExtensionWithDependency } from "@/hooks/useExtensionWithDependency";
import { useSyntaxHighlightingExtension } from "@/hooks/useSyntaxHighlightingExtension";
import { useThemeNormalized } from "@/hooks/useThemeNormalized";
import { lineOffsetExtension } from "@/lib/extensions/lineOffsetExtension";
import { SearchResultFile, SearchResultRange } from "@/lib/schemas";
import { getRepoCodeHostInfo } from "@/lib/utils";
import { DoubleArrowDownIcon, DoubleArrowUpIcon, FileIcon } from "@radix-ui/react-icons";
import { Scrollbar } from "@radix-ui/react-scroll-area";
import CodeMirror, { Decoration, DecorationSet, EditorState, EditorView, ReactCodeMirrorRef, StateField, Transaction } from '@uiw/react-codemirror';
import clsx from "clsx";
import Image from "next/image";
import { useMemo, useRef, useState } from "react";
const MAX_MATCHES_TO_PREVIEW = 3;
interface SearchResultsPanelProps {
fileMatches: SearchResultFile[];
onOpenFileMatch: (fileMatch: SearchResultFile, matchIndex: number) => void;
}
export const SearchResultsPanel = ({
fileMatches,
onOpenFileMatch,
}: SearchResultsPanelProps) => {
return (
<ScrollArea className="h-full">
{fileMatches.map((fileMatch, index) => (
<FilePreview
key={index}
file={fileMatch}
onOpenFile={(matchIndex) => {
onOpenFileMatch(fileMatch, matchIndex);
}}
/>
))}
<Scrollbar orientation="vertical" />
</ScrollArea>
)
}
interface FilePreviewProps {
file: SearchResultFile;
onOpenFile: (matchIndex: number) => void;
}
const FilePreview = ({
file,
onOpenFile,
}: FilePreviewProps) => {
const [showAll, setShowAll] = useState(false);
const matchCount = useMemo(() => {
return file.ChunkMatches.length;
}, [file]);
const matches = useMemo(() => {
const sortedMatches = file.ChunkMatches.sort((a, b) => {
return a.ContentStart.LineNumber - b.ContentStart.LineNumber;
});
if (!showAll) {
return sortedMatches.slice(0, MAX_MATCHES_TO_PREVIEW);
}
return sortedMatches;
}, [file, showAll]);
const { repoIcon, repoName, repoLink } = useMemo(() => {
const info = getRepoCodeHostInfo(file.Repository);
if (info) {
return {
repoName: info.repoName,
repoLink: info.repoLink,
repoIcon: <Image
src={info.icon}
alt={info.costHostName}
className="w-4 h-4 dark:invert"
/>
}
}
return {
repoName: file.Repository,
repoLink: undefined,
repoIcon: <FileIcon className="w-4 h-4" />
}
}, [file]);
const isMoreContentButtonVisible = useMemo(() => {
return matchCount > MAX_MATCHES_TO_PREVIEW;
}, [matchCount]);
return (
<div className="flex flex-col">
<div className="bg-cyan-200 dark:bg-cyan-900 primary-foreground px-2 py-0.5 flex flex-row items-center justify-between border">
<div className="flex flex-row gap-2 items-center">
{repoIcon}
<span
className={clsx("font-medium", {
"cursor-pointer hover:underline": repoLink,
})}
onClick={() => {
if (repoLink) {
window.open(repoLink, "_blank");
}
}}
>
{repoName}
</span>
<span>· {file.FileName}</span>
</div>
</div>
{matches.map((match, index) => {
const content = atob(match.Content);
// If it's just the title, don't show a code preview
if (match.FileName) {
return null;
}
const lineOffset = match.ContentStart.LineNumber - 1;
return (
<div
key={index}
className="cursor-pointer"
onClick={() => {
const matchIndex = matches.slice(0, index).reduce((acc, match) => {
return acc + match.Ranges.length;
}, 0);
onOpenFile(matchIndex);
}}
>
<CodePreview
content={content}
language={file.Language}
ranges={match.Ranges}
lineOffset={lineOffset}
/>
{(index !== matches.length - 1 || isMoreContentButtonVisible) && (
<Separator className="dark:bg-gray-400" />
)}
</div>
);
})}
{isMoreContentButtonVisible && (
<div className="px-4 bg-accent">
<p
onClick={() => setShowAll(!showAll)}
className="text-blue-500 cursor-pointer text-sm flex flex-row items-center gap-2"
>
{showAll ? <DoubleArrowUpIcon className="w-3 h-3" /> : <DoubleArrowDownIcon className="w-3 h-3" />}
{showAll ? `Show fewer matches` : `Show ${matchCount - MAX_MATCHES_TO_PREVIEW} more matches`}
</p>
</div>
)}
</div>
);
}
const markDecoration = Decoration.mark({
class: "cm-searchMatch"
});
const cmTheme = EditorView.baseTheme({
"&light .cm-searchMatch": {
border: "1px #6b7280ff",
},
"&dark .cm-searchMatch": {
border: "1px #d1d5dbff",
},
});
const CodePreview = ({
content,
language,
ranges,
lineOffset,
}: {
content: string,
language: string,
ranges: SearchResultRange[],
lineOffset: number,
}) => {
const editorRef = useRef<ReactCodeMirrorRef>(null);
const { theme } = useThemeNormalized();
const syntaxHighlighting = useSyntaxHighlightingExtension(language, editorRef.current?.view);
const rangeHighlighting = useExtensionWithDependency(editorRef.current?.view ?? null, () => {
return [
StateField.define<DecorationSet>({
create(editorState: EditorState) {
const document = editorState.doc;
const decorations = ranges
.sort((a, b) => {
return a.Start.ByteOffset - b.Start.ByteOffset;
})
.map(({ Start, End }) => {
const from = document.line(Start.LineNumber - lineOffset).from + Start.Column - 1;
const to = document.line(End.LineNumber - lineOffset).from + End.Column - 1;
return markDecoration.range(from, to);
});
return Decoration.set(decorations);
},
update(highlights: DecorationSet, _transaction: Transaction) {
return highlights;
},
provide: (field) => EditorView.decorations.from(field),
}),
cmTheme
];
}, [ranges]);
const extensions = useMemo(() => {
return [
syntaxHighlighting,
lineOffsetExtension(lineOffset),
rangeHighlighting,
];
}, [syntaxHighlighting, lineOffset, rangeHighlighting]);
return (
<CodeMirror
ref={editorRef}
readOnly={true}
editable={false}
value={content}
theme={theme === "dark" ? "dark" : "light"}
basicSetup={{
lineNumbers: true,
syntaxHighlighting: true,
// Disable all this other stuff...
... {
foldGutter: false,
highlightActiveLineGutter: false,
highlightSpecialChars: false,
history: false,
drawSelection: false,
dropCursor: false,
allowMultipleSelections: false,
indentOnInput: false,
bracketMatching: false,
closeBrackets: false,
autocompletion: false,
rectangularSelection: false,
crosshairCursor: false,
highlightActiveLine: false,
highlightSelectionMatches: false,
closeBracketsKeymap: false,
defaultKeymap: false,
searchKeymap: false,
historyKeymap: false,
foldKeymap: false,
completionKeymap: false,
lintKeymap: false,
}
}}
extensions={extensions}
/>
)
}

26
src/hooks/useSyntaxHighlightingExtension.ts Normal file
View file

@ -0,0 +1,26 @@
'use client';
import { EditorView } from "@codemirror/view";
import { useExtensionWithDependency } from "./useExtensionWithDependency";
import { javascript } from "@codemirror/lang-javascript";
export const useSyntaxHighlightingExtension = (language: string, view: EditorView | undefined) => {
const extension = useExtensionWithDependency(
view ?? null,
() => {
switch (language.toLowerCase()) {
case "typescript":
case "javascript":
return javascript({
jsx: true,
typescript: true,
});
default:
return [];
}
},
[language]
);
return extension;
}

22
src/hooks/useThemeNormalized.ts Normal file
View file

@ -0,0 +1,22 @@
'use client';
import { useTheme as useThemeBase } from "next-themes";
import { useMemo } from "react";
export const useThemeNormalized = (defaultTheme: "light" | "dark" = "light") => {
const { theme: _theme, systemTheme, setTheme } = useThemeBase();
const theme = useMemo(() => {
if (_theme === "system") {
return systemTheme ?? defaultTheme;
}
return _theme ?? defaultTheme;
}, [_theme, systemTheme]);
return {
theme,
systemTheme,
setTheme,
};
}

6
src/lib/environment.ts
View file

@ -3,6 +3,12 @@ const getEnv = (env: string | undefined, defaultValue = '') => {
return env ?? defaultValue;
}
const getEnvNumber = (env: string | undefined, defaultValue: number = 0) => {
return Number(env) ?? defaultValue;
}
export const ZOEKT_WEBSERVER_URL = getEnv(process.env.ZOEKT_WEBSERVER_URL, "http://localhost:6070");
export const SHARD_MAX_MATCH_COUNT = getEnvNumber(process.env.SHARD_MAX_MATCH_COUNT, 10000);
export const TOTAL_MAX_MATCH_COUNT = getEnvNumber(process.env.TOTAL_MAX_MATCH_COUNT, 100000);
export const NODE_ENV = process.env.NODE_ENV;

1
src/lib/errorCodes.ts
View file

@ -3,4 +3,5 @@ export enum ErrorCode {
MISSING_REQUIRED_QUERY_PARAMETER = 'MISSING_REQUIRED_QUERY_PARAMETER',
REPOSITORY_NOT_FOUND = 'REPOSITORY_NOT_FOUND',
FILE_NOT_FOUND = 'FILE_NOT_FOUND',
INVALID_REQUEST_BODY = 'INVALID_REQUEST_BODY',
}

20
src/lib/extensions/lineOffsetExtension.ts Normal file
View file

@ -0,0 +1,20 @@
import { Compartment } from "@codemirror/state";
import { lineNumbers } from "@codemirror/view";
const gutter = new Compartment();
/**
* Offsets the line numbers by the given amount
* @see: https://discuss.codemirror.net/t/codemirror-6-offset-line-numbers/2675/8
*/
export const lineOffsetExtension = (lineOffset: number) => {
const lines = lineNumbers({
formatNumber: (n) => {
return (n + lineOffset).toString();
}
});
return [
gutter.of(lines)
]
}

54
src/lib/extensions/searchResultHighlightExtension.ts
View file

@ -1,24 +1,16 @@
import { EditorSelection, Extension, StateEffect, StateField, Text, Transaction } from "@codemirror/state";
import { Decoration, DecorationSet, EditorView } from "@codemirror/view";
import { ZoektMatch } from "../types";
const matchMark = Decoration.mark({
class: "tq-searchMatch"
});
const selectedMatchMark = Decoration.mark({
class: "tq-searchMatch-selected"
});
import { SearchResultRange } from "../schemas";
const setMatchState = StateEffect.define<{
selectedMatchIndex: number,
matches: ZoektMatch[],
ranges: SearchResultRange[],
}>();
const getMatchRange = (match: ZoektMatch, document: Text) => {
const line = document.line(match.LineNum);
const fragment = match.Fragments[0];
const from = line.from + fragment.Pre.length;
const to = from + fragment.Match.length;
const convertToCodeMirrorRange = (range: SearchResultRange, document: Text) => {
const { Start, End } = range;
const from = document.line(Start.LineNumber).from + Start.Column - 1;
const to = document.line(End.LineNumber).from + End.Column - 1;
return { from, to };
}
@ -32,12 +24,14 @@ const matchHighlighter = StateField.define<DecorationSet>({
for (const effect of transaction.effects) {
if (effect.is(setMatchState)) {
const { matches, selectedMatchIndex } = effect.value;
const { ranges, selectedMatchIndex } = effect.value;
const decorations = matches
.filter((match) => match.LineNum > 0)
.map((match, index) => {
const { from, to } = getMatchRange(match, transaction.newDoc);
const decorations = ranges
.sort((a, b) => {
return a.Start.ByteOffset - b.Start.ByteOffset;
})
.map((range, index) => {
const { from, to } = convertToCodeMirrorRange(range, transaction.newDoc);
const mark = index === selectedMatchIndex ? selectedMatchMark : matchMark;
return mark.range(from, to);
});
@ -51,6 +45,13 @@ const matchHighlighter = StateField.define<DecorationSet>({
provide: (field) => EditorView.decorations.from(field),
});
const matchMark = Decoration.mark({
class: "tq-searchMatch"
});
const selectedMatchMark = Decoration.mark({
class: "tq-searchMatch-selected"
});
const highlightTheme = EditorView.baseTheme({
"&light .tq-searchMatch": {
border: "1px dotted #6b7280ff",
@ -64,34 +65,27 @@ const highlightTheme = EditorView.baseTheme({
},
"&dark .tq-searchMatch-selected": {
backgroundColor: "#00ff007a",
}
});
export const markMatches = (selectedMatchIndex: number, matches: ZoektMatch[], view: EditorView) => {
export const highlightRanges = (selectedMatchIndex: number, ranges: SearchResultRange[], view: EditorView) => {
const setState = setMatchState.of({
selectedMatchIndex,
matches,
ranges,
});
const effects = []
effects.push(setState);
if (selectedMatchIndex >= 0 && selectedMatchIndex < matches.length) {
const match = matches[selectedMatchIndex];
// Don't scroll if the match is on the filename.
if (match.LineNum > 0) {
const { from, to } = getMatchRange(match, view.state.doc);
if (selectedMatchIndex >= 0 && selectedMatchIndex < ranges.length) {
const { from, to } = convertToCodeMirrorRange(ranges[selectedMatchIndex], view.state.doc);
const selection = EditorSelection.range(from, to);
effects.push(EditorView.scrollIntoView(selection, {
y: "start",
}));
}
};
view.dispatch({ effects });
return true;
}
export const searchResultHighlightExtension = (): Extension => {

68
src/lib/schemas.ts Normal file
View file

@ -0,0 +1,68 @@
import { z } from "zod";
export type SearchRequest = z.infer<typeof searchRequestSchema>;
export const searchRequestSchema = z.object({
query: z.string(),
numResults: z.number(),
whole: z.optional(z.boolean()),
});
export type SearchResponse = z.infer<typeof searchResponseSchema>;
export type SearchResult = SearchResponse["Result"];
export type SearchResultFile = SearchResult["Files"][number];
export type SearchResultFileMatch = SearchResultFile["ChunkMatches"][number];
export type SearchResultRange = z.infer<typeof rangeSchema>;
export type SearchResultLocation = z.infer<typeof locationSchema>;
// @see : https://github.com/TaqlaAI/zoekt/blob/main/api.go#L212
const locationSchema = z.object({
// 0-based byte offset from the beginning of the file
ByteOffset: z.number(),
// 1-based line number from the beginning of the file
LineNumber: z.number(),
// 1-based column number (in runes) from the beginning of line
Column: z.number(),
});
const rangeSchema = z.object({
Start: locationSchema,
End: locationSchema,
});
export const searchResponseSchema = z.object({
Result: z.object({
Duration: z.number(),
FileCount: z.number(),
Files: z.array(z.object({
FileName: z.string(),
Repository: z.string(),
Version: z.string(),
Language: z.string(),
Branches: z.array(z.string()),
ChunkMatches: z.array(z.object({
Content: z.string(),
Ranges: z.array(rangeSchema),
FileName: z.boolean(),
ContentStart: locationSchema,
Score: z.number(),
})),
Checksum: z.string(),
Score: z.number(),
// Set if `whole` is true.
Content: z.optional(z.string()),
})),
}),
});
export type FileSourceRequest = z.infer<typeof fileSourceRequestSchema>;
export const fileSourceRequestSchema = z.object({
fileName: z.string(),
repository: z.string()
});
export type FileSourceResponse = z.infer<typeof fileSourceResponseSchema>;
export const fileSourceResponseSchema = z.object({
source: z.string(),
});

56
src/lib/server/searchService.ts Normal file
View file

@ -0,0 +1,56 @@
import { SHARD_MAX_MATCH_COUNT, TOTAL_MAX_MATCH_COUNT } from "../environment";
import { FileSourceRequest, FileSourceResponse, SearchRequest, SearchResponse, searchResponseSchema } from "../schemas";
import { fileNotFound, invalidZoektResponse, ServiceError } from "../serviceError";
import { isServiceError } from "../utils";
import { zoektFetch } from "./zoektClient";
export const search = async ({ query, numResults, whole }: SearchRequest): Promise<SearchResponse | ServiceError> => {
const body = JSON.stringify({
q: query,
// @see: https://github.com/TaqlaAI/zoekt/blob/main/api.go#L892
opts: {
NumContextLines: 2,
ChunkMatches: true,
MaxMatchDisplayCount: numResults,
Whole: !!whole,
ShardMaxMatchCount: SHARD_MAX_MATCH_COUNT,
TotalMaxMatchCount: TOTAL_MAX_MATCH_COUNT,
}
});
const searchResponse = await zoektFetch({
path: "/api/search",
body,
method: "POST",
});
if (!searchResponse.ok) {
return invalidZoektResponse(searchResponse);
}
const searchBody = await searchResponse.json();
return searchResponseSchema.parse(searchBody);
}
export const getFileSource = async ({ fileName, repository }: FileSourceRequest): Promise<FileSourceResponse | ServiceError> => {
const searchResponse = await search({
query: `${fileName} repo:${repository}`,
numResults: 1,
whole: true,
});
if (isServiceError(searchResponse)) {
return searchResponse;
}
const files = searchResponse.Result.Files;
if (files.length === 0) {
return fileNotFound(fileName, repository);
}
const source = files[0].Content ?? '';
return {
source
}
}

33
src/lib/server/zoektClient.ts Normal file
View file

@ -0,0 +1,33 @@
import { ZOEKT_WEBSERVER_URL } from "../environment"
interface ZoektRequest {
path: string,
body: string,
method: string,
}
export const zoektFetch = async ({
path,
body,
method,
}: ZoektRequest) => {
const start = Date.now();
const response = await fetch(
new URL(path, ZOEKT_WEBSERVER_URL),
{
method,
headers: {
"Content-Type": "application/json",
},
body,
}
);
const duration = Date.now() - start;
console.log(`[zoektClient] ${method} ${path} ${response.status} ${duration}ms`);
// @todo : add metrics
return response;
}

46
src/lib/serviceError.ts
View file

@ -1,13 +1,14 @@
import { StatusCodes } from "http-status-codes";
import { ErrorCode } from "./errorCodes";
import { ZodError } from "zod";
export interface ServiceErrorArgs {
export interface ServiceError {
statusCode: StatusCodes;
errorCode: ErrorCode;
message: string;
}
export const serviceError = ({ statusCode, errorCode, message }: ServiceErrorArgs) => {
export const serviceErrorResponse = ({ statusCode, errorCode, message }: ServiceError) => {
return Response.json({
statusCode,
errorCode,
@ -17,10 +18,45 @@ export const serviceError = ({ statusCode, errorCode, message }: ServiceErrorArg
});
}
export const missingQueryParam = (name: string) => {
return serviceError({
export const missingQueryParam = (name: string): ServiceError => {
return {
statusCode: StatusCodes.BAD_REQUEST,
errorCode: ErrorCode.MISSING_REQUIRED_QUERY_PARAMETER,
message: `Missing required query parameter: ${name}`,
});
};
}
export const schemaValidationError = (error: ZodError): ServiceError => {
return {
statusCode: StatusCodes.BAD_REQUEST,
errorCode: ErrorCode.INVALID_REQUEST_BODY,
message: `Schema validation failed with: ${error.message}`,
};
}
export const invalidZoektResponse = async (zoektResponse: Response): Promise<ServiceError> => {
const zoektMessage = await (async () => {
try {
const zoektResponseBody = await zoektResponse.json();
if (zoektResponseBody.Error) {
return zoektResponseBody.Error;
}
} catch (_e) {
return "Unknown error";
}
})();
return {
statusCode: StatusCodes.INTERNAL_SERVER_ERROR,
errorCode: ErrorCode.INVALID_REQUEST_BODY,
message: `Zoekt request failed with status code ${zoektResponse.status} and message: "${zoektMessage}"`,
};
}
export const fileNotFound = async (fileName: string, repository: string): Promise<ServiceError> => {
return {
statusCode: StatusCodes.NOT_FOUND,
errorCode: ErrorCode.FILE_NOT_FOUND,
message: `File "${fileName}" not found in repository "${repository}"`,
};
}

44
src/lib/types.ts
View file

@ -1,48 +1,4 @@
export const pathQueryParamName = "path";
export const repoQueryParamName = "repo";
export type GetSourceResponse = {
content: string;
encoding: string;
}
export interface ZoektMatch {
URL: string,
FileName: string,
LineNum: number,
Fragments: {
Pre: string,
Match: string,
Post: string
}[]
}
export interface ZoektFileMatch {
FileName: string,
Repo: string,
Language: string,
Matches: ZoektMatch[],
URL: string,
}
export interface ZoektResult {
QueryStr: string,
FileMatches: ZoektFileMatch[] | null,
Stats: {
// Duration in nanoseconds
Duration: number,
}
}
export interface ZoektSearchResponse {
result: ZoektResult,
}
export interface ZoektPrintResponse {
Content: string,
Encoding: string,
}
export type KeymapType = "default" | "vim";

9
src/lib/utils.ts
View file

@ -2,6 +2,7 @@ import { type ClassValue, clsx } from "clsx"
import { twMerge } from "tailwind-merge"
import githubLogo from "../../public/github.svg";
import gitlabLogo from "../../public/gitlab.svg";
import { ServiceError } from "./serviceError";
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs))
@ -72,3 +73,11 @@ export const getCodeHostFilePreviewLink = (repoName: string, filePath: string):
return undefined;
}
export const isServiceError = (data: unknown): data is ServiceError => {
return typeof data === 'object' &&
data !== null &&
'statusCode' in data &&
'errorCode' in data &&
'message' in data;
}

2
supervisord.conf
View file

@ -13,7 +13,7 @@ stdout_logfile_maxbytes=0
redirect_stderr=true
[program:zoekt-webserver]
command=zoekt-webserver -index %(ENV_DATA_CACHE_DIR)s/index
command=zoekt-webserver -index %(ENV_DATA_CACHE_DIR)s/index -rpc
autostart=true
autorestart=true
startretries=3