diff --git a/src/app/api/(client)/client.ts b/src/app/api/(client)/client.ts new file mode 100644 index 00000000..0990f962 --- /dev/null +++ b/src/app/api/(client)/client.ts @@ -0,0 +1,30 @@ +'use client'; + +import { FileSourceResponse, fileSourceResponseSchema, SearchRequest, SearchResponse, searchResponseSchema } from "@/lib/schemas"; + +export const search = async (body: SearchRequest): Promise => { + const result = await fetch(`/api/search`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }).then(response => response.json()); + + return searchResponseSchema.parse(result); +} + +export const fetchFileSource = async (fileName: string, repository: string): Promise => { + const result = await fetch(`/api/source`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + fileName, + repository, + }), + }).then(response => response.json()); + + return fileSourceResponseSchema.parse(result); +} \ No newline at end of file diff --git a/src/app/api/(server)/search/route.ts b/src/app/api/(server)/search/route.ts new file mode 100644 index 00000000..fa9496b4 --- /dev/null +++ b/src/app/api/(server)/search/route.ts @@ -0,0 +1,24 @@ +'use server'; + +import { search } from "@/lib/server/searchService"; +import { searchRequestSchema } from "@/lib/schemas"; +import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { NextRequest } from "next/server"; + +export const POST = async (request: NextRequest) => { + const body = await request.json(); + const parsed = await searchRequestSchema.safeParseAsync(body); + if (!parsed.success) { + return serviceErrorResponse( + schemaValidationError(parsed.error) + ); + } + + const response = await search(parsed.data); + if (isServiceError(response)) { + return serviceErrorResponse(response); + } + + return Response.json(response); +} \ No newline at end of file diff --git a/src/app/api/(server)/source/route.ts b/src/app/api/(server)/source/route.ts new file mode 100644 index 00000000..2c245374 --- /dev/null +++ b/src/app/api/(server)/source/route.ts @@ -0,0 +1,24 @@ +'use server'; + +import { fileSourceRequestSchema } from "@/lib/schemas"; +import { getFileSource } from "@/lib/server/searchService"; +import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { NextRequest } from "next/server"; + +export const POST = async (request: NextRequest) => { + const body = await request.json(); + const parsed = await fileSourceRequestSchema.safeParseAsync(body); + if (!parsed.success) { + return serviceErrorResponse( + schemaValidationError(parsed.error) + ); + } + + const response = await getFileSource(parsed.data); + if (isServiceError(response)) { + return serviceErrorResponse(response); + } + + return Response.json(response); +} diff --git a/src/app/api/search/route.ts b/src/app/api/search/route.ts deleted file mode 100644 index dfa2e7fc..00000000 --- a/src/app/api/search/route.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { ZOEKT_WEBSERVER_URL } from '@/lib/environment'; -import { createPathWithQueryParams } from '@/lib/utils'; -import { type NextRequest } from 'next/server' - -export async function GET(request: NextRequest) { - // @todo: proper error handling - const searchParams = request.nextUrl.searchParams; - const query = searchParams.get('query'); - const numResults = searchParams.get('numResults'); - - const url = createPathWithQueryParams( - `${ZOEKT_WEBSERVER_URL}/search`, - ["q", query], - ["num", numResults], - ["format", "json"], - ); - const res = await fetch(url); - const data = await res.json(); - - return Response.json({ ...data }) -} \ No newline at end of file diff --git a/src/app/api/source/route.ts b/src/app/api/source/route.ts deleted file mode 100644 index d2a4d1a2..00000000 --- a/src/app/api/source/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -"use server"; - -import { missingQueryParam } from "@/lib/serviceError"; -import { StatusCodes } from "http-status-codes"; -import { NextRequest } from "next/server"; -import { GetSourceResponse, pathQueryParamName, repoQueryParamName, ZoektPrintResponse } from "@/lib/types"; -import { ZOEKT_WEBSERVER_URL } from "@/lib/environment"; -import { createPathWithQueryParams } from "@/lib/utils"; - -/** - * Returns the content of a source file at the given path. - * - * Usage: - * GET /api/source?path=&repo= - */ -export async function GET(request: NextRequest) { - const searchParams = request.nextUrl.searchParams; - const filepath = searchParams.get(pathQueryParamName); - const repo = searchParams.get(repoQueryParamName); - - if (!filepath) { - return missingQueryParam(pathQueryParamName); - } - - if (!repo) { - return missingQueryParam(repoQueryParamName); - } - - const url = createPathWithQueryParams( - `${ZOEKT_WEBSERVER_URL}/print`, - ["f", filepath], - ["r", repo], - ["format", "json"], - ); - - const res = await fetch(url); - const data = await res.json() as ZoektPrintResponse; - - return Response.json( - { - content: data.Content, - encoding: data.Encoding, - } satisfies GetSourceResponse, - { - status: StatusCodes.OK - } - ); -} diff --git a/src/app/search/codePreview.tsx b/src/app/search/codePreviewPanel.tsx similarity index 83% rename from src/app/search/codePreview.tsx rename to src/app/search/codePreviewPanel.tsx index ab2443cd..e472909f 100644 --- a/src/app/search/codePreview.tsx +++ b/src/app/search/codePreviewPanel.tsx @@ -4,9 +4,11 @@ import { Button } from "@/components/ui/button"; import { ScrollArea } from "@/components/ui/scroll-area"; import { useExtensionWithDependency } from "@/hooks/useExtensionWithDependency"; import { useKeymapType } from "@/hooks/useKeymapType"; +import { useSyntaxHighlightingExtension } from "@/hooks/useSyntaxHighlightingExtension"; +import { useThemeNormalized } from "@/hooks/useThemeNormalized"; import { gutterWidthExtension } from "@/lib/extensions/gutterWidthExtension"; -import { markMatches, searchResultHighlightExtension } from "@/lib/extensions/searchResultHighlightExtension"; -import { ZoektMatch } from "@/lib/types"; +import { highlightRanges, searchResultHighlightExtension } from "@/lib/extensions/searchResultHighlightExtension"; +import { SearchResultFileMatch } from "@/lib/schemas"; import { defaultKeymap } from "@codemirror/commands"; import { javascript } from "@codemirror/lang-javascript"; import { search } from "@codemirror/search"; @@ -17,42 +19,33 @@ import { vim } from "@replit/codemirror-vim"; import CodeMirror, { ReactCodeMirrorRef } from '@uiw/react-codemirror'; import clsx from "clsx"; import { ArrowDown, ArrowUp } from "lucide-react"; -import { useTheme } from "next-themes"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; export interface CodePreviewFile { content: string; filepath: string; link?: string; - matches: ZoektMatch[]; + matches: SearchResultFileMatch[]; + language: string; } -interface CodePreviewProps { +interface CodePreviewPanelProps { file?: CodePreviewFile; selectedMatchIndex: number; onSelectedMatchIndexChange: (index: number) => void; onClose: () => void; } -export const CodePreview = ({ +export const CodePreviewPanel = ({ file, selectedMatchIndex, onSelectedMatchIndexChange, onClose, -}: CodePreviewProps) => { +}: CodePreviewPanelProps) => { const editorRef = useRef(null); - const { theme: _theme, systemTheme } = useTheme(); const [ keymapType ] = useKeymapType(); - - const theme = useMemo(() => { - if (_theme === "system") { - return systemTheme ?? "light"; - } - - return _theme ?? "light"; - }, [_theme, systemTheme]); - + const { theme } = useThemeNormalized(); const [gutterWidth, setGutterWidth] = useState(0); const keymapExtension = useExtensionWithDependency( @@ -68,11 +61,14 @@ export const CodePreview = ({ [keymapType] ); + const syntaxHighlighting = useSyntaxHighlightingExtension(file?.language ?? '', editorRef.current?.view); + const extensions = useMemo(() => { return [ keymapExtension, gutterWidthExtension, javascript(), + syntaxHighlighting, searchResultHighlightExtension(), search({ top: true, @@ -84,15 +80,25 @@ export const CodePreview = ({ } }), ]; - }, [keymapExtension]); + }, [keymapExtension, syntaxHighlighting]); + + const ranges = useMemo(() => { + if (!file || !file.matches.length) { + return []; + } + + return file.matches.flatMap((match) => { + return match.Ranges; + }) + }, [file]); useEffect(() => { if (!file || !editorRef.current?.view) { return; } - markMatches(selectedMatchIndex, file.matches, editorRef.current.view); - }, [file, file?.matches, selectedMatchIndex]); + highlightRanges(selectedMatchIndex, ranges, editorRef.current.view); + }, [ranges, selectedMatchIndex]); const onUpClicked = useCallback(() => { onSelectedMatchIndexChange(selectedMatchIndex - 1); @@ -126,7 +132,7 @@ export const CodePreview = ({
-

{`${selectedMatchIndex + 1} of ${file?.matches.length}`}

+

{`${selectedMatchIndex + 1} of ${ranges.length}`}

diff --git a/src/app/search/page.tsx b/src/app/search/page.tsx index c34a9dca..521a64dd 100644 --- a/src/app/search/page.tsx +++ b/src/app/search/page.tsx @@ -7,8 +7,7 @@ import { } from "@/components/ui/resizable"; import { Separator } from "@/components/ui/separator"; import { useNonEmptyQueryParam } from "@/hooks/useNonEmptyQueryParam"; -import { GetSourceResponse, pathQueryParamName, repoQueryParamName, ZoektFileMatch, ZoektSearchResponse } from "@/lib/types"; -import { createPathWithQueryParams, getCodeHostFilePreviewLink } from "@/lib/utils"; +import { getCodeHostFilePreviewLink } from "@/lib/utils"; import { SymbolIcon } from "@radix-ui/react-icons"; import { useQuery } from "@tanstack/react-query"; import Image from "next/image"; @@ -17,9 +16,11 @@ import logoDark from "../../../public/sb_logo_dark.png"; import logoLight from "../../../public/sb_logo_light.png"; import { SearchBar } from "../searchBar"; import { SettingsDropdown } from "../settingsDropdown"; -import { CodePreview, CodePreviewFile } from "./codePreview"; -import { SearchResults } from "./searchResults"; +import { CodePreviewPanel, CodePreviewFile } from "./codePreviewPanel"; +import { SearchResultsPanel } from "./searchResultsPanel"; import { useRouter } from "next/navigation"; +import { fetchFileSource, search } from "../api/(client)/client"; +import { SearchResultFile } from "@/lib/schemas"; export default function SearchPage() { const router = useRouter(); @@ -27,30 +28,28 @@ export default function SearchPage() { const numResults = useNonEmptyQueryParam("numResults") ?? "100"; const [selectedMatchIndex, setSelectedMatchIndex] = useState(0); - const [selectedFile, setSelectedFile] = useState(undefined); + const [selectedFile, setSelectedFile] = useState(undefined); const { data: searchResponse, isLoading } = useQuery({ queryKey: ["search", searchQuery, numResults], - queryFn: async (): Promise => { - console.log("Fetching search results"); - const result = await fetch(`/api/search?query=${searchQuery}&numResults=${numResults}`) - .then(response => response.json()); - console.log("Done"); - return result; - }, + queryFn: () => search({ + query: searchQuery, + numResults: parseInt(numResults), + }), enabled: searchQuery.length > 0, }); - const { fileMatches, searchDurationMs } = useMemo((): { fileMatches: ZoektFileMatch[], searchDurationMs: number } => { + const { fileMatches, searchDurationMs } = useMemo((): { fileMatches: SearchResultFile[], searchDurationMs: number } => { if (!searchResponse) { return { fileMatches: [], searchDurationMs: 0, }; } + return { - fileMatches: searchResponse.result.FileMatches ?? [], - searchDurationMs: Math.round(searchResponse.result.Stats.Duration / 1000000), + fileMatches: searchResponse.Result.Files ?? [], + searchDurationMs: Math.round(searchResponse.Result.Duration / 1000000), } }, [searchResponse]); @@ -100,7 +99,7 @@ export default function SearchPage() { {/* Search Results & Code Preview */} - { setSelectedFile(fileMatch); @@ -126,7 +125,7 @@ export default function SearchPage() { } interface CodePreviewWrapperProps { - fileMatch?: ZoektFileMatch; + fileMatch?: SearchResultFile; onClose: () => void; selectedMatchIndex: number; onSelectedMatchIndexChange: (index: number) => void; @@ -140,33 +139,25 @@ const CodePreviewWrapper = ({ }: CodePreviewWrapperProps) => { const { data: file } = useQuery({ - queryKey: ["source", fileMatch?.FileName, fileMatch?.Repo], + queryKey: ["source", fileMatch?.FileName, fileMatch?.Repository], queryFn: async (): Promise => { if (!fileMatch) { return undefined; } - const url = createPathWithQueryParams( - `/api/source`, - [pathQueryParamName, fileMatch.FileName], - [repoQueryParamName, fileMatch.Repo] - ); + return fetchFileSource(fileMatch.FileName, fileMatch.Repository) + .then(({ source }) => { + // @todo : refector this to use the templates provided by zoekt. + const link = getCodeHostFilePreviewLink(fileMatch.Repository, fileMatch.FileName) - return fetch(url) - .then(response => response.json()) - .then((body: GetSourceResponse) => { - if (body.encoding !== "base64") { - throw new Error("Expected base64 encoding"); - } - - const content = atob(body.content); - const link = getCodeHostFilePreviewLink(fileMatch.Repo, fileMatch.FileName) + const decodedSource = atob(source); return { - content, + content: decodedSource, filepath: fileMatch.FileName, - matches: fileMatch.Matches, + matches: fileMatch.ChunkMatches, link: link, + language: fileMatch.Language, }; }); }, @@ -174,7 +165,7 @@ const CodePreviewWrapper = ({ }); return ( - void; -} - -export const SearchResults = ({ - fileMatches, - onOpenFileMatch, -}: SearchResultsProps) => { - return ( - -
- {fileMatches.map((fileMatch, index) => ( - { - onOpenFileMatch(fileMatch, matchIndex); - }} - /> - ))} -
- - - ) -} - -interface FileMatchProps { - match: ZoektFileMatch; - onOpenFile: (matchIndex: number) => void; -} - -const FileMatch = ({ - match, - onOpenFile, -}: FileMatchProps) => { - - const [showAll, setShowAll] = useState(false); - const matchCount = useMemo(() => { - return match.Matches.length; - }, [match]); - - const matches = useMemo(() => { - const sortedMatches = match.Matches.sort((a, b) => { - return a.LineNum - b.LineNum; - }); - - if (!showAll) { - return sortedMatches.slice(0, MAX_MATCHES_TO_PREVIEW); - } - - return sortedMatches; - }, [match, showAll]); - - const { repoIcon, repoName, repoLink } = useMemo(() => { - const info = getRepoCodeHostInfo(match.Repo); - if (info) { - return { - repoName: info.repoName, - repoLink: info.repoLink, - repoIcon: {info.costHostName} - } - } - - return { - repoName: match.Repo, - repoLink: undefined, - repoIcon: - } - }, [match]); - - return ( -
-
- {repoIcon} - { - if (repoLink) { - window.open(repoLink, "_blank"); - } - }} - > - {repoName} - - · {match.FileName} -
- {matches.map((match, index) => { - const fragment = match.Fragments[0]; - - return ( -
{ - onOpenFile(index); - }} - > -

{match.LineNum > 0 ? match.LineNum : "file match"}: {fragment.Pre}{fragment.Match}{fragment.Post}

- -
- ); - })} - {matchCount > MAX_MATCHES_TO_PREVIEW && ( -
-

setShowAll(!showAll)} - className="text-blue-500 cursor-pointer text-sm flex flex-row items-center gap-2" - > - {showAll ? : } - {showAll ? `Show fewer matching lines` : `Show ${matchCount - MAX_MATCHES_TO_PREVIEW} more matching lines`} -

-
- )} -
- ); -} diff --git a/src/app/search/searchResultsPanel.tsx b/src/app/search/searchResultsPanel.tsx new file mode 100644 index 00000000..2f7b0570 --- /dev/null +++ b/src/app/search/searchResultsPanel.tsx @@ -0,0 +1,269 @@ +'use client'; + +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Separator } from "@/components/ui/separator"; +import { useExtensionWithDependency } from "@/hooks/useExtensionWithDependency"; +import { useSyntaxHighlightingExtension } from "@/hooks/useSyntaxHighlightingExtension"; +import { useThemeNormalized } from "@/hooks/useThemeNormalized"; +import { lineOffsetExtension } from "@/lib/extensions/lineOffsetExtension"; +import { SearchResultFile, SearchResultRange } from "@/lib/schemas"; +import { getRepoCodeHostInfo } from "@/lib/utils"; +import { DoubleArrowDownIcon, DoubleArrowUpIcon, FileIcon } from "@radix-ui/react-icons"; +import { Scrollbar } from "@radix-ui/react-scroll-area"; +import CodeMirror, { Decoration, DecorationSet, EditorState, EditorView, ReactCodeMirrorRef, StateField, Transaction } from '@uiw/react-codemirror'; +import clsx from "clsx"; +import Image from "next/image"; +import { useMemo, useRef, useState } from "react"; + +const MAX_MATCHES_TO_PREVIEW = 3; + +interface SearchResultsPanelProps { + fileMatches: SearchResultFile[]; + onOpenFileMatch: (fileMatch: SearchResultFile, matchIndex: number) => void; +} + +export const SearchResultsPanel = ({ + fileMatches, + onOpenFileMatch, +}: SearchResultsPanelProps) => { + return ( + + {fileMatches.map((fileMatch, index) => ( + { + onOpenFileMatch(fileMatch, matchIndex); + }} + /> + ))} + + + ) +} + +interface FilePreviewProps { + file: SearchResultFile; + onOpenFile: (matchIndex: number) => void; +} + +const FilePreview = ({ + file, + onOpenFile, +}: FilePreviewProps) => { + + const [showAll, setShowAll] = useState(false); + const matchCount = useMemo(() => { + return file.ChunkMatches.length; + }, [file]); + + const matches = useMemo(() => { + const sortedMatches = file.ChunkMatches.sort((a, b) => { + return a.ContentStart.LineNumber - b.ContentStart.LineNumber; + }); + + if (!showAll) { + return sortedMatches.slice(0, MAX_MATCHES_TO_PREVIEW); + } + + return sortedMatches; + }, [file, showAll]); + + const { repoIcon, repoName, repoLink } = useMemo(() => { + const info = getRepoCodeHostInfo(file.Repository); + if (info) { + return { + repoName: info.repoName, + repoLink: info.repoLink, + repoIcon: {info.costHostName} + } + } + + return { + repoName: file.Repository, + repoLink: undefined, + repoIcon: + } + }, [file]); + + const isMoreContentButtonVisible = useMemo(() => { + return matchCount > MAX_MATCHES_TO_PREVIEW; + }, [matchCount]); + + return ( +
+
+
+ {repoIcon} + { + if (repoLink) { + window.open(repoLink, "_blank"); + } + }} + > + {repoName} + + · {file.FileName} +
+
+ {matches.map((match, index) => { + const content = atob(match.Content); + + // If it's just the title, don't show a code preview + if (match.FileName) { + return null; + } + + const lineOffset = match.ContentStart.LineNumber - 1; + + return ( +
{ + const matchIndex = matches.slice(0, index).reduce((acc, match) => { + return acc + match.Ranges.length; + }, 0); + onOpenFile(matchIndex); + }} + > + + {(index !== matches.length - 1 || isMoreContentButtonVisible) && ( + + )} +
+ ); + })} + {isMoreContentButtonVisible && ( +
+

setShowAll(!showAll)} + className="text-blue-500 cursor-pointer text-sm flex flex-row items-center gap-2" + > + {showAll ? : } + {showAll ? `Show fewer matches` : `Show ${matchCount - MAX_MATCHES_TO_PREVIEW} more matches`} +

+
+ )} +
+ ); +} + +const markDecoration = Decoration.mark({ + class: "cm-searchMatch" +}); + +const cmTheme = EditorView.baseTheme({ + "&light .cm-searchMatch": { + border: "1px #6b7280ff", + }, + "&dark .cm-searchMatch": { + border: "1px #d1d5dbff", + }, +}); + +const CodePreview = ({ + content, + language, + ranges, + lineOffset, +}: { + content: string, + language: string, + ranges: SearchResultRange[], + lineOffset: number, +}) => { + const editorRef = useRef(null); + const { theme } = useThemeNormalized(); + + const syntaxHighlighting = useSyntaxHighlightingExtension(language, editorRef.current?.view); + + const rangeHighlighting = useExtensionWithDependency(editorRef.current?.view ?? null, () => { + return [ + StateField.define({ + create(editorState: EditorState) { + const document = editorState.doc; + + const decorations = ranges + .sort((a, b) => { + return a.Start.ByteOffset - b.Start.ByteOffset; + }) + .map(({ Start, End }) => { + const from = document.line(Start.LineNumber - lineOffset).from + Start.Column - 1; + const to = document.line(End.LineNumber - lineOffset).from + End.Column - 1; + return markDecoration.range(from, to); + }); + + return Decoration.set(decorations); + }, + update(highlights: DecorationSet, _transaction: Transaction) { + return highlights; + }, + provide: (field) => EditorView.decorations.from(field), + }), + cmTheme + ]; + }, [ranges]); + + const extensions = useMemo(() => { + return [ + syntaxHighlighting, + lineOffsetExtension(lineOffset), + rangeHighlighting, + ]; + }, [syntaxHighlighting, lineOffset, rangeHighlighting]); + + return ( + + ) +} diff --git a/src/hooks/useSyntaxHighlightingExtension.ts b/src/hooks/useSyntaxHighlightingExtension.ts new file mode 100644 index 00000000..5b0a1831 --- /dev/null +++ b/src/hooks/useSyntaxHighlightingExtension.ts @@ -0,0 +1,26 @@ +'use client'; + +import { EditorView } from "@codemirror/view"; +import { useExtensionWithDependency } from "./useExtensionWithDependency"; +import { javascript } from "@codemirror/lang-javascript"; + +export const useSyntaxHighlightingExtension = (language: string, view: EditorView | undefined) => { + const extension = useExtensionWithDependency( + view ?? null, + () => { + switch (language.toLowerCase()) { + case "typescript": + case "javascript": + return javascript({ + jsx: true, + typescript: true, + }); + default: + return []; + } + }, + [language] + ); + + return extension; +} \ No newline at end of file diff --git a/src/hooks/useThemeNormalized.ts b/src/hooks/useThemeNormalized.ts new file mode 100644 index 00000000..65008e83 --- /dev/null +++ b/src/hooks/useThemeNormalized.ts @@ -0,0 +1,22 @@ +'use client'; + +import { useTheme as useThemeBase } from "next-themes"; +import { useMemo } from "react"; + +export const useThemeNormalized = (defaultTheme: "light" | "dark" = "light") => { + const { theme: _theme, systemTheme, setTheme } = useThemeBase(); + + const theme = useMemo(() => { + if (_theme === "system") { + return systemTheme ?? defaultTheme; + } + + return _theme ?? defaultTheme; + }, [_theme, systemTheme]); + + return { + theme, + systemTheme, + setTheme, + }; +} \ No newline at end of file diff --git a/src/lib/environment.ts b/src/lib/environment.ts index 07b3fa5d..d7717f10 100644 --- a/src/lib/environment.ts +++ b/src/lib/environment.ts @@ -3,6 +3,12 @@ const getEnv = (env: string | undefined, defaultValue = '') => { return env ?? defaultValue; } +const getEnvNumber = (env: string | undefined, defaultValue: number = 0) => { + return Number(env) ?? defaultValue; +} + export const ZOEKT_WEBSERVER_URL = getEnv(process.env.ZOEKT_WEBSERVER_URL, "http://localhost:6070"); +export const SHARD_MAX_MATCH_COUNT = getEnvNumber(process.env.SHARD_MAX_MATCH_COUNT, 10000); +export const TOTAL_MAX_MATCH_COUNT = getEnvNumber(process.env.TOTAL_MAX_MATCH_COUNT, 100000); export const NODE_ENV = process.env.NODE_ENV; diff --git a/src/lib/errorCodes.ts b/src/lib/errorCodes.ts index d5f7b2e6..54c0f5d8 100644 --- a/src/lib/errorCodes.ts +++ b/src/lib/errorCodes.ts @@ -3,4 +3,5 @@ export enum ErrorCode { MISSING_REQUIRED_QUERY_PARAMETER = 'MISSING_REQUIRED_QUERY_PARAMETER', REPOSITORY_NOT_FOUND = 'REPOSITORY_NOT_FOUND', FILE_NOT_FOUND = 'FILE_NOT_FOUND', + INVALID_REQUEST_BODY = 'INVALID_REQUEST_BODY', } diff --git a/src/lib/extensions/lineOffsetExtension.ts b/src/lib/extensions/lineOffsetExtension.ts new file mode 100644 index 00000000..60f890e0 --- /dev/null +++ b/src/lib/extensions/lineOffsetExtension.ts @@ -0,0 +1,20 @@ +import { Compartment } from "@codemirror/state"; +import { lineNumbers } from "@codemirror/view"; + +const gutter = new Compartment(); + +/** + * Offsets the line numbers by the given amount + * @see: https://discuss.codemirror.net/t/codemirror-6-offset-line-numbers/2675/8 + */ +export const lineOffsetExtension = (lineOffset: number) => { + const lines = lineNumbers({ + formatNumber: (n) => { + return (n + lineOffset).toString(); + } + }); + + return [ + gutter.of(lines) + ] +} \ No newline at end of file diff --git a/src/lib/extensions/searchResultHighlightExtension.ts b/src/lib/extensions/searchResultHighlightExtension.ts index 8f2eac11..223d0285 100644 --- a/src/lib/extensions/searchResultHighlightExtension.ts +++ b/src/lib/extensions/searchResultHighlightExtension.ts @@ -1,24 +1,16 @@ import { EditorSelection, Extension, StateEffect, StateField, Text, Transaction } from "@codemirror/state"; import { Decoration, DecorationSet, EditorView } from "@codemirror/view"; -import { ZoektMatch } from "../types"; - -const matchMark = Decoration.mark({ - class: "tq-searchMatch" -}); -const selectedMatchMark = Decoration.mark({ - class: "tq-searchMatch-selected" -}); +import { SearchResultRange } from "../schemas"; const setMatchState = StateEffect.define<{ selectedMatchIndex: number, - matches: ZoektMatch[], + ranges: SearchResultRange[], }>(); -const getMatchRange = (match: ZoektMatch, document: Text) => { - const line = document.line(match.LineNum); - const fragment = match.Fragments[0]; - const from = line.from + fragment.Pre.length; - const to = from + fragment.Match.length; +const convertToCodeMirrorRange = (range: SearchResultRange, document: Text) => { + const { Start, End } = range; + const from = document.line(Start.LineNumber).from + Start.Column - 1; + const to = document.line(End.LineNumber).from + End.Column - 1; return { from, to }; } @@ -32,12 +24,14 @@ const matchHighlighter = StateField.define({ for (const effect of transaction.effects) { if (effect.is(setMatchState)) { - const { matches, selectedMatchIndex } = effect.value; + const { ranges, selectedMatchIndex } = effect.value; - const decorations = matches - .filter((match) => match.LineNum > 0) - .map((match, index) => { - const { from, to } = getMatchRange(match, transaction.newDoc); + const decorations = ranges + .sort((a, b) => { + return a.Start.ByteOffset - b.Start.ByteOffset; + }) + .map((range, index) => { + const { from, to } = convertToCodeMirrorRange(range, transaction.newDoc); const mark = index === selectedMatchIndex ? selectedMatchMark : matchMark; return mark.range(from, to); }); @@ -51,6 +45,13 @@ const matchHighlighter = StateField.define({ provide: (field) => EditorView.decorations.from(field), }); +const matchMark = Decoration.mark({ + class: "tq-searchMatch" +}); +const selectedMatchMark = Decoration.mark({ + class: "tq-searchMatch-selected" +}); + const highlightTheme = EditorView.baseTheme({ "&light .tq-searchMatch": { border: "1px dotted #6b7280ff", @@ -64,34 +65,27 @@ const highlightTheme = EditorView.baseTheme({ }, "&dark .tq-searchMatch-selected": { backgroundColor: "#00ff007a", - } }); -export const markMatches = (selectedMatchIndex: number, matches: ZoektMatch[], view: EditorView) => { +export const highlightRanges = (selectedMatchIndex: number, ranges: SearchResultRange[], view: EditorView) => { const setState = setMatchState.of({ selectedMatchIndex, - matches, + ranges, }); const effects = [] effects.push(setState); - if (selectedMatchIndex >= 0 && selectedMatchIndex < matches.length) { - const match = matches[selectedMatchIndex]; - - // Don't scroll if the match is on the filename. - if (match.LineNum > 0) { - const { from, to } = getMatchRange(match, view.state.doc); - const selection = EditorSelection.range(from, to); - effects.push(EditorView.scrollIntoView(selection, { - y: "start", - })); - } + if (selectedMatchIndex >= 0 && selectedMatchIndex < ranges.length) { + const { from, to } = convertToCodeMirrorRange(ranges[selectedMatchIndex], view.state.doc); + const selection = EditorSelection.range(from, to); + effects.push(EditorView.scrollIntoView(selection, { + y: "start", + })); }; view.dispatch({ effects }); - return true; } export const searchResultHighlightExtension = (): Extension => { diff --git a/src/lib/schemas.ts b/src/lib/schemas.ts new file mode 100644 index 00000000..af0b0961 --- /dev/null +++ b/src/lib/schemas.ts @@ -0,0 +1,68 @@ +import { z } from "zod"; + +export type SearchRequest = z.infer; +export const searchRequestSchema = z.object({ + query: z.string(), + numResults: z.number(), + whole: z.optional(z.boolean()), +}); + + +export type SearchResponse = z.infer; +export type SearchResult = SearchResponse["Result"]; +export type SearchResultFile = SearchResult["Files"][number]; +export type SearchResultFileMatch = SearchResultFile["ChunkMatches"][number]; +export type SearchResultRange = z.infer; +export type SearchResultLocation = z.infer; + +// @see : https://github.com/TaqlaAI/zoekt/blob/main/api.go#L212 +const locationSchema = z.object({ + // 0-based byte offset from the beginning of the file + ByteOffset: z.number(), + // 1-based line number from the beginning of the file + LineNumber: z.number(), + // 1-based column number (in runes) from the beginning of line + Column: z.number(), +}); + +const rangeSchema = z.object({ + Start: locationSchema, + End: locationSchema, +}); + +export const searchResponseSchema = z.object({ + Result: z.object({ + Duration: z.number(), + FileCount: z.number(), + Files: z.array(z.object({ + FileName: z.string(), + Repository: z.string(), + Version: z.string(), + Language: z.string(), + Branches: z.array(z.string()), + ChunkMatches: z.array(z.object({ + Content: z.string(), + Ranges: z.array(rangeSchema), + FileName: z.boolean(), + ContentStart: locationSchema, + Score: z.number(), + })), + Checksum: z.string(), + Score: z.number(), + // Set if `whole` is true. + Content: z.optional(z.string()), + })), + }), +}); + +export type FileSourceRequest = z.infer; +export const fileSourceRequestSchema = z.object({ + fileName: z.string(), + repository: z.string() +}); + +export type FileSourceResponse = z.infer; + +export const fileSourceResponseSchema = z.object({ + source: z.string(), +}); \ No newline at end of file diff --git a/src/lib/server/searchService.ts b/src/lib/server/searchService.ts new file mode 100644 index 00000000..636b4894 --- /dev/null +++ b/src/lib/server/searchService.ts @@ -0,0 +1,56 @@ +import { SHARD_MAX_MATCH_COUNT, TOTAL_MAX_MATCH_COUNT } from "../environment"; +import { FileSourceRequest, FileSourceResponse, SearchRequest, SearchResponse, searchResponseSchema } from "../schemas"; +import { fileNotFound, invalidZoektResponse, ServiceError } from "../serviceError"; +import { isServiceError } from "../utils"; +import { zoektFetch } from "./zoektClient"; + +export const search = async ({ query, numResults, whole }: SearchRequest): Promise => { + const body = JSON.stringify({ + q: query, + // @see: https://github.com/TaqlaAI/zoekt/blob/main/api.go#L892 + opts: { + NumContextLines: 2, + ChunkMatches: true, + MaxMatchDisplayCount: numResults, + Whole: !!whole, + ShardMaxMatchCount: SHARD_MAX_MATCH_COUNT, + TotalMaxMatchCount: TOTAL_MAX_MATCH_COUNT, + } + }); + + const searchResponse = await zoektFetch({ + path: "/api/search", + body, + method: "POST", + }); + + if (!searchResponse.ok) { + return invalidZoektResponse(searchResponse); + } + + const searchBody = await searchResponse.json(); + return searchResponseSchema.parse(searchBody); +} + +export const getFileSource = async ({ fileName, repository }: FileSourceRequest): Promise => { + const searchResponse = await search({ + query: `${fileName} repo:${repository}`, + numResults: 1, + whole: true, + }); + + if (isServiceError(searchResponse)) { + return searchResponse; + } + + const files = searchResponse.Result.Files; + + if (files.length === 0) { + return fileNotFound(fileName, repository); + } + + const source = files[0].Content ?? ''; + return { + source + } +} \ No newline at end of file diff --git a/src/lib/server/zoektClient.ts b/src/lib/server/zoektClient.ts new file mode 100644 index 00000000..55dd9230 --- /dev/null +++ b/src/lib/server/zoektClient.ts @@ -0,0 +1,33 @@ +import { ZOEKT_WEBSERVER_URL } from "../environment" + + +interface ZoektRequest { + path: string, + body: string, + method: string, +} + +export const zoektFetch = async ({ + path, + body, + method, +}: ZoektRequest) => { + const start = Date.now(); + + const response = await fetch( + new URL(path, ZOEKT_WEBSERVER_URL), + { + method, + headers: { + "Content-Type": "application/json", + }, + body, + } + ); + + const duration = Date.now() - start; + console.log(`[zoektClient] ${method} ${path} ${response.status} ${duration}ms`); + // @todo : add metrics + + return response; +} \ No newline at end of file diff --git a/src/lib/serviceError.ts b/src/lib/serviceError.ts index 5b926da6..d7a413da 100644 --- a/src/lib/serviceError.ts +++ b/src/lib/serviceError.ts @@ -1,13 +1,14 @@ import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "./errorCodes"; +import { ZodError } from "zod"; -export interface ServiceErrorArgs { +export interface ServiceError { statusCode: StatusCodes; errorCode: ErrorCode; message: string; } -export const serviceError = ({ statusCode, errorCode, message }: ServiceErrorArgs) => { +export const serviceErrorResponse = ({ statusCode, errorCode, message }: ServiceError) => { return Response.json({ statusCode, errorCode, @@ -17,10 +18,45 @@ export const serviceError = ({ statusCode, errorCode, message }: ServiceErrorArg }); } -export const missingQueryParam = (name: string) => { - return serviceError({ +export const missingQueryParam = (name: string): ServiceError => { + return { statusCode: StatusCodes.BAD_REQUEST, errorCode: ErrorCode.MISSING_REQUIRED_QUERY_PARAMETER, message: `Missing required query parameter: ${name}`, - }); + }; +} + +export const schemaValidationError = (error: ZodError): ServiceError => { + return { + statusCode: StatusCodes.BAD_REQUEST, + errorCode: ErrorCode.INVALID_REQUEST_BODY, + message: `Schema validation failed with: ${error.message}`, + }; +} + +export const invalidZoektResponse = async (zoektResponse: Response): Promise => { + const zoektMessage = await (async () => { + try { + const zoektResponseBody = await zoektResponse.json(); + if (zoektResponseBody.Error) { + return zoektResponseBody.Error; + } + } catch (_e) { + return "Unknown error"; + } + })(); + + return { + statusCode: StatusCodes.INTERNAL_SERVER_ERROR, + errorCode: ErrorCode.INVALID_REQUEST_BODY, + message: `Zoekt request failed with status code ${zoektResponse.status} and message: "${zoektMessage}"`, + }; +} + +export const fileNotFound = async (fileName: string, repository: string): Promise => { + return { + statusCode: StatusCodes.NOT_FOUND, + errorCode: ErrorCode.FILE_NOT_FOUND, + message: `File "${fileName}" not found in repository "${repository}"`, + }; } \ No newline at end of file diff --git a/src/lib/types.ts b/src/lib/types.ts index 3b8b9a25..f137e381 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -1,48 +1,4 @@ - - export const pathQueryParamName = "path"; export const repoQueryParamName = "repo"; -export type GetSourceResponse = { - content: string; - encoding: string; -} - -export interface ZoektMatch { - URL: string, - FileName: string, - LineNum: number, - Fragments: { - Pre: string, - Match: string, - Post: string - }[] -} - -export interface ZoektFileMatch { - FileName: string, - Repo: string, - Language: string, - Matches: ZoektMatch[], - URL: string, -} - -export interface ZoektResult { - QueryStr: string, - FileMatches: ZoektFileMatch[] | null, - Stats: { - // Duration in nanoseconds - Duration: number, - } -} - -export interface ZoektSearchResponse { - result: ZoektResult, -} - -export interface ZoektPrintResponse { - Content: string, - Encoding: string, -} - export type KeymapType = "default" | "vim"; \ No newline at end of file diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 2c90833d..ed507ed3 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -2,6 +2,7 @@ import { type ClassValue, clsx } from "clsx" import { twMerge } from "tailwind-merge" import githubLogo from "../../public/github.svg"; import gitlabLogo from "../../public/gitlab.svg"; +import { ServiceError } from "./serviceError"; export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)) @@ -71,4 +72,12 @@ export const getCodeHostFilePreviewLink = (repoName: string, filePath: string): } return undefined; -} \ No newline at end of file +} + +export const isServiceError = (data: unknown): data is ServiceError => { + return typeof data === 'object' && + data !== null && + 'statusCode' in data && + 'errorCode' in data && + 'message' in data; +} diff --git a/supervisord.conf b/supervisord.conf index c947b513..ee3b6d8c 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -13,7 +13,7 @@ stdout_logfile_maxbytes=0 redirect_stderr=true [program:zoekt-webserver] -command=zoekt-webserver -index %(ENV_DATA_CACHE_DIR)s/index +command=zoekt-webserver -index %(ENV_DATA_CACHE_DIR)s/index -rpc autostart=true autorestart=true startretries=3