diff --git a/.env.development b/.env.development index 330a591d..57d75115 100644 --- a/.env.development +++ b/.env.development @@ -21,7 +21,7 @@ AUTH_URL="http://localhost:3000" DATA_CACHE_DIR=${PWD}/.sourcebot # Path to the sourcebot cache dir (ex. ~/sourcebot/.sourcebot) SOURCEBOT_PUBLIC_KEY_PATH=${PWD}/public.pem -# CONFIG_PATH=${PWD}/config.json # Path to the sourcebot config file (if one exists) +CONFIG_PATH=${PWD}/config.json # Path to the sourcebot config file (if one exists) # Email # EMAIL_FROM_ADDRESS="" # The from address for transactional emails. diff --git a/CHANGELOG.md b/CHANGELOG.md index f066f12c..d915f327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] + + ### Added - Added `ALWAYS_INDEX_FILE_PATTERNS` environment variable to allow specifying a comma seperated list of glob patterns matching file paths that should always be indexed, regardless of size or # of trigrams. [#631](https://github.com/sourcebot-dev/sourcebot/pull/631) - Added button to explore menu to toggle cross-repository search. [#647](https://github.com/sourcebot-dev/sourcebot/pull/647) +- Added server side telemetry for search metrics. [#652](https://github.com/sourcebot-dev/sourcebot/pull/652) ### Fixed - Fixed issue where single quotes could not be used in search queries. [#629](https://github.com/sourcebot-dev/sourcebot/pull/629) diff --git a/packages/mcp/src/client.ts b/packages/mcp/src/client.ts index 3754c605..fdb3440e 100644 --- a/packages/mcp/src/client.ts +++ b/packages/mcp/src/client.ts @@ -8,7 +8,6 @@ export const search = async (request: SearchRequest): Promise response.json()); @@ -43,7 +41,6 @@ export const getFileSource = async (request: FileSourceRequest): Promise { if (isServiceError(data)) { @@ -75,6 +76,7 @@ export const useSuggestionsData = ({ query: `sym:${suggestionQuery.length > 0 ? suggestionQuery : ".*"}`, matches: 15, contextLines: 1, + source: 'search-bar-symbol-suggestions' }), select: (data): Suggestion[] => { if (isServiceError(data)) { diff --git a/packages/web/src/app/[domain]/search/useStreamedSearch.ts b/packages/web/src/app/[domain]/search/useStreamedSearch.ts index b4f079fd..181b8a62 100644 --- a/packages/web/src/app/[domain]/search/useStreamedSearch.ts +++ b/packages/web/src/app/[domain]/search/useStreamedSearch.ts @@ -129,7 +129,8 @@ export const useStreamedSearch = ({ query, matches, contextLines, whole, isRegex whole, isRegexEnabled, isCaseSensitivityEnabled, - }), + source: 'sourcebot-web-client' + } satisfies SearchRequest), signal: abortControllerRef.current.signal, }); diff --git a/packages/web/src/app/api/(server)/search/route.ts b/packages/web/src/app/api/(server)/search/route.ts index 92ba4f2a..e215d3c3 100644 --- a/packages/web/src/app/api/(server)/search/route.ts +++ b/packages/web/src/app/api/(server)/search/route.ts @@ -4,6 +4,7 @@ import { search, searchRequestSchema } from "@/features/search"; import { isServiceError } from "@/lib/utils"; import { NextRequest } from "next/server"; import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"; +import { captureEvent } from "@/lib/posthog"; export const POST = async (request: NextRequest) => { const body = await request.json(); @@ -16,8 +17,14 @@ export const POST = async (request: NextRequest) => { const { query, + source, ...options } = parsed.data; + + await captureEvent('api_code_search_request', { + source: source ?? 'unknown', + type: 'blocking', + }); const response = await search({ queryType: 'string', @@ -28,5 +35,6 @@ export const POST = async (request: NextRequest) => { if (isServiceError(response)) { return serviceErrorResponse(response); } + return Response.json(response); } \ No newline at end of file diff --git a/packages/web/src/app/api/(server)/stream_search/route.ts b/packages/web/src/app/api/(server)/stream_search/route.ts index 03057978..47f1426b 100644 --- a/packages/web/src/app/api/(server)/stream_search/route.ts +++ b/packages/web/src/app/api/(server)/stream_search/route.ts @@ -1,6 +1,7 @@ 'use server'; import { streamSearch, searchRequestSchema } from '@/features/search'; +import { captureEvent } from '@/lib/posthog'; import { schemaValidationError, serviceErrorResponse } from '@/lib/serviceError'; import { isServiceError } from '@/lib/utils'; import { NextRequest } from 'next/server'; @@ -15,9 +16,15 @@ export const POST = async (request: NextRequest) => { const { query, + source, ...options } = parsed.data; + await captureEvent('api_code_search_request', { + source: source ?? 'unknown', + type: 'streamed', + }); + const stream = await streamSearch({ queryType: 'string', query, diff --git a/packages/web/src/features/chat/components/chatBox/useSuggestionsData.ts b/packages/web/src/features/chat/components/chatBox/useSuggestionsData.ts index c1aeff8b..d41367d5 100644 --- a/packages/web/src/features/chat/components/chatBox/useSuggestionsData.ts +++ b/packages/web/src/features/chat/components/chatBox/useSuggestionsData.ts @@ -47,6 +47,7 @@ export const useSuggestionsData = ({ query: query.join(' '), matches: 10, contextLines: 1, + source: 'chat-file-suggestions' })) }, select: (data): FileSuggestion[] => { diff --git a/packages/web/src/features/search/searchApi.ts b/packages/web/src/features/search/searchApi.ts index cf362c86..ff2fb0da 100644 --- a/packages/web/src/features/search/searchApi.ts +++ b/packages/web/src/features/search/searchApi.ts @@ -2,13 +2,12 @@ import { sew } from "@/actions"; import { getRepoPermissionFilterForUser } from "@/prisma"; import { withOptionalAuthV2 } from "@/withAuthV2"; import { PrismaClient, UserWithAccounts } from "@sourcebot/db"; -import { createLogger, env, hasEntitlement } from "@sourcebot/shared"; +import { env, hasEntitlement } from "@sourcebot/shared"; import { QueryIR } from './ir'; import { parseQuerySyntaxIntoIR } from './parser'; import { SearchOptions } from "./types"; import { createZoektSearchRequest, zoektSearch, zoektStreamSearch } from './zoektSearcher'; -const logger = createLogger("searchApi"); type QueryStringSearchRequest = { queryType: 'string'; diff --git a/packages/web/src/features/search/types.ts b/packages/web/src/features/search/types.ts index 90f50182..c90cfdd1 100644 --- a/packages/web/src/features/search/types.ts +++ b/packages/web/src/features/search/types.ts @@ -94,6 +94,7 @@ export type SearchOptions = z.infer; export const searchRequestSchema = z.object({ query: z.string(), // The zoekt query to execute. + source: z.string().optional(), // The source of the search request. ...searchOptionsSchema.shape, }); export type SearchRequest = z.infer; diff --git a/packages/web/src/lib/posthog.ts b/packages/web/src/lib/posthog.ts new file mode 100644 index 00000000..916151ca --- /dev/null +++ b/packages/web/src/lib/posthog.ts @@ -0,0 +1,65 @@ +import { PostHog } from 'posthog-node' +import { env } from '@sourcebot/shared' +import { RequestCookies } from 'next/dist/compiled/@edge-runtime/cookies'; +import * as Sentry from "@sentry/nextjs"; +import { PosthogEvent, PosthogEventMap } from './posthogEvents'; +import { cookies } from 'next/headers'; + +/** + * @note: This is a subset of the properties stored in the + * ph_phc__posthog cookie. + */ +export type PostHogCookie = { + distinct_id: string; +} + +const isPostHogCookie = (cookie: unknown): cookie is PostHogCookie => { + return typeof cookie === 'object' && + cookie !== null && + 'distinct_id' in cookie; +} + +/** +* Attempts to retrieve the PostHog cookie from the given cookie store, returning +* undefined if the cookie is not found or is invalid. +*/ +const getPostHogCookie = (cookieStore: Pick): PostHogCookie | undefined => { + const phCookieKey = `ph_${env.POSTHOG_PAPIK}_posthog`; + const cookie = cookieStore.get(phCookieKey); + + if (!cookie) { + return undefined; + } + + const parsedCookie = (() => { + try { + return JSON.parse(cookie.value); + } catch (e) { + Sentry.captureException(e); + return null; + } + })(); + + if (isPostHogCookie(parsedCookie)) { + return parsedCookie; + } + + return undefined; +} + +export async function captureEvent(event: E, properties: PosthogEventMap[E]) { + const cookieStore = await cookies(); + const cookie = getPostHogCookie(cookieStore); + + const posthog = new PostHog(env.POSTHOG_PAPIK, { + host: 'https://us.i.posthog.com', + flushAt: 1, + flushInterval: 0 + }) + + posthog.capture({ + event, + properties, + distinctId: cookie?.distinct_id ?? '', + }); +} \ No newline at end of file diff --git a/packages/web/src/lib/posthogEvents.ts b/packages/web/src/lib/posthogEvents.ts index 6b340fe9..0e64c509 100644 --- a/packages/web/src/lib/posthogEvents.ts +++ b/packages/web/src/lib/posthogEvents.ts @@ -313,6 +313,11 @@ export type PosthogEventMap = { durationMs: number, // Whether or not the user is searching all repositories. isGlobalSearchEnabled: boolean, - } + }, + ////////////////////////////////////////////////////////////////// + api_code_search_request: { + source: string; + type: 'streamed' | 'blocking'; + }, } export type PosthogEvent = keyof PosthogEventMap; \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index d17ab7e1..3b4b44f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4599,6 +4599,15 @@ __metadata: languageName: node linkType: hard +"@posthog/core@npm:1.6.0": + version: 1.6.0 + resolution: "@posthog/core@npm:1.6.0" + dependencies: + cross-spawn: "npm:^7.0.6" + checksum: 10c0/28aa907bb21b18587bc5f47c44349ebc834b37d9a4cedb1a18d7b673d4d7cdad2120dda80deceaee707b2f52333e9a08a8e591e1fc4066521ce05e820b76309f + languageName: node + linkType: hard + "@prisma/client@npm:6.2.1": version: 6.2.1 resolution: "@prisma/client@npm:6.2.1" @@ -8245,6 +8254,7 @@ __metadata: parse-diff: "npm:^0.11.1" postcss: "npm:^8" posthog-js: "npm:^1.161.5" + posthog-node: "npm:^5.15.0" pretty-bytes: "npm:^6.1.1" psl: "npm:^1.15.0" react: "npm:^19.2.1" @@ -17235,6 +17245,15 @@ __metadata: languageName: node linkType: hard +"posthog-node@npm:^5.15.0": + version: 5.15.0 + resolution: "posthog-node@npm:5.15.0" + dependencies: + "@posthog/core": "npm:1.6.0" + checksum: 10c0/7db929453cefc9b2d0017e1f7c9ffe7e6ecd2a495ee9861bd5e8f3873f72fa29a318dd7bccf10eb15639e1860aab396d4be502af88afba96ed15ac8b46d57e9d + languageName: node + linkType: hard + "preact-render-to-string@npm:6.5.11": version: 6.5.11 resolution: "preact-render-to-string@npm:6.5.11"