From 46544dc1d836a3f494ac43d5d11d9daba9f26817 Mon Sep 17 00:00:00 2001 From: bkellam Date: Tue, 2 Dec 2025 19:24:01 -0800 Subject: [PATCH] wip on ondemand indexing --- packages/backend/src/api.ts | 35 ++++++ packages/backend/src/repoCompileUtils.ts | 116 +++++++++++------- .../web/src/app/askgh/[owner]/[repo]/page.tsx | 16 +++ .../web/src/features/workerApi/actions.ts | 17 ++- packages/web/src/middleware.ts | 35 ------ 5 files changed, 137 insertions(+), 82 deletions(-) create mode 100644 packages/web/src/app/askgh/[owner]/[repo]/page.tsx delete mode 100644 packages/web/src/middleware.ts diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index 5c7e2547..3d67247c 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -7,6 +7,8 @@ import z from 'zod'; import { ConnectionManager } from './connectionManager.js'; import { PromClient } from './promClient.js'; import { RepoIndexManager } from './repoIndexManager.js'; +import { createGitHubRepoRecord } from './repoCompileUtils.js'; +import { Octokit } from '@octokit/rest'; const logger = createLogger('api'); const PORT = 3060; @@ -33,6 +35,7 @@ export class Api { app.post('/api/sync-connection', this.syncConnection.bind(this)); app.post('/api/index-repo', this.indexRepo.bind(this)); + app.post(`/api/experimental/add-github-repo`, this.addGithubRepo.bind(this)); this.server = app.listen(PORT, () => { logger.info(`API server is running on port ${PORT}`); @@ -92,6 +95,38 @@ export class Api { res.status(200).json({ jobId }); } + private async addGithubRepo(req: Request, res: Response) { + const schema = z.object({ + owner: z.string(), + repo: z.string(), + }).strict(); + + const parsed = schema.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: parsed.error.message }); + return; + } + + const octokit = new Octokit(); + const response = await octokit.rest.repos.get({ + owner: parsed.data.owner, + repo: parsed.data.repo, + }); + + const record = createGitHubRepoRecord({ + repo: response.data, + hostUrl: 'https://github.com', + }); + + const repo = await this.prisma.repo.create({ + data: record, + }); + + const [jobId ] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); + + res.status(200).json({ jobId }); + } + public async dispose() { return new Promise((resolve, reject) => { this.server.close((err) => { diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 5b2c0349..3ea1f533 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -1,5 +1,5 @@ import { GithubConnectionConfig } from '@sourcebot/schemas/v3/github.type'; -import { getGitHubReposFromConfig } from "./github.js"; +import { getGitHubReposFromConfig, OctokitRepository } from "./github.js"; import { getGitLabReposFromConfig } from "./gitlab.js"; import { getGiteaReposFromConfig } from "./gitea.js"; import { getGerritReposFromConfig } from "./gerrit.js"; @@ -45,60 +45,23 @@ export const compileGithubConfig = async ( const warnings = gitHubReposResult.warnings; const hostUrl = config.url ?? 'https://github.com'; - const repoNameRoot = new URL(hostUrl) - .toString() - .replace(/^https?:\/\//, ''); const repos = gitHubRepos.map((repo) => { - const repoDisplayName = repo.full_name; - const repoName = path.join(repoNameRoot, repoDisplayName); - const cloneUrl = new URL(repo.clone_url!); - const isPublic = repo.private === false; + const record = createGitHubRepoRecord({ + repo, + hostUrl, + branches: config.revisions?.branches ?? undefined, + tags: config.revisions?.tags ?? undefined, + }) - logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`); - - const record: RepoData = { - external_id: repo.id.toString(), - external_codeHostType: 'github', - external_codeHostUrl: hostUrl, - cloneUrl: cloneUrl.toString(), - webUrl: repo.html_url, - name: repoName, - displayName: repoDisplayName, - imageUrl: repo.owner.avatar_url, - isFork: repo.fork, - isArchived: !!repo.archived, - isPublic: isPublic, - org: { - connect: { - id: SINGLE_TENANT_ORG_ID, - }, - }, + return { + ...record, connections: { create: { connectionId: connectionId, } }, - metadata: { - gitConfig: { - 'zoekt.web-url-type': 'github', - 'zoekt.web-url': repo.html_url, - 'zoekt.name': repoName, - 'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(), - 'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(), - 'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(), - 'zoekt.github-forks': (repo.forks_count ?? 0).toString(), - 'zoekt.archived': marshalBool(repo.archived), - 'zoekt.fork': marshalBool(repo.fork), - 'zoekt.public': marshalBool(isPublic), - 'zoekt.display-name': repoDisplayName, - }, - branches: config.revisions?.branches ?? undefined, - tags: config.revisions?.tags ?? undefined, - } satisfies RepoMetadata, }; - - return record; }) return { @@ -107,6 +70,67 @@ export const compileGithubConfig = async ( }; } +export const createGitHubRepoRecord = ({ + repo, + hostUrl, + branches, + tags, +}: { + repo: OctokitRepository, + hostUrl: string, + branches?: string[], + tags?: string[], +}) => { + const repoNameRoot = new URL(hostUrl) + .toString() + .replace(/^https?:\/\//, ''); + + const repoDisplayName = repo.full_name; + const repoName = path.join(repoNameRoot, repoDisplayName); + const cloneUrl = new URL(repo.clone_url!); + const isPublic = repo.private === false; + + logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`); + + const record: Prisma.RepoCreateInput = { + external_id: repo.id.toString(), + external_codeHostType: 'github', + external_codeHostUrl: hostUrl, + cloneUrl: cloneUrl.toString(), + webUrl: repo.html_url, + name: repoName, + displayName: repoDisplayName, + imageUrl: repo.owner.avatar_url, + isFork: repo.fork, + isArchived: !!repo.archived, + isPublic: isPublic, + org: { + connect: { + id: SINGLE_TENANT_ORG_ID, + }, + }, + metadata: { + gitConfig: { + 'zoekt.web-url-type': 'github', + 'zoekt.web-url': repo.html_url, + 'zoekt.name': repoName, + 'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(), + 'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(), + 'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(), + 'zoekt.github-forks': (repo.forks_count ?? 0).toString(), + 'zoekt.archived': marshalBool(repo.archived), + 'zoekt.fork': marshalBool(repo.fork), + 'zoekt.public': marshalBool(isPublic), + 'zoekt.display-name': repoDisplayName, + }, + branches, + tags, + } satisfies RepoMetadata, + }; + + return record; +} + export const compileGitlabConfig = async ( config: GitlabConnectionConfig, connectionId: number): Promise => { diff --git a/packages/web/src/app/askgh/[owner]/[repo]/page.tsx b/packages/web/src/app/askgh/[owner]/[repo]/page.tsx new file mode 100644 index 00000000..b4e94750 --- /dev/null +++ b/packages/web/src/app/askgh/[owner]/[repo]/page.tsx @@ -0,0 +1,16 @@ +import { addGithubRepo } from "@/features/workerApi/actions"; + +interface PageProps { + params: Promise<{ owner: string; repo: string }>; +} + +export default async function GitHubRepoPage(props: PageProps) { + const params = await props.params; + const { owner, repo } = params; + + const response = await addGithubRepo(owner, repo); + + return

{JSON.stringify(response, null, 2)}

; +} + + diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts index a9f1fc46..69e6764c 100644 --- a/packages/web/src/features/workerApi/actions.ts +++ b/packages/web/src/features/workerApi/actions.ts @@ -2,7 +2,7 @@ import { sew } from "@/actions"; import { unexpectedError } from "@/lib/serviceError"; -import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2"; +import { withAuthV2, withMinimumOrgRole, withOptionalAuthV2 } from "@/withAuthV2"; import { OrgRole } from "@sourcebot/db"; import z from "zod"; @@ -57,3 +57,18 @@ export const indexRepo = async (repoId: number) => sew(() => }) ) ); + +export const addGithubRepo = async (owner: string, repo: string) => sew(() => + withOptionalAuthV2(async () => { + const response = await fetch(`${WORKER_API_URL}/api/experimental/add-github-repo`, { + method: 'POST', + body: JSON.stringify({ owner, repo }), + headers: { + 'Content-Type': 'application/json', + }, + }); + + const data = await response.json(); + return data; + }) +); \ No newline at end of file diff --git a/packages/web/src/middleware.ts b/packages/web/src/middleware.ts deleted file mode 100644 index b59e207d..00000000 --- a/packages/web/src/middleware.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { NextResponse } from 'next/server' -import type { NextRequest } from 'next/server' -import { SINGLE_TENANT_ORG_DOMAIN } from '@/lib/constants' - -export async function middleware(request: NextRequest) { - const url = request.nextUrl.clone(); - - if ( - url.pathname.startsWith('/login') || - url.pathname.startsWith('/redeem') || - url.pathname.startsWith('/signup') || - url.pathname.startsWith('/invite') || - url.pathname.startsWith('/onboard') - ) { - return NextResponse.next(); - } - - const pathSegments = url.pathname.split('/').filter(Boolean); - const currentDomain = pathSegments[0]; - - // If we're already on the correct domain path, allow - if (currentDomain === SINGLE_TENANT_ORG_DOMAIN) { - return NextResponse.next(); - } - - url.pathname = `/${SINGLE_TENANT_ORG_DOMAIN}${pathSegments.length > 1 ? '/' + pathSegments.slice(1).join('/') : ''}`; - return NextResponse.redirect(url); -} - -export const config = { - // https://nextjs.org/docs/app/building-your-application/routing/middleware#matcher - matcher: [ - '/((?!api|_next/static|ingest|_next/image|favicon.ico|sitemap.xml|robots.txt|manifest.json|logo_192.png|logo_512.png|sb_logo_light_large.png|arrow.png|placeholder_avatar.png|sb_logo_dark_small.png|sb_logo_light_small.png).*)', - ], -}