wip on ondemand indexing

packages/backend/src/api.ts

@@ -7,6 +7,8 @@ import z from 'zod';
 import { ConnectionManager } from './connectionManager.js';
 import { PromClient } from './promClient.js';
 import { RepoIndexManager } from './repoIndexManager.js';
+import { createGitHubRepoRecord } from './repoCompileUtils.js';
+import { Octokit } from '@octokit/rest';
 
 const logger = createLogger('api');
 const PORT = 3060;
@@ -33,6 +35,7 @@ export class Api {
 
         app.post('/api/sync-connection', this.syncConnection.bind(this));
         app.post('/api/index-repo', this.indexRepo.bind(this));
+        app.post(`/api/experimental/add-github-repo`, this.addGithubRepo.bind(this));
 
         this.server = app.listen(PORT, () => {
             logger.info(`API server is running on port ${PORT}`);
@@ -92,6 +95,38 @@ export class Api {
         res.status(200).json({ jobId });
     }
 
+    private async addGithubRepo(req: Request, res: Response) {
+        const schema = z.object({
+            owner: z.string(),
+            repo: z.string(),
+        }).strict();
+
+        const parsed = schema.safeParse(req.body);
+        if (!parsed.success) {
+            res.status(400).json({ error: parsed.error.message });
+            return;
+        }
+
+        const octokit = new Octokit();
+        const response = await octokit.rest.repos.get({
+            owner: parsed.data.owner,
+            repo: parsed.data.repo,
+        });
+
+        const record = createGitHubRepoRecord({
+            repo: response.data,
+            hostUrl: 'https://github.com',
+        });
+
+        const repo = await this.prisma.repo.create({
+            data: record,
+        });
+
+        const [jobId ] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX);
+
+        res.status(200).json({ jobId });
+    }
+
     public async dispose() {
         return new Promise<void>((resolve, reject) => {
             this.server.close((err) => {

packages/backend/src/repoCompileUtils.ts

@@ -1,5 +1,5 @@
 import { GithubConnectionConfig } from '@sourcebot/schemas/v3/github.type';
-import { getGitHubReposFromConfig } from "./github.js";
+import { getGitHubReposFromConfig, OctokitRepository } from "./github.js";
 import { getGitLabReposFromConfig } from "./gitlab.js";
 import { getGiteaReposFromConfig } from "./gitea.js";
 import { getGerritReposFromConfig } from "./gerrit.js";
@@ -45,11 +45,46 @@ export const compileGithubConfig = async (
     const warnings = gitHubReposResult.warnings;
 
     const hostUrl = config.url ?? 'https://github.com';
+
+    const repos = gitHubRepos.map((repo) => {
+        const record = createGitHubRepoRecord({
+            repo,
+            hostUrl,
+            branches: config.revisions?.branches ?? undefined,
+            tags: config.revisions?.tags ?? undefined,
+        })
+
+        return {
+            ...record,
+            connections: {
+                create: {
+                    connectionId: connectionId,
+                }
+            },
+        };
+    })
+
+    return {
+        repoData: repos,
+        warnings,
+    };
+}
+
+export const createGitHubRepoRecord = ({
+    repo,
+    hostUrl,
+    branches,
+    tags,
+}: {
+    repo: OctokitRepository,
+    hostUrl: string,
+    branches?: string[],
+    tags?: string[],
+}) => {
     const repoNameRoot = new URL(hostUrl)
         .toString()
         .replace(/^https?:\/\//, '');
 
-    const repos = gitHubRepos.map((repo) => {
     const repoDisplayName = repo.full_name;
     const repoName = path.join(repoNameRoot, repoDisplayName);
     const cloneUrl = new URL(repo.clone_url!);
@@ -57,7 +92,7 @@ export const compileGithubConfig = async (
 
     logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
 
-        const record: RepoData = {
+    const record: Prisma.RepoCreateInput = {
         external_id: repo.id.toString(),
         external_codeHostType: 'github',
         external_codeHostUrl: hostUrl,
@@ -74,11 +109,6 @@ export const compileGithubConfig = async (
                 id: SINGLE_TENANT_ORG_ID,
             },
         },
-            connections: {
-                create: {
-                    connectionId: connectionId,
-                }
-            },
         metadata: {
             gitConfig: {
                 'zoekt.web-url-type': 'github',
@@ -93,18 +123,12 @@ export const compileGithubConfig = async (
                 'zoekt.public': marshalBool(isPublic),
                 'zoekt.display-name': repoDisplayName,
             },
-                branches: config.revisions?.branches ?? undefined,
+            branches,
-                tags: config.revisions?.tags ?? undefined,
+            tags,
         } satisfies RepoMetadata,
     };
 
     return record;
-    })
-
-    return {
-        repoData: repos,
-        warnings,
-    };
 }
 
 export const compileGitlabConfig = async (

packages/web/src/app/askgh/[owner]/[repo]/page.tsx

@@ -0,0 +1,16 @@
+import { addGithubRepo } from "@/features/workerApi/actions";
+
+interface PageProps {
+    params: Promise<{ owner: string; repo: string }>;
+}
+
+export default async function GitHubRepoPage(props: PageProps) {
+    const params = await props.params;
+    const { owner, repo } = params;
+
+    const response = await addGithubRepo(owner, repo);
+
+    return <p>{JSON.stringify(response, null, 2)}</p>;
+}
+
+

packages/web/src/features/workerApi/actions.ts

@@ -2,7 +2,7 @@
 
 import { sew } from "@/actions";
 import { unexpectedError } from "@/lib/serviceError";
-import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2";
+import { withAuthV2, withMinimumOrgRole, withOptionalAuthV2 } from "@/withAuthV2";
 import { OrgRole } from "@sourcebot/db";
 import z from "zod";
 
@@ -57,3 +57,18 @@ export const indexRepo = async (repoId: number) => sew(() =>
         })
     )
 );
+
+export const addGithubRepo = async (owner: string, repo: string) => sew(() =>
+    withOptionalAuthV2(async () => {
+        const response = await fetch(`${WORKER_API_URL}/api/experimental/add-github-repo`, {
+            method: 'POST',
+            body: JSON.stringify({ owner, repo }),
+            headers: {
+                'Content-Type': 'application/json',
+            },
+        });
+
+        const data = await response.json();
+        return data;
+    })
+);

packages/web/src/middleware.ts

@@ -1,35 +0,0 @@
-import { NextResponse } from 'next/server'
-import type { NextRequest } from 'next/server'
-import { SINGLE_TENANT_ORG_DOMAIN } from '@/lib/constants'
-
-export async function middleware(request: NextRequest) {
-    const url = request.nextUrl.clone();
-
-    if (
-        url.pathname.startsWith('/login') ||
-        url.pathname.startsWith('/redeem') ||
-        url.pathname.startsWith('/signup') ||
-        url.pathname.startsWith('/invite') ||
-        url.pathname.startsWith('/onboard')
-    ) {
-        return NextResponse.next();
-    }
-
-    const pathSegments = url.pathname.split('/').filter(Boolean);
-    const currentDomain = pathSegments[0];
-
-    // If we're already on the correct domain path, allow
-    if (currentDomain === SINGLE_TENANT_ORG_DOMAIN) {
-        return NextResponse.next();
-    }
-
-    url.pathname = `/${SINGLE_TENANT_ORG_DOMAIN}${pathSegments.length > 1 ? '/' + pathSegments.slice(1).join('/') : ''}`;
-    return NextResponse.redirect(url);
-}
-
-export const config = {
-    // https://nextjs.org/docs/app/building-your-application/routing/middleware#matcher
-    matcher: [
-        '/((?!api|_next/static|ingest|_next/image|favicon.ico|sitemap.xml|robots.txt|manifest.json|logo_192.png|logo_512.png|sb_logo_light_large.png|arrow.png|placeholder_avatar.png|sb_logo_dark_small.png|sb_logo_light_small.png).*)',
-    ],
-}