ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-11 20:05:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

wip on ondemand indexing

Browse source
This commit is contained in:
bkellam 2025-12-02 19:24:01 -08:00
parent 91caf129ed
commit 46544dc1d8
5 changed files with 137 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
packages/backend/src/api.ts
View file

@ -7,6 +7,8 @@ import z from 'zod';
import { ConnectionManager } from './connectionManager.js';
import { PromClient } from './promClient.js';
import { RepoIndexManager } from './repoIndexManager.js';
import { createGitHubRepoRecord } from './repoCompileUtils.js';
import { Octokit } from '@octokit/rest';
const logger = createLogger('api');
const PORT = 3060;
@ -33,6 +35,7 @@ export class Api {
app.post('/api/sync-connection', this.syncConnection.bind(this));
app.post('/api/index-repo', this.indexRepo.bind(this));
app.post(`/api/experimental/add-github-repo`, this.addGithubRepo.bind(this));
this.server = app.listen(PORT, () => {
logger.info(`API server is running on port ${PORT}`);
@ -92,6 +95,38 @@ export class Api {
res.status(200).json({ jobId });
}
private async addGithubRepo(req: Request, res: Response) {
const schema = z.object({
owner: z.string(),
repo: z.string(),
}).strict();
const parsed = schema.safeParse(req.body);
if (!parsed.success) {
res.status(400).json({ error: parsed.error.message });
return;
}
const octokit = new Octokit();
const response = await octokit.rest.repos.get({
owner: parsed.data.owner,
repo: parsed.data.repo,
});
const record = createGitHubRepoRecord({
repo: response.data,
hostUrl: 'https://github.com',
});
const repo = await this.prisma.repo.create({
data: record,
});
const [jobId ] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX);
res.status(200).json({ jobId });
}
public async dispose() {
return new Promise<void>((resolve, reject) => {
this.server.close((err) => {

116
packages/backend/src/repoCompileUtils.ts
View file

@ -1,5 +1,5 @@
import { GithubConnectionConfig } from '@sourcebot/schemas/v3/github.type';
import { getGitHubReposFromConfig } from "./github.js";
import { getGitHubReposFromConfig, OctokitRepository } from "./github.js";
import { getGitLabReposFromConfig } from "./gitlab.js";
import { getGiteaReposFromConfig } from "./gitea.js";
import { getGerritReposFromConfig } from "./gerrit.js";
@ -45,60 +45,23 @@ export const compileGithubConfig = async (
const warnings = gitHubReposResult.warnings;
const hostUrl = config.url ?? 'https://github.com';
const repoNameRoot = new URL(hostUrl)
.toString()
.replace(/^https?:\/\//, '');
const repos = gitHubRepos.map((repo) => {
const repoDisplayName = repo.full_name;
const repoName = path.join(repoNameRoot, repoDisplayName);
const cloneUrl = new URL(repo.clone_url!);
const isPublic = repo.private === false;
const record = createGitHubRepoRecord({
repo,
hostUrl,
branches: config.revisions?.branches ?? undefined,
tags: config.revisions?.tags ?? undefined,
})
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
const record: RepoData = {
external_id: repo.id.toString(),
external_codeHostType: 'github',
external_codeHostUrl: hostUrl,
cloneUrl: cloneUrl.toString(),
webUrl: repo.html_url,
name: repoName,
displayName: repoDisplayName,
imageUrl: repo.owner.avatar_url,
isFork: repo.fork,
isArchived: !!repo.archived,
isPublic: isPublic,
org: {
connect: {
id: SINGLE_TENANT_ORG_ID,
},
},
return {
...record,
connections: {
create: {
connectionId: connectionId,
}
},
metadata: {
gitConfig: {
'zoekt.web-url-type': 'github',
'zoekt.web-url': repo.html_url,
'zoekt.name': repoName,
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
'zoekt.archived': marshalBool(repo.archived),
'zoekt.fork': marshalBool(repo.fork),
'zoekt.public': marshalBool(isPublic),
'zoekt.display-name': repoDisplayName,
},
branches: config.revisions?.branches ?? undefined,
tags: config.revisions?.tags ?? undefined,
} satisfies RepoMetadata,
};
return record;
})
return {
@ -107,6 +70,67 @@ export const compileGithubConfig = async (
};
}
export const createGitHubRepoRecord = ({
repo,
hostUrl,
branches,
tags,
}: {
repo: OctokitRepository,
hostUrl: string,
branches?: string[],
tags?: string[],
}) => {
const repoNameRoot = new URL(hostUrl)
.toString()
.replace(/^https?:\/\//, '');
const repoDisplayName = repo.full_name;
const repoName = path.join(repoNameRoot, repoDisplayName);
const cloneUrl = new URL(repo.clone_url!);
const isPublic = repo.private === false;
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
const record: Prisma.RepoCreateInput = {
external_id: repo.id.toString(),
external_codeHostType: 'github',
external_codeHostUrl: hostUrl,
cloneUrl: cloneUrl.toString(),
webUrl: repo.html_url,
name: repoName,
displayName: repoDisplayName,
imageUrl: repo.owner.avatar_url,
isFork: repo.fork,
isArchived: !!repo.archived,
isPublic: isPublic,
org: {
connect: {
id: SINGLE_TENANT_ORG_ID,
},
},
metadata: {
gitConfig: {
'zoekt.web-url-type': 'github',
'zoekt.web-url': repo.html_url,
'zoekt.name': repoName,
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
'zoekt.archived': marshalBool(repo.archived),
'zoekt.fork': marshalBool(repo.fork),
'zoekt.public': marshalBool(isPublic),
'zoekt.display-name': repoDisplayName,
},
branches,
tags,
} satisfies RepoMetadata,
};
return record;
}
export const compileGitlabConfig = async (
config: GitlabConnectionConfig,
connectionId: number): Promise<CompileResult> => {

16
packages/web/src/app/askgh/[owner]/[repo]/page.tsx Normal file
View file

@ -0,0 +1,16 @@
import { addGithubRepo } from "@/features/workerApi/actions";
interface PageProps {
params: Promise<{ owner: string; repo: string }>;
}
export default async function GitHubRepoPage(props: PageProps) {
const params = await props.params;
const { owner, repo } = params;
const response = await addGithubRepo(owner, repo);
return <p>{JSON.stringify(response, null, 2)}</p>;
}

17
packages/web/src/features/workerApi/actions.ts
View file

@ -2,7 +2,7 @@
import { sew } from "@/actions";
import { unexpectedError } from "@/lib/serviceError";
import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2";
import { withAuthV2, withMinimumOrgRole, withOptionalAuthV2 } from "@/withAuthV2";
import { OrgRole } from "@sourcebot/db";
import z from "zod";
@ -57,3 +57,18 @@ export const indexRepo = async (repoId: number) => sew(() =>
})
)
);
export const addGithubRepo = async (owner: string, repo: string) => sew(() =>
withOptionalAuthV2(async () => {
const response = await fetch(`${WORKER_API_URL}/api/experimental/add-github-repo`, {
method: 'POST',
body: JSON.stringify({ owner, repo }),
headers: {
'Content-Type': 'application/json',
},
});
const data = await response.json();
return data;
})
);

35
packages/web/src/middleware.ts
View file

@ -1,35 +0,0 @@
import { NextResponse } from 'next/server'
import type { NextRequest } from 'next/server'
import { SINGLE_TENANT_ORG_DOMAIN } from '@/lib/constants'
export async function middleware(request: NextRequest) {
const url = request.nextUrl.clone();
if (
url.pathname.startsWith('/login') ||
url.pathname.startsWith('/redeem') ||
url.pathname.startsWith('/signup') ||
url.pathname.startsWith('/invite') ||
url.pathname.startsWith('/onboard')
) {
return NextResponse.next();
}
const pathSegments = url.pathname.split('/').filter(Boolean);
const currentDomain = pathSegments[0];
// If we're already on the correct domain path, allow
if (currentDomain === SINGLE_TENANT_ORG_DOMAIN) {
return NextResponse.next();
}
url.pathname = `/${SINGLE_TENANT_ORG_DOMAIN}${pathSegments.length > 1 ? '/' + pathSegments.slice(1).join('/') : ''}`;
return NextResponse.redirect(url);
}
export const config = {
// https://nextjs.org/docs/app/building-your-application/routing/middleware#matcher
matcher: [
'/((?!api|_next/static|ingest|_next/image|favicon.ico|sitemap.xml|robots.txt|manifest.json|logo_192.png|logo_512.png|sb_logo_light_large.png|arrow.png|placeholder_avatar.png|sb_logo_dark_small.png|sb_logo_light_small.png).*)',
],
}