ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-14 13:25:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add branch table to repos detail view

Browse source
This commit is contained in:
bkellam 2025-10-25 14:19:21 -04:00
parent c16ef08a7c
commit 5a670bfdb9
12 changed files with 342 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
packages/backend/src/git.ts
View file

@ -278,6 +278,16 @@ export const getCommitHashForRefName = async ({
refName: string,
}) => {
const git = createGitClientForPath(path);
const rev = await git.revparse(refName);
return rev;
try {
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
const rev = await git.revparse(`${refName}^{commit}`);
return rev;
// @note: Was hitting errors when the repository is empty,
// so we're catching the error and returning undefined.
} catch (error: unknown) {
console.error(error);
return undefined;
}
}

2
packages/backend/src/repoCompileUtils.ts
View file

@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
import { createLogger } from '@sourcebot/logger';
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
import { RepoMetadata } from './types.js';
import path from 'path';
import { glob } from 'glob';
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
import assert from 'assert';
import GitUrlParse from 'git-url-parse';
import { RepoMetadata } from '@sourcebot/shared';
export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;

74
packages/backend/src/repoIndexManager.ts
View file

@ -1,15 +1,18 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, getCommitHashForRefName, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
import { RepoWithConnections, Settings } from "./types.js";
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
import { indexGitRepository } from './zoekt.js';
@ -61,7 +64,7 @@ export class RepoIndexManager {
concurrency: this.settings.maxRepoIndexingJobConcurrency,
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
logger: true,
}: {}),
} : {}),
});
this.worker.on('completed', this.onJobCompleted.bind(this));
@ -263,7 +266,16 @@ export class RepoIndexManager {
try {
if (jobType === RepoIndexingJobType.INDEX) {
await this.indexRepository(repo, logger, abortController.signal);
const revisions = await this.indexRepository(repo, logger, abortController.signal);
await this.db.repoIndexingJob.update({
where: { id },
data: {
metadata: {
indexedRevisions: revisions,
} satisfies RepoIndexingJobMetadata,
},
});
} else if (jobType === RepoIndexingJobType.CLEANUP) {
await this.cleanupRepository(repo, logger);
}
@ -285,7 +297,7 @@ export class RepoIndexManager {
// If the repo path exists but it is not a valid git repository root, this indicates
// that the repository is in a bad state. To fix, we remove the directory and perform
// a fresh clone.
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot( { path: repoPath } ))) {
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
const isValidGitRepo = await isPathAValidGitRepoRoot({
path: repoPath,
signal,
@ -354,10 +366,54 @@ export class RepoIndexManager {
});
}
let revisions = [
'HEAD'
];
if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);
revisions = [
...revisions,
...matchingBranches
];
}
if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);
revisions = [
...revisions,
...matchingTags
];
}
// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}
logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
const indexDuration_s = durationMs / 1000;
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);
return revisions;
}
private async cleanupRepository(repo: Repo, logger: Logger) {
@ -399,11 +455,17 @@ export class RepoIndexManager {
refName: 'HEAD',
});
const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);
const repo = await this.db.repo.update({
where: { id: jobData.repoId },
data: {
indexedAt: new Date(),
indexedCommitHash: commitHash,
metadata: {
...(jobData.repo.metadata as RepoMetadata),
indexedRevisions: jobMetadata.indexedRevisions,
} satisfies RepoMetadata,
}
});

28
packages/backend/src/types.ts
View file

@ -1,36 +1,8 @@
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";
export type Settings = Required<SettingsSchema>;
// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),
/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),
/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
});
export type RepoMetadata = z.infer<typeof repoMetadataSchema>;
// @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>;

50
packages/backend/src/zoekt.ts
View file

@ -1,61 +1,15 @@
import { Repo } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { exec } from "child_process";
import micromatch from "micromatch";
import { INDEX_CACHE_DIR } from "./constants.js";
import { getBranches, getTags } from "./git.js";
import { captureEvent } from "./posthog.js";
import { repoMetadataSchema, Settings } from "./types.js";
import { Settings } from "./types.js";
import { getRepoPath, getShardPrefix } from "./utils.js";
const logger = createLogger('zoekt');
export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
let revisions = [
'HEAD'
];
export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
const { path: repoPath } = getRepoPath(repo);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repoMetadataSchema.parse(repo.metadata);
if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);
revisions = [
...revisions,
...matchingBranches
];
}
if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);
revisions = [
...revisions,
...matchingTags
];
}
// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}
const command = [
'zoekt-git-index',

2
packages/db/prisma/migrations/20251024214005_add_metadata_field_to_repo_job_table/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "RepoIndexingJob" ADD COLUMN "metadata" JSONB;

3
packages/db/prisma/schema.prisma
View file

@ -38,7 +38,7 @@ model Repo {
isFork Boolean
isArchived Boolean
isPublic Boolean @default(false)
metadata Json /// For schema see repoMetadataSchema in packages/backend/src/types.ts
metadata Json /// For schema see repoMetadataSchema in packages/shared/src/types.ts
cloneUrl String
webUrl String?
connections RepoToConnection[]
@ -84,6 +84,7 @@ model RepoIndexingJob {
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
completedAt DateTime?
metadata Json? /// For schema see repoIndexingJobMetadataSchema in packages/shared/src/types.ts
errorMessage String?

8
packages/shared/src/index.server.ts
View file

@ -9,6 +9,14 @@ export type {
Plan,
Entitlement,
} from "./entitlements.js";
export type {
RepoMetadata,
RepoIndexingJobMetadata,
} from "./types.js";
export {
repoMetadataSchema,
repoIndexingJobMetadataSchema,
} from "./types.js";
export {
base64Decode,
loadConfig,

42
packages/shared/src/types.ts
View file

@ -1,3 +1,45 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
import { z } from "zod";
export type ConfigSettings = Required<SettingsSchema>;
// Structure of the `metadata` field in the `Repo` table.
//
// @WARNING: If you modify this schema, please make sure it is backwards
// compatible with any prior versions of the schema!!
// @NOTE: If you move this schema, please update the comment in schema.prisma
// to point to the new location.
export const repoMetadataSchema = z.object({
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig: z.record(z.string(), z.string()).optional(),
/**
* A list of branches to index. Glob patterns are supported.
*/
branches: z.array(z.string()).optional(),
/**
* A list of tags to index. Glob patterns are supported.
*/
tags: z.array(z.string()).optional(),
/**
* A list of revisions that were indexed for the repo.
*/
indexedRevisions: z.array(z.string()).optional(),
});
export type RepoMetadata = z.infer<typeof repoMetadataSchema>;
export const repoIndexingJobMetadataSchema = z.object({
/**
* A list of revisions that were indexed for the repo.
*/
indexedRevisions: z.array(z.string()).optional(),
});
export type RepoIndexingJobMetadata = z.infer<typeof repoIndexingJobMetadataSchema>;

32
packages/web/src/app/[domain]/repos/[id]/page.tsx
View file

@ -17,6 +17,8 @@ import { RepoJobsTable } from "../components/repoJobsTable"
import { getConfigSettings } from "@sourcebot/shared"
import { env } from "@/env.mjs"
import { DisplayDate } from "../../components/DisplayDate"
import { RepoBranchesTable } from "../components/repoBranchesTable"
import { repoMetadataSchema } from "@sourcebot/shared"
export default async function RepoDetailPage({ params }: { params: Promise<{ id: string }> }) {
const { id } = await params
@ -47,6 +49,8 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
return undefined;
})();
const repoMetadata = repoMetadataSchema.parse(repo.metadata);
return (
<div className="container mx-auto">
<div className="mb-6">
@ -99,7 +103,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
</CardTitle>
</CardHeader>
<CardContent>
<DisplayDate date={repo.createdAt} className="text-2xl font-semibold"/>
<DisplayDate date={repo.createdAt} className="text-2xl font-semibold" />
</CardContent>
</Card>
@ -118,7 +122,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
</CardTitle>
</CardHeader>
<CardContent>
{repo.indexedAt ? <DisplayDate date={repo.indexedAt} className="text-2xl font-semibold"/> : "Never" }
{repo.indexedAt ? <DisplayDate date={repo.indexedAt} className="text-2xl font-semibold" /> : "Never"}
</CardContent>
</Card>
@ -137,15 +141,35 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
</CardTitle>
</CardHeader>
<CardContent>
{nextIndexAttempt ? <DisplayDate date={nextIndexAttempt} className="text-2xl font-semibold"/> : "-" }
{nextIndexAttempt ? <DisplayDate date={nextIndexAttempt} className="text-2xl font-semibold" /> : "-"}
</CardContent>
</Card>
</div>
{repoMetadata.indexedRevisions && (
<Card className="mb-8">
<CardHeader>
<div className="flex items-center gap-2">
<CardTitle>Indexed Branches</CardTitle>
</div>
<CardDescription>Branches that have been indexed for this repository. <Link href="https://docs.sourcebot.dev/docs/features/search/multi-branch-indexing" target="_blank" className="text-link hover:underline">Docs</Link></CardDescription>
</CardHeader>
<CardContent>
<Suspense fallback={<Skeleton className="h-64 w-full" />}>
<RepoBranchesTable
indexRevisions={repoMetadata.indexedRevisions}
repoWebUrl={repo.webUrl}
repoCodeHostType={repo.external_codeHostType}
/>
</Suspense>
</CardContent>
</Card>
)}
<Card>
<CardHeader>
<CardTitle>Indexing Jobs</CardTitle>
<CardDescription>History of all indexing and cleanup jobs for this repository</CardDescription>
<CardDescription>History of all indexing and cleanup jobs for this repository.</CardDescription>
</CardHeader>
<CardContent>
<Suspense fallback={<Skeleton className="h-96 w-full" />}>

141
packages/web/src/app/[domain]/repos/components/repoBranchesTable.tsx Normal file
View file

@ -0,0 +1,141 @@
"use client"
import * as React from "react"
import {
type ColumnDef,
type ColumnFiltersState,
type SortingState,
flexRender,
getCoreRowModel,
getFilteredRowModel,
getPaginationRowModel,
getSortedRowModel,
useReactTable,
} from "@tanstack/react-table"
import { Button } from "@/components/ui/button"
import { Input } from "@/components/ui/input"
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
import { CodeHostType, getCodeHostBrowseAtBranchUrl } from "@/lib/utils"
import Link from "next/link"
type RepoBranchesTableProps = {
indexRevisions: string[];
repoWebUrl: string | null;
repoCodeHostType: string;
}
export const RepoBranchesTable = ({ indexRevisions, repoWebUrl, repoCodeHostType }: RepoBranchesTableProps) => {
const [sorting, setSorting] = React.useState<SortingState>([])
const [columnFilters, setColumnFilters] = React.useState<ColumnFiltersState>([])
const columns = React.useMemo<ColumnDef<string>[]>(() => {
return [
{
accessorKey: "refName",
header: "Revision",
cell: ({ row }) => {
const refName = row.original;
const shortRefName = refName.replace(/^refs\/(heads|tags)\//, "");
const branchUrl = getCodeHostBrowseAtBranchUrl({
webUrl: repoWebUrl,
codeHostType: repoCodeHostType as CodeHostType,
branchName: refName,
});
return branchUrl ? (
<Link
href={branchUrl}
className="font-mono text-sm text-link hover:underline"
target="_blank"
>
{shortRefName}
</Link>
) : (
<span
className="font-mono text-sm text-muted-foreground"
title="This revision is not indexed"
>
{shortRefName}
</span>
)
},
}
]
}, [repoCodeHostType, repoWebUrl]);
const table = useReactTable({
data: indexRevisions,
columns,
getCoreRowModel: getCoreRowModel(),
getPaginationRowModel: getPaginationRowModel(),
getSortedRowModel: getSortedRowModel(),
getFilteredRowModel: getFilteredRowModel(),
onSortingChange: setSorting,
onColumnFiltersChange: setColumnFilters,
state: {
sorting,
columnFilters,
},
initialState: {
pagination: {
pageSize: 5,
},
},
})
return (
<div className="space-y-4">
<div className="flex items-center gap-2">
<Input
placeholder="Filter branches..."
value={(table.getColumn("refName")?.getFilterValue() as string) ?? ""}
onChange={(event) => table.getColumn("refName")?.setFilterValue(event.target.value)}
className="max-w-sm"
/>
</div>
<div className="rounded-md border">
<Table>
<TableHeader>
{table.getHeaderGroups().map((headerGroup) => (
<TableRow key={headerGroup.id}>
{headerGroup.headers.map((header) => (
<TableHead key={header.id}>
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
</TableHead>
))}
</TableRow>
))}
</TableHeader>
<TableBody>
{table.getRowModel().rows?.length ? (
table.getRowModel().rows.map((row) => (
<TableRow key={row.id}>
{row.getVisibleCells().map((cell) => (
<TableCell key={cell.id}>{flexRender(cell.column.columnDef.cell, cell.getContext())}</TableCell>
))}
</TableRow>
))
) : (
<TableRow>
<TableCell colSpan={columns.length} className="h-24 text-center">
No branches found.
</TableCell>
</TableRow>
)}
</TableBody>
</Table>
</div>
<div className="flex items-center justify-end space-x-2">
<Button variant="outline" size="sm" onClick={() => table.previousPage()} disabled={!table.getCanPreviousPage()}>
Previous
</Button>
<Button variant="outline" size="sm" onClick={() => table.nextPage()} disabled={!table.getCanNextPage()}>
Next
</Button>
</div>
</div>
)
}

32
packages/web/src/lib/utils.ts
View file

@ -352,6 +352,38 @@ export const getCodeHostCommitUrl = ({
}
}
export const getCodeHostBrowseAtBranchUrl = ({
webUrl,
codeHostType,
branchName,
}: {
webUrl?: string | null,
codeHostType: CodeHostType,
branchName: string,
}) => {
if (!webUrl) {
return undefined;
}
switch (codeHostType) {
case 'github':
return `${webUrl}/tree/${branchName}`;
case 'gitlab':
return `${webUrl}/-/tree/${branchName}`;
case 'gitea':
return `${webUrl}/src/branch/${branchName}`;
case 'azuredevops':
return `${webUrl}?branch=${branchName}`;
case 'bitbucket-cloud':
return `${webUrl}?at=${branchName}`;
case 'bitbucket-server':
return `${webUrl}?at=${branchName}`;
case 'gerrit':
case 'generic-git-host':
return undefined;
}
}
export const isAuthSupportedForCodeHost = (codeHostType: CodeHostType): boolean => {
switch (codeHostType) {
case "github":