ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-12 12:25:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add back revision support (#215)

Browse source
This commit is contained in:
Brendan Kellam 2025-02-27 09:39:55 -08:00 committed by GitHub
parent 3d6c7dcca5
commit 46be0440b0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 191 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
packages/backend/src/git.ts
View file

@ -39,3 +39,20 @@ export const fetchRepository = async (path: string, onProgress?: (event: SimpleG
] ]
); );
} }
export const getBranches = async (path: string) => {
const git = simpleGit();
const branches = await git.cwd({
path,
}).branch();
return branches.all;
}
export const getTags = async (path: string) => {
const git = simpleGit();
const tags = await git.cwd({
path,
}).tags();
return tags.all;
}

77
packages/backend/src/gitea.ts
View file

@ -1,6 +1,6 @@
import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js'; import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type'; import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type';
import { getTokenFromConfig, measure, fetchWithRetry } from './utils.js'; import { getTokenFromConfig, measure } from './utils.js';
import fetch from 'cross-fetch'; import fetch from 'cross-fetch';
import { createLogger } from './logger.js'; import { createLogger } from './logger.js';
import micromatch from 'micromatch'; import micromatch from 'micromatch';
@ -56,49 +56,6 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
return true; return true;
}); });
if (config.revisions) {
if (config.revisions.branches) {
const branchGlobs = config.revisions.branches;
allRepos = await Promise.all(
allRepos.map(async (repo) => {
const [owner, name] = repo.full_name!.split('/');
let branches = (await fetchWithRetry(
() => getBranchesForRepo(owner, name, api),
`branches for ${owner}/${name}`,
logger
)).map(branch => branch.name!);
branches = micromatch.match(branches, branchGlobs);
return {
...repo,
branches,
};
})
)
}
if (config.revisions.tags) {
const tagGlobs = config.revisions.tags;
allRepos = await Promise.all(
allRepos.map(async (allRepos) => {
const [owner, name] = allRepos.full_name!.split('/');
let tags = (await fetchWithRetry(
() => getTagsForRepo(owner, name, api),
`tags for ${owner}/${name}`,
logger
)).map(tag => tag.name!);
tags = micromatch.match(tags, tagGlobs);
return {
...allRepos,
tags,
};
})
)
}
}
let repos = allRepos let repos = allRepos
.filter((repo) => { .filter((repo) => {
const isExcluded = shouldExcludeRepo({ const isExcluded = shouldExcludeRepo({
@ -158,38 +115,6 @@ const shouldExcludeRepo = ({
return shouldExclude; return shouldExclude;
} }
const getTagsForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
try {
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
const { durationMs, data: tags } = await measure(() =>
paginate((page) => api.repos.repoListTags(owner, repo, {
page
}))
);
logger.debug(`Found ${tags.length} tags in repo ${owner}/${repo} in ${durationMs}ms.`);
return tags;
} catch (e) {
logger.error(`Failed to fetch tags for repo ${owner}/${repo}.`, e);
throw e;
}
}
const getBranchesForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
try {
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
const { durationMs, data: branches } = await measure(() =>
paginate((page) => api.repos.repoListBranches(owner, repo, {
page
}))
);
logger.debug(`Found ${branches.length} branches in repo ${owner}/${repo} in ${durationMs}ms.`);
return branches;
} catch (e) {
logger.error(`Failed to fetch branches for repo ${owner}/${repo}.`, e);
throw e;
}
}
const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => { const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
const results = await Promise.allSettled(users.map(async (user) => { const results = await Promise.allSettled(users.map(async (user) => {
try { try {

1
packages/backend/src/posthogEvents.ts
View file

@ -18,6 +18,7 @@ export type PosthogEventMap = {
connectionId: number, connectionId: number,
repoCount: number, repoCount: number,
}, },
revisions_truncated: {},
////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////
} }

2
packages/backend/src/promClient.ts
View file

@ -1,5 +1,5 @@
import express, { Request, Response } from 'express'; import express, { Request, Response } from 'express';
import client, { Registry, Counter, Gauge, Histogram } from 'prom-client'; import client, { Registry, Counter, Gauge } from 'prom-client';
export class PromClient { export class PromClient {
private registry: Registry; private registry: Registry;

83
packages/backend/src/repoCompileUtils.ts
View file

@ -7,6 +7,7 @@ import { Prisma, PrismaClient } from '@sourcebot/db';
import { WithRequired } from "./types.js" import { WithRequired } from "./types.js"
import { marshalBool } from "./utils.js"; import { marshalBool } from "./utils.js";
import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { RepoMetadata } from './types.js';
export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>; export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
@ -54,17 +55,21 @@ export const compileGithubConfig = async (
} }
}, },
metadata: { metadata: {
'zoekt.web-url-type': 'github', gitConfig: {
'zoekt.web-url': repo.html_url, 'zoekt.web-url-type': 'github',
'zoekt.name': repoName, 'zoekt.web-url': repo.html_url,
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(), 'zoekt.name': repoName,
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(), 'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(), 'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
'zoekt.github-forks': (repo.forks_count ?? 0).toString(), 'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
'zoekt.archived': marshalBool(repo.archived), 'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
'zoekt.fork': marshalBool(repo.fork), 'zoekt.archived': marshalBool(repo.archived),
'zoekt.public': marshalBool(repo.private === false) 'zoekt.fork': marshalBool(repo.fork),
}, 'zoekt.public': marshalBool(repo.private === false),
},
branches: config.revisions?.branches ?? undefined,
tags: config.revisions?.tags ?? undefined,
} satisfies RepoMetadata,
}; };
return record; return record;
@ -113,15 +118,19 @@ export const compileGitlabConfig = async (
} }
}, },
metadata: { metadata: {
'zoekt.web-url-type': 'gitlab', gitConfig: {
'zoekt.web-url': projectUrl, 'zoekt.web-url-type': 'gitlab',
'zoekt.name': project.path_with_namespace, 'zoekt.web-url': projectUrl,
'zoekt.gitlab-stars': (project.stargazers_count ?? 0).toString(), 'zoekt.name': project.path_with_namespace,
'zoekt.gitlab-forks': (project.forks_count ?? 0).toString(), 'zoekt.gitlab-stars': (project.stargazers_count ?? 0).toString(),
'zoekt.archived': marshalBool(project.archived), 'zoekt.gitlab-forks': (project.forks_count ?? 0).toString(),
'zoekt.fork': marshalBool(isFork), 'zoekt.archived': marshalBool(project.archived),
'zoekt.public': marshalBool(project.private === false) 'zoekt.fork': marshalBool(isFork),
}, 'zoekt.public': marshalBool(project.private === false)
},
branches: config.revisions?.branches ?? undefined,
tags: config.revisions?.tags ?? undefined,
} satisfies RepoMetadata,
}; };
return record; return record;
@ -168,13 +177,17 @@ export const compileGiteaConfig = async (
} }
}, },
metadata: { metadata: {
'zoekt.web-url-type': 'gitea', gitConfig: {
'zoekt.web-url': repo.html_url!, 'zoekt.web-url-type': 'gitea',
'zoekt.name': repo.full_name!, 'zoekt.web-url': repo.html_url!,
'zoekt.archived': marshalBool(repo.archived), 'zoekt.name': repo.full_name!,
'zoekt.fork': marshalBool(repo.fork!), 'zoekt.archived': marshalBool(repo.archived),
'zoekt.public': marshalBool(repo.internal === false && repo.private === false), 'zoekt.fork': marshalBool(repo.fork!),
}, 'zoekt.public': marshalBool(repo.internal === false && repo.private === false),
},
branches: config.revisions?.branches ?? undefined,
tags: config.revisions?.tags ?? undefined,
} satisfies RepoMetadata,
}; };
return record; return record;
@ -227,13 +240,15 @@ export const compileGerritConfig = async (
} }
}, },
metadata: { metadata: {
'zoekt.web-url-type': 'gitiles', gitConfig: {
'zoekt.web-url': webUrl, 'zoekt.web-url-type': 'gitiles',
'zoekt.name': repoId, 'zoekt.web-url': webUrl,
'zoekt.archived': marshalBool(false), 'zoekt.name': repoId,
'zoekt.fork': marshalBool(false), 'zoekt.archived': marshalBool(false),
'zoekt.public': marshalBool(true), 'zoekt.fork': marshalBool(false),
}, 'zoekt.public': marshalBool(true),
},
} satisfies RepoMetadata,
}; };
return record; return record;

6
packages/backend/src/repoManager.ts
View file

@ -3,7 +3,7 @@ import { Redis } from 'ioredis';
import { createLogger } from "./logger.js"; import { createLogger } from "./logger.js";
import { Connection, PrismaClient, Repo, RepoToConnection, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db"; import { Connection, PrismaClient, Repo, RepoToConnection, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db";
import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { AppContext, Settings } from "./types.js"; import { AppContext, Settings, RepoMetadata } from "./types.js";
import { getRepoPath, getTokenFromConfig, measure, getShardPrefix } from "./utils.js"; import { getRepoPath, getTokenFromConfig, measure, getShardPrefix } from "./utils.js";
import { cloneRepository, fetchRepository } from "./git.js"; import { cloneRepository, fetchRepository } from "./git.js";
import { existsSync, rmSync, readdirSync, rm } from 'fs'; import { existsSync, rmSync, readdirSync, rm } from 'fs';
@ -187,7 +187,7 @@ export class RepoManager implements IRepoManager {
let cloneDuration_s: number | undefined = undefined; let cloneDuration_s: number | undefined = undefined;
const repoPath = getRepoPath(repo, this.ctx); const repoPath = getRepoPath(repo, this.ctx);
const metadata = repo.metadata as Record<string, string>; const metadata = repo.metadata as RepoMetadata;
// If the repo was already in the indexing state, this job was likely killed and picked up again. As a result, // If the repo was already in the indexing state, this job was likely killed and picked up again. As a result,
// to ensure the repo state is valid, we delete the repo if it exists so we get a fresh clone // to ensure the repo state is valid, we delete the repo if it exists so we get a fresh clone
@ -223,7 +223,7 @@ export class RepoManager implements IRepoManager {
cloneUrl = url.toString(); cloneUrl = url.toString();
} }
const { durationMs } = await measure(() => cloneRepository(cloneUrl, repoPath, metadata, ({ method, stage, progress }) => { const { durationMs } = await measure(() => cloneRepository(cloneUrl, repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) //this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
})); }));
cloneDuration_s = durationMs / 1000; cloneDuration_s = durationMs / 1000;

23
packages/backend/src/types.ts
View file

@ -49,6 +49,29 @@ export type Settings = {
gcGracePeriodMs: number; gcGracePeriodMs: number;
} }
/**
* Structure of the `metadata` field in the `Repo` table.
*/
export type RepoMetadata = {
/**
* A set of key-value pairs that will be used as git config
* variables when cloning the repo.
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
*/
gitConfig?: Record<string, string>;
/**
* A list of branches to index. Glob patterns are supported.
*/
branches?: string[];
/**
* A list of tags to index. Glob patterns are supported.
*/
tags?: string[];
}
// @see : https://stackoverflow.com/a/61132308 // @see : https://stackoverflow.com/a/61132308
export type DeepPartial<T> = T extends object ? { export type DeepPartial<T> = T extends object ? {
[P in keyof T]?: DeepPartial<T[P]>; [P in keyof T]?: DeepPartial<T[P]>;

54
packages/backend/src/zoekt.ts
View file

@ -1,17 +1,63 @@
import { exec } from "child_process"; import { exec } from "child_process";
import { AppContext } from "./types.js"; import { AppContext, RepoMetadata } from "./types.js";
import { Repo } from "@sourcebot/db"; import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js"; import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js"; import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js"; import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
import { createLogger } from "./logger.js";
import { captureEvent } from "./posthog.js";
const logger = createLogger('zoekt');
export const indexGitRepository = async (repo: Repo, ctx: AppContext) => { export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
const revisions = [ let revisions = [
'HEAD' 'HEAD'
]; ];
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const repoPath = getRepoPath(repo, ctx); const repoPath = getRepoPath(repo, ctx);
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
const metadata = repo.metadata as RepoMetadata;
if (metadata.branches) {
const branchGlobs = metadata.branches
const allBranches = await getBranches(repoPath);
const matchingBranches =
allBranches
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
.map((branch) => `refs/heads/${branch}`);
revisions = [
...revisions,
...matchingBranches
];
}
if (metadata.tags) {
const tagGlobs = metadata.tags;
const allTags = await getTags(repoPath);
const matchingTags =
allTags
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
.map((tag) => `refs/tags/${tag}`);
revisions = [
...revisions,
...matchingTags
];
}
// zoekt has a limit of 64 branches/tags to index.
if (revisions.length > 64) {
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
captureEvent('backend_revisions_truncated', {
repoId: repo.id,
revisionCount: revisions.length,
});
revisions = revisions.slice(0, 64);
}
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`; const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {

6
packages/schemas/src/v3/connection.schema.ts
View file

@ -154,11 +154,11 @@ const schema = {
}, },
"revisions": { "revisions": {
"type": "object", "type": "object",
"description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.",
"properties": { "properties": {
"branches": { "branches": {
"type": "array", "type": "array",
"description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },
@ -175,7 +175,7 @@ const schema = {
}, },
"tags": { "tags": {
"type": "array", "type": "array",
"description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },

6
packages/schemas/src/v3/connection.type.ts
View file

@ -77,15 +77,15 @@ export interface Token {
secret: string; secret: string;
} }
/** /**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
*/ */
export interface GitRevisions { export interface GitRevisions {
/** /**
* List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.
*/ */
branches?: string[]; branches?: string[];
/** /**
* List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.
*/ */
tags?: string[]; tags?: string[];
} }

6
packages/schemas/src/v3/gitea.schema.ts
View file

@ -97,11 +97,11 @@ const schema = {
}, },
"revisions": { "revisions": {
"type": "object", "type": "object",
"description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.",
"properties": { "properties": {
"branches": { "branches": {
"type": "array", "type": "array",
"description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },
@ -118,7 +118,7 @@ const schema = {
}, },
"tags": { "tags": {
"type": "array", "type": "array",
"description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },

6
packages/schemas/src/v3/gitea.type.ts
View file

@ -48,15 +48,15 @@ export interface Token {
secret: string; secret: string;
} }
/** /**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
*/ */
export interface GitRevisions { export interface GitRevisions {
/** /**
* List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.
*/ */
branches?: string[]; branches?: string[];
/** /**
* List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.
*/ */
tags?: string[]; tags?: string[];
} }

6
packages/schemas/src/v3/github.schema.ts
View file

@ -150,11 +150,11 @@ const schema = {
}, },
"revisions": { "revisions": {
"type": "object", "type": "object",
"description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.",
"properties": { "properties": {
"branches": { "branches": {
"type": "array", "type": "array",
"description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },
@ -171,7 +171,7 @@ const schema = {
}, },
"tags": { "tags": {
"type": "array", "type": "array",
"description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },

6
packages/schemas/src/v3/github.type.ts
View file

@ -71,15 +71,15 @@ export interface Token {
secret: string; secret: string;
} }
/** /**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
*/ */
export interface GitRevisions { export interface GitRevisions {
/** /**
* List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.
*/ */
branches?: string[]; branches?: string[];
/** /**
* List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.
*/ */
tags?: string[]; tags?: string[];
} }

6
packages/schemas/src/v3/gitlab.schema.ts
View file

@ -139,11 +139,11 @@ const schema = {
}, },
"revisions": { "revisions": {
"type": "object", "type": "object",
"description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.",
"properties": { "properties": {
"branches": { "branches": {
"type": "array", "type": "array",
"description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },
@ -160,7 +160,7 @@ const schema = {
}, },
"tags": { "tags": {
"type": "array", "type": "array",
"description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },

6
packages/schemas/src/v3/gitlab.type.ts
View file

@ -62,15 +62,15 @@ export interface Token {
secret: string; secret: string;
} }
/** /**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
*/ */
export interface GitRevisions { export interface GitRevisions {
/** /**
* List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.
*/ */
branches?: string[]; branches?: string[];
/** /**
* List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.
*/ */
tags?: string[]; tags?: string[];
} }

6
packages/schemas/src/v3/shared.schema.ts
View file

@ -18,11 +18,11 @@ const schema = {
}, },
"GitRevisions": { "GitRevisions": {
"type": "object", "type": "object",
"description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.",
"properties": { "properties": {
"branches": { "branches": {
"type": "array", "type": "array",
"description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },
@ -39,7 +39,7 @@ const schema = {
}, },
"tags": { "tags": {
"type": "array", "type": "array",
"description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },

6
packages/schemas/src/v3/shared.type.ts
View file

@ -14,18 +14,18 @@ export interface Token {
secret: string; secret: string;
} }
/** /**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
* *
* This interface was referenced by `Shared`'s JSON-Schema * This interface was referenced by `Shared`'s JSON-Schema
* via the `definition` "GitRevisions". * via the `definition` "GitRevisions".
*/ */
export interface GitRevisions { export interface GitRevisions {
/** /**
* List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.
*/ */
branches?: string[]; branches?: string[];
/** /**
* List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.
*/ */
tags?: string[]; tags?: string[];
} }

23
packages/web/src/app/[domain]/search/page.tsx
View file

@ -134,30 +134,11 @@ const SearchPageInternal = () => {
}; };
} }
const isBranchFilteringEnabled = searchResponse.isBranchFilteringEnabled;
let fileMatches = searchResponse.Result.Files ?? [];
// We only want to show matches for the default branch when
// the user isn't explicitly filtering by branch.
measureSync(() => {
if (!isBranchFilteringEnabled) {
fileMatches = fileMatches.filter(match => {
// @note : this case handles local repos that don't have any branches.
if (!match.Branches) {
return true;
}
return match.Branches.includes("HEAD");
});
}
}, "search.branchFiltering");
return { return {
fileMatches, fileMatches: searchResponse.Result.Files ?? [],
searchDurationMs: Math.round(searchResponse.Result.Duration / 1000000), searchDurationMs: Math.round(searchResponse.Result.Duration / 1000000),
totalMatchCount: searchResponse.Result.MatchCount, totalMatchCount: searchResponse.Result.MatchCount,
isBranchFilteringEnabled, isBranchFilteringEnabled: searchResponse.isBranchFilteringEnabled,
repoUrlTemplates: searchResponse.Result.RepoURLs, repoUrlTemplates: searchResponse.Result.RepoURLs,
} }
}, [searchResponse]); }, [searchResponse]);

16
packages/web/src/lib/server/searchService.ts
View file

@ -40,6 +40,17 @@ export const search = async ({ query, maxMatchDisplayCount, whole}: SearchReques
query = query.replaceAll(prefix, zoektPrefix); query = query.replaceAll(prefix, zoektPrefix);
} }
const isBranchFilteringEnabled = (
query.includes(zoektPrefixes.branch) ||
query.includes(zoektPrefixes.branchShort)
);
// We only want to show matches for the default branch when
// the user isn't explicitly filtering by branch.
if (!isBranchFilteringEnabled) {
query = query.concat(` branch:HEAD`);
}
const body = JSON.stringify({ const body = JSON.stringify({
q: query, q: query,
// @see: https://github.com/sourcebot-dev/zoekt/blob/main/api.go#L892 // @see: https://github.com/sourcebot-dev/zoekt/blob/main/api.go#L892
@ -76,11 +87,6 @@ export const search = async ({ query, maxMatchDisplayCount, whole}: SearchReques
return unexpectedError(`Something went wrong while parsing the response from zoekt`); return unexpectedError(`Something went wrong while parsing the response from zoekt`);
} }
const isBranchFilteringEnabled = (
query.includes(zoektPrefixes.branch) ||
query.includes(zoektPrefixes.branchShort)
)
return { return {
...parsedSearchResponse.data, ...parsedSearchResponse.data,
isBranchFilteringEnabled, isBranchFilteringEnabled,

6
schemas/v3/shared.json
View file

@ -17,11 +17,11 @@
}, },
"GitRevisions": { "GitRevisions": {
"type": "object", "type": "object",
"description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.",
"properties": { "properties": {
"branches": { "branches": {
"type": "array", "type": "array",
"description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },
@ -38,7 +38,7 @@
}, },
"tags": { "tags": {
"type": "array", "type": "array",
"description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.",
"items": { "items": {
"type": "string" "type": "string"
}, },