ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-12 04:15:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Declarative connection configuration (#235)

Browse source
This commit is contained in:
Brendan Kellam 2025-03-21 12:27:33 -07:00 committed by GitHub
parent 2b17fd6702
commit 4c52059ecc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
44 changed files with 1632 additions and 381 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.env.development
View file

@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
# You can generate a new secret with:
# openssl rand -base64 33
# @see: https://authjs.dev/getting-started/deployment#auth_secret
AUTH_SECRET="secret"
AUTH_SECRET="00000000000000000000000000000000000000000000"
AUTH_URL="http://localhost:3000"
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
# AUTH_GITHUB_CLIENT_ID=""
@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"
# Generated using:
# openssl rand -base64 24
SOURCEBOT_ENCRYPTION_KEY="secret"
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"
SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
# NEXT_PUBLIC_SOURCEBOT_VERSION=
# CONFIG_MAX_REPOS_NO_TOKEN=
# SOURCEBOT_ROOT_DOMAIN=
# NODE_ENV=
# SOURCEBOT_TENANCY_MODE=mutli
# SOURCEBOT_TENANCY_MODE=single

4
package.json
View file

@ -7,13 +7,13 @@
"build": "yarn workspaces run build",
"test": "yarn workspaces run test",
"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",
"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
},

1
packages/backend/package.json
View file

@ -34,6 +34,7 @@
"@sourcebot/schemas": "^0.1.0",
"@t3-oss/env-core": "^0.12.0",
"@types/express": "^5.0.0",
"ajv": "^8.17.1",
"argparse": "^2.0.1",
"bullmq": "^5.34.10",
"cross-fetch": "^4.0.0",

4
packages/backend/src/env.ts
View file

@ -41,8 +41,8 @@ export const env = createEnv({
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),
INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string().optional(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,

3
packages/backend/src/gitea.ts
View file

@ -12,8 +12,7 @@ import { env } from './env.js';
const logger = createLogger('Gitea');
export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;
const api = giteaApi(config.url ?? 'https://gitea.com', {
token: token,

10
packages/backend/src/github.ts
View file

@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
}
export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token;
const secretKey = tokenResult?.secretKey;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;
const octokit = new Octokit({
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
auth: token,
...(config.url ? {
baseUrl: `${config.url}/api/v3`
} : {}),
@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o
if (isHttpError(error, 401)) {
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
secretKey,
...(config.token && 'secret' in config.token ? {
secretKey: config.token.secret,
} : {}),
});
Sentry.captureException(e);
throw e;

3
packages/backend/src/gitlab.ts
View file

@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";
export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;
const api = new Gitlab({
...(token ? {

2
packages/backend/src/index.ts
View file

@ -42,7 +42,6 @@ const parser = new ArgumentParser({
});
type Arguments = {
configPath: string;
cacheDir: string;
}
@ -67,7 +66,6 @@ const context: AppContext = {
indexPath,
reposPath,
cachePath: cacheDir,
configPath: args.configPath,
}
const prisma = new PrismaClient();

43
packages/backend/src/main.ts
View file

@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
import { RepoManager } from './repoManager.js';
import { env } from './env.js';
import { PromClient } from './promClient.js';
import { isRemotePath } from './utils.js';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import { Ajv } from "ajv";
const logger = createLogger('main');
const ajv = new Ajv({
validateFormats: false,
});
const getSettings = async (configPath?: string) => {
if (!configPath) {
return DEFAULT_SETTINGS;
}
const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, { encoding: 'utf-8' });
}
})();
const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}
return {
...DEFAULT_SETTINGS,
...config.settings,
}
}
export const main = async (db: PrismaClient, context: AppContext) => {
const redis = new Redis(env.REDIS_URL, {
@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
process.exit(1);
});
const settings = DEFAULT_SETTINGS;
if (env.INDEX_CONCURRENCY_MULTIPLE) {
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
}
const settings = await getSettings(env.CONFIG_PATH);
const promClient = new PromClient();

10
packages/backend/src/repoManager.ts
View file

@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
import os from 'os';
import { PromClient } from './promClient.js';
import * as Sentry from "@sentry/node";
interface IRepoManager {
blockingPollLoop: () => void;
dispose: () => void;
@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {
const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
if (config.token) {
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
token = tokenResult?.token;
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
if (token) {
break;
}
@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
this.logger.info(`Fetching ${repo.id}...`);
const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
fetchDuration_s = durationMs / 1000;
@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
}
const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
}));
cloneDuration_s = durationMs / 1000;
@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
}
this.logger.info(`Indexing ${repo.id}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);

47
packages/backend/src/types.ts
View file

@ -1,3 +1,5 @@
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
export type AppContext = {
/**
* Path to the repos cache directory.
@ -10,52 +12,9 @@ export type AppContext = {
indexPath: string;
cachePath: string;
configPath: string;
}
export type Settings = {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs: number;
}
export type Settings = Required<SettingsSchema>;
/**
* Structure of the `metadata` field in the `Repo` table.

73
packages/backend/src/utils.ts
View file

@ -21,44 +21,49 @@ export const marshalBool = (value?: boolean) => {
return !!value ? '1' : '0';
}
export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => {
if (!db) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
message: `No database connection provided.`,
});
Sentry.captureException(e);
throw e;
}
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});
if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
throw e;
}
const decryptedSecret = decrypt(secret.iv, secret.encryptedValue);
return {
token: decryptedSecret,
secretKey,
};
}
export const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}
export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => {
if ('secret' in token) {
const secretKey = token.secret;
const secret = await db.secret.findUnique({
where: {
orgId_key: {
key: secretKey,
orgId
}
}
});
if (!secret) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Secret with key ${secretKey} not found for org ${orgId}`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
const decryptedToken = decrypt(secret.iv, secret.encryptedValue);
return decryptedToken;
} else {
const envToken = process.env[token.env];
if (!envToken) {
const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
message: `Environment variable ${token.env} not found.`,
});
Sentry.captureException(e);
logger?.error(e.metadata.message);
throw e;
}
return envToken;
}
}
export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
let absolutePath = localPath;
if (!path.isAbsolute(absolutePath)) {

7
packages/backend/src/zoekt.ts
View file

@ -1,8 +1,7 @@
import { exec } from "child_process";
import { AppContext, RepoMetadata } from "./types.js";
import { AppContext, RepoMetadata, Settings } from "./types.js";
import { Repo } from "@sourcebot/db";
import { getRepoPath } from "./utils.js";
import { DEFAULT_SETTINGS } from "./constants.js";
import { getShardPrefix } from "./utils.js";
import { getBranches, getTags } from "./git.js";
import micromatch from "micromatch";
@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js";
const logger = createLogger('zoekt');
export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => {
let revisions = [
'HEAD'
];
@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
revisions = revisions.slice(0, 64);
}
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`;
return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {

8
packages/db/prisma/migrations/20250320215449_unique_connection_name_constraint_within_org/migration.sql Normal file
View file

@ -0,0 +1,8 @@
/*
Warnings:
- A unique constraint covering the columns `[name,orgId]` on the table `Connection` will be added. If there are existing duplicate values, this will fail.
*/
-- CreateIndex
CREATE UNIQUE INDEX "Connection_name_orgId_key" ON "Connection"("name", "orgId");

2
packages/db/prisma/schema.prisma
View file

@ -79,6 +79,8 @@ model Connection {
// The organization that owns this connection
org Org @relation(fields: [orgId], references: [id], onDelete: Cascade)
orgId Int
@@unique([name, orgId])
}
model RepoToConnection {

37
packages/schemas/src/v3/connection.schema.ts
View file

@ -19,17 +19,34 @@ const schema = {
"secret": "SECRET_KEY"
}
],
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
"anyOf": [
{
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
}
},
"required": [
"secret"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"env": {
"type": "string",
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
}
},
"required": [
"env"
],
"additionalProperties": false
}
},
"required": [
"secret"
],
"additionalProperties": false
]
},
"url": {
"type": "string",

78
packages/schemas/src/v3/connection.type.ts
View file

@ -11,7 +11,22 @@ export interface GithubConnectionConfig {
* GitHub Configuration
*/
type: "github";
token?: Token;
/**
* A Personal Access Token (PAT).
*/
token?:
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
/**
* The URL of the GitHub host. Defaults to https://github.com
*/
@ -67,15 +82,6 @@ export interface GithubConnectionConfig {
};
revisions?: GitRevisions;
}
/**
* A Personal Access Token (PAT).
*/
export interface Token {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
/**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
*/
@ -94,7 +100,22 @@ export interface GitlabConnectionConfig {
* GitLab Configuration
*/
type: "gitlab";
token?: Token1;
/**
* An authentication token.
*/
token?:
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
/**
* The URL of the GitLab host. Defaults to https://gitlab.com
*/
@ -141,21 +162,27 @@ export interface GitlabConnectionConfig {
};
revisions?: GitRevisions;
}
/**
* An authentication token.
*/
export interface Token1 {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
export interface GiteaConnectionConfig {
/**
* Gitea Configuration
*/
type: "gitea";
token?: Token2;
/**
* A Personal Access Token (PAT).
*/
token?:
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
/**
* The URL of the Gitea host. Defaults to https://gitea.com
*/
@ -188,15 +215,6 @@ export interface GiteaConnectionConfig {
};
revisions?: GitRevisions;
}
/**
* A Personal Access Token (PAT).
*/
export interface Token2 {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
export interface GerritConnectionConfig {
/**
* Gerrit Configuration

37
packages/schemas/src/v3/gitea.schema.ts
View file

@ -15,17 +15,34 @@ const schema = {
"secret": "SECRET_KEY"
}
],
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
"anyOf": [
{
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
}
},
"required": [
"secret"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"env": {
"type": "string",
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
}
},
"required": [
"env"
],
"additionalProperties": false
}
},
"required": [
"secret"
],
"additionalProperties": false
]
},
"url": {
"type": "string",

26
packages/schemas/src/v3/gitea.type.ts
View file

@ -5,7 +5,22 @@ export interface GiteaConnectionConfig {
* Gitea Configuration
*/
type: "gitea";
token?: Token;
/**
* A Personal Access Token (PAT).
*/
token?:
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
/**
* The URL of the Gitea host. Defaults to https://gitea.com
*/
@ -38,15 +53,6 @@ export interface GiteaConnectionConfig {
};
revisions?: GitRevisions;
}
/**
* A Personal Access Token (PAT).
*/
export interface Token {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
/**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
*/

37
packages/schemas/src/v3/github.schema.ts
View file

@ -15,17 +15,34 @@ const schema = {
"secret": "SECRET_KEY"
}
],
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
"anyOf": [
{
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
}
},
"required": [
"secret"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"env": {
"type": "string",
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
}
},
"required": [
"env"
],
"additionalProperties": false
}
},
"required": [
"secret"
],
"additionalProperties": false
]
},
"url": {
"type": "string",

26
packages/schemas/src/v3/github.type.ts
View file

@ -5,7 +5,22 @@ export interface GithubConnectionConfig {
* GitHub Configuration
*/
type: "github";
token?: Token;
/**
* A Personal Access Token (PAT).
*/
token?:
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
/**
* The URL of the GitHub host. Defaults to https://github.com
*/
@ -61,15 +76,6 @@ export interface GithubConnectionConfig {
};
revisions?: GitRevisions;
}
/**
* A Personal Access Token (PAT).
*/
export interface Token {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
/**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
*/

37
packages/schemas/src/v3/gitlab.schema.ts
View file

@ -15,17 +15,34 @@ const schema = {
"secret": "SECRET_KEY"
}
],
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
"anyOf": [
{
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
}
},
"required": [
"secret"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"env": {
"type": "string",
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
}
},
"required": [
"env"
],
"additionalProperties": false
}
},
"required": [
"secret"
],
"additionalProperties": false
]
},
"url": {
"type": "string",

26
packages/schemas/src/v3/gitlab.type.ts
View file

@ -5,7 +5,22 @@ export interface GitlabConnectionConfig {
* GitLab Configuration
*/
type: "gitlab";
token?: Token;
/**
* An authentication token.
*/
token?:
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
/**
* The URL of the GitLab host. Defaults to https://gitlab.com
*/
@ -52,15 +67,6 @@ export interface GitlabConnectionConfig {
};
revisions?: GitRevisions;
}
/**
* An authentication token.
*/
export interface Token {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
/**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
*/

580
packages/schemas/src/v3/index.schema.ts Normal file
View file

@ -0,0 +1,580 @@
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
const schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"title": "SourcebotConfig",
"definitions": {
"Settings": {
"type": "object",
"description": "Defines the globabl settings for Sourcebot.",
"properties": {
"maxFileSize": {
"type": "number",
"description": "The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed."
},
"maxTrigramCount": {
"type": "number",
"description": "The maximum number of trigrams per document. Files that exceed this maximum will not be indexed."
},
"reindexIntervalMs": {
"type": "number",
"description": "The interval (in milliseconds) at which the indexer should re-index all repositories."
},
"resyncConnectionPollingIntervalMs": {
"type": "number",
"description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced."
},
"reindexRepoPollingIntervalMs": {
"type": "number",
"description": "The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed."
},
"indexConcurrencyMultiple": {
"type": "number",
"description": "The multiple of the number of CPUs to use for indexing."
},
"configSyncConcurrencyMultiple": {
"type": "number",
"description": "The multiple of the number of CPUs to use for syncing the configuration."
},
"gcConcurrencyMultiple": {
"type": "number",
"description": "The multiple of the number of CPUs to use for garbage collection."
},
"gcGracePeriodMs": {
"type": "number",
"description": "The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded."
},
"repoIndexTimeoutMs": {
"type": "number",
"description": "The timeout (in milliseconds) for a repo indexing to timeout."
}
},
"additionalProperties": false
}
},
"properties": {
"$schema": {
"type": "string"
},
"settings": {
"$ref": "#/definitions/Settings"
},
"connections": {
"type": "object",
"description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.",
"patternProperties": {
"^[a-zA-Z0-9_-]+$": {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ConnectionConfig",
"oneOf": [
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"title": "GithubConnectionConfig",
"properties": {
"type": {
"const": "github",
"description": "GitHub Configuration"
},
"token": {
"description": "A Personal Access Token (PAT).",
"examples": [
{
"secret": "SECRET_KEY"
}
],
"anyOf": [
{
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
}
},
"required": [
"secret"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"env": {
"type": "string",
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
}
},
"required": [
"env"
],
"additionalProperties": false
}
]
},
"url": {
"type": "string",
"format": "url",
"default": "https://github.com",
"description": "The URL of the GitHub host. Defaults to https://github.com",
"examples": [
"https://github.com",
"https://github.example.com"
],
"pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
},
"users": {
"type": "array",
"items": {
"type": "string",
"pattern": "^[\\w.-]+$"
},
"default": [],
"examples": [
[
"torvalds",
"DHH"
]
],
"description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property."
},
"orgs": {
"type": "array",
"items": {
"type": "string",
"pattern": "^[\\w.-]+$"
},
"default": [],
"examples": [
[
"my-org-name"
],
[
"sourcebot-dev",
"commaai"
]
],
"description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property."
},
"repos": {
"type": "array",
"items": {
"type": "string",
"pattern": "^[\\w.-]+\\/[\\w.-]+$"
},
"default": [],
"description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'."
},
"topics": {
"type": "array",
"items": {
"type": "string"
},
"minItems": 1,
"default": [],
"description": "List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.",
"examples": [
[
"docs",
"core"
]
]
},
"exclude": {
"type": "object",
"properties": {
"forks": {
"type": "boolean",
"default": false,
"description": "Exclude forked repositories from syncing."
},
"archived": {
"type": "boolean",
"default": false,
"description": "Exclude archived repositories from syncing."
},
"repos": {
"type": "array",
"items": {
"type": "string"
},
"default": [],
"description": "List of individual repositories to exclude from syncing. Glob patterns are supported."
},
"topics": {
"type": "array",
"items": {
"type": "string"
},
"default": [],
"description": "List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.",
"examples": [
[
"tests",
"ci"
]
]
},
"size": {
"type": "object",
"description": "Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned.",
"properties": {
"min": {
"type": "integer",
"description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing."
},
"max": {
"type": "integer",
"description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing."
}
},
"additionalProperties": false
}
},
"additionalProperties": false
},
"revisions": {
"type": "object",
"description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.",
"properties": {
"branches": {
"type": "array",
"description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.",
"items": {
"type": "string"
},
"examples": [
[
"main",
"release/*"
],
[
"**"
]
],
"default": []
},
"tags": {
"type": "array",
"description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.",
"items": {
"type": "string"
},
"examples": [
[
"latest",
"v2.*.*"
],
[
"**"
]
],
"default": []
}
},
"additionalProperties": false
}
},
"required": [
"type"
],
"additionalProperties": false
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"title": "GitlabConnectionConfig",
"properties": {
"type": {
"const": "gitlab",
"description": "GitLab Configuration"
},
"token": {
"$ref": "#/properties/connections/patternProperties/%5E%5Ba-zA-Z0-9_-%5D%2B%24/oneOf/0/properties/token",
"description": "An authentication token.",
"examples": [
{
"secret": "SECRET_KEY"
}
]
},
"url": {
"type": "string",
"format": "url",
"default": "https://gitlab.com",
"description": "The URL of the GitLab host. Defaults to https://gitlab.com",
"examples": [
"https://gitlab.com",
"https://gitlab.example.com"
],
"pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
},
"all": {
"type": "boolean",
"default": false,
"description": "Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com ."
},
"users": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property."
},
"groups": {
"type": "array",
"items": {
"type": "string"
},
"examples": [
[
"my-group"
],
[
"my-group/sub-group-a",
"my-group/sub-group-b"
]
],
"description": "List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`)."
},
"projects": {
"type": "array",
"items": {
"type": "string"
},
"examples": [
[
"my-group/my-project"
],
[
"my-group/my-sub-group/my-project"
]
],
"description": "List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/"
},
"topics": {
"type": "array",
"items": {
"type": "string"
},
"minItems": 1,
"description": "List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.",
"examples": [
[
"docs",
"core"
]
]
},
"exclude": {
"type": "object",
"properties": {
"forks": {
"type": "boolean",
"default": false,
"description": "Exclude forked projects from syncing."
},
"archived": {
"type": "boolean",
"default": false,
"description": "Exclude archived projects from syncing."
},
"projects": {
"type": "array",
"items": {
"type": "string"
},
"default": [],
"examples": [
[
"my-group/my-project"
]
],
"description": "List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/"
},
"topics": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.",
"examples": [
[
"tests",
"ci"
]
]
}
},
"additionalProperties": false
},
"revisions": {
"$ref": "#/properties/connections/patternProperties/%5E%5Ba-zA-Z0-9_-%5D%2B%24/oneOf/0/properties/revisions"
}
},
"required": [
"type"
],
"additionalProperties": false
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"title": "GiteaConnectionConfig",
"properties": {
"type": {
"const": "gitea",
"description": "Gitea Configuration"
},
"token": {
"$ref": "#/properties/connections/patternProperties/%5E%5Ba-zA-Z0-9_-%5D%2B%24/oneOf/0/properties/token",
"description": "A Personal Access Token (PAT).",
"examples": [
{
"secret": "SECRET_KEY"
}
]
},
"url": {
"type": "string",
"format": "url",
"default": "https://gitea.com",
"description": "The URL of the Gitea host. Defaults to https://gitea.com",
"examples": [
"https://gitea.com",
"https://gitea.example.com"
],
"pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
},
"orgs": {
"type": "array",
"items": {
"type": "string"
},
"examples": [
[
"my-org-name"
]
],
"description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:organization scope."
},
"repos": {
"type": "array",
"items": {
"type": "string",
"pattern": "^[\\w.-]+\\/[\\w.-]+$"
},
"description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'."
},
"users": {
"type": "array",
"items": {
"type": "string"
},
"examples": [
[
"username-1",
"username-2"
]
],
"description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:user scope."
},
"exclude": {
"type": "object",
"properties": {
"forks": {
"type": "boolean",
"default": false,
"description": "Exclude forked repositories from syncing."
},
"archived": {
"type": "boolean",
"default": false,
"description": "Exclude archived repositories from syncing."
},
"repos": {
"type": "array",
"items": {
"type": "string"
},
"default": [],
"description": "List of individual repositories to exclude from syncing. Glob patterns are supported."
}
},
"additionalProperties": false
},
"revisions": {
"$ref": "#/properties/connections/patternProperties/%5E%5Ba-zA-Z0-9_-%5D%2B%24/oneOf/0/properties/revisions"
}
},
"required": [
"type"
],
"additionalProperties": false
},
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"title": "GerritConnectionConfig",
"properties": {
"type": {
"const": "gerrit",
"description": "Gerrit Configuration"
},
"url": {
"type": "string",
"format": "url",
"description": "The URL of the Gerrit host.",
"examples": [
"https://gerrit.example.com"
],
"pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
},
"projects": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported",
"examples": [
[
"project1/repo1",
"project2/**"
]
]
},
"exclude": {
"type": "object",
"properties": {
"projects": {
"type": "array",
"items": {
"type": "string"
},
"examples": [
[
"project1/repo1",
"project2/**"
]
],
"description": "List of specific projects to exclude from syncing."
}
},
"additionalProperties": false
}
},
"required": [
"type",
"url"
],
"additionalProperties": false
}
]
}
},
"additionalProperties": false
}
},
"additionalProperties": false
} as const;
export { schema as indexSchema };

299
packages/schemas/src/v3/index.type.ts Normal file
View file

@ -0,0 +1,299 @@
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
/**
* This interface was referenced by `undefined`'s JSON-Schema definition
* via the `patternProperty` "^[a-zA-Z0-9_-]+$".
*/
export type ConnectionConfig =
| GithubConnectionConfig
| GitlabConnectionConfig
| GiteaConnectionConfig
| GerritConnectionConfig;
export interface SourcebotConfig {
$schema?: string;
settings?: Settings;
/**
* Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.
*/
connections?: {
[k: string]: ConnectionConfig;
};
}
/**
* Defines the globabl settings for Sourcebot.
*
* This interface was referenced by `SourcebotConfig`'s JSON-Schema
* via the `definition` "Settings".
*/
export interface Settings {
/**
* The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed.
*/
maxFileSize?: number;
/**
* The maximum number of trigrams per document. Files that exceed this maximum will not be indexed.
*/
maxTrigramCount?: number;
/**
* The interval (in milliseconds) at which the indexer should re-index all repositories.
*/
reindexIntervalMs?: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced.
*/
resyncConnectionPollingIntervalMs?: number;
/**
* The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
*/
reindexRepoPollingIntervalMs?: number;
/**
* The multiple of the number of CPUs to use for indexing.
*/
indexConcurrencyMultiple?: number;
/**
* The multiple of the number of CPUs to use for syncing the configuration.
*/
configSyncConcurrencyMultiple?: number;
/**
* The multiple of the number of CPUs to use for garbage collection.
*/
gcConcurrencyMultiple?: number;
/**
* The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded.
*/
gcGracePeriodMs?: number;
/**
* The timeout (in milliseconds) for a repo indexing to timeout.
*/
repoIndexTimeoutMs?: number;
}
export interface GithubConnectionConfig {
/**
* GitHub Configuration
*/
type: "github";
/**
* A Personal Access Token (PAT).
*/
token?:
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
/**
* The URL of the GitHub host. Defaults to https://github.com
*/
url?: string;
/**
* List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property.
*/
users?: string[];
/**
* List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property.
*/
orgs?: string[];
/**
* List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'.
*/
repos?: string[];
/**
* List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.
*
* @minItems 1
*/
topics?: string[];
exclude?: {
/**
* Exclude forked repositories from syncing.
*/
forks?: boolean;
/**
* Exclude archived repositories from syncing.
*/
archived?: boolean;
/**
* List of individual repositories to exclude from syncing. Glob patterns are supported.
*/
repos?: string[];
/**
* List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.
*/
topics?: string[];
/**
* Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned.
*/
size?: {
/**
* Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing.
*/
min?: number;
/**
* Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing.
*/
max?: number;
};
};
revisions?: GitRevisions;
}
/**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.
*/
export interface GitRevisions {
/**
* List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.
*/
branches?: string[];
/**
* List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.
*/
tags?: string[];
}
export interface GitlabConnectionConfig {
/**
* GitLab Configuration
*/
type: "gitlab";
/**
* An authentication token.
*/
token?:
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
/**
* The URL of the GitLab host. Defaults to https://gitlab.com
*/
url?: string;
/**
* Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com .
*/
all?: boolean;
/**
* List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property.
*/
users?: string[];
/**
* List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`).
*/
groups?: string[];
/**
* List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/
*/
projects?: string[];
/**
* List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.
*
* @minItems 1
*/
topics?: string[];
exclude?: {
/**
* Exclude forked projects from syncing.
*/
forks?: boolean;
/**
* Exclude archived projects from syncing.
*/
archived?: boolean;
/**
* List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/
*/
projects?: string[];
/**
* List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.
*/
topics?: string[];
};
revisions?: GitRevisions;
}
export interface GiteaConnectionConfig {
/**
* Gitea Configuration
*/
type: "gitea";
/**
* A Personal Access Token (PAT).
*/
token?:
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
/**
* The URL of the Gitea host. Defaults to https://gitea.com
*/
url?: string;
/**
* List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:organization scope.
*/
orgs?: string[];
/**
* List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'.
*/
repos?: string[];
/**
* List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:user scope.
*/
users?: string[];
exclude?: {
/**
* Exclude forked repositories from syncing.
*/
forks?: boolean;
/**
* Exclude archived repositories from syncing.
*/
archived?: boolean;
/**
* List of individual repositories to exclude from syncing. Glob patterns are supported.
*/
repos?: string[];
};
revisions?: GitRevisions;
}
export interface GerritConnectionConfig {
/**
* Gerrit Configuration
*/
type: "gerrit";
/**
* The URL of the Gerrit host.
*/
url: string;
/**
* List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported
*/
projects?: string[];
exclude?: {
/**
* List of specific projects to exclude from syncing.
*/
projects?: string[];
};
}

37
packages/schemas/src/v3/shared.schema.ts
View file

@ -4,17 +4,34 @@ const schema = {
"type": "object",
"definitions": {
"Token": {
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
"anyOf": [
{
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
}
},
"required": [
"secret"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"env": {
"type": "string",
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
}
},
"required": [
"env"
],
"additionalProperties": false
}
},
"required": [
"secret"
],
"additionalProperties": false
]
},
"GitRevisions": {
"type": "object",

24
packages/schemas/src/v3/shared.type.ts
View file

@ -1,17 +1,25 @@
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
export interface Shared {
[k: string]: unknown;
}
/**
* This interface was referenced by `Shared`'s JSON-Schema
* via the `definition` "Token".
*/
export interface Token {
/**
* The name of the secret that contains the token.
*/
secret: string;
export type Token =
| {
/**
* The name of the secret that contains the token.
*/
secret: string;
}
| {
/**
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
*/
env: string;
};
export interface Shared {
[k: string]: unknown;
}
/**
* The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.

1
packages/web/package.json
View file

@ -129,6 +129,7 @@
"react-resizable-panels": "^2.1.1",
"server-only": "^0.0.1",
"sharp": "^0.33.5",
"strip-json-comments": "^5.0.1",
"stripe": "^17.6.0",
"tailwind-merge": "^2.5.2",
"tailwind-scrollbar-hide": "^1.1.7",

287
packages/web/src/actions.ts
View file

@ -1,6 +1,7 @@
'use server';
import Ajv from "ajv";
import * as Sentry from '@sentry/nextjs';
import { auth } from "./auth";
import { notAuthenticated, notFound, ServiceError, unexpectedError, orgInvalidSubscription, secretAlreadyExists, stripeClientNotInitialized } from "@/lib/serviceError";
import { prisma } from "@/prisma";
@ -11,7 +12,7 @@ import { githubSchema } from "@sourcebot/schemas/v3/github.schema";
import { gitlabSchema } from "@sourcebot/schemas/v3/gitlab.schema";
import { giteaSchema } from "@sourcebot/schemas/v3/gitea.schema";
import { gerritSchema } from "@sourcebot/schemas/v3/gerrit.schema";
import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, GerritConnectionConfig, ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
import { decrypt, encrypt } from "@sourcebot/crypto"
import { getConnection } from "./data/connection";
import { ConnectionSyncStatus, Prisma, OrgRole, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db";
@ -32,6 +33,22 @@ const ajv = new Ajv({
validateFormats: false,
});
/**
* "Service Error Wrapper".
*
* Captures any thrown exceptions and converts them to a unexpected
* service error. Also logs them with Sentry.
*/
export const sew = async <T>(fn: () => Promise<T>): Promise<T | ServiceError> => {
try {
return await fn();
} catch (e) {
Sentry.captureException(e);
console.error(e);
return unexpectedError(`An unexpected error occurred. Please try again later.`);
}
}
export const withAuth = async <T>(fn: (session: Session) => Promise<T>, allowSingleTenantUnauthedAccess: boolean = false) => {
const session = await auth();
if (!session) {
@ -117,7 +134,9 @@ export const withTenancyModeEnforcement = async<T>(mode: TenancyMode, fn: () =>
return fn();
}
export const createOrg = (name: string, domain: string): Promise<{ id: number } | ServiceError> =>
////// Actions ///////
export const createOrg = (name: string, domain: string): Promise<{ id: number } | ServiceError> => sew(() =>
withTenancyModeEnforcement('multi', () =>
withAuth(async (session) => {
const org = await prisma.org.create({
@ -140,9 +159,9 @@ export const createOrg = (name: string, domain: string): Promise<{ id: number }
return {
id: org.id,
}
}));
})));
export const updateOrgName = async (name: string, domain: string) =>
export const updateOrgName = async (name: string, domain: string) => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const { success } = orgNameSchema.safeParse(name);
@ -163,9 +182,9 @@ export const updateOrgName = async (name: string, domain: string) =>
success: true,
}
}, /* minRequiredRole = */ OrgRole.OWNER)
);
));
export const updateOrgDomain = async (newDomain: string, existingDomain: string) =>
export const updateOrgDomain = async (newDomain: string, existingDomain: string) => sew(() =>
withTenancyModeEnforcement('multi', () =>
withAuth((session) =>
withOrgMembership(session, existingDomain, async ({ orgId }) => {
@ -187,9 +206,9 @@ export const updateOrgDomain = async (newDomain: string, existingDomain: string)
success: true,
}
}, /* minRequiredRole = */ OrgRole.OWNER)
));
)));
export const completeOnboarding = async (domain: string): Promise<{ success: boolean } | ServiceError> =>
export const completeOnboarding = async (domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const org = await prisma.org.findUnique({
@ -230,9 +249,9 @@ export const completeOnboarding = async (domain: string): Promise<{ success: boo
success: true,
}
})
);
));
export const getSecrets = (domain: string): Promise<{ createdAt: Date; key: string; }[] | ServiceError> =>
export const getSecrets = (domain: string): Promise<{ createdAt: Date; key: string; }[] | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const secrets = await prisma.secret.findMany({
@ -249,44 +268,41 @@ export const getSecrets = (domain: string): Promise<{ createdAt: Date; key: stri
key: secret.key,
createdAt: secret.createdAt,
}));
}));
})));
export const createSecret = async (key: string, value: string, domain: string): Promise<{ success: boolean } | ServiceError> =>
export const createSecret = async (key: string, value: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
try {
const encrypted = encrypt(value);
const existingSecret = await prisma.secret.findUnique({
where: {
orgId_key: {
orgId,
key,
}
}
});
if (existingSecret) {
return secretAlreadyExists();
}
await prisma.secret.create({
data: {
const encrypted = encrypt(value);
const existingSecret = await prisma.secret.findUnique({
where: {
orgId_key: {
orgId,
key,
encryptedValue: encrypted.encryptedData,
iv: encrypted.iv,
}
});
} catch {
return unexpectedError(`Failed to create secret`);
}
});
if (existingSecret) {
return secretAlreadyExists();
}
await prisma.secret.create({
data: {
orgId,
key,
encryptedValue: encrypted.encryptedData,
iv: encrypted.iv,
}
});
return {
success: true,
}
}));
})));
export const checkIfSecretExists = async (key: string, domain: string): Promise<boolean | ServiceError> =>
export const checkIfSecretExists = async (key: string, domain: string): Promise<boolean | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const secret = await prisma.secret.findUnique({
@ -299,9 +315,9 @@ export const checkIfSecretExists = async (key: string, domain: string): Promise<
});
return !!secret;
}));
})));
export const deleteSecret = async (key: string, domain: string): Promise<{ success: boolean } | ServiceError> =>
export const deleteSecret = async (key: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
await prisma.secret.delete({
@ -316,10 +332,10 @@ export const deleteSecret = async (key: string, domain: string): Promise<{ succe
return {
success: true,
}
}));
})));
export const getConnections = async (domain: string, filter: { status?: ConnectionSyncStatus[] } = {}) =>
export const getConnections = async (domain: string, filter: { status?: ConnectionSyncStatus[] } = {}) => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const connections = await prisma.connection.findMany({
@ -352,10 +368,9 @@ export const getConnections = async (domain: string, filter: { status?: Connecti
repoIndexingStatus: repo.repoIndexingStatus,
})),
}));
})
);
})));
export const getConnectionInfo = async (connectionId: number, domain: string) =>
export const getConnectionInfo = async (connectionId: number, domain: string) => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const connection = await prisma.connection.findUnique({
@ -382,10 +397,9 @@ export const getConnectionInfo = async (connectionId: number, domain: string) =>
syncedAt: connection.syncedAt ?? undefined,
numLinkedRepos: connection.repos.length,
}
})
);
})));
export const getRepos = async (domain: string, filter: { status?: RepoIndexingStatus[], connectionId?: number } = {}) =>
export const getRepos = async (domain: string, filter: { status?: RepoIndexingStatus[], connectionId?: number } = {}) => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const repos = await prisma.repo.findMany({
@ -425,9 +439,9 @@ export const getRepos = async (domain: string, filter: { status?: RepoIndexingSt
repoIndexingStatus: repo.repoIndexingStatus,
}));
}
), /* allowSingleTenantUnauthedAccess = */ true);
), /* allowSingleTenantUnauthedAccess = */ true));
export const createConnection = async (name: string, type: string, connectionConfig: string, domain: string): Promise<{ id: number } | ServiceError> =>
export const createConnection = async (name: string, type: string, connectionConfig: string, domain: string): Promise<{ id: number } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const parsedConfig = parseConnectionConfig(type, connectionConfig);
@ -435,6 +449,23 @@ export const createConnection = async (name: string, type: string, connectionCon
return parsedConfig;
}
const existingConnectionWithName = await prisma.connection.findUnique({
where: {
name_orgId: {
orgId,
name,
}
}
});
if (existingConnectionWithName) {
return {
statusCode: StatusCodes.BAD_REQUEST,
errorCode: ErrorCode.CONNECTION_ALREADY_EXISTS,
message: "A connection with this name already exists.",
} satisfies ServiceError;
}
const connection = await prisma.connection.create({
data: {
orgId,
@ -448,9 +479,9 @@ export const createConnection = async (name: string, type: string, connectionCon
id: connection.id,
}
})
);
));
export const updateConnectionDisplayName = async (connectionId: number, name: string, domain: string): Promise<{ success: boolean } | ServiceError> =>
export const updateConnectionDisplayName = async (connectionId: number, name: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const connection = await getConnection(connectionId, orgId);
@ -458,6 +489,23 @@ export const updateConnectionDisplayName = async (connectionId: number, name: st
return notFound();
}
const existingConnectionWithName = await prisma.connection.findUnique({
where: {
name_orgId: {
orgId,
name,
}
}
});
if (existingConnectionWithName) {
return {
statusCode: StatusCodes.BAD_REQUEST,
errorCode: ErrorCode.CONNECTION_ALREADY_EXISTS,
message: "A connection with this name already exists.",
} satisfies ServiceError;
}
await prisma.connection.update({
where: {
id: connectionId,
@ -471,9 +519,10 @@ export const updateConnectionDisplayName = async (connectionId: number, name: st
return {
success: true,
}
}));
})
));
export const updateConnectionConfigAndScheduleSync = async (connectionId: number, config: string, domain: string): Promise<{ success: boolean } | ServiceError> =>
export const updateConnectionConfigAndScheduleSync = async (connectionId: number, config: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const connection = await getConnection(connectionId, orgId);
@ -510,9 +559,10 @@ export const updateConnectionConfigAndScheduleSync = async (connectionId: number
return {
success: true,
}
}));
})
));
export const flagConnectionForSync = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> =>
export const flagConnectionForSync = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const connection = await getConnection(connectionId, orgId);
@ -532,9 +582,10 @@ export const flagConnectionForSync = async (connectionId: number, domain: string
return {
success: true,
}
}));
})
));
export const flagReposForIndex = async (repoIds: number[], domain: string) =>
export const flagReposForIndex = async (repoIds: number[], domain: string) => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
await prisma.repo.updateMany({
@ -551,9 +602,9 @@ export const flagReposForIndex = async (repoIds: number[], domain: string) =>
success: true,
}
})
);
));
export const deleteConnection = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> =>
export const deleteConnection = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const connection = await getConnection(connectionId, orgId);
@ -571,16 +622,17 @@ export const deleteConnection = async (connectionId: number, domain: string): Pr
return {
success: true,
}
}));
})
));
export const getCurrentUserRole = async (domain: string): Promise<OrgRole | ServiceError> =>
export const getCurrentUserRole = async (domain: string): Promise<OrgRole | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ userRole }) => {
return userRole;
})
);
));
export const createInvites = async (emails: string[], domain: string): Promise<{ success: boolean } | ServiceError> =>
export const createInvites = async (emails: string[], domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
// Check for existing invites
@ -691,9 +743,9 @@ export const createInvites = async (emails: string[], domain: string): Promise<{
success: true,
}
}, /* minRequiredRole = */ OrgRole.OWNER)
);
));
export const cancelInvite = async (inviteId: string, domain: string): Promise<{ success: boolean } | ServiceError> =>
export const cancelInvite = async (inviteId: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const invite = await prisma.invite.findUnique({
@ -717,9 +769,9 @@ export const cancelInvite = async (inviteId: string, domain: string): Promise<{
success: true,
}
}, /* minRequiredRole = */ OrgRole.OWNER)
);
));
export const getMe = async () =>
export const getMe = async () => sew(() =>
withAuth(async (session) => {
const user = await prisma.user.findUnique({
where: {
@ -749,9 +801,9 @@ export const getMe = async () =>
name: org.org.name,
}))
}
});
}));
export const redeemInvite = async (inviteId: string): Promise<{ success: boolean } | ServiceError> =>
export const redeemInvite = async (inviteId: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth(async () => {
const invite = await prisma.invite.findUnique({
where: {
@ -818,9 +870,9 @@ export const redeemInvite = async (inviteId: string): Promise<{ success: boolean
return {
success: true,
}
});
}));
export const getInviteInfo = async (inviteId: string) =>
export const getInviteInfo = async (inviteId: string) => sew(() =>
withAuth(async () => {
const user = await getMe();
if (isServiceError(user)) {
@ -860,9 +912,9 @@ export const getInviteInfo = async (inviteId: string) =>
email: user.email!,
}
}
});
}));
export const transferOwnership = async (newOwnerId: string, domain: string): Promise<{ success: boolean } | ServiceError> =>
export const transferOwnership = async (newOwnerId: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const currentUserId = session.user.id;
@ -921,9 +973,9 @@ export const transferOwnership = async (newOwnerId: string, domain: string): Pro
success: true,
}
}, /* minRequiredRole = */ OrgRole.OWNER)
);
));
export const createOnboardingSubscription = async (domain: string) =>
export const createOnboardingSubscription = async (domain: string) => sew(() =>
withAuth(async (session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const org = await prisma.org.findUnique({
@ -1028,9 +1080,9 @@ export const createOnboardingSubscription = async (domain: string) =>
}, /* minRequiredRole = */ OrgRole.OWNER)
);
));
export const createStripeCheckoutSession = async (domain: string) =>
export const createStripeCheckoutSession = async (domain: string) => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const org = await prisma.org.findUnique({
@ -1090,9 +1142,9 @@ export const createStripeCheckoutSession = async (domain: string) =>
url: stripeSession.url,
}
})
);
));
export const getCustomerPortalSessionLink = async (domain: string): Promise<string | ServiceError> =>
export const getCustomerPortalSessionLink = async (domain: string): Promise<string | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const org = await prisma.org.findUnique({
@ -1117,16 +1169,16 @@ export const getCustomerPortalSessionLink = async (domain: string): Promise<stri
return portalSession.url;
}, /* minRequiredRole = */ OrgRole.OWNER)
);
));
export const fetchSubscription = (domain: string): Promise<Stripe.Subscription | ServiceError> =>
export const fetchSubscription = (domain: string): Promise<Stripe.Subscription | ServiceError> => sew(() =>
withAuth(async (session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
return _fetchSubscriptionForOrg(orgId, prisma);
})
);
));
export const getSubscriptionBillingEmail = async (domain: string): Promise<string | ServiceError> =>
export const getSubscriptionBillingEmail = async (domain: string): Promise<string | ServiceError> => sew(() =>
withAuth(async (session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const org = await prisma.org.findUnique({
@ -1149,9 +1201,9 @@ export const getSubscriptionBillingEmail = async (domain: string): Promise<strin
}
return customer.email!;
})
);
));
export const changeSubscriptionBillingEmail = async (domain: string, newEmail: string): Promise<{ success: boolean } | ServiceError> =>
export const changeSubscriptionBillingEmail = async (domain: string, newEmail: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const org = await prisma.org.findUnique({
@ -1176,9 +1228,9 @@ export const changeSubscriptionBillingEmail = async (domain: string, newEmail: s
success: true,
}
}, /* minRequiredRole = */ OrgRole.OWNER)
);
));
export const checkIfOrgDomainExists = async (domain: string): Promise<boolean | ServiceError> =>
export const checkIfOrgDomainExists = async (domain: string): Promise<boolean | ServiceError> => sew(() =>
withAuth(async () => {
const org = await prisma.org.findFirst({
where: {
@ -1187,9 +1239,9 @@ export const checkIfOrgDomainExists = async (domain: string): Promise<boolean |
});
return !!org;
});
}));
export const removeMemberFromOrg = async (memberId: string, domain: string): Promise<{ success: boolean } | ServiceError> =>
export const removeMemberFromOrg = async (memberId: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth(async (session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const targetMember = await prisma.userToOrg.findUnique({
@ -1246,9 +1298,9 @@ export const removeMemberFromOrg = async (memberId: string, domain: string): Pro
success: true,
}
}, /* minRequiredRole = */ OrgRole.OWNER)
);
));
export const leaveOrg = async (domain: string): Promise<{ success: boolean } | ServiceError> =>
export const leaveOrg = async (domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
withAuth(async (session) =>
withOrgMembership(session, domain, async ({ orgId, userRole }) => {
if (userRole === OrgRole.OWNER) {
@ -1300,9 +1352,9 @@ export const leaveOrg = async (domain: string): Promise<{ success: boolean } | S
success: true,
}
})
);
));
export const getSubscriptionData = async (domain: string) =>
export const getSubscriptionData = async (domain: string) => sew(() =>
withAuth(async (session) =>
withOrgMembership(session, domain, async () => {
const subscription = await fetchSubscription(domain);
@ -1322,9 +1374,9 @@ export const getSubscriptionData = async (domain: string) =>
status: subscription.status,
}
})
);
));
export const getOrgMembers = async (domain: string) =>
export const getOrgMembers = async (domain: string) => sew(() =>
withAuth(async (session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const members = await prisma.userToOrg.findMany({
@ -1345,9 +1397,9 @@ export const getOrgMembers = async (domain: string) =>
joinedAt: member.joinedAt,
}));
})
);
));
export const getOrgInvites = async (domain: string) =>
export const getOrgInvites = async (domain: string) => sew(() =>
withAuth(async (session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const invites = await prisma.invite.findMany({
@ -1362,11 +1414,12 @@ export const getOrgInvites = async (domain: string) =>
createdAt: invite.createdAt,
}));
})
);
));
export const dismissMobileUnsupportedSplashScreen = async () => {
export const dismissMobileUnsupportedSplashScreen = async () => sew(async () => {
await cookies().set(MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, 'true');
}
return true;
});
////// Helpers ///////
@ -1442,41 +1495,35 @@ const parseConnectionConfig = (connectionType: string, config: string) => {
} satisfies ServiceError;
}
if ('token' in parsedConfig && parsedConfig.token && 'env' in parsedConfig.token) {
return {
statusCode: StatusCodes.BAD_REQUEST,
errorCode: ErrorCode.INVALID_REQUEST_BODY,
message: "Environment variables are not supported for connections created in the web UI. Please use a secret instead.",
} satisfies ServiceError;
}
const { numRepos, hasToken } = (() => {
switch (connectionType) {
switch (parsedConfig.type) {
case "gitea":
case "github": {
const githubConfig = parsedConfig as GithubConnectionConfig;
return {
numRepos: githubConfig.repos?.length,
hasToken: !!githubConfig.token,
numRepos: parsedConfig.repos?.length,
hasToken: !!parsedConfig.token,
}
}
case "gitlab": {
const gitlabConfig = parsedConfig as GitlabConnectionConfig;
return {
numRepos: gitlabConfig.projects?.length,
hasToken: !!gitlabConfig.token,
}
}
case "gitea": {
const giteaConfig = parsedConfig as GiteaConnectionConfig;
return {
numRepos: giteaConfig.repos?.length,
hasToken: !!giteaConfig.token,
numRepos: parsedConfig.projects?.length,
hasToken: !!parsedConfig.token,
}
}
case "gerrit": {
const gerritConfig = parsedConfig as GerritConnectionConfig;
return {
numRepos: gerritConfig.projects?.length,
numRepos: parsedConfig.projects?.length,
hasToken: true, // gerrit doesn't use a token atm
}
}
default:
return {
numRepos: undefined,
hasToken: true
}
}
})();

2
packages/web/src/app/[domain]/components/navigationMenu.tsx
View file

@ -27,7 +27,7 @@ export const NavigationMenu = async ({
const subscription = IS_BILLING_ENABLED ? await getSubscriptionData(domain) : null;
return (
<div className="flex flex-col w-screen h-fit">
<div className="flex flex-col w-screen h-fit bg-background">
<div className="flex flex-row justify-between items-center py-1.5 px-3">
<div className="flex flex-row items-center">
<Link

8
packages/web/src/app/[domain]/connections/layout.tsx
View file

@ -1,12 +1,18 @@
import { auth } from "@/auth";
import { NavigationMenu } from "../components/navigationMenu";
import { redirect } from "next/navigation";
export default function Layout({
export default async function Layout({
children,
params: { domain },
}: Readonly<{
children: React.ReactNode;
params: { domain: string };
}>) {
const session = await auth();
if (!session) {
return redirect(`/${domain}`);
}
return (
<div className="min-h-screen flex flex-col">

6
packages/web/src/app/[domain]/settings/(general)/page.tsx
View file

@ -3,9 +3,9 @@ import { isServiceError } from "@/lib/utils";
import { getCurrentUserRole } from "@/actions";
import { getOrgFromDomain } from "@/data/org";
import { ChangeOrgDomainCard } from "./components/changeOrgDomainCard";
import { env } from "@/env.mjs";
import { ServiceErrorException } from "@/lib/serviceError";
import { ErrorCode } from "@/lib/errorCodes";
import { headers } from "next/headers";
interface GeneralSettingsPageProps {
params: {
domain: string;
@ -27,6 +27,8 @@ export default async function GeneralSettingsPage({ params: { domain } }: Genera
});
}
const host = (await headers()).get('host') ?? '';
return (
<div className="flex flex-col gap-6">
<div>
@ -41,7 +43,7 @@ export default async function GeneralSettingsPage({ params: { domain } }: Genera
<ChangeOrgDomainCard
orgDomain={org.domain}
currentUserRole={currentUserRole}
rootDomain={env.SOURCEBOT_ROOT_DOMAIN}
rootDomain={host}
/>
</div>
)

13
packages/web/src/app/[domain]/settings/layout.tsx
View file

@ -3,17 +3,24 @@ import { SidebarNav } from "./components/sidebar-nav"
import { NavigationMenu } from "../components/navigationMenu"
import { Header } from "./components/header";
import { IS_BILLING_ENABLED } from "@/lib/stripe";
import { redirect } from "next/navigation";
import { auth } from "@/auth";
export const metadata: Metadata = {
title: "Settings",
}
export default function SettingsLayout({
export default async function SettingsLayout({
children,
params: { domain },
}: Readonly<{
children: React.ReactNode;
params: { domain: string };
}>) {
const session = await auth();
if (!session) {
return redirect(`/${domain}`);
}
const sidebarNavItems = [
{
@ -37,9 +44,9 @@ export default function SettingsLayout({
]
return (
<div className="min-h-screen flex flex-col">
<div className="min-h-screen flex flex-col bg-backgroundSecondary">
<NavigationMenu domain={domain} />
<div className="flex-grow flex justify-center p-4 bg-backgroundSecondary relative">
<div className="flex-grow flex justify-center p-4 relative">
<div className="w-full max-w-6xl p-6">
<Header className="w-full">
<h1 className="text-3xl">Settings</h1>

6
packages/web/src/app/api/(server)/repos/route.ts
View file

@ -2,7 +2,7 @@
import { listRepositories } from "@/lib/server/searchService";
import { NextRequest } from "next/server";
import { withAuth, withOrgMembership } from "@/actions";
import { sew, withAuth, withOrgMembership } from "@/actions";
import { isServiceError } from "@/lib/utils";
import { serviceErrorResponse } from "@/lib/serviceError";
@ -17,10 +17,10 @@ export const GET = async (request: NextRequest) => {
}
const getRepos = (domain: string) =>
const getRepos = (domain: string) => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const response = await listRepositories(orgId);
return response;
}
), /* allowSingleTenantUnauthedAccess */ true);
), /* allowSingleTenantUnauthedAccess */ true));

6
packages/web/src/app/api/(server)/search/route.ts
View file

@ -4,7 +4,7 @@ import { search } from "@/lib/server/searchService";
import { searchRequestSchema } from "@/lib/schemas";
import { isServiceError } from "@/lib/utils";
import { NextRequest } from "next/server";
import { withAuth, withOrgMembership } from "@/actions";
import { sew, withAuth, withOrgMembership } from "@/actions";
import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError";
import { SearchRequest } from "@/lib/types";
@ -25,10 +25,10 @@ export const POST = async (request: NextRequest) => {
return Response.json(response);
}
const postSearch = (request: SearchRequest, domain: string) =>
const postSearch = (request: SearchRequest, domain: string) => sew(() =>
withAuth((session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const response = await search(request, orgId);
return response;
}
), /* allowSingleTenantUnauthedAccess */ true);
), /* allowSingleTenantUnauthedAccess */ true));

6
packages/web/src/app/api/(server)/source/route.ts
View file

@ -5,7 +5,7 @@ import { getFileSource } from "@/lib/server/searchService";
import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError";
import { isServiceError } from "@/lib/utils";
import { NextRequest } from "next/server";
import { withAuth, withOrgMembership } from "@/actions";
import { sew, withAuth, withOrgMembership } from "@/actions";
import { FileSourceRequest } from "@/lib/types";
export const POST = async (request: NextRequest) => {
@ -27,10 +27,10 @@ export const POST = async (request: NextRequest) => {
}
const postSource = (request: FileSourceRequest, domain: string) =>
const postSource = (request: FileSourceRequest, domain: string) => sew(() =>
withAuth(async (session) =>
withOrgMembership(session, domain, async ({ orgId }) => {
const response = await getFileSource(request, orgId);
return response;
}
), /* allowSingleTenantUnauthedAccess */ true);
), /* allowSingleTenantUnauthedAccess */ true));

3
packages/web/src/app/login/components/credentialsForm.tsx
View file

@ -35,9 +35,10 @@ export const CredentialsForm = ({ callbackUrl }: CredentialsFormProps) => {
password: values.password,
redirectTo: callbackUrl ?? "/"
})
.finally(() => {
.catch(() => {
setIsLoading(false);
});
// signIn will redirect on success, so don't set isLoading to false
}
return (

6
packages/web/src/app/onboard/page.tsx
View file

@ -4,7 +4,7 @@ import { redirect } from "next/navigation";
import { OnboardHeader } from "./components/onboardHeader";
import { OnboardingSteps } from "@/lib/constants";
import { LogoutEscapeHatch } from "../components/logoutEscapeHatch";
import { env } from "@/env.mjs";
import { headers } from "next/headers";
export default async function Onboarding() {
const session = await auth();
@ -12,6 +12,8 @@ export default async function Onboarding() {
redirect("/login");
}
const host = (await headers()).get('host') ?? '';
return (
<div className="flex flex-col items-center min-h-screen py-12 px-4 sm:px-12 bg-backgroundSecondary relative">
<OnboardHeader
@ -19,7 +21,7 @@ export default async function Onboarding() {
description="Create a organization for your team to search and share code across your repositories."
step={OnboardingSteps.CreateOrg}
/>
<OrgCreateForm rootDomain={env.SOURCEBOT_ROOT_DOMAIN} />
<OrgCreateForm rootDomain={host} />
<LogoutEscapeHatch className="absolute top-0 right-0 p-4 sm:p-12" />
</div>
);

2
packages/web/src/env.mjs
View file

@ -38,13 +38,13 @@ export const env = createEnv({
// Misc
CONFIG_MAX_REPOS_NO_TOKEN: numberSchema.default(Number.MAX_SAFE_INTEGER),
SOURCEBOT_ROOT_DOMAIN: z.string().default("localhost:3000"),
NODE_ENV: z.enum(["development", "test", "production"]),
SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default('false'),
DATABASE_URL: z.string().url(),
SOURCEBOT_TENANCY_MODE: tenancyModeSchema.default("single"),
SOURCEBOT_AUTH_ENABLED: booleanSchema.default('true'),
CONFIG_PATH: z.string().optional(),
},
// @NOTE: Make sure you destructure all client variables in the
// `experimental__runtimeEnv` block below.

84
packages/web/src/initialize.ts
View file

@ -1,12 +1,26 @@
import { OrgRole } from '@sourcebot/db';
import { ConnectionSyncStatus, OrgRole, Prisma } from '@sourcebot/db';
import { env } from './env.mjs';
import { prisma } from "@/prisma";
import { SINGLE_TENANT_USER_ID, SINGLE_TENANT_ORG_ID, SINGLE_TENANT_ORG_DOMAIN, SINGLE_TENANT_ORG_NAME, SINGLE_TENANT_USER_EMAIL } from './lib/constants';
import { readFile } from 'fs/promises';
import stripJsonComments from 'strip-json-comments';
import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
import { ConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
import Ajv from 'ajv';
const ajv = new Ajv({
validateFormats: false,
});
if (env.SOURCEBOT_AUTH_ENABLED === 'false' && env.SOURCEBOT_TENANCY_MODE === 'multi') {
throw new Error('SOURCEBOT_AUTH_ENABLED must be true when SOURCEBOT_TENANCY_MODE is multi');
}
const isRemotePath = (path: string) => {
return path.startsWith('https://') || path.startsWith('http://');
}
const initSingleTenancy = async () => {
await prisma.org.upsert({
where: {
@ -50,6 +64,74 @@ const initSingleTenancy = async () => {
}
});
}
// Load any connections defined declaratively in the config file.
const configPath = env.CONFIG_PATH;
if (configPath) {
const configContent = await (async () => {
if (isRemotePath(configPath)) {
const response = await fetch(configPath);
if (!response.ok) {
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
}
return response.text();
} else {
return readFile(configPath, {
encoding: 'utf-8',
});
}
})();
const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
const isValidConfig = ajv.validate(indexSchema, config);
if (!isValidConfig) {
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
}
if (config.connections) {
for (const [key, newConnectionConfig] of Object.entries(config.connections)) {
const currentConnection = await prisma.connection.findUnique({
where: {
name_orgId: {
name: key,
orgId: SINGLE_TENANT_ORG_ID,
}
},
select: {
config: true,
}
});
const currentConnectionConfig = currentConnection ? currentConnection.config as unknown as ConnectionConfig : undefined;
const syncNeededOnUpdate = currentConnectionConfig && JSON.stringify(currentConnectionConfig) !== JSON.stringify(newConnectionConfig);
const connectionDb = await prisma.connection.upsert({
where: {
name_orgId: {
name: key,
orgId: SINGLE_TENANT_ORG_ID,
}
},
update: {
config: newConnectionConfig as unknown as Prisma.InputJsonValue,
syncStatus: syncNeededOnUpdate ? ConnectionSyncStatus.SYNC_NEEDED : undefined,
},
create: {
name: key,
connectionType: newConnectionConfig.type,
config: newConnectionConfig as unknown as Prisma.InputJsonValue,
org: {
connect: {
id: SINGLE_TENANT_ORG_ID,
}
}
}
});
console.log(`Upserted connection with name '${key}'. Connection ID: ${connectionDb.id}`);
}
}
}
}
if (env.SOURCEBOT_TENANCY_MODE === 'single') {

1
packages/web/src/lib/errorCodes.ts
View file

@ -14,6 +14,7 @@ export enum ErrorCode {
INVALID_CREDENTIALS = 'INVALID_CREDENTIALS',
INSUFFICIENT_PERMISSIONS = 'INSUFFICIENT_PERMISSIONS',
CONNECTION_NOT_FAILED = 'CONNECTION_NOT_FAILED',
CONNECTION_ALREADY_EXISTS = 'CONNECTION_ALREADY_EXISTS',
OWNER_CANNOT_LEAVE_ORG = 'OWNER_CANNOT_LEAVE_ORG',
INVALID_INVITE = 'INVALID_INVITE',
STRIPE_CHECKOUT_ERROR = 'STRIPE_CHECKOUT_ERROR',

73
schemas/v3/index.json Normal file
View file

@ -0,0 +1,73 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"title": "SourcebotConfig",
"definitions": {
"Settings": {
"type": "object",
"description": "Defines the globabl settings for Sourcebot.",
"properties": {
"maxFileSize": {
"type": "number",
"description": "The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed."
},
"maxTrigramCount": {
"type": "number",
"description": "The maximum number of trigrams per document. Files that exceed this maximum will not be indexed."
},
"reindexIntervalMs": {
"type": "number",
"description": "The interval (in milliseconds) at which the indexer should re-index all repositories."
},
"resyncConnectionPollingIntervalMs": {
"type": "number",
"description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced."
},
"reindexRepoPollingIntervalMs": {
"type": "number",
"description": "The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed."
},
"indexConcurrencyMultiple": {
"type": "number",
"description": "The multiple of the number of CPUs to use for indexing."
},
"configSyncConcurrencyMultiple": {
"type": "number",
"description": "The multiple of the number of CPUs to use for syncing the configuration."
},
"gcConcurrencyMultiple": {
"type": "number",
"description": "The multiple of the number of CPUs to use for garbage collection."
},
"gcGracePeriodMs": {
"type": "number",
"description": "The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded."
},
"repoIndexTimeoutMs": {
"type": "number",
"description": "The timeout (in milliseconds) for a repo indexing to timeout."
}
},
"additionalProperties": false
}
},
"properties": {
"$schema": {
"type": "string"
},
"settings": {
"$ref": "#/definitions/Settings"
},
"connections": {
"type": "object",
"description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.",
"patternProperties": {
"^[a-zA-Z0-9_-]+$": {
"$ref": "./connection.json"
}
},
"additionalProperties": false
}
},
"additionalProperties": false
}

37
schemas/v3/shared.json
View file

@ -3,17 +3,34 @@
"type": "object",
"definitions": {
"Token": {
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
"anyOf": [
{
"type": "object",
"properties": {
"secret": {
"type": "string",
"description": "The name of the secret that contains the token."
}
},
"required": [
"secret"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"env": {
"type": "string",
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
}
},
"required": [
"env"
],
"additionalProperties": false
}
},
"required": [
"secret"
],
"additionalProperties": false
]
},
"GitRevisions": {
"type": "object",

4
yarn.lock
View file

@ -4175,7 +4175,7 @@ ajv@^6.12.4:
ajv@^8.17.1:
version "8.17.1"
resolved "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz"
resolved "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz#37d9a5c776af6bc92d7f4f9510eba4c0a60d11a6"
integrity sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==
dependencies:
fast-deep-equal "^3.1.3"
@ -9600,7 +9600,7 @@ strip-json-comments@^3.1.1:
strip-json-comments@^5.0.1:
version "5.0.1"
resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.1.tgz"
resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.1.tgz#0d8b7d01b23848ed7dbdf4baaaa31a8250d8cfa0"
integrity sha512-0fk9zBqO67Nq5M/m45qHCJxylV/DhBlIOVExqgOMiCCrzrhU6tCibRXNqE3jwJLftzE9SNuZtYbpzcO+i9FiKw==
stripe@^17.6.0: