ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-12 04:15:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Use shared env object in backend

Browse source
This commit is contained in:
bkellam 2025-11-02 12:02:22 -08:00
parent 1d14ddc322
commit c8fecc02fe
15 changed files with 290 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

282
docs/snippets/schemas/v3/index.schema.mdx
View file

@ -135,6 +135,117 @@
}
},
"additionalProperties": false
},
"EnvironmentOverrides": {
"type": "object",
"description": "Environment variable overrides.",
"not": {
"$comment": "List of environment variables that are not allowed to be overridden.",
"anyOf": [
{
"required": [
"CONFIG_PATH"
]
}
]
},
"patternProperties": {
"^[a-zA-Z0-9_-]+$": {
"oneOf": [
{
"type": "object",
"properties": {
"type": {
"const": "token"
},
"value": {
"anyOf": [
{
"type": "object",
"properties": {
"env": {
"type": "string",
"description": "The name of the environment variable that contains the token."
}
},
"required": [
"env"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"googleCloudSecret": {
"type": "string",
"description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
}
},
"required": [
"googleCloudSecret"
],
"additionalProperties": false
}
]
}
},
"required": [
"type",
"value"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"type": {
"const": "string"
},
"value": {
"type": "string"
}
},
"required": [
"type",
"value"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"type": {
"const": "number"
},
"value": {
"type": "number"
}
},
"required": [
"type",
"value"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"type": {
"const": "boolean"
},
"value": {
"type": "boolean"
}
},
"required": [
"type",
"value"
],
"additionalProperties": false
}
]
}
}
}
},
"properties": {
@ -279,25 +390,29 @@
},
"additionalProperties": false
},
"connections": {
"environmentOverrides": {
"type": "object",
"description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.",
"description": "Environment variable overrides.",
"not": {
"$comment": "List of environment variables that are not allowed to be overridden.",
"anyOf": [
{
"required": [
"CONFIG_PATH"
]
}
]
},
"patternProperties": {
"^[a-zA-Z0-9_-]+$": {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ConnectionConfig",
"oneOf": [
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"title": "GithubConnectionConfig",
"properties": {
"type": {
"const": "github",
"description": "GitHub Configuration"
"const": "token"
},
"token": {
"description": "A Personal Access Token (PAT).",
"value": {
"anyOf": [
{
"type": "object",
@ -326,6 +441,113 @@
"additionalProperties": false
}
]
}
},
"required": [
"type",
"value"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"type": {
"const": "string"
},
"value": {
"type": "string"
}
},
"required": [
"type",
"value"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"type": {
"const": "number"
},
"value": {
"type": "number"
}
},
"required": [
"type",
"value"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"type": {
"const": "boolean"
},
"value": {
"type": "boolean"
}
},
"required": [
"type",
"value"
],
"additionalProperties": false
}
]
}
}
},
"connections": {
"type": "object",
"description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.",
"patternProperties": {
"^[a-zA-Z0-9_-]+$": {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ConnectionConfig",
"oneOf": [
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"title": "GithubConnectionConfig",
"properties": {
"type": {
"const": "github",
"description": "GitHub Configuration"
},
"token": {
"anyOf": [
{
"type": "object",
"properties": {
"env": {
"type": "string",
"description": "The name of the environment variable that contains the token."
}
},
"required": [
"env"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"googleCloudSecret": {
"type": "string",
"description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
}
},
"required": [
"googleCloudSecret"
],
"additionalProperties": false
}
],
"description": "A Personal Access Token (PAT)."
},
"url": {
"type": "string",
@ -505,7 +727,6 @@
"description": "GitLab Configuration"
},
"token": {
"description": "An authentication token.",
"anyOf": [
{
"type": "object",
@ -533,7 +754,8 @@
],
"additionalProperties": false
}
]
],
"description": "An authentication token."
},
"url": {
"type": "string",
@ -707,7 +929,6 @@
"description": "Gitea Configuration"
},
"token": {
"description": "A Personal Access Token (PAT).",
"anyOf": [
{
"type": "object",
@ -735,7 +956,8 @@
],
"additionalProperties": false
}
]
],
"description": "A Personal Access Token (PAT)."
},
"url": {
"type": "string",
@ -974,7 +1196,6 @@
"description": "The username to use for authentication. Only needed if token is an app password."
},
"token": {
"description": "An authentication token.",
"anyOf": [
{
"type": "object",
@ -1002,7 +1223,8 @@
],
"additionalProperties": false
}
]
],
"description": "An authentication token."
},
"url": {
"type": "string",
@ -1142,7 +1364,6 @@
"description": "Azure DevOps Configuration"
},
"token": {
"description": "A Personal Access Token (PAT).",
"anyOf": [
{
"type": "object",
@ -1170,7 +1391,8 @@
],
"additionalProperties": false
}
]
],
"description": "A Personal Access Token (PAT)."
},
"url": {
"type": "string",
@ -1426,7 +1648,6 @@
"description": "Optional display name."
},
"accessKeyId": {
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.",
"anyOf": [
{
"type": "object",
@ -1454,10 +1675,10 @@
],
"additionalProperties": false
}
]
],
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable."
},
"accessKeySecret": {
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.",
"anyOf": [
{
"type": "object",
@ -1485,10 +1706,10 @@
],
"additionalProperties": false
}
]
],
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable."
},
"sessionToken": {
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.",
"anyOf": [
{
"type": "object",
@ -1516,7 +1737,8 @@
],
"additionalProperties": false
}
]
],
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable."
},
"region": {
"type": "string",
@ -2855,7 +3077,6 @@
"description": "Optional display name."
},
"accessKeyId": {
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.",
"anyOf": [
{
"type": "object",
@ -2883,10 +3104,10 @@
],
"additionalProperties": false
}
]
],
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable."
},
"accessKeySecret": {
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.",
"anyOf": [
{
"type": "object",
@ -2914,10 +3135,10 @@
],
"additionalProperties": false
}
]
],
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable."
},
"sessionToken": {
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.",
"anyOf": [
{
"type": "object",
@ -2945,7 +3166,8 @@
],
"additionalProperties": false
}
]
],
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable."
},
"region": {
"type": "string",

3
packages/backend/src/connectionManager.ts
View file

@ -2,10 +2,9 @@ import * as Sentry from "@sentry/node";
import { Connection, ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
import { loadConfig } from "@sourcebot/shared";
import { loadConfig, env } from "@sourcebot/shared";
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import { env } from "./env.js";
import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js";
import { Settings } from "./types.js";
import { groupmqLifecycleExceptionWrapper } from "./utils.js";

2
packages/backend/src/constants.ts
View file

@ -1,5 +1,5 @@
import { CodeHostType } from "@sourcebot/db";
import { env } from "./env.js";
import { env } from "@sourcebot/shared";
import path from "path";
export const SINGLE_TENANT_ORG_ID = 1;

3
packages/backend/src/ee/accountPermissionSyncer.ts
View file

@ -1,13 +1,12 @@
import * as Sentry from "@sentry/node";
import { PrismaClient, AccountPermissionSyncJobStatus, Account } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { env, hasEntitlement } from "@sourcebot/shared";
import { Job, Queue, Worker } from "bullmq";
import { Redis } from "ioredis";
import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js";
import { env } from "../env.js";
import { createOctokitFromToken, getReposForAuthenticatedUser } from "../github.js";
import { createGitLabFromOAuthToken, getProjectsForAuthenticatedUser } from "../gitlab.js";
import { hasEntitlement } from "@sourcebot/shared";
import { Settings } from "../types.js";
const LOG_TAG = 'user-permission-syncer';

9
packages/backend/src/ee/githubAppManager.ts
View file

@ -1,10 +1,9 @@
import { loadConfig } from "@sourcebot/shared";
import { env } from "../env.js";
import { createLogger } from "@sourcebot/logger";
import { App } from "@octokit/app";
import { getTokenFromConfig } from "@sourcebot/crypto";
import { PrismaClient } from "@sourcebot/db";
import { App } from "@octokit/app";
import { createLogger } from "@sourcebot/logger";
import { GitHubAppConfig } from "@sourcebot/schemas/v3/index.type";
import { env, loadConfig } from "@sourcebot/shared";
const logger = createLogger('githubAppManager');
const GITHUB_DEFAULT_DEPLOYMENT_HOSTNAME = 'github.com';
@ -45,7 +44,7 @@ export class GithubAppManager {
public async init(db: PrismaClient) {
this.db = db;
const config = await loadConfig(env.CONFIG_PATH!);
const config = await loadConfig(env.CONFIG_PATH);
if (!config.apps) {
return;
}

3
packages/backend/src/ee/repoPermissionSyncer.ts
View file

@ -1,11 +1,10 @@
import * as Sentry from "@sentry/node";
import { PrismaClient, Repo, RepoPermissionSyncJobStatus } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { hasEntitlement } from "@sourcebot/shared";
import { env, hasEntitlement } from "@sourcebot/shared";
import { Job, Queue, Worker } from 'bullmq';
import { Redis } from 'ioredis';
import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js";
import { env } from "../env.js";
import { createOctokitFromToken, getRepoCollaborators, GITHUB_CLOUD_HOSTNAME } from "../github.js";
import { createGitLabFromPersonalAccessToken, getProjectMembers } from "../gitlab.js";
import { Settings } from "../types.js";

64
packages/backend/src/env.ts
View file

@ -1,64 +0,0 @@
import { createEnv } from "@t3-oss/env-core";
import { z } from "zod";
import dotenv from 'dotenv';
// Booleans are specified as 'true' or 'false' strings.
const booleanSchema = z.enum(["true", "false"]);
// Numbers are treated as strings in .env files.
// coerce helps us convert them to numbers.
// @see: https://zod.dev/?id=coercion-for-primitives
const numberSchema = z.coerce.number();
dotenv.config({
path: './.env',
});
dotenv.config({
path: './.env.local',
override: true
});
export const env = createEnv({
server: {
SOURCEBOT_ENCRYPTION_KEY: z.string(),
SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default("false"),
SOURCEBOT_INSTALL_ID: z.string().default("unknown"),
NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"),
DATA_CACHE_DIR: z.string(),
NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(),
FALLBACK_GITHUB_CLOUD_TOKEN: z.string().optional(),
FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(),
FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(),
REDIS_URL: z.string().url().default("redis://localhost:6379"),
REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0),
REDIS_REMOVE_ON_FAIL: numberSchema.default(100),
NEXT_PUBLIC_SENTRY_BACKEND_DSN: z.string().optional(),
NEXT_PUBLIC_SENTRY_ENVIRONMENT: z.string().optional(),
LOGTAIL_TOKEN: z.string().optional(),
LOGTAIL_HOST: z.string().url().optional(),
SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"),
DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'),
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
CONFIG_PATH: z.string(),
CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000),
REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60),
GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10),
EXPERIMENT_EE_PERMISSION_SYNC_ENABLED: booleanSchema.default('false'),
AUTH_EE_GITHUB_BASE_URL: z.string().optional(),
AUTH_EE_GITLAB_BASE_URL: z.string().default("https://gitlab.com"),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
});

8
packages/backend/src/git.ts
View file

@ -1,8 +1,8 @@
import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git';
import { mkdir } from 'node:fs/promises';
import { env } from './env.js';
import { dirname, resolve } from 'node:path';
import { env } from "@sourcebot/shared";
import { existsSync } from 'node:fs';
import { mkdir } from 'node:fs/promises';
import { dirname, resolve } from 'node:path';
import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git';
type onProgressFn = (event: SimpleGitProgressEvent) => void;

14
packages/backend/src/gitea.ts
View file

@ -1,13 +1,13 @@
import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type';
import { measure } from './utils.js';
import fetch from 'cross-fetch';
import * as Sentry from "@sentry/node";
import { getTokenFromConfig } from "@sourcebot/crypto";
import { createLogger } from '@sourcebot/logger';
import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type';
import { env } from "@sourcebot/shared";
import fetch from 'cross-fetch';
import { Api, giteaApi, Repository as GiteaRepository, HttpResponse } from 'gitea-js';
import micromatch from 'micromatch';
import { processPromiseResults, throwIfAnyFailed } from './connectionUtils.js';
import * as Sentry from "@sentry/node";
import { env } from './env.js';
import { getTokenFromConfig } from "@sourcebot/crypto";
import { measure } from './utils.js';
const logger = createLogger('gitea');
const GITEA_CLOUD_HOSTNAME = "gitea.com";

5
packages/backend/src/github.ts
View file

@ -1,15 +1,14 @@
import { Octokit } from "@octokit/rest";
import * as Sentry from "@sentry/node";
import { getTokenFromConfig } from "@sourcebot/crypto";
import { createLogger } from "@sourcebot/logger";
import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
import { hasEntitlement } from "@sourcebot/shared";
import { env, hasEntitlement } from "@sourcebot/shared";
import micromatch from "micromatch";
import pLimit from "p-limit";
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
import { GithubAppManager } from "./ee/githubAppManager.js";
import { env } from "./env.js";
import { fetchWithRetry, measure } from "./utils.js";
import { getTokenFromConfig } from "@sourcebot/crypto";
export const GITHUB_CLOUD_HOSTNAME = "github.com";

12
packages/backend/src/gitlab.ts
View file

@ -1,12 +1,12 @@
import { Gitlab, ProjectSchema } from "@gitbeaker/rest";
import micromatch from "micromatch";
import { createLogger } from "@sourcebot/logger";
import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"
import { measure, fetchWithRetry } from "./utils.js";
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
import * as Sentry from "@sentry/node";
import { env } from "./env.js";
import { getTokenFromConfig } from "@sourcebot/crypto";
import { createLogger } from "@sourcebot/logger";
import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
import { env } from "@sourcebot/shared";
import micromatch from "micromatch";
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
import { fetchWithRetry, measure } from "./utils.js";
const logger = createLogger('gitlab');
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";

11
packages/backend/src/index.ts
View file

@ -2,7 +2,7 @@ import "./instrument.js";
import { PrismaClient } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { env, getConfigSettings, hasEntitlement } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { mkdir } from 'fs/promises';
import { Redis } from 'ioredis';
@ -12,7 +12,6 @@ import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { GithubAppManager } from "./ee/githubAppManager.js";
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js";
import { env } from "./env.js";
import { PromClient } from './promClient.js';
import { RepoIndexManager } from "./repoIndexManager.js";
@ -29,7 +28,13 @@ if (!existsSync(indexPath)) {
await mkdir(indexPath, { recursive: true });
}
const prisma = new PrismaClient();
const prisma = new PrismaClient({
datasources: {
db: {
url: env.DATABASE_URL,
},
},
});
const redis = new Redis(env.REDIS_URL, {
maxRetriesPerRequest: null

2
packages/backend/src/instrument.ts
View file

@ -1,6 +1,6 @@
import * as Sentry from "@sentry/node";
import { env } from "./env.js";
import { createLogger } from "@sourcebot/logger";
import { env } from "@sourcebot/shared";
const logger = createLogger('instrument');

2
packages/backend/src/posthog.ts
View file

@ -1,6 +1,6 @@
import { env } from "@sourcebot/shared";
import { PostHog } from 'posthog-node';
import { PosthogEvent, PosthogEventMap } from './posthogEvents.js';
import { env } from './env.js';
let posthog: PostHog | undefined = undefined;

3
packages/backend/src/repoIndexManager.ts
View file

@ -1,14 +1,13 @@
import * as Sentry from '@sentry/node';
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
import { createLogger, Logger } from "@sourcebot/logger";
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
import { env, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from '@sourcebot/shared';
import { existsSync } from 'fs';
import { readdir, rm } from 'fs/promises';
import { Job, Queue, ReservedJob, Worker } from "groupmq";
import { Redis } from 'ioredis';
import micromatch from 'micromatch';
import { INDEX_CACHE_DIR } from './constants.js';
import { env } from './env.js';
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
import { captureEvent } from './posthog.js';
import { PromClient } from './promClient.js';