diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index ee045243..115df223 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -134,6 +134,117 @@ const schema = { } }, "additionalProperties": false + }, + "EnvironmentOverrides": { + "type": "object", + "description": "Environment variable overrides.", + "not": { + "$comment": "List of environment variables that are not allowed to be overridden.", + "anyOf": [ + { + "required": [ + "CONFIG_PATH" + ] + } + ] + }, + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "token" + }, + "value": { + "anyOf": [ + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "googleCloudSecret": { + "type": "string", + "description": "The resource name of a Google Cloud secret. Must be in the format `projects//secrets//versions/`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets" + } + }, + "required": [ + "googleCloudSecret" + ], + "additionalProperties": false + } + ] + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "number" + }, + "value": { + "type": "number" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "boolean" + }, + "value": { + "type": "boolean" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + } + } } }, "properties": { @@ -278,25 +389,29 @@ const schema = { }, "additionalProperties": false }, - "connections": { + "environmentOverrides": { "type": "object", - "description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.", + "description": "Environment variable overrides.", + "not": { + "$comment": "List of environment variables that are not allowed to be overridden.", + "anyOf": [ + { + "required": [ + "CONFIG_PATH" + ] + } + ] + }, "patternProperties": { "^[a-zA-Z0-9_-]+$": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "ConnectionConfig", "oneOf": [ { - "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "title": "GithubConnectionConfig", "properties": { "type": { - "const": "github", - "description": "GitHub Configuration" + "const": "token" }, - "token": { - "description": "A Personal Access Token (PAT).", + "value": { "anyOf": [ { "type": "object", @@ -325,6 +440,113 @@ const schema = { "additionalProperties": false } ] + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "number" + }, + "value": { + "type": "number" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "boolean" + }, + "value": { + "type": "boolean" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + } + } + }, + "connections": { + "type": "object", + "description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.", + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ConnectionConfig", + "oneOf": [ + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "GithubConnectionConfig", + "properties": { + "type": { + "const": "github", + "description": "GitHub Configuration" + }, + "token": { + "anyOf": [ + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "googleCloudSecret": { + "type": "string", + "description": "The resource name of a Google Cloud secret. Must be in the format `projects//secrets//versions/`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets" + } + }, + "required": [ + "googleCloudSecret" + ], + "additionalProperties": false + } + ], + "description": "A Personal Access Token (PAT)." }, "url": { "type": "string", @@ -504,7 +726,6 @@ const schema = { "description": "GitLab Configuration" }, "token": { - "description": "An authentication token.", "anyOf": [ { "type": "object", @@ -532,7 +753,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "An authentication token." }, "url": { "type": "string", @@ -706,7 +928,6 @@ const schema = { "description": "Gitea Configuration" }, "token": { - "description": "A Personal Access Token (PAT).", "anyOf": [ { "type": "object", @@ -734,7 +955,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "A Personal Access Token (PAT)." }, "url": { "type": "string", @@ -973,7 +1195,6 @@ const schema = { "description": "The username to use for authentication. Only needed if token is an app password." }, "token": { - "description": "An authentication token.", "anyOf": [ { "type": "object", @@ -1001,7 +1222,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "An authentication token." }, "url": { "type": "string", @@ -1141,7 +1363,6 @@ const schema = { "description": "Azure DevOps Configuration" }, "token": { - "description": "A Personal Access Token (PAT).", "anyOf": [ { "type": "object", @@ -1169,7 +1390,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "A Personal Access Token (PAT)." }, "url": { "type": "string", @@ -1425,7 +1647,6 @@ const schema = { "description": "Optional display name." }, "accessKeyId": { - "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.", "anyOf": [ { "type": "object", @@ -1453,10 +1674,10 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable." }, "accessKeySecret": { - "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.", "anyOf": [ { "type": "object", @@ -1484,10 +1705,10 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable." }, "sessionToken": { - "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.", "anyOf": [ { "type": "object", @@ -1515,7 +1736,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable." }, "region": { "type": "string", @@ -2854,7 +3076,6 @@ const schema = { "description": "Optional display name." }, "accessKeyId": { - "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.", "anyOf": [ { "type": "object", @@ -2882,10 +3103,10 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable." }, "accessKeySecret": { - "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.", "anyOf": [ { "type": "object", @@ -2913,10 +3134,10 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable." }, "sessionToken": { - "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.", "anyOf": [ { "type": "object", @@ -2944,7 +3165,8 @@ const schema = { ], "additionalProperties": false } - ] + ], + "description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable." }, "region": { "type": "string", diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 0e88d8ab..4464782c 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -44,6 +44,7 @@ export interface SourcebotConfig { contexts?: { [k: string]: SearchContext; }; + environmentOverrides?: EnvironmentOverrides; /** * Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode. */ @@ -159,6 +160,47 @@ export interface SearchContext { */ description?: string; } +/** + * Environment variable overrides. + * + * This interface was referenced by `SourcebotConfig`'s JSON-Schema + * via the `definition` "EnvironmentOverrides". + */ +export interface EnvironmentOverrides { + /** + * This interface was referenced by `EnvironmentOverrides`'s JSON-Schema definition + * via the `patternProperty` "^[a-zA-Z0-9_-]+$". + */ + [k: string]: + | { + type: "token"; + value: + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + } + | { + /** + * The resource name of a Google Cloud secret. Must be in the format `projects//secrets//versions/`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets + */ + googleCloudSecret: string; + }; + } + | { + type: "string"; + value: string; + } + | { + type: "number"; + value: number; + } + | { + type: "boolean"; + value: boolean; + }; +} export interface GithubConnectionConfig { /** * GitHub Configuration diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts index c1162923..667dedaa 100644 --- a/packages/shared/src/env.ts +++ b/packages/shared/src/env.ts @@ -1,21 +1,107 @@ import { createEnv } from "@t3-oss/env-core"; import { z } from "zod"; import { SOURCEBOT_CLOUD_ENVIRONMENT } from "./constants.js"; +import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type"; +import { getTokenFromConfig } from "@sourcebot/crypto"; +import { loadConfig } from "./utils.js"; + +// Booleans are specified as 'true' or 'false' strings. +const booleanSchema = z.enum(["true", "false"]); + +// Numbers are treated as strings in .env files. +// coerce helps us convert them to numbers. +// @see: https://zod.dev/?id=coercion-for-primitives +const numberSchema = z.coerce.number(); + + +const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotConfig): Promise> => { + if (!config.environmentOverrides) { + return {}; + } + + const resolved: Record = {}; + console.debug('resolving environment variable overrides'); + + for (const [key, override] of Object.entries(config.environmentOverrides)) { + switch (override.type) { + case 'token': + resolved[key] = await getTokenFromConfig(override.value); + break; + case 'boolean': + resolved[key] = override.value ? 'true' : 'false'; + break; + case 'number': + resolved[key] = override.value.toString(); + break; + case 'string': + resolved[key] = override.value; + break; + } + } + + return resolved; +} + +// Merge process.env with environment variables resolved from config.json +const runtimeEnv = await (async () => { + const configPath = process.env.CONFIG_PATH; + if (!configPath) { + return process.env; + } + + const config = await loadConfig(configPath); + const overrides = await resolveEnvironmentVariableOverridesFromConfig(config); + return { + ...process.env, + ...overrides, + } +})(); export const env = createEnv({ server: { SOURCEBOT_EE_LICENSE_KEY: z.string().optional(), SOURCEBOT_PUBLIC_KEY_PATH: z.string(), + + SOURCEBOT_ENCRYPTION_KEY: z.string(), + SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default("false"), + SOURCEBOT_INSTALL_ID: z.string().default("unknown"), + + DATA_CACHE_DIR: z.string(), + + FALLBACK_GITHUB_CLOUD_TOKEN: z.string().optional(), + FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(), + FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(), + + REDIS_URL: z.string().url(), + REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0), + REDIS_REMOVE_ON_FAIL: numberSchema.default(100), + + LOGTAIL_TOKEN: z.string().optional(), + LOGTAIL_HOST: z.string().url().optional(), + SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"), + DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'), + + DATABASE_URL: z.string().url(), + CONFIG_PATH: z.string(), + + CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000), + REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60), + + GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10), + + EXPERIMENT_EE_PERMISSION_SYNC_ENABLED: booleanSchema.default('false'), + AUTH_EE_GITHUB_BASE_URL: z.string().optional(), + AUTH_EE_GITLAB_BASE_URL: z.string().default("https://gitlab.com"), }, client: { NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(), + NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"), + NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(), + NEXT_PUBLIC_SENTRY_BACKEND_DSN: z.string().optional(), + NEXT_PUBLIC_SENTRY_ENVIRONMENT: z.string().optional(), }, clientPrefix: "NEXT_PUBLIC_", - runtimeEnvStrict: { - SOURCEBOT_EE_LICENSE_KEY: process.env.SOURCEBOT_EE_LICENSE_KEY, - SOURCEBOT_PUBLIC_KEY_PATH: process.env.SOURCEBOT_PUBLIC_KEY_PATH, - NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT, - }, + runtimeEnv, emptyStringAsUndefined: true, skipValidation: process.env.SKIP_ENV_VALIDATION === "1", }); \ No newline at end of file diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index f3303c14..b841c6d9 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -24,4 +24,7 @@ export { isRemotePath, getConfigSettings, } from "./utils.js"; -export * from "./constants.js"; \ No newline at end of file +export * from "./constants.js"; +export { + env +} from "./env.js"; \ No newline at end of file diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 392a2eb9..038d0169 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -83,6 +83,90 @@ }, "SearchContext": { "$ref": "./searchContext.json" + }, + "EnvironmentOverrides": { + "type": "object", + "description": "Environment variable overrides.", + "not": { + "$comment": "List of environment variables that are not allowed to be overridden.", + "anyOf": [ + { + "required": [ + "CONFIG_PATH" + ] + } + ] + }, + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "token" + }, + "value": { + "$ref": "./shared.json#/definitions/Token" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "number" + }, + "value": { + "type": "number" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "boolean" + }, + "value": { + "type": "boolean" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + } + } } }, "properties": { @@ -102,6 +186,9 @@ }, "additionalProperties": false }, + "environmentOverrides": { + "$ref": "#/definitions/EnvironmentOverrides" + }, "connections": { "type": "object", "description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.",