diff --git a/Dockerfile b/Dockerfile index e69ccaf2..ae90c8c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -173,7 +173,6 @@ ENV DATA_DIR=/data ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot ENV DATABASE_DATA_DIR=$DATA_CACHE_DIR/db ENV REDIS_DATA_DIR=$DATA_CACHE_DIR/redis -ENV REDIS_URL="redis://localhost:6379" ENV SRC_TENANT_ENFORCEMENT_MODE=strict ENV SOURCEBOT_PUBLIC_KEY_PATH=/app/public.pem diff --git a/entrypoint.sh b/entrypoint.sh index b031b326..cf90a377 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,28 +1,57 @@ #!/bin/sh + +# Exit immediately if a command fails set -e +# Disable auto-exporting of variables +set +a -# Check if DATABASE_URL is not set -if [ -z "$DATABASE_URL" ]; then - # Check if the individual database variables are set and construct the URL - if [ -n "$DATABASE_HOST" ] && [ -n "$DATABASE_USERNAME" ] && [ -n "$DATABASE_PASSWORD" ] && [ -n "$DATABASE_NAME" ]; then - DATABASE_URL="postgresql://${DATABASE_USERNAME}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}" +# If a CONFIG_PATH is set, resolve the environment overrides from the config file. +# The overrides will be written into variables scopped to the current shell. This is +# required in case one of the variables used in this entrypoint is overriden (e.g., +# DATABASE_URL, REDIS_URL, etc.) +if [ -n "$CONFIG_PATH" ]; then + echo -e "\e[34m[Info] Resolving environment overrides from $CONFIG_PATH...\e[0m" - if [ -n "$DATABASE_ARGS" ]; then - DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS" - fi + set +e # Disable exist on error so we can capture EXIT_CODE + OVERRIDES_OUTPUT=$(SKIP_ENV_VALIDATION=1 yarn tool:resolve-env-overrides 2>&1) + EXIT_CODE=$? + set -e # Re-enable exit on error - export DATABASE_URL + if [ $EXIT_CODE -eq 0 ]; then + eval "$OVERRIDES_OUTPUT" else - # Otherwise, fallback to a default value - DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot" - export DATABASE_URL + echo -e "\e[31m[Error] Failed to resolve environment overrides.\e[0m" + echo "$OVERRIDES_OUTPUT" + exit 1 fi fi -if [ "$DATABASE_URL" = "postgresql://postgres@localhost:5432/sourcebot" ]; then - DATABASE_EMBEDDED="true" +# Descontruct the database URL from the individual variables if DATABASE_URL is not set +if [ -z "$DATABASE_URL" ] && [ -n "$DATABASE_HOST" ] && [ -n "$DATABASE_USERNAME" ] && [ -n "$DATABASE_PASSWORD" ] && [ -n "$DATABASE_NAME" ]; then + DATABASE_URL="postgresql://${DATABASE_USERNAME}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}" + + if [ -n "$DATABASE_ARGS" ]; then + DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS" + fi fi +if [ -z "$DATABASE_URL" ]; then + echo -e "\e[34m[Info] DATABASE_URL is not set. Using embeded database.\e[0m" + export DATABASE_EMBEDDED="true" + export DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot" +else + export DATABASE_EMBEDDED="false" +fi + +if [ -z "$REDIS_URL" ]; then + echo -e "\e[34m[Info] REDIS_URL is not set. Using embeded redis.\e[0m" + export REDIS_EMBEDDED="true" + export REDIS_URL="redis://localhost:6379" +else + export REDIS_EMBEDDED="false" +fi + + echo -e "\e[34m[Info] Sourcebot version: $NEXT_PUBLIC_SOURCEBOT_VERSION\e[0m" # If we don't have a PostHog key, then we need to disable telemetry. @@ -59,7 +88,7 @@ if [ "$DATABASE_EMBEDDED" = "true" ] && [ ! -d "$DATABASE_DATA_DIR" ]; then fi # Create the redis data directory if it doesn't exist -if [ ! -d "$REDIS_DATA_DIR" ]; then +if [ "$REDIS_EMBEDDED" = "true" ] && [ ! -d "$REDIS_DATA_DIR" ]; then mkdir -p $REDIS_DATA_DIR fi @@ -149,7 +178,6 @@ fi echo "{\"version\": \"$NEXT_PUBLIC_SOURCEBOT_VERSION\", \"install_id\": \"$SOURCEBOT_INSTALL_ID\"}" > "$FIRST_RUN_FILE" - # Start the database and wait for it to be ready before starting any other service if [ "$DATABASE_EMBEDDED" = "true" ]; then su postgres -c "postgres -D $DATABASE_DATA_DIR" & @@ -171,7 +199,7 @@ fi # Run a Database migration echo -e "\e[34m[Info] Running database migration...\e[0m" -yarn workspace @sourcebot/db prisma:migrate:prod +DATABASE_URL="$DATABASE_URL" yarn workspace @sourcebot/db prisma:migrate:prod # Create the log directory mkdir -p /var/log/sourcebot diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index ffdff86a..6c1cbeba 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -2,7 +2,7 @@ import "./instrument.js"; import { PrismaClient } from "@sourcebot/db"; import { createLogger } from "@sourcebot/shared"; -import { env, getConfigSettings, hasEntitlement } from '@sourcebot/shared'; +import { env, getConfigSettings, hasEntitlement, getDBConnectionString } from '@sourcebot/shared'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; import { Redis } from 'ioredis'; @@ -31,7 +31,7 @@ if (!existsSync(indexPath)) { const prisma = new PrismaClient({ datasources: { db: { - url: env.DATABASE_URL, + url: getDBConnectionString(), }, }, }); diff --git a/packages/shared/package.json b/packages/shared/package.json index dfe8dc16..23d4463b 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -6,7 +6,8 @@ "scripts": { "build": "tsc", "build:watch": "tsc-watch --preserveWatchOutput", - "postinstall": "yarn build" + "postinstall": "yarn build", + "tool:resolve-env-overrides": "tsx tools/resolveEnvOverrides.ts" }, "dependencies": { "@google-cloud/secret-manager": "^6.1.1", @@ -26,6 +27,7 @@ "@types/micromatch": "^4.0.9", "@types/node": "^22.7.5", "tsc-watch": "6.2.1", + "tsx": "^4.19.1", "typescript": "^5.7.3" }, "exports": { diff --git a/packages/shared/src/db.ts b/packages/shared/src/db.ts new file mode 100644 index 00000000..b3588829 --- /dev/null +++ b/packages/shared/src/db.ts @@ -0,0 +1,15 @@ +import { env } from "./env.server.js"; + +export const getDBConnectionString = (): string | undefined => { + if (env.DATABASE_URL) { + return env.DATABASE_URL; + } + else if (env.DATABASE_HOST && env.DATABASE_USERNAME && env.DATABASE_PASSWORD && env.DATABASE_NAME) { + let databaseUrl = `postgresql://${env.DATABASE_USERNAME}:${env.DATABASE_PASSWORD}@${env.DATABASE_HOST}/${env.DATABASE_NAME}`; + if (env.DATABASE_ARGS) { + databaseUrl += `?${env.DATABASE_ARGS}`; + } + + return databaseUrl; + } +} \ No newline at end of file diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 9b8dcdd0..87e48758 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -1,9 +1,9 @@ import { createEnv } from "@t3-oss/env-core"; import { z } from "zod"; -import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type"; -import { getTokenFromConfig } from "./crypto.js"; import { loadConfig } from "./utils.js"; import { tenancyModeSchema } from "./types.js"; +import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type"; +import { getTokenFromConfig } from "./crypto.js"; // Booleans are specified as 'true' or 'false' strings. const booleanSchema = z.enum(["true", "false"]); @@ -13,8 +13,7 @@ const booleanSchema = z.enum(["true", "false"]); // @see: https://zod.dev/?id=coercion-for-primitives const numberSchema = z.coerce.number(); - -const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotConfig): Promise> => { +export const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotConfig): Promise> => { if (!config.environmentOverrides) { return {}; } @@ -22,7 +21,6 @@ const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotCo const resolved: Record = {}; const start = performance.now(); - console.debug('resolving environment variable overrides'); for (const [key, override] of Object.entries(config.environmentOverrides)) { switch (override.type) { @@ -122,7 +120,16 @@ export const env = createEnv({ CONFIG_MAX_REPOS_NO_TOKEN: numberSchema.default(Number.MAX_SAFE_INTEGER), NODE_ENV: z.enum(["development", "test", "production"]), SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default('false'), - DATABASE_URL: z.string().url(), + + // Database variables + // Either DATABASE_URL or DATABASE_HOST, DATABASE_USERNAME, DATABASE_PASSWORD, and DATABASE_NAME must be set. + // @see: shared/src/db.ts for more details. + DATABASE_URL: z.string().url().optional(), + DATABASE_HOST: z.string().optional(), + DATABASE_USERNAME: z.string().optional(), + DATABASE_PASSWORD: z.string().optional(), + DATABASE_NAME: z.string().optional(), + DATABASE_ARGS: z.string().optional(), SOURCEBOT_TENANCY_MODE: tenancyModeSchema.default("single"), CONFIG_PATH: z.string(), diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index c5bcad10..fabe608e 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -27,7 +27,8 @@ export { } from "./utils.js"; export * from "./constants.js"; export { - env + env, + resolveEnvironmentVariableOverridesFromConfig, } from "./env.server.js"; export { createLogger, @@ -42,4 +43,7 @@ export { hashSecret, generateApiKey, verifySignature, -} from "./crypto.js"; \ No newline at end of file +} from "./crypto.js"; +export { + getDBConnectionString, +} from "./db.js"; \ No newline at end of file diff --git a/packages/shared/tools/resolveEnvOverrides.ts b/packages/shared/tools/resolveEnvOverrides.ts new file mode 100644 index 00000000..def001c3 --- /dev/null +++ b/packages/shared/tools/resolveEnvOverrides.ts @@ -0,0 +1,30 @@ +// The following script loads the config.json file and resolves any environment variable overrides. +// It then writes then to stdout in the format of `KEY="VALUE"`. +// This is used by entrypoint.sh to set them as variables. +(async () => { + if (!process.env.CONFIG_PATH) { + console.error('CONFIG_PATH is not set'); + process.exit(1); + } + + // Silence all console logs so we don't pollute stdout. + const originalConsoleLog = console.log; + console.log = () => {}; + console.debug = () => {}; + console.info = () => {}; + console.warn = () => {}; + // console.error = () => {}; // Keep errors + + const { loadConfig } = await import("../src/utils.js"); + const { resolveEnvironmentVariableOverridesFromConfig } = await import("../src/env.server.js"); + + const config = await loadConfig(process.env.CONFIG_PATH); + const overrides = await resolveEnvironmentVariableOverridesFromConfig(config); + + for (const [key, value] of Object.entries(overrides)) { + const escapedValue = value.replace(/"/g, '\\"'); + originalConsoleLog(`${key}="${escapedValue}"`); + } + + process.exit(0); +})(); \ No newline at end of file diff --git a/packages/web/src/prisma.ts b/packages/web/src/prisma.ts index 52a69cd3..0d520de7 100644 --- a/packages/web/src/prisma.ts +++ b/packages/web/src/prisma.ts @@ -1,11 +1,13 @@ import 'server-only'; -import { env } from "@sourcebot/shared"; +import { env, getDBConnectionString } from "@sourcebot/shared"; import { Prisma, PrismaClient } from "@sourcebot/db"; import { hasEntitlement } from "@sourcebot/shared"; // @see: https://authjs.dev/getting-started/adapters/prisma const globalForPrisma = globalThis as unknown as { prisma: PrismaClient } +const dbConnectionString = getDBConnectionString(); + // @NOTE: In almost all cases, the userScopedPrismaClientExtension should be used // (since actions & queries are scoped to a particular user). There are some exceptions // (e.g., in initialize.ts). @@ -14,15 +16,15 @@ const globalForPrisma = globalThis as unknown as { prisma: PrismaClient } // all of the actions & queries to use the userScopedPrismaClientExtension to avoid // accidental misuse. export const prisma = globalForPrisma.prisma || new PrismaClient({ - // @note: even though DATABASE_URL is of type string, we need to check if it's defined - // because this code will be executed at build time, and env.DATABASE_URL will be undefined. - ...(env.DATABASE_URL ? { + // @note: this code is evaluated at build time, and will throw exceptions if these env vars are not set. + // Here we explicitly check if the DATABASE_URL or the individual database variables are set, and only + ...(dbConnectionString !== undefined ? { datasources: { db: { - url: env.DATABASE_URL, + url: dbConnectionString, }, } - } : {}) + }: {}), }) if (env.NODE_ENV !== "production") globalForPrisma.prisma = prisma diff --git a/supervisord.conf b/supervisord.conf index eda6d430..19d30850 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -36,7 +36,7 @@ redirect_stderr=true [program:redis] command=redis-server --dir %(ENV_REDIS_DATA_DIR)s priority=10 -autostart=true +autostart=%(ENV_REDIS_EMBEDDED)s autorestart=true startretries=3 stdout_logfile=/dev/fd/1 diff --git a/yarn.lock b/yarn.lock index 11e9c654..2f19bfa9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8006,6 +8006,7 @@ __metadata: strip-json-comments: "npm:^5.0.1" triple-beam: "npm:^1.4.1" tsc-watch: "npm:6.2.1" + tsx: "npm:^4.19.1" typescript: "npm:^5.7.3" winston: "npm:^3.15.0" zod: "npm:^3.25.74"