wip on splitting the services out into seperate docker files

This commit is contained in:
bkellam 2025-07-26 16:44:41 -07:00
parent 3e50469cf7
commit d607d9b468
8 changed files with 460 additions and 2 deletions

View file

@ -189,6 +189,7 @@ ENV DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot"
ENV REDIS_URL="redis://localhost:6379" ENV REDIS_URL="redis://localhost:6379"
ENV SRC_TENANT_ENFORCEMENT_MODE=strict ENV SRC_TENANT_ENFORCEMENT_MODE=strict
ENV SOURCEBOT_PUBLIC_KEY_PATH=/app/public.pem ENV SOURCEBOT_PUBLIC_KEY_PATH=/app/public.pem
ENV ZOEKT_WEBSERVER_URL=http://localhost:6070
# Valid values are: debug, info, warn, error # Valid values are: debug, info, warn, error
ENV SOURCEBOT_LOG_LEVEL=info ENV SOURCEBOT_LOG_LEVEL=info

31
Dockerfile.zoekt Normal file
View file

@ -0,0 +1,31 @@
FROM golang:1.23.4-alpine3.19 AS zoekt-builder
RUN apk add --no-cache ca-certificates
WORKDIR /zoekt
COPY vendor/zoekt/go.mod vendor/zoekt/go.sum ./
RUN go mod download
COPY vendor/zoekt ./
RUN CGO_ENABLED=0 GOOS=linux go build -o /cmd/zoekt-webserver ./cmd/zoekt-webserver
FROM alpine:3.19 AS runner
RUN apk add --no-cache ca-certificates bind-tools tini
# Install ctags
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
# Set up data directory
ENV DATA_DIR /data/index
RUN mkdir -p ${DATA_DIR}
# Copy the zoekt-webserver binary
COPY --from=zoekt-builder /cmd/zoekt-webserver /usr/local/bin/
# zoekt-webserver has a large stable heap size (10s of gigs), and as such the
# default GOGC=100 could be better tuned. https://dave.cheney.net/tag/gogc
# In go1.18 the GC changed significantly and from experimentation we tuned it
# down from 50 to 25.
ENV GOGC=25
ENTRYPOINT ["/sbin/tini", "--"]
CMD ["zoekt-webserver", "-index", "/data/index", "-rpc"]

111
docker-compose.yml Normal file
View file

@ -0,0 +1,111 @@
services:
sourcebot-web:
build:
context: .
dockerfile: packages/web/Dockerfile
args:
- NEXT_PUBLIC_SOURCEBOT_VERSION=${NEXT_PUBLIC_SOURCEBOT_VERSION}
- NEXT_PUBLIC_POSTHOG_PAPIK=${NEXT_PUBLIC_POSTHOG_PAPIK}
- NEXT_PUBLIC_SENTRY_ENVIRONMENT=${NEXT_PUBLIC_SENTRY_ENVIRONMENT}
restart: always
ports:
- 3000:3000
volumes:
- ./config.json:/app/config.json:ro
- sourcebot_data:/data
depends_on:
postgres:
condition: service_healthy
zoekt:
condition: service_healthy
environment:
- AUTH_URL=http://localhost:3000
- AUTH_SECRET=mysecret # CHANGEME
- DATA_CACHE_DIR=${DATA_CACHE_DIR:-/data}
- CONFIG_PATH=/app/config.json
- DATABASE_URL=postgresql://postgres:postgres@postgres:5432/postgres # CHANGEME
- SOURCEBOT_ENCRYPTION_KEY=00000000000000000000000000000000 # CHANGEME
- ZOEKT_WEBSERVER_URL=http://zoekt:6070 # CHANGEME
worker:
build:
context: .
dockerfile: packages/backend/Dockerfile
args:
- NEXT_PUBLIC_SOURCEBOT_VERSION=${NEXT_PUBLIC_SOURCEBOT_VERSION}
- NEXT_PUBLIC_SENTRY_BACKEND_DSN=${NEXT_PUBLIC_SENTRY_BACKEND_DSN}
- SENTRY_ORG=${SENTRY_ORG}
- SENTRY_BACKEND_PROJECT=${SENTRY_BACKEND_PROJECT}
- SENTRY_SMUAT=${SENTRY_SMUAT}
restart: always
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
ports:
- 8080:8080
volumes:
- sourcebot_data:/data
environment:
- DATABASE_URL=postgresql://postgres:postgres@postgres:5432/postgres # CHANGEME
- DATA_CACHE_DIR=${DATA_CACHE_DIR:-/data}
- SOURCEBOT_ENCRYPTION_KEY=00000000000000000000000000000000 # CHANGEME
- REDIS_URL=redis://redis:6379 # CHANGEME
zoekt:
build:
context: .
dockerfile: Dockerfile.zoekt
ports:
- 127.0.0.1:6070:6070
volumes:
- sourcebot_data:/data
healthcheck:
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:6070/healthz"]
interval: 3s
timeout: 3s
retries: 10
postgres:
image: docker.io/postgres:${POSTGRES_VERSION:-latest}
restart: always
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 3s
timeout: 3s
retries: 10
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres # CHANGEME
POSTGRES_DB: postgres
ports:
- 127.0.0.1:5432:5432
volumes:
- sourcebot_postgres_data:/var/lib/postgresql/data
redis:
image: docker.io/redis:${REDIS_VERSION:-latest}
restart: always
# CHANGEME: row below to secure redis password
# command: >
# --requirepass ${REDIS_AUTH:-myredissecret}
ports:
- 127.0.0.1:6379:6379
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 3s
timeout: 10s
retries: 10
volumes:
- sourcebot_redis_data:/data
volumes:
sourcebot_data:
driver: local
sourcebot_postgres_data:
driver: local
sourcebot_redis_data:
driver: local

131
packages/backend/Dockerfile Normal file
View file

@ -0,0 +1,131 @@
# ------ Global scope variables ------
ARG NEXT_PUBLIC_SOURCEBOT_VERSION
ARG NEXT_PUBLIC_SENTRY_BACKEND_DSN
ARG SENTRY_ORG
ARG SENTRY_BACKEND_PROJECT
ARG SENTRY_SMUAT
FROM node:20-alpine3.19 AS node-alpine
FROM golang:1.23.4-alpine3.19 AS go-alpine
# ------ Build Zoekt ------
FROM go-alpine AS zoekt-builder
RUN apk add --no-cache ca-certificates
WORKDIR /zoekt
COPY vendor/zoekt/go.mod vendor/zoekt/go.sum ./
RUN go mod download
COPY vendor/zoekt ./
RUN CGO_ENABLED=0 GOOS=linux go build -o /cmd/ ./cmd/...
# ------ Build shared libraries ------
FROM node-alpine AS shared-libs-builder
WORKDIR /app
COPY package.json yarn.lock* .yarnrc.yml ./
COPY .yarn ./.yarn
COPY ./packages/db ./packages/db
COPY ./packages/schemas ./packages/schemas
COPY ./packages/crypto ./packages/crypto
COPY ./packages/error ./packages/error
COPY ./packages/logger ./packages/logger
COPY ./packages/shared ./packages/shared
RUN yarn workspace @sourcebot/db install
RUN yarn workspace @sourcebot/schemas install
RUN yarn workspace @sourcebot/crypto install
RUN yarn workspace @sourcebot/error install
RUN yarn workspace @sourcebot/logger install
RUN yarn workspace @sourcebot/shared install
# ------ Build Backend ------
FROM node-alpine AS backend-builder
ENV SKIP_ENV_VALIDATION=1
# Set environment variables
ARG NEXT_PUBLIC_SOURCEBOT_VERSION
ENV NEXT_PUBLIC_SOURCEBOT_VERSION=$NEXT_PUBLIC_SOURCEBOT_VERSION
ARG SENTRY_ORG
ENV SENTRY_ORG=$SENTRY_ORG
ARG SENTRY_BACKEND_PROJECT
ENV SENTRY_BACKEND_PROJECT=$SENTRY_BACKEND_PROJECT
ARG SENTRY_SMUAT
ENV SENTRY_SMUAT=$SENTRY_SMUAT
WORKDIR /app
COPY package.json yarn.lock* .yarnrc.yml ./
COPY .yarn ./.yarn
COPY ./schemas ./schemas
COPY ./packages/backend ./packages/backend
COPY --from=shared-libs-builder /app/node_modules ./node_modules
COPY --from=shared-libs-builder /app/packages/db ./packages/db
COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
COPY --from=shared-libs-builder /app/packages/error ./packages/error
COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
RUN yarn workspace @sourcebot/backend install
RUN yarn workspace @sourcebot/backend build
# Upload source maps to Sentry if we have the necessary build-time args
RUN if [ -n "$SENTRY_SMUAT" ] && [ -n "$SENTRY_ORG" ] && [ -n "$SENTRY_BACKEND_PROJECT" ] && [ -n "$NEXT_PUBLIC_SOURCEBOT_VERSION" ]; then \
apk add --no-cache curl; \
curl -sL https://sentry.io/get-cli/ | sh; \
sentry-cli login --auth-token $SENTRY_SMUAT; \
sentry-cli sourcemaps inject --org $SENTRY_ORG --project $SENTRY_BACKEND_PROJECT --release $NEXT_PUBLIC_SOURCEBOT_VERSION ./packages/backend/dist; \
sentry-cli sourcemaps upload --org $SENTRY_ORG --project $SENTRY_BACKEND_PROJECT --release $NEXT_PUBLIC_SOURCEBOT_VERSION ./packages/backend/dist; \
fi
ENV SKIP_ENV_VALIDATION=0
# ------ Runtime ------
FROM node-alpine AS runner
# Set environment variables
ARG NEXT_PUBLIC_SOURCEBOT_VERSION
ENV NEXT_PUBLIC_SOURCEBOT_VERSION=$NEXT_PUBLIC_SOURCEBOT_VERSION
ARG NEXT_PUBLIC_SENTRY_BACKEND_DSN
ENV NEXT_PUBLIC_SENTRY_BACKEND_DSN=$NEXT_PUBLIC_SENTRY_BACKEND_DSN
WORKDIR /app
ENV NODE_ENV=production
ENV SRC_TENANT_ENFORCEMENT_MODE=strict
ENV SOURCEBOT_PUBLIC_KEY_PATH=/app/public.pem
ENV SOURCEBOT_LOG_LEVEL=info
# Install system dependencies
RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget curl perl jq openssl util-linux unzip
# Configure zoekt and ctags
COPY vendor/zoekt/install-ctags-alpine.sh .
RUN ./install-ctags-alpine.sh && rm install-ctags-alpine.sh
# Copy zoekt binaries (backend specifically needs these for git indexing)
COPY --from=zoekt-builder \
/cmd/zoekt-git-index \
/cmd/zoekt-indexserver \
/cmd/zoekt-mirror-github \
/cmd/zoekt-mirror-gitiles \
/cmd/zoekt-mirror-bitbucket-server \
/cmd/zoekt-mirror-gitlab \
/cmd/zoekt-mirror-gerrit \
/cmd/zoekt-webserver \
/cmd/zoekt-index \
/usr/local/bin/
# Copy application files
COPY package.json yarn.lock* .yarnrc.yml public.pem ./
COPY .yarn ./.yarn
COPY --from=backend-builder /app/node_modules ./node_modules
COPY --from=backend-builder /app/packages/backend ./packages/backend
COPY --from=shared-libs-builder /app/packages/db ./packages/db
COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
COPY --from=shared-libs-builder /app/packages/error ./packages/error
COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
ENTRYPOINT ["/sbin/tini", "--"]
CMD ["node", "./packages/backend/dist/index.js"]

View file

@ -22,6 +22,7 @@ dotenv.config({
export const env = createEnv({ export const env = createEnv({
server: { server: {
SOURCEBOT_ENCRYPTION_KEY: z.string(), SOURCEBOT_ENCRYPTION_KEY: z.string(),
SOURCEBOT_PUBLIC_KEY_PATH: z.string(),
SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default("false"), SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default("false"),
SOURCEBOT_INSTALL_ID: z.string().default("unknown"), SOURCEBOT_INSTALL_ID: z.string().default("unknown"),
NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"), NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"),
@ -34,7 +35,7 @@ export const env = createEnv({
FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(), FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(),
FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(), FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(),
REDIS_URL: z.string().url().default("redis://localhost:6379"), REDIS_URL: z.string().url(),
REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0), REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0),
REDIS_REMOVE_ON_FAIL: numberSchema.default(100), REDIS_REMOVE_ON_FAIL: numberSchema.default(100),

169
packages/web/Dockerfile Normal file
View file

@ -0,0 +1,169 @@
# ------ Global scope variables ------
# Set of global build arguments.
# These are considered "public" and will be baked into the image.
# The convention is to prefix these with `NEXT_PUBLIC_` so that
# they can be optionally be passed as client-side environment variables
# in the webapp.
# @see: https://docs.docker.com/build/building/variables/#scoping
ARG NEXT_PUBLIC_SOURCEBOT_VERSION
# PAPIK = Project API Key
# Note that this key does not need to be kept secret, so it's not
# necessary to use Docker build secrets here.
# @see: https://posthog.com/tutorials/api-capture-events#authenticating-with-the-project-api-key
ARG NEXT_PUBLIC_POSTHOG_PAPIK
ARG NEXT_PUBLIC_SENTRY_ENVIRONMENT
ARG NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT
ARG NEXT_PUBLIC_SENTRY_WEBAPP_DSN
ARG NEXT_PUBLIC_SENTRY_BACKEND_DSN
ARG NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY
ARG NEXT_PUBLIC_LANGFUSE_BASE_URL
# Sentry source map upload arguments
ARG SENTRY_ORG
ARG SENTRY_WEBAPP_PROJECT
ARG SENTRY_SMUAT
FROM node:20-alpine3.19 AS node-alpine
# ----------------------------------
# ------ Build shared libraries ------
FROM node-alpine AS shared-libs-builder
WORKDIR /app
# Copy package management files
COPY package.json yarn.lock* .yarnrc.yml ./
COPY .yarn ./.yarn
# Copy shared packages
COPY ./packages/db ./packages/db
COPY ./packages/schemas ./packages/schemas
COPY ./packages/crypto ./packages/crypto
COPY ./packages/error ./packages/error
COPY ./packages/logger ./packages/logger
COPY ./packages/shared ./packages/shared
# Install shared dependencies
RUN yarn workspace @sourcebot/db install
RUN yarn workspace @sourcebot/schemas install
RUN yarn workspace @sourcebot/crypto install
RUN yarn workspace @sourcebot/error install
RUN yarn workspace @sourcebot/logger install
RUN yarn workspace @sourcebot/shared install
# ------------------------------------
# ------ Build Web ------
FROM node-alpine AS web-builder
ENV SKIP_ENV_VALIDATION=1
# Set build-time environment variables
ARG NEXT_PUBLIC_SOURCEBOT_VERSION
ENV NEXT_PUBLIC_SOURCEBOT_VERSION=$NEXT_PUBLIC_SOURCEBOT_VERSION
ARG NEXT_PUBLIC_POSTHOG_PAPIK
ENV NEXT_PUBLIC_POSTHOG_PAPIK=$NEXT_PUBLIC_POSTHOG_PAPIK
ARG NEXT_PUBLIC_SENTRY_ENVIRONMENT
ENV NEXT_PUBLIC_SENTRY_ENVIRONMENT=$NEXT_PUBLIC_SENTRY_ENVIRONMENT
ARG NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT
ENV NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT=$NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT
ARG NEXT_PUBLIC_SENTRY_WEBAPP_DSN
ENV NEXT_PUBLIC_SENTRY_WEBAPP_DSN=$NEXT_PUBLIC_SENTRY_WEBAPP_DSN
ARG NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY
ENV NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY=$NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY
ARG NEXT_PUBLIC_LANGFUSE_BASE_URL
ENV NEXT_PUBLIC_LANGFUSE_BASE_URL=$NEXT_PUBLIC_LANGFUSE_BASE_URL
# Sentry configuration for source map upload
ARG SENTRY_ORG
ENV SENTRY_ORG=$SENTRY_ORG
ARG SENTRY_WEBAPP_PROJECT
ENV SENTRY_WEBAPP_PROJECT=$SENTRY_WEBAPP_PROJECT
ENV SENTRY_RELEASE=$NEXT_PUBLIC_SOURCEBOT_VERSION
ARG SENTRY_SMUAT
ENV SENTRY_SMUAT=$SENTRY_SMUAT
RUN apk add --no-cache libc6-compat
WORKDIR /app
# Copy package management files
COPY package.json yarn.lock* .yarnrc.yml ./
COPY .yarn ./.yarn
# Copy web package
COPY ./packages/web ./packages/web
# Copy shared libraries from previous stage
COPY --from=shared-libs-builder /app/node_modules ./node_modules
COPY --from=shared-libs-builder /app/packages/db ./packages/db
COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
COPY --from=shared-libs-builder /app/packages/error ./packages/error
COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
# Install web dependencies (fixes arm64 timeouts)
RUN yarn workspace @sourcebot/web install
# Build the web application
ENV NEXT_TELEMETRY_DISABLED=1
RUN yarn workspace @sourcebot/web build
ENV SKIP_ENV_VALIDATION=0
# ------------------------------
# ------ Runner ------
FROM node-alpine AS runner
# Set runtime environment variables
ARG NEXT_PUBLIC_SOURCEBOT_VERSION
ENV NEXT_PUBLIC_SOURCEBOT_VERSION=$NEXT_PUBLIC_SOURCEBOT_VERSION
ARG NEXT_PUBLIC_POSTHOG_PAPIK
ENV NEXT_PUBLIC_POSTHOG_PAPIK=$NEXT_PUBLIC_POSTHOG_PAPIK
ARG NEXT_PUBLIC_SENTRY_ENVIRONMENT
ENV NEXT_PUBLIC_SENTRY_ENVIRONMENT=$NEXT_PUBLIC_SENTRY_ENVIRONMENT
ARG NEXT_PUBLIC_SENTRY_WEBAPP_DSN
ENV NEXT_PUBLIC_SENTRY_WEBAPP_DSN=$NEXT_PUBLIC_SENTRY_WEBAPP_DSN
ARG NEXT_PUBLIC_SENTRY_BACKEND_DSN
ENV NEXT_PUBLIC_SENTRY_BACKEND_DSN=$NEXT_PUBLIC_SENTRY_BACKEND_DSN
ARG NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY
ENV NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY=$NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY
ARG NEXT_PUBLIC_LANGFUSE_BASE_URL
ENV NEXT_PUBLIC_LANGFUSE_BASE_URL=$NEXT_PUBLIC_LANGFUSE_BASE_URL
ENV SOURCEBOT_PUBLIC_KEY_PATH=/app/public.pem
WORKDIR /app
ENV NODE_ENV=production
ENV NEXT_TELEMETRY_DISABLED=1
RUN apk add --no-cache tini git
RUN echo "Sourcebot Web Version: $NEXT_PUBLIC_SOURCEBOT_VERSION"
# Copy package management files
COPY package.json yarn.lock* .yarnrc.yml public.pem ./
COPY .yarn ./.yarn
# Copy built web application
COPY --from=web-builder /app/packages/web/public ./packages/web/public
COPY --from=web-builder /app/packages/web/.next/standalone ./
COPY --from=web-builder /app/packages/web/.next/static ./packages/web/.next/static
# Copy shared dependencies
COPY --from=web-builder /app/node_modules ./node_modules
COPY --from=web-builder /app/packages/db ./packages/db
COPY --from=web-builder /app/packages/schemas ./packages/schemas
COPY --from=web-builder /app/packages/crypto ./packages/crypto
COPY --from=web-builder /app/packages/error ./packages/error
COPY --from=web-builder /app/packages/logger ./packages/logger
COPY --from=web-builder /app/packages/shared ./packages/shared
COPY ./packages/web/entrypoint.sh ./packages/web/entrypoint.sh
RUN chmod +x ./packages/web/entrypoint.sh
EXPOSE 3000
ENV PORT=3000
ENV HOSTNAME="0.0.0.0"
ENTRYPOINT ["/sbin/tini", "--", "./packages/web/entrypoint.sh"]
CMD ["node", "packages/web/server.js"]
# ------------------------------

View file

@ -0,0 +1,13 @@
#!/bin/sh
if [ -z "$DATABASE_URL" ]; then
echo "Error: Required environment variable DATABASE_URL is not set. Provide a valid PostgreSQL connection string."
exit 1
fi
# Run a Database migration
echo -e "\e[34m[Info] Running database migration...\e[0m"
yarn workspace @sourcebot/db prisma:migrate:prod
# Run the command passed to the script
exec "$@"

View file

@ -14,7 +14,7 @@ const numberSchema = z.coerce.number();
export const env = createEnv({ export const env = createEnv({
server: { server: {
// Zoekt // Zoekt
ZOEKT_WEBSERVER_URL: z.string().url().default("http://localhost:6070"), ZOEKT_WEBSERVER_URL: z.string().url(),
SHARD_MAX_MATCH_COUNT: numberSchema.default(10000), SHARD_MAX_MATCH_COUNT: numberSchema.default(10000),
TOTAL_MAX_MATCH_COUNT: numberSchema.default(100000), TOTAL_MAX_MATCH_COUNT: numberSchema.default(100000),
ZOEKT_MAX_WALL_TIME_MS: numberSchema.default(10000), ZOEKT_MAX_WALL_TIME_MS: numberSchema.default(10000),
@ -57,6 +57,7 @@ export const env = createEnv({
DATA_CACHE_DIR: z.string(), DATA_CACHE_DIR: z.string(),
SOURCEBOT_PUBLIC_KEY_PATH: z.string(), SOURCEBOT_PUBLIC_KEY_PATH: z.string(),
SOURCEBOT_ENCRYPTION_KEY: z.string(),
// Email // Email
SMTP_CONNECTION_URL: z.string().url().optional(), SMTP_CONNECTION_URL: z.string().url().optional(),