mirror of
https://github.com/sourcebot-dev/sourcebot.git
synced 2025-12-11 20:05:25 +00:00
feat(web,worker): Environment overrides (#597)
This commit is contained in:
Brendan Kellam
GitHub
parent
5fde901356
commit
1908051daa
140 changed files with 1280 additions and 825 deletions
|
|
@ -84,7 +84,7 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
|
|||
# NEXT_PUBLIC_SOURCEBOT_VERSION=
|
||||
|
||||
# CONFIG_MAX_REPOS_NO_TOKEN=
|
||||
# NODE_ENV=
|
||||
NODE_ENV=development
|
||||
# SOURCEBOT_TENANCY_MODE=single
|
||||
|
||||
# NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT=
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||
## Added
|
||||
- [Experimental][Sourcebot EE] Added GitLab permission syncing. [#585](https://github.com/sourcebot-dev/sourcebot/pull/585)
|
||||
- [Sourcebot EE] Added external identity provider config and support for multiple accounts. [#595](https://github.com/sourcebot-dev/sourcebot/pull/595)
|
||||
- Added ability to configure environment variables from the config. [#597](https://github.com/sourcebot-dev/sourcebot/pull/597)
|
||||
|
||||
### Fixed
|
||||
- [ask sb] Fixed issue where reasoning tokens would appear in `text` content for openai compatible models. [#582](https://github.com/sourcebot-dev/sourcebot/pull/582)
|
||||
|
|
|
|||
16
Dockerfile
16
Dockerfile
|
|
@ -42,16 +42,10 @@ COPY package.json yarn.lock* .yarnrc.yml ./
|
|||
COPY .yarn ./.yarn
|
||||
COPY ./packages/db ./packages/db
|
||||
COPY ./packages/schemas ./packages/schemas
|
||||
COPY ./packages/crypto ./packages/crypto
|
||||
COPY ./packages/error ./packages/error
|
||||
COPY ./packages/logger ./packages/logger
|
||||
COPY ./packages/shared ./packages/shared
|
||||
|
||||
RUN yarn workspace @sourcebot/db install
|
||||
RUN yarn workspace @sourcebot/schemas install
|
||||
RUN yarn workspace @sourcebot/crypto install
|
||||
RUN yarn workspace @sourcebot/error install
|
||||
RUN yarn workspace @sourcebot/logger install
|
||||
RUN yarn workspace @sourcebot/shared install
|
||||
# ------------------------------------
|
||||
|
||||
|
|
@ -97,9 +91,6 @@ COPY ./packages/web ./packages/web
|
|||
COPY --from=shared-libs-builder /app/node_modules ./node_modules
|
||||
COPY --from=shared-libs-builder /app/packages/db ./packages/db
|
||||
COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
|
||||
COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
|
||||
COPY --from=shared-libs-builder /app/packages/error ./packages/error
|
||||
COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
|
||||
COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
|
||||
|
||||
# Fixes arm64 timeouts
|
||||
|
|
@ -138,9 +129,6 @@ COPY ./packages/backend ./packages/backend
|
|||
COPY --from=shared-libs-builder /app/node_modules ./node_modules
|
||||
COPY --from=shared-libs-builder /app/packages/db ./packages/db
|
||||
COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
|
||||
COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
|
||||
COPY --from=shared-libs-builder /app/packages/error ./packages/error
|
||||
COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
|
||||
COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
|
||||
RUN yarn workspace @sourcebot/backend install
|
||||
RUN yarn workspace @sourcebot/backend build
|
||||
|
|
@ -185,7 +173,6 @@ ENV DATA_DIR=/data
|
|||
ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
|
||||
ENV DATABASE_DATA_DIR=$DATA_CACHE_DIR/db
|
||||
ENV REDIS_DATA_DIR=$DATA_CACHE_DIR/redis
|
||||
ENV REDIS_URL="redis://localhost:6379"
|
||||
ENV SRC_TENANT_ENFORCEMENT_MODE=strict
|
||||
ENV SOURCEBOT_PUBLIC_KEY_PATH=/app/public.pem
|
||||
|
||||
|
|
@ -225,9 +212,6 @@ COPY --from=backend-builder /app/packages/backend ./packages/backend
|
|||
COPY --from=shared-libs-builder /app/node_modules ./node_modules
|
||||
COPY --from=shared-libs-builder /app/packages/db ./packages/db
|
||||
COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
|
||||
COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
|
||||
COPY --from=shared-libs-builder /app/packages/error ./packages/error
|
||||
COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
|
||||
COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
|
||||
|
||||
# Configure dependencies
|
||||
|
|
|
|||
4
Makefile
4
Makefile
|
|
@ -28,10 +28,6 @@ clean:
|
|||
packages/db/dist \
|
||||
packages/schemas/node_modules \
|
||||
packages/schemas/dist \
|
||||
packages/crypto/node_modules \
|
||||
packages/crypto/dist \
|
||||
packages/error/node_modules \
|
||||
packages/error/dist \
|
||||
packages/mcp/node_modules \
|
||||
packages/mcp/dist \
|
||||
packages/shared/node_modules \
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@ title: Config File
|
|||
sidebarTitle: Config file
|
||||
---
|
||||
|
||||
import ConfigSchema from '/snippets/schemas/v3/index.schema.mdx'
|
||||
import EnvironmentOverridesSchema from '/snippets/schemas/v3/environmentOverrides.schema.mdx'
|
||||
|
||||
When self-hosting Sourcebot, you **must** provide it a config file. This is done by defining a config file in a volume that's mounted to Sourcebot, and providing the path to this
|
||||
file in the `CONFIG_PATH` environment variable. For example:
|
||||
|
||||
|
|
@ -49,3 +52,103 @@ The following are settings that can be provided in your config file to modify So
|
|||
| `enablePublicAccess` **(deprecated)** | boolean | false | — | Use the `FORCE_ENABLE_ANONYMOUS_ACCESS` environment variable instead. |
|
||||
| `experiment_repoDrivenPermissionSyncIntervalMs` | number | 24 hours | 1 | Interval at which the repo permission syncer should run. |
|
||||
| `experiment_userDrivenPermissionSyncIntervalMs` | number | 24 hours | 1 | Interval at which the user permission syncer should run. |
|
||||
|
||||
# Tokens
|
||||
|
||||
Tokens are used to securely pass secrets to Sourcebot in a config file. They are used in various places, including connections, language model providers, auth providers, etc. Tokens can be passed as either environment variables or Google Cloud secrets:
|
||||
|
||||
<AccordionGroup>
|
||||
<Accordion title="Environment Variables">
|
||||
```json
|
||||
{
|
||||
"token": {
|
||||
"env": "TOKEN_NAME"
|
||||
}
|
||||
}
|
||||
```
|
||||
</Accordion>
|
||||
<Accordion title="Google Cloud Secrets">
|
||||
```json
|
||||
{
|
||||
"token": {
|
||||
"googleCloudSecret": "projects/<project-id>/secrets/<secret-name>/versions/<version-id>"
|
||||
}
|
||||
}
|
||||
```
|
||||
</Accordion>
|
||||
</AccordionGroup>
|
||||
|
||||
# Overriding environment variables from the config
|
||||
|
||||
You can override / set environment variables from the config file by using the `environmentOverrides` property. Overrides can be of type `string`, `number`, `boolean`, or a [token](/docs/configuration/config-file#tokens). Tokens are useful when you want to configure a environment variable using a Google Cloud Secret or other supported secret management service.
|
||||
|
||||
<AccordionGroup>
|
||||
<Accordion title="Token">
|
||||
```jsonc
|
||||
{
|
||||
"environmentOverrides": {
|
||||
"DATABASE_URL": {
|
||||
"type": "token",
|
||||
"value": {
|
||||
"googleCloudSecret": "projects/<id>/secrets/postgres-connection-string/versions/latest"
|
||||
}
|
||||
},
|
||||
"REDIS_URL": {
|
||||
"type": "token",
|
||||
"value": {
|
||||
"googleCloudSecret": "projects/<id>/secrets/redis-connection-string/versions/latest"
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
```
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="String">
|
||||
```jsonc
|
||||
{
|
||||
"environmentOverrides": {
|
||||
"EMAIL_FROM_ADDRESS": {
|
||||
"type": "string",
|
||||
"value": "hello@sourcebot.dev"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Number">
|
||||
```jsonc
|
||||
{
|
||||
"environmentOverrides": {
|
||||
"SOURCEBOT_CHAT_MODEL_TEMPERATURE": {
|
||||
"type": "number",
|
||||
"value": 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Boolean">
|
||||
```jsonc
|
||||
{
|
||||
"environmentOverrides": {
|
||||
"SOURCEBOT_TELEMETRY_DISABLED": {
|
||||
"type": "boolean",
|
||||
"value": false
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
</Accordion>
|
||||
</AccordionGroup>
|
||||
|
||||
|
||||
**Note:** Overrides are **not** set as system environment variables, and instead are resolved at runtime on startup and stored in memory.
|
||||
|
||||
<Accordion title="Schema reference">
|
||||
[schemas/v3/environmentOverrides.json](https://github.com/sourcebot-dev/sourcebot/blob/main/schemas/v3/environmentOverrides.json)
|
||||
|
||||
<EnvironmentOverridesSchema />
|
||||
</Accordion>
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
---
|
||||
title: Environment variables
|
||||
sidebarTitle: Environment variables
|
||||
mode: "wide"
|
||||
---
|
||||
|
||||
<Note>This page provides a detailed reference of all environment variables supported by Sourcebot. If you're just looking to get up and running, we recommend starting with the [deployment guide](/docs/deployment-guide) instead.</Note>
|
||||
|
|
@ -71,3 +70,6 @@ The following environment variables allow you to configure your Sourcebot deploy
|
|||
| `REVIEW_AGENT_LOGGING_ENABLED` | `true` | <p>Enables/disables logging for the review agent. Logs are saved in `DATA_CACHE_DIR/review-agent`</p> |
|
||||
| `REVIEW_AGENT_REVIEW_COMMAND` | `review` | <p>The command used to trigger a code review by the review agent.</p> |
|
||||
|
||||
### Overriding environment variables from the config
|
||||
|
||||
You can override environment variables from the config file by using the `environmentOverrides` property. See [this doc](/docs/configuration/config-file#overriding-environment-variables-from-the-config) for more info.
|
||||
|
|
@ -20,11 +20,6 @@ External identity providers can be used for [authentication](/docs/configuration
|
|||
"provider": "github",
|
||||
"purpose": "account_linking",
|
||||
"accountLinkingRequired": true,
|
||||
/*
|
||||
Secrets are provided through environment variables. Set the secret into
|
||||
an env var and provide the name here to tell Sourcebot where to get
|
||||
the value
|
||||
*/
|
||||
"clientId": {
|
||||
"env": "GITHUB_IDENTITY_PROVIDER_CLIENT_ID"
|
||||
},
|
||||
|
|
@ -45,6 +40,8 @@ the value
|
|||
}
|
||||
```
|
||||
|
||||
Secret values (such as `clientId` and `clientSecret`) can be provided as environment variables or Google Cloud secrets via [tokens](/docs/configuration/config-file#tokens).
|
||||
|
||||
# Supported External Identity Providers
|
||||
|
||||
Sourcebot uses [Auth.js](https://authjs.dev/) to connect to external identity providers. If there's a provider supported by Auth.js that you don't see below, please submit a
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ If you're not familiar with Sourcebot [connections](/docs/connections/overview),
|
|||
Azure Devops Cloud requires you to provide a PAT in order to index your repositories. To learn how to create PAT, check out the [Azure Devops docs](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows).
|
||||
Sourcebot needs the `Read` access for the `Code` scope in order to find and clone your repos.
|
||||
|
||||
Next, provide the access token via an environment variable which is referenced in the `token` property:
|
||||
Next, provide the access [token](/docs/configuration/config-file#tokens) via an environment variable which is referenced in the `token` property:
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Environment Variable">
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ If you're not familiar with Sourcebot [connections](/docs/connections/overview),
|
|||
Azure Devops Server requires you to provide a PAT in order to index your repositories. To learn how to create PAT, check out the [Azure Devops docs](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows).
|
||||
Sourcebot needs the `Read` access for the `Code` scope in order to find and clone your repos.
|
||||
|
||||
Next, provide the access token via an environment variable which is referenced in the `token` property:
|
||||
Next, provide the access [token](/docs/configuration/config-file#tokens) via an environment variable which is referenced in the `token` property:
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Environment Variable">
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ If you're not familiar with Sourcebot [connections](/docs/connections/overview),
|
|||
|
||||
## Authenticating with Bitbucket Cloud
|
||||
|
||||
In order to index private repositories, you'll need to provide authentication credentials. You can do this using an `App Password` or an `Access Token`
|
||||
In order to index private repositories, you'll need to provide authentication credentials via a [token](/docs/configuration/config-file#tokens). You can do this using an `App Password` or an `Access Token`
|
||||
|
||||
<Tabs>
|
||||
<Tab title="App Password">
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ If you're not familiar with Sourcebot [connections](/docs/connections/overview),
|
|||
|
||||
## Authenticating with Bitbucket Data Center
|
||||
|
||||
In order to index private repositories, you'll need to provide an access token to Sourcebot.
|
||||
In order to index private repositories, you'll need to provide an access token to Sourcebot via a [token](/docs/configuration/config-file#tokens).
|
||||
|
||||
Create an access token for the desired scope (repo, project, or workspace). Visit the official [Bitbucket Data Center docs](https://confluence.atlassian.com/bitbucketserver/http-access-tokens-939515499.html)
|
||||
for more info.
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ In order to index private repositories, you'll need to generate a Gitea access t
|
|||
|
||||
|
||||
|
||||
Next, provide the access token via an environment variable which is referenced in the `token` property:
|
||||
Next, provide the access token via an environment variable [token](/docs/configuration/config-file#tokens) which is referenced in the `token` property:
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Environment Variable">
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ In order to index private repositories, you'll need to generate a access token a
|
|||
</Accordion>
|
||||
</AccordionGroup>
|
||||
|
||||
Next, provide the access token via an environment variable which is referenced in the `token` property:
|
||||
Next, provide the access token via an environment variable [token](/docs/configuration/config-file#tokens) which is referenced in the `token` property:
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Environment Variable">
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ In order to index private projects, you'll need to generate a GitLab Personal Ac
|
|||
|
||||
|
||||
|
||||
Next, provide the PAT via an environment variable which is referenced in the `token` property:
|
||||
Next, provide the PAT via an environment variable [token](/docs/configuration/config-file#tokens) which is referenced in the `token` property:
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Environment Variable">
|
||||
|
|
|
|||
115
docs/snippets/schemas/v3/environmentOverrides.schema.mdx
Normal file
115
docs/snippets/schemas/v3/environmentOverrides.schema.mdx
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
{/* THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! */}
|
||||
```json
|
||||
{
|
||||
"type": "object",
|
||||
"description": "Environment variable overrides.",
|
||||
"title": "EnvironmentOverrides",
|
||||
"not": {
|
||||
"$comment": "List of environment variables that are not allowed to be overridden.",
|
||||
"anyOf": [
|
||||
{
|
||||
"required": [
|
||||
"CONFIG_PATH"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"patternProperties": {
|
||||
"^[a-zA-Z0-9_-]+$": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "token"
|
||||
},
|
||||
"value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"env": {
|
||||
"type": "string",
|
||||
"description": "The name of the environment variable that contains the token."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"env"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"googleCloudSecret": {
|
||||
"type": "string",
|
||||
"description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"googleCloudSecret"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "string"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "number"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "boolean"
|
||||
},
|
||||
"value": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -279,25 +279,30 @@
|
|||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"connections": {
|
||||
"environmentOverrides": {
|
||||
"type": "object",
|
||||
"description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.",
|
||||
"description": "Environment variable overrides.",
|
||||
"title": "EnvironmentOverrides",
|
||||
"not": {
|
||||
"$comment": "List of environment variables that are not allowed to be overridden.",
|
||||
"anyOf": [
|
||||
{
|
||||
"required": [
|
||||
"CONFIG_PATH"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"patternProperties": {
|
||||
"^[a-zA-Z0-9_-]+$": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ConnectionConfig",
|
||||
"oneOf": [
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"title": "GithubConnectionConfig",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "github",
|
||||
"description": "GitHub Configuration"
|
||||
"const": "token"
|
||||
},
|
||||
"token": {
|
||||
"description": "A Personal Access Token (PAT).",
|
||||
"value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -326,6 +331,113 @@
|
|||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "string"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "number"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "boolean"
|
||||
},
|
||||
"value": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"connections": {
|
||||
"type": "object",
|
||||
"description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.",
|
||||
"patternProperties": {
|
||||
"^[a-zA-Z0-9_-]+$": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ConnectionConfig",
|
||||
"oneOf": [
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"title": "GithubConnectionConfig",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "github",
|
||||
"description": "GitHub Configuration"
|
||||
},
|
||||
"token": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"env": {
|
||||
"type": "string",
|
||||
"description": "The name of the environment variable that contains the token."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"env"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"googleCloudSecret": {
|
||||
"type": "string",
|
||||
"description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"googleCloudSecret"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
],
|
||||
"description": "A Personal Access Token (PAT)."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -505,7 +617,6 @@
|
|||
"description": "GitLab Configuration"
|
||||
},
|
||||
"token": {
|
||||
"description": "An authentication token.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -533,7 +644,8 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "An authentication token."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -707,7 +819,6 @@
|
|||
"description": "Gitea Configuration"
|
||||
},
|
||||
"token": {
|
||||
"description": "A Personal Access Token (PAT).",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -735,7 +846,8 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "A Personal Access Token (PAT)."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -974,7 +1086,6 @@
|
|||
"description": "The username to use for authentication. Only needed if token is an app password."
|
||||
},
|
||||
"token": {
|
||||
"description": "An authentication token.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1002,7 +1113,8 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "An authentication token."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -1142,7 +1254,6 @@
|
|||
"description": "Azure DevOps Configuration"
|
||||
},
|
||||
"token": {
|
||||
"description": "A Personal Access Token (PAT).",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1170,7 +1281,8 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "A Personal Access Token (PAT)."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -1426,7 +1538,6 @@
|
|||
"description": "Optional display name."
|
||||
},
|
||||
"accessKeyId": {
|
||||
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1454,10 +1565,10 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable."
|
||||
},
|
||||
"accessKeySecret": {
|
||||
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1485,10 +1596,10 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable."
|
||||
},
|
||||
"sessionToken": {
|
||||
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1516,7 +1627,8 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable."
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
|
|
@ -2855,7 +2967,6 @@
|
|||
"description": "Optional display name."
|
||||
},
|
||||
"accessKeyId": {
|
||||
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -2883,10 +2994,10 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable."
|
||||
},
|
||||
"accessKeySecret": {
|
||||
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -2914,10 +3025,10 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable."
|
||||
},
|
||||
"sessionToken": {
|
||||
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -2945,7 +3056,8 @@
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable."
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
|
|
|
|||
|
|
@ -1,28 +1,57 @@
|
|||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# Check if DATABASE_URL is not set
|
||||
if [ -z "$DATABASE_URL" ]; then
|
||||
# Check if the individual database variables are set and construct the URL
|
||||
if [ -n "$DATABASE_HOST" ] && [ -n "$DATABASE_USERNAME" ] && [ -n "$DATABASE_PASSWORD" ] && [ -n "$DATABASE_NAME" ]; then
|
||||
# Exit immediately if a command fails
|
||||
set -e
|
||||
# Disable auto-exporting of variables
|
||||
set +a
|
||||
|
||||
# If a CONFIG_PATH is set, resolve the environment overrides from the config file.
|
||||
# The overrides will be written into variables scopped to the current shell. This is
|
||||
# required in case one of the variables used in this entrypoint is overriden (e.g.,
|
||||
# DATABASE_URL, REDIS_URL, etc.)
|
||||
if [ -n "$CONFIG_PATH" ]; then
|
||||
echo -e "\e[34m[Info] Resolving environment overrides from $CONFIG_PATH...\e[0m"
|
||||
|
||||
set +e # Disable exist on error so we can capture EXIT_CODE
|
||||
OVERRIDES_OUTPUT=$(SKIP_ENV_VALIDATION=1 yarn tool:resolve-env-overrides 2>&1)
|
||||
EXIT_CODE=$?
|
||||
set -e # Re-enable exit on error
|
||||
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
eval "$OVERRIDES_OUTPUT"
|
||||
else
|
||||
echo -e "\e[31m[Error] Failed to resolve environment overrides.\e[0m"
|
||||
echo "$OVERRIDES_OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Descontruct the database URL from the individual variables if DATABASE_URL is not set
|
||||
if [ -z "$DATABASE_URL" ] && [ -n "$DATABASE_HOST" ] && [ -n "$DATABASE_USERNAME" ] && [ -n "$DATABASE_PASSWORD" ] && [ -n "$DATABASE_NAME" ]; then
|
||||
DATABASE_URL="postgresql://${DATABASE_USERNAME}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}"
|
||||
|
||||
if [ -n "$DATABASE_ARGS" ]; then
|
||||
DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS"
|
||||
fi
|
||||
|
||||
export DATABASE_URL
|
||||
else
|
||||
# Otherwise, fallback to a default value
|
||||
DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot"
|
||||
export DATABASE_URL
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$DATABASE_URL" = "postgresql://postgres@localhost:5432/sourcebot" ]; then
|
||||
DATABASE_EMBEDDED="true"
|
||||
if [ -z "$DATABASE_URL" ]; then
|
||||
echo -e "\e[34m[Info] DATABASE_URL is not set. Using embeded database.\e[0m"
|
||||
export DATABASE_EMBEDDED="true"
|
||||
export DATABASE_URL="postgresql://postgres@localhost:5432/sourcebot"
|
||||
else
|
||||
export DATABASE_EMBEDDED="false"
|
||||
fi
|
||||
|
||||
if [ -z "$REDIS_URL" ]; then
|
||||
echo -e "\e[34m[Info] REDIS_URL is not set. Using embeded redis.\e[0m"
|
||||
export REDIS_EMBEDDED="true"
|
||||
export REDIS_URL="redis://localhost:6379"
|
||||
else
|
||||
export REDIS_EMBEDDED="false"
|
||||
fi
|
||||
|
||||
|
||||
echo -e "\e[34m[Info] Sourcebot version: $NEXT_PUBLIC_SOURCEBOT_VERSION\e[0m"
|
||||
|
||||
# If we don't have a PostHog key, then we need to disable telemetry.
|
||||
|
|
@ -59,7 +88,7 @@ if [ "$DATABASE_EMBEDDED" = "true" ] && [ ! -d "$DATABASE_DATA_DIR" ]; then
|
|||
fi
|
||||
|
||||
# Create the redis data directory if it doesn't exist
|
||||
if [ ! -d "$REDIS_DATA_DIR" ]; then
|
||||
if [ "$REDIS_EMBEDDED" = "true" ] && [ ! -d "$REDIS_DATA_DIR" ]; then
|
||||
mkdir -p $REDIS_DATA_DIR
|
||||
fi
|
||||
|
||||
|
|
@ -149,7 +178,6 @@ fi
|
|||
|
||||
echo "{\"version\": \"$NEXT_PUBLIC_SOURCEBOT_VERSION\", \"install_id\": \"$SOURCEBOT_INSTALL_ID\"}" > "$FIRST_RUN_FILE"
|
||||
|
||||
|
||||
# Start the database and wait for it to be ready before starting any other service
|
||||
if [ "$DATABASE_EMBEDDED" = "true" ]; then
|
||||
su postgres -c "postgres -D $DATABASE_DATA_DIR" &
|
||||
|
|
@ -171,7 +199,7 @@ fi
|
|||
|
||||
# Run a Database migration
|
||||
echo -e "\e[34m[Info] Running database migration...\e[0m"
|
||||
yarn workspace @sourcebot/db prisma:migrate:prod
|
||||
DATABASE_URL="$DATABASE_URL" yarn workspace @sourcebot/db prisma:migrate:prod
|
||||
|
||||
# Create the log directory
|
||||
mkdir -p /var/log/sourcebot
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@
|
|||
"packages/*"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "cross-env SKIP_ENV_VALIDATION=1 yarn workspaces foreach -A run build",
|
||||
"test": "yarn workspaces foreach -A run test",
|
||||
"build": "cross-env SKIP_ENV_VALIDATION=1 yarn workspaces foreach --all --topological run build",
|
||||
"test": "yarn workspaces foreach --all --topological run test",
|
||||
"dev": "concurrently --kill-others --names \"zoekt,worker,web,mcp,schemas\" 'yarn dev:zoekt' 'yarn dev:backend' 'yarn dev:web' 'yarn watch:mcp' 'yarn watch:schemas'",
|
||||
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
|
||||
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
|
||||
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset",
|
||||
"dev:prisma:db:push": "yarn with-env yarn workspace @sourcebot/db prisma:db:push",
|
||||
"build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db,@sourcebot/shared}' run build"
|
||||
"build:deps": "yarn workspaces foreach --recursive --topological --from '{@sourcebot/schemas,@sourcebot/db,@sourcebot/shared}' run build"
|
||||
},
|
||||
"devDependencies": {
|
||||
"concurrently": "^9.2.1",
|
||||
|
|
|
|||
|
|
@ -29,13 +29,9 @@
|
|||
"@sentry/cli": "^2.42.2",
|
||||
"@sentry/node": "^9.3.0",
|
||||
"@sentry/profiling-node": "^9.3.0",
|
||||
"@sourcebot/crypto": "workspace:*",
|
||||
"@sourcebot/db": "workspace:*",
|
||||
"@sourcebot/error": "workspace:*",
|
||||
"@sourcebot/logger": "workspace:*",
|
||||
"@sourcebot/schemas": "workspace:*",
|
||||
"@sourcebot/shared": "workspace:*",
|
||||
"@t3-oss/env-core": "^0.12.0",
|
||||
"@types/express": "^5.0.0",
|
||||
"argparse": "^2.0.1",
|
||||
"azure-devops-node-api": "^15.1.1",
|
||||
|
|
@ -55,6 +51,6 @@
|
|||
"posthog-node": "^4.2.1",
|
||||
"prom-client": "^15.1.3",
|
||||
"simple-git": "^3.27.0",
|
||||
"zod": "^3.24.3"
|
||||
"zod": "^3.25.74"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,12 @@
|
|||
import { AzureDevOpsConnectionConfig } from "@sourcebot/schemas/v3/azuredevops.type";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { measure, fetchWithRetry } from "./utils.js";
|
||||
import micromatch from "micromatch";
|
||||
import { BackendException, BackendError } from "@sourcebot/error";
|
||||
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
|
||||
import * as Sentry from "@sentry/node";
|
||||
import * as azdev from "azure-devops-node-api";
|
||||
import { GitRepository } from "azure-devops-node-api/interfaces/GitInterfaces.js";
|
||||
import { getTokenFromConfig } from "@sourcebot/crypto";
|
||||
import { getTokenFromConfig } from "@sourcebot/shared";
|
||||
|
||||
const logger = createLogger('azuredevops');
|
||||
const AZUREDEVOPS_CLOUD_HOSTNAME = "dev.azure.com";
|
||||
|
|
@ -36,9 +35,7 @@ export const getAzureDevOpsReposFromConfig = async (
|
|||
undefined;
|
||||
|
||||
if (!token) {
|
||||
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
|
||||
message: 'Azure DevOps requires a Personal Access Token',
|
||||
});
|
||||
const e = new Error('Azure DevOps requires a Personal Access Token');
|
||||
Sentry.captureException(e);
|
||||
throw e;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { createBitbucketCloudClient } from "@coderabbitai/bitbucket/cloud";
|
|||
import { createBitbucketServerClient } from "@coderabbitai/bitbucket/server";
|
||||
import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type";
|
||||
import type { ClientOptions, ClientPathsWithMethod } from "openapi-fetch";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { measure, fetchWithRetry } from "./utils.js";
|
||||
import * as Sentry from "@sentry/node";
|
||||
import {
|
||||
|
|
@ -11,7 +11,7 @@ import {
|
|||
import { SchemaRestRepository as ServerRepository } from "@coderabbitai/bitbucket/server/openapi";
|
||||
import { processPromiseResults } from "./connectionUtils.js";
|
||||
import { throwIfAnyFailed } from "./connectionUtils.js";
|
||||
import { getTokenFromConfig } from "@sourcebot/crypto";
|
||||
import { getTokenFromConfig } from "@sourcebot/shared";
|
||||
|
||||
const logger = createLogger('bitbucket');
|
||||
const BITBUCKET_CLOUD_GIT = 'https://bitbucket.org';
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { Prisma, PrismaClient } from "@sourcebot/db";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
|
||||
import { loadConfig } from "@sourcebot/shared";
|
||||
import chokidar, { FSWatcher } from 'chokidar';
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
import * as Sentry from "@sentry/node";
|
||||
import { Connection, ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
|
||||
import { loadConfig } from "@sourcebot/shared";
|
||||
import { loadConfig, env } from "@sourcebot/shared";
|
||||
import { Job, Queue, ReservedJob, Worker } from "groupmq";
|
||||
import { Redis } from 'ioredis';
|
||||
import { env } from "./env.js";
|
||||
import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js";
|
||||
import { Settings } from "./types.js";
|
||||
import { groupmqLifecycleExceptionWrapper } from "./utils.js";
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { CodeHostType } from "@sourcebot/db";
|
||||
import { env } from "./env.js";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import path from "path";
|
||||
|
||||
export const SINGLE_TENANT_ORG_ID = 1;
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
import * as Sentry from "@sentry/node";
|
||||
import { PrismaClient, AccountPermissionSyncJobStatus, Account } from "@sourcebot/db";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { PrismaClient, AccountPermissionSyncJobStatus, Account} from "@sourcebot/db";
|
||||
import { env, hasEntitlement, createLogger } from "@sourcebot/shared";
|
||||
import { Job, Queue, Worker } from "bullmq";
|
||||
import { Redis } from "ioredis";
|
||||
import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js";
|
||||
import { env } from "../env.js";
|
||||
import { createOctokitFromToken, getReposForAuthenticatedUser } from "../github.js";
|
||||
import { createGitLabFromOAuthToken, getProjectsForAuthenticatedUser } from "../gitlab.js";
|
||||
import { hasEntitlement } from "@sourcebot/shared";
|
||||
import { Settings } from "../types.js";
|
||||
|
||||
const LOG_TAG = 'user-permission-syncer';
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
import { loadConfig } from "@sourcebot/shared";
|
||||
import { env } from "../env.js";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { getTokenFromConfig } from "@sourcebot/crypto";
|
||||
import { PrismaClient } from "@sourcebot/db";
|
||||
import { App } from "@octokit/app";
|
||||
import { getTokenFromConfig } from "@sourcebot/shared";
|
||||
import { PrismaClient } from "@sourcebot/db";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { GitHubAppConfig } from "@sourcebot/schemas/v3/index.type";
|
||||
import { env, loadConfig } from "@sourcebot/shared";
|
||||
|
||||
const logger = createLogger('githubAppManager');
|
||||
const GITHUB_DEFAULT_DEPLOYMENT_HOSTNAME = 'github.com';
|
||||
|
|
@ -45,7 +44,7 @@ export class GithubAppManager {
|
|||
|
||||
public async init(db: PrismaClient) {
|
||||
this.db = db;
|
||||
const config = await loadConfig(env.CONFIG_PATH!);
|
||||
const config = await loadConfig(env.CONFIG_PATH);
|
||||
if (!config.apps) {
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
import * as Sentry from "@sentry/node";
|
||||
import { PrismaClient, Repo, RepoPermissionSyncJobStatus } from "@sourcebot/db";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { hasEntitlement } from "@sourcebot/shared";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { env, hasEntitlement } from "@sourcebot/shared";
|
||||
import { Job, Queue, Worker } from 'bullmq';
|
||||
import { Redis } from 'ioredis';
|
||||
import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js";
|
||||
import { env } from "../env.js";
|
||||
import { createOctokitFromToken, getRepoCollaborators, GITHUB_CLOUD_HOSTNAME } from "../github.js";
|
||||
import { createGitLabFromPersonalAccessToken, getProjectMembers } from "../gitlab.js";
|
||||
import { Settings } from "../types.js";
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import micromatch from "micromatch";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { PrismaClient } from "@sourcebot/db";
|
||||
import { getPlan, hasEntitlement, SOURCEBOT_SUPPORT_EMAIL } from "@sourcebot/shared";
|
||||
import { SearchContext } from "@sourcebot/schemas/v3/index.type";
|
||||
|
|
|
|||
|
|
@ -1,64 +0,0 @@
|
|||
import { createEnv } from "@t3-oss/env-core";
|
||||
import { z } from "zod";
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
// Booleans are specified as 'true' or 'false' strings.
|
||||
const booleanSchema = z.enum(["true", "false"]);
|
||||
|
||||
// Numbers are treated as strings in .env files.
|
||||
// coerce helps us convert them to numbers.
|
||||
// @see: https://zod.dev/?id=coercion-for-primitives
|
||||
const numberSchema = z.coerce.number();
|
||||
|
||||
dotenv.config({
|
||||
path: './.env',
|
||||
});
|
||||
|
||||
dotenv.config({
|
||||
path: './.env.local',
|
||||
override: true
|
||||
});
|
||||
|
||||
export const env = createEnv({
|
||||
server: {
|
||||
SOURCEBOT_ENCRYPTION_KEY: z.string(),
|
||||
SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default("false"),
|
||||
SOURCEBOT_INSTALL_ID: z.string().default("unknown"),
|
||||
NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"),
|
||||
|
||||
DATA_CACHE_DIR: z.string(),
|
||||
|
||||
NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(),
|
||||
|
||||
FALLBACK_GITHUB_CLOUD_TOKEN: z.string().optional(),
|
||||
FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(),
|
||||
FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(),
|
||||
|
||||
REDIS_URL: z.string().url().default("redis://localhost:6379"),
|
||||
REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0),
|
||||
REDIS_REMOVE_ON_FAIL: numberSchema.default(100),
|
||||
|
||||
NEXT_PUBLIC_SENTRY_BACKEND_DSN: z.string().optional(),
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT: z.string().optional(),
|
||||
|
||||
LOGTAIL_TOKEN: z.string().optional(),
|
||||
LOGTAIL_HOST: z.string().url().optional(),
|
||||
SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"),
|
||||
DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'),
|
||||
|
||||
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
|
||||
CONFIG_PATH: z.string(),
|
||||
|
||||
CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000),
|
||||
REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60),
|
||||
|
||||
GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10),
|
||||
|
||||
EXPERIMENT_EE_PERMISSION_SYNC_ENABLED: booleanSchema.default('false'),
|
||||
AUTH_EE_GITHUB_BASE_URL: z.string().optional(),
|
||||
AUTH_EE_GITLAB_BASE_URL: z.string().default("https://gitlab.com"),
|
||||
},
|
||||
runtimeEnv: process.env,
|
||||
emptyStringAsUndefined: true,
|
||||
skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
|
||||
});
|
||||
|
|
@ -1,11 +1,8 @@
|
|||
import { GerritConnectionConfig } from "@sourcebot/schemas/v3/index.type";
|
||||
import { createLogger } from '@sourcebot/shared';
|
||||
import fetch from 'cross-fetch';
|
||||
import { GerritConnectionConfig } from "@sourcebot/schemas/v3/index.type"
|
||||
import { createLogger } from '@sourcebot/logger';
|
||||
import micromatch from "micromatch";
|
||||
import { measure, fetchWithRetry } from './utils.js';
|
||||
import { BackendError } from '@sourcebot/error';
|
||||
import { BackendException } from '@sourcebot/error';
|
||||
import * as Sentry from "@sentry/node";
|
||||
import { fetchWithRetry, measure } from './utils.js';
|
||||
|
||||
// https://gerrit-review.googlesource.com/Documentation/rest-api.html
|
||||
interface GerritProjects {
|
||||
|
|
@ -39,26 +36,10 @@ export const getGerritReposFromConfig = async (config: GerritConnectionConfig):
|
|||
const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
|
||||
|
||||
let { durationMs, data: projects } = await measure(async () => {
|
||||
try {
|
||||
const fetchFn = () => fetchAllProjects(url);
|
||||
return fetchWithRetry(fetchFn, `projects from ${url}`, logger);
|
||||
} catch (err) {
|
||||
Sentry.captureException(err);
|
||||
if (err instanceof BackendException) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
logger.error(`Failed to fetch projects from ${url}`, err);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
if (!projects) {
|
||||
const e = new Error(`Failed to fetch projects from ${url}`);
|
||||
Sentry.captureException(e);
|
||||
throw e;
|
||||
}
|
||||
|
||||
// include repos by glob if specified in config
|
||||
if (config.projects) {
|
||||
projects = projects.filter((project) => {
|
||||
|
|
@ -91,27 +72,9 @@ const fetchAllProjects = async (url: string): Promise<GerritProject[]> => {
|
|||
logger.debug(`Fetching projects from Gerrit at ${endpointWithParams}`);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(endpointWithParams);
|
||||
if (!response.ok) {
|
||||
logger.error(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${response.status}`);
|
||||
const e = new BackendException(BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS, {
|
||||
status: response.status,
|
||||
});
|
||||
Sentry.captureException(e);
|
||||
throw e;
|
||||
}
|
||||
} catch (err) {
|
||||
Sentry.captureException(err);
|
||||
if (err instanceof BackendException) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
const status = (err as any).code;
|
||||
logger.error(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${status}`);
|
||||
throw new BackendException(BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS, {
|
||||
status: status,
|
||||
});
|
||||
throw new Error(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${response.status}`);
|
||||
}
|
||||
|
||||
const text = await response.text();
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { env } from './env.js';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { existsSync } from 'node:fs';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git';
|
||||
|
||||
type onProgressFn = (event: SimpleGitProgressEvent) => void;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
|
||||
import * as Sentry from "@sentry/node";
|
||||
import { getTokenFromConfig } from "@sourcebot/shared";
|
||||
import { createLogger } from '@sourcebot/shared';
|
||||
import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type';
|
||||
import { measure } from './utils.js';
|
||||
import { env } from "@sourcebot/shared";
|
||||
import fetch from 'cross-fetch';
|
||||
import { createLogger } from '@sourcebot/logger';
|
||||
import { Api, giteaApi, Repository as GiteaRepository, HttpResponse } from 'gitea-js';
|
||||
import micromatch from 'micromatch';
|
||||
import { processPromiseResults, throwIfAnyFailed } from './connectionUtils.js';
|
||||
import * as Sentry from "@sentry/node";
|
||||
import { env } from './env.js';
|
||||
import { getTokenFromConfig } from "@sourcebot/crypto";
|
||||
import { measure } from './utils.js';
|
||||
|
||||
const logger = createLogger('gitea');
|
||||
const GITEA_CLOUD_HOSTNAME = "gitea.com";
|
||||
|
|
|
|||
|
|
@ -1,15 +1,14 @@
|
|||
import { Octokit } from "@octokit/rest";
|
||||
import * as Sentry from "@sentry/node";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { getTokenFromConfig } from "@sourcebot/shared";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
|
||||
import { hasEntitlement } from "@sourcebot/shared";
|
||||
import { env, hasEntitlement } from "@sourcebot/shared";
|
||||
import micromatch from "micromatch";
|
||||
import pLimit from "p-limit";
|
||||
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
|
||||
import { GithubAppManager } from "./ee/githubAppManager.js";
|
||||
import { env } from "./env.js";
|
||||
import { fetchWithRetry, measure } from "./utils.js";
|
||||
import { getTokenFromConfig } from "@sourcebot/crypto";
|
||||
|
||||
export const GITHUB_CLOUD_HOSTNAME = "github.com";
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import { Gitlab, ProjectSchema } from "@gitbeaker/rest";
|
||||
import micromatch from "micromatch";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"
|
||||
import { measure, fetchWithRetry } from "./utils.js";
|
||||
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
|
||||
import * as Sentry from "@sentry/node";
|
||||
import { env } from "./env.js";
|
||||
import { getTokenFromConfig } from "@sourcebot/crypto";
|
||||
import { getTokenFromConfig } from "@sourcebot/shared";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import micromatch from "micromatch";
|
||||
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
|
||||
import { fetchWithRetry, measure } from "./utils.js";
|
||||
|
||||
const logger = createLogger('gitlab');
|
||||
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import "./instrument.js";
|
||||
|
||||
import { PrismaClient } from "@sourcebot/db";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { env, getConfigSettings, hasEntitlement, getDBConnectionString } from '@sourcebot/shared';
|
||||
import { existsSync } from 'fs';
|
||||
import { mkdir } from 'fs/promises';
|
||||
import { Redis } from 'ioredis';
|
||||
|
|
@ -12,7 +12,6 @@ import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
|
|||
import { GithubAppManager } from "./ee/githubAppManager.js";
|
||||
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
|
||||
import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js";
|
||||
import { env } from "./env.js";
|
||||
import { PromClient } from './promClient.js';
|
||||
import { RepoIndexManager } from "./repoIndexManager.js";
|
||||
|
||||
|
|
@ -29,7 +28,13 @@ if (!existsSync(indexPath)) {
|
|||
await mkdir(indexPath, { recursive: true });
|
||||
}
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
const prisma = new PrismaClient({
|
||||
datasources: {
|
||||
db: {
|
||||
url: getDBConnectionString(),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const redis = new Redis(env.REDIS_URL, {
|
||||
maxRetriesPerRequest: null
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import * as Sentry from "@sentry/node";
|
||||
import { env } from "./env.js";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { env } from "@sourcebot/shared/client";
|
||||
|
||||
const logger = createLogger('instrument');
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
import { env as clientEnv } from "@sourcebot/shared/client";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { PostHog } from 'posthog-node';
|
||||
import { PosthogEvent, PosthogEventMap } from './posthogEvents.js';
|
||||
import { env } from './env.js';
|
||||
|
||||
let posthog: PostHog | undefined = undefined;
|
||||
|
||||
if (env.NEXT_PUBLIC_POSTHOG_PAPIK) {
|
||||
if (clientEnv.NEXT_PUBLIC_POSTHOG_PAPIK) {
|
||||
posthog = new PostHog(
|
||||
env.NEXT_PUBLIC_POSTHOG_PAPIK,
|
||||
clientEnv.NEXT_PUBLIC_POSTHOG_PAPIK,
|
||||
{
|
||||
host: "https://us.i.posthog.com",
|
||||
}
|
||||
|
|
@ -23,7 +24,7 @@ export function captureEvent<E extends PosthogEvent>(event: E, properties: Posth
|
|||
event: event,
|
||||
properties: {
|
||||
...properties,
|
||||
sourcebot_version: env.NEXT_PUBLIC_SOURCEBOT_VERSION,
|
||||
sourcebot_version: clientEnv.NEXT_PUBLIC_SOURCEBOT_VERSION,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import express, { Request, Response } from 'express';
|
||||
import { Server } from 'http';
|
||||
import client, { Registry, Counter, Gauge } from 'prom-client';
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
|
||||
const logger = createLogger('prometheus-client');
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { SchemaRepository as BitbucketCloudRepository } from "@coderabbitai/bitb
|
|||
import { CodeHostType, Prisma } from '@sourcebot/db';
|
||||
import { WithRequired } from "./types.js"
|
||||
import { marshalBool } from "./utils.js";
|
||||
import { createLogger } from '@sourcebot/logger';
|
||||
import { createLogger } from '@sourcebot/shared';
|
||||
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
|
||||
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
|
||||
import path from 'path';
|
||||
|
|
|
|||
|
|
@ -1,14 +1,13 @@
|
|||
import * as Sentry from '@sentry/node';
|
||||
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
|
||||
import { createLogger, Logger } from "@sourcebot/logger";
|
||||
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
|
||||
import { createLogger, Logger } from "@sourcebot/shared";
|
||||
import { env, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from '@sourcebot/shared';
|
||||
import { existsSync } from 'fs';
|
||||
import { readdir, rm } from 'fs/promises';
|
||||
import { Job, Queue, ReservedJob, Worker } from "groupmq";
|
||||
import { Redis } from 'ioredis';
|
||||
import micromatch from 'micromatch';
|
||||
import { INDEX_CACHE_DIR } from './constants.js';
|
||||
import { env } from './env.js';
|
||||
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
|
||||
import { captureEvent } from './posthog.js';
|
||||
import { PromClient } from './promClient.js';
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { Logger } from "winston";
|
|||
import { RepoAuthCredentials, RepoWithConnections } from "./types.js";
|
||||
import path from 'path';
|
||||
import { Repo } from "@sourcebot/db";
|
||||
import { getTokenFromConfig } from "@sourcebot/crypto";
|
||||
import { getTokenFromConfig } from "@sourcebot/shared";
|
||||
import * as Sentry from "@sentry/node";
|
||||
import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
|
||||
import { GithubAppManager } from "./ee/githubAppManager.js";
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { Repo } from "@sourcebot/db";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { exec } from "child_process";
|
||||
import { INDEX_CACHE_DIR } from "./constants.js";
|
||||
import { Settings } from "./types.js";
|
||||
|
|
|
|||
1
packages/crypto/.gitignore
vendored
1
packages/crypto/.gitignore
vendored
|
|
@ -1 +0,0 @@
|
|||
.env.local
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
{
|
||||
"name": "@sourcebot/crypto",
|
||||
"version": "0.1.0",
|
||||
"main": "dist/index.js",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"postinstall": "yarn build"
|
||||
},
|
||||
"dependencies": {
|
||||
"@google-cloud/secret-manager": "^6.1.1",
|
||||
"@sourcebot/db": "*",
|
||||
"@sourcebot/schemas": "*",
|
||||
"dotenv": "^16.4.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.7.5",
|
||||
"typescript": "^5.7.3"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
import dotenv from 'dotenv';
|
||||
|
||||
export const getEnv = (env: string | undefined, defaultValue?: string) => {
|
||||
return env ?? defaultValue;
|
||||
}
|
||||
|
||||
dotenv.config({
|
||||
path: './.env.local',
|
||||
override: true
|
||||
});
|
||||
|
||||
// @note: You can use https://generate-random.org/encryption-key-generator to create a new 32 byte key
|
||||
export const SOURCEBOT_ENCRYPTION_KEY = getEnv(process.env.SOURCEBOT_ENCRYPTION_KEY);
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
import { SecretManagerServiceClient } from "@google-cloud/secret-manager";
|
||||
import { Token } from "@sourcebot/schemas/v3/shared.type";
|
||||
|
||||
export const getTokenFromConfig = async (token: Token): Promise<string> => {
|
||||
if ('env' in token) {
|
||||
const envToken = process.env[token.env];
|
||||
if (!envToken) {
|
||||
throw new Error(`Environment variable ${token.env} not found.`);
|
||||
}
|
||||
|
||||
return envToken;
|
||||
} else if ('googleCloudSecret' in token) {
|
||||
try {
|
||||
const client = new SecretManagerServiceClient();
|
||||
const [response] = await client.accessSecretVersion({
|
||||
name: token.googleCloudSecret,
|
||||
});
|
||||
|
||||
if (!response.payload?.data) {
|
||||
throw new Error(`Secret ${token.googleCloudSecret} not found.`);
|
||||
}
|
||||
|
||||
return response.payload.data.toString();
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to access Google Cloud secret ${token.googleCloudSecret}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
} else {
|
||||
throw new Error('Invalid token configuration');
|
||||
}
|
||||
};
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "Node16",
|
||||
"lib": ["ES2023"],
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"strict": true,
|
||||
"noImplicitAny": true,
|
||||
"strictNullChecks": true,
|
||||
"moduleResolution": "Node16",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"skipLibCheck": true,
|
||||
"isolatedModules": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
|
||||
|
|
@ -25,7 +25,6 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@prisma/client": "6.2.1",
|
||||
"@sourcebot/logger": "workspace:*",
|
||||
"@types/readline-sync": "^1.4.8",
|
||||
"readline-sync": "^1.4.10"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import { ArgumentParser } from "argparse";
|
|||
import { migrateDuplicateConnections } from "./scripts/migrate-duplicate-connections";
|
||||
import { injectAuditData } from "./scripts/inject-audit-data";
|
||||
import { confirmAction } from "./utils";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { injectRepoData } from "./scripts/inject-repo-data";
|
||||
import { testRepoQueryPerf } from "./scripts/test-repo-query-perf";
|
||||
|
||||
|
|
@ -23,19 +22,17 @@ parser.add_argument("--url", { required: true, help: "Database URL" });
|
|||
parser.add_argument("--script", { required: true, help: "Script to run" });
|
||||
const args = parser.parse_args();
|
||||
|
||||
const logger = createLogger('db-script-runner');
|
||||
|
||||
(async () => {
|
||||
if (!(args.script in scripts)) {
|
||||
logger.error("Invalid script");
|
||||
console.error("Invalid script");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const selectedScript = scripts[args.script];
|
||||
|
||||
logger.info("\nTo confirm:");
|
||||
logger.info(`- Database URL: ${args.url}`);
|
||||
logger.info(`- Script: ${args.script}`);
|
||||
console.log("\nTo confirm:");
|
||||
console.log(`- Database URL: ${args.url}`);
|
||||
console.log(`- Script: ${args.script}`);
|
||||
|
||||
confirmAction();
|
||||
|
||||
|
|
@ -45,7 +42,7 @@ const logger = createLogger('db-script-runner');
|
|||
|
||||
await selectedScript.run(prisma);
|
||||
|
||||
logger.info("\nDone.");
|
||||
console.log("\nDone.");
|
||||
process.exit(0);
|
||||
})();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
import { Script } from "../scriptRunner";
|
||||
import { PrismaClient } from "../../dist";
|
||||
import { confirmAction } from "../utils";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
|
||||
const logger = createLogger('inject-audit-data');
|
||||
|
||||
// Generate realistic audit data for analytics testing
|
||||
// Simulates 50 engineers with varying activity patterns
|
||||
|
|
@ -17,11 +14,11 @@ export const injectAuditData: Script = {
|
|||
});
|
||||
|
||||
if (!org) {
|
||||
logger.error(`Organization with id ${orgId} not found. Please create it first.`);
|
||||
console.error(`Organization with id ${orgId} not found. Please create it first.`);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Injecting audit data for organization: ${org.name} (${org.domain})`);
|
||||
console.log(`Injecting audit data for organization: ${org.name} (${org.domain})`);
|
||||
|
||||
// Generate 50 fake user IDs
|
||||
const userIds = Array.from({ length: 50 }, (_, i) => `user_${String(i + 1).padStart(3, '0')}`);
|
||||
|
|
@ -38,7 +35,7 @@ export const injectAuditData: Script = {
|
|||
const startDate = new Date();
|
||||
startDate.setDate(startDate.getDate() - 90);
|
||||
|
||||
logger.info(`Generating data from ${startDate.toISOString().split('T')[0]} to ${endDate.toISOString().split('T')[0]}`);
|
||||
console.log(`Generating data from ${startDate.toISOString().split('T')[0]} to ${endDate.toISOString().split('T')[0]}`);
|
||||
|
||||
confirmAction();
|
||||
|
||||
|
|
@ -125,9 +122,9 @@ export const injectAuditData: Script = {
|
|||
}
|
||||
}
|
||||
|
||||
logger.info(`\nAudit data injection complete!`);
|
||||
logger.info(`Users: ${userIds.length}`);
|
||||
logger.info(`Date range: ${startDate.toISOString().split('T')[0]} to ${endDate.toISOString().split('T')[0]}`);
|
||||
console.log(`\nAudit data injection complete!`);
|
||||
console.log(`Users: ${userIds.length}`);
|
||||
console.log(`Date range: ${startDate.toISOString().split('T')[0]} to ${endDate.toISOString().split('T')[0]}`);
|
||||
|
||||
// Show some statistics
|
||||
const stats = await prisma.audit.groupBy({
|
||||
|
|
@ -136,9 +133,9 @@ export const injectAuditData: Script = {
|
|||
_count: { action: true }
|
||||
});
|
||||
|
||||
logger.info('\nAction breakdown:');
|
||||
console.log('\nAction breakdown:');
|
||||
stats.forEach(stat => {
|
||||
logger.info(` ${stat.action}: ${stat._count.action}`);
|
||||
console.log(` ${stat.action}: ${stat._count.action}`);
|
||||
});
|
||||
},
|
||||
};
|
||||
|
|
@ -1,8 +1,5 @@
|
|||
import { Script } from "../scriptRunner";
|
||||
import { PrismaClient } from "../../dist";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
|
||||
const logger = createLogger('inject-repo-data');
|
||||
|
||||
const NUM_REPOS = 100000;
|
||||
|
||||
|
|
@ -35,7 +32,7 @@ export const injectRepoData: Script = {
|
|||
});
|
||||
|
||||
|
||||
logger.info(`Creating ${NUM_REPOS} repos...`);
|
||||
console.log(`Creating ${NUM_REPOS} repos...`);
|
||||
|
||||
for (let i = 0; i < NUM_REPOS; i++) {
|
||||
await prisma.repo.create({
|
||||
|
|
@ -59,6 +56,6 @@ export const injectRepoData: Script = {
|
|||
});
|
||||
}
|
||||
|
||||
logger.info(`Created ${NUM_REPOS} repos.`);
|
||||
console.log(`Created ${NUM_REPOS} repos.`);
|
||||
}
|
||||
};
|
||||
|
|
@ -1,9 +1,6 @@
|
|||
import { Script } from "../scriptRunner";
|
||||
import { PrismaClient } from "../../dist";
|
||||
import { confirmAction } from "../utils";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
|
||||
const logger = createLogger('migrate-duplicate-connections');
|
||||
|
||||
// Handles duplicate connections by renaming them to be unique.
|
||||
// @see: 20250320215449_unique_connection_name_constraint_within_org
|
||||
|
|
@ -18,7 +15,7 @@ export const migrateDuplicateConnections: Script = {
|
|||
},
|
||||
})).filter(({ _count }) => _count._all > 1);
|
||||
|
||||
logger.info(`Found ${duplicates.reduce((acc, { _count }) => acc + _count._all, 0)} duplicate connections.`);
|
||||
console.log(`Found ${duplicates.reduce((acc, { _count }) => acc + _count._all, 0)} duplicate connections.`);
|
||||
|
||||
confirmAction();
|
||||
|
||||
|
|
@ -40,7 +37,7 @@ export const migrateDuplicateConnections: Script = {
|
|||
const connection = connections[i];
|
||||
const newName = `${name}-${i + 1}`;
|
||||
|
||||
logger.info(`Migrating connection with id ${connection.id} from name=${name} to name=${newName}`);
|
||||
console.log(`Migrating connection with id ${connection.id} from name=${name} to name=${newName}`);
|
||||
|
||||
await prisma.connection.update({
|
||||
where: { id: connection.id },
|
||||
|
|
@ -50,6 +47,6 @@ export const migrateDuplicateConnections: Script = {
|
|||
}
|
||||
}
|
||||
|
||||
logger.info(`Migrated ${migrated} connections.`);
|
||||
console.log(`Migrated ${migrated} connections.`);
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
import { Script } from "../scriptRunner";
|
||||
import { PrismaClient } from "../../dist";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
|
||||
const logger = createLogger('test-repo-query-perf');
|
||||
|
||||
export const testRepoQueryPerf: Script = {
|
||||
run: async (prisma: PrismaClient) => {
|
||||
|
|
@ -23,6 +20,6 @@ export const testRepoQueryPerf: Script = {
|
|||
});
|
||||
|
||||
const durationMs = Date.now() - start;
|
||||
logger.info(`Found ${allRepos.length} repos in ${durationMs}ms`);
|
||||
console.log(`Found ${allRepos.length} repos in ${durationMs}ms`);
|
||||
}
|
||||
};
|
||||
|
|
@ -1,17 +1,14 @@
|
|||
import readline from 'readline-sync';
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
|
||||
const logger = createLogger('db-utils');
|
||||
|
||||
export const confirmAction = (message: string = "Are you sure you want to proceed? [N/y]") => {
|
||||
const response = readline.question(message).toLowerCase();
|
||||
if (response !== 'y') {
|
||||
logger.info("Aborted.");
|
||||
console.log("Aborted.");
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
export const abort = () => {
|
||||
logger.info("Aborted.");
|
||||
console.log("Aborted.");
|
||||
process.exit(0);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
{
|
||||
"name": "@sourcebot/error",
|
||||
"main": "dist/index.js",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"postinstall": "yarn build"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.7.5",
|
||||
"typescript": "^5.7.3"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
export enum BackendError {
|
||||
CONNECTION_SYNC_SECRET_DNE = 'CONNECTION_SYNC_SECRET_DNE',
|
||||
CONNECTION_SYNC_INVALID_TOKEN = 'CONNECTION_SYNC_INVALID_TOKEN',
|
||||
CONNECTION_SYNC_SYSTEM_ERROR = 'CONNECTION_SYNC_SYSTEM_ERROR',
|
||||
CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS = 'CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS',
|
||||
CONNECTION_SYNC_CONNECTION_NOT_FOUND = 'CONNECTION_SYNC_CONNECTION_NOT_FOUND',
|
||||
}
|
||||
|
||||
export class BackendException extends Error {
|
||||
constructor(
|
||||
public readonly code: BackendError,
|
||||
public readonly metadata: Record<string, unknown> = {}
|
||||
) {
|
||||
super(code);
|
||||
this.name = 'BackendException';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES6",
|
||||
"module": "CommonJS",
|
||||
"lib": ["ES6"],
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"strict": true,
|
||||
"noImplicitAny": true,
|
||||
"strictNullChecks": true,
|
||||
"moduleResolution": "node",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"skipLibCheck": true,
|
||||
"isolatedModules": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
2
packages/logger/.gitignore
vendored
2
packages/logger/.gitignore
vendored
|
|
@ -1,2 +0,0 @@
|
|||
dist/
|
||||
*.tsbuildinfo
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
{
|
||||
"name": "@sourcebot/logger",
|
||||
"version": "0.1.0",
|
||||
"main": "dist/index.js",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"postinstall": "yarn build"
|
||||
},
|
||||
"dependencies": {
|
||||
"@logtail/node": "^0.5.2",
|
||||
"@logtail/winston": "^0.5.2",
|
||||
"@t3-oss/env-core": "^0.12.0",
|
||||
"dotenv": "^16.4.5",
|
||||
"triple-beam": "^1.4.1",
|
||||
"winston": "^3.15.0",
|
||||
"zod": "^3.24.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.7.5",
|
||||
"typescript": "^5.7.3"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
import { createEnv } from "@t3-oss/env-core";
|
||||
import { z } from "zod";
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
// Booleans are specified as 'true' or 'false' strings.
|
||||
const booleanSchema = z.enum(["true", "false"]);
|
||||
|
||||
dotenv.config({
|
||||
path: './.env',
|
||||
});
|
||||
|
||||
dotenv.config({
|
||||
path: './.env.local',
|
||||
override: true
|
||||
});
|
||||
|
||||
export const env = createEnv({
|
||||
server: {
|
||||
SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"),
|
||||
SOURCEBOT_STRUCTURED_LOGGING_ENABLED: booleanSchema.default("false"),
|
||||
SOURCEBOT_STRUCTURED_LOGGING_FILE: z.string().optional(),
|
||||
LOGTAIL_TOKEN: z.string().optional(),
|
||||
LOGTAIL_HOST: z.string().url().optional(),
|
||||
},
|
||||
runtimeEnv: process.env,
|
||||
emptyStringAsUndefined: true,
|
||||
skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
|
||||
});
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"lib": ["ES2023"],
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"strict": true,
|
||||
"noImplicitAny": true,
|
||||
"strictNullChecks": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"skipLibCheck": true,
|
||||
"isolatedModules": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
114
packages/schemas/src/v3/environmentOverrides.schema.ts
Normal file
114
packages/schemas/src/v3/environmentOverrides.schema.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
|
||||
const schema = {
|
||||
"type": "object",
|
||||
"description": "Environment variable overrides.",
|
||||
"title": "EnvironmentOverrides",
|
||||
"not": {
|
||||
"$comment": "List of environment variables that are not allowed to be overridden.",
|
||||
"anyOf": [
|
||||
{
|
||||
"required": [
|
||||
"CONFIG_PATH"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"patternProperties": {
|
||||
"^[a-zA-Z0-9_-]+$": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "token"
|
||||
},
|
||||
"value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"env": {
|
||||
"type": "string",
|
||||
"description": "The name of the environment variable that contains the token."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"env"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"googleCloudSecret": {
|
||||
"type": "string",
|
||||
"description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"googleCloudSecret"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "string"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "number"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "boolean"
|
||||
},
|
||||
"value": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
} as const;
|
||||
export { schema as environmentOverridesSchema };
|
||||
40
packages/schemas/src/v3/environmentOverrides.type.ts
Normal file
40
packages/schemas/src/v3/environmentOverrides.type.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
|
||||
|
||||
/**
|
||||
* Environment variable overrides.
|
||||
*/
|
||||
export interface EnvironmentOverrides {
|
||||
/**
|
||||
* This interface was referenced by `EnvironmentOverrides`'s JSON-Schema definition
|
||||
* via the `patternProperty` "^[a-zA-Z0-9_-]+$".
|
||||
*/
|
||||
[k: string]:
|
||||
| {
|
||||
type: "token";
|
||||
value:
|
||||
| {
|
||||
/**
|
||||
* The name of the environment variable that contains the token.
|
||||
*/
|
||||
env: string;
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets
|
||||
*/
|
||||
googleCloudSecret: string;
|
||||
};
|
||||
}
|
||||
| {
|
||||
type: "string";
|
||||
value: string;
|
||||
}
|
||||
| {
|
||||
type: "number";
|
||||
value: number;
|
||||
}
|
||||
| {
|
||||
type: "boolean";
|
||||
value: boolean;
|
||||
};
|
||||
}
|
||||
|
|
@ -278,25 +278,30 @@ const schema = {
|
|||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"connections": {
|
||||
"environmentOverrides": {
|
||||
"type": "object",
|
||||
"description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.",
|
||||
"description": "Environment variable overrides.",
|
||||
"title": "EnvironmentOverrides",
|
||||
"not": {
|
||||
"$comment": "List of environment variables that are not allowed to be overridden.",
|
||||
"anyOf": [
|
||||
{
|
||||
"required": [
|
||||
"CONFIG_PATH"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"patternProperties": {
|
||||
"^[a-zA-Z0-9_-]+$": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ConnectionConfig",
|
||||
"oneOf": [
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"title": "GithubConnectionConfig",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "github",
|
||||
"description": "GitHub Configuration"
|
||||
"const": "token"
|
||||
},
|
||||
"token": {
|
||||
"description": "A Personal Access Token (PAT).",
|
||||
"value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -325,6 +330,113 @@ const schema = {
|
|||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "string"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "number"
|
||||
},
|
||||
"value": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "boolean"
|
||||
},
|
||||
"value": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"value"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"connections": {
|
||||
"type": "object",
|
||||
"description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.",
|
||||
"patternProperties": {
|
||||
"^[a-zA-Z0-9_-]+$": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ConnectionConfig",
|
||||
"oneOf": [
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"title": "GithubConnectionConfig",
|
||||
"properties": {
|
||||
"type": {
|
||||
"const": "github",
|
||||
"description": "GitHub Configuration"
|
||||
},
|
||||
"token": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"env": {
|
||||
"type": "string",
|
||||
"description": "The name of the environment variable that contains the token."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"env"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"googleCloudSecret": {
|
||||
"type": "string",
|
||||
"description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"googleCloudSecret"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
],
|
||||
"description": "A Personal Access Token (PAT)."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -504,7 +616,6 @@ const schema = {
|
|||
"description": "GitLab Configuration"
|
||||
},
|
||||
"token": {
|
||||
"description": "An authentication token.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -532,7 +643,8 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "An authentication token."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -706,7 +818,6 @@ const schema = {
|
|||
"description": "Gitea Configuration"
|
||||
},
|
||||
"token": {
|
||||
"description": "A Personal Access Token (PAT).",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -734,7 +845,8 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "A Personal Access Token (PAT)."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -973,7 +1085,6 @@ const schema = {
|
|||
"description": "The username to use for authentication. Only needed if token is an app password."
|
||||
},
|
||||
"token": {
|
||||
"description": "An authentication token.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1001,7 +1112,8 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "An authentication token."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -1141,7 +1253,6 @@ const schema = {
|
|||
"description": "Azure DevOps Configuration"
|
||||
},
|
||||
"token": {
|
||||
"description": "A Personal Access Token (PAT).",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1169,7 +1280,8 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "A Personal Access Token (PAT)."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
|
|
@ -1425,7 +1537,6 @@ const schema = {
|
|||
"description": "Optional display name."
|
||||
},
|
||||
"accessKeyId": {
|
||||
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1453,10 +1564,10 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable."
|
||||
},
|
||||
"accessKeySecret": {
|
||||
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1484,10 +1595,10 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable."
|
||||
},
|
||||
"sessionToken": {
|
||||
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -1515,7 +1626,8 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable."
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
|
|
@ -2854,7 +2966,6 @@ const schema = {
|
|||
"description": "Optional display name."
|
||||
},
|
||||
"accessKeyId": {
|
||||
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -2882,10 +2993,10 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional access key ID to use with the model. Defaults to the `AWS_ACCESS_KEY_ID` environment variable."
|
||||
},
|
||||
"accessKeySecret": {
|
||||
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -2913,10 +3024,10 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional secret access key to use with the model. Defaults to the `AWS_SECRET_ACCESS_KEY` environment variable."
|
||||
},
|
||||
"sessionToken": {
|
||||
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable.",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
|
|
@ -2944,7 +3055,8 @@ const schema = {
|
|||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
],
|
||||
"description": "Optional session token to use with the model. Defaults to the `AWS_SESSION_TOKEN` environment variable."
|
||||
},
|
||||
"region": {
|
||||
"type": "string",
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ export interface SourcebotConfig {
|
|||
contexts?: {
|
||||
[k: string]: SearchContext;
|
||||
};
|
||||
environmentOverrides?: EnvironmentOverrides;
|
||||
/**
|
||||
* Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.
|
||||
*/
|
||||
|
|
@ -159,6 +160,44 @@ export interface SearchContext {
|
|||
*/
|
||||
description?: string;
|
||||
}
|
||||
/**
|
||||
* Environment variable overrides.
|
||||
*/
|
||||
export interface EnvironmentOverrides {
|
||||
/**
|
||||
* This interface was referenced by `EnvironmentOverrides`'s JSON-Schema definition
|
||||
* via the `patternProperty` "^[a-zA-Z0-9_-]+$".
|
||||
*/
|
||||
[k: string]:
|
||||
| {
|
||||
type: "token";
|
||||
value:
|
||||
| {
|
||||
/**
|
||||
* The name of the environment variable that contains the token.
|
||||
*/
|
||||
env: string;
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets
|
||||
*/
|
||||
googleCloudSecret: string;
|
||||
};
|
||||
}
|
||||
| {
|
||||
type: "string";
|
||||
value: string;
|
||||
}
|
||||
| {
|
||||
type: "number";
|
||||
value: number;
|
||||
}
|
||||
| {
|
||||
type: "boolean";
|
||||
value: boolean;
|
||||
};
|
||||
}
|
||||
export interface GithubConnectionConfig {
|
||||
/**
|
||||
* GitHub Configuration
|
||||
|
|
|
|||
|
|
@ -6,23 +6,28 @@
|
|||
"scripts": {
|
||||
"build": "tsc",
|
||||
"build:watch": "tsc-watch --preserveWatchOutput",
|
||||
"postinstall": "yarn build"
|
||||
"postinstall": "yarn build",
|
||||
"tool:resolve-env-overrides": "tsx tools/resolveEnvOverrides.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@sourcebot/crypto": "workspace:*",
|
||||
"@google-cloud/secret-manager": "^6.1.1",
|
||||
"@logtail/node": "^0.5.2",
|
||||
"@logtail/winston": "^0.5.2",
|
||||
"@sourcebot/db": "workspace:*",
|
||||
"@sourcebot/logger": "workspace:*",
|
||||
"@sourcebot/schemas": "workspace:*",
|
||||
"@t3-oss/env-core": "^0.12.0",
|
||||
"ajv": "^8.17.1",
|
||||
"micromatch": "^4.0.8",
|
||||
"strip-json-comments": "^5.0.1",
|
||||
"zod": "^3.24.3"
|
||||
"triple-beam": "^1.4.1",
|
||||
"winston": "^3.15.0",
|
||||
"zod": "^3.25.74"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/micromatch": "^4.0.9",
|
||||
"@types/node": "^22.7.5",
|
||||
"tsc-watch": "6.2.1",
|
||||
"tsx": "^4.19.1",
|
||||
"typescript": "^5.7.3"
|
||||
},
|
||||
"exports": {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import crypto from 'crypto';
|
||||
import fs from 'fs';
|
||||
import { SOURCEBOT_ENCRYPTION_KEY } from './environment';
|
||||
import { env } from './env.server.js';
|
||||
import { Token } from '@sourcebot/schemas/v3/shared.type';
|
||||
import { SecretManagerServiceClient } from "@google-cloud/secret-manager";
|
||||
|
||||
const algorithm = 'aes-256-cbc';
|
||||
const ivLength = 16; // 16 bytes for CBC
|
||||
|
|
@ -12,11 +14,7 @@ const generateIV = (): Buffer => {
|
|||
};
|
||||
|
||||
export function encrypt(text: string): { iv: string; encryptedData: string } {
|
||||
if (!SOURCEBOT_ENCRYPTION_KEY) {
|
||||
throw new Error('Encryption key is not set');
|
||||
}
|
||||
|
||||
const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'ascii');
|
||||
const encryptionKey = Buffer.from(env.SOURCEBOT_ENCRYPTION_KEY, 'ascii');
|
||||
|
||||
const iv = generateIV();
|
||||
const cipher = crypto.createCipheriv(algorithm, encryptionKey, iv);
|
||||
|
|
@ -28,18 +26,10 @@ export function encrypt(text: string): { iv: string; encryptedData: string } {
|
|||
}
|
||||
|
||||
export function hashSecret(text: string): string {
|
||||
if (!SOURCEBOT_ENCRYPTION_KEY) {
|
||||
throw new Error('Encryption key is not set');
|
||||
}
|
||||
|
||||
return crypto.createHmac('sha256', SOURCEBOT_ENCRYPTION_KEY).update(text).digest('hex');
|
||||
return crypto.createHmac('sha256', env.SOURCEBOT_ENCRYPTION_KEY).update(text).digest('hex');
|
||||
}
|
||||
|
||||
export function generateApiKey(): { key: string; hash: string } {
|
||||
if (!SOURCEBOT_ENCRYPTION_KEY) {
|
||||
throw new Error('Encryption key is not set');
|
||||
}
|
||||
|
||||
const secret = crypto.randomBytes(32).toString('hex');
|
||||
const hash = hashSecret(secret);
|
||||
|
||||
|
|
@ -50,11 +40,7 @@ export function generateApiKey(): { key: string; hash: string } {
|
|||
}
|
||||
|
||||
export function decrypt(iv: string, encryptedText: string): string {
|
||||
if (!SOURCEBOT_ENCRYPTION_KEY) {
|
||||
throw new Error('Encryption key is not set');
|
||||
}
|
||||
|
||||
const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'ascii');
|
||||
const encryptionKey = Buffer.from(env.SOURCEBOT_ENCRYPTION_KEY, 'ascii');
|
||||
|
||||
const ivBuffer = Buffer.from(iv, 'hex');
|
||||
const encryptedBuffer = Buffer.from(encryptedText, 'hex');
|
||||
|
|
@ -92,4 +78,30 @@ export function verifySignature(data: string, signature: string, publicKeyPath:
|
|||
}
|
||||
}
|
||||
|
||||
export { getTokenFromConfig } from './tokenUtils.js';
|
||||
export const getTokenFromConfig = async (token: Token): Promise<string> => {
|
||||
if ('env' in token) {
|
||||
const envToken = process.env[token.env];
|
||||
if (!envToken) {
|
||||
throw new Error(`Environment variable ${token.env} not found.`);
|
||||
}
|
||||
|
||||
return envToken;
|
||||
} else if ('googleCloudSecret' in token) {
|
||||
try {
|
||||
const client = new SecretManagerServiceClient();
|
||||
const [response] = await client.accessSecretVersion({
|
||||
name: token.googleCloudSecret,
|
||||
});
|
||||
|
||||
if (!response.payload?.data) {
|
||||
throw new Error(`Secret ${token.googleCloudSecret} not found.`);
|
||||
}
|
||||
|
||||
return response.payload.data.toString();
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to access Google Cloud secret ${token.googleCloudSecret}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
} else {
|
||||
throw new Error('Invalid token configuration');
|
||||
}
|
||||
};
|
||||
15
packages/shared/src/db.ts
Normal file
15
packages/shared/src/db.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import { env } from "./env.server.js";
|
||||
|
||||
export const getDBConnectionString = (): string | undefined => {
|
||||
if (env.DATABASE_URL) {
|
||||
return env.DATABASE_URL;
|
||||
}
|
||||
else if (env.DATABASE_HOST && env.DATABASE_USERNAME && env.DATABASE_PASSWORD && env.DATABASE_NAME) {
|
||||
let databaseUrl = `postgresql://${env.DATABASE_USERNAME}:${env.DATABASE_PASSWORD}@${env.DATABASE_HOST}/${env.DATABASE_NAME}`;
|
||||
if (env.DATABASE_ARGS) {
|
||||
databaseUrl += `?${env.DATABASE_ARGS}`;
|
||||
}
|
||||
|
||||
return databaseUrl;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +1,10 @@
|
|||
import { base64Decode } from "./utils.js";
|
||||
import { z } from "zod";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { verifySignature } from "@sourcebot/crypto";
|
||||
import { env } from "./env.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
import { env } from "./env.server.js";
|
||||
import { env as clientEnv } from "./env.client.js";
|
||||
import { SOURCEBOT_SUPPORT_EMAIL, SOURCEBOT_UNLIMITED_SEATS } from "./constants.js";
|
||||
import { verifySignature } from "./crypto.js";
|
||||
|
||||
const logger = createLogger('entitlements');
|
||||
|
||||
|
|
@ -89,8 +90,8 @@ export const getLicenseKey = (): LicenseKeyPayload | null => {
|
|||
}
|
||||
|
||||
export const getPlan = (): Plan => {
|
||||
if (env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT) {
|
||||
if (env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo") {
|
||||
if (clientEnv.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT) {
|
||||
if (clientEnv.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo") {
|
||||
return "cloud:demo";
|
||||
}
|
||||
|
||||
|
|
|
|||
27
packages/shared/src/env.client.ts
Normal file
27
packages/shared/src/env.client.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { createEnv } from "@t3-oss/env-core";
|
||||
import { z } from "zod";
|
||||
import { SOURCEBOT_CLOUD_ENVIRONMENT } from "./constants.js";
|
||||
|
||||
export const env = createEnv({
|
||||
clientPrefix: "NEXT_PUBLIC_",
|
||||
client: {
|
||||
NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(),
|
||||
NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"),
|
||||
NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(),
|
||||
NEXT_PUBLIC_SENTRY_BACKEND_DSN: z.string().optional(),
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT: z.string().optional(),
|
||||
NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: z.string().optional(),
|
||||
NEXT_PUBLIC_LANGFUSE_BASE_URL: z.string().optional()
|
||||
},
|
||||
runtimeEnvStrict: {
|
||||
NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT,
|
||||
NEXT_PUBLIC_SOURCEBOT_VERSION: process.env.NEXT_PUBLIC_SOURCEBOT_VERSION,
|
||||
NEXT_PUBLIC_POSTHOG_PAPIK: process.env.NEXT_PUBLIC_POSTHOG_PAPIK,
|
||||
NEXT_PUBLIC_SENTRY_BACKEND_DSN: process.env.NEXT_PUBLIC_SENTRY_BACKEND_DSN,
|
||||
NEXT_PUBLIC_SENTRY_ENVIRONMENT: process.env.NEXT_PUBLIC_SENTRY_ENVIRONMENT,
|
||||
NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY,
|
||||
NEXT_PUBLIC_LANGFUSE_BASE_URL: process.env.NEXT_PUBLIC_LANGFUSE_BASE_URL,
|
||||
},
|
||||
emptyStringAsUndefined: true,
|
||||
skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
|
||||
});
|
||||
|
|
@ -1,16 +1,65 @@
|
|||
import { createEnv } from "@t3-oss/env-nextjs";
|
||||
import { createEnv } from "@t3-oss/env-core";
|
||||
import { z } from "zod";
|
||||
import { SOURCEBOT_CLOUD_ENVIRONMENT } from "@sourcebot/shared/client";
|
||||
import { loadConfig } from "./utils.js";
|
||||
import { tenancyModeSchema } from "./types.js";
|
||||
import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
|
||||
import { getTokenFromConfig } from "./crypto.js";
|
||||
|
||||
// Booleans are specified as 'true' or 'false' strings.
|
||||
const booleanSchema = z.enum(["true", "false"]);
|
||||
export const tenancyModeSchema = z.enum(["multi", "single"]);
|
||||
|
||||
// Numbers are treated as strings in .env files.
|
||||
// coerce helps us convert them to numbers.
|
||||
// @see: https://zod.dev/?id=coercion-for-primitives
|
||||
const numberSchema = z.coerce.number();
|
||||
|
||||
export const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotConfig): Promise<Record<string, string>> => {
|
||||
if (!config.environmentOverrides) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const resolved: Record<string, string> = {};
|
||||
|
||||
const start = performance.now();
|
||||
|
||||
for (const [key, override] of Object.entries(config.environmentOverrides)) {
|
||||
switch (override.type) {
|
||||
case 'token':
|
||||
resolved[key] = await getTokenFromConfig(override.value);
|
||||
break;
|
||||
case 'boolean':
|
||||
resolved[key] = override.value ? 'true' : 'false';
|
||||
break;
|
||||
case 'number':
|
||||
resolved[key] = override.value.toString();
|
||||
break;
|
||||
case 'string':
|
||||
resolved[key] = override.value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const end = performance.now();
|
||||
console.debug(`resolved environment variable overrides in ${end - start}ms`);
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
// Merge process.env with environment variables resolved from config.json
|
||||
const runtimeEnv = await (async () => {
|
||||
const configPath = process.env.CONFIG_PATH;
|
||||
if (!configPath) {
|
||||
return process.env;
|
||||
}
|
||||
|
||||
const config = await loadConfig(configPath);
|
||||
const overrides = await resolveEnvironmentVariableOverridesFromConfig(config);
|
||||
return {
|
||||
...process.env,
|
||||
...overrides,
|
||||
}
|
||||
})();
|
||||
|
||||
export const env = createEnv({
|
||||
server: {
|
||||
// Zoekt
|
||||
|
|
@ -18,7 +67,6 @@ export const env = createEnv({
|
|||
|
||||
// Auth
|
||||
FORCE_ENABLE_ANONYMOUS_ACCESS: booleanSchema.default('false'),
|
||||
|
||||
AUTH_SECRET: z.string(),
|
||||
AUTH_URL: z.string().url(),
|
||||
AUTH_CREDENTIALS_LOGIN_ENABLED: booleanSchema.default('true'),
|
||||
|
|
@ -72,10 +120,19 @@ export const env = createEnv({
|
|||
CONFIG_MAX_REPOS_NO_TOKEN: numberSchema.default(Number.MAX_SAFE_INTEGER),
|
||||
NODE_ENV: z.enum(["development", "test", "production"]),
|
||||
SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default('false'),
|
||||
DATABASE_URL: z.string().url(),
|
||||
|
||||
// Database variables
|
||||
// Either DATABASE_URL or DATABASE_HOST, DATABASE_USERNAME, DATABASE_PASSWORD, and DATABASE_NAME must be set.
|
||||
// @see: shared/src/db.ts for more details.
|
||||
DATABASE_URL: z.string().url().optional(),
|
||||
DATABASE_HOST: z.string().optional(),
|
||||
DATABASE_USERNAME: z.string().optional(),
|
||||
DATABASE_PASSWORD: z.string().optional(),
|
||||
DATABASE_NAME: z.string().optional(),
|
||||
DATABASE_ARGS: z.string().optional(),
|
||||
|
||||
SOURCEBOT_TENANCY_MODE: tenancyModeSchema.default("single"),
|
||||
CONFIG_PATH: z.string().optional(),
|
||||
CONFIG_PATH: z.string(),
|
||||
|
||||
// Misc UI flags
|
||||
SECURITY_CARD_ENABLED: booleanSchema.default('false'),
|
||||
|
|
@ -137,31 +194,30 @@ export const env = createEnv({
|
|||
// @NOTE: Take care to update actions.ts when changing the name of this.
|
||||
EXPERIMENT_SELF_SERVE_REPO_INDEXING_GITHUB_TOKEN: z.string().optional(),
|
||||
EXPERIMENT_EE_PERMISSION_SYNC_ENABLED: booleanSchema.default('false'),
|
||||
},
|
||||
// @NOTE: Please make sure of the following:
|
||||
// - Make sure you destructure all client variables in
|
||||
// the `experimental__runtimeEnv` block below.
|
||||
// - Update the Dockerfile to pass these variables as build-args.
|
||||
client: {
|
||||
// PostHog
|
||||
NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(),
|
||||
|
||||
// Misc
|
||||
NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default('unknown'),
|
||||
SOURCEBOT_ENCRYPTION_KEY: z.string(),
|
||||
SOURCEBOT_INSTALL_ID: z.string().default("unknown"),
|
||||
|
||||
NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(),
|
||||
FALLBACK_GITHUB_CLOUD_TOKEN: z.string().optional(),
|
||||
FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(),
|
||||
FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(),
|
||||
|
||||
NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: z.string().optional(),
|
||||
NEXT_PUBLIC_LANGFUSE_BASE_URL: z.string().optional()
|
||||
REDIS_URL: z.string().url().default("redis://localhost:6379"),
|
||||
REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0),
|
||||
REDIS_REMOVE_ON_FAIL: numberSchema.default(100),
|
||||
|
||||
DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'),
|
||||
|
||||
CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000),
|
||||
REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60),
|
||||
|
||||
GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10),
|
||||
|
||||
SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"),
|
||||
SOURCEBOT_STRUCTURED_LOGGING_ENABLED: booleanSchema.default("false"),
|
||||
SOURCEBOT_STRUCTURED_LOGGING_FILE: z.string().optional(),
|
||||
},
|
||||
// For Next.js >= 13.4.4, you only need to destructure client variables:
|
||||
experimental__runtimeEnv: {
|
||||
NEXT_PUBLIC_POSTHOG_PAPIK: process.env.NEXT_PUBLIC_POSTHOG_PAPIK,
|
||||
NEXT_PUBLIC_SOURCEBOT_VERSION: process.env.NEXT_PUBLIC_SOURCEBOT_VERSION,
|
||||
NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT,
|
||||
NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY,
|
||||
NEXT_PUBLIC_LANGFUSE_BASE_URL: process.env.NEXT_PUBLIC_LANGFUSE_BASE_URL,
|
||||
},
|
||||
skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
|
||||
runtimeEnv,
|
||||
emptyStringAsUndefined: true,
|
||||
skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
|
||||
});
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
import { createEnv } from "@t3-oss/env-core";
|
||||
import { z } from "zod";
|
||||
import { SOURCEBOT_CLOUD_ENVIRONMENT } from "./constants.js";
|
||||
|
||||
export const env = createEnv({
|
||||
server: {
|
||||
SOURCEBOT_EE_LICENSE_KEY: z.string().optional(),
|
||||
SOURCEBOT_PUBLIC_KEY_PATH: z.string(),
|
||||
},
|
||||
client: {
|
||||
NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(),
|
||||
},
|
||||
clientPrefix: "NEXT_PUBLIC_",
|
||||
runtimeEnvStrict: {
|
||||
SOURCEBOT_EE_LICENSE_KEY: process.env.SOURCEBOT_EE_LICENSE_KEY,
|
||||
SOURCEBOT_PUBLIC_KEY_PATH: process.env.SOURCEBOT_PUBLIC_KEY_PATH,
|
||||
NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT,
|
||||
},
|
||||
emptyStringAsUndefined: true,
|
||||
skipValidation: process.env.SKIP_ENV_VALIDATION === "1",
|
||||
});
|
||||
|
|
@ -1,2 +1,4 @@
|
|||
|
||||
export * from "./constants.js";
|
||||
export {
|
||||
env
|
||||
} from "./env.client.js";
|
||||
|
|
@ -16,6 +16,7 @@ export type {
|
|||
export {
|
||||
repoMetadataSchema,
|
||||
repoIndexingJobMetadataSchema,
|
||||
tenancyModeSchema,
|
||||
} from "./types.js";
|
||||
export {
|
||||
base64Decode,
|
||||
|
|
@ -25,3 +26,24 @@ export {
|
|||
getConfigSettings,
|
||||
} from "./utils.js";
|
||||
export * from "./constants.js";
|
||||
export {
|
||||
env,
|
||||
resolveEnvironmentVariableOverridesFromConfig,
|
||||
} from "./env.server.js";
|
||||
export {
|
||||
createLogger,
|
||||
} from "./logger.js";
|
||||
export type {
|
||||
Logger,
|
||||
} from "./logger.js";
|
||||
export {
|
||||
getTokenFromConfig,
|
||||
encrypt,
|
||||
decrypt,
|
||||
hashSecret,
|
||||
generateApiKey,
|
||||
verifySignature,
|
||||
} from "./crypto.js";
|
||||
export {
|
||||
getDBConnectionString,
|
||||
} from "./db.js";
|
||||
|
|
@ -2,7 +2,7 @@ import winston, { format, Logger } from 'winston';
|
|||
import { Logtail } from '@logtail/node';
|
||||
import { LogtailTransport } from '@logtail/winston';
|
||||
import { MESSAGE } from 'triple-beam';
|
||||
import { env } from './env.js';
|
||||
import { env } from './env.server.js';
|
||||
|
||||
/**
|
||||
* Logger configuration with support for structured JSON logging.
|
||||
|
|
@ -16,7 +16,7 @@ import { env } from './env.js';
|
|||
* - Logs will be formatted as: "timestamp level: [label] message"
|
||||
*/
|
||||
|
||||
const { combine, colorize, timestamp, prettyPrint, errors, printf, label: labelFn, json } = format;
|
||||
const { combine, colorize, timestamp, errors, printf, label: labelFn, json } = format;
|
||||
|
||||
const datadogFormat = format((info) => {
|
||||
info.status = info.level.toLowerCase();
|
||||
|
|
@ -43,3 +43,5 @@ export const repoIndexingJobMetadataSchema = z.object({
|
|||
});
|
||||
|
||||
export type RepoIndexingJobMetadata = z.infer<typeof repoIndexingJobMetadataSchema>;
|
||||
|
||||
export const tenancyModeSchema = z.enum(["multi", "single"]);
|
||||
30
packages/shared/tools/resolveEnvOverrides.ts
Normal file
30
packages/shared/tools/resolveEnvOverrides.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
// The following script loads the config.json file and resolves any environment variable overrides.
|
||||
// It then writes then to stdout in the format of `KEY="VALUE"`.
|
||||
// This is used by entrypoint.sh to set them as variables.
|
||||
(async () => {
|
||||
if (!process.env.CONFIG_PATH) {
|
||||
console.error('CONFIG_PATH is not set');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Silence all console logs so we don't pollute stdout.
|
||||
const originalConsoleLog = console.log;
|
||||
console.log = () => {};
|
||||
console.debug = () => {};
|
||||
console.info = () => {};
|
||||
console.warn = () => {};
|
||||
// console.error = () => {}; // Keep errors
|
||||
|
||||
const { loadConfig } = await import("../src/utils.js");
|
||||
const { resolveEnvironmentVariableOverridesFromConfig } = await import("../src/env.server.js");
|
||||
|
||||
const config = await loadConfig(process.env.CONFIG_PATH);
|
||||
const overrides = await resolveEnvironmentVariableOverridesFromConfig(config);
|
||||
|
||||
for (const [key, value] of Object.entries(overrides)) {
|
||||
const escapedValue = value.replace(/"/g, '\\"');
|
||||
originalConsoleLog(`${key}="${escapedValue}"`);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
})();
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
await import("./src/env.mjs");
|
||||
import { withSentryConfig } from "@sentry/nextjs";
|
||||
|
||||
|
||||
|
|
@ -8,7 +7,7 @@ const nextConfig = {
|
|||
|
||||
// This is required when using standalone builds.
|
||||
// @see: https://env.t3.gg/docs/nextjs#create-your-schema
|
||||
transpilePackages: ["@t3-oss/env-nextjs", "@t3-oss/env-core"],
|
||||
transpilePackages: ["@t3-oss/env-core"],
|
||||
|
||||
// @see : https://posthog.com/docs/advanced/proxy/nextjs
|
||||
async rewrites() {
|
||||
|
|
|
|||
|
|
@ -90,16 +90,12 @@
|
|||
"@sentry/nextjs": "^9",
|
||||
"@shopify/lang-jsonc": "^1.0.0",
|
||||
"@sourcebot/codemirror-lang-tcl": "^1.0.12",
|
||||
"@sourcebot/crypto": "workspace:*",
|
||||
"@sourcebot/db": "workspace:*",
|
||||
"@sourcebot/error": "workspace:*",
|
||||
"@sourcebot/logger": "workspace:*",
|
||||
"@sourcebot/schemas": "workspace:*",
|
||||
"@sourcebot/shared": "workspace:*",
|
||||
"@ssddanbrown/codemirror-lang-twig": "^1.0.0",
|
||||
"@stripe/react-stripe-js": "^3.1.1",
|
||||
"@stripe/stripe-js": "^5.6.0",
|
||||
"@t3-oss/env-nextjs": "^0.12.0",
|
||||
"@tailwindcss/typography": "^0.5.16",
|
||||
"@tanstack/react-query": "^5.53.3",
|
||||
"@tanstack/react-table": "^8.20.5",
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
// https://docs.sentry.io/platforms/javascript/guides/nextjs/
|
||||
|
||||
import * as Sentry from "@sentry/nextjs";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
|
||||
const logger = createLogger('sentry-server-config');
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
'use server';
|
||||
|
||||
import { getAuditService } from "@/ee/features/audit/factory";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { addUserToOrganization, orgHasAvailability } from "@/lib/authUtils";
|
||||
import { ErrorCode } from "@/lib/errorCodes";
|
||||
import { notAuthenticated, notFound, orgNotFound, ServiceError, ServiceErrorException, unexpectedError } from "@/lib/serviceError";
|
||||
|
|
@ -9,9 +9,9 @@ import { getOrgMetadata, isHttpError, isServiceError } from "@/lib/utils";
|
|||
import { prisma } from "@/prisma";
|
||||
import { render } from "@react-email/components";
|
||||
import * as Sentry from '@sentry/nextjs';
|
||||
import { generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto";
|
||||
import { generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/shared";
|
||||
import { ApiKey, ConnectionSyncJobStatus, Org, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType, StripeSubscriptionStatus } from "@sourcebot/db";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type";
|
||||
import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
|
||||
import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import Link from "next/link";
|
||||
import { NavigationMenu } from "../components/navigationMenu";
|
||||
import { FaCogs } from "react-icons/fa";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared";
|
||||
|
||||
const agents = [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { RepositoryCarousel } from "../components/repositoryCarousel";
|
|||
import { NavigationMenu } from "../components/navigationMenu";
|
||||
import { Separator } from "@/components/ui/separator";
|
||||
import { DemoCards } from "./components/demoCards";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { loadJsonFile } from "@sourcebot/shared";
|
||||
import { DemoExamples, demoExamplesSchema } from "@/types";
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { NavigationMenu as NavigationMenuBase } from "@/components/ui/navigation
|
|||
import { Separator } from "@/components/ui/separator";
|
||||
import { getSubscriptionInfo } from "@/ee/features/billing/actions";
|
||||
import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { ServiceErrorException } from "@/lib/serviceError";
|
||||
import { isServiceError } from "@/lib/utils";
|
||||
import { DiscordLogoIcon, GitHubLogoIcon } from "@radix-ui/react-icons";
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ import { useKeymapType } from "@/hooks/useKeymapType"
|
|||
import { useSession } from "next-auth/react";
|
||||
import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar";
|
||||
import { signOut } from "next-auth/react"
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared/client";
|
||||
import posthog from "posthog-js";
|
||||
import { useDomain } from "@/hooks/useDomain";
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import { getSubscriptionInfo } from "@/ee/features/billing/actions";
|
|||
import { PendingApprovalCard } from "./components/pendingApproval";
|
||||
import { SubmitJoinRequest } from "./components/submitJoinRequest";
|
||||
import { hasEntitlement } from "@sourcebot/shared";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { GcpIapAuth } from "./components/gcpIapAuth";
|
||||
import { getAnonymousAccessStatus, getMemberApprovalRequired } from "@/actions";
|
||||
import { JoinOrganizationCard } from "@/app/components/joinOrganizationCard";
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { Button } from "@/components/ui/button"
|
|||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"
|
||||
import { Skeleton } from "@/components/ui/skeleton"
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
|
||||
import { env } from "@/env.mjs"
|
||||
import { env } from "@sourcebot/shared"
|
||||
import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants"
|
||||
import { ServiceErrorException } from "@/lib/serviceError"
|
||||
import { cn, getCodeHostInfoForRepo, isServiceError } from "@/lib/utils"
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { DisplayDate } from "@/app/[domain]/components/DisplayDate";
|
|||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants";
|
||||
import { notFound, ServiceErrorException } from "@/lib/serviceError";
|
||||
import { isServiceError } from "@/lib/utils";
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ import { getConnectionStats, getMe, getOrgAccountRequests } from "@/actions";
|
|||
import { ServiceErrorException } from "@/lib/serviceError";
|
||||
import { getOrgFromDomain } from "@/data/org";
|
||||
import { OrgRole } from "@prisma/client";
|
||||
import { env } from "@/env.mjs";
|
||||
import { hasEntitlement } from "@sourcebot/shared";
|
||||
import { env } from "@sourcebot/shared/client";
|
||||
|
||||
interface LayoutProps {
|
||||
children: React.ReactNode;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { Info, Mail } from "lucide-react";
|
|||
import { getOrgMembers } from "@/actions";
|
||||
import { isServiceError } from "@/lib/utils";
|
||||
import { notFound, ServiceErrorException } from "@/lib/serviceError";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared/client";
|
||||
|
||||
interface LicensePageProps {
|
||||
params: Promise<{
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import { isServiceError } from "@/lib/utils";
|
|||
import Link from "next/link";
|
||||
import { ArrowLeftIcon } from "@radix-ui/react-icons";
|
||||
import { LogoutEscapeHatch } from "@/app/components/logoutEscapeHatch";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe";
|
||||
import { getSubscriptionInfo } from "@/ee/features/billing/actions";
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { prisma } from "@/prisma";
|
|||
import { LanguageModelV2 as AISDKLanguageModelV2 } from "@ai-sdk/provider";
|
||||
import * as Sentry from "@sentry/nextjs";
|
||||
import { OrgRole } from "@sourcebot/db";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import {
|
||||
createUIMessageStream,
|
||||
createUIMessageStreamResponse,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { isServiceError } from "@/lib/utils";
|
|||
import { serviceErrorResponse } from "@/lib/serviceError";
|
||||
import { StatusCodes } from "http-status-codes";
|
||||
import { ErrorCode } from "@/lib/errorCodes";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { getEntitlements } from "@sourcebot/shared";
|
||||
|
||||
export const GET = async (request: NextRequest) => {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2";
|
|||
import { OrgRole } from "@sourcebot/db";
|
||||
import { isServiceError } from "@/lib/utils";
|
||||
import { serviceErrorResponse, missingQueryParam, notFound } from "@/lib/serviceError";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { NextRequest } from "next/server";
|
||||
import { StatusCodes } from "http-status-codes";
|
||||
import { ErrorCode } from "@/lib/errorCodes";
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2";
|
|||
import { OrgRole } from "@sourcebot/db";
|
||||
import { isServiceError } from "@/lib/utils";
|
||||
import { serviceErrorResponse } from "@/lib/serviceError";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
import { getAuditService } from "@/ee/features/audit/factory";
|
||||
|
||||
const logger = createLogger('ee-users-api');
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
'use server';
|
||||
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
|
||||
const logger = createLogger('health-check');
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ import Stripe from 'stripe';
|
|||
import { prisma } from '@/prisma';
|
||||
import { StripeSubscriptionStatus } from '@sourcebot/db';
|
||||
import { stripeClient } from '@/ee/features/billing/stripe';
|
||||
import { env } from '@/env.mjs';
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { env } from '@sourcebot/shared';
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
|
||||
const logger = createLogger('stripe-webhook');
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared/client";
|
||||
import { GetVersionResponse } from "@/lib/types";
|
||||
|
||||
// Note: In Next.JS 14, GET methods with no params are cached by default at build time.
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@ import { NextRequest } from "next/server";
|
|||
import { App, Octokit } from "octokit";
|
||||
import { WebhookEventDefinition} from "@octokit/webhooks/types";
|
||||
import { EndpointDefaults } from "@octokit/types";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared";
|
||||
import { processGitHubPullRequest } from "@/features/agents/review-agent/app";
|
||||
import { throttling } from "@octokit/plugin-throttling";
|
||||
import fs from "fs";
|
||||
import { GitHubPullRequest } from "@/features/agents/review-agent/types";
|
||||
import { createLogger } from "@sourcebot/logger";
|
||||
import { createLogger } from "@sourcebot/shared";
|
||||
|
||||
const logger = createLogger('github-webhook');
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
'use client';
|
||||
|
||||
import React, { useState, useEffect } from "react";
|
||||
import { env } from "@/env.mjs";
|
||||
import { env } from "@sourcebot/shared/client";
|
||||
|
||||
interface AuthSecurityNoticeProps {
|
||||
closable?: boolean;
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue