diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a06cef1..81304f2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Text highlighting clarity. [#342](https://github.com/sourcebot-dev/sourcebot/pull/342) +- Clean up successful and failed jobs in Redis queues. [#343](https://github.com/sourcebot-dev/sourcebot/pull/343) ## [4.2.0] - 2025-06-09 diff --git a/docs/docs/configuration/environment-variables.mdx b/docs/docs/configuration/environment-variables.mdx index d868619e..88e12bce 100644 --- a/docs/docs/configuration/environment-variables.mdx +++ b/docs/docs/configuration/environment-variables.mdx @@ -23,6 +23,8 @@ The following environment variables allow you to configure your Sourcebot deploy | `EMAIL_FROM_ADDRESS` | `-` |
The email address that transactional emails will be sent from. See [this doc](/docs/configuration/transactional-emails) for more info.
| | `REDIS_DATA_DIR` | `$DATA_CACHE_DIR/redis` |The data directory for the default Redis instance.
| | `REDIS_URL` | `redis://localhost:6379` |Connection string of your Redis instance. By default, a Redis database is automatically provisioned at startup within the container.
| +| `REDIS_REMOVE_ON_COMPLETE` | `0` |Controls how many completed jobs are allowed to remain in Redis queues
| +| `REDIS_REMOVE_ON_FAIL` | `100` |Controls how many failed jobs are allowed to remain in Redis queues
| | `SHARD_MAX_MATCH_COUNT` | `10000` |The maximum shard count per query
| | `SMTP_CONNECTION_URL` | `-` |The url to the SMTP service used for sending transactional emails. See [this doc](/docs/configuration/transactional-emails) for more info.
| | `SOURCEBOT_ENCRYPTION_KEY` | Automatically generated at startup if no value is provided. Generated using `openssl rand -base64 24` |Used to encrypt connection secrets and generate API keys.
| diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index ac31826c..3ce88f49 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -64,6 +64,9 @@ export class ConnectionManager implements IConnectionManager { connectionName: connection.name, orgId: connection.orgId, config: connectionConfig, + }, { + removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, + removeOnFail: env.REDIS_REMOVE_ON_FAIL, }); this.logger.info(`Added job to queue for connection ${connection.name} (id: ${connection.id})`); }).catch((err: unknown) => { diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 40a6a1cc..d8beba19 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -35,6 +35,8 @@ export const env = createEnv({ FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(), REDIS_URL: z.string().url().default("redis://localhost:6379"), + REDIS_REMOVE_ON_COMPLETE: numberSchema.default(0), + REDIS_REMOVE_ON_FAIL: numberSchema.default(100), NEXT_PUBLIC_SENTRY_BACKEND_DSN: z.string().optional(), NEXT_PUBLIC_SENTRY_ENVIRONMENT: z.string().optional(), diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts index 8f16a9d0..50dc0478 100644 --- a/packages/backend/src/repoManager.ts +++ b/packages/backend/src/repoManager.ts @@ -10,6 +10,7 @@ import { existsSync, readdirSync, promises } from 'fs'; import { indexGitRepository } from "./zoekt.js"; import { PromClient } from './promClient.js'; import * as Sentry from "@sentry/node"; +import { env } from './env.js'; interface IRepoManager { validateIndexedReposHaveShards: () => Promise