diff --git a/CHANGELOG.md b/CHANGELOG.md index c416a295..3c8bc43e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added config option `settings.reindexInterval` and `settings.resyncInterval` to control how often the index should be re-indexed and re-synced. ([#134](https://github.com/sourcebot-dev/sourcebot/pull/134)) + ## [2.6.2] - 2024-12-13 ### Added diff --git a/demo-site-config.json b/demo-site-config.json index 2b3a47fc..e6aa8572 100644 --- a/demo-site-config.json +++ b/demo-site-config.json @@ -1,7 +1,9 @@ { "$schema": "./schemas/v2/index.json", "settings": { - "autoDeleteStaleRepos": true + "autoDeleteStaleRepos": true, + "reindexInterval": 86400000, // 24 hours + "resyncInterval": 86400000 // 24 hours }, "repos": [ { diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts index 01a2ffc5..94b8c764 100644 --- a/packages/backend/src/constants.ts +++ b/packages/backend/src/constants.ts @@ -1,19 +1,11 @@ import { Settings } from "./types.js"; -/** - * The interval to reindex a given repository. - */ -export const REINDEX_INTERVAL_MS = 1000 * 60 * 60; - -/** - * The interval to re-sync the config. - */ -export const RESYNC_CONFIG_INTERVAL_MS = 1000 * 60 * 60 * 24; - /** * Default settings. */ export const DEFAULT_SETTINGS: Settings = { maxFileSize: 2 * 1024 * 1024, // 2MB in bytes autoDeleteStaleRepos: true, + reindexInterval: 1000 * 60 * 60, // 1 hour in milliseconds + resyncInterval: 1000 * 60 * 60 * 24, // 1 day in milliseconds } \ No newline at end of file diff --git a/packages/backend/src/db.test.ts b/packages/backend/src/db.test.ts index ed5d9391..53b73e9a 100644 --- a/packages/backend/src/db.test.ts +++ b/packages/backend/src/db.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'vitest'; -import { DEFAULT_DB_DATA, migration_addDeleteStaleRepos, migration_addMaxFileSize, migration_addSettings, Schema } from './db'; +import { DEFAULT_DB_DATA, migration_addDeleteStaleRepos, migration_addMaxFileSize, migration_addReindexInterval, migration_addResyncInterval, migration_addSettings, Schema } from './db'; import { DEFAULT_SETTINGS } from './constants'; import { DeepPartial } from './types'; import { Low } from 'lowdb'; @@ -60,4 +60,66 @@ test('migration_addDeleteStaleRepos adds the `autoDeleteStaleRepos` field with t autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos, } }); -}); \ No newline at end of file +}); + +test('migration_addReindexInterval adds the `reindexInterval` field with the default value if it does not exist', () => { + const schema: DeepPartial = { + settings: { + maxFileSize: DEFAULT_SETTINGS.maxFileSize, + autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos, + }, + } + + const migratedSchema = migration_addReindexInterval(schema as Schema); + expect(migratedSchema).toStrictEqual({ + settings: { + maxFileSize: DEFAULT_SETTINGS.maxFileSize, + autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos, + reindexInterval: DEFAULT_SETTINGS.reindexInterval, + } + }); +}); + +test('migration_addReindexInterval preserves existing reindexInterval value if already set', () => { + const customInterval = 60; + const schema: DeepPartial = { + settings: { + maxFileSize: DEFAULT_SETTINGS.maxFileSize, + reindexInterval: customInterval, + }, + } + + const migratedSchema = migration_addReindexInterval(schema as Schema); + expect(migratedSchema.settings.reindexInterval).toBe(customInterval); +}); + +test('migration_addResyncInterval adds the `resyncInterval` field with the default value if it does not exist', () => { + const schema: DeepPartial = { + settings: { + maxFileSize: DEFAULT_SETTINGS.maxFileSize, + autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos, + }, + } + + const migratedSchema = migration_addResyncInterval(schema as Schema); + expect(migratedSchema).toStrictEqual({ + settings: { + maxFileSize: DEFAULT_SETTINGS.maxFileSize, + autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos, + resyncInterval: DEFAULT_SETTINGS.resyncInterval, + } + }); +}); + +test('migration_addResyncInterval preserves existing resyncInterval value if already set', () => { + const customInterval = 120; + const schema: DeepPartial = { + settings: { + maxFileSize: DEFAULT_SETTINGS.maxFileSize, + resyncInterval: customInterval, + }, + } + + const migratedSchema = migration_addResyncInterval(schema as Schema); + expect(migratedSchema.settings.resyncInterval).toBe(customInterval); +}); diff --git a/packages/backend/src/db.ts b/packages/backend/src/db.ts index 3eed377b..c7f3e778 100644 --- a/packages/backend/src/db.ts +++ b/packages/backend/src/db.ts @@ -56,6 +56,8 @@ export const applyMigrations = async (db: Database) => { schema = migration_addSettings(schema, log); schema = migration_addMaxFileSize(schema, log); schema = migration_addDeleteStaleRepos(schema, log); + schema = migration_addReindexInterval(schema, log); + schema = migration_addResyncInterval(schema, log); return schema; }); } @@ -89,9 +91,33 @@ export const migration_addMaxFileSize = (schema: Schema, log?: (name: string) => */ export const migration_addDeleteStaleRepos = (schema: Schema, log?: (name: string) => void) => { if (schema.settings.autoDeleteStaleRepos === undefined) { - log?.("deleteStaleRepos"); + log?.("addDeleteStaleRepos"); schema.settings.autoDeleteStaleRepos = DEFAULT_SETTINGS.autoDeleteStaleRepos; } + return schema; +} + +/** + * @see: https://github.com/sourcebot-dev/sourcebot/pull/134 + */ +export const migration_addReindexInterval = (schema: Schema, log?: (name: string) => void) => { + if (schema.settings.reindexInterval === undefined) { + log?.("addReindexInterval"); + schema.settings.reindexInterval = DEFAULT_SETTINGS.reindexInterval; + } + + return schema; +} + +/** + * @see: https://github.com/sourcebot-dev/sourcebot/pull/134 + */ +export const migration_addResyncInterval = (schema: Schema, log?: (name: string) => void) => { + if (schema.settings.resyncInterval === undefined) { + log?.("addResyncInterval"); + schema.settings.resyncInterval = DEFAULT_SETTINGS.resyncInterval; + } + return schema; } \ No newline at end of file diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 8cb744b2..b9b818fe 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -10,7 +10,7 @@ import { cloneRepository, fetchRepository } from "./git.js"; import { createLogger } from "./logger.js"; import { createRepository, Database, loadDB, updateRepository, updateSettings } from './db.js'; import { arraysEqualShallow, isRemotePath, measure } from "./utils.js"; -import { DEFAULT_SETTINGS, REINDEX_INTERVAL_MS, RESYNC_CONFIG_INTERVAL_MS } from "./constants.js"; +import { DEFAULT_SETTINGS } from "./constants.js"; import stripJsonComments from 'strip-json-comments'; import { indexGitRepository, indexLocalRepository } from "./zoekt.js"; import { getLocalRepoFromConfig, initLocalRepoFileWatchers } from "./local.js"; @@ -205,6 +205,8 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, const updatedSettings: Settings = { maxFileSize: config.settings?.maxFileSize ?? DEFAULT_SETTINGS.maxFileSize, autoDeleteStaleRepos: config.settings?.autoDeleteStaleRepos ?? DEFAULT_SETTINGS.autoDeleteStaleRepos, + reindexInterval: config.settings?.reindexInterval ?? DEFAULT_SETTINGS.reindexInterval, + resyncInterval: config.settings?.resyncInterval ?? DEFAULT_SETTINGS.resyncInterval, } const _isAllRepoReindexingRequired = isAllRepoReindexingRequired(db.data.settings, updatedSettings); await updateSettings(updatedSettings, db); @@ -345,11 +347,10 @@ export const main = async (context: AppContext) => { }); } - // Re-sync every 24 hours + // Re-sync at a fixed interval setInterval(() => { - logger.info(`Re-syncing configuration file ${context.configPath}`); _syncConfig(); - }, RESYNC_CONFIG_INTERVAL_MS); + }, db.data.settings.resyncInterval); // Sync immediately on startup await _syncConfig(); @@ -369,7 +370,7 @@ export const main = async (context: AppContext) => { continue; } - if (lastIndexed.getTime() > Date.now() - REINDEX_INTERVAL_MS) { + if (lastIndexed.getTime() > (Date.now() - db.data.settings.reindexInterval)) { continue; } diff --git a/packages/backend/src/schemas/v2.ts b/packages/backend/src/schemas/v2.ts index 0cbc4aa0..3ef76dca 100644 --- a/packages/backend/src/schemas/v2.ts +++ b/packages/backend/src/schemas/v2.ts @@ -25,6 +25,14 @@ export interface Settings { * Automatically delete stale repositories from the index. Defaults to true. */ autoDeleteStaleRepos?: boolean; + /** + * The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds). + */ + reindexInterval?: number; + /** + * The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds). + */ + resyncInterval?: number; } export interface GitHubConfig { /** diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 97ded5d8..c1029b07 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -44,8 +44,22 @@ export type AppContext = { } export type Settings = { + /** + * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed. + */ maxFileSize: number; + /** + * Automatically delete stale repositories from the index. Defaults to true. + */ autoDeleteStaleRepos: boolean; + /** + * The interval (in milliseconds) at which the indexer should re-index all repositories. + */ + reindexInterval: number; + /** + * The interval (in milliseconds) at which the configuration file should be re-synced. + */ + resyncInterval: number; } // @see : https://stackoverflow.com/a/61132308 diff --git a/schemas/v2/index.json b/schemas/v2/index.json index 649dbab0..c1bc3966 100644 --- a/schemas/v2/index.json +++ b/schemas/v2/index.json @@ -534,6 +534,18 @@ "type": "boolean", "description": "Automatically delete stale repositories from the index. Defaults to true.", "default": true + }, + "reindexInterval": { + "type": "integer", + "description": "The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds).", + "default": 3600000, + "minimum": 1 + }, + "resyncInterval": { + "type": "integer", + "description": "The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds).", + "default": 86400000, + "minimum": 1 } }, "additionalProperties": false