Add reindexInterval and resyncInterval to config settings (#134)

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Added config option `settings.reindexInterval` and `settings.resyncInterval` to control how often the index should be re-indexed and re-synced. ([#134](https://github.com/sourcebot-dev/sourcebot/pull/134))
+
 ## [2.6.2] - 2024-12-13
 
 ### Added

demo-site-config.json

@@ -1,7 +1,9 @@
 {
     "$schema": "./schemas/v2/index.json",
     "settings": {
-        "autoDeleteStaleRepos": true
+        "autoDeleteStaleRepos": true,
+        "reindexInterval": 86400000, // 24 hours
+        "resyncInterval": 86400000 // 24 hours
     },
     "repos": [
         {

packages/backend/src/constants.ts

@@ -1,19 +1,11 @@
 import { Settings } from "./types.js";
 
-/**
- * The interval to reindex a given repository.
- */
-export const REINDEX_INTERVAL_MS = 1000 * 60 * 60;
-
-/**
- * The interval to re-sync the config.
- */
-export const RESYNC_CONFIG_INTERVAL_MS = 1000 * 60 * 60 * 24;
-
 /**
  * Default settings.
  */
 export const DEFAULT_SETTINGS: Settings = {
     maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
     autoDeleteStaleRepos: true,
+    reindexInterval: 1000 * 60 * 60, // 1 hour in milliseconds
+    resyncInterval: 1000 * 60 * 60 * 24, // 1 day in milliseconds
 }

packages/backend/src/db.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from 'vitest';
-import { DEFAULT_DB_DATA, migration_addDeleteStaleRepos, migration_addMaxFileSize, migration_addSettings, Schema } from './db';
+import { DEFAULT_DB_DATA, migration_addDeleteStaleRepos, migration_addMaxFileSize, migration_addReindexInterval, migration_addResyncInterval, migration_addSettings, Schema } from './db';
 import { DEFAULT_SETTINGS } from './constants';
 import { DeepPartial } from './types';
 import { Low } from 'lowdb';
@@ -61,3 +61,65 @@ test('migration_addDeleteStaleRepos adds the `autoDeleteStaleRepos` field with t
         }
     });
 });
+
+test('migration_addReindexInterval adds the `reindexInterval` field with the default value if it does not exist', () => {
+    const schema: DeepPartial<Schema> = {
+        settings: {
+            maxFileSize: DEFAULT_SETTINGS.maxFileSize,
+            autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
+        },
+    }
+
+    const migratedSchema = migration_addReindexInterval(schema as Schema);
+    expect(migratedSchema).toStrictEqual({
+        settings: {
+            maxFileSize: DEFAULT_SETTINGS.maxFileSize,
+            autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
+            reindexInterval: DEFAULT_SETTINGS.reindexInterval,
+        }
+    });
+});
+
+test('migration_addReindexInterval preserves existing reindexInterval value if already set', () => {
+    const customInterval = 60;
+    const schema: DeepPartial<Schema> = {
+        settings: {
+            maxFileSize: DEFAULT_SETTINGS.maxFileSize,
+            reindexInterval: customInterval,
+        },
+    }
+
+    const migratedSchema = migration_addReindexInterval(schema as Schema);
+    expect(migratedSchema.settings.reindexInterval).toBe(customInterval);
+});
+
+test('migration_addResyncInterval adds the `resyncInterval` field with the default value if it does not exist', () => {
+    const schema: DeepPartial<Schema> = {
+        settings: {
+            maxFileSize: DEFAULT_SETTINGS.maxFileSize,
+            autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
+        },
+    }
+
+    const migratedSchema = migration_addResyncInterval(schema as Schema);
+    expect(migratedSchema).toStrictEqual({
+        settings: {
+            maxFileSize: DEFAULT_SETTINGS.maxFileSize,
+            autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
+            resyncInterval: DEFAULT_SETTINGS.resyncInterval,
+        }
+    });
+});
+
+test('migration_addResyncInterval preserves existing resyncInterval value if already set', () => {
+    const customInterval = 120;
+    const schema: DeepPartial<Schema> = {
+        settings: {
+            maxFileSize: DEFAULT_SETTINGS.maxFileSize,
+            resyncInterval: customInterval,
+        },
+    }
+
+    const migratedSchema = migration_addResyncInterval(schema as Schema);
+    expect(migratedSchema.settings.resyncInterval).toBe(customInterval);
+});

packages/backend/src/db.ts

@@ -56,6 +56,8 @@ export const applyMigrations = async (db: Database) => {
         schema = migration_addSettings(schema, log);
         schema = migration_addMaxFileSize(schema, log);
         schema = migration_addDeleteStaleRepos(schema, log);
+        schema = migration_addReindexInterval(schema, log);
+        schema = migration_addResyncInterval(schema, log);
         return schema;
     });
 }
@@ -89,9 +91,33 @@ export const migration_addMaxFileSize = (schema: Schema, log?: (name: string) =>
  */
 export const migration_addDeleteStaleRepos = (schema: Schema, log?: (name: string) => void) => {
     if (schema.settings.autoDeleteStaleRepos === undefined) {
-        log?.("deleteStaleRepos");
+        log?.("addDeleteStaleRepos");
         schema.settings.autoDeleteStaleRepos = DEFAULT_SETTINGS.autoDeleteStaleRepos;
     }
 
     return schema;
 }
+
+/**
+ * @see: https://github.com/sourcebot-dev/sourcebot/pull/134
+ */
+export const migration_addReindexInterval = (schema: Schema, log?: (name: string) => void) => {
+    if (schema.settings.reindexInterval === undefined) {
+        log?.("addReindexInterval");
+        schema.settings.reindexInterval = DEFAULT_SETTINGS.reindexInterval;
+    }
+
+    return schema;
+}
+
+/**
+ * @see: https://github.com/sourcebot-dev/sourcebot/pull/134
+ */
+export const migration_addResyncInterval = (schema: Schema, log?: (name: string) => void) => {
+    if (schema.settings.resyncInterval === undefined) {
+        log?.("addResyncInterval");
+        schema.settings.resyncInterval = DEFAULT_SETTINGS.resyncInterval;
+    }
+
+    return schema;
+}

packages/backend/src/main.ts

@@ -10,7 +10,7 @@ import { cloneRepository, fetchRepository } from "./git.js";
 import { createLogger } from "./logger.js";
 import { createRepository, Database, loadDB, updateRepository, updateSettings } from './db.js';
 import { arraysEqualShallow, isRemotePath, measure } from "./utils.js";
-import { DEFAULT_SETTINGS, REINDEX_INTERVAL_MS, RESYNC_CONFIG_INTERVAL_MS } from "./constants.js";
+import { DEFAULT_SETTINGS } from "./constants.js";
 import stripJsonComments from 'strip-json-comments';
 import { indexGitRepository, indexLocalRepository } from "./zoekt.js";
 import { getLocalRepoFromConfig, initLocalRepoFileWatchers } from "./local.js";
@@ -205,6 +205,8 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
     const updatedSettings: Settings = {
         maxFileSize: config.settings?.maxFileSize ?? DEFAULT_SETTINGS.maxFileSize,
         autoDeleteStaleRepos: config.settings?.autoDeleteStaleRepos ?? DEFAULT_SETTINGS.autoDeleteStaleRepos,
+        reindexInterval: config.settings?.reindexInterval ?? DEFAULT_SETTINGS.reindexInterval,
+        resyncInterval: config.settings?.resyncInterval ?? DEFAULT_SETTINGS.resyncInterval,
     }
     const _isAllRepoReindexingRequired = isAllRepoReindexingRequired(db.data.settings, updatedSettings);
     await updateSettings(updatedSettings, db);
@@ -345,11 +347,10 @@ export const main = async (context: AppContext) => {
         });
     }
 
-    // Re-sync every 24 hours
+    // Re-sync at a fixed interval
     setInterval(() => {
-        logger.info(`Re-syncing configuration file ${context.configPath}`);
         _syncConfig();
-    }, RESYNC_CONFIG_INTERVAL_MS);
+    }, db.data.settings.resyncInterval);
 
     // Sync immediately on startup
     await _syncConfig();
@@ -369,7 +370,7 @@ export const main = async (context: AppContext) => {
                 continue;
             }
 
-            if (lastIndexed.getTime() > Date.now() - REINDEX_INTERVAL_MS) {
+            if (lastIndexed.getTime() > (Date.now() - db.data.settings.reindexInterval)) {
                 continue;
             }
 

packages/backend/src/schemas/v2.ts

@@ -25,6 +25,14 @@ export interface Settings {
    * Automatically delete stale repositories from the index. Defaults to true.
    */
   autoDeleteStaleRepos?: boolean;
+  /**
+   * The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds).
+   */
+  reindexInterval?: number;
+  /**
+   * The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds).
+   */
+  resyncInterval?: number;
 }
 export interface GitHubConfig {
   /**

packages/backend/src/types.ts

@@ -44,8 +44,22 @@ export type AppContext = {
 }
 
 export type Settings = {
+    /**
+     * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed.
+     */
     maxFileSize: number;
+    /**
+     * Automatically delete stale repositories from the index. Defaults to true.
+     */
     autoDeleteStaleRepos: boolean;
+    /**
+     * The interval (in milliseconds) at which the indexer should re-index all repositories.
+     */
+    reindexInterval: number;
+    /**
+     * The interval (in milliseconds) at which the configuration file should be re-synced.
+     */
+    resyncInterval: number;
 }
 
 // @see : https://stackoverflow.com/a/61132308

schemas/v2/index.json

@@ -534,6 +534,18 @@
                     "type": "boolean",
                     "description": "Automatically delete stale repositories from the index. Defaults to true.",
                     "default": true
+                },
+                "reindexInterval": {
+                    "type": "integer",
+                    "description": "The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds).",
+                    "default": 3600000,
+                    "minimum": 1
+                },
+                "resyncInterval": {
+                    "type": "integer",
+                    "description": "The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds).",
+                    "default": 86400000,
+                    "minimum": 1
                 }
             },
             "additionalProperties": false