Local directory support (#56)

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Local directory indexing support. ([#56](https://github.com/sourcebot-dev/sourcebot/pull/56))
+
 ## [2.2.0] - 2024-10-30
 
 ### Added

Dockerfile

@@ -76,6 +76,7 @@ COPY --from=zoekt-builder \
     /cmd/zoekt-mirror-gitlab \
     /cmd/zoekt-mirror-gerrit \
     /cmd/zoekt-webserver \
+    /cmd/zoekt-index \
     /usr/local/bin/
 
 # Configure the webapp

README.md

@@ -267,6 +267,37 @@ docker run -e <b>GITEA_TOKEN=my-secret-token</b> /* additional args */ ghcr.io/s
 
 If you're using a self-hosted GitLab or GitHub instance with a custom domain, you can specify the domain in your config file. See [configs/self-hosted.json](configs/self-hosted.json) for examples.
 
+## Searching a local directory
+
+Local directories can be searched by using the `local` type in your config file:
+
+```jsonc
+{
+    "$schema": "https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v2/index.json",
+    "repos": [
+        {
+            "type": "local",
+            "path": "/repos/my-repo",
+            // re-index files when a change is detected
+            "watch": true,
+            "exclude": {
+                // exclude paths from being indexed
+                "paths": [
+                    "node_modules",
+                    "build"
+                ]
+            }
+        }
+    ]
+}
+```
+
+You'll need to mount the directory as a volume when running Sourcebot:
+
+<pre>
+docker run <b>-v /path/to/my-repo:/repos/my-repo</b> /* additional args */ ghcr.io/sourcebot-dev/sourcebot:latest
+</pre>
+
 ## Build from source
 >[!NOTE]
 > Building from source is only required if you'd like to contribute. The recommended way to use Sourcebot is to use the [pre-built docker image](https://github.com/sourcebot-dev/sourcebot/pkgs/container/sourcebot).

configs/auth.json

@@ -17,6 +17,13 @@
                 "my-group"
             ]
         },
+        {
+            "type": "gitea",
+            "token": "gitea-token",
+            "orgs": [
+                "my-org"
+            ]
+        },
 
         // You can also store the token in a environment variable and then
         // references it from the config.
@@ -34,6 +41,15 @@
             "groups": [
                 "my-group"
             ]
+        },
+        {
+            "type": "gitea",
+            "token": {
+                "env": "GITEA_TOKEN_ENV_VAR"
+            },
+            "orgs": [
+                "my-org"
+            ]
         }
     ]
 }

configs/basic.json

@@ -37,6 +37,28 @@
             "projects": [
                 "my-group/project1"
             ]
+        },
+        // From Gitea, include:
+        // - all public repos owned by user `my-user`
+        // - all public repos owned by organization `my-org`
+        // - repo `my-org/my-repo`
+        {
+            "type": "gitea",
+            "token": "my-token",
+            "users": [
+                "my-user"
+            ],
+            "orgs": [
+                "my-org"
+            ],
+            "repos": [
+                "my-org/my-repo"
+            ]
+        },
+        // Index a local repository
+        {
+            "type": "local",
+            "path": "/path/to/local/repo"
         }
     ]
 }

configs/filter.json

@@ -37,6 +37,25 @@
                     "my-group/project2"
                 ]
             }
-        }
+        },
+
+        // Include all repos in my-org, except:
+        // - repo1 & repo2
+        // - repos that are archived or forks
+        {
+            "type": "gitea",
+            "token": "my-token",
+            "orgs": [
+                "my-org"
+            ],
+            "exclude": {
+                "archived": true,
+                "forks": true,
+                "repos": [
+                    "my-org/repo1",
+                    "my-org/repo2"
+                ]
+            }
+        },
     ]
 }

configs/local-repo.json

@@ -0,0 +1,32 @@
+{
+    "$schema": "../schemas/v2/index.json",
+    "repos": [
+        {
+            "type": "local",
+            "path": "/path/to/local/repo"
+        },
+        // Relative paths are relative to the config file
+        {
+            "type": "local",
+            "path": "../../relative/path/to/local/repo"
+        },
+        // File watcher can be disabled (enabled by default)
+        {
+            "type": "local",
+            "path": "/path/to/local/repo",
+            "watch": false
+        },
+        // Exclude paths can be specified
+        {
+            "type": "local",
+            "path": "/path/to/local/repo",
+            "exclude": {
+                "paths": [
+                    ".git",
+                    "node_modules",
+                    "dist"
+                ]
+            }
+        }
+    ]
+}

configs/self-hosted.json

@@ -14,6 +14,13 @@
             "groups": [
                 "my-group"
             ]
+        },
+        {
+            "type": "gitea",
+            "url": "https://gitea.example.com",
+            "orgs": [
+                "my-org-name"
+            ]
         }
     ]
 }

packages/backend/src/db.ts

@@ -14,7 +14,7 @@ export const loadDB = async (ctx: AppContext): Promise<Database> => {
     const db = await JSONFilePreset<Schema>(`${ctx.cachePath}/db.json`, { repos: {} });
     return db;
 }
-export const updateRepository = async (repoId: string, data: Partial<Repository>, db: Database) => {
+export const updateRepository = async (repoId: string, data: Repository, db: Database) => {
     db.data.repos[repoId] = {
         ...db.data.repos[repoId],
         ...data,

packages/backend/src/git.ts

@@ -1,11 +1,11 @@
-import { Repository } from './types.js';
+import { GitRepository } from './types.js';
 import { simpleGit, SimpleGitProgressEvent } from 'simple-git';
 import { existsSync } from 'fs';
 import { createLogger } from './logger.js';
 
 const logger = createLogger('git');
 
-export const cloneRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
+export const cloneRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
     if (existsSync(repo.path)) {
         logger.warn(`${repo.id} already exists. Skipping clone.`)
         return;
@@ -34,7 +34,7 @@ export const cloneRepository = async (repo: Repository, onProgress?: (event: Sim
 }
 
 
-export const fetchRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
+export const fetchRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => {
     const git = simpleGit({
         progress: onProgress,
     });

packages/backend/src/gitea.ts

@@ -1,7 +1,7 @@
 import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
 import { GiteaConfig } from './schemas/v2.js';
 import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from './utils.js';
-import { AppContext, Repository } from './types.js';
+import { AppContext, GitRepository } from './types.js';
 import fetch from 'cross-fetch';
 import { createLogger } from './logger.js';
 import path from 'path';
@@ -33,7 +33,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
         allRepos = allRepos.concat(_repos);
     }
 
-    let repos: Repository[] = allRepos
+    let repos: GitRepository[] = allRepos
         .map((repo) => {
             const hostname = config.url ? new URL(config.url).hostname : 'gitea.com';
             const repoId = `${hostname}/${repo.full_name!}`;
@@ -45,6 +45,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
             }
 
             return {
+                vcs: 'git',
                 name: repo.full_name!,
                 id: repoId,
                 cloneUrl: cloneUrl.toString(),
@@ -60,7 +61,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
                     'zoekt.fork': marshalBool(repo.fork!),
                     'zoekt.public': marshalBool(repo.internal === false && repo.private === false),
                 }
-            } satisfies Repository;
+            } satisfies GitRepository;
         });
     
     if (config.exclude) {

packages/backend/src/github.ts

@@ -1,7 +1,7 @@
 import { Octokit } from "@octokit/rest";
 import { GitHubConfig } from "./schemas/v2.js";
 import { createLogger } from "./logger.js";
-import { AppContext, Repository } from "./types.js";
+import { AppContext, GitRepository } from "./types.js";
 import path from 'path';
 import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool } from "./utils.js";
 
@@ -50,7 +50,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
     }
 
     // Marshall results to our type
-    let repos: Repository[] = allRepos
+    let repos: GitRepository[] = allRepos
         .filter((repo) => {
             if (!repo.clone_url) {
                 logger.warn(`Repository ${repo.name} missing property 'clone_url'. Excluding.`)
@@ -69,6 +69,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
             }
             
             return {
+                vcs: 'git',
                 name: repo.full_name,
                 id: repoId,
                 cloneUrl: cloneUrl.toString(),
@@ -88,7 +89,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
                     'zoekt.fork': marshalBool(repo.fork),
                     'zoekt.public': marshalBool(repo.private === false)
                 }
-            } satisfies Repository;
+            } satisfies GitRepository;
         });
 
     if (config.exclude) {

packages/backend/src/gitlab.ts

@@ -2,7 +2,7 @@ import { Gitlab, ProjectSchema } from "@gitbeaker/rest";
 import { GitLabConfig } from "./schemas/v2.js";
 import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from "./utils.js";
 import { createLogger } from "./logger.js";
-import { AppContext, Repository } from "./types.js";
+import { AppContext, GitRepository } from "./types.js";
 import path from 'path';
 
 const logger = createLogger("GitLab");
@@ -59,7 +59,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
         allProjects = allProjects.concat(_projects);
     }
 
-    let repos: Repository[] = allProjects
+    let repos: GitRepository[] = allProjects
         .map((project) => {
             const hostname = config.url ? new URL(config.url).hostname : "gitlab.com";
             const repoId = `${hostname}/${project.path_with_namespace}`;
@@ -73,6 +73,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
             }
 
             return {
+                vcs: 'git',
                 name: project.path_with_namespace,
                 id: repoId,
                 cloneUrl: cloneUrl.toString(),
@@ -90,7 +91,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
                     'zoekt.fork': marshalBool(isFork),
                     'zoekt.public': marshalBool(project.visibility === 'public'),
                 }
-            } satisfies Repository;
+            } satisfies GitRepository;
         });
 
     if (config.exclude) {

packages/backend/src/index.ts

@@ -1,19 +1,20 @@
 import { ArgumentParser } from "argparse";
 import { mkdir, readFile } from 'fs/promises';
-import { existsSync, watch } from 'fs';
+import { existsSync, watch, FSWatcher } from 'fs';
-import { exec } from "child_process";
 import path from 'path';
 import { SourcebotConfigurationSchema } from "./schemas/v2.js";
 import { getGitHubReposFromConfig } from "./github.js";
 import { getGitLabReposFromConfig } from "./gitlab.js";
 import { getGiteaReposFromConfig } from "./gitea.js";
-import { AppContext, Repository } from "./types.js";
+import { AppContext, LocalRepository, GitRepository, Repository } from "./types.js";
 import { cloneRepository, fetchRepository } from "./git.js";
 import { createLogger } from "./logger.js";
 import { createRepository, Database, loadDB, updateRepository } from './db.js';
 import { isRemotePath, measure } from "./utils.js";
 import { REINDEX_INTERVAL_MS, RESYNC_CONFIG_INTERVAL_MS } from "./constants.js";
 import stripJsonComments from 'strip-json-comments';
+import { indexGitRepository, indexLocalRepository } from "./zoekt.js";
+import { getLocalRepoFromConfig, initLocalRepoFileWatchers } from "./local.js";
 
 const logger = createLogger('main');
 
@@ -26,19 +27,32 @@ type Arguments = {
     cacheDir: string;
 }
 
-const indexRepository = async (repo: Repository, ctx: AppContext) => {
+const syncGitRepository = async (repo: GitRepository, ctx: AppContext) => {
-    return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
+    if (existsSync(repo.path)) {
-        exec(`zoekt-git-index -index ${ctx.indexPath} ${repo.path}`, (error, stdout, stderr) => {
+        logger.info(`Fetching ${repo.id}...`);
-            if (error) {
+        const { durationMs } = await measure(() => fetchRepository(repo, ({ method, stage , progress}) => {
-                reject(error);
+            logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
-                return;
+        }));
+        process.stdout.write('\n');
+        logger.info(`Fetched ${repo.id} in ${durationMs / 1000}s`);
+    } else {
+        logger.info(`Cloning ${repo.id}...`);
+        const { durationMs } = await measure(() => cloneRepository(repo, ({ method, stage, progress }) => {
+            logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
+        }));
+        process.stdout.write('\n');
+        logger.info(`Cloned ${repo.id} in ${durationMs / 1000}s`);
     }
-            resolve({
+
-                stdout,
+    logger.info(`Indexing ${repo.id}...`);
-                stderr
+    const { durationMs } = await measure(() => indexGitRepository(repo, ctx));
-            });
+    logger.info(`Indexed ${repo.id} in ${durationMs / 1000}s`);
-        })
+}
-    });
+
+const syncLocalRepository = async (repo: LocalRepository, ctx: AppContext, signal?: AbortSignal) => {
+    logger.info(`Indexing ${repo.id}...`);
+    const { durationMs } = await measure(() => indexLocalRepository(repo, ctx, signal));
+    logger.info(`Indexed ${repo.id} in ${durationMs / 1000}s`);
 }
 
 const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, ctx: AppContext) => {
@@ -81,6 +95,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
                 configRepos.push(...giteaRepos);
                 break;
             }
+            case 'local': {
+                const repo = getLocalRepoFromConfig(repoConfig, ctx);
+                configRepos.push(repo);
+                break;
+            }
         }
     }
 
@@ -167,7 +186,7 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
     
     let abortController = new AbortController();
     let isSyncing = false;
-    const _syncConfig = () => {
+    const _syncConfig = async () => {
         if (isSyncing) {
             abortController.abort();
             abortController = new AbortController();
@@ -175,12 +194,12 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
 
         logger.info(`Syncing configuration file ${args.configPath} ...`);
         isSyncing = true;
-        measure(() => syncConfig(args.configPath, db, abortController.signal, context))
+
-            .then(({ durationMs }) => {
+        try {
+            const { durationMs } = await measure(() => syncConfig(args.configPath, db, abortController.signal, context))
             logger.info(`Synced configuration file ${args.configPath} in ${durationMs / 1000}s`);
             isSyncing = false;
-            })
+        } catch (err: any) {
-            .catch((err) => {
             if (err.name === "AbortError") {
                 // @note: If we're aborting, we don't want to set isSyncing to false
                 // since it implies another sync is in progress.
@@ -189,6 +208,13 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
                 logger.error(`Failed to sync configuration file ${args.configPath} with error:`);
                 console.log(err);
             }
+        }
+
+        const localRepos = Object.values(db.data.repos).filter(repo => repo.vcs === 'local');
+        initLocalRepoFileWatchers(localRepos, async (repo, signal) => {
+            logger.info(`Change detected to local repository ${repo.id}. Re-syncing...`);
+            await syncLocalRepository(repo, context, signal);
+            await db.update(({ repos }) => repos[repo.id].lastIndexedDate = new Date().toUTCString());
         });
     }
 
@@ -207,7 +233,7 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
     }, RESYNC_CONFIG_INTERVAL_MS);
 
     // Sync immediately on startup
-    _syncConfig();
+    await _syncConfig();
 
     while (true) {
         const repos = db.data.repos;
@@ -223,25 +249,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
             }
 
             try {
-                if (existsSync(repo.path)) {
+                if (repo.vcs === 'git') {
-                    logger.info(`Fetching ${repo.id}...`);
+                    await syncGitRepository(repo, context);
-                    const { durationMs } = await measure(() => fetchRepository(repo, ({ method, stage , progress}) => {
+                } else if (repo.vcs === 'local') {
-                        logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
+                    await syncLocalRepository(repo, context);
-                    }));
-                    process.stdout.write('\n');
-                    logger.info(`Fetched ${repo.id} in ${durationMs / 1000}s`);
-                } else {
-                    logger.info(`Cloning ${repo.id}...`);
-                    const { durationMs } = await measure(() => cloneRepository(repo, ({ method, stage, progress }) => {
-                        logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
-                    }));
-                    process.stdout.write('\n');
-                    logger.info(`Cloned ${repo.id} in ${durationMs / 1000}s`);
                 }
-
-                logger.info(`Indexing ${repo.id}...`);
-                const { durationMs } = await measure(() => indexRepository(repo, context));
-                logger.info(`Indexed ${repo.id} in ${durationMs / 1000}s`);
             } catch (err: any) {
                 // @todo : better error handling here..
                 logger.error(err);

packages/backend/src/local.ts

@@ -0,0 +1,71 @@
+import { existsSync, FSWatcher, statSync, watch } from "fs";
+import { createLogger } from "./logger.js";
+import { LocalConfig } from "./schemas/v2.js";
+import { AppContext, LocalRepository } from "./types.js";
+import { resolvePathRelativeToConfig } from "./utils.js";
+import path from "path";
+
+const logger = createLogger('local');
+const fileWatchers = new Map<string, FSWatcher>();
+const abortControllers = new Map<string, AbortController>();
+
+
+export const getLocalRepoFromConfig = (config: LocalConfig, ctx: AppContext) => {
+    const repoPath = resolvePathRelativeToConfig(config.path, ctx.configPath);
+    logger.debug(`Resolved path '${config.path}' to '${repoPath}'`);
+
+    if (!existsSync(repoPath)) {
+        throw new Error(`The local repository path '${repoPath}' referenced in ${ctx.configPath} does not exist`);
+    }
+
+    const stat = statSync(repoPath);
+    if (!stat.isDirectory()) {
+        throw new Error(`The local repository path '${repoPath}' referenced in ${ctx.configPath} is not a directory`);
+    }
+
+    const repo: LocalRepository = {
+        vcs: 'local',
+        name: path.basename(repoPath),
+        id: repoPath,
+        path: repoPath,
+        isStale: false,
+        excludedPaths: config.exclude?.paths ?? [],
+        watch: config.watch ?? true,
+    }
+
+    return repo;
+}
+
+export const initLocalRepoFileWatchers = (repos: LocalRepository[], onUpdate: (repo: LocalRepository, ac: AbortSignal) => Promise<void>) => {
+    // Close all existing watchers
+    fileWatchers.forEach((watcher) => {
+        watcher.close();
+    });
+
+    repos
+        .filter(repo => !repo.isStale && repo.watch)
+        .forEach((repo) => {
+            logger.info(`Watching local repository ${repo.id} for changes...`);
+            const watcher = watch(repo.path, async () => {
+                const existingController = abortControllers.get(repo.id);
+                if (existingController) {
+                    existingController.abort();
+                }
+
+                const controller = new AbortController();
+                abortControllers.set(repo.id, controller);
+
+                try {
+                    await onUpdate(repo, controller.signal);
+                } catch (err: any) {
+                    if (err.name !== 'AbortError') {
+                        logger.error(`Error while watching local repository ${repo.id} for changes:`);
+                        console.log(err);
+                    } else {
+                        logger.debug(`Aborting watch for local repository ${repo.id} due to abort signal`);
+                    }
+                }
+            });
+            fileWatchers.set(repo.id, watcher);
+        });
+}

packages/backend/src/schemas/v2.ts

@@ -1,6 +1,6 @@
 // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
 
-export type Repos = GitHubConfig | GitLabConfig | GiteaConfig;
+export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | LocalConfig;
 
 /**
  * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index.
@@ -153,3 +153,23 @@ export interface GiteaConfig {
     repos?: string[];
   };
 }
+export interface LocalConfig {
+  /**
+   * Local Configuration
+   */
+  type: "local";
+  /**
+   * Path to the local directory to sync with. Relative paths are relative to the configuration file's directory.
+   */
+  path: string;
+  /**
+   * Enables a file watcher that will automatically re-sync when changes are made within `path` (recursively). Defaults to true.
+   */
+  watch?: boolean;
+  exclude?: {
+    /**
+     * List of paths relative to the provided `path` to exclude from the index. .git, .hg, and .svn are always exluded.
+     */
+    paths?: string[];
+  };
+}

packages/backend/src/types.ts

@@ -1,34 +1,29 @@
 
-export type Repository = {
+interface BaseRepository {
-    /**
+    vcs: 'git' | 'local';
-     * Name of the repository (e.g., 'sourcebot-dev/sourcebot')
-     */
-    name: string;
-
-    /**
-     * The unique identifier for the repository. (e.g., `github.com/sourcebot-dev/sourcebot`)
-     */
     id: string;
-    
+    name: string;
-    /**
-     * The .git url for the repository
-     */
-    cloneUrl: string;
-
-    /**
-     * Path to where the repository is cloned
-     */
     path: string;
-
-    gitConfigMetadata?: Record<string, string>;
-
-    lastIndexedDate?: string;
-
     isStale: boolean;
-    isFork: boolean;
+    lastIndexedDate?: string;
-    isArchived: boolean;
+    isFork?: boolean;
+    isArchived?: boolean;
 }
 
+export interface GitRepository extends BaseRepository {
+    vcs: 'git';
+    cloneUrl: string;
+    gitConfigMetadata?: Record<string, string>;
+}
+
+export interface LocalRepository extends BaseRepository {
+    vcs: 'local';
+    excludedPaths: string[];
+    watch: boolean;
+}
+
+export type Repository = GitRepository | LocalRepository;
+
 export type AppContext = {
     /**
      * Path to the repos cache directory.

packages/backend/src/utils.ts

@@ -1,5 +1,6 @@
 import { Logger } from "winston";
 import { AppContext, Repository } from "./types.js";
+import path from 'path';
 
 export const measure = async <T>(cb : () => Promise<T>) => {
     const start = Date.now();
@@ -15,9 +16,9 @@ export const marshalBool = (value?: boolean) => {
     return !!value ? '1' : '0';
 }
 
-export const excludeForkedRepos = (repos: Repository[], logger?: Logger) => {
+export const excludeForkedRepos = <T extends Repository>(repos: T[], logger?: Logger) => {
     return repos.filter((repo) => {
-        if (repo.isFork) {
+        if (!!repo.isFork) {
             logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.forks is true`);
             return false;
         }
@@ -25,9 +26,9 @@ export const excludeForkedRepos = (repos: Repository[], logger?: Logger) => {
     });
 }
 
-export const excludeArchivedRepos = (repos: Repository[], logger?: Logger) => {
+export const excludeArchivedRepos = <T extends Repository>(repos: T[], logger?: Logger) => {
     return repos.filter((repo) => {
-        if (repo.isArchived) {
+        if (!!repo.isArchived) {
             logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.archived is true`);
             return false;
         }
@@ -35,7 +36,7 @@ export const excludeArchivedRepos = (repos: Repository[], logger?: Logger) => {
     });
 }
 
-export const excludeReposByName = (repos: Repository[], excludedRepoNames: string[], logger?: Logger) => {
+export const excludeReposByName = <T extends Repository>(repos: T[], excludedRepoNames: string[], logger?: Logger) => {
     const excludedRepos = new Set(excludedRepoNames);
     return repos.filter((repo) => {
         if (excludedRepos.has(repo.name)) {
@@ -60,3 +61,16 @@ export const getTokenFromConfig = (token: string | { env: string }, ctx: AppCont
 export const isRemotePath = (path: string) => {
     return path.startsWith('https://') || path.startsWith('http://');
 }
+
+export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
+    let absolutePath = localPath;
+    if (!path.isAbsolute(absolutePath)) {
+        if (absolutePath.startsWith('~')) {
+            absolutePath = path.join(process.env.HOME ?? '', absolutePath.slice(1));
+        }
+
+        absolutePath = path.resolve(path.dirname(configPath), absolutePath);
+    }
+
+    return absolutePath;
+}

packages/backend/src/zoekt.ts

@@ -0,0 +1,37 @@
+import { exec } from "child_process";
+import { AppContext, GitRepository, LocalRepository } from "./types.js";
+
+const ALWAYS_EXCLUDED_DIRS = ['.git', '.hg', '.svn'];
+
+export const indexGitRepository = async (repo: GitRepository, ctx: AppContext) => {
+    return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
+        exec(`zoekt-git-index -index ${ctx.indexPath} ${repo.path}`, (error, stdout, stderr) => {
+            if (error) {
+                reject(error);
+                return;
+            }
+            resolve({
+                stdout,
+                stderr
+            });
+        })
+    });
+}
+
+export const indexLocalRepository = async (repo: LocalRepository, ctx: AppContext, signal?: AbortSignal) => {
+    const excludedDirs = [...ALWAYS_EXCLUDED_DIRS, repo.excludedPaths];
+    const command = `zoekt-index -index ${ctx.indexPath} -ignore_dirs ${excludedDirs.join(',')} ${repo.path}`;
+
+    return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
+        exec(command, { signal }, (error, stdout, stderr) => {
+            if (error) {
+                reject(error);
+                return;
+            }
+            resolve({
+                stdout,
+                stderr
+            });
+        })
+    });
+}

packages/web/src/app/repos/columns.tsx

@@ -49,6 +49,11 @@ export const columns: ColumnDef<RepositoryColumnInfo>[] = [
         header: "Branches",
         cell: ({ row }) => {
             const branches = row.original.branches;
+
+            if (branches.length === 0) {
+                return <div>N/A</div>;
+            }
+
             return (
                 <div className="flex flex-col gap-2">
                     {branches.map(({ name, version }, index) => {

packages/web/src/app/repos/repositoryTable.tsx

@@ -13,7 +13,7 @@ export const RepositoryTable = async () => {
     const repos = _repos.List.Repos.map((repo): RepositoryColumnInfo => {
         return {
             name: repo.Repository.Name,
-            branches: repo.Repository.Branches.map((branch) => {
+            branches: (repo.Repository.Branches ?? []).map((branch) => {
                 return {
                     name: branch.Name,
                     version: branch.Version,

packages/web/src/lib/schemas.ts

@@ -53,9 +53,9 @@ export const searchResponseSchema = z.object({
         Files: z.array(z.object({
             FileName: z.string(),
             Repository: z.string(),
-            Version: z.string(),
+            Version: z.string().optional(),
             Language: z.string(),
-            Branches: z.array(z.string()),
+            Branches: z.array(z.string()).optional(),
             ChunkMatches: z.array(z.object({
                 Content: z.string(),
                 Ranges: z.array(rangeSchema),
@@ -113,11 +113,11 @@ export const repositorySchema = z.object({
     Branches: z.array(z.object({
         Name: z.string(),
         Version: z.string(),
-    })),
+    })).nullable(),
     CommitURLTemplate: z.string(),
     FileURLTemplate: z.string(),
     LineFragmentTemplate: z.string(),
-    RawConfig: z.record(z.string(), z.string()),
+    RawConfig: z.record(z.string(), z.string()).nullable(),
     Rank: z.number(),
     IndexOptions: z.string(),
     HasSymbols: z.boolean(),

schemas/v2/index.json

@@ -304,6 +304,54 @@
             ],
             "additionalProperties": false
         },
+        "LocalConfig": {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "const": "local",
+                    "description": "Local Configuration"
+                },
+                "path": {
+                    "type": "string",
+                    "description": "Path to the local directory to sync with. Relative paths are relative to the configuration file's directory.",
+                    "pattern": ".+"
+                },
+                "watch": {
+                    "type": "boolean",
+                    "default": true,
+                    "description": "Enables a file watcher that will automatically re-sync when changes are made within `path` (recursively). Defaults to true."
+                },
+                "exclude": {
+                    "type": "object",
+                    "properties": {
+                        "paths": {
+                            "type": "array",
+                            "items": {
+                                "type": "string",
+                                "pattern": ".+"
+                            },
+                            "description": "List of paths relative to the provided `path` to exclude from the index. .git, .hg, and .svn are always exluded.",
+                            "default": [],
+                            "examples": [
+                                [
+                                    "node_modules",
+                                    "bin",
+                                    "dist",
+                                    "build",
+                                    "out"
+                                ]
+                            ]
+                        }
+                    },
+                    "additionalProperties": false
+                }
+            },
+            "required": [
+                "type",
+                "path"
+            ],
+            "additionalProperties": false
+        },
         "Repos": {
             "anyOf": [
                 {
@@ -314,6 +362,9 @@
                 },
                 {
                     "$ref": "#/definitions/GiteaConfig"
+                },
+                {
+                    "$ref": "#/definitions/LocalConfig"
                 }
             ]
         }