diff --git a/CHANGELOG.md b/CHANGELOG.md index 827714d9..baf12cf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Local directory indexing support. ([#56](https://github.com/sourcebot-dev/sourcebot/pull/56)) + ## [2.2.0] - 2024-10-30 ### Added diff --git a/Dockerfile b/Dockerfile index 3af8b3e5..7e5a1955 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,7 @@ COPY --from=zoekt-builder \ /cmd/zoekt-mirror-gitlab \ /cmd/zoekt-mirror-gerrit \ /cmd/zoekt-webserver \ + /cmd/zoekt-index \ /usr/local/bin/ # Configure the webapp diff --git a/README.md b/README.md index e23c0920..f4c7b21a 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,37 @@ docker run -e GITEA_TOKEN=my-secret-token /* additional args */ ghcr.io/s If you're using a self-hosted GitLab or GitHub instance with a custom domain, you can specify the domain in your config file. See [configs/self-hosted.json](configs/self-hosted.json) for examples. +## Searching a local directory + +Local directories can be searched by using the `local` type in your config file: + +```jsonc +{ + "$schema": "https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v2/index.json", + "repos": [ + { + "type": "local", + "path": "/repos/my-repo", + // re-index files when a change is detected + "watch": true, + "exclude": { + // exclude paths from being indexed + "paths": [ + "node_modules", + "build" + ] + } + } + ] +} +``` + +You'll need to mount the directory as a volume when running Sourcebot: + +
+docker run -v /path/to/my-repo:/repos/my-repo /* additional args */ ghcr.io/sourcebot-dev/sourcebot:latest
+
+ ## Build from source >[!NOTE] > Building from source is only required if you'd like to contribute. The recommended way to use Sourcebot is to use the [pre-built docker image](https://github.com/sourcebot-dev/sourcebot/pkgs/container/sourcebot). diff --git a/configs/auth.json b/configs/auth.json index ce328e02..14274eb9 100644 --- a/configs/auth.json +++ b/configs/auth.json @@ -17,6 +17,13 @@ "my-group" ] }, + { + "type": "gitea", + "token": "gitea-token", + "orgs": [ + "my-org" + ] + }, // You can also store the token in a environment variable and then // references it from the config. @@ -34,6 +41,15 @@ "groups": [ "my-group" ] + }, + { + "type": "gitea", + "token": { + "env": "GITEA_TOKEN_ENV_VAR" + }, + "orgs": [ + "my-org" + ] } ] } \ No newline at end of file diff --git a/configs/basic.json b/configs/basic.json index 38ddcb8b..78b8588e 100644 --- a/configs/basic.json +++ b/configs/basic.json @@ -37,6 +37,28 @@ "projects": [ "my-group/project1" ] + }, + // From Gitea, include: + // - all public repos owned by user `my-user` + // - all public repos owned by organization `my-org` + // - repo `my-org/my-repo` + { + "type": "gitea", + "token": "my-token", + "users": [ + "my-user" + ], + "orgs": [ + "my-org" + ], + "repos": [ + "my-org/my-repo" + ] + }, + // Index a local repository + { + "type": "local", + "path": "/path/to/local/repo" } ] } \ No newline at end of file diff --git a/configs/filter.json b/configs/filter.json index f072cd7e..1e31d524 100644 --- a/configs/filter.json +++ b/configs/filter.json @@ -37,6 +37,25 @@ "my-group/project2" ] } - } + }, + + // Include all repos in my-org, except: + // - repo1 & repo2 + // - repos that are archived or forks + { + "type": "gitea", + "token": "my-token", + "orgs": [ + "my-org" + ], + "exclude": { + "archived": true, + "forks": true, + "repos": [ + "my-org/repo1", + "my-org/repo2" + ] + } + }, ] } \ No newline at end of file diff --git a/configs/local-repo.json b/configs/local-repo.json new file mode 100644 index 00000000..8c1647d4 --- /dev/null +++ b/configs/local-repo.json @@ -0,0 +1,32 @@ +{ + "$schema": "../schemas/v2/index.json", + "repos": [ + { + "type": "local", + "path": "/path/to/local/repo" + }, + // Relative paths are relative to the config file + { + "type": "local", + "path": "../../relative/path/to/local/repo" + }, + // File watcher can be disabled (enabled by default) + { + "type": "local", + "path": "/path/to/local/repo", + "watch": false + }, + // Exclude paths can be specified + { + "type": "local", + "path": "/path/to/local/repo", + "exclude": { + "paths": [ + ".git", + "node_modules", + "dist" + ] + } + } + ] +} \ No newline at end of file diff --git a/configs/self-hosted.json b/configs/self-hosted.json index 0ea32be3..96695d1f 100644 --- a/configs/self-hosted.json +++ b/configs/self-hosted.json @@ -14,6 +14,13 @@ "groups": [ "my-group" ] + }, + { + "type": "gitea", + "url": "https://gitea.example.com", + "orgs": [ + "my-org-name" + ] } ] } \ No newline at end of file diff --git a/packages/backend/src/db.ts b/packages/backend/src/db.ts index 84ea69fd..6cc59cd8 100644 --- a/packages/backend/src/db.ts +++ b/packages/backend/src/db.ts @@ -14,7 +14,7 @@ export const loadDB = async (ctx: AppContext): Promise => { const db = await JSONFilePreset(`${ctx.cachePath}/db.json`, { repos: {} }); return db; } -export const updateRepository = async (repoId: string, data: Partial, db: Database) => { +export const updateRepository = async (repoId: string, data: Repository, db: Database) => { db.data.repos[repoId] = { ...db.data.repos[repoId], ...data, diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index 7d5947a2..3e7d57df 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -1,11 +1,11 @@ -import { Repository } from './types.js'; +import { GitRepository } from './types.js'; import { simpleGit, SimpleGitProgressEvent } from 'simple-git'; import { existsSync } from 'fs'; import { createLogger } from './logger.js'; const logger = createLogger('git'); -export const cloneRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => { +export const cloneRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => { if (existsSync(repo.path)) { logger.warn(`${repo.id} already exists. Skipping clone.`) return; @@ -34,7 +34,7 @@ export const cloneRepository = async (repo: Repository, onProgress?: (event: Sim } -export const fetchRepository = async (repo: Repository, onProgress?: (event: SimpleGitProgressEvent) => void) => { +export const fetchRepository = async (repo: GitRepository, onProgress?: (event: SimpleGitProgressEvent) => void) => { const git = simpleGit({ progress: onProgress, }); diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index e26d22b8..0d7273a7 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -1,7 +1,7 @@ import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js'; import { GiteaConfig } from './schemas/v2.js'; import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from './utils.js'; -import { AppContext, Repository } from './types.js'; +import { AppContext, GitRepository } from './types.js'; import fetch from 'cross-fetch'; import { createLogger } from './logger.js'; import path from 'path'; @@ -33,7 +33,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte allRepos = allRepos.concat(_repos); } - let repos: Repository[] = allRepos + let repos: GitRepository[] = allRepos .map((repo) => { const hostname = config.url ? new URL(config.url).hostname : 'gitea.com'; const repoId = `${hostname}/${repo.full_name!}`; @@ -45,6 +45,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte } return { + vcs: 'git', name: repo.full_name!, id: repoId, cloneUrl: cloneUrl.toString(), @@ -60,7 +61,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte 'zoekt.fork': marshalBool(repo.fork!), 'zoekt.public': marshalBool(repo.internal === false && repo.private === false), } - } satisfies Repository; + } satisfies GitRepository; }); if (config.exclude) { diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index e25b8c5e..52aec540 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -1,7 +1,7 @@ import { Octokit } from "@octokit/rest"; import { GitHubConfig } from "./schemas/v2.js"; import { createLogger } from "./logger.js"; -import { AppContext, Repository } from "./types.js"; +import { AppContext, GitRepository } from "./types.js"; import path from 'path'; import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool } from "./utils.js"; @@ -50,7 +50,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo } // Marshall results to our type - let repos: Repository[] = allRepos + let repos: GitRepository[] = allRepos .filter((repo) => { if (!repo.clone_url) { logger.warn(`Repository ${repo.name} missing property 'clone_url'. Excluding.`) @@ -69,6 +69,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo } return { + vcs: 'git', name: repo.full_name, id: repoId, cloneUrl: cloneUrl.toString(), @@ -88,7 +89,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo 'zoekt.fork': marshalBool(repo.fork), 'zoekt.public': marshalBool(repo.private === false) } - } satisfies Repository; + } satisfies GitRepository; }); if (config.exclude) { diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index 9918c13d..8bb78ea7 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -2,7 +2,7 @@ import { Gitlab, ProjectSchema } from "@gitbeaker/rest"; import { GitLabConfig } from "./schemas/v2.js"; import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from "./utils.js"; import { createLogger } from "./logger.js"; -import { AppContext, Repository } from "./types.js"; +import { AppContext, GitRepository } from "./types.js"; import path from 'path'; const logger = createLogger("GitLab"); @@ -59,7 +59,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon allProjects = allProjects.concat(_projects); } - let repos: Repository[] = allProjects + let repos: GitRepository[] = allProjects .map((project) => { const hostname = config.url ? new URL(config.url).hostname : "gitlab.com"; const repoId = `${hostname}/${project.path_with_namespace}`; @@ -73,6 +73,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon } return { + vcs: 'git', name: project.path_with_namespace, id: repoId, cloneUrl: cloneUrl.toString(), @@ -90,7 +91,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon 'zoekt.fork': marshalBool(isFork), 'zoekt.public': marshalBool(project.visibility === 'public'), } - } satisfies Repository; + } satisfies GitRepository; }); if (config.exclude) { diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 871dbdbb..332cffb5 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -1,19 +1,20 @@ import { ArgumentParser } from "argparse"; import { mkdir, readFile } from 'fs/promises'; -import { existsSync, watch } from 'fs'; -import { exec } from "child_process"; +import { existsSync, watch, FSWatcher } from 'fs'; import path from 'path'; import { SourcebotConfigurationSchema } from "./schemas/v2.js"; import { getGitHubReposFromConfig } from "./github.js"; import { getGitLabReposFromConfig } from "./gitlab.js"; import { getGiteaReposFromConfig } from "./gitea.js"; -import { AppContext, Repository } from "./types.js"; +import { AppContext, LocalRepository, GitRepository, Repository } from "./types.js"; import { cloneRepository, fetchRepository } from "./git.js"; import { createLogger } from "./logger.js"; import { createRepository, Database, loadDB, updateRepository } from './db.js'; import { isRemotePath, measure } from "./utils.js"; import { REINDEX_INTERVAL_MS, RESYNC_CONFIG_INTERVAL_MS } from "./constants.js"; import stripJsonComments from 'strip-json-comments'; +import { indexGitRepository, indexLocalRepository } from "./zoekt.js"; +import { getLocalRepoFromConfig, initLocalRepoFileWatchers } from "./local.js"; const logger = createLogger('main'); @@ -26,19 +27,32 @@ type Arguments = { cacheDir: string; } -const indexRepository = async (repo: Repository, ctx: AppContext) => { - return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { - exec(`zoekt-git-index -index ${ctx.indexPath} ${repo.path}`, (error, stdout, stderr) => { - if (error) { - reject(error); - return; - } - resolve({ - stdout, - stderr - }); - }) - }); +const syncGitRepository = async (repo: GitRepository, ctx: AppContext) => { + if (existsSync(repo.path)) { + logger.info(`Fetching ${repo.id}...`); + const { durationMs } = await measure(() => fetchRepository(repo, ({ method, stage , progress}) => { + logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) + })); + process.stdout.write('\n'); + logger.info(`Fetched ${repo.id} in ${durationMs / 1000}s`); + } else { + logger.info(`Cloning ${repo.id}...`); + const { durationMs } = await measure(() => cloneRepository(repo, ({ method, stage, progress }) => { + logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) + })); + process.stdout.write('\n'); + logger.info(`Cloned ${repo.id} in ${durationMs / 1000}s`); + } + + logger.info(`Indexing ${repo.id}...`); + const { durationMs } = await measure(() => indexGitRepository(repo, ctx)); + logger.info(`Indexed ${repo.id} in ${durationMs / 1000}s`); +} + +const syncLocalRepository = async (repo: LocalRepository, ctx: AppContext, signal?: AbortSignal) => { + logger.info(`Indexing ${repo.id}...`); + const { durationMs } = await measure(() => indexLocalRepository(repo, ctx, signal)); + logger.info(`Indexed ${repo.id} in ${durationMs / 1000}s`); } const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, ctx: AppContext) => { @@ -81,6 +95,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, configRepos.push(...giteaRepos); break; } + case 'local': { + const repo = getLocalRepoFromConfig(repoConfig, ctx); + configRepos.push(repo); + break; + } } } @@ -167,7 +186,7 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, let abortController = new AbortController(); let isSyncing = false; - const _syncConfig = () => { + const _syncConfig = async () => { if (isSyncing) { abortController.abort(); abortController = new AbortController(); @@ -175,21 +194,28 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, logger.info(`Syncing configuration file ${args.configPath} ...`); isSyncing = true; - measure(() => syncConfig(args.configPath, db, abortController.signal, context)) - .then(({ durationMs }) => { - logger.info(`Synced configuration file ${args.configPath} in ${durationMs / 1000}s`); + + try { + const { durationMs } = await measure(() => syncConfig(args.configPath, db, abortController.signal, context)) + logger.info(`Synced configuration file ${args.configPath} in ${durationMs / 1000}s`); + isSyncing = false; + } catch (err: any) { + if (err.name === "AbortError") { + // @note: If we're aborting, we don't want to set isSyncing to false + // since it implies another sync is in progress. + } else { isSyncing = false; - }) - .catch((err) => { - if (err.name === "AbortError") { - // @note: If we're aborting, we don't want to set isSyncing to false - // since it implies another sync is in progress. - } else { - isSyncing = false; - logger.error(`Failed to sync configuration file ${args.configPath} with error:`); - console.log(err); - } - }); + logger.error(`Failed to sync configuration file ${args.configPath} with error:`); + console.log(err); + } + } + + const localRepos = Object.values(db.data.repos).filter(repo => repo.vcs === 'local'); + initLocalRepoFileWatchers(localRepos, async (repo, signal) => { + logger.info(`Change detected to local repository ${repo.id}. Re-syncing...`); + await syncLocalRepository(repo, context, signal); + await db.update(({ repos }) => repos[repo.id].lastIndexedDate = new Date().toUTCString()); + }); } // Re-sync on file changes if the config file is local @@ -207,7 +233,7 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, }, RESYNC_CONFIG_INTERVAL_MS); // Sync immediately on startup - _syncConfig(); + await _syncConfig(); while (true) { const repos = db.data.repos; @@ -223,25 +249,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal, } try { - if (existsSync(repo.path)) { - logger.info(`Fetching ${repo.id}...`); - const { durationMs } = await measure(() => fetchRepository(repo, ({ method, stage , progress}) => { - logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) - })); - process.stdout.write('\n'); - logger.info(`Fetched ${repo.id} in ${durationMs / 1000}s`); - } else { - logger.info(`Cloning ${repo.id}...`); - const { durationMs } = await measure(() => cloneRepository(repo, ({ method, stage, progress }) => { - logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) - })); - process.stdout.write('\n'); - logger.info(`Cloned ${repo.id} in ${durationMs / 1000}s`); + if (repo.vcs === 'git') { + await syncGitRepository(repo, context); + } else if (repo.vcs === 'local') { + await syncLocalRepository(repo, context); } - - logger.info(`Indexing ${repo.id}...`); - const { durationMs } = await measure(() => indexRepository(repo, context)); - logger.info(`Indexed ${repo.id} in ${durationMs / 1000}s`); } catch (err: any) { // @todo : better error handling here.. logger.error(err); diff --git a/packages/backend/src/local.ts b/packages/backend/src/local.ts new file mode 100644 index 00000000..6e415546 --- /dev/null +++ b/packages/backend/src/local.ts @@ -0,0 +1,71 @@ +import { existsSync, FSWatcher, statSync, watch } from "fs"; +import { createLogger } from "./logger.js"; +import { LocalConfig } from "./schemas/v2.js"; +import { AppContext, LocalRepository } from "./types.js"; +import { resolvePathRelativeToConfig } from "./utils.js"; +import path from "path"; + +const logger = createLogger('local'); +const fileWatchers = new Map(); +const abortControllers = new Map(); + + +export const getLocalRepoFromConfig = (config: LocalConfig, ctx: AppContext) => { + const repoPath = resolvePathRelativeToConfig(config.path, ctx.configPath); + logger.debug(`Resolved path '${config.path}' to '${repoPath}'`); + + if (!existsSync(repoPath)) { + throw new Error(`The local repository path '${repoPath}' referenced in ${ctx.configPath} does not exist`); + } + + const stat = statSync(repoPath); + if (!stat.isDirectory()) { + throw new Error(`The local repository path '${repoPath}' referenced in ${ctx.configPath} is not a directory`); + } + + const repo: LocalRepository = { + vcs: 'local', + name: path.basename(repoPath), + id: repoPath, + path: repoPath, + isStale: false, + excludedPaths: config.exclude?.paths ?? [], + watch: config.watch ?? true, + } + + return repo; +} + +export const initLocalRepoFileWatchers = (repos: LocalRepository[], onUpdate: (repo: LocalRepository, ac: AbortSignal) => Promise) => { + // Close all existing watchers + fileWatchers.forEach((watcher) => { + watcher.close(); + }); + + repos + .filter(repo => !repo.isStale && repo.watch) + .forEach((repo) => { + logger.info(`Watching local repository ${repo.id} for changes...`); + const watcher = watch(repo.path, async () => { + const existingController = abortControllers.get(repo.id); + if (existingController) { + existingController.abort(); + } + + const controller = new AbortController(); + abortControllers.set(repo.id, controller); + + try { + await onUpdate(repo, controller.signal); + } catch (err: any) { + if (err.name !== 'AbortError') { + logger.error(`Error while watching local repository ${repo.id} for changes:`); + console.log(err); + } else { + logger.debug(`Aborting watch for local repository ${repo.id} due to abort signal`); + } + } + }); + fileWatchers.set(repo.id, watcher); + }); +} \ No newline at end of file diff --git a/packages/backend/src/schemas/v2.ts b/packages/backend/src/schemas/v2.ts index 7450cd06..4d99fa8f 100644 --- a/packages/backend/src/schemas/v2.ts +++ b/packages/backend/src/schemas/v2.ts @@ -1,6 +1,6 @@ // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! -export type Repos = GitHubConfig | GitLabConfig | GiteaConfig; +export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | LocalConfig; /** * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index. @@ -153,3 +153,23 @@ export interface GiteaConfig { repos?: string[]; }; } +export interface LocalConfig { + /** + * Local Configuration + */ + type: "local"; + /** + * Path to the local directory to sync with. Relative paths are relative to the configuration file's directory. + */ + path: string; + /** + * Enables a file watcher that will automatically re-sync when changes are made within `path` (recursively). Defaults to true. + */ + watch?: boolean; + exclude?: { + /** + * List of paths relative to the provided `path` to exclude from the index. .git, .hg, and .svn are always exluded. + */ + paths?: string[]; + }; +} diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 6d8253b9..c53ef438 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,34 +1,29 @@ -export type Repository = { - /** - * Name of the repository (e.g., 'sourcebot-dev/sourcebot') - */ - name: string; - - /** - * The unique identifier for the repository. (e.g., `github.com/sourcebot-dev/sourcebot`) - */ +interface BaseRepository { + vcs: 'git' | 'local'; id: string; - - /** - * The .git url for the repository - */ - cloneUrl: string; - - /** - * Path to where the repository is cloned - */ + name: string; path: string; - - gitConfigMetadata?: Record; - - lastIndexedDate?: string; - isStale: boolean; - isFork: boolean; - isArchived: boolean; + lastIndexedDate?: string; + isFork?: boolean; + isArchived?: boolean; } +export interface GitRepository extends BaseRepository { + vcs: 'git'; + cloneUrl: string; + gitConfigMetadata?: Record; +} + +export interface LocalRepository extends BaseRepository { + vcs: 'local'; + excludedPaths: string[]; + watch: boolean; +} + +export type Repository = GitRepository | LocalRepository; + export type AppContext = { /** * Path to the repos cache directory. diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 6cdfb833..adc2b2d8 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,5 +1,6 @@ import { Logger } from "winston"; import { AppContext, Repository } from "./types.js"; +import path from 'path'; export const measure = async (cb : () => Promise) => { const start = Date.now(); @@ -15,9 +16,9 @@ export const marshalBool = (value?: boolean) => { return !!value ? '1' : '0'; } -export const excludeForkedRepos = (repos: Repository[], logger?: Logger) => { +export const excludeForkedRepos = (repos: T[], logger?: Logger) => { return repos.filter((repo) => { - if (repo.isFork) { + if (!!repo.isFork) { logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.forks is true`); return false; } @@ -25,9 +26,9 @@ export const excludeForkedRepos = (repos: Repository[], logger?: Logger) => { }); } -export const excludeArchivedRepos = (repos: Repository[], logger?: Logger) => { +export const excludeArchivedRepos = (repos: T[], logger?: Logger) => { return repos.filter((repo) => { - if (repo.isArchived) { + if (!!repo.isArchived) { logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.archived is true`); return false; } @@ -35,7 +36,7 @@ export const excludeArchivedRepos = (repos: Repository[], logger?: Logger) => { }); } -export const excludeReposByName = (repos: Repository[], excludedRepoNames: string[], logger?: Logger) => { +export const excludeReposByName = (repos: T[], excludedRepoNames: string[], logger?: Logger) => { const excludedRepos = new Set(excludedRepoNames); return repos.filter((repo) => { if (excludedRepos.has(repo.name)) { @@ -59,4 +60,17 @@ export const getTokenFromConfig = (token: string | { env: string }, ctx: AppCont export const isRemotePath = (path: string) => { return path.startsWith('https://') || path.startsWith('http://'); +} + +export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => { + let absolutePath = localPath; + if (!path.isAbsolute(absolutePath)) { + if (absolutePath.startsWith('~')) { + absolutePath = path.join(process.env.HOME ?? '', absolutePath.slice(1)); + } + + absolutePath = path.resolve(path.dirname(configPath), absolutePath); + } + + return absolutePath; } \ No newline at end of file diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts new file mode 100644 index 00000000..7eb495f9 --- /dev/null +++ b/packages/backend/src/zoekt.ts @@ -0,0 +1,37 @@ +import { exec } from "child_process"; +import { AppContext, GitRepository, LocalRepository } from "./types.js"; + +const ALWAYS_EXCLUDED_DIRS = ['.git', '.hg', '.svn']; + +export const indexGitRepository = async (repo: GitRepository, ctx: AppContext) => { + return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { + exec(`zoekt-git-index -index ${ctx.indexPath} ${repo.path}`, (error, stdout, stderr) => { + if (error) { + reject(error); + return; + } + resolve({ + stdout, + stderr + }); + }) + }); +} + +export const indexLocalRepository = async (repo: LocalRepository, ctx: AppContext, signal?: AbortSignal) => { + const excludedDirs = [...ALWAYS_EXCLUDED_DIRS, repo.excludedPaths]; + const command = `zoekt-index -index ${ctx.indexPath} -ignore_dirs ${excludedDirs.join(',')} ${repo.path}`; + + return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { + exec(command, { signal }, (error, stdout, stderr) => { + if (error) { + reject(error); + return; + } + resolve({ + stdout, + stderr + }); + }) + }); +} \ No newline at end of file diff --git a/packages/web/src/app/repos/columns.tsx b/packages/web/src/app/repos/columns.tsx index d0477e44..c1101e59 100644 --- a/packages/web/src/app/repos/columns.tsx +++ b/packages/web/src/app/repos/columns.tsx @@ -49,6 +49,11 @@ export const columns: ColumnDef[] = [ header: "Branches", cell: ({ row }) => { const branches = row.original.branches; + + if (branches.length === 0) { + return
N/A
; + } + return (
{branches.map(({ name, version }, index) => { diff --git a/packages/web/src/app/repos/repositoryTable.tsx b/packages/web/src/app/repos/repositoryTable.tsx index 2e7d0da6..81ec5d11 100644 --- a/packages/web/src/app/repos/repositoryTable.tsx +++ b/packages/web/src/app/repos/repositoryTable.tsx @@ -13,7 +13,7 @@ export const RepositoryTable = async () => { const repos = _repos.List.Repos.map((repo): RepositoryColumnInfo => { return { name: repo.Repository.Name, - branches: repo.Repository.Branches.map((branch) => { + branches: (repo.Repository.Branches ?? []).map((branch) => { return { name: branch.Name, version: branch.Version, diff --git a/packages/web/src/lib/schemas.ts b/packages/web/src/lib/schemas.ts index ee40b7d3..cac793cf 100644 --- a/packages/web/src/lib/schemas.ts +++ b/packages/web/src/lib/schemas.ts @@ -53,9 +53,9 @@ export const searchResponseSchema = z.object({ Files: z.array(z.object({ FileName: z.string(), Repository: z.string(), - Version: z.string(), + Version: z.string().optional(), Language: z.string(), - Branches: z.array(z.string()), + Branches: z.array(z.string()).optional(), ChunkMatches: z.array(z.object({ Content: z.string(), Ranges: z.array(rangeSchema), @@ -113,11 +113,11 @@ export const repositorySchema = z.object({ Branches: z.array(z.object({ Name: z.string(), Version: z.string(), - })), + })).nullable(), CommitURLTemplate: z.string(), FileURLTemplate: z.string(), LineFragmentTemplate: z.string(), - RawConfig: z.record(z.string(), z.string()), + RawConfig: z.record(z.string(), z.string()).nullable(), Rank: z.number(), IndexOptions: z.string(), HasSymbols: z.boolean(), diff --git a/schemas/v2/index.json b/schemas/v2/index.json index ed864e35..9cdfb6f9 100644 --- a/schemas/v2/index.json +++ b/schemas/v2/index.json @@ -304,6 +304,54 @@ ], "additionalProperties": false }, + "LocalConfig": { + "type": "object", + "properties": { + "type": { + "const": "local", + "description": "Local Configuration" + }, + "path": { + "type": "string", + "description": "Path to the local directory to sync with. Relative paths are relative to the configuration file's directory.", + "pattern": ".+" + }, + "watch": { + "type": "boolean", + "default": true, + "description": "Enables a file watcher that will automatically re-sync when changes are made within `path` (recursively). Defaults to true." + }, + "exclude": { + "type": "object", + "properties": { + "paths": { + "type": "array", + "items": { + "type": "string", + "pattern": ".+" + }, + "description": "List of paths relative to the provided `path` to exclude from the index. .git, .hg, and .svn are always exluded.", + "default": [], + "examples": [ + [ + "node_modules", + "bin", + "dist", + "build", + "out" + ] + ] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "path" + ], + "additionalProperties": false + }, "Repos": { "anyOf": [ { @@ -314,6 +362,9 @@ }, { "$ref": "#/definitions/GiteaConfig" + }, + { + "$ref": "#/definitions/LocalConfig" } ] }