implement raw remote git repo support (#152)

* implement raw remote git repo support

* add changelog entry

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added support for creating share links to snippets of code. ([#149](https://github.com/sourcebot-dev/sourcebot/pull/149))
+- Added support for indexing raw remote git repository. ([#152](https://github.com/sourcebot-dev/sourcebot/pull/152))
 
 ## [2.6.3] - 2024-12-18
 

packages/backend/src/git.ts

@@ -1,7 +1,9 @@
-import { GitRepository } from './types.js';
+import { GitRepository, AppContext } from './types.js';
 import { simpleGit, SimpleGitProgressEvent } from 'simple-git';
 import { existsSync } from 'fs';
 import { createLogger } from './logger.js';
+import { GitConfig } from './schemas/v2.js';
+import path from 'path';
 
 const logger = createLogger('git');
 
@@ -49,3 +51,80 @@ export const fetchRepository = async (repo: GitRepository, onProgress?: (event:
         ]
     );
 }
+
+const isValidGitRepo = async (url: string): Promise<boolean> => {
+    const git = simpleGit();
+    try {
+        await git.listRemote([url]);
+        return true;
+    } catch (error) {
+        logger.debug(`Error checking if ${url} is a valid git repo: ${error}`);
+        return false;
+    }
+}
+
+const stripProtocolAndGitSuffix = (url: string): string => {
+    return url.replace(/^[a-zA-Z]+:\/\//, '').replace(/\.git$/, '');
+}
+
+const getRepoNameFromUrl = (url: string): string => {
+    const strippedUrl = stripProtocolAndGitSuffix(url);
+    return strippedUrl.split('/').slice(-2).join('/');
+}
+
+export const getGitRepoFromConfig = async (config: GitConfig, ctx: AppContext) => {
+    const repoValid = await isValidGitRepo(config.url);
+    if (!repoValid) {
+        logger.error(`Git repo provided in config with url ${config.url} is not valid`);
+        return null;
+    }
+
+    const cloneUrl = config.url;
+    const repoId = stripProtocolAndGitSuffix(cloneUrl);
+    const repoName = getRepoNameFromUrl(config.url);
+    const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
+    const repo: GitRepository = {
+        vcs: 'git',
+        id: repoId,
+        name: repoName,
+        path: repoPath,
+        isStale: false,
+        cloneUrl: cloneUrl,
+        branches: [],
+        tags: [],
+    }
+
+    if (config.revisions) {
+        if (config.revisions.branches) {
+            const branchGlobs = config.revisions.branches;
+            const git = simpleGit();
+            const branchList = await git.listRemote(['--heads', cloneUrl]);
+            const branches = branchList
+                .split('\n')
+                .map(line => line.split('\t')[1])
+                .filter(Boolean)
+                .map(branch => branch.replace('refs/heads/', ''));
+
+            repo.branches = branches.filter(branch => 
+                branchGlobs.some(glob => new RegExp(glob).test(branch))
+            );
+        }
+
+        if (config.revisions.tags) {
+            const tagGlobs = config.revisions.tags;
+            const git = simpleGit();
+            const tagList = await git.listRemote(['--tags', cloneUrl]);
+            const tags = tagList
+                .split('\n')
+                .map(line => line.split('\t')[1])
+                .filter(Boolean)
+                .map(tag => tag.replace('refs/tags/', ''));
+
+            repo.tags = tags.filter(tag => 
+                tagGlobs.some(glob => new RegExp(glob).test(tag))
+            );
+        }
+    }
+
+    return repo;
+}

packages/backend/src/main.ts

@@ -6,7 +6,7 @@ import { getGitLabReposFromConfig } from "./gitlab.js";
 import { getGiteaReposFromConfig } from "./gitea.js";
 import { getGerritReposFromConfig } from "./gerrit.js";
 import { AppContext, LocalRepository, GitRepository, Repository, Settings } from "./types.js";
-import { cloneRepository, fetchRepository } from "./git.js";
+import { cloneRepository, fetchRepository, getGitRepoFromConfig } from "./git.js";
 import { createLogger } from "./logger.js";
 import { createRepository, Database, loadDB, updateRepository, updateSettings } from './db.js';
 import { arraysEqualShallow, isRemotePath, measure } from "./utils.js";
@@ -245,6 +245,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
                 configRepos.push(repo);
                 break;
             }
+            case 'git': {
+                const gitRepo = await  getGitRepoFromConfig(repoConfig, ctx);
+                gitRepo && configRepos.push(gitRepo);
+                break;
+            }
         }
     }
 

packages/backend/src/schemas/v2.ts

@@ -1,6 +1,6 @@
 // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
 
-export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig;
+export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig | GitConfig;
 
 /**
  * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index.
@@ -268,3 +268,14 @@ export interface LocalConfig {
     paths?: string[];
   };
 }
+export interface GitConfig {
+  /**
+   * Git Configuration
+   */
+  type: "git";
+  /**
+   * The URL to the git repository.
+   */
+  url: string;
+  revisions?: GitRevisions;
+}

schemas/v2/index.json

@@ -516,6 +516,28 @@
             ],
             "additionalProperties": false
         },
+        "GitConfig": {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "const": "git",
+                    "description": "Git Configuration"
+                },
+                "url": {
+                    "type": "string",
+                    "format": "url",
+                    "description": "The URL to the git repository."
+                },
+                "revisions": {
+                    "$ref": "#/definitions/GitRevisions"
+                }
+            },
+            "required": [
+                "type",
+                "url"
+            ],
+            "additionalProperties": false
+        },
         "Repos": {
             "anyOf": [
                 {
@@ -532,6 +554,9 @@
                 },
                 {
                     "$ref": "#/definitions/LocalConfig"
+                },
+                {
+                    "$ref": "#/definitions/GitConfig"
                 }
             ]
         },