feat(worker,web): Improved connection management (#579)

CHANGELOG.md

@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
 - Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
 - Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)
+- Improved connections table. [#579](https://github.com/sourcebot-dev/sourcebot/pull/579)
 
 ### Removed
 - Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)

docs/snippets/schemas/v3/app.schema.mdx

@@ -3,11 +3,9 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "AppConfig",
-  "oneOf": [
-    {
-      "$schema": "http://json-schema.org/draft-07/schema#",
+  "definitions": {
+    "GitHubAppConfig": {
       "type": "object",
-      "title": "GithubAppConfig",
       "properties": {
         "type": {
           "const": "githubApp",
@@ -61,19 +59,70 @@
       },
       "required": [
         "type",
-        "id"
+        "id",
+        "privateKey"
       ],
-      "oneOf": [
-        {
-          "required": [
-            "privateKey"
+      "additionalProperties": false
+    }
+  },
+  "oneOf": [
+    {
+      "type": "object",
+      "properties": {
+        "type": {
+          "const": "githubApp",
+          "description": "GitHub App Configuration"
+        },
+        "deploymentHostname": {
+          "type": "string",
+          "format": "hostname",
+          "default": "github.com",
+          "description": "The hostname of the GitHub App deployment.",
+          "examples": [
+            "github.com",
+            "github.example.com"
           ]
         },
-        {
-          "required": [
-            "privateKeyPath"
+        "id": {
+          "type": "string",
+          "description": "The ID of the GitHub App."
+        },
+        "privateKey": {
+          "description": "The private key of the GitHub App.",
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "secret": {
+                  "type": "string",
+                  "description": "The name of the secret that contains the token."
+                }
+              },
+              "required": [
+                "secret"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            }
           ]
         }
+      },
+      "required": [
+        "type",
+        "id",
+        "privateKey"
       ],
       "additionalProperties": false
     }

docs/snippets/schemas/v3/index.schema.mdx

@@ -4280,11 +4280,9 @@
       "items": {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "title": "AppConfig",
-        "oneOf": [
-          {
-            "$schema": "http://json-schema.org/draft-07/schema#",
+        "definitions": {
+          "GitHubAppConfig": {
             "type": "object",
-            "title": "GithubAppConfig",
             "properties": {
               "type": {
                 "const": "githubApp",
@@ -4338,19 +4336,70 @@
             },
             "required": [
               "type",
-              "id"
+              "id",
+              "privateKey"
             ],
-            "oneOf": [
-              {
-                "required": [
-                  "privateKey"
+            "additionalProperties": false
+          }
+        },
+        "oneOf": [
+          {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "githubApp",
+                "description": "GitHub App Configuration"
+              },
+              "deploymentHostname": {
+                "type": "string",
+                "format": "hostname",
+                "default": "github.com",
+                "description": "The hostname of the GitHub App deployment.",
+                "examples": [
+                  "github.com",
+                  "github.example.com"
                 ]
               },
-              {
-                "required": [
-                  "privateKeyPath"
-                ]
+              "id": {
+                "type": "string",
+                "description": "The ID of the GitHub App."
+              },
+              "privateKey": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "secret": {
+                        "type": "string",
+                        "description": "The name of the secret that contains the token."
+                      }
+                    },
+                    "required": [
+                      "secret"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  }
+                ],
+                "description": "The private key of the GitHub App."
               }
+            },
+            "required": [
+              "type",
+              "id",
+              "privateKey"
             ],
             "additionalProperties": false
           }

packages/backend/package.json

@@ -40,6 +40,7 @@
         "argparse": "^2.0.1",
         "azure-devops-node-api": "^15.1.1",
         "bullmq": "^5.34.10",
+        "chokidar": "^4.0.3",
         "cross-fetch": "^4.0.0",
         "dotenv": "^16.4.5",
         "express": "^4.21.2",

packages/backend/src/azuredevops.ts

@@ -1,6 +1,6 @@
 import { AzureDevOpsConnectionConfig } from "@sourcebot/schemas/v3/azuredevops.type";
 import { createLogger } from "@sourcebot/logger";
-import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js";
+import { measure, fetchWithRetry } from "./utils.js";
 import micromatch from "micromatch";
 import { PrismaClient } from "@sourcebot/db";
 import { BackendException, BackendError } from "@sourcebot/error";
@@ -8,6 +8,7 @@ import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
 import * as Sentry from "@sentry/node";
 import * as azdev from "azure-devops-node-api";
 import { GitRepository } from "azure-devops-node-api/interfaces/GitInterfaces.js";
+import { getTokenFromConfig } from "@sourcebot/crypto";
 
 const logger = createLogger('azuredevops');
 const AZUREDEVOPS_CLOUD_HOSTNAME = "dev.azure.com";
@@ -34,7 +35,7 @@ export const getAzureDevOpsReposFromConfig = async (
     const baseUrl = config.url || `https://${AZUREDEVOPS_CLOUD_HOSTNAME}`;
 
     const token = config.token ?
-        await getTokenFromConfig(config.token, orgId, db, logger) :
+        await getTokenFromConfig(config.token, orgId, db) :
         undefined;
 
     if (!token) {
@@ -47,47 +48,39 @@ export const getAzureDevOpsReposFromConfig = async (
 
     const useTfsPath = config.useTfsPath || false;
     let allRepos: GitRepository[] = [];
-    let notFound: {
-        users: string[],
-        orgs: string[],
-        repos: string[],
-    } = {
-        users: [],
-        orgs: [],
-        repos: [],
-    };
+    let allWarnings: string[] = [];
 
     if (config.orgs) {
-        const { validRepos, notFoundOrgs } = await getReposForOrganizations(
+        const { repos, warnings } = await getReposForOrganizations(
             config.orgs, 
             baseUrl,
             token,
             useTfsPath
         );
-        allRepos = allRepos.concat(validRepos);
-        notFound.orgs = notFoundOrgs;
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.projects) {
-        const { validRepos, notFoundProjects } = await getReposForProjects(
+        const { repos, warnings } = await getReposForProjects(
             config.projects,
             baseUrl,
             token,
             useTfsPath
         );
-        allRepos = allRepos.concat(validRepos);
-        notFound.repos = notFound.repos.concat(notFoundProjects);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.repos) {
-        const { validRepos, notFoundRepos } = await getRepos(
+        const { repos, warnings } = await getRepos(
             config.repos,
             baseUrl,
             token,
             useTfsPath
         );
-        allRepos = allRepos.concat(validRepos);
-        notFound.repos = notFound.repos.concat(notFoundRepos);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     let repos = allRepos
@@ -103,8 +96,8 @@ export const getAzureDevOpsReposFromConfig = async (
     logger.debug(`Found ${repos.length} total repositories.`);
 
     return {
-        validRepos: repos,
-        notFound,
+        repos,
+        warnings: allWarnings,
     };
 };
 
@@ -221,10 +214,11 @@ async function getReposForOrganizations(
 
             // Check if it's a 404-like error (organization not found)
             if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
-                logger.error(`Organization ${org} not found or no access`);
+                const warning = `Organization ${org} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: org
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw error;
@@ -232,11 +226,11 @@ async function getReposForOrganizations(
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults<GitRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<GitRepository>(results);
 
     return {
-        validRepos,
-        notFoundOrgs,
+        repos,
+        warnings,
     };
 }
 
@@ -274,10 +268,11 @@ async function getReposForProjects(
             logger.error(`Failed to fetch repositories for project ${project}.`, error);
 
             if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
-                logger.error(`Project ${project} not found or no access`);
+                const warning = `Project ${project} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: project
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw error;
@@ -285,11 +280,11 @@ async function getReposForProjects(
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults<GitRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<GitRepository>(results);
 
     return {
-        validRepos,
-        notFoundProjects,
+        repos,
+        warnings,
     };
 }
 
@@ -328,10 +323,11 @@ async function getRepos(
             logger.error(`Failed to fetch repository ${repo}.`, error);
 
             if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
-                logger.error(`Repository ${repo} not found or no access`);
+                const warning = `Repository ${repo} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: repo
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw error;
@@ -339,10 +335,10 @@ async function getRepos(
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults<GitRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<GitRepository>(results);
 
     return {
-        validRepos,
-        notFoundRepos,
+        repos,
+        warnings,
     };
 }

packages/backend/src/bitbucket.ts

@@ -4,7 +4,7 @@ import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type"
 import type { ClientOptions, ClientPathsWithMethod } from "openapi-fetch";
 import { createLogger } from "@sourcebot/logger";
 import { PrismaClient } from "@sourcebot/db";
-import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js";
+import { measure, fetchWithRetry } from "./utils.js";
 import * as Sentry from "@sentry/node";
 import {
     SchemaRepository as CloudRepository,
@@ -12,6 +12,7 @@ import {
 import { SchemaRestRepository as ServerRepository } from "@coderabbitai/bitbucket/server/openapi";
 import { processPromiseResults } from "./connectionUtils.js";
 import { throwIfAnyFailed } from "./connectionUtils.js";
+import { getTokenFromConfig } from "@sourcebot/crypto";
 
 const logger = createLogger('bitbucket');
 const BITBUCKET_CLOUD_GIT = 'https://bitbucket.org';
@@ -27,9 +28,9 @@ interface BitbucketClient {
     apiClient: any;
     baseUrl: string;
     gitUrl: string;
-    getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundWorkspaces: string[]}>;
-    getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundProjects: string[]}>;
-    getRepos: (client: BitbucketClient, repos: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundRepos: string[]}>;
+    getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>;
+    getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>;
+    getRepos: (client: BitbucketClient, repos: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>;
     shouldExcludeRepo: (repo: BitbucketRepository, config: BitbucketConnectionConfig) => boolean;
 }
 
@@ -59,7 +60,7 @@ type ServerPaginatedResponse<T> = {
 
 export const getBitbucketReposFromConfig = async (config: BitbucketConnectionConfig, orgId: number, db: PrismaClient) => {
     const token = config.token ?
-        await getTokenFromConfig(config.token, orgId, db, logger) :
+        await getTokenFromConfig(config.token, orgId, db) :
         undefined;
 
     if (config.deploymentType === 'server' && !config.url) {
@@ -71,32 +72,24 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon
         cloudClient(config.user, token);
 
     let allRepos: BitbucketRepository[] = [];
-    let notFound: {
-        orgs: string[],
-        users: string[],
-        repos: string[],
-    } = {
-        orgs: [],
-        users: [],
-        repos: [],
-    };
+    let allWarnings: string[] = [];
 
     if (config.workspaces) {
-        const { validRepos, notFoundWorkspaces } = await client.getReposForWorkspace(client, config.workspaces);
-        allRepos = allRepos.concat(validRepos);
-        notFound.orgs = notFoundWorkspaces;
+        const { repos, warnings } = await client.getReposForWorkspace(client, config.workspaces);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.projects) {
-        const { validRepos, notFoundProjects } = await client.getReposForProjects(client, config.projects);
-        allRepos = allRepos.concat(validRepos);
-        notFound.orgs = notFoundProjects;
+        const { repos, warnings } = await client.getReposForProjects(client, config.projects);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.repos) {
-        const { validRepos, notFoundRepos } = await client.getRepos(client, config.repos);
-        allRepos = allRepos.concat(validRepos);
-        notFound.repos = notFoundRepos;
+        const { repos, warnings } = await client.getRepos(client, config.repos);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     const filteredRepos = allRepos.filter((repo) => {
@@ -104,8 +97,8 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon
     });
 
     return {
-        validRepos: filteredRepos,
-        notFound,
+        repos: filteredRepos,
+        warnings: allWarnings,
     };
 }
 
@@ -186,7 +179,7 @@ function parseUrl(url: string): { path: string; query: Record<string, string>; }
 }
 
 
-async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{validRepos: CloudRepository[], notFoundWorkspaces: string[]}> {
+async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> {
     const results = await Promise.allSettled(workspaces.map(async (workspace) => {
         try {
             logger.debug(`Fetching all repos for workspace ${workspace}...`);
@@ -221,10 +214,11 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st
 
             const status = e?.cause?.response?.status;
             if (status == 404) {
-                logger.error(`Workspace ${workspace} not found or invalid access`)
+                const warning = `Workspace ${workspace} not found or invalid access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: workspace
+                    type: 'warning' as const,
+                    warning
                 }
             }
             throw e;
@@ -232,21 +226,22 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundWorkspaces } = processPromiseResults(results);
+    const { validItems: repos, warnings } = processPromiseResults(results);
     return {
-        validRepos,
-        notFoundWorkspaces,
+        repos,
+        warnings,
     };
 }
 
-async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{validRepos: CloudRepository[], notFoundProjects: string[]}> {
+async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> {
     const results = await Promise.allSettled(projects.map(async (project) => {
         const [workspace, project_name] = project.split('/');
         if (!workspace || !project_name) {
-            logger.error(`Invalid project ${project}`);
+            const warning = `Invalid project ${project}`;
+            logger.warn(warning);
             return {
-                type: 'notFound' as const,
-                value: project
+                type: 'warning' as const,
+                warning
             }
         }
 
@@ -282,10 +277,11 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin
 
             const status = e?.cause?.response?.status;
             if (status == 404) {
-                logger.error(`Project ${project_name} not found in ${workspace} or invalid access`)
+                const warning = `Project ${project_name} not found in ${workspace} or invalid access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: project
+                    type: 'warning' as const,
+                    warning
                 }
             }
             throw e;
@@ -293,21 +289,22 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results);
+    const { validItems: repos, warnings } = processPromiseResults(results);
     return {
-        validRepos,
-        notFoundProjects
+        repos,
+        warnings
     }
 }
 
-async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise<{validRepos: CloudRepository[], notFoundRepos: string[]}> {
-    const results = await Promise.allSettled(repos.map(async (repo) => {
+async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> {
+    const results = await Promise.allSettled(repoList.map(async (repo) => {
         const [workspace, repo_slug] = repo.split('/');
         if (!workspace || !repo_slug) {
-            logger.error(`Invalid repo ${repo}`);
+            const warning = `Invalid repo ${repo}`;
+            logger.warn(warning);
             return {
-                type: 'notFound' as const,
-                value: repo
+                type: 'warning' as const,
+                warning
             };
         }
 
@@ -329,10 +326,11 @@ async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise<
 
             const status = e?.cause?.response?.status;
             if (status === 404) {
-                logger.error(`Repo ${repo} not found in ${workspace} or invalid access`);
+                const warning = `Repo ${repo} not found in ${workspace} or invalid access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: repo
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw e;
@@ -340,10 +338,10 @@ async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise<
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results);
+    const { validItems: repos, warnings } = processPromiseResults(results);
     return {
-        validRepos,
-        notFoundRepos
+        repos,
+        warnings
     };
 }
 
@@ -434,15 +432,16 @@ const getPaginatedServer = async <T>(
     return results;
 }
 
-async function serverGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{validRepos: ServerRepository[], notFoundWorkspaces: string[]}> {
+async function serverGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> {
+    const warnings = workspaces.map(workspace => `Workspaces are not supported in Bitbucket Server: ${workspace}`);
     logger.debug('Workspaces are not supported in Bitbucket Server');
     return {
-        validRepos: [],
-        notFoundWorkspaces: workspaces
+        repos: [],
+        warnings
     };
 }
 
-async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{validRepos: ServerRepository[], notFoundProjects: string[]}> {
+async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> {
     const results = await Promise.allSettled(projects.map(async (project) => {
         try {
             logger.debug(`Fetching all repos for project ${project}...`);
@@ -477,10 +476,11 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri
 
             const status = e?.cause?.response?.status;
             if (status == 404) {
-                logger.error(`Project ${project} not found or invalid access`);
+                const warning = `Project ${project} not found or invalid access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: project
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw e;
@@ -488,21 +488,22 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results);
+    const { validItems: repos, warnings } = processPromiseResults(results);
     return {
-        validRepos,
-        notFoundProjects
+        repos,
+        warnings
     };
 }
 
-async function serverGetRepos(client: BitbucketClient, repos: string[]): Promise<{validRepos: ServerRepository[], notFoundRepos: string[]}> {
-    const results = await Promise.allSettled(repos.map(async (repo) => {
+async function serverGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> {
+    const results = await Promise.allSettled(repoList.map(async (repo) => {
         const [project, repo_slug] = repo.split('/');
         if (!project || !repo_slug) {
-            logger.error(`Invalid repo ${repo}`);
+            const warning = `Invalid repo ${repo}`;
+            logger.warn(warning);
             return {
-                type: 'notFound' as const,
-                value: repo
+                type: 'warning' as const,
+                warning
             };
         }
 
@@ -524,10 +525,11 @@ async function serverGetRepos(client: BitbucketClient, repos: string[]): Promise
 
             const status = e?.cause?.response?.status;
             if (status === 404) {
-                logger.error(`Repo ${repo} not found in project ${project} or invalid access`);
+                const warning = `Repo ${repo} not found in project ${project} or invalid access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: repo
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw e;
@@ -535,10 +537,10 @@ async function serverGetRepos(client: BitbucketClient, repos: string[]): Promise
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results);
+    const { validItems: repos, warnings } = processPromiseResults(results);
     return {
-        validRepos,
-        notFoundRepos
+        repos,
+        warnings
     };
 }
 

packages/backend/src/configManager.ts

@@ -0,0 +1,126 @@
+import { Prisma, PrismaClient } from "@sourcebot/db";
+import { createLogger } from "@sourcebot/logger";
+import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
+import { loadConfig } from "@sourcebot/shared";
+import chokidar, { FSWatcher } from 'chokidar';
+import { ConnectionManager } from "./connectionManager.js";
+import { SINGLE_TENANT_ORG_ID } from "./constants.js";
+import { syncSearchContexts } from "./ee/syncSearchContexts.js";
+
+const logger = createLogger('config-manager');
+
+export class ConfigManager {
+    private watcher: FSWatcher;
+
+    constructor(
+        private db: PrismaClient,
+        private connectionManager: ConnectionManager,
+        configPath: string,
+    ) {
+        this.watcher = chokidar.watch(configPath, {
+            ignoreInitial: true,           // Don't fire events for existing files
+            awaitWriteFinish: {
+                stabilityThreshold: 100,   // File size stable for 100ms
+                pollInterval: 100          // Check every 100ms
+            },
+            atomic: true                   // Handle atomic writes (temp file + rename)
+        });
+
+        this.watcher.on('change', async () => {
+            logger.info(`Config file ${configPath} changed. Syncing config.`);
+            try {
+                await this.syncConfig(configPath);
+            } catch (error) {
+                logger.error(`Failed to sync config: ${error}`);
+            }
+        });
+
+        this.syncConfig(configPath);
+    }
+
+    private syncConfig = async (configPath: string) => {
+        const config = await loadConfig(configPath);
+
+        await this.syncConnections(config.connections);
+        await syncSearchContexts({
+            contexts: config.contexts,
+            orgId: SINGLE_TENANT_ORG_ID,
+            db: this.db,
+        });
+    }
+
+    private syncConnections = async (connections?: { [key: string]: ConnectionConfig }) => {
+        if (connections) {
+            for (const [key, newConnectionConfig] of Object.entries(connections)) {
+                const existingConnection = await this.db.connection.findUnique({
+                    where: {
+                        name_orgId: {
+                            name: key,
+                            orgId: SINGLE_TENANT_ORG_ID,
+                        }
+                    }
+                });
+
+
+                const existingConnectionConfig = existingConnection ? existingConnection.config as unknown as ConnectionConfig : undefined;
+                const connectionNeedsSyncing =
+                    !existingConnection ||
+                    (JSON.stringify(existingConnectionConfig) !== JSON.stringify(newConnectionConfig));
+
+                // Either update the existing connection or create a new one.
+                const connection = existingConnection ?
+                    await this.db.connection.update({
+                        where: {
+                            id: existingConnection.id,
+                        },
+                        data: {
+                            config: newConnectionConfig as unknown as Prisma.InputJsonValue,
+                            isDeclarative: true,
+                        }
+                    }) :
+                    await this.db.connection.create({
+                        data: {
+                            name: key,
+                            config: newConnectionConfig as unknown as Prisma.InputJsonValue,
+                            connectionType: newConnectionConfig.type,
+                            isDeclarative: true,
+                            org: {
+                                connect: {
+                                    id: SINGLE_TENANT_ORG_ID,
+                                }
+                            }
+                        }
+                    });
+
+                if (connectionNeedsSyncing) {
+                    const [jobId] = await this.connectionManager.createJobs([connection]);
+                    logger.info(`Change detected for connection '${key}' (id: ${connection.id}). Created sync job ${jobId}.`);
+                }
+            }
+        }
+
+        // Delete any connections that are no longer in the config.
+        const deletedConnections = await this.db.connection.findMany({
+            where: {
+                isDeclarative: true,
+                name: {
+                    notIn: Object.keys(connections ?? {}),
+                },
+                orgId: SINGLE_TENANT_ORG_ID,
+            }
+        });
+
+        for (const connection of deletedConnections) {
+            logger.info(`Deleting connection with name '${connection.name}'. Connection ID: ${connection.id}`);
+            await this.db.connection.delete({
+                where: {
+                    id: connection.id,
+                }
+            })
+        }
+    }
+
+    public dispose = async () => {
+        await this.watcher.close();
+    }
+}

packages/backend/src/connectionManager.ts

@@ -1,212 +1,219 @@
-import { Connection, ConnectionSyncStatus, PrismaClient, Prisma } from "@sourcebot/db";
-import { Job, Queue, Worker } from 'bullmq';
-import { Settings } from "./types.js";
-import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
-import { createLogger } from "@sourcebot/logger";
-import { Redis } from 'ioredis';
-import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig, compileBitbucketConfig, compileAzureDevOpsConfig, compileGenericGitHostConfig } from "./repoCompileUtils.js";
-import { BackendError, BackendException } from "@sourcebot/error";
-import { captureEvent } from "./posthog.js";
-import { env } from "./env.js";
 import * as Sentry from "@sentry/node";
-import { loadConfig, syncSearchContexts } from "@sourcebot/shared";
+import { Connection, ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db";
+import { createLogger } from "@sourcebot/logger";
+import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
+import { loadConfig } from "@sourcebot/shared";
+import { Job, Queue, ReservedJob, Worker } from "groupmq";
+import { Redis } from 'ioredis';
+import { env } from "./env.js";
+import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js";
+import { Settings } from "./types.js";
+import { groupmqLifecycleExceptionWrapper } from "./utils.js";
+import { syncSearchContexts } from "./ee/syncSearchContexts.js";
+import { captureEvent } from "./posthog.js";
+import { PromClient } from "./promClient.js";
 
-const QUEUE_NAME = 'connectionSyncQueue';
+const LOG_TAG = 'connection-manager';
+const logger = createLogger(LOG_TAG);
+const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`);
 
 type JobPayload = {
+    jobId: string,
     connectionId: number,
     connectionName: string,
     orgId: number,
-    config: ConnectionConfig,
 };
 
 type JobResult = {
     repoCount: number,
 }
 
+const JOB_TIMEOUT_MS = 1000 * 60 * 60 * 2; // 2 hour timeout
+
 export class ConnectionManager {
     private worker: Worker;
     private queue: Queue<JobPayload>;
-    private logger = createLogger('connection-manager');
     private interval?: NodeJS.Timeout;
 
     constructor(
         private db: PrismaClient,
         private settings: Settings,
         redis: Redis,
+        private promClient: PromClient,
     ) {
-        this.queue = new Queue<JobPayload>(QUEUE_NAME, {
-            connection: redis,
+        this.queue = new Queue<JobPayload>({
+            redis,
+            namespace: 'connection-sync-queue',
+            jobTimeoutMs: JOB_TIMEOUT_MS,
+            maxAttempts: 3,
+            logger: env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true',
         });
-        this.worker = new Worker(QUEUE_NAME, this.runSyncJob.bind(this), {
-            connection: redis,
+
+        this.worker = new Worker<JobPayload>({
+            queue: this.queue,
+            maxStalledCount: 1,
+            handler: this.runJob.bind(this),
             concurrency: this.settings.maxConnectionSyncJobConcurrency,
+            ...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
+                logger: true,
+            } : {}),
         });
-        this.worker.on('completed', this.onSyncJobCompleted.bind(this));
-        this.worker.on('failed', this.onSyncJobFailed.bind(this));
-    }
 
-    public async scheduleConnectionSync(connection: Connection) {
-        await this.db.$transaction(async (tx) => {
-            await tx.connection.update({
-                where: { id: connection.id },
-                data: { syncStatus: ConnectionSyncStatus.IN_SYNC_QUEUE },
-            });
-
-            const connectionConfig = connection.config as unknown as ConnectionConfig;
-
-            await this.queue.add('connectionSyncJob', {
-                connectionId: connection.id,
-                connectionName: connection.name,
-                orgId: connection.orgId,
-                config: connectionConfig,
-            }, {
-                removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE,
-                removeOnFail: env.REDIS_REMOVE_ON_FAIL,
-            });
-            this.logger.info(`Added job to queue for connection ${connection.name} (id: ${connection.id})`);
-        }).catch((err: unknown) => {
-            this.logger.error(`Failed to add job to queue for connection ${connection.name} (id: ${connection.id}): ${err}`);
-        });
+        this.worker.on('completed', this.onJobCompleted.bind(this));
+        this.worker.on('failed', this.onJobFailed.bind(this));
+        this.worker.on('stalled', this.onJobStalled.bind(this));
+        this.worker.on('error', this.onWorkerError.bind(this));
     }
 
     public startScheduler() {
-        this.logger.debug('Starting scheduler');
+        logger.debug('Starting scheduler');
         this.interval = setInterval(async () => {
             const thresholdDate = new Date(Date.now() - this.settings.resyncConnectionIntervalMs);
+            const timeoutDate = new Date(Date.now() - JOB_TIMEOUT_MS);
+
             const connections = await this.db.connection.findMany({
                 where: {
-                    OR: [
-                        // When the connection needs to be synced, we want to sync it immediately.
+                    AND: [
                         {
-                            syncStatus: ConnectionSyncStatus.SYNC_NEEDED,
-                        },
-                        // When the connection has already been synced, we only want to re-sync if the re-sync interval has elapsed
-                        // (or if the date isn't set for some reason).
-                        {
-                            AND: [
-                                {
-                                    OR: [
-                                        { syncStatus: ConnectionSyncStatus.SYNCED },
-                                        { syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS },
-                                    ]
-                                },
-                                {
-                                    OR: [
-                                        { syncedAt: null },
-                                        { syncedAt: { lt: thresholdDate } },
-                                    ]
-                                }
+                            OR: [
+                                { syncedAt: null },
+                                { syncedAt: { lt: thresholdDate } },
                             ]
+                        },
+                        {
+                            NOT: {
+                                syncJobs: {
+                                    some: {
+                                        OR: [
+                                            // Don't schedule if there are active jobs that were created within the threshold date.
+                                            // This handles the case where a job is stuck in a pending state and will never be scheduled.
+                                            {
+                                                AND: [
+                                                    { status: { in: [ConnectionSyncJobStatus.PENDING, ConnectionSyncJobStatus.IN_PROGRESS] } },
+                                                    { createdAt: { gt: timeoutDate } },
+                                                ]
+                                            },
+                                            // Don't schedule if there are recent failed jobs (within the threshold date).
+                                            {
+                                                AND: [
+                                                    { status: ConnectionSyncJobStatus.FAILED },
+                                                    { completedAt: { gt: thresholdDate } },
+                                                ]
+                                            }
+                                        ]
+                                    }
+                                }
+                            }
                         }
                     ]
                 }
             });
-            for (const connection of connections) {
-                await this.scheduleConnectionSync(connection);
+
+            if (connections.length > 0) {
+                await this.createJobs(connections);
             }
         }, this.settings.resyncConnectionPollingIntervalMs);
+
+        this.worker.run();
     }
 
-    private async runSyncJob(job: Job<JobPayload>): Promise<JobResult> {
-        const { config, orgId, connectionName } = job.data;
+
+    public async createJobs(connections: Connection[]) {
+        const jobs = await this.db.connectionSyncJob.createManyAndReturn({
+            data: connections.map(connection => ({
+                connectionId: connection.id,
+            })),
+            include: {
+                connection: true,
+            }
+        });
+
+        for (const job of jobs) {
+            await this.queue.add({
+                groupId: `connection:${job.connectionId}`,
+                data: {
+                    jobId: job.id,
+                    connectionId: job.connectionId,
+                    connectionName: job.connection.name,
+                    orgId: job.connection.orgId,
+                },
+                jobId: job.id,
+            });
+
+            this.promClient.pendingConnectionSyncJobs.inc({ connection: job.connection.name });
+        }
+
+        return jobs.map(job => job.id);
+    }
+
+    private async runJob(job: ReservedJob<JobPayload>): Promise<JobResult> {
+        const { jobId, connectionName } = job.data;
+        const logger = createJobLogger(jobId);
+        logger.info(`Running connection sync job ${jobId} for connection ${connectionName} (id: ${job.data.connectionId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`);
+
+        this.promClient.pendingConnectionSyncJobs.dec({ connection: connectionName });
+        this.promClient.activeConnectionSyncJobs.inc({ connection: connectionName });
+
         // @note: We aren't actually doing anything with this atm.
         const abortController = new AbortController();
 
-        const connection = await this.db.connection.findUnique({
+        const { connection: { config: rawConnectionConfig, orgId } } = await this.db.connectionSyncJob.update({
             where: {
-                id: job.data.connectionId,
-            },
-        });
-
-        if (!connection) {
-            const e = new BackendException(BackendError.CONNECTION_SYNC_CONNECTION_NOT_FOUND, {
-                message: `Connection ${job.data.connectionId} not found`,
-            });
-            Sentry.captureException(e);
-            throw e;
-        }
-
-        // Reset the syncStatusMetadata to an empty object at the start of the sync job
-        await this.db.connection.update({
-            where: {
-                id: job.data.connectionId,
+                id: jobId,
             },
             data: {
-                syncStatus: ConnectionSyncStatus.SYNCING,
-                syncStatusMetadata: {}
-            }
-        })
-
-
-        let result: {
-            repoData: RepoData[],
-            notFound: {
-                users: string[],
-                orgs: string[],
-                repos: string[],
-            }
-        } = {
-            repoData: [],
-            notFound: {
-                users: [],
-                orgs: [],
-                repos: [],
-            }
-        };
-
-        try {
-            result = await (async () => {
-                switch (config.type) {
-                    case 'github': {
-                        return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController);
-                    }
-                    case 'gitlab': {
-                        return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db);
-                    }
-                    case 'gitea': {
-                        return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db);
-                    }
-                    case 'gerrit': {
-                        return await compileGerritConfig(config, job.data.connectionId, orgId);
-                    }
-                    case 'bitbucket': {
-                        return await compileBitbucketConfig(config, job.data.connectionId, orgId, this.db);
-                    }
-                    case 'azuredevops': {
-                        return await compileAzureDevOpsConfig(config, job.data.connectionId, orgId, this.db, abortController);
-                    }
-                    case 'git': {
-                        return await compileGenericGitHostConfig(config, job.data.connectionId, orgId);
+                status: ConnectionSyncJobStatus.IN_PROGRESS,
+            },
+            select: {
+                connection: {
+                    select: {
+                        config: true,
+                        orgId: true,
                     }
                 }
-            })();
-        } catch (err) {
-            this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId} (${connectionName}): ${err}`);
-            Sentry.captureException(err);
+            },
+        });
 
-            if (err instanceof BackendException) {
-                throw err;
-            } else {
-                throw new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
-                    message: `Failed to compile repo data for connection ${job.data.connectionId}`,
-                });
+        const config = rawConnectionConfig as unknown as ConnectionConfig;
+
+        const result = await (async () => {
+            switch (config.type) {
+                case 'github': {
+                    return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController);
+                }
+                case 'gitlab': {
+                    return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db);
+                }
+                case 'gitea': {
+                    return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db);
+                }
+                case 'gerrit': {
+                    return await compileGerritConfig(config, job.data.connectionId, orgId);
+                }
+                case 'bitbucket': {
+                    return await compileBitbucketConfig(config, job.data.connectionId, orgId, this.db);
+                }
+                case 'azuredevops': {
+                    return await compileAzureDevOpsConfig(config, job.data.connectionId, orgId, this.db);
+                }
+                case 'git': {
+                    return await compileGenericGitHostConfig(config, job.data.connectionId, orgId);
+                }
             }
-        }
+        })();
        
-        let { repoData, notFound } = result;
+        let { repoData, warnings } = result;
 
-        // Push the information regarding not found users, orgs, and repos to the connection's syncStatusMetadata. Note that 
-        // this won't be overwritten even if the connection job fails
-        await this.db.connection.update({
+        await this.db.connectionSyncJob.update({
             where: {
-                id: job.data.connectionId,
+                id: jobId,
             },
             data: {
-                syncStatusMetadata: { notFound }
-            }
+                warningMessages: warnings,
+            },
         });
 
+
         // Filter out any duplicates by external_id and external_codeHostUrl.
         repoData = repoData.filter((repo, index, self) => {
             return index === self.findIndex(r =>
@@ -233,7 +240,7 @@ export class ConnectionManager {
                 }
             });
             const deleteDuration = performance.now() - deleteStart;
-            this.logger.info(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`);
+            logger.info(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`);
 
             const totalUpsertStart = performance.now();
             for (const repo of repoData) {
@@ -250,10 +257,10 @@ export class ConnectionManager {
                     create: repo,
                 })
                 const upsertDuration = performance.now() - upsertStart;
-                this.logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`);
+                logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`);
             }
             const totalUpsertDuration = performance.now() - totalUpsertStart;
-            this.logger.info(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`);
+            logger.info(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`);
         }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS });
 
         return {
@@ -262,106 +269,124 @@ export class ConnectionManager {
     }
 
 
-    private async onSyncJobCompleted(job: Job<JobPayload>, result: JobResult) {
-        this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`);
-        const { connectionId, orgId } = job.data;
+    private onJobCompleted = async (job: Job<JobPayload>) =>
+        groupmqLifecycleExceptionWrapper('onJobCompleted', logger, async () => {
+            const logger = createJobLogger(job.id);
+            const { connectionId, connectionName, orgId } = job.data;
 
-        let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
-            where: { id: connectionId },
-            select: { syncStatusMetadata: true }
-        }))?.syncStatusMetadata as Record<string, unknown> ?? {};
-        const { notFound } = syncStatusMetadata as {
-            notFound: {
-                users: string[],
-                orgs: string[],
-                repos: string[],
-            }
-        };
-
-        await this.db.connection.update({
-            where: {
-                id: connectionId,
-            },
-            data: {
-                syncStatus:
-                    notFound.users.length > 0 ||
-                        notFound.orgs.length > 0 ||
-                        notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
-                syncedAt: new Date()
-            }
-        });
-
-        // After a connection has synced, we need to re-sync the org's search contexts as
-        // there may be new repos that match the search context's include/exclude patterns.
-        if (env.CONFIG_PATH) {
-            try {
-                const config = await loadConfig(env.CONFIG_PATH);
-
-                await syncSearchContexts({
-                    db: this.db,
-                    orgId,
-                    contexts: config.contexts,
-                });
-            } catch (err) {
-                this.logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`);
-                Sentry.captureException(err);
-            }
-        }
-
-
-        captureEvent('backend_connection_sync_job_completed', {
-            connectionId: connectionId,
-            repoCount: result.repoCount,
-        });
-    }
-
-    private async onSyncJobFailed(job: Job<JobPayload> | undefined, err: unknown) {
-        this.logger.info(`Connection sync job for connection ${job?.data.connectionName} (id: ${job?.data.connectionId}, jobId: ${job?.id}) failed with error: ${err}`);
-        Sentry.captureException(err, {
-            tags: {
-                connectionid: job?.data.connectionId,
-                jobId: job?.id,
-                queue: QUEUE_NAME,
-            }
-        });
-
-        if (job) {
-            const { connectionId } = job.data;
-
-            captureEvent('backend_connection_sync_job_failed', {
-                connectionId: connectionId,
-                error: err instanceof BackendException ? err.code : 'UNKNOWN',
-            });
-
-            // We may have pushed some metadata during the execution of the job, so we make sure to not overwrite the metadata here
-            let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
-                where: { id: connectionId },
-                select: { syncStatusMetadata: true }
-            }))?.syncStatusMetadata as Record<string, unknown> ?? {};
-
-            if (err instanceof BackendException) {
-                syncStatusMetadata = {
-                    ...syncStatusMetadata,
-                    error: err.code,
-                    ...err.metadata,
-                }
-            } else {
-                syncStatusMetadata = {
-                    ...syncStatusMetadata,
-                    error: 'UNKNOWN',
-                }
-            }
-
-            await this.db.connection.update({
+            await this.db.connectionSyncJob.update({
                 where: {
-                    id: connectionId,
+                    id: job.id,
                 },
                 data: {
-                    syncStatus: ConnectionSyncStatus.FAILED,
-                    syncStatusMetadata: syncStatusMetadata as Prisma.InputJsonValue,
+                    status: ConnectionSyncJobStatus.COMPLETED,
+                    completedAt: new Date(),
+                    connection: {
+                        update: {
+                            syncedAt: new Date(),
+                        }
+                    }
                 }
             });
-        }
+
+            // After a connection has synced, we need to re-sync the org's search contexts as
+            // there may be new repos that match the search context's include/exclude patterns.
+            if (env.CONFIG_PATH) {
+                try {
+                    const config = await loadConfig(env.CONFIG_PATH);
+
+                    await syncSearchContexts({
+                        db: this.db,
+                        orgId,
+                        contexts: config.contexts,
+                    });
+                } catch (err) {
+                    logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`);
+                    Sentry.captureException(err);
+                }
+            }
+
+            logger.info(`Connection sync job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) completed`);
+
+            this.promClient.activeConnectionSyncJobs.dec({ connection: connectionName });
+            this.promClient.connectionSyncJobSuccessTotal.inc({ connection: connectionName });
+
+            const result = job.returnvalue as JobResult;
+            captureEvent('backend_connection_sync_job_completed', {
+                connectionId: connectionId,
+                repoCount: result.repoCount,
+            });
+        });
+
+    private onJobFailed = async (job: Job<JobPayload>) =>
+        groupmqLifecycleExceptionWrapper('onJobFailed', logger, async () => {
+            const logger = createJobLogger(job.id);
+
+            const attempt = job.attemptsMade + 1;
+            const wasLastAttempt = attempt >= job.opts.attempts;
+
+            if (wasLastAttempt) {
+                const { connection } = await this.db.connectionSyncJob.update({
+                    where: { id: job.id },
+                    data: {
+                        status: ConnectionSyncJobStatus.FAILED,
+                        completedAt: new Date(),
+                        errorMessage: job.failedReason,
+                    },
+                    select: {
+                        connection: true,
+                    }
+                });
+
+                this.promClient.activeConnectionSyncJobs.dec({ connection: connection.name });
+                this.promClient.connectionSyncJobFailTotal.inc({ connection: connection.name });
+
+                logger.error(`Failed job ${job.id} for connection ${connection.name} (id: ${connection.id}). Attempt ${attempt} / ${job.opts.attempts}. Failing job.`);
+            } else {
+                const connection = await this.db.connection.findUniqueOrThrow({
+                    where: { id: job.data.connectionId },
+                });
+
+                this.promClient.connectionSyncJobReattemptsTotal.inc({ connection: connection.name });
+
+                logger.warn(`Failed job ${job.id} for connection ${connection.name} (id: ${connection.id}). Attempt ${attempt} / ${job.opts.attempts}. Retrying.`);
+            }
+
+            captureEvent('backend_connection_sync_job_failed', {
+                connectionId: job.data.connectionId,
+                error: job.failedReason,
+            });
+        });
+
+    private onJobStalled = async (jobId: string) =>
+        groupmqLifecycleExceptionWrapper('onJobStalled', logger, async () => {
+            const logger = createJobLogger(jobId);
+            const { connection } = await this.db.connectionSyncJob.update({
+                where: { id: jobId },
+                data: {
+                    status: ConnectionSyncJobStatus.FAILED,
+                    completedAt: new Date(),
+                    errorMessage: 'Job stalled',
+                },
+                select: {
+                    connection: true,
+                }
+            });
+
+            this.promClient.activeConnectionSyncJobs.dec({ connection: connection.name });
+            this.promClient.connectionSyncJobFailTotal.inc({ connection: connection.name });
+
+            logger.error(`Job ${jobId} stalled for connection ${connection.name} (id: ${connection.id})`);
+
+            captureEvent('backend_connection_sync_job_failed', {
+                connectionId: connection.id,
+                error: 'Job stalled',
+            });
+        });
+
+    private async onWorkerError(error: Error) {
+        Sentry.captureException(error);
+        logger.error(`Connection syncer worker error.`, error);
     }
 
     public async dispose() {

packages/backend/src/connectionUtils.ts

@@ -5,21 +5,21 @@ type ValidResult<T> = {
     data: T[];
 };
 
-type NotFoundResult = {
-    type: 'notFound';
-    value: string;
+type WarningResult = {
+    type: 'warning';
+    warning: string;
 };
 
-type CustomResult<T> = ValidResult<T> | NotFoundResult;
+type CustomResult<T> = ValidResult<T> | WarningResult;
 
 export function processPromiseResults<T>(
     results: PromiseSettledResult<CustomResult<T>>[],
 ): {
     validItems: T[];
-    notFoundItems: string[];
+    warnings: string[];
 } {
     const validItems: T[] = [];
-    const notFoundItems: string[] = [];
+    const warnings: string[] = [];
 
     results.forEach(result => {
         if (result.status === 'fulfilled') {
@@ -27,14 +27,14 @@ export function processPromiseResults<T>(
             if (value.type === 'valid') {
                 validItems.push(...value.data);
             } else {
-                notFoundItems.push(value.value);
+                warnings.push(value.warning);
             }
         }
     });
 
     return {
         validItems,
-        notFoundItems,
+        warnings,
     };
 }
 

packages/backend/src/constants.ts

@@ -1,6 +1,8 @@
 import { env } from "./env.js";
 import path from "path";
 
+export const SINGLE_TENANT_ORG_ID = 1;
+
 export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
     'github',
 ];

packages/backend/src/ee/githubAppManager.ts

@@ -1,10 +1,10 @@
-import { GithubAppConfig, SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
 import { loadConfig } from "@sourcebot/shared";
 import { env } from "../env.js";
 import { createLogger } from "@sourcebot/logger";
-import { getTokenFromConfig } from "../utils.js";
+import { getTokenFromConfig } from "@sourcebot/crypto";
 import { PrismaClient } from "@sourcebot/db";
 import { App } from "@octokit/app";
+import { GitHubAppConfig } from "@sourcebot/schemas/v3/index.type";
 
 const logger = createLogger('githubAppManager');
 const GITHUB_DEFAULT_DEPLOYMENT_HOSTNAME = 'github.com';
@@ -53,7 +53,7 @@ export class GithubAppManager {
             return;
         }
 
-        const githubApps = config.apps.filter(app => app.type === 'githubApp') as GithubAppConfig[];
+        const githubApps = config.apps.filter(app => app.type === 'githubApp') as GitHubAppConfig[];
         logger.info(`Found ${githubApps.length} GitHub apps in config`);
 
         for (const app of githubApps) {
@@ -62,7 +62,7 @@ export class GithubAppManager {
             // @todo: we should move SINGLE_TENANT_ORG_ID to shared package or just remove the need to pass this in 
             // when resolving tokens
             const SINGLE_TENANT_ORG_ID = 1;
-            const privateKey = await getTokenFromConfig(app.privateKey, SINGLE_TENANT_ORG_ID, this.db!);
+            const privateKey = await getTokenFromConfig(app.privateKey, SINGLE_TENANT_ORG_ID, this.db);
 
             const octokitApp = new App({
                 appId: Number(app.id),

packages/backend/src/ee/syncSearchContexts.ts

@@ -1,8 +1,7 @@
 import micromatch from "micromatch";
 import { createLogger } from "@sourcebot/logger";
 import { PrismaClient } from "@sourcebot/db";
-import { getPlan, hasEntitlement } from "../entitlements.js";
-import { SOURCEBOT_SUPPORT_EMAIL } from "../constants.js";
+import { getPlan, hasEntitlement, SOURCEBOT_SUPPORT_EMAIL } from "@sourcebot/shared";
 import { SearchContext } from "@sourcebot/schemas/v3/index.type";
 
 const logger = createLogger('sync-search-contexts');

packages/backend/src/env.ts

@@ -47,7 +47,7 @@ export const env = createEnv({
         DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'),
 
         DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
-        CONFIG_PATH: z.string().optional(),
+        CONFIG_PATH: z.string(),
 
         CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000),
         REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60),

packages/backend/src/gerrit.ts

@@ -37,7 +37,6 @@ const logger = createLogger('gerrit');
 
 export const getGerritReposFromConfig = async (config: GerritConnectionConfig): Promise<GerritProject[]> => {
    const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
-   const hostname = new URL(config.url).hostname;
 
    let { durationMs, data: projects } = await measure(async () => {
       try {

packages/backend/src/git.ts

@@ -35,6 +35,11 @@ const createGitClientForPath = (path: string, onProgress?: onProgressFn, signal?
              * parent directory.
              */
             GIT_CEILING_DIRECTORIES: parentPath,
+            /**
+             * Disable git credential prompts. This ensures that git operations will fail
+             * immediately if credentials are not available, rather than prompting for input.
+             */
+            GIT_TERMINAL_PROMPT: '0',
         })
         .cwd({
             path,

packages/backend/src/gitea.ts

@@ -1,6 +1,6 @@
 import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
 import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type';
-import { getTokenFromConfig, measure } from './utils.js';
+import { measure } from './utils.js';
 import fetch from 'cross-fetch';
 import { createLogger } from '@sourcebot/logger';
 import micromatch from 'micromatch';
@@ -8,6 +8,7 @@ import { PrismaClient } from '@sourcebot/db';
 import { processPromiseResults, throwIfAnyFailed } from './connectionUtils.js';
 import * as Sentry from "@sentry/node";
 import { env } from './env.js';
+import { getTokenFromConfig } from "@sourcebot/crypto";
 
 const logger = createLogger('gitea');
 const GITEA_CLOUD_HOSTNAME = "gitea.com";
@@ -18,7 +19,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
         GITEA_CLOUD_HOSTNAME;
 
     const token = config.token ?
-        await getTokenFromConfig(config.token, orgId, db, logger) :
+        await getTokenFromConfig(config.token, orgId, db) :
         hostname === GITEA_CLOUD_HOSTNAME ?
         env.FALLBACK_GITEA_CLOUD_TOKEN :
         undefined;
@@ -29,32 +30,24 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
     });
 
     let allRepos: GiteaRepository[] = [];
-    let notFound: {
-        users: string[],
-        orgs: string[],
-        repos: string[],
-    } = {
-        users: [],
-        orgs: [],
-        repos: [],
-    };
+    let allWarnings: string[] = [];
 
     if (config.orgs) {
-        const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, api);
-        allRepos = allRepos.concat(validRepos);
-        notFound.orgs = notFoundOrgs;
+        const { repos, warnings } = await getReposForOrgs(config.orgs, api);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.repos) {
-        const { validRepos, notFoundRepos } = await getRepos(config.repos, api);
-        allRepos = allRepos.concat(validRepos);
-        notFound.repos = notFoundRepos;
+        const { repos, warnings } = await getRepos(config.repos, api);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.users) {
-        const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, api);
-        allRepos = allRepos.concat(validRepos);
-        notFound.users = notFoundUsers;
+        const { repos, warnings } = await getReposOwnedByUsers(config.users, api);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
     
     allRepos = allRepos.filter(repo => repo.full_name !== undefined);
@@ -78,8 +71,8 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
     
     logger.debug(`Found ${repos.length} total repositories.`);
     return {
-        validRepos: repos,
-        notFound,
+        repos,
+        warnings: allWarnings,
     };
 }
 
@@ -145,10 +138,11 @@ const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
             Sentry.captureException(e);
 
             if (e?.status === 404) {
-                logger.error(`User ${user} not found or no access`);
+                const warning = `User ${user} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: user
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw e;
@@ -156,11 +150,11 @@ const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults<GiteaRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<GiteaRepository>(results);
 
     return {
-        validRepos,
-        notFoundUsers,
+        repos,
+        warnings,
     };
 }
 
@@ -185,10 +179,11 @@ const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
             Sentry.captureException(e);
 
             if (e?.status === 404) {
-                logger.error(`Organization ${org} not found or no access`);
+                const warning = `Organization ${org} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: org
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw e;
@@ -196,16 +191,16 @@ const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults<GiteaRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<GiteaRepository>(results);
 
     return {
-        validRepos,
-        notFoundOrgs,
+        repos,
+        warnings,
     };
 }
 
-const getRepos = async <T>(repos: string[], api: Api<T>) => {
-    const results = await Promise.allSettled(repos.map(async (repo) => {
+const getRepos = async <T>(repoList: string[], api: Api<T>) => {
+    const results = await Promise.allSettled(repoList.map(async (repo) => {
         try {
             logger.debug(`Fetching repository info for ${repo}...`);
 
@@ -223,10 +218,11 @@ const getRepos = async <T>(repos: string[], api: Api<T>) => {
             Sentry.captureException(e);
 
             if (e?.status === 404) {
-                logger.error(`Repository ${repo} not found or no access`);
+                const warning = `Repository ${repo} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: repo
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw e;
@@ -234,11 +230,11 @@ const getRepos = async <T>(repos: string[], api: Api<T>) => {
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults<GiteaRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<GiteaRepository>(results);
 
     return {
-        validRepos,
-        notFoundRepos,
+        repos,
+        warnings,
     };
 }
 

packages/backend/src/github.ts

@@ -1,15 +1,15 @@
 import { Octokit } from "@octokit/rest";
-import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
-import { createLogger } from "@sourcebot/logger";
-import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js";
-import micromatch from "micromatch";
-import { PrismaClient } from "@sourcebot/db";
-import { BackendException, BackendError } from "@sourcebot/error";
-import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
 import * as Sentry from "@sentry/node";
-import { env } from "./env.js";
-import { GithubAppManager } from "./ee/githubAppManager.js";
+import { PrismaClient } from "@sourcebot/db";
+import { createLogger } from "@sourcebot/logger";
+import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
 import { hasEntitlement } from "@sourcebot/shared";
+import micromatch from "micromatch";
+import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
+import { GithubAppManager } from "./ee/githubAppManager.js";
+import { env } from "./env.js";
+import { fetchWithRetry, measure } from "./utils.js";
+import { getTokenFromConfig } from "@sourcebot/crypto";
 
 export const GITHUB_CLOUD_HOSTNAME = "github.com";
 const logger = createLogger('github');
@@ -92,13 +92,13 @@ const getOctokitWithGithubApp = async (
     }
 }
 
-export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
+export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal): Promise<{ repos: OctokitRepository[], warnings: string[] }> => {
     const hostname = config.url ?
         new URL(config.url).hostname :
         GITHUB_CLOUD_HOSTNAME;
 
     const token = config.token ?
-        await getTokenFromConfig(config.token, orgId, db, logger) :
+        await getTokenFromConfig(config.token, orgId, db) :
         hostname === GITHUB_CLOUD_HOSTNAME ?
             env.FALLBACK_GITHUB_CLOUD_TOKEN :
             undefined;
@@ -108,57 +108,36 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o
         url: config.url,
     });
 
+
     if (isAuthenticated) {
         try {
             await octokit.rest.users.getAuthenticated();
         } catch (error) {
             Sentry.captureException(error);
-
-            if (isHttpError(error, 401)) {
-                const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
-                    ...(config.token && 'secret' in config.token ? {
-                        secretKey: config.token.secret,
-                    } : {}),
-                });
-                Sentry.captureException(e);
-                throw e;
-            }
-
-            const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, {
-                message: `Failed to authenticate with GitHub`,
-            });
-            Sentry.captureException(e);
-            throw e;
+            logger.error(`Failed to authenticate with GitHub`, error);
+            throw error;
         }
     }
 
     let allRepos: OctokitRepository[] = [];
-    let notFound: {
-        users: string[],
-        orgs: string[],
-        repos: string[],
-    } = {
-        users: [],
-        orgs: [],
-        repos: [],
-    };
+    let allWarnings: string[] = [];
 
     if (config.orgs) {
-        const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, octokit, signal, config.url);
-        allRepos = allRepos.concat(validRepos);
-        notFound.orgs = notFoundOrgs;
+        const { repos, warnings } = await getReposForOrgs(config.orgs, octokit, signal, config.url);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.repos) {
-        const { validRepos, notFoundRepos } = await getRepos(config.repos, octokit, signal, config.url);
-        allRepos = allRepos.concat(validRepos);
-        notFound.repos = notFoundRepos;
+        const { repos, warnings } = await getRepos(config.repos, octokit, signal, config.url);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.users) {
-        const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, octokit, signal, config.url);
-        allRepos = allRepos.concat(validRepos);
-        notFound.users = notFoundUsers;
+        const { repos, warnings } = await getReposOwnedByUsers(config.users, octokit, signal, config.url);
+        allRepos = allRepos.concat(repos);
+        allWarnings = allWarnings.concat(warnings);
     }
 
     let repos = allRepos
@@ -177,8 +156,8 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o
     logger.debug(`Found ${repos.length} total repositories.`);
 
     return {
-        validRepos: repos,
-        notFound,
+        repos,
+        warnings: allWarnings,
     };
 }
 
@@ -256,10 +235,11 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A
             logger.error(`Failed to fetch repositories for user ${user}.`, error);
 
             if (isHttpError(error, 404)) {
-                logger.error(`User ${user} not found or no access`);
+                const warning = `User ${user} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: user
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw error;
@@ -267,18 +247,18 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults<OctokitRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<OctokitRepository>(results);
 
     return {
-        validRepos,
-        notFoundUsers,
+        repos,
+        warnings,
     };
 }
 
 const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal, url?: string) => {
     const results = await Promise.allSettled(orgs.map(async (org) => {
         try {
-            logger.info(`Fetching repository info for org ${org}...`);
+            logger.debug(`Fetching repository info for org ${org}...`);
 
             const octokitToUse = await getOctokitWithGithubApp(octokit, org, url, `org ${org}`);
             const { durationMs, data } = await measure(async () => {
@@ -293,7 +273,7 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi
                 return fetchWithRetry(fetchFn, `org ${org}`, logger);
             });
 
-            logger.info(`Found ${data.length} in org ${org} in ${durationMs}ms.`);
+            logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`);
             return {
                 type: 'valid' as const,
                 data
@@ -303,10 +283,11 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi
             logger.error(`Failed to fetch repositories for org ${org}.`, error);
 
             if (isHttpError(error, 404)) {
-                logger.error(`Organization ${org} not found or no access`);
+                const warning = `Organization ${org} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: org
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw error;
@@ -314,11 +295,11 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults<OctokitRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<OctokitRepository>(results);
 
     return {
-        validRepos,
-        notFoundOrgs,
+        repos,
+        warnings,
     };
 }
 
@@ -326,7 +307,7 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna
     const results = await Promise.allSettled(repoList.map(async (repo) => {
         try {
             const [owner, repoName] = repo.split('/');
-            logger.info(`Fetching repository info for ${repo}...`);
+            logger.debug(`Fetching repository info for ${repo}...`);
 
             const octokitToUse = await getOctokitWithGithubApp(octokit, owner, url, `repo ${repo}`);
             const { durationMs, data: result } = await measure(async () => {
@@ -341,7 +322,7 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna
                 return fetchWithRetry(fetchFn, repo, logger);
             });
 
-            logger.info(`Found info for repository ${repo} in ${durationMs}ms`);
+            logger.debug(`Found info for repository ${repo} in ${durationMs}ms`);
             return {
                 type: 'valid' as const,
                 data: [result.data]
@@ -352,10 +333,11 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna
             logger.error(`Failed to fetch repository ${repo}.`, error);
 
             if (isHttpError(error, 404)) {
-                logger.error(`Repository ${repo} not found or no access`);
+                const warning = `Repository ${repo} not found or no access`;
+                logger.warn(warning);
                 return {
-                    type: 'notFound' as const,
-                    value: repo
+                    type: 'warning' as const,
+                    warning
                 };
             }
             throw error;
@@ -363,11 +345,11 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna
     }));
 
     throwIfAnyFailed(results);
-    const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults<OctokitRepository>(results);
+    const { validItems: repos, warnings } = processPromiseResults<OctokitRepository>(results);
 
     return {
-        validRepos,
-        notFoundRepos,
+        repos,
+        warnings,
     };
 }
 

packages/backend/src/gitlab.ts

@@ -2,11 +2,12 @@ import { Gitlab, ProjectSchema } from "@gitbeaker/rest";
 import micromatch from "micromatch";
 import { createLogger } from "@sourcebot/logger";
 import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"
-import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js";
+import { measure, fetchWithRetry } from "./utils.js";
 import { PrismaClient } from "@sourcebot/db";
 import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
 import * as Sentry from "@sentry/node";
 import { env } from "./env.js";
+import { getTokenFromConfig } from "@sourcebot/crypto";
 
 const logger = createLogger('gitlab');
 export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";
@@ -17,7 +18,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
         GITLAB_CLOUD_HOSTNAME;
 
     const token = config.token ?
-        await getTokenFromConfig(config.token, orgId, db, logger) :
+        await getTokenFromConfig(config.token, orgId, db) :
         hostname === GITLAB_CLOUD_HOSTNAME ?
         env.FALLBACK_GITLAB_CLOUD_TOKEN :
         undefined;
@@ -33,15 +34,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
     });
 
     let allRepos: ProjectSchema[] = [];
-    let notFound: {
-        orgs: string[],
-        users: string[],
-        repos: string[],
-    } = {
-        orgs: [],
-        users: [],
-        repos: [],
-    };
+    let allWarnings: string[] = [];
 
     if (config.all === true) {
         if (hostname !== GITLAB_CLOUD_HOSTNAME) {
@@ -61,7 +54,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
                 throw e;
             }
         } else {
-            logger.warn(`Ignoring option all:true in config : host is ${GITLAB_CLOUD_HOSTNAME}`);
+            const warning = `Ignoring option all:true in config : host is ${GITLAB_CLOUD_HOSTNAME}`;
+            logger.warn(warning);
+            allWarnings = allWarnings.concat(warning);
         }
     }
 
@@ -87,10 +82,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
 
                 const status = e?.cause?.response?.status;
                 if (status === 404) {
-                    logger.error(`Group ${group} not found or no access`);
+                    const warning = `Group ${group} not found or no access`;
+                    logger.warn(warning);
                     return {
-                        type: 'notFound' as const,
-                        value: group
+                        type: 'warning' as const,
+                        warning
                     };
                 }
                 throw e;
@@ -98,9 +94,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
         }));
 
         throwIfAnyFailed(results);
-        const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results);
+        const { validItems: validRepos, warnings } = processPromiseResults(results);
         allRepos = allRepos.concat(validRepos);
-        notFound.orgs = notFoundOrgs;
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.users) {
@@ -124,10 +120,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
 
                 const status = e?.cause?.response?.status;
                 if (status === 404) {
-                    logger.error(`User ${user} not found or no access`);
+                    const warning = `User ${user} not found or no access`;
+                    logger.warn(warning);
                     return {
-                        type: 'notFound' as const,
-                        value: user
+                        type: 'warning' as const,
+                        warning
                     };
                 }
                 throw e;
@@ -135,9 +132,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
         }));
 
         throwIfAnyFailed(results);
-        const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results);
+        const { validItems: validRepos, warnings } = processPromiseResults(results);
         allRepos = allRepos.concat(validRepos);
-        notFound.users = notFoundUsers;
+        allWarnings = allWarnings.concat(warnings);
     }
 
     if (config.projects) {
@@ -160,10 +157,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
                 const status = e?.cause?.response?.status;
 
                 if (status === 404) {
-                    logger.error(`Project ${project} not found or no access`);
+                    const warning = `Project ${project} not found or no access`;
+                    logger.warn(warning);
                     return {
-                        type: 'notFound' as const,
-                        value: project
+                        type: 'warning' as const,
+                        warning
                     };
                 }
                 throw e;
@@ -171,9 +169,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
         }));
 
         throwIfAnyFailed(results);
-        const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results);
+        const { validItems: validRepos, warnings } = processPromiseResults(results);
         allRepos = allRepos.concat(validRepos);
-        notFound.repos = notFoundRepos;
+        allWarnings = allWarnings.concat(warnings);
     }
 
     let repos = allRepos
@@ -192,8 +190,8 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
     logger.debug(`Found ${repos.length} total repositories.`);
 
     return {
-        validRepos: repos,
-        notFound,
+        repos,
+        warnings: allWarnings,
     };
 }
 

packages/backend/src/index.ts

@@ -6,14 +6,15 @@ import { getConfigSettings, hasEntitlement } from '@sourcebot/shared';
 import { existsSync } from 'fs';
 import { mkdir } from 'fs/promises';
 import { Redis } from 'ioredis';
+import { ConfigManager } from "./configManager.js";
 import { ConnectionManager } from './connectionManager.js';
 import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
+import { GithubAppManager } from "./ee/githubAppManager.js";
 import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
 import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
-import { GithubAppManager } from "./ee/githubAppManager.js";
 import { env } from "./env.js";
-import { RepoIndexManager } from "./repoIndexManager.js";
 import { PromClient } from './promClient.js';
+import { RepoIndexManager } from "./repoIndexManager.js";
 
 
 const logger = createLogger('backend-entrypoint');
@@ -49,10 +50,11 @@ if (hasEntitlement('github-app')) {
     await GithubAppManager.getInstance().init(prisma);
 }
 
-const connectionManager = new ConnectionManager(prisma, settings, redis);
+const connectionManager = new ConnectionManager(prisma, settings, redis, promClient);
 const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis);
 const userPermissionSyncer = new UserPermissionSyncer(prisma, settings, redis);
 const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient);
+const configManager = new ConfigManager(prisma, connectionManager, env.CONFIG_PATH);
 
 connectionManager.startScheduler();
 repoIndexManager.startScheduler();
@@ -66,6 +68,8 @@ else if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && hasEntitlement(
     userPermissionSyncer.startScheduler();
 }
 
+logger.info('Worker started.');
+
 const cleanup = async (signal: string) => {
     logger.info(`Received ${signal}, cleaning up...`);
 
@@ -79,6 +83,7 @@ const cleanup = async (signal: string) => {
                 repoPermissionSyncer.dispose(),
                 userPermissionSyncer.dispose(),
                 promClient.dispose(),
+                configManager.dispose(),
             ]),
             new Promise((_, reject) =>
                 setTimeout(() => reject(new Error('Shutdown timeout')), shutdownTimeout)

packages/backend/src/promClient.ts

@@ -16,6 +16,12 @@ export class PromClient {
     public repoIndexJobFailTotal: Counter<string>;
     public repoIndexJobSuccessTotal: Counter<string>;
 
+    public activeConnectionSyncJobs: Gauge<string>;
+    public pendingConnectionSyncJobs: Gauge<string>;
+    public connectionSyncJobReattemptsTotal: Counter<string>;
+    public connectionSyncJobFailTotal: Counter<string>;
+    public connectionSyncJobSuccessTotal: Counter<string>;
+
     public readonly PORT = 3060;
 
     constructor() {
@@ -56,6 +62,41 @@ export class PromClient {
         });
         this.registry.registerMetric(this.repoIndexJobSuccessTotal);
 
+        this.activeConnectionSyncJobs = new Gauge({
+            name: 'active_connection_sync_jobs',
+            help: 'The number of connection sync jobs in progress',
+            labelNames: ['connection'],
+        });
+        this.registry.registerMetric(this.activeConnectionSyncJobs);
+
+        this.pendingConnectionSyncJobs = new Gauge({
+            name: 'pending_connection_sync_jobs',
+            help: 'The number of connection sync jobs waiting in queue',
+            labelNames: ['connection'],
+        });
+        this.registry.registerMetric(this.pendingConnectionSyncJobs);
+
+        this.connectionSyncJobReattemptsTotal = new Counter({
+            name: 'connection_sync_job_reattempts',
+            help: 'The number of connection sync job reattempts',
+            labelNames: ['connection'],
+        });
+        this.registry.registerMetric(this.connectionSyncJobReattemptsTotal);
+
+        this.connectionSyncJobFailTotal = new Counter({
+            name: 'connection_sync_job_fails',
+            help: 'The number of connection sync job fails',
+            labelNames: ['connection'],
+        });
+        this.registry.registerMetric(this.connectionSyncJobFailTotal);
+
+        this.connectionSyncJobSuccessTotal = new Counter({
+            name: 'connection_sync_job_successes',
+            help: 'The number of connection sync job successes',
+            labelNames: ['connection'],
+        });
+        this.registry.registerMetric(this.connectionSyncJobSuccessTotal);
+
         client.collectDefaultMetrics({
             register: this.registry,
         });

packages/backend/src/repoCompileUtils.ts

@@ -24,22 +24,20 @@ export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
 
 const logger = createLogger('repo-compile-utils');
 
+type CompileResult = {
+    repoData: RepoData[],
+    warnings: string[],
+}
+
 export const compileGithubConfig = async (
     config: GithubConnectionConfig,
     connectionId: number,
     orgId: number,
     db: PrismaClient,
-    abortController: AbortController): Promise<{
-        repoData: RepoData[],
-        notFound: {
-            users: string[],
-            orgs: string[],
-            repos: string[],
-        }
-    }> => {
+    abortController: AbortController): Promise<CompileResult> => {
     const gitHubReposResult = await getGitHubReposFromConfig(config, orgId, db, abortController.signal);
-    const gitHubRepos = gitHubReposResult.validRepos;
-    const notFound = gitHubReposResult.notFound;
+    const gitHubRepos = gitHubReposResult.repos;
+    const warnings = gitHubReposResult.warnings;
 
     const hostUrl = config.url ?? 'https://github.com';
     const repoNameRoot = new URL(hostUrl)
@@ -100,7 +98,7 @@ export const compileGithubConfig = async (
 
     return {
         repoData: repos,
-        notFound,
+        warnings,
     };
 }
 
@@ -108,11 +106,11 @@ export const compileGitlabConfig = async (
     config: GitlabConnectionConfig,
     connectionId: number,
     orgId: number,
-    db: PrismaClient) => {
+    db: PrismaClient): Promise<CompileResult> => {
 
     const gitlabReposResult = await getGitLabReposFromConfig(config, orgId, db);
-    const gitlabRepos = gitlabReposResult.validRepos;
-    const notFound = gitlabReposResult.notFound;
+    const gitlabRepos = gitlabReposResult.repos;
+    const warnings = gitlabReposResult.warnings;
 
     const hostUrl = config.url ?? 'https://gitlab.com';
     const repoNameRoot = new URL(hostUrl)
@@ -177,7 +175,7 @@ export const compileGitlabConfig = async (
 
     return {
         repoData: repos,
-        notFound,
+        warnings,
     };
 }
 
@@ -185,11 +183,11 @@ export const compileGiteaConfig = async (
     config: GiteaConnectionConfig,
     connectionId: number,
     orgId: number,
-    db: PrismaClient) => {
+    db: PrismaClient): Promise<CompileResult> => {
 
     const giteaReposResult = await getGiteaReposFromConfig(config, orgId, db);
-    const giteaRepos = giteaReposResult.validRepos;
-    const notFound = giteaReposResult.notFound;
+    const giteaRepos = giteaReposResult.repos;
+    const warnings = giteaReposResult.warnings;
 
     const hostUrl = config.url ?? 'https://gitea.com';
     const repoNameRoot = new URL(hostUrl)
@@ -248,14 +246,14 @@ export const compileGiteaConfig = async (
 
     return {
         repoData: repos,
-        notFound,
+        warnings,
     };
 }
 
 export const compileGerritConfig = async (
     config: GerritConnectionConfig,
     connectionId: number,
-    orgId: number) => {
+    orgId: number): Promise<CompileResult> => {
 
     const gerritRepos = await getGerritReposFromConfig(config);
     const hostUrl = config.url;
@@ -329,11 +327,7 @@ export const compileGerritConfig = async (
 
     return {
         repoData: repos,
-        notFound: {
-            users: [],
-            orgs: [],
-            repos: [],
-        }
+        warnings: [],
     };
 }
 
@@ -341,11 +335,11 @@ export const compileBitbucketConfig = async (
     config: BitbucketConnectionConfig,
     connectionId: number,
     orgId: number,
-    db: PrismaClient) => {
+    db: PrismaClient): Promise<CompileResult> => {
 
     const bitbucketReposResult = await getBitbucketReposFromConfig(config, orgId, db);
-    const bitbucketRepos = bitbucketReposResult.validRepos;
-    const notFound = bitbucketReposResult.notFound;
+    const bitbucketRepos = bitbucketReposResult.repos;
+    const warnings = bitbucketReposResult.warnings;
 
     const hostUrl = config.url ?? 'https://bitbucket.org';
     const repoNameRoot = new URL(hostUrl)
@@ -450,7 +444,7 @@ export const compileBitbucketConfig = async (
 
     return {
         repoData: repos,
-        notFound,
+        warnings,
     };
 }
 
@@ -458,7 +452,7 @@ export const compileGenericGitHostConfig = async (
     config: GenericGitHostConnectionConfig,
     connectionId: number,
     orgId: number,
-) => {
+): Promise<CompileResult> => {
     const configUrl = new URL(config.url);
     if (configUrl.protocol === 'file:') {
         return compileGenericGitHostConfig_file(config, orgId, connectionId);
@@ -476,7 +470,7 @@ export const compileGenericGitHostConfig_file = async (
     config: GenericGitHostConnectionConfig,
     orgId: number,
     connectionId: number,
-) => {
+): Promise<CompileResult> => {
     const configUrl = new URL(config.url);
     assert(configUrl.protocol === 'file:', 'config.url must be a file:// URL');
 
@@ -486,30 +480,24 @@ export const compileGenericGitHostConfig_file = async (
     });
 
     const repos: RepoData[] = [];
-    const notFound: {
-        users: string[],
-        orgs: string[],
-        repos: string[],
-    } = {
-        users: [],
-        orgs: [],
-        repos: [],
-    };
+    const warnings: string[] = [];
     
     await Promise.all(repoPaths.map(async (repoPath) => {
         const isGitRepo = await isPathAValidGitRepoRoot({
             path: repoPath,
         });
         if (!isGitRepo) {
-            logger.warn(`Skipping ${repoPath} - not a git repository.`);
-            notFound.repos.push(repoPath);
+            const warning = `Skipping ${repoPath} - not a git repository.`;
+            logger.warn(warning);
+            warnings.push(warning);
             return;
         }
 
         const origin = await getOriginUrl(repoPath);
         if (!origin) {
-            logger.warn(`Skipping ${repoPath} - remote.origin.url not found in git config.`);
-            notFound.repos.push(repoPath);
+            const warning = `Skipping ${repoPath} - remote.origin.url not found in git config.`;
+            logger.warn(warning);
+            warnings.push(warning);
             return;
         }
 
@@ -552,7 +540,7 @@ export const compileGenericGitHostConfig_file = async (
 
     return {
         repoData: repos,
-        notFound,
+        warnings,
     }
 }
 
@@ -561,27 +549,21 @@ export const compileGenericGitHostConfig_url = async (
     config: GenericGitHostConnectionConfig,
     orgId: number,
     connectionId: number,
-) => {
+): Promise<CompileResult> => {
     const remoteUrl = new URL(config.url);
     assert(remoteUrl.protocol === 'http:' || remoteUrl.protocol === 'https:', 'config.url must be a http:// or https:// URL');
 
-    const notFound: {
-        users: string[],
-        orgs: string[],
-        repos: string[],
-    } = {
-        users: [],
-        orgs: [],
-        repos: [],
-    };
+    const warnings: string[] = [];
 
     // Validate that we are dealing with a valid git repo.
     const isGitRepo = await isUrlAValidGitRepo(remoteUrl.toString());
     if (!isGitRepo) {
-        notFound.repos.push(remoteUrl.toString());
+        const warning = `Skipping ${remoteUrl.toString()} - not a git repository.`;
+        logger.warn(warning);
+        warnings.push(warning);
         return {
             repoData: [],
-            notFound,
+            warnings,
         }
     }
 
@@ -616,7 +598,7 @@ export const compileGenericGitHostConfig_url = async (
 
     return {
         repoData: [repo],
-        notFound,
+        warnings,
     }
 }
 
@@ -624,12 +606,11 @@ export const compileAzureDevOpsConfig = async (
     config: AzureDevOpsConnectionConfig,
     connectionId: number,
     orgId: number,
-    db: PrismaClient,
-    abortController: AbortController) => {
+    db: PrismaClient): Promise<CompileResult> => {
 
     const azureDevOpsReposResult = await getAzureDevOpsReposFromConfig(config, orgId, db);
-    const azureDevOpsRepos = azureDevOpsReposResult.validRepos;
-    const notFound = azureDevOpsReposResult.notFound;
+    const azureDevOpsRepos = azureDevOpsReposResult.repos;
+    const warnings = azureDevOpsReposResult.warnings;
 
     const hostUrl = config.url ?? 'https://dev.azure.com';
     const repoNameRoot = new URL(hostUrl)
@@ -699,6 +680,6 @@ export const compileAzureDevOpsConfig = async (
 
     return {
         repoData: repos,
-        notFound,
+        warnings,
     };
 }

packages/backend/src/repoIndexManager.ts

@@ -149,7 +149,8 @@ export class RepoIndexManager {
     }
 
     private async scheduleCleanupJobs() {
-        const thresholdDate = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs);
+        const gcGracePeriodMs = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs);
+        const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs);
 
         const reposToCleanup = await this.db.repo.findMany({
             where: {
@@ -158,9 +159,8 @@ export class RepoIndexManager {
                 },
                 OR: [
                     { indexedAt: null },
-                    { indexedAt: { lt: thresholdDate } },
+                    { indexedAt: { lt: gcGracePeriodMs } },
                 ],
-                // Don't schedule if there are active jobs that were created within the threshold date.
                 NOT: {
                     jobs: {
                         some: {
@@ -178,7 +178,7 @@ export class RepoIndexManager {
                                 },
                                 {
                                     createdAt: {
-                                        gt: thresholdDate,
+                                        gt: timeoutDate,
                                     }
                                 }
                             ]

packages/backend/src/utils.ts

@@ -2,8 +2,7 @@ import { Logger } from "winston";
 import { RepoAuthCredentials, RepoWithConnections } from "./types.js";
 import path from 'path';
 import { PrismaClient, Repo } from "@sourcebot/db";
-import { getTokenFromConfig as getTokenFromConfigBase } from "@sourcebot/crypto";
-import { BackendException, BackendError } from "@sourcebot/error";
+import { getTokenFromConfig } from "@sourcebot/crypto";
 import * as Sentry from "@sentry/node";
 import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
 import { GithubAppManager } from "./ee/githubAppManager.js";
@@ -24,22 +23,6 @@ export const marshalBool = (value?: boolean) => {
     return !!value ? '1' : '0';
 }
 
-export const getTokenFromConfig = async (token: any, orgId: number, db: PrismaClient, logger?: Logger) => {
-    try {
-        return await getTokenFromConfigBase(token, orgId, db);
-    } catch (error: unknown) {
-        if (error instanceof Error) {
-            const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, {
-                message: error.message,
-            });
-            Sentry.captureException(e);
-            logger?.error(error.message);
-            throw e;
-        }
-        throw error;
-    }
-};
-
 export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => {
     let absolutePath = localPath;
     if (!path.isAbsolute(absolutePath)) {
@@ -156,7 +139,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P
         if (connection.connectionType === 'github') {
             const config = connection.config as unknown as GithubConnectionConfig;
             if (config.token) {
-                const token = await getTokenFromConfig(config.token, connection.orgId, db, logger);
+                const token = await getTokenFromConfig(config.token, connection.orgId, db);
                 return {
                     hostUrl: config.url,
                     token,
@@ -171,7 +154,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P
         } else if (connection.connectionType === 'gitlab') {
             const config = connection.config as unknown as GitlabConnectionConfig;
             if (config.token) {
-                const token = await getTokenFromConfig(config.token, connection.orgId, db, logger);
+                const token = await getTokenFromConfig(config.token, connection.orgId, db);
                 return {
                     hostUrl: config.url,
                     token,
@@ -187,7 +170,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P
         } else if (connection.connectionType === 'gitea') {
             const config = connection.config as unknown as GiteaConnectionConfig;
             if (config.token) {
-                const token = await getTokenFromConfig(config.token, connection.orgId, db, logger);
+                const token = await getTokenFromConfig(config.token, connection.orgId, db);
                 return {
                     hostUrl: config.url,
                     token,
@@ -202,7 +185,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P
         } else if (connection.connectionType === 'bitbucket') {
             const config = connection.config as unknown as BitbucketConnectionConfig;
             if (config.token) {
-                const token = await getTokenFromConfig(config.token, connection.orgId, db, logger);
+                const token = await getTokenFromConfig(config.token, connection.orgId, db);
                 const username = config.user ?? 'x-token-auth';
                 return {
                     hostUrl: config.url,
@@ -219,7 +202,7 @@ export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: P
         } else if (connection.connectionType === 'azuredevops') {
             const config = connection.config as unknown as AzureDevOpsConnectionConfig;
             if (config.token) {
-                const token = await getTokenFromConfig(config.token, connection.orgId, db, logger);
+                const token = await getTokenFromConfig(config.token, connection.orgId, db);
 
                 // For ADO server, multiple auth schemes may be supported. If the ADO deployment supports NTLM, the git clone will default
                 // to this over basic auth. As a result, we cannot embed the token in the clone URL and must force basic auth by passing in the token

packages/db/prisma/migrations/20251026194617_add_connection_job_table/migration.sql

@@ -0,0 +1,30 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `syncStatus` on the `Connection` table. All the data in the column will be lost.
+  - You are about to drop the column `syncStatusMetadata` on the `Connection` table. All the data in the column will be lost.
+
+*/
+-- CreateEnum
+CREATE TYPE "ConnectionSyncJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED');
+
+-- AlterTable
+ALTER TABLE "Connection" DROP COLUMN "syncStatus",
+DROP COLUMN "syncStatusMetadata";
+
+-- CreateTable
+CREATE TABLE "ConnectionSyncJob" (
+    "id" TEXT NOT NULL,
+    "status" "ConnectionSyncJobStatus" NOT NULL DEFAULT 'PENDING',
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "completedAt" TIMESTAMP(3),
+    "warningMessages" TEXT[],
+    "errorMessage" TEXT,
+    "connectionId" INTEGER NOT NULL,
+
+    CONSTRAINT "ConnectionSyncJob_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "ConnectionSyncJob" ADD CONSTRAINT "ConnectionSyncJob_connectionId_fkey" FOREIGN KEY ("connectionId") REFERENCES "Connection"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/db/prisma/migrations/20251026194628_ensure_single_tenant_org/migration.sql

@@ -0,0 +1,13 @@
+-- Installs the pgcrypto extension. Required for the gen_random_uuid() function.
+-- @see: https://www.prisma.io/docs/orm/prisma-migrate/workflows/native-database-functions#how-to-install-a-postgresql-extension-as-part-of-a-migration
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+-- Ensure single tenant organization exists
+INSERT INTO "Org" (id, name, domain, "inviteLinkId", "createdAt", "updatedAt")
+VALUES (1, 'default', '~', gen_random_uuid(), NOW(), NOW())
+ON CONFLICT (id) DO NOTHING;
+
+-- Backfill inviteLinkId for any existing orgs that don't have one
+UPDATE "Org"
+SET "inviteLinkId" = gen_random_uuid()
+WHERE "inviteLinkId" IS NULL;

packages/db/prisma/schema.prisma

@@ -132,15 +132,15 @@ model Connection {
   isDeclarative      Boolean              @default(false)
   createdAt          DateTime             @default(now())
   updatedAt          DateTime             @updatedAt
-  /// When the connection was last synced successfully.
-  syncedAt           DateTime?
   repos              RepoToConnection[]
-  syncStatus         ConnectionSyncStatus @default(SYNC_NEEDED)
-  syncStatusMetadata Json?
 
   // The type of connection (e.g., github, gitlab, etc.)
   connectionType String
 
+  syncJobs ConnectionSyncJob[]
+  /// When the connection was last synced successfully.
+  syncedAt           DateTime?
+
   // The organization that owns this connection
   org   Org @relation(fields: [orgId], references: [id], onDelete: Cascade)
   orgId Int
@@ -148,6 +148,27 @@ model Connection {
   @@unique([name, orgId])
 }
 
+enum ConnectionSyncJobStatus {
+  PENDING
+  IN_PROGRESS
+  COMPLETED
+  FAILED
+}
+
+model ConnectionSyncJob {
+  id          String                      @id @default(cuid())
+  status      ConnectionSyncJobStatus @default(PENDING)
+  createdAt   DateTime                    @default(now())
+  updatedAt   DateTime                    @updatedAt
+  completedAt DateTime?
+
+  warningMessages String[]
+  errorMessage String?
+
+  connection   Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade)
+  connectionId Int
+}
+
 model RepoToConnection {
   addedAt DateTime @default(now())
 

packages/schemas/src/v3/app.schema.ts

@@ -2,11 +2,9 @@
 const schema = {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "AppConfig",
-  "oneOf": [
-    {
-      "$schema": "http://json-schema.org/draft-07/schema#",
+  "definitions": {
+    "GitHubAppConfig": {
       "type": "object",
-      "title": "GithubAppConfig",
       "properties": {
         "type": {
           "const": "githubApp",
@@ -60,19 +58,70 @@ const schema = {
       },
       "required": [
         "type",
-        "id"
+        "id",
+        "privateKey"
       ],
-      "oneOf": [
-        {
-          "required": [
-            "privateKey"
+      "additionalProperties": false
+    }
+  },
+  "oneOf": [
+    {
+      "type": "object",
+      "properties": {
+        "type": {
+          "const": "githubApp",
+          "description": "GitHub App Configuration"
+        },
+        "deploymentHostname": {
+          "type": "string",
+          "format": "hostname",
+          "default": "github.com",
+          "description": "The hostname of the GitHub App deployment.",
+          "examples": [
+            "github.com",
+            "github.example.com"
           ]
         },
-        {
-          "required": [
-            "privateKeyPath"
+        "id": {
+          "type": "string",
+          "description": "The ID of the GitHub App."
+        },
+        "privateKey": {
+          "description": "The private key of the GitHub App.",
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "secret": {
+                  "type": "string",
+                  "description": "The name of the secret that contains the token."
+                }
+              },
+              "required": [
+                "secret"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            }
           ]
         }
+      },
+      "required": [
+        "type",
+        "id",
+        "privateKey"
       ],
       "additionalProperties": false
     }

packages/schemas/src/v3/app.type.ts

@@ -1,6 +1,34 @@
 // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
 
-export type AppConfig = GithubAppConfig;
-export type GithubAppConfig = {
-  [k: string]: unknown;
-};
+export type AppConfig = GitHubAppConfig;
+
+export interface GitHubAppConfig {
+  /**
+   * GitHub App Configuration
+   */
+  type: "githubApp";
+  /**
+   * The hostname of the GitHub App deployment.
+   */
+  deploymentHostname?: string;
+  /**
+   * The ID of the GitHub App.
+   */
+  id: string;
+  /**
+   * The private key of the GitHub App.
+   */
+  privateKey:
+    | {
+        /**
+         * The name of the secret that contains the token.
+         */
+        secret: string;
+      }
+    | {
+        /**
+         * The name of the environment variable that contains the token. Only supported in declarative connection configs.
+         */
+        env: string;
+      };
+}

packages/schemas/src/v3/githubApp.schema.ts

@@ -1,75 +0,0 @@
-// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
-const schema = {
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "type": "object",
-  "title": "GithubAppConfig",
-  "properties": {
-    "type": {
-      "const": "githubApp",
-      "description": "GitHub App Configuration"
-    },
-    "deploymentHostname": {
-      "type": "string",
-      "format": "hostname",
-      "default": "github.com",
-      "description": "The hostname of the GitHub App deployment.",
-      "examples": [
-        "github.com",
-        "github.example.com"
-      ]
-    },
-    "id": {
-      "type": "string",
-      "description": "The ID of the GitHub App."
-    },
-    "privateKey": {
-      "description": "The private key of the GitHub App.",
-      "anyOf": [
-        {
-          "type": "object",
-          "properties": {
-            "secret": {
-              "type": "string",
-              "description": "The name of the secret that contains the token."
-            }
-          },
-          "required": [
-            "secret"
-          ],
-          "additionalProperties": false
-        },
-        {
-          "type": "object",
-          "properties": {
-            "env": {
-              "type": "string",
-              "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
-            }
-          },
-          "required": [
-            "env"
-          ],
-          "additionalProperties": false
-        }
-      ]
-    }
-  },
-  "required": [
-    "type",
-    "id"
-  ],
-  "oneOf": [
-    {
-      "required": [
-        "privateKey"
-      ]
-    },
-    {
-      "required": [
-        "privateKeyPath"
-      ]
-    }
-  ],
-  "additionalProperties": false
-} as const;
-export { schema as githubAppSchema };

packages/schemas/src/v3/githubApp.type.ts

@@ -1,34 +0,0 @@
-// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
-
-export type GithubAppConfig = {
-  /**
-   * GitHub App Configuration
-   */
-  type: "githubApp";
-  /**
-   * The hostname of the GitHub App deployment.
-   */
-  deploymentHostname?: string;
-  /**
-   * The ID of the GitHub App.
-   */
-  id: string;
-  /**
-   * The private key of the GitHub App.
-   */
-  privateKey?:
-    | {
-        /**
-         * The name of the secret that contains the token.
-         */
-        secret: string;
-      }
-    | {
-        /**
-         * The name of the environment variable that contains the token. Only supported in declarative connection configs.
-         */
-        env: string;
-      };
-} & {
-  [k: string]: unknown;
-};

packages/schemas/src/v3/index.schema.ts

@@ -4279,11 +4279,9 @@ const schema = {
       "items": {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "title": "AppConfig",
-        "oneOf": [
-          {
-            "$schema": "http://json-schema.org/draft-07/schema#",
+        "definitions": {
+          "GitHubAppConfig": {
             "type": "object",
-            "title": "GithubAppConfig",
             "properties": {
               "type": {
                 "const": "githubApp",
@@ -4337,19 +4335,70 @@ const schema = {
             },
             "required": [
               "type",
-              "id"
+              "id",
+              "privateKey"
             ],
-            "oneOf": [
-              {
-                "required": [
-                  "privateKey"
+            "additionalProperties": false
+          }
+        },
+        "oneOf": [
+          {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "githubApp",
+                "description": "GitHub App Configuration"
+              },
+              "deploymentHostname": {
+                "type": "string",
+                "format": "hostname",
+                "default": "github.com",
+                "description": "The hostname of the GitHub App deployment.",
+                "examples": [
+                  "github.com",
+                  "github.example.com"
                 ]
               },
-              {
-                "required": [
-                  "privateKeyPath"
-                ]
+              "id": {
+                "type": "string",
+                "description": "The ID of the GitHub App."
+              },
+              "privateKey": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "secret": {
+                        "type": "string",
+                        "description": "The name of the secret that contains the token."
+                      }
+                    },
+                    "required": [
+                      "secret"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  }
+                ],
+                "description": "The private key of the GitHub App."
               }
+            },
+            "required": [
+              "type",
+              "id",
+              "privateKey"
             ],
             "additionalProperties": false
           }

packages/schemas/src/v3/index.type.ts

@@ -25,10 +25,7 @@ export type LanguageModel =
   | OpenAICompatibleLanguageModel
   | OpenRouterLanguageModel
   | XaiLanguageModel;
-export type AppConfig = GithubAppConfig;
-export type GithubAppConfig = {
-  [k: string]: unknown;
-};
+export type AppConfig = GitHubAppConfig;
 
 export interface SourcebotConfig {
   $schema?: string;
@@ -1073,3 +1070,33 @@ export interface XaiLanguageModel {
   baseUrl?: string;
   headers?: LanguageModelHeaders;
 }
+export interface GitHubAppConfig {
+  /**
+   * GitHub App Configuration
+   */
+  type: "githubApp";
+  /**
+   * The hostname of the GitHub App deployment.
+   */
+  deploymentHostname?: string;
+  /**
+   * The ID of the GitHub App.
+   */
+  id: string;
+  /**
+   * The private key of the GitHub App.
+   */
+  privateKey:
+    | {
+        /**
+         * The name of the secret that contains the token.
+         */
+        secret: string;
+      }
+    | {
+        /**
+         * The name of the environment variable that contains the token. Only supported in declarative connection configs.
+         */
+        env: string;
+      };
+}

packages/shared/src/index.server.ts

@@ -24,7 +24,4 @@ export {
     isRemotePath,
     getConfigSettings,
 } from "./utils.js";
-export {
-    syncSearchContexts,
-} from "./ee/syncSearchContexts.js";
 export * from "./constants.js";

packages/web/package.json

@@ -113,7 +113,6 @@
     "ai": "^5.0.45",
     "ajv": "^8.17.1",
     "bcryptjs": "^3.0.2",
-    "chokidar": "^4.0.3",
     "class-variance-authority": "^0.7.0",
     "client-only": "^0.0.1",
     "clsx": "^2.1.1",

packages/web/src/actions.ts

@@ -10,7 +10,7 @@ import { prisma } from "@/prisma";
 import { render } from "@react-email/components";
 import * as Sentry from '@sentry/nextjs';
 import { encrypt, generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto";
-import { ApiKey, Org, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType, StripeSubscriptionStatus } from "@sourcebot/db";
+import { ApiKey, ConnectionSyncJobStatus, Org, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType, StripeSubscriptionStatus } from "@sourcebot/db";
 import { createLogger } from "@sourcebot/logger";
 import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type";
 import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
@@ -30,7 +30,7 @@ import JoinRequestSubmittedEmail from "./emails/joinRequestSubmittedEmail";
 import { AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME, MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SOURCEBOT_SUPPORT_EMAIL } from "./lib/constants";
 import { orgDomainSchema, orgNameSchema, repositoryQuerySchema } from "./lib/schemas";
 import { ApiKeyPayload, TenancyMode } from "./lib/types";
-import { withOptionalAuthV2 } from "./withAuthV2";
+import { withAuthV2, withOptionalAuthV2 } from "./withAuthV2";
 
 const logger = createLogger('web-actions');
 const auditService = getAuditService();
@@ -593,6 +593,7 @@ export const getReposStats = async () => sew(() =>
             prisma.repo.count({
                 where: {
                     orgId: org.id,
+                    indexedAt: null,
                     jobs: {
                         some: {
                             type: RepoIndexingJobType.INDEX,
@@ -604,7 +605,6 @@ export const getReposStats = async () => sew(() =>
                             }
                         },
                     },
-                    indexedAt: null,
                 }
             }),
             prisma.repo.count({
@@ -625,6 +625,42 @@ export const getReposStats = async () => sew(() =>
     })
 )
 
+export const getConnectionStats = async () => sew(() =>
+    withAuthV2(async ({ org, prisma }) => {
+        const [
+            numberOfConnections,
+            numberOfConnectionsWithFirstTimeSyncJobsInProgress,
+        ] = await Promise.all([
+            prisma.connection.count({
+                where: {
+                    orgId: org.id,
+                }
+            }),
+            prisma.connection.count({
+                where: {
+                    orgId: org.id,
+                    syncedAt: null,
+                    syncJobs: {
+                        some: {
+                            status: {
+                                in: [
+                                    ConnectionSyncJobStatus.PENDING,
+                                    ConnectionSyncJobStatus.IN_PROGRESS,
+                                ]
+                            }
+                        }
+                    }
+                }
+            })
+        ]);
+
+        return {
+            numberOfConnections,
+            numberOfConnectionsWithFirstTimeSyncJobsInProgress,
+        };
+    })
+);
+
 export const getRepoInfoByName = async (repoName: string) => sew(() =>
     withOptionalAuthV2(async ({ org, prisma }) => {
         // @note: repo names are represented by their remote url

packages/web/src/app/[domain]/chat/components/demoCards.tsx

@@ -7,7 +7,7 @@ import { Badge } from "@/components/ui/badge";
 import { Card } from "@/components/ui/card";
 import { CardContent } from "@/components/ui/card";
 import { DemoExamples, DemoSearchExample, DemoSearchScope } from "@/types";
-import { cn, getCodeHostIcon } from "@/lib/utils";
+import { cn, CodeHostType, getCodeHostIcon } from "@/lib/utils";
 import useCaptureEvent from "@/hooks/useCaptureEvent";
 import { SearchScopeInfoCard } from "@/features/chat/components/chatBox/searchScopeInfoCard";
 
@@ -41,25 +41,23 @@ export const DemoCards = ({
         }
 
         if (searchScope.codeHostType) {
-            const codeHostIcon = getCodeHostIcon(searchScope.codeHostType);
-            if (codeHostIcon) {
-                // When selected, icons need to match the inverted badge colors
-                // In light mode selected: light icon on dark bg (invert)
-                // In dark mode selected: dark icon on light bg (no invert, override dark:invert)
-                const selectedIconClass = isSelected
-                    ? "invert dark:invert-0"
-                    : codeHostIcon.className;
+            const codeHostIcon = getCodeHostIcon(searchScope.codeHostType as CodeHostType);
+            // When selected, icons need to match the inverted badge colors
+            // In light mode selected: light icon on dark bg (invert)
+            // In dark mode selected: dark icon on light bg (no invert, override dark:invert)
+            const selectedIconClass = isSelected
+                ? "invert dark:invert-0"
+                : codeHostIcon.className;
 
-                return (
-                    <Image
-                        src={codeHostIcon.src}
-                        alt={`${searchScope.codeHostType} icon`}
-                        width={size}
-                        height={size}
-                        className={cn(sizeClass, selectedIconClass)}
-                    />
-                );
-            }
+            return (
+                <Image
+                    src={codeHostIcon.src}
+                    alt={`${searchScope.codeHostType} icon`}
+                    width={size}
+                    height={size}
+                    className={cn(sizeClass, selectedIconClass)}
+                />
+            );
         }
 
         return <Code className={cn(sizeClass, colorClass)} />;

packages/web/src/app/[domain]/components/backButton.tsx

@@ -0,0 +1,20 @@
+import { cn } from "@/lib/utils";
+import { ArrowLeft } from "lucide-react"
+import Link from "next/link"
+
+interface BackButtonProps {
+    href: string;
+    name: string;
+    className?: string;
+}
+
+export function BackButton({ href, name, className }: BackButtonProps) {
+    return (
+        <Link href={href} className={cn("inline-flex items-center text-link transition-colors group", className)}>
+            <span className="inline-flex items-center gap-1.5 border-b border-transparent group-hover:border-link pb-0.5">
+                <ArrowLeft className="h-4 w-4" />
+                <span>{name}</span>
+            </span>
+        </Link>
+    )
+}

packages/web/src/app/[domain]/components/navigationMenu/index.tsx

@@ -1,4 +1,4 @@
-import { getRepos, getReposStats } from "@/actions";
+import { getConnectionStats, getRepos, getReposStats } from "@/actions";
 import { SourcebotLogo } from "@/app/components/sourcebotLogo";
 import { auth } from "@/auth";
 import { Button } from "@/components/ui/button";
@@ -39,6 +39,11 @@ export const NavigationMenu = async ({
         throw new ServiceErrorException(repoStats);
     }
 
+    const connectionStats = isAuthenticated ? await getConnectionStats() : null;
+    if (isServiceError(connectionStats)) {
+        throw new ServiceErrorException(connectionStats);
+    }
+
     const sampleRepos = await getRepos({
         where: {
             jobs: {
@@ -93,7 +98,12 @@ export const NavigationMenu = async ({
                         <NavigationItems
                             domain={domain}
                             numberOfRepos={numberOfRepos}
-                            numberOfReposWithFirstTimeIndexingJobsInProgress={numberOfReposWithFirstTimeIndexingJobsInProgress}
+                            isReposButtonNotificationDotVisible={numberOfReposWithFirstTimeIndexingJobsInProgress > 0}
+                            isSettingsButtonNotificationDotVisible={
+                                connectionStats ?
+                                    connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0 :
+                                    false
+                            }
                             isAuthenticated={isAuthenticated}
                         />
                     </NavigationMenuBase>

packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx

@@ -3,20 +3,23 @@
 import { NavigationMenuItem, NavigationMenuLink, NavigationMenuList, navigationMenuTriggerStyle } from "@/components/ui/navigation-menu";
 import { Badge } from "@/components/ui/badge";
 import { cn, getShortenedNumberDisplayString } from "@/lib/utils";
-import { SearchIcon, MessageCircleIcon, BookMarkedIcon, SettingsIcon, CircleIcon } from "lucide-react";
+import { SearchIcon, MessageCircleIcon, BookMarkedIcon, SettingsIcon } from "lucide-react";
 import { usePathname } from "next/navigation";
+import { NotificationDot } from "../notificationDot";
 
 interface NavigationItemsProps {
     domain: string;
     numberOfRepos: number;
-    numberOfReposWithFirstTimeIndexingJobsInProgress: number;
+    isReposButtonNotificationDotVisible: boolean;
+    isSettingsButtonNotificationDotVisible: boolean;
     isAuthenticated: boolean;
 }
 
 export const NavigationItems = ({
     domain,
     numberOfRepos,
-    numberOfReposWithFirstTimeIndexingJobsInProgress,
+    isReposButtonNotificationDotVisible,
+    isSettingsButtonNotificationDotVisible,
     isAuthenticated,
 }: NavigationItemsProps) => {
     const pathname = usePathname();
@@ -59,9 +62,7 @@ export const NavigationItems = ({
                     <span className="mr-2">Repositories</span>
                     <Badge variant="secondary" className="px-1.5 relative">
                         {getShortenedNumberDisplayString(numberOfRepos)}
-                        {numberOfReposWithFirstTimeIndexingJobsInProgress > 0 && (
-                            <CircleIcon className="absolute -right-0.5 -top-0.5 h-2 w-2 text-green-600" fill="currentColor" />
-                        )}
+                        {isReposButtonNotificationDotVisible && <NotificationDot className="absolute -right-0.5 -top-0.5" />}
                     </Badge>
                 </NavigationMenuLink>
                 {isActive(`/${domain}/repos`) && <ActiveIndicator />}
@@ -74,6 +75,7 @@ export const NavigationItems = ({
                     >
                         <SettingsIcon className="w-4 h-4 mr-1" />
                         Settings
+                        {isSettingsButtonNotificationDotVisible && <NotificationDot className="absolute -right-0.5 -top-0.5" />}
                     </NavigationMenuLink>
                     {isActive(`/${domain}/settings`) && <ActiveIndicator />}
                 </NavigationMenuItem>

packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx

@@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button";
 import { Separator } from "@/components/ui/separator";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
 import { useDomain } from "@/hooks/useDomain";
+import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants";
 import { RepositoryQuery } from "@/lib/types";
 import { getCodeHostInfoForRepo, getShortenedNumberDisplayString } from "@/lib/utils";
 import clsx from "clsx";
@@ -110,13 +111,14 @@ const RepoItem = ({ repo }: { repo: RepositoryQuery }) => {
 
 
     return (
-        <div
+        <Link
             className={clsx("flex flex-row items-center gap-2 border rounded-md p-2 text-clip")}
+            href={`/${SINGLE_TENANT_ORG_DOMAIN}/repos/${repo.repoId}`}
         >
             {repoIcon}
             <span className="text-sm truncate">
                 {displayName}
             </span>
-        </div>
+        </Link>
     )
 }

packages/web/src/app/[domain]/components/notificationDot.tsx

@@ -0,0 +1,9 @@
+import { cn } from "@/lib/utils"
+
+interface NotificationDotProps {
+    className?: string
+}
+
+export const NotificationDot = ({ className }: NotificationDotProps) => {
+    return <div className={cn("w-2 h-2 rounded-full bg-green-600", className)} />
+}

packages/web/src/app/[domain]/components/repositoryCarousel.tsx

@@ -37,7 +37,7 @@ export function RepositoryCarousel({
                         <span className="text-sm text-muted-foreground">
                             <>
                                 Create a{" "}
-                                <Link href={`https://docs.sourcebot.dev/docs/connections/overview`} className="text-blue-500 hover:underline inline-flex items-center gap-1">
+                                <Link href={`/${domain}/settings/connections`} className="text-blue-500 hover:underline inline-flex items-center gap-1">
                                     connection
                                 </Link>{" "}
                                 to start indexing repositories

packages/web/src/app/[domain]/repos/[id]/page.tsx

@@ -4,21 +4,21 @@ import { Button } from "@/components/ui/button"
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"
 import { Skeleton } from "@/components/ui/skeleton"
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
+import { env } from "@/env.mjs"
 import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants"
 import { ServiceErrorException } from "@/lib/serviceError"
 import { cn, getCodeHostInfoForRepo, isServiceError } from "@/lib/utils"
 import { withOptionalAuthV2 } from "@/withAuthV2"
-import { ChevronLeft, ExternalLink, Info } from "lucide-react"
+import { getConfigSettings, repoMetadataSchema } from "@sourcebot/shared"
+import { ExternalLink, Info } from "lucide-react"
 import Image from "next/image"
 import Link from "next/link"
 import { notFound } from "next/navigation"
 import { Suspense } from "react"
-import { RepoJobsTable } from "../components/repoJobsTable"
-import { getConfigSettings } from "@sourcebot/shared"
-import { env } from "@/env.mjs"
+import { BackButton } from "../../components/backButton"
 import { DisplayDate } from "../../components/DisplayDate"
 import { RepoBranchesTable } from "../components/repoBranchesTable"
-import { repoMetadataSchema } from "@sourcebot/shared"
+import { RepoJobsTable } from "../components/repoJobsTable"
 
 export default async function RepoDetailPage({ params }: { params: Promise<{ id: string }> }) {
     const { id } = await params
@@ -52,14 +52,13 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
     const repoMetadata = repoMetadataSchema.parse(repo.metadata);
 
     return (
-        <div className="container mx-auto">
+        <>
             <div className="mb-6">
-                <Button variant="ghost" asChild className="mb-4">
-                    <Link href={`/${SINGLE_TENANT_ORG_DOMAIN}/repos`}>
-                        <ChevronLeft className="mr-2 h-4 w-4" />
-                        Back to repositories
-                    </Link>
-                </Button>
+                <BackButton
+                    href={`/${SINGLE_TENANT_ORG_DOMAIN}/repos`}
+                    name="Back to repositories"
+                    className="mb-2"
+                />
 
                 <div className="flex items-start justify-between">
                     <div>
@@ -103,7 +102,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
                         </CardTitle>
                     </CardHeader>
                     <CardContent>
-                        <DisplayDate date={repo.createdAt} className="text-2xl font-semibold" />
+                        <span className="text-2xl font-semibold"><DisplayDate date={repo.createdAt} /></span>
                     </CardContent>
                 </Card>
 
@@ -122,7 +121,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
                         </CardTitle>
                     </CardHeader>
                     <CardContent>
-                        {repo.indexedAt ? <DisplayDate date={repo.indexedAt} className="text-2xl font-semibold" /> : "Never"}
+                        <span className="text-2xl font-semibold">{repo.indexedAt ? <DisplayDate date={repo.indexedAt} /> : "Never"}</span>
                     </CardContent>
                 </Card>
 
@@ -141,7 +140,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
                         </CardTitle>
                     </CardHeader>
                     <CardContent>
-                        {nextIndexAttempt ? <DisplayDate date={nextIndexAttempt} className="text-2xl font-semibold" /> : "-"}
+                        <span className="text-2xl font-semibold">{nextIndexAttempt ? <DisplayDate date={nextIndexAttempt} /> : "-"}</span>
                     </CardContent>
                 </Card>
             </div>
@@ -168,7 +167,7 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
 
             <Card>
                 <CardHeader>
-                    <CardTitle>Indexing Jobs</CardTitle>
+                    <CardTitle>Indexing History</CardTitle>
                     <CardDescription>History of all indexing and cleanup jobs for this repository.</CardDescription>
                 </CardHeader>
                 <CardContent>
@@ -177,16 +176,17 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
                     </Suspense>
                 </CardContent>
             </Card>
-        </div>
+        </>
     )
 }
 
 const getRepoWithJobs = async (repoId: number) => sew(() =>
-    withOptionalAuthV2(async ({ prisma }) => {
+    withOptionalAuthV2(async ({ prisma, org }) => {
 
         const repo = await prisma.repo.findUnique({
             where: {
                 id: repoId,
+                orgId: org.id,
             },
             include: {
                 jobs: {

packages/web/src/app/[domain]/repos/components/reposTable.tsx

@@ -37,6 +37,7 @@ import { useRouter } from "next/navigation"
 import { useToast } from "@/components/hooks/use-toast";
 import { DisplayDate } from "../../components/DisplayDate"
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
+import { NotificationDot } from "../../components/notificationDot"
 
 // @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS
 
@@ -53,6 +54,7 @@ export type Repo = {
     imageUrl: string | null
     indexedCommitHash: string | null
     latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null
+    isFirstTimeIndex: boolean
 }
 
 const statusBadgeVariants = cva("", {
@@ -111,14 +113,32 @@ export const columns: ColumnDef<Repo>[] = [
                             {repo.displayName?.charAt(0) ?? repo.name.charAt(0)}
                         </div>
                     )}
-                    <Link href={getBrowsePath({
-                        repoName: repo.name,
-                        path: '/',
-                        pathType: 'tree',
-                        domain: SINGLE_TENANT_ORG_DOMAIN,
-                    })} className="font-medium hover:underline">
+
+                    {/* Link to the details page (instead of browse) when the repo is indexing
+                        as the code will not be available yet */}
+                    <Link
+                        href={repo.isFirstTimeIndex ? `/${SINGLE_TENANT_ORG_DOMAIN}/repos/${repo.id}` : getBrowsePath({
+                            repoName: repo.name,
+                            path: '/',
+                            pathType: 'tree',
+                            domain: SINGLE_TENANT_ORG_DOMAIN,
+                        })}
+                        className="font-medium hover:underline"
+                    >
                         {repo.displayName || repo.name}
                     </Link>
+                    {repo.isFirstTimeIndex && (
+                        <Tooltip>
+                            <TooltipTrigger asChild>
+                                <span>
+                                    <NotificationDot className="ml-1.5" />
+                                </span>
+                            </TooltipTrigger>
+                            <TooltipContent>
+                                <span>This is the first time Sourcebot is indexing this repository. It may take a few minutes to complete.</span>
+                            </TooltipContent>
+                        </Tooltip>
+                    )}
                 </div>
             )
         },
@@ -150,7 +170,7 @@ export const columns: ColumnDef<Repo>[] = [
             }
 
             return (
-                <DisplayDate date={indexedAt} className="ml-3"/>
+                <DisplayDate date={indexedAt} className="ml-3" />
             )
         }
     },
@@ -177,11 +197,11 @@ export const columns: ColumnDef<Repo>[] = [
 
             const HashComponent = commitUrl ? (
                 <Link
-                href={commitUrl}
-                className="font-mono text-sm text-link hover:underline"
-            >
-                {smallHash}
-            </Link>
+                    href={commitUrl}
+                    className="font-mono text-sm text-link hover:underline"
+                >
+                    {smallHash}
+                </Link>
             ) : (
                 <span className="font-mono text-sm text-muted-foreground">
                     {smallHash}
@@ -331,7 +351,7 @@ export const ReposTable = ({ data }: { data: Repo[] }) => {
                 </Button>
             </div>
             <div className="rounded-md border">
-                <Table style={{ tableLayout: 'fixed', width: '100%' }}>
+                <Table style={{ width: '100%' }}>
                     <TableHeader>
                         {table.getHeaderGroups().map((headerGroup) => (
                             <TableRow key={headerGroup.id}>

packages/web/src/app/[domain]/repos/layout.tsx

@@ -1,4 +1,11 @@
+import { InfoIcon } from "lucide-react";
 import { NavigationMenu } from "../components/navigationMenu";
+import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants";
+import Link from "next/link";
+import { getCurrentUserRole, getReposStats } from "@/actions";
+import { isServiceError } from "@/lib/utils";
+import { ServiceErrorException } from "@/lib/serviceError";
+import { OrgRole } from "@sourcebot/db";
 
 interface LayoutProps {
     children: React.ReactNode;
@@ -12,11 +19,28 @@ export default async function Layout(
     const { domain } = params;
     const { children } = props;
 
+    const repoStats = await getReposStats();
+    if (isServiceError(repoStats)) {
+        throw new ServiceErrorException(repoStats);
+    }
+
+    const userRoleInOrg = await getCurrentUserRole(domain);
+
     return (
         <div className="min-h-screen flex flex-col">
             <NavigationMenu domain={domain} />
+            {(repoStats.numberOfRepos === 0 && userRoleInOrg === OrgRole.OWNER) && (
+                <div className="w-full flex flex-row justify-center items-center bg-accent py-0.5">
+                    <InfoIcon className="w-4 h-4 mr-1" />
+                    <span><span className="font-medium">No repositories configured.</span> Create a <Link href={`/${SINGLE_TENANT_ORG_DOMAIN}/settings/connections`} className="text-link hover:underline">connection</Link> to get started.</span>
+                </div>
+            )}
             <main className="flex-grow flex justify-center p-4 bg-backgroundSecondary relative">
-                <div className="w-full max-w-6xl rounded-lg p-6">{children}</div>
+                <div className="w-full max-w-6xl rounded-lg p-6">
+                    <div className="container mx-auto">
+                        {children}
+                    </div>
+                </div>
             </main>
         </div>
     )

packages/web/src/app/[domain]/repos/page.tsx

@@ -3,16 +3,33 @@ import { ServiceErrorException } from "@/lib/serviceError";
 import { isServiceError } from "@/lib/utils";
 import { withOptionalAuthV2 } from "@/withAuthV2";
 import { ReposTable } from "./components/reposTable";
+import { RepoIndexingJobStatus } from "@sourcebot/db";
 
 export default async function ReposPage() {
 
-    const repos = await getReposWithLatestJob();
-    if (isServiceError(repos)) {
-        throw new ServiceErrorException(repos);
+    const _repos = await getReposWithLatestJob();
+    if (isServiceError(_repos)) {
+        throw new ServiceErrorException(_repos);
     }
 
+    const repos = _repos
+        .map((repo) => ({
+            ...repo,
+            latestJobStatus: repo.jobs.length > 0 ? repo.jobs[0].status : null,
+            isFirstTimeIndex: repo.indexedAt === null && repo.jobs.filter((job) => job.status === RepoIndexingJobStatus.PENDING || job.status === RepoIndexingJobStatus.IN_PROGRESS).length > 0,
+        }))
+        .sort((a, b) => {
+            if (a.isFirstTimeIndex && !b.isFirstTimeIndex) {
+                return -1;
+            }
+            if (!a.isFirstTimeIndex && b.isFirstTimeIndex) {
+                return 1;
+            }
+            return a.name.localeCompare(b.name);
+        });
+
     return (
-        <div className="container mx-auto">
+        <>
             <div className="mb-6">
                 <h1 className="text-3xl font-semibold">Repositories</h1>
                 <p className="text-muted-foreground mt-2">View and manage your code repositories and their indexing status.</p>
@@ -27,16 +44,17 @@ export default async function ReposPage() {
                 createdAt: repo.createdAt,
                 webUrl: repo.webUrl,
                 imageUrl: repo.imageUrl,
-                latestJobStatus: repo.jobs.length > 0 ? repo.jobs[0].status : null,
+                latestJobStatus: repo.latestJobStatus,
+                isFirstTimeIndex: repo.isFirstTimeIndex,
                 codeHostType: repo.external_codeHostType,
                 indexedCommitHash: repo.indexedCommitHash,
             }))} />
-        </div>
+        </>
     )
 }
 
 const getReposWithLatestJob = async () => sew(() =>
-    withOptionalAuthV2(async ({ prisma }) => {
+    withOptionalAuthV2(async ({ prisma, org }) => {
         const repos = await prisma.repo.findMany({
             include: {
                 jobs: {
@@ -48,6 +66,9 @@ const getReposWithLatestJob = async () => sew(() =>
             },
             orderBy: {
                 name: 'asc'
+            },
+            where: {
+                orgId: org.id,
             }
         });
         return repos;

packages/web/src/app/[domain]/settings/components/header.tsx

@@ -1,22 +0,0 @@
-import { Separator } from "@/components/ui/separator";
-import { cn } from "@/lib/utils";
-import clsx from "clsx";
-
-interface HeaderProps {
-    children: React.ReactNode;
-    withTopMargin?: boolean;
-    className?: string;
-}
-
-export const Header = ({
-    children,
-    withTopMargin = true,
-    className,
-}: HeaderProps) => {
-    return (
-        <div className={cn("mb-16", className)}>
-            {children}
-            <Separator className={clsx("absolute left-0 right-0", { "mt-12": withTopMargin })} />
-        </div>
-    )
-}

packages/web/src/app/[domain]/settings/components/sidebar-nav.tsx

@@ -1,44 +1,54 @@
 "use client"
 
-import React from "react"
+import { buttonVariants } from "@/components/ui/button"
+import { NotificationDot } from "@/app/[domain]/components/notificationDot"
+import { cn } from "@/lib/utils"
 import Link from "next/link"
 import { usePathname } from "next/navigation"
-import { cn } from "@/lib/utils"
-import { buttonVariants } from "@/components/ui/button"
+import React from "react"
+
+export type SidebarNavItem = {
+    href: string
+    hrefRegex?: string
+    title: React.ReactNode
+    isNotificationDotVisible?: boolean
+}
 
 interface SidebarNavProps extends React.HTMLAttributes<HTMLElement> {
-  items: {
-    href: string
-    title: React.ReactNode
-  }[]
+    items: SidebarNavItem[]
 }
 
 export function SidebarNav({ className, items, ...props }: SidebarNavProps) {
-  const pathname = usePathname()
+    const pathname = usePathname()
 
-  return (
-    <nav
-      className={cn(
-        "flex flex-col space-x-2 lg:space-x-0 lg:space-y-1",
-        className
-      )}
-      {...props}
-    >
-      {items.map((item) => (
-        <Link
-          key={item.href}
-          href={item.href}
-          className={cn(
-            buttonVariants({ variant: "ghost" }),
-            pathname === item.href
-              ? "bg-muted hover:bg-muted"
-              : "hover:bg-transparent hover:underline",
-            "justify-start"
-          )}
+    return (
+        <nav
+            className={cn(
+                "flex flex-col space-x-2 lg:space-x-0 lg:space-y-1",
+                className
+            )}
+            {...props}
         >
-          {item.title}
-        </Link>
-      ))}
-    </nav>
-  )
+            {items.map((item) => {
+                const isActive = item.hrefRegex ? new RegExp(item.hrefRegex).test(pathname) : pathname === item.href;
+
+                return (
+                    <Link
+                        key={item.href}
+                        href={item.href}
+                        className={cn(
+                            buttonVariants({ variant: "ghost" }),
+                            isActive
+                                ? "bg-muted hover:bg-muted"
+                                : "hover:bg-transparent hover:underline",
+                            "justify-start"
+                        )}
+                    >
+                        {item.title}
+                        {item.isNotificationDotVisible && <NotificationDot className="ml-1.5" />}
+                    </Link>
+                )
+            })}
+        </nav>
+    )
 }

packages/web/src/app/[domain]/settings/connections/[id]/page.tsx

@@ -0,0 +1,204 @@
+import { sew } from "@/actions";
+import { BackButton } from "@/app/[domain]/components/backButton";
+import { DisplayDate } from "@/app/[domain]/components/DisplayDate";
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
+import { Skeleton } from "@/components/ui/skeleton";
+import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
+import { env } from "@/env.mjs";
+import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants";
+import { notFound, ServiceErrorException } from "@/lib/serviceError";
+import { CodeHostType, isServiceError } from "@/lib/utils";
+import { withAuthV2 } from "@/withAuthV2";
+import { AzureDevOpsConnectionConfig, BitbucketConnectionConfig, GenericGitHostConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GithubConnectionConfig, GitlabConnectionConfig } from "@sourcebot/schemas/v3/index.type";
+import { getConfigSettings } from "@sourcebot/shared";
+import { Info } from "lucide-react";
+import Link from "next/link";
+import { Suspense } from "react";
+import { ConnectionJobsTable } from "../components/connectionJobsTable";
+
+interface ConnectionDetailPageProps {
+    params: Promise<{
+        id: string
+    }>
+}
+
+
+export default async function ConnectionDetailPage(props: ConnectionDetailPageProps) {
+    const params = await props.params;
+    const { id } = params;
+
+    const connection = await getConnectionWithJobs(Number.parseInt(id));
+    if (isServiceError(connection)) {
+        throw new ServiceErrorException(connection);
+    }
+
+    const configSettings = await getConfigSettings(env.CONFIG_PATH);
+
+    const nextSyncAttempt = (() => {
+        const latestJob = connection.syncJobs.length > 0 ? connection.syncJobs[0] : null;
+        if (!latestJob) {
+            return undefined;
+        }
+
+        if (latestJob.completedAt) {
+            return new Date(latestJob.completedAt.getTime() + configSettings.resyncConnectionIntervalMs);
+        }
+
+        return undefined;
+    })();
+
+    const codeHostUrl = (() => {
+        const connectionType = connection.connectionType as CodeHostType;
+        switch (connectionType) {
+            case 'github': {
+                const config = connection.config as unknown as GithubConnectionConfig;
+                return config.url ?? 'https://github.com';
+            }
+            case 'gitlab': {
+                const config = connection.config as unknown as GitlabConnectionConfig;
+                return config.url ?? 'https://gitlab.com';
+            }
+            case 'gitea': {
+                const config = connection.config as unknown as GiteaConnectionConfig;
+                return config.url ?? 'https://gitea.com';
+            }
+            case 'gerrit': {
+                const config = connection.config as unknown as GerritConnectionConfig;
+                return config.url;
+            }
+            case 'bitbucket-server': {
+                const config = connection.config as unknown as BitbucketConnectionConfig;
+                return config.url!;
+            }
+            case 'bitbucket-cloud': {
+                const config = connection.config as unknown as BitbucketConnectionConfig;
+                return config.url ?? 'https://bitbucket.org';
+            }
+            case 'azuredevops': {
+                const config = connection.config as unknown as AzureDevOpsConnectionConfig;
+                return config.url ?? 'https://dev.azure.com';
+            }
+            case 'generic-git-host': {
+                const config = connection.config as unknown as GenericGitHostConnectionConfig;
+                return config.url;
+            }
+        }
+    })();
+
+    return (
+        <div>
+            <BackButton
+                href={`/${SINGLE_TENANT_ORG_DOMAIN}/settings/connections`}
+                name="Back to connections"
+                className="mb-2"
+            />
+            <div className="flex flex-col gap-2 mb-6">
+                <h1 className="text-3xl font-semibold">{connection.name}</h1>
+
+                <Link
+                    href={codeHostUrl}
+                    target="_blank"
+                    rel="noopener noreferrer"
+                    className="hover:underline text-muted-foreground"
+                >
+                    {codeHostUrl}
+                </Link>
+            </div>
+
+            <div className="grid gap-4 md:grid-cols-3 mb-8">
+                <Card>
+                    <CardHeader className="pb-3">
+                        <CardTitle className="text-sm font-medium flex items-center gap-1.5">
+                            Created
+                            <Tooltip>
+                                <TooltipTrigger asChild>
+                                    <Info className="h-3.5 w-3.5 text-muted-foreground cursor-help" />
+                                </TooltipTrigger>
+                                <TooltipContent>
+                                    <p>When this connection was first added to Sourcebot</p>
+                                </TooltipContent>
+                            </Tooltip>
+                        </CardTitle>
+                    </CardHeader>
+                    <CardContent>
+                        <span className="text-2xl font-semibold"><DisplayDate date={connection.createdAt} /></span>
+                    </CardContent>
+                </Card>
+
+                <Card>
+                    <CardHeader className="pb-3">
+                        <CardTitle className="text-sm font-medium flex items-center gap-1.5">
+                            Last synced
+                            <Tooltip>
+                                <TooltipTrigger asChild>
+                                    <Info className="h-3.5 w-3.5 text-muted-foreground cursor-help" />
+                                </TooltipTrigger>
+                                <TooltipContent>
+                                    <p>The last time this connection was successfully synced</p>
+                                </TooltipContent>
+                            </Tooltip>
+                        </CardTitle>
+                    </CardHeader>
+                    <CardContent>
+                        <span className="text-2xl font-semibold">{connection.syncedAt ? <DisplayDate date={connection.syncedAt} /> : "Never"}</span>
+                    </CardContent>
+                </Card>
+
+                <Card>
+                    <CardHeader className="pb-3">
+                        <CardTitle className="text-sm font-medium flex items-center gap-1.5">
+                            Scheduled
+                            <Tooltip>
+                                <TooltipTrigger asChild>
+                                    <Info className="h-3.5 w-3.5 text-muted-foreground cursor-help" />
+                                </TooltipTrigger>
+                                <TooltipContent>
+                                    <p>When the connection will be resynced next. Modifying the config will also trigger a resync.</p>
+                                </TooltipContent>
+                            </Tooltip>
+                        </CardTitle>
+                    </CardHeader>
+                    <CardContent>
+                        <span className="text-2xl font-semibold">{nextSyncAttempt ? <DisplayDate date={nextSyncAttempt} /> : "-"}</span>
+                    </CardContent>
+                </Card>
+            </div>
+
+            <Card>
+                <CardHeader>
+                    <CardTitle>Sync History</CardTitle>
+                    <CardDescription>History of all sync jobs for this connection.</CardDescription>
+                </CardHeader>
+                <CardContent>
+                    <Suspense fallback={<Skeleton className="h-96 w-full" />}>
+                        <ConnectionJobsTable data={connection.syncJobs} />
+                    </Suspense>
+                </CardContent>
+            </Card>
+        </div>
+    )
+}
+
+const getConnectionWithJobs = async (id: number) => sew(() =>
+    withAuthV2(async ({ prisma, org }) => {
+        const connection = await prisma.connection.findUnique({
+            where: {
+                id,
+                orgId: org.id,
+            },
+            include: {
+                syncJobs: {
+                    orderBy: {
+                        createdAt: 'desc',
+                    },
+                },
+            },
+        });
+
+        if (!connection) {
+            return notFound();
+        }
+
+        return connection;
+    })
+)

packages/web/src/app/[domain]/settings/connections/components/connectionJobsTable.tsx

@@ -0,0 +1,311 @@
+"use client"
+
+import { Badge } from "@/components/ui/badge"
+import { Button } from "@/components/ui/button"
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
+import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"
+import {
+    type ColumnDef,
+    type ColumnFiltersState,
+    type SortingState,
+    type VisibilityState,
+    flexRender,
+    getCoreRowModel,
+    getFilteredRowModel,
+    getPaginationRowModel,
+    getSortedRowModel,
+    useReactTable,
+} from "@tanstack/react-table"
+import { cva } from "class-variance-authority"
+import { AlertCircle, AlertTriangle, ArrowUpDown, RefreshCwIcon } from "lucide-react"
+import * as React from "react"
+import { CopyIconButton } from "@/app/[domain]/components/copyIconButton"
+import { useMemo } from "react"
+import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter"
+import { useRouter } from "next/navigation"
+import { useToast } from "@/components/hooks/use-toast"
+import { DisplayDate } from "@/app/[domain]/components/DisplayDate"
+
+
+export type ConnectionSyncJob = {
+    id: string
+    status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED"
+    createdAt: Date
+    updatedAt: Date
+    completedAt: Date | null
+    errorMessage: string | null
+    warningMessages: string[]
+}
+
+const statusBadgeVariants = cva("", {
+    variants: {
+        status: {
+            PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80",
+            IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90",
+            COMPLETED: "bg-green-600 text-white hover:bg-green-700",
+            FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90",
+        },
+    },
+})
+
+const getStatusBadge = (status: ConnectionSyncJob["status"]) => {
+    const labels = {
+        PENDING: "Pending",
+        IN_PROGRESS: "In Progress",
+        COMPLETED: "Completed",
+        FAILED: "Failed",
+    }
+
+    return <Badge className={statusBadgeVariants({ status })}>{labels[status]}</Badge>
+}
+
+const getDuration = (start: Date, end: Date | null) => {
+    if (!end) return "-"
+    const diff = end.getTime() - start.getTime()
+    const minutes = Math.floor(diff / 60000)
+    const seconds = Math.floor((diff % 60000) / 1000)
+    return `${minutes}m ${seconds}s`
+}
+
+export const columns: ColumnDef<ConnectionSyncJob>[] = [
+    {
+        accessorKey: "status",
+        header: "Status",
+        cell: ({ row }) => {
+            const job = row.original
+            return (
+                <div className="flex items-center gap-2">
+                    {getStatusBadge(row.getValue("status"))}
+                    {job.errorMessage ? (
+                        <TooltipProvider>
+                            <Tooltip>
+                                <TooltipTrigger>
+                                    <AlertCircle className="h-4 w-4 text-destructive" />
+                                </TooltipTrigger>
+                                <TooltipContent className="max-w-[750px] max-h-96 overflow-scroll p-4">
+                                    <LightweightCodeHighlighter
+                                        language="text"
+                                        lineNumbers={true}
+                                        renderWhitespace={false}
+                                    >
+                                        {job.errorMessage}
+                                    </LightweightCodeHighlighter>
+                                </TooltipContent>
+                            </Tooltip>
+                        </TooltipProvider>
+                    ) : job.warningMessages.length > 0 ? (
+                        <TooltipProvider>
+                            <Tooltip>
+                                <TooltipTrigger>
+                                    <AlertTriangle className="h-4 w-4 text-warning" />
+                                </TooltipTrigger>
+                                <TooltipContent className="max-w-[750px] max-h-96 overflow-scroll p-4">
+                                    <p className="text-sm font-medium mb-2">{job.warningMessages.length} warning(s) while syncing:</p>
+                                    <div className="flex flex-col gap-1">
+                                        {job.warningMessages.map((warning, index) => (
+                                            <div
+                                                key={index}
+                                                className="text-sm font-mono flex flex-row items-center gap-1.5"
+                                            >
+                                                <span>{index + 1}.</span>
+                                                <span className="text-warning">{warning}</span>
+                                            </div>
+                                        ))}
+                                    </div>
+                                </TooltipContent>
+                            </Tooltip>
+                        </TooltipProvider>
+                    ) : null}
+                </div>
+            )
+        },
+        filterFn: (row, id, value) => {
+            return value.includes(row.getValue(id))
+        },
+    },
+    {
+        accessorKey: "createdAt",
+        header: ({ column }) => {
+            return (
+                <Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}>
+                    Started
+                    <ArrowUpDown className="ml-2 h-4 w-4" />
+                </Button>
+            )
+        },
+        cell: ({ row }) => <DisplayDate date={row.getValue("createdAt") as Date} className="ml-3" />,
+    },
+    {
+        accessorKey: "completedAt",
+        header: ({ column }) => {
+            return (
+                <Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}>
+                    Completed
+                    <ArrowUpDown className="ml-2 h-4 w-4" />
+                </Button>
+            )
+        },
+        cell: ({ row }) => {
+            const completedAt = row.getValue("completedAt") as Date | null;
+            if (!completedAt) {
+                return "-";
+            }
+
+            return <DisplayDate date={completedAt} className="ml-3" />
+        },
+    },
+    {
+        id: "duration",
+        header: "Duration",
+        cell: ({ row }) => {
+            const job = row.original
+            return getDuration(job.createdAt, job.completedAt)
+        },
+    },
+    {
+        accessorKey: "id",
+        header: "Job ID",
+        cell: ({ row }) => {
+            const id = row.getValue("id") as string
+            return (
+                <div className="flex items-center gap-2">
+                    <code className="text-xs text-muted-foreground">{id}</code>
+                    <CopyIconButton onCopy={() => {
+                        navigator.clipboard.writeText(id);
+                        return true;
+                    }} />
+                </div>
+            )
+        },
+    },
+]
+
+export const ConnectionJobsTable = ({ data }: { data: ConnectionSyncJob[] }) => {
+    const [sorting, setSorting] = React.useState<SortingState>([{ id: "createdAt", desc: true }])
+    const [columnFilters, setColumnFilters] = React.useState<ColumnFiltersState>([])
+    const [columnVisibility, setColumnVisibility] = React.useState<VisibilityState>({})
+    const router = useRouter();
+    const { toast } = useToast();
+
+    const table = useReactTable({
+        data,
+        columns,
+        onSortingChange: setSorting,
+        onColumnFiltersChange: setColumnFilters,
+        getCoreRowModel: getCoreRowModel(),
+        getPaginationRowModel: getPaginationRowModel(),
+        getSortedRowModel: getSortedRowModel(),
+        getFilteredRowModel: getFilteredRowModel(),
+        onColumnVisibilityChange: setColumnVisibility,
+        state: {
+            sorting,
+            columnFilters,
+            columnVisibility,
+        },
+    })
+
+    const {
+        numCompleted,
+        numInProgress,
+        numPending,
+        numFailed,
+    } = useMemo(() => {
+        return {
+            numCompleted: data.filter((job) => job.status === "COMPLETED").length,
+            numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length,
+            numPending: data.filter((job) => job.status === "PENDING").length,
+            numFailed: data.filter((job) => job.status === "FAILED").length,
+        };
+    }, [data]);
+
+    return (
+        <div className="w-full">
+            <div className="flex items-center gap-4 py-4">
+                <Select
+                    value={(table.getColumn("status")?.getFilterValue() as string) ?? "all"}
+                    onValueChange={(value) => table.getColumn("status")?.setFilterValue(value === "all" ? "" : value)}
+                >
+                    <SelectTrigger className="w-[180px]">
+                        <SelectValue placeholder="Filter by status" />
+                    </SelectTrigger>
+                    <SelectContent>
+                        <SelectItem value="all">Filter by status</SelectItem>
+                        <SelectItem value="COMPLETED">Completed ({numCompleted})</SelectItem>
+                        <SelectItem value="IN_PROGRESS">In progress ({numInProgress})</SelectItem>
+                        <SelectItem value="PENDING">Pending ({numPending})</SelectItem>
+                        <SelectItem value="FAILED">Failed ({numFailed})</SelectItem>
+                    </SelectContent>
+                </Select>
+
+                <Button
+                    variant="outline"
+                    className="ml-auto"
+                    onClick={() => {
+                        router.refresh();
+                        toast({
+                            description: "Page refreshed",
+                        });
+                    }}
+                >
+                    <RefreshCwIcon className="w-3 h-3" />
+                    Refresh
+                </Button>
+            </div>
+
+            <div className="rounded-md border">
+                <Table>
+                    <TableHeader>
+                        {table.getHeaderGroups().map((headerGroup) => (
+                            <TableRow key={headerGroup.id}>
+                                {headerGroup.headers.map((header) => {
+                                    return (
+                                        <TableHead key={header.id}>
+                                            {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
+                                        </TableHead>
+                                    )
+                                })}
+                            </TableRow>
+                        ))}
+                    </TableHeader>
+                    <TableBody>
+                        {table.getRowModel().rows?.length ? (
+                            table.getRowModel().rows.map((row) => (
+                                <TableRow key={row.id}>
+                                    {row.getVisibleCells().map((cell) => (
+                                        <TableCell key={cell.id}>{flexRender(cell.column.columnDef.cell, cell.getContext())}</TableCell>
+                                    ))}
+                                </TableRow>
+                            ))
+                        ) : (
+                            <TableRow>
+                                <TableCell colSpan={columns.length} className="h-24 text-center">
+                                    No sync jobs found.
+                                </TableCell>
+                            </TableRow>
+                        )}
+                    </TableBody>
+                </Table>
+            </div>
+
+            <div className="flex items-center justify-end space-x-2 py-4">
+                <div className="flex-1 text-sm text-muted-foreground">
+                    {table.getFilteredRowModel().rows.length} job(s) total
+                </div>
+                <div className="space-x-2">
+                    <Button
+                        variant="outline"
+                        size="sm"
+                        onClick={() => table.previousPage()}
+                        disabled={!table.getCanPreviousPage()}
+                    >
+                        Previous
+                    </Button>
+                    <Button variant="outline" size="sm" onClick={() => table.nextPage()} disabled={!table.getCanNextPage()}>
+                        Next
+                    </Button>
+                </div>
+            </div>
+        </div>
+    )
+}

packages/web/src/app/[domain]/settings/connections/components/connectionsTable.tsx

@@ -0,0 +1,295 @@
+"use client"
+
+import { DisplayDate } from "@/app/[domain]/components/DisplayDate"
+import { NotificationDot } from "@/app/[domain]/components/notificationDot"
+import { useToast } from "@/components/hooks/use-toast"
+import { Badge } from "@/components/ui/badge"
+import { Button } from "@/components/ui/button"
+import { Input } from "@/components/ui/input"
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
+import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
+import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
+import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants"
+import { CodeHostType, getCodeHostIcon } from "@/lib/utils"
+import {
+    type ColumnDef,
+    type ColumnFiltersState,
+    type SortingState,
+    type VisibilityState,
+    flexRender,
+    getCoreRowModel,
+    getFilteredRowModel,
+    getPaginationRowModel,
+    getSortedRowModel,
+    useReactTable,
+} from "@tanstack/react-table"
+import { cva } from "class-variance-authority"
+import { ArrowUpDown, RefreshCwIcon } from "lucide-react"
+import Image from "next/image"
+import Link from "next/link"
+import { useRouter } from "next/navigation"
+import { useMemo, useState } from "react"
+
+
+export type Connection = {
+    id: number
+    name: string
+    syncedAt: Date | null
+    codeHostType: CodeHostType
+    latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null
+    isFirstTimeSync: boolean
+}
+
+const statusBadgeVariants = cva("", {
+    variants: {
+        status: {
+            PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80",
+            IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90",
+            COMPLETED: "bg-green-600 text-white hover:bg-green-700",
+            FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90",
+            NO_JOBS: "bg-secondary text-secondary-foreground hover:bg-secondary/80",
+        },
+    },
+})
+
+const getStatusBadge = (status: Connection["latestJobStatus"]) => {
+    if (!status) {
+        return <Badge className={statusBadgeVariants({ status: "NO_JOBS" })}>No Jobs</Badge>
+    }
+
+    const labels = {
+        PENDING: "Pending",
+        IN_PROGRESS: "In Progress",
+        COMPLETED: "Completed",
+        FAILED: "Failed",
+    }
+
+    return <Badge className={statusBadgeVariants({ status })}>{labels[status]}</Badge>
+}
+
+export const columns: ColumnDef<Connection>[] = [
+    {
+        accessorKey: "name",
+        size: 400,
+        header: ({ column }) => {
+            return (
+                <Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}>
+                    Name
+                    <ArrowUpDown className="ml-2 h-4 w-4" />
+                </Button>
+            )
+        },
+        cell: ({ row }) => {
+            const connection = row.original;
+            const codeHostIcon = getCodeHostIcon(connection.codeHostType);
+
+            return (
+                <div className="flex flex-row gap-2 items-center">
+                    <Image
+                        src={codeHostIcon.src}
+                        alt={`${connection.codeHostType} logo`}
+                        className={codeHostIcon.className}
+                        width={20}
+                        height={20}
+                    />
+                    <Link href={`/${SINGLE_TENANT_ORG_DOMAIN}/settings/connections/${connection.id}`} className="font-medium hover:underline">
+                        {connection.name}
+                    </Link>
+                    {connection.isFirstTimeSync && (
+                        <Tooltip>
+                            <TooltipTrigger asChild>
+                                <span>
+                                    <NotificationDot className="ml-1.5" />
+                                </span>
+                            </TooltipTrigger>
+                            <TooltipContent>
+                                <span>This is the first time Sourcebot is syncing this connection. It may take a few minutes to complete.</span>
+                            </TooltipContent>
+                        </Tooltip>
+                    )}
+                </div>
+            )
+        },
+    },
+    {
+        accessorKey: "latestJobStatus",
+        size: 150,
+        header: "Lastest status",
+        cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")),
+    },
+    {
+        accessorKey: "syncedAt",
+        size: 200,
+        header: ({ column }) => {
+            return (
+                <Button
+                    variant="ghost"
+                    onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
+                >
+                    Last synced
+                    <ArrowUpDown className="ml-2 h-4 w-4" />
+                </Button>
+            )
+        },
+        cell: ({ row }) => {
+            const syncedAt = row.getValue("syncedAt") as Date | null;
+            if (!syncedAt) {
+                return "-";
+            }
+
+            return (
+                <DisplayDate date={syncedAt} className="ml-3" />
+            )
+        }
+    },
+]
+
+export const ConnectionsTable = ({ data }: { data: Connection[] }) => {
+    const [sorting, setSorting] = useState<SortingState>([])
+    const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])
+    const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({})
+    const [rowSelection, setRowSelection] = useState({})
+    const router = useRouter();
+    const { toast } = useToast();
+
+    const {
+        numCompleted,
+        numInProgress,
+        numPending,
+        numFailed,
+        numNoJobs,
+    } = useMemo(() => {
+        return {
+            numCompleted: data.filter((connection) => connection.latestJobStatus === "COMPLETED").length,
+            numInProgress: data.filter((connection) => connection.latestJobStatus === "IN_PROGRESS").length,
+            numPending: data.filter((connection) => connection.latestJobStatus === "PENDING").length,
+            numFailed: data.filter((connection) => connection.latestJobStatus === "FAILED").length,
+            numNoJobs: data.filter((connection) => connection.latestJobStatus === null).length,
+        }
+    }, [data]);
+
+    const table = useReactTable({
+        data,
+        columns,
+        onSortingChange: setSorting,
+        onColumnFiltersChange: setColumnFilters,
+        getCoreRowModel: getCoreRowModel(),
+        getPaginationRowModel: getPaginationRowModel(),
+        getSortedRowModel: getSortedRowModel(),
+        getFilteredRowModel: getFilteredRowModel(),
+        onColumnVisibilityChange: setColumnVisibility,
+        onRowSelectionChange: setRowSelection,
+        columnResizeMode: 'onChange',
+        enableColumnResizing: false,
+        state: {
+            sorting,
+            columnFilters,
+            columnVisibility,
+            rowSelection,
+        },
+    })
+
+    return (
+        <div className="w-full">
+            <div className="flex items-center gap-4 py-4">
+                <Input
+                    placeholder="Filter connections..."
+                    value={(table.getColumn("name")?.getFilterValue() as string) ?? ""}
+                    onChange={(event) => table.getColumn("name")?.setFilterValue(event.target.value)}
+                    className="max-w-sm"
+                />
+                <Select
+                    value={(table.getColumn("latestJobStatus")?.getFilterValue() as string) ?? "all"}
+                    onValueChange={(value) => {
+                        table.getColumn("latestJobStatus")?.setFilterValue(value === "all" ? "" : value)
+                    }}
+                >
+                    <SelectTrigger className="w-[180px]">
+                        <SelectValue placeholder="Filter by status" />
+                    </SelectTrigger>
+                    <SelectContent>
+                        <SelectItem value="all">Filter by status</SelectItem>
+                        <SelectItem value="COMPLETED">Completed ({numCompleted})</SelectItem>
+                        <SelectItem value="IN_PROGRESS">In progress ({numInProgress})</SelectItem>
+                        <SelectItem value="PENDING">Pending ({numPending})</SelectItem>
+                        <SelectItem value="FAILED">Failed ({numFailed})</SelectItem>
+                        <SelectItem value="null">No status ({numNoJobs})</SelectItem>
+                    </SelectContent>
+                </Select>
+                <Button
+                    variant="outline"
+                    className="ml-auto"
+                    onClick={() => {
+                        router.refresh();
+                        toast({
+                            description: "Page refreshed",
+                        });
+                    }}
+                >
+                    <RefreshCwIcon className="w-3 h-3" />
+                    Refresh
+                </Button>
+            </div>
+            <div className="rounded-md border">
+                <Table style={{ width: '100%' }}>
+                    <TableHeader>
+                        {table.getHeaderGroups().map((headerGroup) => (
+                            <TableRow key={headerGroup.id}>
+                                {headerGroup.headers.map((header) => {
+                                    return (
+                                        <TableHead
+                                            key={header.id}
+                                            style={{ width: `${header.getSize()}px` }}
+                                        >
+                                            {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
+                                        </TableHead>
+                                    )
+                                })}
+                            </TableRow>
+                        ))}
+                    </TableHeader>
+                    <TableBody>
+                        {table.getRowModel().rows?.length ? (
+                            table.getRowModel().rows.map((row) => (
+                                <TableRow key={row.id} data-state={row.getIsSelected() && "selected"}>
+                                    {row.getVisibleCells().map((cell) => (
+                                        <TableCell
+                                            key={cell.id}
+                                            style={{ width: `${cell.column.getSize()}px` }}
+                                        >
+                                            {flexRender(cell.column.columnDef.cell, cell.getContext())}
+                                        </TableCell>
+                                    ))}
+                                </TableRow>
+                            ))
+                        ) : (
+                            <TableRow>
+                                <TableCell colSpan={columns.length} className="h-24 text-center">
+                                    No results.
+                                </TableCell>
+                            </TableRow>
+                        )}
+                    </TableBody>
+                </Table>
+            </div>
+            <div className="flex items-center justify-end space-x-2 py-4">
+                <div className="flex-1 text-sm text-muted-foreground">
+                    {table.getFilteredRowModel().rows.length} {data.length > 1 ? 'connections' : 'connection'} total
+                </div>
+                <div className="space-x-2">
+                    <Button
+                        variant="outline"
+                        size="sm"
+                        onClick={() => table.previousPage()}
+                        disabled={!table.getCanPreviousPage()}
+                    >
+                        Previous
+                    </Button>
+                    <Button variant="outline" size="sm" onClick={() => table.nextPage()} disabled={!table.getCanNextPage()}>
+                        Next
+                    </Button>
+                </div>
+            </div>
+        </div>
+    )
+}

packages/web/src/app/[domain]/settings/connections/layout.tsx

@@ -0,0 +1,39 @@
+import { getMe } from "@/actions";
+import { getOrgFromDomain } from "@/data/org";
+import { ServiceErrorException } from "@/lib/serviceError";
+import { notFound } from "next/navigation";
+import { isServiceError } from "@/lib/utils";
+import { OrgRole } from "@sourcebot/db";
+
+
+interface ConnectionsLayoutProps {
+    children: React.ReactNode;
+    params: Promise<{
+        domain: string
+    }>;
+}
+
+export default async function ConnectionsLayout({ children, params }: ConnectionsLayoutProps) {
+    const { domain } = await params;
+
+    const org = await getOrgFromDomain(domain);
+    if (!org) {
+        throw new Error("Organization not found");
+    }
+
+    const me = await getMe();
+    if (isServiceError(me)) {
+        throw new ServiceErrorException(me);
+    }
+
+    const userRoleInOrg = me.memberships.find((membership) => membership.id === org.id)?.role;
+    if (!userRoleInOrg) {
+        throw new Error("User role not found");
+    }
+
+    if (userRoleInOrg !== OrgRole.OWNER) {
+        return notFound();
+    }
+
+    return children;
+}

packages/web/src/app/[domain]/settings/connections/page.tsx

@@ -0,0 +1,77 @@
+import { sew } from "@/actions";
+import { ServiceErrorException } from "@/lib/serviceError";
+import { CodeHostType, isServiceError } from "@/lib/utils";
+import { withAuthV2 } from "@/withAuthV2";
+import Link from "next/link";
+import { ConnectionsTable } from "./components/connectionsTable";
+import { ConnectionSyncJobStatus } from "@prisma/client";
+
+const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/overview";
+
+export default async function ConnectionsPage() {
+    const _connections = await getConnectionsWithLatestJob();
+    if (isServiceError(_connections)) {
+        throw new ServiceErrorException(_connections);
+    }
+
+    // Sort connections so that first time syncs are at the top.
+    const connections = _connections
+        .map((connection) => ({
+            ...connection,
+            isFirstTimeSync: connection.syncedAt === null && connection.syncJobs.filter((job) => job.status === ConnectionSyncJobStatus.PENDING || job.status === ConnectionSyncJobStatus.IN_PROGRESS).length > 0,
+            latestJobStatus: connection.syncJobs.length > 0 ? connection.syncJobs[0].status : null,
+        }))
+        .sort((a, b) => {
+            if (a.isFirstTimeSync && !b.isFirstTimeSync) {
+                return -1;
+            }
+            if (!a.isFirstTimeSync && b.isFirstTimeSync) {
+                return 1;
+            }
+            return a.name.localeCompare(b.name);
+        });
+
+    return (
+        <div className="flex flex-col gap-6">
+            <div>
+                <h3 className="text-lg font-medium">Code Host Connections</h3>
+                <p className="text-sm text-muted-foreground">Manage your connections to external code hosts. <Link href={DOCS_URL} target="_blank" className="text-link hover:underline">Learn more</Link></p>
+            </div>
+            <ConnectionsTable data={connections.map((connection) => ({
+                id: connection.id,
+                name: connection.name,
+                codeHostType: connection.connectionType as CodeHostType,
+                syncedAt: connection.syncedAt,
+                latestJobStatus: connection.latestJobStatus,
+                isFirstTimeSync: connection.isFirstTimeSync,
+            }))} />
+        </div>
+    )
+}
+
+const getConnectionsWithLatestJob = async () => sew(() =>
+    withAuthV2(async ({ prisma, org }) => {
+        const connections = await prisma.connection.findMany({
+            where: {
+                orgId: org.id,
+            },
+            include: {
+                _count: {
+                    select: {
+                        syncJobs: true,
+                    }
+                },
+                syncJobs: {
+                    orderBy: {
+                        createdAt: 'desc'
+                    },
+                    take: 1
+                },
+            },
+            orderBy: {
+                name: 'asc'
+            },
+        });
+
+        return connections;
+    }));

packages/web/src/app/[domain]/settings/layout.tsx

@@ -1,13 +1,12 @@
 import React from "react"
 import { Metadata } from "next"
-import { SidebarNav } from "./components/sidebar-nav"
+import { SidebarNav, SidebarNavItem } from "./components/sidebar-nav"
 import { NavigationMenu } from "../components/navigationMenu"
-import { Header } from "./components/header";
 import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe";
 import { redirect } from "next/navigation";
 import { auth } from "@/auth";
 import { isServiceError } from "@/lib/utils";
-import { getMe, getOrgAccountRequests } from "@/actions";
+import { getConnectionStats, getMe, getOrgAccountRequests } from "@/actions";
 import { ServiceErrorException } from "@/lib/serviceError";
 import { getOrgFromDomain } from "@/data/org";
 import { OrgRole } from "@prisma/client";
@@ -64,7 +63,12 @@ export default async function SettingsLayout(
         numJoinRequests = requests.length;
     }
 
-    const sidebarNavItems = [
+    const connectionStats = await getConnectionStats();
+    if (isServiceError(connectionStats)) {
+        throw new ServiceErrorException(connectionStats);
+    }
+
+    const sidebarNavItems: SidebarNavItem[] = [
         {
             title: "General",
             href: `/${domain}/settings`,
@@ -94,6 +98,14 @@ export default async function SettingsLayout(
             ),
             href: `/${domain}/settings/members`,
         }] : []),
+        ...(userRoleInOrg === OrgRole.OWNER ? [
+            {
+                title: "Connections",
+                href: `/${domain}/settings/connections`,
+                hrefRegex: `/${domain}/settings/connections(/[^/]+)?$`,
+                isNotificationDotVisible: connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0,
+            }
+        ] : []),
         {
             title: "Secrets",
             href: `/${domain}/settings/secrets`,
@@ -115,21 +127,24 @@ export default async function SettingsLayout(
     ]
 
     return (
-        <div className="min-h-screen flex flex-col bg-backgroundSecondary">
+        <div className="min-h-screen flex flex-col">
             <NavigationMenu domain={domain} />
-            <div className="flex-grow flex justify-center p-4 relative">
-                <div className="w-full max-w-6xl p-6">
-                    <Header className="w-full">
-                        <h1 className="text-3xl">Settings</h1>
-                    </Header>
-                    <div className="flex flex-row gap-10 mt-20">
-                        <aside className="lg:w-48">
-                            <SidebarNav items={sidebarNavItems} />
-                        </aside>
-                        <div className="w-full rounded-lg">{children}</div>
+            <main className="flex-grow flex justify-center p-4 bg-backgroundSecondary relative">
+                <div className="w-full max-w-6xl rounded-lg p-6">
+                    <div className="container mx-auto">
+                        <div className="mb-16">
+                            <h1 className="text-3xl font-semibold">Settings</h1>
+                        </div>
+                        <div className="flex flex-row gap-10">
+                            <aside className="lg:w-48">
+                                <SidebarNav items={sidebarNavItems} />
+                            </aside>
+                            <div className="w-full rounded-lg">{children}</div>
+                        </div>
                     </div>
                 </div>
-            </div>
+            </main>
         </div>
     )
 }
+

packages/web/src/app/[domain]/settings/secrets/components/importSecretCard.tsx

@@ -27,7 +27,7 @@ export const ImportSecretCard = ({ className }: ImportSecretCardProps) => {
                 <CardContent className="flex flex-row gap-4 w-full justify-center">
                     <CodeHostIconButton
                         name="GitHub"
-                        logo={getCodeHostIcon("github")!}
+                        logo={getCodeHostIcon("github")}
                         onClick={() => {
                             setSelectedCodeHost("github");
                             setIsImportSecretDialogOpen(true);
@@ -35,7 +35,7 @@ export const ImportSecretCard = ({ className }: ImportSecretCardProps) => {
                     />
                     <CodeHostIconButton
                         name="GitLab"
-                        logo={getCodeHostIcon("gitlab")!}
+                        logo={getCodeHostIcon("gitlab")}
                         onClick={() => {
                             setSelectedCodeHost("gitlab");
                             setIsImportSecretDialogOpen(true);
@@ -43,7 +43,7 @@ export const ImportSecretCard = ({ className }: ImportSecretCardProps) => {
                     />
                     <CodeHostIconButton
                         name="Gitea"
-                        logo={getCodeHostIcon("gitea")!}
+                        logo={getCodeHostIcon("gitea")}
                         onClick={() => {
                             setSelectedCodeHost("gitea");
                             setIsImportSecretDialogOpen(true);

packages/web/src/app/api/(server)/stripe/route.ts

@@ -2,7 +2,7 @@ import { headers } from 'next/headers';
 import { NextRequest } from 'next/server';
 import Stripe from 'stripe';
 import { prisma } from '@/prisma';
-import { ConnectionSyncStatus, StripeSubscriptionStatus } from '@sourcebot/db';
+import { StripeSubscriptionStatus } from '@sourcebot/db';
 import { stripeClient } from '@/ee/features/billing/stripe';
 import { env } from '@/env.mjs';
 import { createLogger } from "@sourcebot/logger";
@@ -85,16 +85,6 @@ export async function POST(req: NextRequest) {
             });
             logger.info(`Org ${org.id} subscription status updated to ACTIVE`);
 
-            // mark all of this org's connections for sync, since their repos may have been previously garbage collected
-            await prisma.connection.updateMany({
-                where: {
-                    orgId: org.id
-                },
-                data: {
-                    syncStatus: ConnectionSyncStatus.SYNC_NEEDED
-                }
-            });
-
             return new Response(JSON.stringify({ received: true }), {
                 status: 200
             });

packages/web/src/features/chat/components/searchScopeIcon.tsx

@@ -1,5 +1,5 @@
-import { cn, getCodeHostIcon } from "@/lib/utils";
-import { FolderIcon, LibraryBigIcon } from "lucide-react";
+import { cn, CodeHostType, getCodeHostIcon } from "@/lib/utils";
+import { LibraryBigIcon } from "lucide-react";
 import Image from "next/image";
 import { SearchScope } from "../types";
 
@@ -13,20 +13,16 @@ export const SearchScopeIcon = ({ searchScope, className = "h-4 w-4" }: SearchSc
         return <LibraryBigIcon className={cn(className, "text-muted-foreground flex-shrink-0")} />;
     } else {
         // Render code host icon for repos
-        const codeHostIcon = searchScope.codeHostType ? getCodeHostIcon(searchScope.codeHostType) : null;
-        if (codeHostIcon) {
-            const size = className.includes('h-3') ? 12 : 16;
-            return (
-                <Image
-                    src={codeHostIcon.src}
-                    alt={`${searchScope.codeHostType} icon`}
-                    width={size}
-                    height={size}
-                    className={cn(className, "flex-shrink-0", codeHostIcon.className)}
-                />
-            );
-        } else {
-            return <FolderIcon className={cn(className, "text-muted-foreground flex-shrink-0")} />;
-        }
+        const codeHostIcon = getCodeHostIcon(searchScope.codeHostType as CodeHostType);
+        const size = className.includes('h-3') ? 12 : 16;
+        return (
+            <Image
+                src={codeHostIcon.src}
+                alt={`${searchScope.codeHostType} icon`}
+                width={size}
+                height={size}
+                className={cn(className, "flex-shrink-0", codeHostIcon.className)}
+            />
+        );
     }
 }; 

packages/web/src/initialize.ts

@@ -1,129 +1,17 @@
-import { ConnectionSyncStatus, OrgRole, Prisma } from '@sourcebot/db';
-import { env } from './env.mjs';
-import { prisma } from "@/prisma";
-import { SINGLE_TENANT_ORG_ID, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SINGLE_TENANT_ORG_NAME } from './lib/constants';
-import chokidar from 'chokidar';
-import { ConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
-import { hasEntitlement, loadConfig, isRemotePath, syncSearchContexts } from '@sourcebot/shared';
-import { isServiceError, getOrgMetadata } from './lib/utils';
-import { ServiceErrorException } from './lib/serviceError';
-import { SOURCEBOT_SUPPORT_EMAIL } from "@/lib/constants";
-import { createLogger } from "@sourcebot/logger";
 import { createGuestUser } from '@/lib/authUtils';
+import { SOURCEBOT_SUPPORT_EMAIL } from "@/lib/constants";
+import { prisma } from "@/prisma";
+import { OrgRole } from '@sourcebot/db';
+import { createLogger } from "@sourcebot/logger";
+import { hasEntitlement, loadConfig } from '@sourcebot/shared';
 import { getOrgFromDomain } from './data/org';
+import { env } from './env.mjs';
+import { SINGLE_TENANT_ORG_DOMAIN, SINGLE_TENANT_ORG_ID, SOURCEBOT_GUEST_USER_ID } from './lib/constants';
+import { ServiceErrorException } from './lib/serviceError';
+import { getOrgMetadata, isServiceError } from './lib/utils';
 
 const logger = createLogger('web-initialize');
 
-const syncConnections = async (connections?: { [key: string]: ConnectionConfig }) => {
-    if (connections) {
-        for (const [key, newConnectionConfig] of Object.entries(connections)) {
-            const currentConnection = await prisma.connection.findUnique({
-                where: {
-                    name_orgId: {
-                        name: key,
-                        orgId: SINGLE_TENANT_ORG_ID,
-                    }
-                },
-                include: {
-                    repos: {
-                        include: {
-                            repo: true,
-                        }
-                    }
-                }
-            });
-
-            const currentConnectionConfig = currentConnection ? currentConnection.config as unknown as ConnectionConfig : undefined;
-            const syncNeededOnUpdate =
-                (currentConnectionConfig && JSON.stringify(currentConnectionConfig) !== JSON.stringify(newConnectionConfig)) ||
-                (currentConnection?.syncStatus === ConnectionSyncStatus.FAILED);
-
-            const connectionDb = await prisma.connection.upsert({
-                where: {
-                    name_orgId: {
-                        name: key,
-                        orgId: SINGLE_TENANT_ORG_ID,
-                    }
-                },
-                update: {
-                    config: newConnectionConfig as unknown as Prisma.InputJsonValue,
-                    syncStatus: syncNeededOnUpdate ? ConnectionSyncStatus.SYNC_NEEDED : undefined,
-                    isDeclarative: true,
-                },
-                create: {
-                    name: key,
-                    connectionType: newConnectionConfig.type,
-                    config: newConnectionConfig as unknown as Prisma.InputJsonValue,
-                    isDeclarative: true,
-                    org: {
-                        connect: {
-                            id: SINGLE_TENANT_ORG_ID,
-                        }
-                    }
-                }
-            });
-
-            logger.info(`Upserted connection with name '${key}'. Connection ID: ${connectionDb.id}`);
-        }
-    }
-
-    // Delete any connections that are no longer in the config.
-    const deletedConnections = await prisma.connection.findMany({
-        where: {
-            isDeclarative: true,
-            name: {
-                notIn: Object.keys(connections ?? {}),
-            },
-            orgId: SINGLE_TENANT_ORG_ID,
-        }
-    });
-
-    for (const connection of deletedConnections) {
-        logger.info(`Deleting connection with name '${connection.name}'. Connection ID: ${connection.id}`);
-        await prisma.connection.delete({
-            where: {
-                id: connection.id,
-            }
-        })
-    }
-}
-
-const syncDeclarativeConfig = async (configPath: string) => {
-    const config = await loadConfig(configPath);
-
-    const forceEnableAnonymousAccess = config.settings?.enablePublicAccess ?? env.FORCE_ENABLE_ANONYMOUS_ACCESS === 'true';
-    if (forceEnableAnonymousAccess) {
-        const hasAnonymousAccessEntitlement = hasEntitlement("anonymous-access");
-        if (!hasAnonymousAccessEntitlement) {
-            logger.warn(`FORCE_ENABLE_ANONYMOUS_ACCESS env var is set to true but anonymous access entitlement is not available. Setting will be ignored.`);
-        } else {
-            const org = await getOrgFromDomain(SINGLE_TENANT_ORG_DOMAIN);
-            if (org) {
-                const currentMetadata = getOrgMetadata(org);
-                const mergedMetadata = {
-                    ...(currentMetadata ?? {}),
-                    anonymousAccessEnabled: true,
-                };
-                
-                await prisma.org.update({
-                    where: { id: org.id },
-                    data: { 
-                        metadata: mergedMetadata,
-                    },
-                });
-                logger.info(`Anonymous access enabled via FORCE_ENABLE_ANONYMOUS_ACCESS environment variable`);
-            }
-        }
-    }
-
-    await syncConnections(config.connections);
-    await syncSearchContexts({
-        contexts: config.contexts,
-        orgId: SINGLE_TENANT_ORG_ID,
-        db: prisma,
-    });
-}
-
 const pruneOldGuestUser = async () => {
     // The old guest user doesn't have the GUEST role
     const guestUser = await prisma.userToOrg.findUnique({
@@ -150,35 +38,6 @@ const pruneOldGuestUser = async () => {
 }
 
 const initSingleTenancy = async () => {
-    // Back fill the inviteId if the org has already been created to prevent needing to wipe the db
-    await prisma.$transaction(async (tx) => {
-        const org = await tx.org.findUnique({
-            where: {
-                id: SINGLE_TENANT_ORG_ID,
-            },
-        });
-
-        if (!org) {
-            await tx.org.create({
-                data: {
-                    id: SINGLE_TENANT_ORG_ID,
-                    name: SINGLE_TENANT_ORG_NAME,
-                    domain: SINGLE_TENANT_ORG_DOMAIN,
-                    inviteLinkId: crypto.randomUUID(),
-                }
-            });
-        } else if (!org.inviteLinkId) {
-            await tx.org.update({
-                where: {
-                    id: SINGLE_TENANT_ORG_ID,
-                },
-                data: {
-                    inviteLinkId: crypto.randomUUID(),
-                }
-            });
-        }
-    });
-
     // This is needed because v4 introduces the GUEST org role as well as making authentication required. 
     // To keep things simple, we'll just delete the old guest user if it exists in the DB
     await pruneOldGuestUser();
@@ -205,30 +64,32 @@ const initSingleTenancy = async () => {
         }
     }
 
-    // Load any connections defined declaratively in the config file.
-    const configPath = env.CONFIG_PATH;
-    if (configPath) {
-        await syncDeclarativeConfig(configPath);
+    // Sync anonymous access config from the config file
+    if (env.CONFIG_PATH) {
+        const config = await loadConfig(env.CONFIG_PATH);
+        const forceEnableAnonymousAccess = config.settings?.enablePublicAccess ?? env.FORCE_ENABLE_ANONYMOUS_ACCESS === 'true';
 
-        // watch for changes assuming it is a local file
-        if (!isRemotePath(configPath)) {
-            const watcher = chokidar.watch(configPath, {
-                ignoreInitial: true,           // Don't fire events for existing files
-                awaitWriteFinish: {
-                    stabilityThreshold: 100,   // File size stable for 100ms
-                    pollInterval: 100          // Check every 100ms
-                },
-                atomic: true                   // Handle atomic writes (temp file + rename)
-            });
+        if (forceEnableAnonymousAccess) {
+            if (!hasAnonymousAccessEntitlement) {
+                logger.warn(`FORCE_ENABLE_ANONYMOUS_ACCESS env var is set to true but anonymous access entitlement is not available. Setting will be ignored.`);
+            } else {
+                const org = await getOrgFromDomain(SINGLE_TENANT_ORG_DOMAIN);
+                if (org) {
+                    const currentMetadata = getOrgMetadata(org);
+                    const mergedMetadata = {
+                        ...(currentMetadata ?? {}),
+                        anonymousAccessEnabled: true,
+                    };
 
-            watcher.on('change', async () => {
-                logger.info(`Config file ${configPath} changed. Re-syncing...`);
-                try {
-                    await syncDeclarativeConfig(configPath);
-                } catch (error) {
-                    logger.error(`Failed to sync config: ${error}`);
+                    await prisma.org.update({
+                        where: { id: org.id },
+                        data: {
+                            metadata: mergedMetadata,
+                        },
+                    });
+                    logger.info(`Anonymous access enabled via FORCE_ENABLE_ANONYMOUS_ACCESS environment variable`);
                 }
-            });
+            }
         }
     }
 }

packages/web/src/lib/syncStatusMetadataSchema.ts

@@ -1,17 +0,0 @@
-import { z } from "zod";
-
-export const NotFoundSchema = z.object({
-  users: z.array(z.string()),
-  orgs: z.array(z.string()),
-  repos: z.array(z.string()),
-});
-
-export const SyncStatusMetadataSchema = z.object({
-  notFound: NotFoundSchema.optional(),
-  error: z.string().optional(),
-  secretKey: z.string().optional(),
-  status: z.number().optional(),
-});
-
-export type NotFoundData = z.infer<typeof NotFoundSchema>;
-export type SyncStatusMetadata = z.infer<typeof SyncStatusMetadataSchema>;

packages/web/src/lib/utils.ts

@@ -145,7 +145,7 @@ export const getAuthProviderInfo = (providerId: string): AuthProviderInfo => {
                 id: "microsoft-entra-id",
                 name: "Microsoft Entra ID",
                 displayName: "Microsoft Entra ID",
-               icon: {
+                icon: {
                     src: microsoftLogo,
                 },
             };
@@ -283,7 +283,7 @@ export const getCodeHostInfoForRepo = (repo: {
     }
 }
 
-export const getCodeHostIcon = (codeHostType: string): { src: string, className?: string } | null => {
+export const getCodeHostIcon = (codeHostType: CodeHostType): { src: string, className?: string } => {
     switch (codeHostType) {
         case "github":
             return {
@@ -315,8 +315,6 @@ export const getCodeHostIcon = (codeHostType: string): { src: string, className?
             return {
                 src: gitLogo,
             }
-        default:
-            return null;
     }
 }
 

schemas/v3/app.json

@@ -1,9 +1,44 @@
 {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "AppConfig",
+    "definitions": {
+        "GitHubAppConfig": {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "const": "githubApp",
+                    "description": "GitHub App Configuration"
+                },
+                "deploymentHostname": {
+                    "type": "string",
+                    "format": "hostname",
+                    "default": "github.com",
+                    "description": "The hostname of the GitHub App deployment.",
+                    "examples": [
+                        "github.com",
+                        "github.example.com"
+                    ]
+                },
+                "id": {
+                    "type": "string",
+                    "description": "The ID of the GitHub App."
+                },
+                "privateKey": {
+                    "$ref": "./shared.json#/definitions/Token",
+                    "description": "The private key of the GitHub App."
+                }
+            },
+            "required": [
+                "type",
+                "id",
+                "privateKey"
+            ],
+            "additionalProperties": false
+        }
+    },
     "oneOf": [
         {
-            "$ref": "./githubApp.json"
+            "$ref": "#/definitions/GitHubAppConfig"
         }
     ]
 }

schemas/v3/githubApp.json

@@ -1,42 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "type": "object",
-    "title": "GithubAppConfig",
-    "properties": {
-        "type": {
-            "const": "githubApp",
-            "description": "GitHub App Configuration"
-        },
-        "deploymentHostname": {
-            "type": "string",
-            "format": "hostname",
-            "default": "github.com",
-            "description": "The hostname of the GitHub App deployment.",
-            "examples": [
-                "github.com",
-                "github.example.com"
-            ]
-        },
-        "id": {
-            "type": "string",
-            "description": "The ID of the GitHub App."
-        },
-        "privateKey": {
-            "$ref": "./shared.json#/definitions/Token",
-            "description": "The private key of the GitHub App."
-        }
-    },
-    "required": [
-        "type",
-        "id"
-    ],
-    "oneOf": [
-        {
-            "required": ["privateKey"]
-        },
-        {
-            "required": ["privateKeyPath"]
-        }
-    ],
-    "additionalProperties": false
-}

yarn.lock

@@ -7796,6 +7796,7 @@ __metadata:
     argparse: "npm:^2.0.1"
     azure-devops-node-api: "npm:^15.1.1"
     bullmq: "npm:^5.34.10"
+    chokidar: "npm:^4.0.3"
     cross-env: "npm:^7.0.3"
     cross-fetch: "npm:^4.0.0"
     dotenv: "npm:^16.4.5"
@@ -8055,7 +8056,6 @@ __metadata:
     ai: "npm:^5.0.45"
     ajv: "npm:^8.17.1"
     bcryptjs: "npm:^3.0.2"
-    chokidar: "npm:^4.0.3"
     class-variance-authority: "npm:^0.7.0"
     client-only: "npm:^0.0.1"
     clsx: "npm:^2.1.1"