diff --git a/Dockerfile b/Dockerfile index 2871ed3a..ff5eff9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,11 @@ COPY package.json yarn.lock* ./ COPY ./packages/db ./packages/db COPY ./packages/schemas ./packages/schemas COPY ./packages/crypto ./packages/crypto +COPY ./packages/error ./packages/error RUN yarn workspace @sourcebot/db install --frozen-lockfile RUN yarn workspace @sourcebot/schemas install --frozen-lockfile RUN yarn workspace @sourcebot/crypto install --frozen-lockfile +RUN yarn workspace @sourcebot/error install --frozen-lockfile # ------ Build Web ------ FROM node-alpine AS web-builder @@ -33,6 +35,7 @@ COPY --from=shared-libs-builder /app/node_modules ./node_modules COPY --from=shared-libs-builder /app/packages/db ./packages/db COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto +COPY --from=shared-libs-builder /app/packages/error ./packages/error # Fixes arm64 timeouts RUN yarn config set registry https://registry.npmjs.org/ @@ -66,6 +69,7 @@ COPY --from=shared-libs-builder /app/node_modules ./node_modules COPY --from=shared-libs-builder /app/packages/db ./packages/db COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto +COPY --from=shared-libs-builder /app/packages/error ./packages/error RUN yarn workspace @sourcebot/backend install --frozen-lockfile RUN yarn workspace @sourcebot/backend build @@ -138,6 +142,7 @@ COPY --from=shared-libs-builder /app/node_modules ./node_modules COPY --from=shared-libs-builder /app/packages/db ./packages/db COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto +COPY --from=shared-libs-builder /app/packages/error ./packages/error # Configure the database RUN mkdir -p /run/postgresql && \ diff --git a/Makefile b/Makefile index b83f7a60..abae628d 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,8 @@ clean: packages/schemas/dist \ packages/crypto/node_modules \ packages/crypto/dist \ + packages/error/node_modules \ + packages/error/dist \ .sourcebot soft-reset: diff --git a/packages/backend/package.json b/packages/backend/package.json index 16987364..01a4a36f 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -34,6 +34,7 @@ "@sourcebot/crypto": "^0.1.0", "@sourcebot/db": "^0.1.0", "@sourcebot/schemas": "^0.1.0", + "@sourcebot/error": "^0.1.0", "simple-git": "^3.27.0", "strip-json-comments": "^5.0.1", "winston": "^3.15.0", diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index 0990c6c6..250d99f5 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -6,6 +6,7 @@ import { createLogger } from "./logger.js"; import os from 'os'; import { Redis } from 'ioredis'; import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig } from "./repoCompileUtils.js"; +import { BackendError, BackendException } from "@sourcebot/error"; interface IConnectionManager { scheduleConnectionSync: (connection: Connection) => Promise; @@ -81,26 +82,93 @@ export class ConnectionManager implements IConnectionManager { // @note: We aren't actually doing anything with this atm. const abortController = new AbortController(); - const repoData: RepoData[] = await (async () => { - switch (config.type) { - case 'github': { - return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController); - } - case 'gitlab': { - return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db); - } - case 'gitea': { - return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db); - } - case 'gerrit': { - return await compileGerritConfig(config, job.data.connectionId, orgId); - } - default: { - return []; - } - } - })(); + const connection = await this.db.connection.findUnique({ + where: { + id: job.data.connectionId, + }, + }); + if (!connection) { + throw new BackendException(BackendError.CONNECTION_SYNC_CONNECTION_NOT_FOUND, { + message: `Connection ${job.data.connectionId} not found`, + }); + } + + // Reset the syncStatusMetadata to an empty object at the start of the sync job + await this.db.connection.update({ + where: { + id: job.data.connectionId, + }, + data: { + syncStatusMetadata: {} + } + }) + + + let result: { + repoData: RepoData[], + notFound: { + users: string[], + orgs: string[], + repos: string[], + } + } = { + repoData: [], + notFound: { + users: [], + orgs: [], + repos: [], + } + }; + + try { + result = await (async () => { + switch (config.type) { + case 'github': { + return await compileGithubConfig(config, job.data.connectionId, orgId, this.db, abortController); + } + case 'gitlab': { + return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db); + } + case 'gitea': { + return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db); + } + case 'gerrit': { + return await compileGerritConfig(config, job.data.connectionId, orgId); + } + default: { + return {repoData: [], notFound: { + users: [], + orgs: [], + repos: [], + }}; + } + } + })(); + } catch (err) { + this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId}: ${err}`); + if (err instanceof BackendException) { + throw err; + } else { + throw new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, { + message: `Failed to compile repo data for connection ${job.data.connectionId}`, + }); + } + } + + const { repoData, notFound } = result; + + // Push the information regarding not found users, orgs, and repos to the connection's syncStatusMetadata. Note that + // this won't be overwritten even if the connection job fails + await this.db.connection.update({ + where: { + id: job.data.connectionId, + }, + data: { + syncStatusMetadata: { notFound } + } + }); + // Filter out any duplicates by external_id and external_codeHostUrl. repoData.filter((repo, index, self) => { return index === self.findIndex(r => @@ -265,16 +333,37 @@ export class ConnectionManager implements IConnectionManager { private async onSyncJobFailed(job: Job | undefined, err: unknown) { this.logger.info(`Connection sync job failed with error: ${err}`); if (job) { + + // We may have pushed some metadata during the execution of the job, so we make sure to not overwrite the metadata here const { connectionId } = job.data; + let syncStatusMetadata: Record = (await this.db.connection.findUnique({ + where: { id: connectionId }, + select: { syncStatusMetadata: true } + }))?.syncStatusMetadata as Record ?? {}; + + if (err instanceof BackendException) { + syncStatusMetadata = { + ...syncStatusMetadata, + error: err.code, + ...err.metadata, + } + } else { + syncStatusMetadata = { + ...syncStatusMetadata, + error: 'UNKNOWN', + } + } + await this.db.connection.update({ where: { id: connectionId, }, data: { syncStatus: ConnectionSyncStatus.FAILED, - syncedAt: new Date() + syncedAt: new Date(), + syncStatusMetadata: syncStatusMetadata as Prisma.InputJsonValue, } - }) + }); } } diff --git a/packages/backend/src/connectionUtils.ts b/packages/backend/src/connectionUtils.ts new file mode 100644 index 00000000..810e9685 --- /dev/null +++ b/packages/backend/src/connectionUtils.ts @@ -0,0 +1,44 @@ +type ValidResult = { + type: 'valid'; + data: T[]; +}; + +type NotFoundResult = { + type: 'notFound'; + value: string; +}; + +type CustomResult = ValidResult | NotFoundResult; + +export function processPromiseResults( + results: PromiseSettledResult>[], +): { + validItems: T[]; + notFoundItems: string[]; +} { + const validItems: T[] = []; + const notFoundItems: string[] = []; + + results.forEach(result => { + if (result.status === 'fulfilled') { + const value = result.value; + if (value.type === 'valid') { + validItems.push(...value.data); + } else { + notFoundItems.push(value.value); + } + } + }); + + return { + validItems, + notFoundItems, + }; +} + +export function throwIfAnyFailed(results: PromiseSettledResult[]) { + const failedResult = results.find(result => result.status === 'rejected'); + if (failedResult) { + throw failedResult.reason; + } +} \ No newline at end of file diff --git a/packages/backend/src/gerrit.ts b/packages/backend/src/gerrit.ts index 336633c4..c6d96d50 100644 --- a/packages/backend/src/gerrit.ts +++ b/packages/backend/src/gerrit.ts @@ -3,6 +3,8 @@ import { GerritConfig } from "@sourcebot/schemas/v2/index.type" import { createLogger } from './logger.js'; import micromatch from "micromatch"; import { measure, marshalBool, excludeReposByName, includeReposByName, fetchWithRetry } from './utils.js'; +import { BackendError } from '@sourcebot/error'; +import { BackendException } from '@sourcebot/error'; // https://gerrit-review.googlesource.com/Documentation/rest-api.html interface GerritProjects { @@ -38,6 +40,10 @@ export const getGerritReposFromConfig = async (config: GerritConfig): Promise fetchAllProjects(url); return fetchWithRetry(fetchFn, `projects from ${url}`, logger); } catch (err) { + if (err instanceof BackendException) { + throw err; + } + logger.error(`Failed to fetch projects from ${url}`, err); return null; } @@ -78,9 +84,25 @@ const fetchAllProjects = async (url: string): Promise => { const endpointWithParams = `${projectsEndpoint}?S=${start}`; logger.debug(`Fetching projects from Gerrit at ${endpointWithParams}`); - const response = await fetch(endpointWithParams); - if (!response.ok) { - throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`); + let response: Response; + try { + response = await fetch(endpointWithParams); + if (!response.ok) { + console.log(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${response.status}`); + throw new BackendException(BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS, { + status: response.status, + }); + } + } catch (err) { + if (err instanceof BackendException) { + throw err; + } + + const status = (err as any).code; + console.log(`Failed to fetch projects from Gerrit at ${endpointWithParams} with status ${status}`); + throw new BackendException(BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS, { + status: status, + }); } const text = await response.text(); diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index b3f20c7d..5fd8f5ba 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -6,43 +6,45 @@ import { createLogger } from './logger.js'; import micromatch from 'micromatch'; import { PrismaClient } from '@sourcebot/db'; import { FALLBACK_GITEA_TOKEN } from './environment.js'; +import { processPromiseResults, throwIfAnyFailed } from './connectionUtils.js'; const logger = createLogger('Gitea'); export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => { - const token = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; + const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; + const token = tokenResult?.token ?? FALLBACK_GITEA_TOKEN; const api = giteaApi(config.url ?? 'https://gitea.com', { - token: token ?? FALLBACK_GITEA_TOKEN, + token: token, customFetch: fetch, }); let allRepos: GiteaRepository[] = []; + let notFound: { + users: string[], + orgs: string[], + repos: string[], + } = { + users: [], + orgs: [], + repos: [], + }; if (config.orgs) { - const _repos = await fetchWithRetry( - () => getReposForOrgs(config.orgs!, api), - `orgs ${config.orgs.join(', ')}`, - logger - ); - allRepos = allRepos.concat(_repos); + const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, api); + allRepos = allRepos.concat(validRepos); + notFound.orgs = notFoundOrgs; } if (config.repos) { - const _repos = await fetchWithRetry( - () => getRepos(config.repos!, api), - `repos ${config.repos.join(', ')}`, - logger - ); - allRepos = allRepos.concat(_repos); + const { validRepos, notFoundRepos } = await getRepos(config.repos, api); + allRepos = allRepos.concat(validRepos); + notFound.repos = notFoundRepos; } if (config.users) { - const _repos = await fetchWithRetry( - () => getReposOwnedByUsers(config.users!, api), - `users ${config.users.join(', ')}`, - logger - ); - allRepos = allRepos.concat(_repos); + const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, api); + allRepos = allRepos.concat(validRepos); + notFound.users = notFoundUsers; } allRepos = allRepos.filter(repo => repo.full_name !== undefined); @@ -108,7 +110,10 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org }); logger.debug(`Found ${repos.length} total repositories.`); - return repos; + return { + validRepos: repos, + notFound, + }; } const shouldExcludeRepo = ({ @@ -186,7 +191,7 @@ const getBranchesForRepo = async (owner: string, repo: string, api: Api) = } const getReposOwnedByUsers = async (users: string[], api: Api) => { - const repos = (await Promise.all(users.map(async (user) => { + const results = await Promise.allSettled(users.map(async (user) => { try { logger.debug(`Fetching repos for user ${user}...`); @@ -197,18 +202,33 @@ const getReposOwnedByUsers = async (users: string[], api: Api) => { ); logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`); - return data; - } catch (e) { - logger.error(`Failed to fetch repos for user ${user}.`, e); + return { + type: 'valid' as const, + data + }; + } catch (e: any) { + if (e?.status === 404) { + logger.error(`User ${user} not found or no access`); + return { + type: 'notFound' as const, + value: user + }; + } throw e; } - }))).flat(); + })); - return repos; + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results); + + return { + validRepos, + notFoundUsers, + }; } const getReposForOrgs = async (orgs: string[], api: Api) => { - return (await Promise.all(orgs.map(async (org) => { + const results = await Promise.allSettled(orgs.map(async (org) => { try { logger.debug(`Fetching repos for org ${org}...`); @@ -220,16 +240,33 @@ const getReposForOrgs = async (orgs: string[], api: Api) => { ); logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`); - return data; - } catch (e) { - logger.error(`Failed to fetch repos for org ${org}.`, e); + return { + type: 'valid' as const, + data + }; + } catch (e: any) { + if (e?.status === 404) { + logger.error(`Organization ${org} not found or no access`); + return { + type: 'notFound' as const, + value: org + }; + } throw e; } - }))).flat(); + })); + + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + + return { + validRepos, + notFoundOrgs, + }; } const getRepos = async (repos: string[], api: Api) => { - return (await Promise.all(repos.map(async (repo) => { + const results = await Promise.allSettled(repos.map(async (repo) => { try { logger.debug(`Fetching repository info for ${repo}...`); @@ -239,13 +276,29 @@ const getRepos = async (repos: string[], api: Api) => { ); logger.debug(`Found repo ${repo} in ${durationMs}ms.`); - - return [response.data]; - } catch (e) { - logger.error(`Failed to fetch repository info for ${repo}.`, e); + return { + type: 'valid' as const, + data: [response.data] + }; + } catch (e: any) { + if (e?.status === 404) { + logger.error(`Repository ${repo} not found or no access`); + return { + type: 'notFound' as const, + value: repo + }; + } throw e; } - }))).flat(); + })); + + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + + return { + validRepos, + notFoundRepos, + }; } // @see : https://docs.gitea.com/development/api-usage#pagination diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index b3e8f15e..794eb199 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -5,6 +5,8 @@ import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js"; import micromatch from "micromatch"; import { PrismaClient } from "@sourcebot/db"; import { FALLBACK_GITHUB_TOKEN } from "./environment.js"; +import { BackendException, BackendError } from "@sourcebot/error"; +import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; const logger = createLogger("GitHub"); export type OctokitRepository = { @@ -28,8 +30,17 @@ export type OctokitRepository = { } } +const isHttpError = (error: unknown, status: number): boolean => { + return error !== null + && typeof error === 'object' + && 'status' in error + && error.status === status; +} + export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => { - const token = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; + const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; + const token = tokenResult?.token; + const secretKey = tokenResult?.secretKey; const octokit = new Octokit({ auth: token ?? FALLBACK_GITHUB_TOKEN, @@ -38,25 +49,52 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o } : {}), }); + if (token) { + try { + await octokit.rest.users.getAuthenticated(); + } catch (error) { + if (isHttpError(error, 401)) { + throw new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, { + secretKey, + }); + } + + throw new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, { + message: `Failed to authenticate with GitHub`, + }); + } + } + let allRepos: OctokitRepository[] = []; + let notFound: { + users: string[], + orgs: string[], + repos: string[], + } = { + users: [], + orgs: [], + repos: [], + }; if (config.orgs) { - const _repos = await getReposForOrgs(config.orgs, octokit, signal); - allRepos = allRepos.concat(_repos); + const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, octokit, signal); + allRepos = allRepos.concat(validRepos); + notFound.orgs = notFoundOrgs; } if (config.repos) { - const _repos = await getRepos(config.repos, octokit, signal); - allRepos = allRepos.concat(_repos); + const { validRepos, notFoundRepos } = await getRepos(config.repos, octokit, signal); + allRepos = allRepos.concat(validRepos); + notFound.repos = notFoundRepos; } if (config.users) { const isAuthenticated = config.token !== undefined; - const _repos = await getReposOwnedByUsers(config.users, isAuthenticated, octokit, signal); - allRepos = allRepos.concat(_repos); + const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, isAuthenticated, octokit, signal); + allRepos = allRepos.concat(validRepos); + notFound.users = notFoundUsers; } - // Marshall results to our type let repos = allRepos .filter((repo) => { const isExcluded = shouldExcludeRepo({ @@ -72,21 +110,10 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o logger.debug(`Found ${repos.length} total repositories.`); - return repos; -} - -export const getGitHubRepoFromId = async (id: string, hostURL: string, token?: string) => { - const octokit = new Octokit({ - auth: token ?? FALLBACK_GITHUB_TOKEN, - ...(hostURL !== 'https://github.com' ? { - baseUrl: `${hostURL}/api/v3` - } : {}) - }); - - const repo = await octokit.request('GET /repositories/:id', { - id, - }); - return repo; + return { + validRepos: repos, + notFound, + }; } export const shouldExcludeRepo = ({ @@ -176,7 +203,7 @@ export const shouldExcludeRepo = ({ } const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, octokit: Octokit, signal: AbortSignal) => { - const repos = (await Promise.all(users.map(async (user) => { + const results = await Promise.allSettled(users.map(async (user) => { try { logger.debug(`Fetching repository info for user ${user}...`); @@ -207,18 +234,33 @@ const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, o }); logger.debug(`Found ${data.length} owned by user ${user} in ${durationMs}ms.`); - return data; - } catch (e) { - logger.error(`Failed to fetch repository info for user ${user}.`, e); - throw e; + return { + type: 'valid' as const, + data + }; + } catch (error) { + if (isHttpError(error, 404)) { + logger.error(`User ${user} not found or no access`); + return { + type: 'notFound' as const, + value: user + }; + } + throw error; } - }))).flat(); + })); - return repos; + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results); + + return { + validRepos, + notFoundUsers, + }; } const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal) => { - const repos = (await Promise.all(orgs.map(async (org) => { + const results = await Promise.allSettled(orgs.map(async (org) => { try { logger.info(`Fetching repository info for org ${org}...`); @@ -235,18 +277,33 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi }); logger.info(`Found ${data.length} in org ${org} in ${durationMs}ms.`); - return data; - } catch (e) { - logger.error(`Failed to fetch repository info for org ${org}.`, e); - throw e; + return { + type: 'valid' as const, + data + }; + } catch (error) { + if (isHttpError(error, 404)) { + logger.error(`Organization ${org} not found or no access`); + return { + type: 'notFound' as const, + value: org + }; + } + throw error; } - }))).flat(); + })); - return repos; + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + + return { + validRepos, + notFoundOrgs, + }; } const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal) => { - const repos = (await Promise.all(repoList.map(async (repo) => { + const results = await Promise.allSettled(repoList.map(async (repo) => { try { const [owner, repoName] = repo.split('/'); logger.info(`Fetching repository info for ${repo}...`); @@ -264,13 +321,28 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna }); logger.info(`Found info for repository ${repo} in ${durationMs}ms`); + return { + type: 'valid' as const, + data: [result.data] + }; - return [result.data]; - } catch (e) { - logger.error(`Failed to fetch repository info for ${repo}.`, e); - throw e; + } catch (error) { + if (isHttpError(error, 404)) { + logger.error(`Repository ${repo} not found or no access`); + return { + type: 'notFound' as const, + value: repo + }; + } + throw error; } - }))).flat(); + })); - return repos; + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + + return { + validRepos, + notFoundRepos, + }; } \ No newline at end of file diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index cce4800a..99a55fae 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -5,24 +5,35 @@ import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type" import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js"; import { PrismaClient } from "@sourcebot/db"; import { FALLBACK_GITLAB_TOKEN } from "./environment.js"; +import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; const logger = createLogger("GitLab"); export const GITLAB_CLOUD_HOSTNAME = "gitlab.com"; export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => { - const token = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; + const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; + const token = tokenResult?.token ?? FALLBACK_GITLAB_TOKEN; + const secretKey = tokenResult?.secretKey; + const api = new Gitlab({ ...(token ? { token, - } : { - token: FALLBACK_GITLAB_TOKEN, - }), + } : {}), ...(config.url ? { host: config.url, } : {}), }); const hostname = config.url ? new URL(config.url).hostname : GITLAB_CLOUD_HOSTNAME; - let allProjects: ProjectSchema[] = []; + let allRepos: ProjectSchema[] = []; + let notFound: { + orgs: string[], + users: string[], + repos: string[], + } = { + orgs: [], + users: [], + repos: [], + }; if (config.all === true) { if (hostname !== GITLAB_CLOUD_HOSTNAME) { @@ -35,7 +46,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o return fetchWithRetry(fetchFn, `all projects in ${config.url}`, logger); }); logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`); - allProjects = allProjects.concat(_projects); + allRepos = allRepos.concat(_projects); } catch (e) { logger.error(`Failed to fetch all projects visible in ${config.url}.`, e); throw e; @@ -46,7 +57,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o } if (config.groups) { - const _projects = (await Promise.all(config.groups.map(async (group) => { + const results = await Promise.allSettled(config.groups.map(async (group) => { try { logger.debug(`Fetching project info for group ${group}...`); const { durationMs, data } = await measure(async () => { @@ -57,18 +68,31 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o return fetchWithRetry(fetchFn, `group ${group}`, logger); }); logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`); - return data; - } catch (e) { - logger.error(`Failed to fetch project info for group ${group}.`, e); + return { + type: 'valid' as const, + data + }; + } catch (e: any) { + const status = e?.cause?.response?.status; + if (status === 404) { + logger.error(`Group ${group} not found or no access`); + return { + type: 'notFound' as const, + value: group + }; + } throw e; } - }))).flat(); + })); - allProjects = allProjects.concat(_projects); + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + allRepos = allRepos.concat(validRepos); + notFound.orgs = notFoundOrgs; } if (config.users) { - const _projects = (await Promise.all(config.users.map(async (user) => { + const results = await Promise.allSettled(config.users.map(async (user) => { try { logger.debug(`Fetching project info for user ${user}...`); const { durationMs, data } = await measure(async () => { @@ -78,18 +102,31 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o return fetchWithRetry(fetchFn, `user ${user}`, logger); }); logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`); - return data; - } catch (e) { - logger.error(`Failed to fetch project info for user ${user}.`, e); + return { + type: 'valid' as const, + data + }; + } catch (e: any) { + const status = e?.cause?.response?.status; + if (status === 404) { + logger.error(`User ${user} not found or no access`); + return { + type: 'notFound' as const, + value: user + }; + } throw e; } - }))).flat(); + })); - allProjects = allProjects.concat(_projects); + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results); + allRepos = allRepos.concat(validRepos); + notFound.users = notFoundUsers; } if (config.projects) { - const _projects = (await Promise.all(config.projects.map(async (project) => { + const results = await Promise.allSettled(config.projects.map(async (project) => { try { logger.debug(`Fetching project info for project ${project}...`); const { durationMs, data } = await measure(async () => { @@ -97,17 +134,31 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o return fetchWithRetry(fetchFn, `project ${project}`, logger); }); logger.debug(`Found project ${project} in ${durationMs}ms.`); - return [data]; - } catch (e) { - logger.error(`Failed to fetch project info for project ${project}.`, e); + return { + type: 'valid' as const, + data: [data] + }; + } catch (e: any) { + const status = e?.cause?.response?.status; + + if (status === 404) { + logger.error(`Project ${project} not found or no access`); + return { + type: 'notFound' as const, + value: project + }; + } throw e; } - }))).flat(); + })); - allProjects = allProjects.concat(_projects); + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + allRepos = allRepos.concat(validRepos); + notFound.repos = notFoundRepos; } - let repos = allProjects + let repos = allRepos .filter((project) => { const isExcluded = shouldExcludeProject({ project, @@ -122,7 +173,10 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o logger.debug(`Found ${repos.length} total repositories.`); - return repos; + return { + validRepos: repos, + notFound, + }; } export const shouldExcludeProject = ({ diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 83359e35..4342faea 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -15,12 +15,22 @@ export const compileGithubConfig = async ( connectionId: number, orgId: number, db: PrismaClient, - abortController: AbortController): Promise => { - const gitHubRepos = await getGitHubReposFromConfig(config, orgId, db, abortController.signal); + abortController: AbortController): Promise<{ + repoData: RepoData[], + notFound: { + users: string[], + orgs: string[], + repos: string[], + } + }> => { + const gitHubReposResult = await getGitHubReposFromConfig(config, orgId, db, abortController.signal); + const gitHubRepos = gitHubReposResult.validRepos; + const notFound = gitHubReposResult.notFound; + const hostUrl = config.url ?? 'https://github.com'; const hostname = config.url ? new URL(config.url).hostname : 'github.com'; - return gitHubRepos.map((repo) => { + const repos = gitHubRepos.map((repo) => { const repoName = `${hostname}/${repo.full_name}`; const cloneUrl = new URL(repo.clone_url!); @@ -59,6 +69,11 @@ export const compileGithubConfig = async ( return record; }) + + return { + repoData: repos, + notFound, + }; } export const compileGitlabConfig = async ( @@ -67,10 +82,13 @@ export const compileGitlabConfig = async ( orgId: number, db: PrismaClient) => { - const gitlabRepos = await getGitLabReposFromConfig(config, orgId, db); + const gitlabReposResult = await getGitLabReposFromConfig(config, orgId, db); + const gitlabRepos = gitlabReposResult.validRepos; + const notFound = gitlabReposResult.notFound; + const hostUrl = config.url ?? 'https://gitlab.com'; - return gitlabRepos.map((project) => { + const repos = gitlabRepos.map((project) => { const projectUrl = `${hostUrl}/${project.path_with_namespace}`; const cloneUrl = new URL(project.http_url_to_repo); const isFork = project.forked_from_project !== undefined; @@ -108,6 +126,11 @@ export const compileGitlabConfig = async ( return record; }) + + return { + repoData: repos, + notFound, + }; } export const compileGiteaConfig = async ( @@ -116,10 +139,13 @@ export const compileGiteaConfig = async ( orgId: number, db: PrismaClient) => { - const giteaRepos = await getGiteaReposFromConfig(config, orgId, db); + const giteaReposResult = await getGiteaReposFromConfig(config, orgId, db); + const giteaRepos = giteaReposResult.validRepos; + const notFound = giteaReposResult.notFound; + const hostUrl = config.url ?? 'https://gitea.com'; - return giteaRepos.map((repo) => { + const repos = giteaRepos.map((repo) => { const cloneUrl = new URL(repo.clone_url!); const record: RepoData = { @@ -153,6 +179,11 @@ export const compileGiteaConfig = async ( return record; }) + + return { + repoData: repos, + notFound, + }; } export const compileGerritConfig = async ( @@ -164,7 +195,7 @@ export const compileGerritConfig = async ( const hostUrl = config.url ?? 'https://gerritcodereview.com'; const hostname = new URL(hostUrl).hostname; - return gerritRepos.map((project) => { + const repos = gerritRepos.map((project) => { const repoId = `${hostname}/${project.name}`; const cloneUrl = new URL(`${config.url}/${encodeURIComponent(project.name)}`); @@ -207,4 +238,13 @@ export const compileGerritConfig = async ( return record; }) + + return { + repoData: repos, + notFound: { + users: [], + orgs: [], + repos: [], + } + }; } \ No newline at end of file diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts index 34d438df..7d39d71d 100644 --- a/packages/backend/src/repoManager.ts +++ b/packages/backend/src/repoManager.ts @@ -207,7 +207,8 @@ export class RepoManager implements IRepoManager { const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig; if (config.token) { - token = await getTokenFromConfig(config.token, connection.orgId, db); + const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db); + token = tokenResult?.token; if (token) { break; } diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index b6135995..2bc3c8f1 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -5,6 +5,7 @@ import micromatch from "micromatch"; import { PrismaClient, Repo } from "@sourcebot/db"; import { decrypt } from "@sourcebot/crypto"; import { Token } from "@sourcebot/schemas/v3/shared.type"; +import { BackendException, BackendError } from "@sourcebot/error"; export const measure = async (cb: () => Promise) => { const start = Date.now(); @@ -90,7 +91,9 @@ export const excludeReposByTopic = (repos: T[], excludedRe export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => { if (!db) { - throw new Error(`Database connection required to retrieve secret`); + throw new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, { + message: `No database connection provided.`, + }); } const secretKey = token.secret; @@ -104,11 +107,16 @@ export const getTokenFromConfig = async (token: Token, orgId: number, db?: Prism }); if (!secret) { - throw new Error(`Secret with key ${secretKey} not found for org ${orgId}`); + throw new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, { + message: `Secret with key ${secretKey} not found for org ${orgId}`, + }); } const decryptedSecret = decrypt(secret.iv, secret.encryptedValue); - return decryptedSecret; + return { + token: decryptedSecret, + secretKey, + }; } export const isRemotePath = (path: string) => { diff --git a/packages/db/prisma/migrations/20250218193104_add_connection_sync_metadata/migration.sql b/packages/db/prisma/migrations/20250218193104_add_connection_sync_metadata/migration.sql new file mode 100644 index 00000000..64f0da89 --- /dev/null +++ b/packages/db/prisma/migrations/20250218193104_add_connection_sync_metadata/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Connection" ADD COLUMN "syncStatusMetadata" JSONB; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index ad45ad0b..b8198890 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -59,21 +59,22 @@ model Repo { } model Connection { - id Int @id @default(autoincrement()) - name String - config Json - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - syncedAt DateTime? - repos RepoToConnection[] - syncStatus ConnectionSyncStatus @default(SYNC_NEEDED) + id Int @id @default(autoincrement()) + name String + config Json + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + syncedAt DateTime? + repos RepoToConnection[] + syncStatus ConnectionSyncStatus @default(SYNC_NEEDED) + syncStatusMetadata Json? // The type of connection (e.g., github, gitlab, etc.) - connectionType String + connectionType String // The organization that owns this connection - org Org @relation(fields: [orgId], references: [id], onDelete: Cascade) - orgId Int + org Org @relation(fields: [orgId], references: [id], onDelete: Cascade) + orgId Int } model RepoToConnection { diff --git a/packages/error/package.json b/packages/error/package.json new file mode 100644 index 00000000..7a88f762 --- /dev/null +++ b/packages/error/package.json @@ -0,0 +1,13 @@ +{ + "name": "@sourcebot/error", + "main": "dist/index.js", + "version": "0.1.0", + "scripts": { + "build": "tsc", + "postinstall": "yarn build" + }, + "devDependencies": { + "@types/node": "^22.7.5", + "typescript": "^5.7.3" + } +} \ No newline at end of file diff --git a/packages/error/src/index.ts b/packages/error/src/index.ts new file mode 100644 index 00000000..f18f4005 --- /dev/null +++ b/packages/error/src/index.ts @@ -0,0 +1,17 @@ +export enum BackendError { + CONNECTION_SYNC_SECRET_DNE = 'CONNECTION_SYNC_SECRET_DNE', + CONNECTION_SYNC_INVALID_TOKEN = 'CONNECTION_SYNC_INVALID_TOKEN', + CONNECTION_SYNC_SYSTEM_ERROR = 'CONNECTION_SYNC_SYSTEM_ERROR', + CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS = 'CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS', + CONNECTION_SYNC_CONNECTION_NOT_FOUND = 'CONNECTION_SYNC_CONNECTION_NOT_FOUND', +} + +export class BackendException extends Error { + constructor( + public readonly code: BackendError, + public readonly metadata: Record = {} + ) { + super(code); + this.name = 'BackendException'; + } +} \ No newline at end of file diff --git a/packages/error/tsconfig.json b/packages/error/tsconfig.json new file mode 100644 index 00000000..a27277b9 --- /dev/null +++ b/packages/error/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES6", + "module": "CommonJS", + "lib": ["ES6"], + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "moduleResolution": "node", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "isolatedModules": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/web/components.json b/packages/web/components.json index 32fffae3..20099419 100644 --- a/packages/web/components.json +++ b/packages/web/components.json @@ -12,7 +12,9 @@ }, "aliases": { "components": "@/components", - "hooks": "@/components/hooks", - "utils": "@/lib/utils" + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" } } \ No newline at end of file diff --git a/packages/web/next.config.mjs b/packages/web/next.config.mjs index a560a21f..6317d972 100644 --- a/packages/web/next.config.mjs +++ b/packages/web/next.config.mjs @@ -27,7 +27,11 @@ const nextConfig = { { protocol: 'https', hostname: 'avatars.githubusercontent.com', - } + }, + { + protocol: 'https', + hostname: 'gitlab.com', + }, ] } diff --git a/packages/web/package.json b/packages/web/package.json index bf03677e..54c9805d 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -67,6 +67,7 @@ "@shopify/lang-jsonc": "^1.0.0", "@sourcebot/crypto": "^0.1.0", "@sourcebot/db": "^0.1.0", + "@sourcebot/error": "^0.1.0", "@sourcebot/schemas": "^0.1.0", "@ssddanbrown/codemirror-lang-twig": "^1.0.0", "@stripe/react-stripe-js": "^3.1.1", diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 6b2be718..639fe0e2 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -14,7 +14,7 @@ import { gerritSchema } from "@sourcebot/schemas/v3/gerrit.schema"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, GerritConnectionConfig, ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { encrypt } from "@sourcebot/crypto" import { getConnection, getLinkedRepos } from "./data/connection"; -import { ConnectionSyncStatus, Prisma, Invite, OrgRole, Connection, Repo, Org } from "@sourcebot/db"; +import { ConnectionSyncStatus, Prisma, Invite, OrgRole, Connection, Repo, Org, RepoIndexingStatus } from "@sourcebot/db"; import { headers } from "next/headers" import { getStripe } from "@/lib/stripe" import { getUser } from "@/data/user"; @@ -22,6 +22,7 @@ import { Session } from "next-auth"; import { STRIPE_PRODUCT_ID, CONFIG_MAX_REPOS_NO_TOKEN } from "@/lib/environment"; import { StripeSubscriptionStatus } from "@sourcebot/db"; import Stripe from "stripe"; +import { SyncStatusMetadataSchema, type NotFoundData } from "@/lib/syncStatusMetadataSchema"; const ajv = new Ajv({ validateFormats: false, }); @@ -179,6 +180,7 @@ export const getConnections = async (domain: string): Promise< id: number, name: string, syncStatus: ConnectionSyncStatus, + syncStatusMetadata: Prisma.JsonValue, connectionType: string, updatedAt: Date, syncedAt?: Date @@ -196,6 +198,7 @@ export const getConnections = async (domain: string): Promise< id: connection.id, name: connection.name, syncStatus: connection.syncStatus, + syncStatusMetadata: connection.syncStatusMetadata, connectionType: connection.connectionType, updatedAt: connection.updatedAt, syncedAt: connection.syncedAt ?? undefined, @@ -203,6 +206,40 @@ export const getConnections = async (domain: string): Promise< }) ); +export const getConnectionFailedRepos = async (connectionId: number, domain: string): Promise<{ repoId: number, repoName: string }[] | ServiceError> => + withAuth((session) => + withOrgMembership(session, domain, async ({ orgId }) => { + const connection = await getConnection(connectionId, orgId); + if (!connection) { + return notFound(); + } + + const linkedRepos = await getLinkedRepos(connectionId, orgId); + + return linkedRepos.filter((repo) => repo.repo.repoIndexingStatus === RepoIndexingStatus.FAILED).map((repo) => ({ + repoId: repo.repo.id, + repoName: repo.repo.name, + })); + }) + ); + +export const getConnectionInProgressRepos = async (connectionId: number, domain: string): Promise<{ repoId: number, repoName: string }[] | ServiceError> => + withAuth((session) => + withOrgMembership(session, domain, async ({ orgId }) => { + const connection = await getConnection(connectionId, orgId); + if (!connection) { + return notFound(); + } + + const linkedRepos = await getLinkedRepos(connectionId, orgId); + + return linkedRepos.filter((repo) => repo.repo.repoIndexingStatus === RepoIndexingStatus.IN_INDEX_QUEUE || repo.repo.repoIndexingStatus === RepoIndexingStatus.INDEXING).map((repo) => ({ + repoId: repo.repo.id, + repoName: repo.repo.name, + })); + }) + ); + export const createConnection = async (name: string, type: string, connectionConfig: string, domain: string): Promise<{ id: number } | ServiceError> => withAuth((session) => @@ -331,14 +368,6 @@ export const flagConnectionForSync = async (connectionId: number, domain: string return notFound(); } - if (connection.syncStatus !== "FAILED") { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.CONNECTION_NOT_FAILED, - message: "Connection is not in a failed state. Cannot flag for sync.", - } satisfies ServiceError; - } - await prisma.connection.update({ where: { id: connection.id, @@ -353,6 +382,36 @@ export const flagConnectionForSync = async (connectionId: number, domain: string } })); +export const flagRepoForIndex = async (repoId: number, domain: string): Promise<{ success: boolean } | ServiceError> => + withAuth((session) => + withOrgMembership(session, domain, async () => { + const repo = await prisma.repo.findUnique({ + where: { + id: repoId, + }, + }); + + if (!repo) { + return notFound(); + } + + await prisma.repo.update({ + where: { + id: repoId, + }, + data: { + repoIndexingStatus: RepoIndexingStatus.NEW, + } + }); + + return { + success: true, + } + }) + ); + + + export const deleteConnection = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { diff --git a/packages/web/src/app/[domain]/components/errorNavIndicator.tsx b/packages/web/src/app/[domain]/components/errorNavIndicator.tsx new file mode 100644 index 00000000..8ce3f833 --- /dev/null +++ b/packages/web/src/app/[domain]/components/errorNavIndicator.tsx @@ -0,0 +1,160 @@ +"use client"; + +import Link from "next/link"; +import { HoverCard, HoverCardTrigger, HoverCardContent } from "@/components/ui/hover-card"; +import { CircleXIcon } from "lucide-react"; +import { useDomain } from "@/hooks/useDomain"; +import { getConnectionFailedRepos, getConnections } from "@/actions"; +import { useState, useEffect } from "react"; +import { isServiceError } from "@/lib/utils"; + +enum ConnectionErrorType { + SYNC_FAILED = "SYNC_FAILED", + REPO_INDEXING_FAILED = "REPO_INDEXING_FAILED", +} + +interface Error { + connectionId?: number; + connectionName?: string; + errorType: ConnectionErrorType; + numRepos?: number; +} + +export const ErrorNavIndicator = () => { + const domain = useDomain(); + const [errors, setErrors] = useState([]); + + useEffect(() => { + const fetchErrors = async () => { + const connections = await getConnections(domain); + const errors: Error[] = []; + if (!isServiceError(connections)) { + for (const connection of connections) { + if (connection.syncStatus === 'FAILED') { + errors.push({ + connectionId: connection.id, + connectionName: connection.name, + errorType: ConnectionErrorType.SYNC_FAILED + }); + } + + const failedRepos = await getConnectionFailedRepos(connection.id, domain); + if (!isServiceError(failedRepos) && failedRepos.length > 0) { + errors.push({ + connectionId: connection.id, + connectionName: connection.name, + numRepos: failedRepos.length, + errorType: ConnectionErrorType.REPO_INDEXING_FAILED + }); + } + } + } + setErrors(prevErrors => { + // Only update if the errors have actually changed + const errorsChanged = prevErrors.length !== errors.length || + prevErrors.some((error, idx) => + error.connectionId !== errors[idx]?.connectionId || + error.connectionName !== errors[idx]?.connectionName || + error.errorType !== errors[idx]?.errorType + ); + return errorsChanged ? errors : prevErrors; + }); + }; + + fetchErrors(); + const intervalId = setInterval(fetchErrors, 1000); + return () => clearInterval(intervalId); + }, [domain]); + + if (errors.length === 0) return null; + + return ( + + + +
+ + {errors.reduce((acc, error) => acc + (error.numRepos || 0), 0) > 0 && ( + {errors.reduce((acc, error) => acc + (error.numRepos || 0), 0)} + )} +
+ + +
+ {errors.filter(e => e.errorType === 'SYNC_FAILED').length > 0 && ( +
+
+
+

Connection Sync Issues

+
+

+ The following connections have failed to sync: +

+
+ {errors + .filter(e => e.errorType === 'SYNC_FAILED') + .slice(0, 10) + .map(error => ( + +
+ {error.connectionName} +
+ + ))} + {errors.filter(e => e.errorType === 'SYNC_FAILED').length > 10 && ( +
+ And {errors.filter(e => e.errorType === 'SYNC_FAILED').length - 10} more... +
+ )} +
+
+ )} + + {errors.filter(e => e.errorType === 'REPO_INDEXING_FAILED').length > 0 && ( +
+
+
+

Repository Indexing Issues

+
+

+ The following connections have repositories that failed to index: +

+
+ {errors + .filter(e => e.errorType === 'REPO_INDEXING_FAILED') + .slice(0, 10) + .map(error => ( + +
+ + {error.connectionName} + + + {error.numRepos} + +
+ + ))} + {errors.filter(e => e.errorType === 'REPO_INDEXING_FAILED').length > 10 && ( +
+ And {errors.filter(e => e.errorType === 'REPO_INDEXING_FAILED').length - 10} more... +
+ )} +
+
+ )} +
+ + + + ); +}; diff --git a/packages/web/src/app/[domain]/components/navigationMenu.tsx b/packages/web/src/app/[domain]/components/navigationMenu.tsx index d625f179..68264acf 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu.tsx @@ -8,6 +8,9 @@ import { redirect } from "next/navigation"; import { OrgSelector } from "./orgSelector"; import { getSubscriptionData } from "@/actions"; import { isServiceError } from "@/lib/utils"; +import { ErrorNavIndicator } from "./errorNavIndicator"; +import { WarningNavIndicator } from "./warningNavIndicator"; +import { ProgressNavIndicator } from "./progressNavIndicator"; import { SourcebotLogo } from "@/app/components/sourcebotLogo"; const SOURCEBOT_DISCORD_URL = "https://discord.gg/6Fhp27x7Pb"; @@ -83,10 +86,13 @@ export const NavigationMenu = async ({
+ + + {!isServiceError(subscription) && subscription.status === "trialing" && ( -
- +
+ {Math.ceil((subscription.nextBillingDate * 1000 - Date.now()) / (1000 * 60 * 60 * 24))} days left in trial diff --git a/packages/web/src/app/[domain]/components/progressNavIndicator.tsx b/packages/web/src/app/[domain]/components/progressNavIndicator.tsx new file mode 100644 index 00000000..b8ebdfc7 --- /dev/null +++ b/packages/web/src/app/[domain]/components/progressNavIndicator.tsx @@ -0,0 +1,97 @@ +"use client"; + +import Link from "next/link"; +import { HoverCard, HoverCardTrigger, HoverCardContent } from "@/components/ui/hover-card"; +import { Loader2Icon } from "lucide-react"; +import { useEffect, useState } from "react"; +import { useDomain } from "@/hooks/useDomain"; +import { getConnectionInProgressRepos, getConnections } from "@/actions"; +import { isServiceError } from "@/lib/utils"; + +interface InProgress { + connectionId: number; + repoId: number; + repoName: string; +} + + +export const ProgressNavIndicator = () => { + const domain = useDomain(); + const [inProgressJobs, setInProgressJobs] = useState([]); + + useEffect(() => { + const fetchInProgressJobs = async () => { + const connections = await getConnections(domain); + if (!isServiceError(connections)) { + const allInProgressRepos: InProgress[] = []; + for (const connection of connections) { + const inProgressRepos = await getConnectionInProgressRepos(connection.id, domain); + if (!isServiceError(inProgressRepos)) { + allInProgressRepos.push(...inProgressRepos.map(repo => ({ + connectionId: connection.id, + ...repo + }))); + } + } + setInProgressJobs(prevJobs => { + // Only update if the jobs have actually changed + const jobsChanged = prevJobs.length !== allInProgressRepos.length || + prevJobs.some((job, idx) => + job.repoId !== allInProgressRepos[idx]?.repoId || + job.repoName !== allInProgressRepos[idx]?.repoName + ); + return jobsChanged ? allInProgressRepos : prevJobs; + }); + } + }; + + fetchInProgressJobs(); + const intervalId = setInterval(fetchInProgressJobs, 1000); + return () => clearInterval(intervalId); + }, [domain]); + + if (inProgressJobs.length === 0) { + return null; + } + + return ( + + + +
+ + {inProgressJobs.length} +
+ + +
+
+
+

Indexing in Progress

+
+

+ The following repositories are currently being indexed: +

+
+ {inProgressJobs.slice(0, 10).map(item => ( + +
+ {item.repoName} +
+ + ))} + {inProgressJobs.length > 10 && ( +
+ And {inProgressJobs.length - 10} more... +
+ )} +
+
+ + + + ); +}; \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/warningNavIndicator.tsx b/packages/web/src/app/[domain]/components/warningNavIndicator.tsx new file mode 100644 index 00000000..8ac60e06 --- /dev/null +++ b/packages/web/src/app/[domain]/components/warningNavIndicator.tsx @@ -0,0 +1,97 @@ +"use client"; + +import Link from "next/link"; +import { HoverCard, HoverCardTrigger, HoverCardContent } from "@/components/ui/hover-card"; +import { AlertTriangleIcon } from "lucide-react"; +import { useDomain } from "@/hooks/useDomain"; +import { getConnections } from "@/actions"; +import { useState } from "react"; +import { useEffect } from "react"; +import { isServiceError } from "@/lib/utils"; +import { SyncStatusMetadataSchema } from "@/lib/syncStatusMetadataSchema"; + +interface Warning { + connectionId?: number; + connectionName?: string; +} + +export const WarningNavIndicator = () => { + const domain = useDomain(); + const [warnings, setWarnings] = useState([]); + + useEffect(() => { + const fetchWarnings = async () => { + const connections = await getConnections(domain); + const warnings: Warning[] = []; + if (!isServiceError(connections)) { + for (const connection of connections) { + const parseResult = SyncStatusMetadataSchema.safeParse(connection.syncStatusMetadata); + if (parseResult.success && parseResult.data.notFound) { + const { notFound } = parseResult.data; + if (notFound.users.length > 0 || notFound.orgs.length > 0 || notFound.repos.length > 0) { + warnings.push({ connectionId: connection.id, connectionName: connection.name }); + } + } + } + } + setWarnings(prevWarnings => { + // Only update if the warnings have actually changed + const warningsChanged = prevWarnings.length !== warnings.length || + prevWarnings.some((warning, idx) => + warning.connectionId !== warnings[idx]?.connectionId || + warning.connectionName !== warnings[idx]?.connectionName + ); + return warningsChanged ? warnings : prevWarnings; + }); + }; + + fetchWarnings(); + const intervalId = setInterval(fetchWarnings, 1000); + return () => clearInterval(intervalId); + }, [domain]); + + if (warnings.length === 0) { + return null; + } + + return ( + + + +
+ + {warnings.length} +
+ + +
+
+
+

Missing References

+
+

+ The following connections have references that could not be found: +

+
+ {warnings.slice(0, 10).map(warning => ( + +
+ {warning.connectionName} +
+ + ))} + {warnings.length > 10 && ( +
+ And {warnings.length - 10} more... +
+ )} +
+
+ + + + ); +}; diff --git a/packages/web/src/app/[domain]/connections/[id]/components/connectionError.tsx b/packages/web/src/app/[domain]/connections/[id]/components/connectionError.tsx new file mode 100644 index 00000000..928cdca7 --- /dev/null +++ b/packages/web/src/app/[domain]/connections/[id]/components/connectionError.tsx @@ -0,0 +1,110 @@ +"use client" + +import { BackendError } from "@sourcebot/error"; +import { Prisma } from "@sourcebot/db"; + +export function DisplayConnectionError({ syncStatusMetadata, onSecretsClick }: { syncStatusMetadata: Prisma.JsonValue, onSecretsClick: () => void }) { + const errorCode = syncStatusMetadata && typeof syncStatusMetadata === 'object' && 'error' in syncStatusMetadata + ? (syncStatusMetadata.error as string) + : undefined; + + switch (errorCode) { + case BackendError.CONNECTION_SYNC_INVALID_TOKEN: + return + case BackendError.CONNECTION_SYNC_SECRET_DNE: + return + case BackendError.CONNECTION_SYNC_SYSTEM_ERROR: + return + case BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS: + return + default: + return + } +} + +function SecretNotFoundError({ syncStatusMetadata, onSecretsClick }: { syncStatusMetadata: Prisma.JsonValue, onSecretsClick: () => void }) { + const secretKey = syncStatusMetadata && typeof syncStatusMetadata === 'object' && 'secretKey' in syncStatusMetadata + ? (syncStatusMetadata.secretKey as string) + : undefined; + + return ( +
+

Secret Not Found

+

+ The secret key provided for this connection was not found. Please ensure your config is referencing a secret + that exists in your{" "} + + , and try again. +

+ {secretKey && ( +

+ Secret Key: {secretKey} +

+ )} +
+ ); +} + +function InvalidTokenError({ syncStatusMetadata, onSecretsClick }: { syncStatusMetadata: Prisma.JsonValue, onSecretsClick: () => void }) { + const secretKey = syncStatusMetadata && typeof syncStatusMetadata === 'object' && 'secretKey' in syncStatusMetadata + ? (syncStatusMetadata.secretKey as string) + : undefined; + + return ( +
+

Invalid Authentication Token

+

+ The authentication token provided for this connection is invalid. Please update your config with a valid token and try again. +

+ {secretKey && ( +

+ Secret Key: +

+ )} +
+ ); +} + +function SystemError() { + return ( +
+

System Error

+

+ An error occurred while syncing this connection. Please try again later. +

+
+ ) +} + +function FailedToFetchGerritProjects({ syncStatusMetadata }: { syncStatusMetadata: Prisma.JsonValue}) { + const status = syncStatusMetadata && typeof syncStatusMetadata === 'object' && 'status' in syncStatusMetadata + ? (syncStatusMetadata.status as number) + : undefined; + + return ( +
+

Failed to Fetch Gerrit Projects

+

+ An error occurred while syncing this connection. Please try again later. +

+ {status && ( +

+ Status: {status} +

+ )} +
+ ) +} + +function UnknownError() { + return ( +
+

Unknown Error

+

+ An unknown error occurred while syncing this connection. Please try again later. +

+
+ ) +} diff --git a/packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx b/packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx new file mode 100644 index 00000000..629ac49d --- /dev/null +++ b/packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx @@ -0,0 +1,64 @@ +import { AlertTriangle } from "lucide-react" +import { Prisma } from "@sourcebot/db" +import { RetrySyncButton } from "./retrySyncButton" +import { SyncStatusMetadataSchema } from "@/lib/syncStatusMetadataSchema" + +interface NotFoundWarningProps { + syncStatusMetadata: Prisma.JsonValue + onSecretsClick: () => void + connectionId: number + domain: string + connectionType: string +} + +export const NotFoundWarning = ({ syncStatusMetadata, onSecretsClick, connectionId, domain, connectionType }: NotFoundWarningProps) => { + const parseResult = SyncStatusMetadataSchema.safeParse(syncStatusMetadata); + if (!parseResult.success || !parseResult.data.notFound) { + return null; + } + + const { notFound } = parseResult.data; + + if (notFound.users.length === 0 && notFound.orgs.length === 0 && notFound.repos.length === 0) { + return null; + } + + return ( +
+
+ +

Unable to fetch all references

+
+

+ Some requested references couldn't be found. Please ensure you've provided the information listed below correctly, and that you've provided a{" "} + {" "} + to access them if they're private. +

+
    + {notFound.users.length > 0 && ( +
  • + Users: + {notFound.users.join(', ')} +
    • + )} + {notFound.orgs.length > 0 && ( +
  • + {connectionType === "gitlab" ? "Groups" : "Organizations"}: + {notFound.orgs.join(', ')} +
    • + )} + {notFound.repos.length > 0 && ( +
  • + {connectionType === "gitlab" ? "Projects" : "Repositories"}: + {notFound.repos.join(', ')} +
    • + )} +
+
+ +
+
+ ) +} diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx index 8797c9bc..c75e37e5 100644 --- a/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx +++ b/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx @@ -3,6 +3,7 @@ import Image from "next/image"; import { StatusIcon } from "../../components/statusIcon"; import { RepoIndexingStatus } from "@sourcebot/db"; import { useMemo } from "react"; +import { RetryRepoIndexButton } from "./repoRetryIndexButton"; interface RepoListItemProps { @@ -10,6 +11,8 @@ interface RepoListItemProps { status: RepoIndexingStatus; imageUrl?: string; indexedAt?: Date; + repoId: number; + domain: string; } export const RepoListItem = ({ @@ -17,6 +20,8 @@ export const RepoListItem = ({ name, indexedAt, status, + repoId, + domain, }: RepoListItemProps) => { const statusDisplayName = useMemo(() => { switch (status) { @@ -47,22 +52,27 @@ export const RepoListItem = ({ />

{name}

-
- -

- {statusDisplayName} - { - ( - status === RepoIndexingStatus.INDEXED || - status === RepoIndexingStatus.FAILED - ) && indexedAt && ( - {` ${getDisplayTime(indexedAt)}`} - ) - } -

+
+ {status === RepoIndexingStatus.FAILED && ( + + )} +
+ +

+ {statusDisplayName} + { + ( + status === RepoIndexingStatus.INDEXED || + status === RepoIndexingStatus.FAILED + ) && indexedAt && ( + {` ${getDisplayTime(indexedAt)}`} + ) + } +

+
) diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx new file mode 100644 index 00000000..080ba450 --- /dev/null +++ b/packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx @@ -0,0 +1,37 @@ +"use client"; + +import { Button } from "@/components/ui/button"; +import { ReloadIcon } from "@radix-ui/react-icons" +import { toast } from "@/components/hooks/use-toast"; +import { flagRepoForIndex } from "@/actions"; +import { isServiceError } from "@/lib/utils"; + +interface RetryRepoIndexButtonProps { + repoId: number; + domain: string; +} + +export const RetryRepoIndexButton = ({ repoId, domain }: RetryRepoIndexButtonProps) => { + return ( + + ); +}; diff --git a/packages/web/src/app/[domain]/connections/[id]/components/retryAllFailedReposButton.tsx b/packages/web/src/app/[domain]/connections/[id]/components/retryAllFailedReposButton.tsx new file mode 100644 index 00000000..559735ca --- /dev/null +++ b/packages/web/src/app/[domain]/connections/[id]/components/retryAllFailedReposButton.tsx @@ -0,0 +1,60 @@ +"use client"; + +import { Button } from "@/components/ui/button"; +import { ReloadIcon } from "@radix-ui/react-icons" +import { toast } from "@/components/hooks/use-toast"; +import { flagRepoForIndex, getConnectionFailedRepos } from "@/actions"; +import { isServiceError } from "@/lib/utils"; + +interface RetryAllFailedReposButtonProps { + connectionId: number; + domain: string; +} + +export const RetryAllFailedReposButton = ({ connectionId, domain }: RetryAllFailedReposButtonProps) => { + return ( + + ); +}; diff --git a/packages/web/src/app/[domain]/connections/[id]/components/retrySyncButton.tsx b/packages/web/src/app/[domain]/connections/[id]/components/retrySyncButton.tsx new file mode 100644 index 00000000..24de0d44 --- /dev/null +++ b/packages/web/src/app/[domain]/connections/[id]/components/retrySyncButton.tsx @@ -0,0 +1,37 @@ +"use client"; + +import { Button } from "@/components/ui/button"; +import { ReloadIcon } from "@radix-ui/react-icons" +import { toast } from "@/components/hooks/use-toast"; +import { flagConnectionForSync } from "@/actions"; +import { isServiceError } from "@/lib/utils"; + +interface RetrySyncButtonProps { + connectionId: number; + domain: string; +} + +export const RetrySyncButton = ({ connectionId, domain }: RetrySyncButtonProps) => { + return ( + + ); +}; diff --git a/packages/web/src/app/[domain]/connections/[id]/page.tsx b/packages/web/src/app/[domain]/connections/[id]/page.tsx index 877aa502..714c1d39 100644 --- a/packages/web/src/app/[domain]/connections/[id]/page.tsx +++ b/packages/web/src/app/[domain]/connections/[id]/page.tsx @@ -1,6 +1,6 @@ -"use client"; +"use client" -import { NotFound } from "@/app/[domain]/components/notFound"; +import { NotFound } from "@/app/[domain]/components/notFound" import { Breadcrumb, BreadcrumbItem, @@ -8,122 +8,125 @@ import { BreadcrumbList, BreadcrumbPage, BreadcrumbSeparator, -} from "@/components/ui/breadcrumb"; -import { ScrollArea } from "@/components/ui/scroll-area"; -import { TabSwitcher } from "@/components/ui/tab-switcher"; -import { Tabs, TabsContent } from "@/components/ui/tabs"; -import { ConnectionIcon } from "../components/connectionIcon"; -import { Header } from "../../components/header"; -import { ConfigSetting } from "./components/configSetting"; -import { DeleteConnectionSetting } from "./components/deleteConnectionSetting"; -import { DisplayNameSetting } from "./components/displayNameSetting"; -import { RepoListItem } from "./components/repoListItem"; -import { useParams, useSearchParams } from "next/navigation"; -import { useEffect, useState } from "react"; -import { Connection, Repo, Org } from "@sourcebot/db"; -import { getConnectionInfoAction, getOrgFromDomainAction, flagConnectionForSync } from "@/actions"; -import { isServiceError } from "@/lib/utils"; -import { Button } from "@/components/ui/button"; -import { ReloadIcon } from "@radix-ui/react-icons"; -import { useToast } from "@/components/hooks/use-toast"; +} from "@/components/ui/breadcrumb" +import { ScrollArea } from "@/components/ui/scroll-area" +import { TabSwitcher } from "@/components/ui/tab-switcher" +import { Tabs, TabsContent } from "@/components/ui/tabs" +import { ConnectionIcon } from "../components/connectionIcon" +import { Header } from "../../components/header" +import { ConfigSetting } from "./components/configSetting" +import { DeleteConnectionSetting } from "./components/deleteConnectionSetting" +import { DisplayNameSetting } from "./components/displayNameSetting" +import { RepoListItem } from "./components/repoListItem" +import { useParams, useSearchParams, useRouter } from "next/navigation" +import { useEffect, useState } from "react" +import type { Connection, Repo, Org } from "@sourcebot/db" +import { getConnectionInfoAction, getOrgFromDomainAction } from "@/actions" +import { isServiceError } from "@/lib/utils" +import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card" +import { DisplayConnectionError } from "./components/connectionError" +import { NotFoundWarning } from "./components/notFoundWarning" +import { RetrySyncButton } from "./components/retrySyncButton" +import { RetryAllFailedReposButton } from "./components/retryAllFailedReposButton" export default function ConnectionManagementPage() { - const params = useParams(); - const searchParams = useSearchParams(); - const { toast } = useToast(); - const [org, setOrg] = useState(null); - const [connection, setConnection] = useState(null); - const [linkedRepos, setLinkedRepos] = useState([]); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); + const params = useParams() + const searchParams = useSearchParams() + const router = useRouter() + const [org, setOrg] = useState(null) + const [connection, setConnection] = useState(null) + const [linkedRepos, setLinkedRepos] = useState([]) + const [loading, setLoading] = useState(true) + const [error, setError] = useState(null) + + const handleSecretsNavigation = () => { + router.push(`/${params.domain}/secrets`) + } useEffect(() => { const loadData = async () => { try { - const orgResult = await getOrgFromDomainAction(params.domain as string); + const orgResult = await getOrgFromDomainAction(params.domain as string) if (isServiceError(orgResult)) { - setError(orgResult.message); - setLoading(false); - return; + setError(orgResult.message) + setLoading(false) + return } - setOrg(orgResult); + setOrg(orgResult) - const connectionId = Number(params.id); + const connectionId = Number(params.id) if (isNaN(connectionId)) { - setError("Invalid connection ID"); - setLoading(false); - return; + setError("Invalid connection ID") + setLoading(false) + return } - const connectionInfoResult = await getConnectionInfoAction(connectionId, params.domain as string); + const connectionInfoResult = await getConnectionInfoAction(connectionId, params.domain as string) if (isServiceError(connectionInfoResult)) { - setError(connectionInfoResult.message); - setLoading(false); - return; + setError(connectionInfoResult.message) + setLoading(false) + return } connectionInfoResult.linkedRepos.sort((a, b) => { // Helper function to get priority of indexing status const getPriority = (status: string) => { switch (status) { - case 'FAILED': return 0; - case 'IN_INDEX_QUEUE': - case 'INDEXING': return 1; - case 'INDEXED': return 2; - default: return 3; + case "FAILED": + return 0 + case "IN_INDEX_QUEUE": + case "INDEXING": + return 1 + case "INDEXED": + return 2 + default: + return 3 } - }; + } - const priorityA = getPriority(a.repoIndexingStatus); - const priorityB = getPriority(b.repoIndexingStatus); + const priorityA = getPriority(a.repoIndexingStatus) + const priorityB = getPriority(b.repoIndexingStatus) // First sort by priority if (priorityA !== priorityB) { - return priorityA - priorityB; + return priorityA - priorityB } // If same priority, sort by createdAt - return new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime(); - }); + return new Date(a.indexedAt ?? new Date()).getTime() - new Date(b.indexedAt ?? new Date()).getTime() + }) - setConnection(connectionInfoResult.connection); - setLinkedRepos(connectionInfoResult.linkedRepos); - setLoading(false); + setConnection(connectionInfoResult.connection) + setLinkedRepos(connectionInfoResult.linkedRepos) + setLoading(false) } catch (err) { - setError(err instanceof Error ? err.message : "An error occurred while loading the connection. If the problem persists, please contact us at team@sourcebot.dev"); - setLoading(false); + setError( + err instanceof Error + ? err.message + : "An error occurred while loading the connection. If the problem persists, please contact us at team@sourcebot.dev", + ) + setLoading(false) } - }; + } - loadData(); - const intervalId = setInterval(loadData, 1000); - return () => clearInterval(intervalId); - }, [params.domain, params.id]); + loadData() + const intervalId = setInterval(loadData, 1000) + return () => clearInterval(intervalId) + }, [params.domain, params.id]) if (loading) { - return
Loading...
; + return
Loading...
} if (error || !org || !connection) { - return ( - - ); + return } - const currentTab = searchParams.get("tab") || "overview"; + const currentTab = searchParams.get("tab") || "overview" return ( - -
+ +
@@ -135,10 +138,8 @@ export default function ConnectionManagementPage() { -
- +
+

{connection.name}

Last Synced At

-

{connection.syncedAt ? new Date(connection.syncedAt).toLocaleDateString() : 'never'}

+

+ {connection.syncedAt ? new Date(connection.syncedAt).toLocaleDateString() : "never"} +

Linked Repositories

@@ -168,74 +171,65 @@ export default function ConnectionManagementPage() {

Status

-
-

{connection.syncStatus}

+
+ {connection.syncStatus === "FAILED" ? ( + + +
+ + {connection.syncStatus} + +
+ + + + + + ) : ( + + {connection.syncStatus} + + )} {connection.syncStatus === "FAILED" && ( - + )}
+
-

Linked Repositories

- +
+

Linked Repositories

+ +
+
- {linkedRepos - .sort((a, b) => { - const aIndexedAt = a.indexedAt ?? new Date(); - const bIndexedAt = b.indexedAt ?? new Date(); - - return bIndexedAt.getTime() - aIndexedAt.getTime(); - }) - .map((repo) => ( - - ))} + {linkedRepos.map((repo) => ( + + ))}
- - + + - + - ); + ) } diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx index d217b9b0..3e7b0449 100644 --- a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx +++ b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx @@ -1,11 +1,11 @@ import { Button } from "@/components/ui/button"; import { getDisplayTime } from "@/lib/utils"; -import Link from "next/link"; import { useMemo } from "react"; import { ConnectionIcon } from "../connectionIcon"; -import { ConnectionSyncStatus } from "@sourcebot/db"; +import { ConnectionSyncStatus, Prisma } from "@sourcebot/db"; import { StatusIcon } from "../statusIcon"; - +import { AlertTriangle, CircleX} from "lucide-react"; +import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; const convertSyncStatus = (status: ConnectionSyncStatus) => { switch (status) { @@ -26,8 +26,10 @@ interface ConnectionListItemProps { name: string; type: string; status: ConnectionSyncStatus; + syncStatusMetadata: Prisma.JsonValue; editedAt: Date; syncedAt?: Date; + failedRepos?: { repoId: number, repoName: string }[]; } export const ConnectionListItem = ({ @@ -35,8 +37,10 @@ export const ConnectionListItem = ({ name, type, status, + syncStatusMetadata, editedAt, syncedAt, + failedRepos, }: ConnectionListItemProps) => { const statusDisplayName = useMemo(() => { switch (status) { @@ -52,46 +56,145 @@ export const ConnectionListItem = ({ } }, [status]); + const { notFoundData, displayNotFoundWarning } = useMemo(() => { + if (!syncStatusMetadata || typeof syncStatusMetadata !== 'object' || !('notFound' in syncStatusMetadata)) { + return { notFoundData: null, displayNotFoundWarning: false }; + } + + const notFoundData = syncStatusMetadata.notFound as { + users: string[], + orgs: string[], + repos: string[], + } + + return { notFoundData, displayNotFoundWarning: notFoundData.users.length > 0 || notFoundData.orgs.length > 0 || notFoundData.repos.length > 0 }; + }, [syncStatusMetadata]); + return ( - -
-
- -
-

{name}

- {`Edited ${getDisplayTime(editedAt)}`} -
-
-
- -

- {statusDisplayName} - { - ( - status === ConnectionSyncStatus.SYNCED || - status === ConnectionSyncStatus.FAILED - ) && syncedAt && ( - {` ${getDisplayTime(syncedAt)}`} - ) - } -

- +
+
+ +
+

{name}

+ {`Edited ${getDisplayTime(editedAt)}`}
+ {failedRepos && failedRepos.length > 0 && ( + + + window.location.href = `connections/${id}`} + /> + + +
+
+ +

Failed to Index Repositories

+
+
+

+ {failedRepos.length} {failedRepos.length === 1 ? 'repository' : 'repositories'} failed to index. This is likely due to temporary server load. +

+
+
+ {failedRepos.slice(0, 10).map(repo => ( + {repo.repoName} + ))} + {failedRepos.length > 10 && ( + + And {failedRepos.length - 10} more... + + )} +
+
+

+ Navigate to the connection for more details and to retry indexing. +

+
+
+ + + )} + {(notFoundData && displayNotFoundWarning) && ( + + + window.location.href = `connections/${id}`} + /> + + +
+
+ +

Unable to fetch all references

+
+

+ Some requested references couldn't be found. Verify the details below and ensure your connection is using a {" "} + {" "} + that has access to any private references. +

+
    + {notFoundData.users.length > 0 && ( +
  • + Users: + {notFoundData.users.join(', ')} +
    • + )} + {notFoundData.orgs.length > 0 && ( +
  • + {type === "gitlab" ? "Groups" : "Organizations"}: + {notFoundData.orgs.join(', ')} +
    • + )} + {notFoundData.repos.length > 0 && ( +
  • + {type === "gitlab" ? "Projects" : "Repositories"}: + {notFoundData.repos.join(', ')} +
    • + )} +
+
+ + + )}
- +
+ +

+ {statusDisplayName} + { + ( + status === ConnectionSyncStatus.SYNCED || + status === ConnectionSyncStatus.FAILED + ) && syncedAt && ( + {` ${getDisplayTime(syncedAt)}`} + ) + } +

+ +
+
) } diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx index 311847ab..6fd523ab 100644 --- a/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx +++ b/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx @@ -5,8 +5,8 @@ import { cn } from "@/lib/utils"; import { useEffect } from "react"; import { InfoCircledIcon } from "@radix-ui/react-icons"; import { useState } from "react"; -import { ConnectionSyncStatus } from "@sourcebot/db"; -import { getConnections } from "@/actions"; +import { ConnectionSyncStatus, Prisma } from "@sourcebot/db"; +import { getConnectionFailedRepos, getConnections } from "@/actions"; import { isServiceError } from "@/lib/utils"; interface ConnectionListProps { @@ -22,8 +22,10 @@ export const ConnectionList = ({ name: string; connectionType: string; syncStatus: ConnectionSyncStatus; + syncStatusMetadata: Prisma.JsonValue; updatedAt: Date; syncedAt?: Date; + failedRepos?: { repoId: number, repoName: string }[]; }[]>([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); @@ -35,7 +37,19 @@ export const ConnectionList = ({ if (isServiceError(result)) { setError(result.message); } else { - setConnections(result); + const connectionsWithFailedRepos = []; + for (const connection of result) { + const failedRepos = await getConnectionFailedRepos(connection.id, domain); + if (isServiceError(failedRepos)) { + setError(`An error occured while fetching the failed repositories for connection ${connection.name}. If the problem persists, please contact us at team@sourcebot.dev`); + } else { + connectionsWithFailedRepos.push({ + ...connection, + failedRepos, + }); + } + } + setConnections(connectionsWithFailedRepos); } setLoading(false); } catch (err) { @@ -69,8 +83,10 @@ export const ConnectionList = ({ name={connection.name} type={connection.connectionType} status={connection.syncStatus} + syncStatusMetadata={connection.syncStatusMetadata} editedAt={connection.updatedAt} syncedAt={connection.syncedAt ?? undefined} + failedRepos={connection.failedRepos} /> )) ) : ( diff --git a/packages/web/src/app/[domain]/connections/components/statusIcon.tsx b/packages/web/src/app/[domain]/connections/components/statusIcon.tsx index b7b6b2bf..edd4cc91 100644 --- a/packages/web/src/app/[domain]/connections/components/statusIcon.tsx +++ b/packages/web/src/app/[domain]/connections/components/statusIcon.tsx @@ -1,6 +1,5 @@ import { cn } from "@/lib/utils"; -import { Cross2Icon } from "@radix-ui/react-icons"; -import { CircleCheckIcon } from "lucide-react"; +import { CircleCheckIcon, CircleXIcon } from "lucide-react"; import { useMemo } from "react"; import { FiLoader } from "react-icons/fi"; @@ -18,7 +17,7 @@ export const StatusIcon = ({ case 'succeeded': return ; case 'failed': - return ; + return ; } }, [className, status]); diff --git a/packages/web/src/app/[domain]/connections/quickActions.ts b/packages/web/src/app/[domain]/connections/quickActions.ts index 21d20cb7..65cb63af 100644 --- a/packages/web/src/app/[domain]/connections/quickActions.ts +++ b/packages/web/src/app/[domain]/connections/quickActions.ts @@ -79,6 +79,16 @@ export const gitlabQuickActions: QuickAction[] = [ ] }), name: "Add a project", + }, + { + fn: (previous: GitlabConnectionConfig) => ({ + ...previous, + users: [ + ...previous.users ?? [], + "" + ] + }), + name: "Add a user", } ] diff --git a/packages/web/src/app/[domain]/secrets/secretsTable.tsx b/packages/web/src/app/[domain]/secrets/secretsTable.tsx index 097d9922..e4a68671 100644 --- a/packages/web/src/app/[domain]/secrets/secretsTable.tsx +++ b/packages/web/src/app/[domain]/secrets/secretsTable.tsx @@ -31,14 +31,17 @@ export const SecretsTable = ({ initialSecrets }: SecretsTableProps) => { const domain = useDomain(); useEffect(() => { - getSecrets(domain).then((keys) => { - if ('keys' in keys) { - setSecrets(keys); - } else { - console.error(keys); + const fetchSecretKeys = async () => { + const keys = await getSecrets(domain); + if (isServiceError(keys)) { + console.error("Failed to fetch secrets:", keys); + return; } - }) - }, []); + setSecrets(keys); + }; + + fetchSecretKeys(); + }, [domain]); const form = useForm>({ resolver: zodResolver(formSchema), diff --git a/packages/web/src/components/ui/hover-card.tsx b/packages/web/src/components/ui/hover-card.tsx new file mode 100644 index 00000000..e54d91cf --- /dev/null +++ b/packages/web/src/components/ui/hover-card.tsx @@ -0,0 +1,29 @@ +"use client" + +import * as React from "react" +import * as HoverCardPrimitive from "@radix-ui/react-hover-card" + +import { cn } from "@/lib/utils" + +const HoverCard = HoverCardPrimitive.Root + +const HoverCardTrigger = HoverCardPrimitive.Trigger + +const HoverCardContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, align = "center", sideOffset = 4, ...props }, ref) => ( + +)) +HoverCardContent.displayName = HoverCardPrimitive.Content.displayName + +export { HoverCard, HoverCardTrigger, HoverCardContent } diff --git a/packages/web/src/lib/syncStatusMetadataSchema.ts b/packages/web/src/lib/syncStatusMetadataSchema.ts new file mode 100644 index 00000000..f9855db9 --- /dev/null +++ b/packages/web/src/lib/syncStatusMetadataSchema.ts @@ -0,0 +1,17 @@ +import { z } from "zod"; + +export const NotFoundSchema = z.object({ + users: z.array(z.string()), + orgs: z.array(z.string()), + repos: z.array(z.string()), +}); + +export const SyncStatusMetadataSchema = z.object({ + notFound: NotFoundSchema.optional(), + error: z.string().optional(), + secretKey: z.string().optional(), + status: z.number().optional(), +}); + +export type NotFoundData = z.infer; +export type SyncStatusMetadata = z.infer; \ No newline at end of file