diff --git a/packages/backend/src/azuredevops.ts b/packages/backend/src/azuredevops.ts index a06b9c09..1f9c089b 100644 --- a/packages/backend/src/azuredevops.ts +++ b/packages/backend/src/azuredevops.ts @@ -47,47 +47,39 @@ export const getAzureDevOpsReposFromConfig = async ( const useTfsPath = config.useTfsPath || false; let allRepos: GitRepository[] = []; - let notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.orgs) { - const { validRepos, notFoundOrgs } = await getReposForOrganizations( + const { repos, warnings } = await getReposForOrganizations( config.orgs, baseUrl, token, useTfsPath ); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundOrgs; + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.projects) { - const { validRepos, notFoundProjects } = await getReposForProjects( + const { repos, warnings } = await getReposForProjects( config.projects, baseUrl, token, useTfsPath ); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFound.repos.concat(notFoundProjects); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { validRepos, notFoundRepos } = await getRepos( + const { repos, warnings } = await getRepos( config.repos, baseUrl, token, useTfsPath ); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFound.repos.concat(notFoundRepos); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } let repos = allRepos @@ -103,8 +95,8 @@ export const getAzureDevOpsReposFromConfig = async ( logger.debug(`Found ${repos.length} total repositories.`); return { - validRepos: repos, - notFound, + repos, + warnings: allWarnings, }; }; @@ -221,10 +213,11 @@ async function getReposForOrganizations( // Check if it's a 404-like error (organization not found) if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { - logger.error(`Organization ${org} not found or no access`); + const warning = `Organization ${org} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: org + type: 'warning' as const, + warning }; } throw error; @@ -232,11 +225,11 @@ async function getReposForOrganizations( })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundOrgs, + repos, + warnings, }; } @@ -274,10 +267,11 @@ async function getReposForProjects( logger.error(`Failed to fetch repositories for project ${project}.`, error); if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { - logger.error(`Project ${project} not found or no access`); + const warning = `Project ${project} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning }; } throw error; @@ -285,11 +279,11 @@ async function getReposForProjects( })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundProjects, + repos, + warnings, }; } @@ -328,10 +322,11 @@ async function getRepos( logger.error(`Failed to fetch repository ${repo}.`, error); if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { - logger.error(`Repository ${repo} not found or no access`); + const warning = `Repository ${repo} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw error; @@ -339,10 +334,10 @@ async function getRepos( })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos, + repos, + warnings, }; } \ No newline at end of file diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index cfa591cc..ae4d6b40 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -27,9 +27,9 @@ interface BitbucketClient { apiClient: any; baseUrl: string; gitUrl: string; - getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundWorkspaces: string[]}>; - getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundProjects: string[]}>; - getRepos: (client: BitbucketClient, repos: string[]) => Promise<{validRepos: BitbucketRepository[], notFoundRepos: string[]}>; + getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; + getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; + getRepos: (client: BitbucketClient, repos: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; shouldExcludeRepo: (repo: BitbucketRepository, config: BitbucketConnectionConfig) => boolean; } @@ -71,32 +71,24 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon cloudClient(config.user, token); let allRepos: BitbucketRepository[] = []; - let notFound: { - orgs: string[], - users: string[], - repos: string[], - } = { - orgs: [], - users: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.workspaces) { - const { validRepos, notFoundWorkspaces } = await client.getReposForWorkspace(client, config.workspaces); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundWorkspaces; + const { repos, warnings } = await client.getReposForWorkspace(client, config.workspaces); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.projects) { - const { validRepos, notFoundProjects } = await client.getReposForProjects(client, config.projects); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundProjects; + const { repos, warnings } = await client.getReposForProjects(client, config.projects); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { validRepos, notFoundRepos } = await client.getRepos(client, config.repos); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFoundRepos; + const { repos, warnings } = await client.getRepos(client, config.repos); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } const filteredRepos = allRepos.filter((repo) => { @@ -104,8 +96,8 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon }); return { - validRepos: filteredRepos, - notFound, + repos: filteredRepos, + warnings: allWarnings, }; } @@ -186,7 +178,7 @@ function parseUrl(url: string): { path: string; query: Record; } } -async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{validRepos: CloudRepository[], notFoundWorkspaces: string[]}> { +async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { const results = await Promise.allSettled(workspaces.map(async (workspace) => { try { logger.debug(`Fetching all repos for workspace ${workspace}...`); @@ -221,10 +213,11 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st const status = e?.cause?.response?.status; if (status == 404) { - logger.error(`Workspace ${workspace} not found or invalid access`) + const warning = `Workspace ${workspace} not found or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: workspace + type: 'warning' as const, + warning } } throw e; @@ -232,21 +225,22 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundWorkspaces } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundWorkspaces, + repos, + warnings, }; } -async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{validRepos: CloudRepository[], notFoundProjects: string[]}> { +async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { const results = await Promise.allSettled(projects.map(async (project) => { const [workspace, project_name] = project.split('/'); if (!workspace || !project_name) { - logger.error(`Invalid project ${project}`); + const warning = `Invalid project ${project}`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning } } @@ -282,10 +276,11 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin const status = e?.cause?.response?.status; if (status == 404) { - logger.error(`Project ${project_name} not found in ${workspace} or invalid access`) + const warning = `Project ${project_name} not found in ${workspace} or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning } } throw e; @@ -293,21 +288,22 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundProjects + repos, + warnings } } -async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise<{validRepos: CloudRepository[], notFoundRepos: string[]}> { - const results = await Promise.allSettled(repos.map(async (repo) => { +async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { + const results = await Promise.allSettled(repoList.map(async (repo) => { const [workspace, repo_slug] = repo.split('/'); if (!workspace || !repo_slug) { - logger.error(`Invalid repo ${repo}`); + const warning = `Invalid repo ${repo}`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } @@ -329,10 +325,11 @@ async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise< const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`Repo ${repo} not found in ${workspace} or invalid access`); + const warning = `Repo ${repo} not found in ${workspace} or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw e; @@ -340,10 +337,10 @@ async function cloudGetRepos(client: BitbucketClient, repos: string[]): Promise< })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos + repos, + warnings }; } @@ -434,15 +431,16 @@ const getPaginatedServer = async ( return results; } -async function serverGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{validRepos: ServerRepository[], notFoundWorkspaces: string[]}> { +async function serverGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { + const warnings = workspaces.map(workspace => `Workspaces are not supported in Bitbucket Server: ${workspace}`); logger.debug('Workspaces are not supported in Bitbucket Server'); return { - validRepos: [], - notFoundWorkspaces: workspaces + repos: [], + warnings }; } -async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{validRepos: ServerRepository[], notFoundProjects: string[]}> { +async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { const results = await Promise.allSettled(projects.map(async (project) => { try { logger.debug(`Fetching all repos for project ${project}...`); @@ -477,10 +475,11 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri const status = e?.cause?.response?.status; if (status == 404) { - logger.error(`Project ${project} not found or invalid access`); + const warning = `Project ${project} not found or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning }; } throw e; @@ -488,21 +487,22 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundProjects + repos, + warnings }; } -async function serverGetRepos(client: BitbucketClient, repos: string[]): Promise<{validRepos: ServerRepository[], notFoundRepos: string[]}> { - const results = await Promise.allSettled(repos.map(async (repo) => { +async function serverGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { + const results = await Promise.allSettled(repoList.map(async (repo) => { const [project, repo_slug] = repo.split('/'); if (!project || !repo_slug) { - logger.error(`Invalid repo ${repo}`); + const warning = `Invalid repo ${repo}`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } @@ -524,10 +524,11 @@ async function serverGetRepos(client: BitbucketClient, repos: string[]): Promise const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`Repo ${repo} not found in project ${project} or invalid access`); + const warning = `Repo ${repo} not found in project ${project} or invalid access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw e; @@ -535,10 +536,10 @@ async function serverGetRepos(client: BitbucketClient, repos: string[]): Promise })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos + repos, + warnings }; } diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index 4179e54a..eaba0638 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -168,18 +168,10 @@ export class ConnectionManager { let result: { repoData: RepoData[], - notFound: { - users: string[], - orgs: string[], - repos: string[], - } + warnings: string[], } = { repoData: [], - notFound: { - users: [], - orgs: [], - repos: [], - } + warnings: [], }; try { @@ -201,7 +193,7 @@ export class ConnectionManager { return await compileBitbucketConfig(config, job.data.connectionId, orgId, this.db); } case 'azuredevops': { - return await compileAzureDevOpsConfig(config, job.data.connectionId, orgId, this.db, abortController); + return await compileAzureDevOpsConfig(config, job.data.connectionId, orgId, this.db); } case 'git': { return await compileGenericGitHostConfig(config, job.data.connectionId, orgId); @@ -221,7 +213,17 @@ export class ConnectionManager { } } - let { repoData } = result; + let { repoData, warnings } = result; + + await this.db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + warningMessages: warnings, + }, + }); + // Filter out any duplicates by external_id and external_codeHostUrl. repoData = repoData.filter((repo, index, self) => { diff --git a/packages/backend/src/connectionUtils.ts b/packages/backend/src/connectionUtils.ts index ca19fa59..074dfe7f 100644 --- a/packages/backend/src/connectionUtils.ts +++ b/packages/backend/src/connectionUtils.ts @@ -5,21 +5,21 @@ type ValidResult = { data: T[]; }; -type NotFoundResult = { - type: 'notFound'; - value: string; +type WarningResult = { + type: 'warning'; + warning: string; }; -type CustomResult = ValidResult | NotFoundResult; +type CustomResult = ValidResult | WarningResult; export function processPromiseResults( results: PromiseSettledResult>[], ): { validItems: T[]; - notFoundItems: string[]; + warnings: string[]; } { const validItems: T[] = []; - const notFoundItems: string[] = []; + const warnings: string[] = []; results.forEach(result => { if (result.status === 'fulfilled') { @@ -27,14 +27,14 @@ export function processPromiseResults( if (value.type === 'valid') { validItems.push(...value.data); } else { - notFoundItems.push(value.value); + warnings.push(value.warning); } } }); return { validItems, - notFoundItems, + warnings, }; } diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index aefe1c24..5dca28d3 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -29,32 +29,24 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org }); let allRepos: GiteaRepository[] = []; - let notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.orgs) { - const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, api); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundOrgs; + const { repos, warnings } = await getReposForOrgs(config.orgs, api); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { validRepos, notFoundRepos } = await getRepos(config.repos, api); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFoundRepos; + const { repos, warnings } = await getRepos(config.repos, api); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.users) { - const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, api); - allRepos = allRepos.concat(validRepos); - notFound.users = notFoundUsers; + const { repos, warnings } = await getReposOwnedByUsers(config.users, api); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } allRepos = allRepos.filter(repo => repo.full_name !== undefined); @@ -78,8 +70,8 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org logger.debug(`Found ${repos.length} total repositories.`); return { - validRepos: repos, - notFound, + repos, + warnings: allWarnings, }; } @@ -145,10 +137,11 @@ const getReposOwnedByUsers = async (users: string[], api: Api) => { Sentry.captureException(e); if (e?.status === 404) { - logger.error(`User ${user} not found or no access`); + const warning = `User ${user} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: user + type: 'warning' as const, + warning }; } throw e; @@ -156,11 +149,11 @@ const getReposOwnedByUsers = async (users: string[], api: Api) => { })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundUsers, + repos, + warnings, }; } @@ -185,10 +178,11 @@ const getReposForOrgs = async (orgs: string[], api: Api) => { Sentry.captureException(e); if (e?.status === 404) { - logger.error(`Organization ${org} not found or no access`); + const warning = `Organization ${org} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: org + type: 'warning' as const, + warning }; } throw e; @@ -196,16 +190,16 @@ const getReposForOrgs = async (orgs: string[], api: Api) => { })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundOrgs, + repos, + warnings, }; } -const getRepos = async (repos: string[], api: Api) => { - const results = await Promise.allSettled(repos.map(async (repo) => { +const getRepos = async (repoList: string[], api: Api) => { + const results = await Promise.allSettled(repoList.map(async (repo) => { try { logger.debug(`Fetching repository info for ${repo}...`); @@ -223,10 +217,11 @@ const getRepos = async (repos: string[], api: Api) => { Sentry.captureException(e); if (e?.status === 404) { - logger.error(`Repository ${repo} not found or no access`); + const warning = `Repository ${repo} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw e; @@ -234,11 +229,11 @@ const getRepos = async (repos: string[], api: Api) => { })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos, + repos, + warnings, }; } diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index ee26a024..0b64eca8 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -92,7 +92,7 @@ const getOctokitWithGithubApp = async ( } } -export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => { +export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal): Promise<{ repos: OctokitRepository[], warnings: string[] }> => { const hostname = config.url ? new URL(config.url).hostname : GITHUB_CLOUD_HOSTNAME; @@ -108,6 +108,7 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o url: config.url, }); + if (isAuthenticated) { try { await octokit.rest.users.getAuthenticated(); @@ -133,32 +134,24 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o } let allRepos: OctokitRepository[] = []; - let notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.orgs) { - const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, octokit, signal, config.url); - allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundOrgs; + const { repos, warnings } = await getReposForOrgs(config.orgs, octokit, signal, config.url); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { validRepos, notFoundRepos } = await getRepos(config.repos, octokit, signal, config.url); - allRepos = allRepos.concat(validRepos); - notFound.repos = notFoundRepos; + const { repos, warnings } = await getRepos(config.repos, octokit, signal, config.url); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } if (config.users) { - const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, octokit, signal, config.url); - allRepos = allRepos.concat(validRepos); - notFound.users = notFoundUsers; + const { repos, warnings } = await getReposOwnedByUsers(config.users, octokit, signal, config.url); + allRepos = allRepos.concat(repos); + allWarnings = allWarnings.concat(warnings); } let repos = allRepos @@ -177,8 +170,8 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o logger.debug(`Found ${repos.length} total repositories.`); return { - validRepos: repos, - notFound, + repos, + warnings: allWarnings, }; } @@ -256,10 +249,11 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A logger.error(`Failed to fetch repositories for user ${user}.`, error); if (isHttpError(error, 404)) { - logger.error(`User ${user} not found or no access`); + const warning = `User ${user} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: user + type: 'warning' as const, + warning }; } throw error; @@ -267,11 +261,11 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundUsers, + repos, + warnings, }; } @@ -303,10 +297,11 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi logger.error(`Failed to fetch repositories for org ${org}.`, error); if (isHttpError(error, 404)) { - logger.error(`Organization ${org} not found or no access`); + const warning = `Organization ${org} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: org + type: 'warning' as const, + warning }; } throw error; @@ -314,11 +309,11 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundOrgs, + repos, + warnings, }; } @@ -352,10 +347,11 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna logger.error(`Failed to fetch repository ${repo}.`, error); if (isHttpError(error, 404)) { - logger.error(`Repository ${repo} not found or no access`); + const warning = `Repository ${repo} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: repo + type: 'warning' as const, + warning }; } throw error; @@ -363,11 +359,11 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: repos, warnings } = processPromiseResults(results); return { - validRepos, - notFoundRepos, + repos, + warnings, }; } diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index d13692d3..de52e64d 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -33,15 +33,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o }); let allRepos: ProjectSchema[] = []; - let notFound: { - orgs: string[], - users: string[], - repos: string[], - } = { - orgs: [], - users: [], - repos: [], - }; + let allWarnings: string[] = []; if (config.all === true) { if (hostname !== GITLAB_CLOUD_HOSTNAME) { @@ -61,7 +53,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o throw e; } } else { - logger.warn(`Ignoring option all:true in config : host is ${GITLAB_CLOUD_HOSTNAME}`); + const warning = `Ignoring option all:true in config : host is ${GITLAB_CLOUD_HOSTNAME}`; + logger.warn(warning); + allWarnings = allWarnings.concat(warning); } } @@ -87,10 +81,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`Group ${group} not found or no access`); + const warning = `Group ${group} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: group + type: 'warning' as const, + warning }; } throw e; @@ -98,9 +93,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); - notFound.orgs = notFoundOrgs; + allWarnings = allWarnings.concat(warnings); } if (config.users) { @@ -124,10 +119,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`User ${user} not found or no access`); + const warning = `User ${user} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: user + type: 'warning' as const, + warning }; } throw e; @@ -135,9 +131,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundUsers } = processPromiseResults(results); + const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); - notFound.users = notFoundUsers; + allWarnings = allWarnings.concat(warnings); } if (config.projects) { @@ -160,10 +156,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o const status = e?.cause?.response?.status; if (status === 404) { - logger.error(`Project ${project} not found or no access`); + const warning = `Project ${project} not found or no access`; + logger.warn(warning); return { - type: 'notFound' as const, - value: project + type: 'warning' as const, + warning }; } throw e; @@ -171,9 +168,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o })); throwIfAnyFailed(results); - const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); - notFound.repos = notFoundRepos; + allWarnings = allWarnings.concat(warnings); } let repos = allRepos @@ -192,8 +189,8 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o logger.debug(`Found ${repos.length} total repositories.`); return { - validRepos: repos, - notFound, + repos, + warnings: allWarnings, }; } diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index f5edb4f6..d78455e0 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -24,22 +24,20 @@ export type RepoData = WithRequired; const logger = createLogger('repo-compile-utils'); +type CompileResult = { + repoData: RepoData[], + warnings: string[], +} + export const compileGithubConfig = async ( config: GithubConnectionConfig, connectionId: number, orgId: number, db: PrismaClient, - abortController: AbortController): Promise<{ - repoData: RepoData[], - notFound: { - users: string[], - orgs: string[], - repos: string[], - } - }> => { + abortController: AbortController): Promise => { const gitHubReposResult = await getGitHubReposFromConfig(config, orgId, db, abortController.signal); - const gitHubRepos = gitHubReposResult.validRepos; - const notFound = gitHubReposResult.notFound; + const gitHubRepos = gitHubReposResult.repos; + const warnings = gitHubReposResult.warnings; const hostUrl = config.url ?? 'https://github.com'; const repoNameRoot = new URL(hostUrl) @@ -100,7 +98,7 @@ export const compileGithubConfig = async ( return { repoData: repos, - notFound, + warnings, }; } @@ -108,11 +106,11 @@ export const compileGitlabConfig = async ( config: GitlabConnectionConfig, connectionId: number, orgId: number, - db: PrismaClient) => { + db: PrismaClient): Promise => { const gitlabReposResult = await getGitLabReposFromConfig(config, orgId, db); - const gitlabRepos = gitlabReposResult.validRepos; - const notFound = gitlabReposResult.notFound; + const gitlabRepos = gitlabReposResult.repos; + const warnings = gitlabReposResult.warnings; const hostUrl = config.url ?? 'https://gitlab.com'; const repoNameRoot = new URL(hostUrl) @@ -177,7 +175,7 @@ export const compileGitlabConfig = async ( return { repoData: repos, - notFound, + warnings, }; } @@ -185,11 +183,11 @@ export const compileGiteaConfig = async ( config: GiteaConnectionConfig, connectionId: number, orgId: number, - db: PrismaClient) => { + db: PrismaClient): Promise => { const giteaReposResult = await getGiteaReposFromConfig(config, orgId, db); - const giteaRepos = giteaReposResult.validRepos; - const notFound = giteaReposResult.notFound; + const giteaRepos = giteaReposResult.repos; + const warnings = giteaReposResult.warnings; const hostUrl = config.url ?? 'https://gitea.com'; const repoNameRoot = new URL(hostUrl) @@ -248,14 +246,14 @@ export const compileGiteaConfig = async ( return { repoData: repos, - notFound, + warnings, }; } export const compileGerritConfig = async ( config: GerritConnectionConfig, connectionId: number, - orgId: number) => { + orgId: number): Promise => { const gerritRepos = await getGerritReposFromConfig(config); const hostUrl = config.url; @@ -329,11 +327,7 @@ export const compileGerritConfig = async ( return { repoData: repos, - notFound: { - users: [], - orgs: [], - repos: [], - } + warnings: [], }; } @@ -341,11 +335,11 @@ export const compileBitbucketConfig = async ( config: BitbucketConnectionConfig, connectionId: number, orgId: number, - db: PrismaClient) => { + db: PrismaClient): Promise => { const bitbucketReposResult = await getBitbucketReposFromConfig(config, orgId, db); - const bitbucketRepos = bitbucketReposResult.validRepos; - const notFound = bitbucketReposResult.notFound; + const bitbucketRepos = bitbucketReposResult.repos; + const warnings = bitbucketReposResult.warnings; const hostUrl = config.url ?? 'https://bitbucket.org'; const repoNameRoot = new URL(hostUrl) @@ -450,7 +444,7 @@ export const compileBitbucketConfig = async ( return { repoData: repos, - notFound, + warnings, }; } @@ -458,7 +452,7 @@ export const compileGenericGitHostConfig = async ( config: GenericGitHostConnectionConfig, connectionId: number, orgId: number, -) => { +): Promise => { const configUrl = new URL(config.url); if (configUrl.protocol === 'file:') { return compileGenericGitHostConfig_file(config, orgId, connectionId); @@ -476,7 +470,7 @@ export const compileGenericGitHostConfig_file = async ( config: GenericGitHostConnectionConfig, orgId: number, connectionId: number, -) => { +): Promise => { const configUrl = new URL(config.url); assert(configUrl.protocol === 'file:', 'config.url must be a file:// URL'); @@ -486,30 +480,24 @@ export const compileGenericGitHostConfig_file = async ( }); const repos: RepoData[] = []; - const notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + const warnings: string[] = []; await Promise.all(repoPaths.map(async (repoPath) => { const isGitRepo = await isPathAValidGitRepoRoot({ path: repoPath, }); if (!isGitRepo) { - logger.warn(`Skipping ${repoPath} - not a git repository.`); - notFound.repos.push(repoPath); + const warning = `Skipping ${repoPath} - not a git repository.`; + logger.warn(warning); + warnings.push(warning); return; } const origin = await getOriginUrl(repoPath); if (!origin) { - logger.warn(`Skipping ${repoPath} - remote.origin.url not found in git config.`); - notFound.repos.push(repoPath); + const warning = `Skipping ${repoPath} - remote.origin.url not found in git config.`; + logger.warn(warning); + warnings.push(warning); return; } @@ -552,7 +540,7 @@ export const compileGenericGitHostConfig_file = async ( return { repoData: repos, - notFound, + warnings, } } @@ -561,27 +549,21 @@ export const compileGenericGitHostConfig_url = async ( config: GenericGitHostConnectionConfig, orgId: number, connectionId: number, -) => { +): Promise => { const remoteUrl = new URL(config.url); assert(remoteUrl.protocol === 'http:' || remoteUrl.protocol === 'https:', 'config.url must be a http:// or https:// URL'); - const notFound: { - users: string[], - orgs: string[], - repos: string[], - } = { - users: [], - orgs: [], - repos: [], - }; + const warnings: string[] = []; // Validate that we are dealing with a valid git repo. const isGitRepo = await isUrlAValidGitRepo(remoteUrl.toString()); if (!isGitRepo) { - notFound.repos.push(remoteUrl.toString()); + const warning = `Skipping ${remoteUrl.toString()} - not a git repository.`; + logger.warn(warning); + warnings.push(warning); return { repoData: [], - notFound, + warnings, } } @@ -616,7 +598,7 @@ export const compileGenericGitHostConfig_url = async ( return { repoData: [repo], - notFound, + warnings, } } @@ -624,12 +606,11 @@ export const compileAzureDevOpsConfig = async ( config: AzureDevOpsConnectionConfig, connectionId: number, orgId: number, - db: PrismaClient, - abortController: AbortController) => { + db: PrismaClient): Promise => { const azureDevOpsReposResult = await getAzureDevOpsReposFromConfig(config, orgId, db); - const azureDevOpsRepos = azureDevOpsReposResult.validRepos; - const notFound = azureDevOpsReposResult.notFound; + const azureDevOpsRepos = azureDevOpsReposResult.repos; + const warnings = azureDevOpsReposResult.warnings; const hostUrl = config.url ?? 'https://dev.azure.com'; const repoNameRoot = new URL(hostUrl) @@ -699,6 +680,6 @@ export const compileAzureDevOpsConfig = async ( return { repoData: repos, - notFound, + warnings, }; } diff --git a/packages/web/src/lib/syncStatusMetadataSchema.ts b/packages/web/src/lib/syncStatusMetadataSchema.ts deleted file mode 100644 index f9855db9..00000000 --- a/packages/web/src/lib/syncStatusMetadataSchema.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { z } from "zod"; - -export const NotFoundSchema = z.object({ - users: z.array(z.string()), - orgs: z.array(z.string()), - repos: z.array(z.string()), -}); - -export const SyncStatusMetadataSchema = z.object({ - notFound: NotFoundSchema.optional(), - error: z.string().optional(), - secretKey: z.string().optional(), - status: z.number().optional(), -}); - -export type NotFoundData = z.infer; -export type SyncStatusMetadata = z.infer; \ No newline at end of file