mirror of
https://github.com/sourcebot-dev/sourcebot.git
synced 2025-12-11 20:05:25 +00:00
Graceful error handling when calling code host apis (#142)
This commit is contained in:
Brendan Kellam
GitHub
parent
4e68dc5032
commit
03aa608e21
9 changed files with 261 additions and 187 deletions
|
|
@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||
- Added config option `settings.reindexInterval` and `settings.resyncInterval` to control how often the index should be re-indexed and re-synced. ([#134](https://github.com/sourcebot-dev/sourcebot/pull/134))
|
||||
- Added `exclude.size` to the GitHub config to allow excluding repositories by size. ([#137](https://github.com/sourcebot-dev/sourcebot/pull/137))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed issue where config synchronization was failing entirely when a single api call fails. ([#142](https://github.com/sourcebot-dev/sourcebot/pull/142))
|
||||
|
||||
## [2.6.2] - 2024-12-13
|
||||
|
||||
### Added
|
||||
|
|
|
|||
|
|
@ -28,9 +28,18 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
|
|||
const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
|
||||
const hostname = new URL(config.url).hostname;
|
||||
|
||||
const { durationMs, data: projects } = await measure(() =>
|
||||
fetchAllProjects(url)
|
||||
);
|
||||
const { durationMs, data: projects } = await measure(async () => {
|
||||
try {
|
||||
return fetchAllProjects(url)
|
||||
} catch (err) {
|
||||
logger.error(`Failed to fetch projects from ${url}`, err);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
if (!projects) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// exclude "All-Projects" and "All-Users" projects
|
||||
delete projects['All-Projects'];
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
|
|||
}
|
||||
|
||||
const getTagsForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
|
||||
try {
|
||||
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
|
||||
const { durationMs, data: tags } = await measure(() =>
|
||||
paginate((page) => api.repos.repoListTags(owner, repo, {
|
||||
|
|
@ -130,9 +131,14 @@ const getTagsForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
|
|||
);
|
||||
logger.debug(`Found ${tags.length} tags in repo ${owner}/${repo} in ${durationMs}ms.`);
|
||||
return tags;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch tags for repo ${owner}/${repo}.`, e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const getBranchesForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
|
||||
try {
|
||||
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
|
||||
const { durationMs, data: branches } = await measure(() =>
|
||||
paginate((page) => api.repos.repoListBranches(owner, repo, {
|
||||
|
|
@ -141,10 +147,15 @@ const getBranchesForRepo = async <T>(owner: string, repo: string, api: Api<T>) =
|
|||
);
|
||||
logger.debug(`Found ${branches.length} branches in repo ${owner}/${repo} in ${durationMs}ms.`);
|
||||
return branches;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch branches for repo ${owner}/${repo}.`, e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
|
||||
const repos = (await Promise.all(users.map(async (user) => {
|
||||
try {
|
||||
logger.debug(`Fetching repos for user ${user}...`);
|
||||
|
||||
const { durationMs, data } = await measure(() =>
|
||||
|
|
@ -155,6 +166,10 @@ const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
|
|||
|
||||
logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`);
|
||||
return data;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch repos for user ${user}.`, e);
|
||||
return [];
|
||||
}
|
||||
}))).flat();
|
||||
|
||||
return repos;
|
||||
|
|
@ -162,6 +177,7 @@ const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
|
|||
|
||||
const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
|
||||
return (await Promise.all(orgs.map(async (org) => {
|
||||
try {
|
||||
logger.debug(`Fetching repos for org ${org}...`);
|
||||
|
||||
const { durationMs, data } = await measure(() =>
|
||||
|
|
@ -173,11 +189,16 @@ const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
|
|||
|
||||
logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`);
|
||||
return data;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch repos for org ${org}.`, e);
|
||||
return [];
|
||||
}
|
||||
}))).flat();
|
||||
}
|
||||
|
||||
const getRepos = async <T>(repos: string[], api: Api<T>) => {
|
||||
return Promise.all(repos.map(async (repo) => {
|
||||
return (await Promise.all(repos.map(async (repo) => {
|
||||
try {
|
||||
logger.debug(`Fetching repository info for ${repo}...`);
|
||||
|
||||
const [owner, repoName] = repo.split('/');
|
||||
|
|
@ -187,8 +208,12 @@ const getRepos = async <T>(repos: string[], api: Api<T>) => {
|
|||
|
||||
logger.debug(`Found repo ${repo} in ${durationMs}ms.`);
|
||||
|
||||
return response.data;
|
||||
}));
|
||||
return [response.data];
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch repository info for ${repo}.`, e);
|
||||
return [];
|
||||
}
|
||||
}))).flat();
|
||||
}
|
||||
|
||||
// @see : https://docs.gitea.com/development/api-usage#pagination
|
||||
|
|
|
|||
|
|
@ -201,8 +201,8 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
|
|||
}
|
||||
|
||||
const getTagsForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
|
||||
try {
|
||||
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
|
||||
|
||||
const { durationMs, data: tags } = await measure(() => octokit.paginate(octokit.repos.listTags, {
|
||||
owner,
|
||||
repo,
|
||||
|
|
@ -214,9 +214,14 @@ const getTagsForRepo = async (owner: string, repo: string, octokit: Octokit, sig
|
|||
|
||||
logger.debug(`Found ${tags.length} tags for repo ${owner}/${repo} in ${durationMs}ms`);
|
||||
return tags;
|
||||
} catch (e) {
|
||||
logger.debug(`Error fetching tags for repo ${owner}/${repo}: ${e}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const getBranchesForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
|
||||
try {
|
||||
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
|
||||
const { durationMs, data: branches } = await measure(() => octokit.paginate(octokit.repos.listBranches, {
|
||||
owner,
|
||||
|
|
@ -228,16 +233,19 @@ const getBranchesForRepo = async (owner: string, repo: string, octokit: Octokit,
|
|||
}));
|
||||
logger.debug(`Found ${branches.length} branches for repo ${owner}/${repo} in ${durationMs}ms`);
|
||||
return branches;
|
||||
} catch (e) {
|
||||
logger.debug(`Error fetching branches for repo ${owner}/${repo}: ${e}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, octokit: Octokit, signal: AbortSignal) => {
|
||||
// @todo : error handling
|
||||
const repos = (await Promise.all(users.map(async (user) => {
|
||||
try {
|
||||
logger.debug(`Fetching repository info for user ${user}...`);
|
||||
const start = Date.now();
|
||||
|
||||
const result = await (() => {
|
||||
const { durationMs, data } = await measure(async () => {
|
||||
if (isAuthenticated) {
|
||||
return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
|
||||
username: user,
|
||||
|
|
@ -257,12 +265,14 @@ const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, o
|
|||
},
|
||||
});
|
||||
}
|
||||
})();
|
||||
});
|
||||
|
||||
const duration = Date.now() - start;
|
||||
logger.debug(`Found ${result.length} owned by user ${user} in ${duration}ms.`);
|
||||
|
||||
return result;
|
||||
logger.debug(`Found ${data.length} owned by user ${user} in ${durationMs}ms.`);
|
||||
return data;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch repository info for user ${user}.`, e);
|
||||
return [];
|
||||
}
|
||||
}))).flat();
|
||||
|
||||
return repos;
|
||||
|
|
@ -270,45 +280,50 @@ const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, o
|
|||
|
||||
const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal) => {
|
||||
const repos = (await Promise.all(orgs.map(async (org) => {
|
||||
try {
|
||||
logger.debug(`Fetching repository info for org ${org}...`);
|
||||
const start = Date.now();
|
||||
|
||||
const result = await octokit.paginate(octokit.repos.listForOrg, {
|
||||
const { durationMs, data } = await measure(() => octokit.paginate(octokit.repos.listForOrg, {
|
||||
org: org,
|
||||
per_page: 100,
|
||||
request: {
|
||||
signal
|
||||
}
|
||||
});
|
||||
}));
|
||||
|
||||
const duration = Date.now() - start;
|
||||
logger.debug(`Found ${result.length} in org ${org} in ${duration}ms.`);
|
||||
|
||||
return result;
|
||||
logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`);
|
||||
return data;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch repository info for org ${org}.`, e);
|
||||
return [];
|
||||
}
|
||||
}))).flat();
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
||||
const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal) => {
|
||||
const repos = await Promise.all(repoList.map(async (repo) => {
|
||||
const repos = (await Promise.all(repoList.map(async (repo) => {
|
||||
try {
|
||||
logger.debug(`Fetching repository info for ${repo}...`);
|
||||
const start = Date.now();
|
||||
|
||||
const [owner, repoName] = repo.split('/');
|
||||
const result = await octokit.repos.get({
|
||||
const { durationMs, data: result } = await measure(() => octokit.repos.get({
|
||||
owner,
|
||||
repo: repoName,
|
||||
request: {
|
||||
signal
|
||||
}
|
||||
});
|
||||
|
||||
const duration = Date.now() - start;
|
||||
logger.debug(`Found info for repository ${repo} in ${duration}ms`);
|
||||
|
||||
return result.data;
|
||||
}));
|
||||
|
||||
logger.debug(`Found info for repository ${repo} in ${durationMs}ms`);
|
||||
|
||||
return [result.data];
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch repository info for ${repo}.`, e);
|
||||
return [];
|
||||
}
|
||||
}))).flat();
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
|
@ -26,12 +26,16 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
|
|||
|
||||
if (config.all === true) {
|
||||
if (hostname !== GITLAB_CLOUD_HOSTNAME) {
|
||||
try {
|
||||
logger.debug(`Fetching all projects visible in ${config.url}...`);
|
||||
const { durationMs, data: _projects } = await measure(() => api.Projects.all({
|
||||
perPage: 100,
|
||||
}));
|
||||
logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`);
|
||||
allProjects = allProjects.concat(_projects);
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch all projects visible in ${config.url}.`, e);
|
||||
}
|
||||
} else {
|
||||
logger.warn(`Ignoring option all:true in ${ctx.configPath} : host is ${GITLAB_CLOUD_HOSTNAME}`);
|
||||
}
|
||||
|
|
@ -39,14 +43,18 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
|
|||
|
||||
if (config.groups) {
|
||||
const _projects = (await Promise.all(config.groups.map(async (group) => {
|
||||
try {
|
||||
logger.debug(`Fetching project info for group ${group}...`);
|
||||
const { durationMs, data } = await measure(() => api.Groups.allProjects(group, {
|
||||
perPage: 100,
|
||||
includeSubgroups: true
|
||||
}));
|
||||
logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`);
|
||||
|
||||
return data;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch project info for group ${group}.`, e);
|
||||
return [];
|
||||
}
|
||||
}))).flat();
|
||||
|
||||
allProjects = allProjects.concat(_projects);
|
||||
|
|
@ -54,24 +62,34 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
|
|||
|
||||
if (config.users) {
|
||||
const _projects = (await Promise.all(config.users.map(async (user) => {
|
||||
try {
|
||||
logger.debug(`Fetching project info for user ${user}...`);
|
||||
const { durationMs, data } = await measure(() => api.Users.allProjects(user, {
|
||||
perPage: 100,
|
||||
}));
|
||||
logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`);
|
||||
return data;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch project info for user ${user}.`, e);
|
||||
return [];
|
||||
}
|
||||
}))).flat();
|
||||
|
||||
allProjects = allProjects.concat(_projects);
|
||||
}
|
||||
|
||||
if (config.projects) {
|
||||
const _projects = await Promise.all(config.projects.map(async (project) => {
|
||||
const _projects = (await Promise.all(config.projects.map(async (project) => {
|
||||
try {
|
||||
logger.debug(`Fetching project info for project ${project}...`);
|
||||
const { durationMs, data } = await measure(() => api.Projects.show(project));
|
||||
logger.debug(`Found project ${project} in ${durationMs}ms.`);
|
||||
return data;
|
||||
}));
|
||||
return [data];
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch project info for project ${project}.`, e);
|
||||
return [];
|
||||
}
|
||||
}))).flat();
|
||||
|
||||
allProjects = allProjects.concat(_projects);
|
||||
}
|
||||
|
|
@ -144,6 +162,7 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
|
|||
if (config.revisions.branches) {
|
||||
const branchGlobs = config.revisions.branches;
|
||||
repos = await Promise.all(repos.map(async (repo) => {
|
||||
try {
|
||||
logger.debug(`Fetching branches for repo ${repo.name}...`);
|
||||
let { durationMs, data } = await measure(() => api.Branches.all(repo.name));
|
||||
logger.debug(`Found ${data.length} branches in repo ${repo.name} in ${durationMs}ms.`);
|
||||
|
|
@ -155,12 +174,17 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
|
|||
...repo,
|
||||
branches,
|
||||
};
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch branches for repo ${repo.name}.`, e);
|
||||
return repo;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
if (config.revisions.tags) {
|
||||
const tagGlobs = config.revisions.tags;
|
||||
repos = await Promise.all(repos.map(async (repo) => {
|
||||
try {
|
||||
logger.debug(`Fetching tags for repo ${repo.name}...`);
|
||||
let { durationMs, data } = await measure(() => api.Tags.all(repo.name));
|
||||
logger.debug(`Found ${data.length} tags in repo ${repo.name} in ${durationMs}ms.`);
|
||||
|
|
@ -172,6 +196,10 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
|
|||
...repo,
|
||||
tags,
|
||||
};
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch tags for repo ${repo.name}.`, e);
|
||||
return repo;
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ const { combine, colorize, timestamp, prettyPrint, errors, printf, label: labelF
|
|||
|
||||
const createLogger = (label: string) => {
|
||||
return winston.createLogger({
|
||||
// @todo: Make log level configurable
|
||||
level: SOURCEBOT_LOG_LEVEL,
|
||||
format: combine(
|
||||
errors({ stack: true }),
|
||||
|
|
|
|||
|
|
@ -132,6 +132,11 @@ export const deleteStaleRepository = async (repo: Repository, db: Database, ctx:
|
|||
});
|
||||
|
||||
logger.info(`Deleted stale repository ${repo.id}`);
|
||||
|
||||
captureEvent('repo_deleted', {
|
||||
vcs: repo.vcs,
|
||||
codeHost: repo.codeHost,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ export type PosthogEventMap = {
|
|||
fetchDuration_s?: number;
|
||||
cloneDuration_s?: number;
|
||||
indexDuration_s?: number;
|
||||
},
|
||||
repo_deleted: {
|
||||
vcs: string;
|
||||
codeHost?: string;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
21
yarn.lock
21
yarn.lock
|
|
@ -5527,16 +5527,8 @@ string-argv@^0.3.1:
|
|||
resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6"
|
||||
integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==
|
||||
|
||||
"string-width-cjs@npm:string-width@^4.2.0":
|
||||
version "4.2.3"
|
||||
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
|
||||
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
|
||||
dependencies:
|
||||
emoji-regex "^8.0.0"
|
||||
is-fullwidth-code-point "^3.0.0"
|
||||
strip-ansi "^6.0.1"
|
||||
|
||||
string-width@^4.1.0:
|
||||
"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0:
|
||||
name string-width-cjs
|
||||
version "4.2.3"
|
||||
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
|
||||
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
|
||||
|
|
@ -5633,14 +5625,7 @@ string_decoder@^1.1.1, string_decoder@^1.3.0:
|
|||
dependencies:
|
||||
safe-buffer "~5.2.0"
|
||||
|
||||
"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
|
||||
version "6.0.1"
|
||||
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
|
||||
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
|
||||
dependencies:
|
||||
ansi-regex "^5.0.1"
|
||||
|
||||
strip-ansi@^6.0.0, strip-ansi@^6.0.1:
|
||||
"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
|
||||
version "6.0.1"
|
||||
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
|
||||
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
|
||||
|
|
|
|||
Loading…
Reference in a new issue