Graceful error handling when calling code host apis (#142)

This commit is contained in:
Brendan Kellam 2024-12-18 19:21:21 -08:00 committed by GitHub
parent 4e68dc5032
commit 03aa608e21
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 261 additions and 187 deletions

View file

@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added config option `settings.reindexInterval` and `settings.resyncInterval` to control how often the index should be re-indexed and re-synced. ([#134](https://github.com/sourcebot-dev/sourcebot/pull/134))
- Added `exclude.size` to the GitHub config to allow excluding repositories by size. ([#137](https://github.com/sourcebot-dev/sourcebot/pull/137))
### Fixed
- Fixed issue where config synchronization was failing entirely when a single api call fails. ([#142](https://github.com/sourcebot-dev/sourcebot/pull/142))
## [2.6.2] - 2024-12-13
### Added

View file

@ -28,9 +28,18 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
const hostname = new URL(config.url).hostname;
const { durationMs, data: projects } = await measure(() =>
fetchAllProjects(url)
);
const { durationMs, data: projects } = await measure(async () => {
try {
return fetchAllProjects(url)
} catch (err) {
logger.error(`Failed to fetch projects from ${url}`, err);
return null;
}
});
if (!projects) {
return [];
}
// exclude "All-Projects" and "All-Users" projects
delete projects['All-Projects'];

View file

@ -122,39 +122,54 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
}
const getTagsForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
const { durationMs, data: tags } = await measure(() =>
paginate((page) => api.repos.repoListTags(owner, repo, {
page
}))
);
logger.debug(`Found ${tags.length} tags in repo ${owner}/${repo} in ${durationMs}ms.`);
return tags;
try {
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
const { durationMs, data: tags } = await measure(() =>
paginate((page) => api.repos.repoListTags(owner, repo, {
page
}))
);
logger.debug(`Found ${tags.length} tags in repo ${owner}/${repo} in ${durationMs}ms.`);
return tags;
} catch (e) {
logger.error(`Failed to fetch tags for repo ${owner}/${repo}.`, e);
return [];
}
}
const getBranchesForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
const { durationMs, data: branches } = await measure(() =>
paginate((page) => api.repos.repoListBranches(owner, repo, {
page
}))
);
logger.debug(`Found ${branches.length} branches in repo ${owner}/${repo} in ${durationMs}ms.`);
return branches;
try {
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
const { durationMs, data: branches } = await measure(() =>
paginate((page) => api.repos.repoListBranches(owner, repo, {
page
}))
);
logger.debug(`Found ${branches.length} branches in repo ${owner}/${repo} in ${durationMs}ms.`);
return branches;
} catch (e) {
logger.error(`Failed to fetch branches for repo ${owner}/${repo}.`, e);
return [];
}
}
const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
const repos = (await Promise.all(users.map(async (user) => {
logger.debug(`Fetching repos for user ${user}...`);
try {
logger.debug(`Fetching repos for user ${user}...`);
const { durationMs, data } = await measure(() =>
paginate((page) => api.users.userListRepos(user, {
page,
}))
);
const { durationMs, data } = await measure(() =>
paginate((page) => api.users.userListRepos(user, {
page,
}))
);
logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`);
return data;
logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`);
return data;
} catch (e) {
logger.error(`Failed to fetch repos for user ${user}.`, e);
return [];
}
}))).flat();
return repos;
@ -162,33 +177,43 @@ const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
return (await Promise.all(orgs.map(async (org) => {
logger.debug(`Fetching repos for org ${org}...`);
try {
logger.debug(`Fetching repos for org ${org}...`);
const { durationMs, data } = await measure(() =>
paginate((page) => api.orgs.orgListRepos(org, {
limit: 100,
page,
}))
);
const { durationMs, data } = await measure(() =>
paginate((page) => api.orgs.orgListRepos(org, {
limit: 100,
page,
}))
);
logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`);
return data;
logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`);
return data;
} catch (e) {
logger.error(`Failed to fetch repos for org ${org}.`, e);
return [];
}
}))).flat();
}
const getRepos = async <T>(repos: string[], api: Api<T>) => {
return Promise.all(repos.map(async (repo) => {
logger.debug(`Fetching repository info for ${repo}...`);
return (await Promise.all(repos.map(async (repo) => {
try {
logger.debug(`Fetching repository info for ${repo}...`);
const [owner, repoName] = repo.split('/');
const { durationMs, data: response } = await measure(() =>
api.repos.repoGet(owner, repoName),
);
const [owner, repoName] = repo.split('/');
const { durationMs, data: response } = await measure(() =>
api.repos.repoGet(owner, repoName),
);
logger.debug(`Found repo ${repo} in ${durationMs}ms.`);
logger.debug(`Found repo ${repo} in ${durationMs}ms.`);
return response.data;
}));
return [response.data];
} catch (e) {
logger.error(`Failed to fetch repository info for ${repo}.`, e);
return [];
}
}))).flat();
}
// @see : https://docs.gitea.com/development/api-usage#pagination

View file

@ -201,68 +201,78 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
}
const getTagsForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
try {
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
const { durationMs, data: tags } = await measure(() => octokit.paginate(octokit.repos.listTags, {
owner,
repo,
per_page: 100,
request: {
signal
}
}));
const { durationMs, data: tags } = await measure(() => octokit.paginate(octokit.repos.listTags, {
owner,
repo,
per_page: 100,
request: {
signal
}
}));
logger.debug(`Found ${tags.length} tags for repo ${owner}/${repo} in ${durationMs}ms`);
return tags;
logger.debug(`Found ${tags.length} tags for repo ${owner}/${repo} in ${durationMs}ms`);
return tags;
} catch (e) {
logger.debug(`Error fetching tags for repo ${owner}/${repo}: ${e}`);
return [];
}
}
const getBranchesForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
const { durationMs, data: branches } = await measure(() => octokit.paginate(octokit.repos.listBranches, {
owner,
repo,
per_page: 100,
request: {
signal
}
}));
logger.debug(`Found ${branches.length} branches for repo ${owner}/${repo} in ${durationMs}ms`);
return branches;
try {
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
const { durationMs, data: branches } = await measure(() => octokit.paginate(octokit.repos.listBranches, {
owner,
repo,
per_page: 100,
request: {
signal
}
}));
logger.debug(`Found ${branches.length} branches for repo ${owner}/${repo} in ${durationMs}ms`);
return branches;
} catch (e) {
logger.debug(`Error fetching branches for repo ${owner}/${repo}: ${e}`);
return [];
}
}
const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, octokit: Octokit, signal: AbortSignal) => {
// @todo : error handling
const repos = (await Promise.all(users.map(async (user) => {
logger.debug(`Fetching repository info for user ${user}...`);
const start = Date.now();
try {
logger.debug(`Fetching repository info for user ${user}...`);
const result = await (() => {
if (isAuthenticated) {
return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
username: user,
visibility: 'all',
affiliation: 'owner',
per_page: 100,
request: {
signal,
},
});
} else {
return octokit.paginate(octokit.repos.listForUser, {
username: user,
per_page: 100,
request: {
signal,
},
});
}
})();
const { durationMs, data } = await measure(async () => {
if (isAuthenticated) {
return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
username: user,
visibility: 'all',
affiliation: 'owner',
per_page: 100,
request: {
signal,
},
});
} else {
return octokit.paginate(octokit.repos.listForUser, {
username: user,
per_page: 100,
request: {
signal,
},
});
}
});
const duration = Date.now() - start;
logger.debug(`Found ${result.length} owned by user ${user} in ${duration}ms.`);
return result;
logger.debug(`Found ${data.length} owned by user ${user} in ${durationMs}ms.`);
return data;
} catch (e) {
logger.error(`Failed to fetch repository info for user ${user}.`, e);
return [];
}
}))).flat();
return repos;
@ -270,45 +280,50 @@ const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, o
const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal) => {
const repos = (await Promise.all(orgs.map(async (org) => {
logger.debug(`Fetching repository info for org ${org}...`);
const start = Date.now();
try {
logger.debug(`Fetching repository info for org ${org}...`);
const result = await octokit.paginate(octokit.repos.listForOrg, {
org: org,
per_page: 100,
request: {
signal
}
});
const { durationMs, data } = await measure(() => octokit.paginate(octokit.repos.listForOrg, {
org: org,
per_page: 100,
request: {
signal
}
}));
const duration = Date.now() - start;
logger.debug(`Found ${result.length} in org ${org} in ${duration}ms.`);
return result;
logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`);
return data;
} catch (e) {
logger.error(`Failed to fetch repository info for org ${org}.`, e);
return [];
}
}))).flat();
return repos;
}
const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal) => {
const repos = await Promise.all(repoList.map(async (repo) => {
logger.debug(`Fetching repository info for ${repo}...`);
const start = Date.now();
const repos = (await Promise.all(repoList.map(async (repo) => {
try {
logger.debug(`Fetching repository info for ${repo}...`);
const [owner, repoName] = repo.split('/');
const result = await octokit.repos.get({
owner,
repo: repoName,
request: {
signal
}
});
const [owner, repoName] = repo.split('/');
const { durationMs, data: result } = await measure(() => octokit.repos.get({
owner,
repo: repoName,
request: {
signal
}
}));
const duration = Date.now() - start;
logger.debug(`Found info for repository ${repo} in ${duration}ms`);
logger.debug(`Found info for repository ${repo} in ${durationMs}ms`);
return result.data;
}));
return [result.data];
} catch (e) {
logger.error(`Failed to fetch repository info for ${repo}.`, e);
return [];
}
}))).flat();
return repos;
}

View file

@ -26,12 +26,16 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
if (config.all === true) {
if (hostname !== GITLAB_CLOUD_HOSTNAME) {
logger.debug(`Fetching all projects visible in ${config.url}...`);
const { durationMs, data: _projects } = await measure(() => api.Projects.all({
perPage: 100,
}));
logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`);
allProjects = allProjects.concat(_projects);
try {
logger.debug(`Fetching all projects visible in ${config.url}...`);
const { durationMs, data: _projects } = await measure(() => api.Projects.all({
perPage: 100,
}));
logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`);
allProjects = allProjects.concat(_projects);
} catch (e) {
logger.error(`Failed to fetch all projects visible in ${config.url}.`, e);
}
} else {
logger.warn(`Ignoring option all:true in ${ctx.configPath} : host is ${GITLAB_CLOUD_HOSTNAME}`);
}
@ -39,14 +43,18 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
if (config.groups) {
const _projects = (await Promise.all(config.groups.map(async (group) => {
logger.debug(`Fetching project info for group ${group}...`);
const { durationMs, data } = await measure(() => api.Groups.allProjects(group, {
perPage: 100,
includeSubgroups: true
}));
logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`);
return data;
try {
logger.debug(`Fetching project info for group ${group}...`);
const { durationMs, data } = await measure(() => api.Groups.allProjects(group, {
perPage: 100,
includeSubgroups: true
}));
logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`);
return data;
} catch (e) {
logger.error(`Failed to fetch project info for group ${group}.`, e);
return [];
}
}))).flat();
allProjects = allProjects.concat(_projects);
@ -54,24 +62,34 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
if (config.users) {
const _projects = (await Promise.all(config.users.map(async (user) => {
logger.debug(`Fetching project info for user ${user}...`);
const { durationMs, data } = await measure(() => api.Users.allProjects(user, {
perPage: 100,
}));
logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`);
return data;
try {
logger.debug(`Fetching project info for user ${user}...`);
const { durationMs, data } = await measure(() => api.Users.allProjects(user, {
perPage: 100,
}));
logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`);
return data;
} catch (e) {
logger.error(`Failed to fetch project info for user ${user}.`, e);
return [];
}
}))).flat();
allProjects = allProjects.concat(_projects);
}
if (config.projects) {
const _projects = await Promise.all(config.projects.map(async (project) => {
logger.debug(`Fetching project info for project ${project}...`);
const { durationMs, data } = await measure(() => api.Projects.show(project));
logger.debug(`Found project ${project} in ${durationMs}ms.`);
return data;
}));
const _projects = (await Promise.all(config.projects.map(async (project) => {
try {
logger.debug(`Fetching project info for project ${project}...`);
const { durationMs, data } = await measure(() => api.Projects.show(project));
logger.debug(`Found project ${project} in ${durationMs}ms.`);
return [data];
} catch (e) {
logger.error(`Failed to fetch project info for project ${project}.`, e);
return [];
}
}))).flat();
allProjects = allProjects.concat(_projects);
}
@ -144,34 +162,44 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
if (config.revisions.branches) {
const branchGlobs = config.revisions.branches;
repos = await Promise.all(repos.map(async (repo) => {
logger.debug(`Fetching branches for repo ${repo.name}...`);
let { durationMs, data } = await measure(() => api.Branches.all(repo.name));
logger.debug(`Found ${data.length} branches in repo ${repo.name} in ${durationMs}ms.`);
try {
logger.debug(`Fetching branches for repo ${repo.name}...`);
let { durationMs, data } = await measure(() => api.Branches.all(repo.name));
logger.debug(`Found ${data.length} branches in repo ${repo.name} in ${durationMs}ms.`);
let branches = data.map((branch) => branch.name);
branches = micromatch.match(branches, branchGlobs);
let branches = data.map((branch) => branch.name);
branches = micromatch.match(branches, branchGlobs);
return {
...repo,
branches,
};
return {
...repo,
branches,
};
} catch (e) {
logger.error(`Failed to fetch branches for repo ${repo.name}.`, e);
return repo;
}
}));
}
if (config.revisions.tags) {
const tagGlobs = config.revisions.tags;
repos = await Promise.all(repos.map(async (repo) => {
logger.debug(`Fetching tags for repo ${repo.name}...`);
let { durationMs, data } = await measure(() => api.Tags.all(repo.name));
logger.debug(`Found ${data.length} tags in repo ${repo.name} in ${durationMs}ms.`);
try {
logger.debug(`Fetching tags for repo ${repo.name}...`);
let { durationMs, data } = await measure(() => api.Tags.all(repo.name));
logger.debug(`Found ${data.length} tags in repo ${repo.name} in ${durationMs}ms.`);
let tags = data.map((tag) => tag.name);
tags = micromatch.match(tags, tagGlobs);
let tags = data.map((tag) => tag.name);
tags = micromatch.match(tags, tagGlobs);
return {
...repo,
tags,
};
return {
...repo,
tags,
};
} catch (e) {
logger.error(`Failed to fetch tags for repo ${repo.name}.`, e);
return repo;
}
}));
}
}

View file

@ -5,7 +5,6 @@ const { combine, colorize, timestamp, prettyPrint, errors, printf, label: labelF
const createLogger = (label: string) => {
return winston.createLogger({
// @todo: Make log level configurable
level: SOURCEBOT_LOG_LEVEL,
format: combine(
errors({ stack: true }),

View file

@ -132,6 +132,11 @@ export const deleteStaleRepository = async (repo: Repository, db: Database, ctx:
});
logger.info(`Deleted stale repository ${repo.id}`);
captureEvent('repo_deleted', {
vcs: repo.vcs,
codeHost: repo.codeHost,
})
}
/**

View file

@ -11,6 +11,10 @@ export type PosthogEventMap = {
fetchDuration_s?: number;
cloneDuration_s?: number;
indexDuration_s?: number;
},
repo_deleted: {
vcs: string;
codeHost?: string;
}
}

View file

@ -5527,16 +5527,8 @@ string-argv@^0.3.1:
resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6"
integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==
"string-width-cjs@npm:string-width@^4.2.0":
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
dependencies:
emoji-regex "^8.0.0"
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.1"
string-width@^4.1.0:
"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0:
name string-width-cjs
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@ -5633,14 +5625,7 @@ string_decoder@^1.1.1, string_decoder@^1.3.0:
dependencies:
safe-buffer "~5.2.0"
"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
dependencies:
ansi-regex "^5.0.1"
strip-ansi@^6.0.0, strip-ansi@^6.0.1:
"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==