diff --git a/CHANGELOG.md b/CHANGELOG.md index aca40e8f..24e03b1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed "The account is already associated with another user" errors with GitLab oauth provider. [#584](https://github.com/sourcebot-dev/sourcebot/pull/584) - Fixed error when viewing a generic git connection in `/settings/connections`. [#588](https://github.com/sourcebot-dev/sourcebot/pull/588) - Fixed issue with an unbounded `Promise.allSettled(...)` when retrieving details from the GitHub API about a large number of repositories (or orgs or users). [#591](https://github.com/sourcebot-dev/sourcebot/pull/591) +- Fixed resource exhaustion (EAGAIN errors) when syncing generic-git-host connections with thousands of repositories. [#593](https://github.com/sourcebot-dev/sourcebot/pull/593) ## Removed - Removed built-in secret manager. [#592](https://github.com/sourcebot-dev/sourcebot/pull/592) diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 77508226..04a8b3b5 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -20,11 +20,17 @@ import assert from 'assert'; import GitUrlParse from 'git-url-parse'; import { RepoMetadata } from '@sourcebot/shared'; import { SINGLE_TENANT_ORG_ID } from './constants.js'; +import pLimit from 'p-limit'; export type RepoData = WithRequired; const logger = createLogger('repo-compile-utils'); +// Limit concurrent git operations to prevent resource exhaustion (EAGAIN errors) +// when processing thousands of repositories simultaneously +const MAX_CONCURRENT_GIT_OPERATIONS = 100; +const gitOperationLimit = pLimit(MAX_CONCURRENT_GIT_OPERATIONS); + type CompileResult = { repoData: RepoData[], warnings: string[], @@ -472,7 +478,7 @@ export const compileGenericGitHostConfig_file = async ( const repos: RepoData[] = []; const warnings: string[] = []; - await Promise.all(repoPaths.map(async (repoPath) => { + await Promise.all(repoPaths.map((repoPath) => gitOperationLimit(async () => { const isGitRepo = await isPathAValidGitRepoRoot({ path: repoPath, }); @@ -526,7 +532,7 @@ export const compileGenericGitHostConfig_file = async ( } repos.push(repo); - })); + }))); return { repoData: repos,