ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-12 04:15:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

refactor gitlab all

Browse source
This commit is contained in:
Alexander Braverman Masis 2025-09-16 16:29:08 -04:00
parent 384aa9ebe6
commit 3876b98df6
1 changed files with 99 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
packages/backend/src/gitlab.ts
View file

@ -4,7 +4,7 @@ import { createLogger } from "@sourcebot/logger";
import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"
import { measure, fetchWithRetry } from "./utils.js";
import { PrismaClient } from "@sourcebot/db";
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
import { processPromiseResults } from "./connectionUtils.js";
import * as Sentry from "@sentry/node";
import { env } from "./env.js";
import { getTokenFromConfig } from "@sourcebot/crypto";
@ -39,15 +39,27 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
if (config.all === true) {
if (hostname !== GITLAB_CLOUD_HOSTNAME) {
try {
logger.debug(`Fetching all projects visible in ${config.url}...`);
const { durationMs, data: _projects } = await measure(async () => {
const fetchFn = () => api.Projects.all({
perPage: 100,
// Fetch all groups
logger.debug(`Fetching all groups visible in ${config.url}...`);
const { durationMs: groupsDuration, data: _groups } = await measure(async () => {
const fetchFn = () => api.Groups.all({ perPage: 100, allAvailable: true });
return fetchWithRetry(fetchFn, `all groups in ${config.url}`, logger);
});
return fetchWithRetry(fetchFn, `all projects in ${config.url}`, logger);
logger.debug(`Found ${_groups.length} groups in ${groupsDuration}ms.`);
config.groups = _groups.map(g => g.full_path);
logger.debug(`Found these groups: ${config.groups.join('\n')}`);
// Fetch all users - too much for sourcebot/gitlab
logger.debug(`Fetching all users visible in ${config.url}...`);
const { durationMs: usersDuration, data: _users } = await measure(async () => {
const fetchFn = () => api.Users.all({ perPage: 100, withoutProjects: false });
return fetchWithRetry(fetchFn, `all users in ${config.url}`, logger);
});
logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`);
allRepos = allRepos.concat(_projects);
logger.debug(`Found ${_users.length} users in ${usersDuration}ms.`);
config.users = _users.map(u => u.username);
} catch (e) {
Sentry.captureException(e);
logger.error(`Failed to fetch all projects visible in ${config.url}.`, e);
@ -61,13 +73,21 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
}
if (config.groups) {
const results = await Promise.allSettled(config.groups.map(async (group) => {
const batchSize = 10;
const allResults = [];
// Process groups in batches of 10
for (let i = 0; i < config.groups.length; i += batchSize) {
const batch = config.groups.slice(i, i + batchSize);
logger.debug(`Processing batch ${i/batchSize + 1} of ${Math.ceil(config.groups.length/batchSize)} (${batch.length} groups)`);
const batchResults = await Promise.allSettled(batch.map(async (group) => {
try {
logger.debug(`Fetching project info for group ${group}...`);
const { durationMs, data } = await measure(async () => {
const fetchFn = () => api.Groups.allProjects(group, {
perPage: 100,
includeSubgroups: true
includeSubgroups: true,
});
return fetchWithRetry(fetchFn, `group ${group}`, logger);
});
@ -83,7 +103,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
const status = e?.cause?.response?.status;
if (status === 404) {
const warning = `Group ${group} not found or no access`;
logger.warn(warning);
logger.error(warning);
return {
type: 'warning' as const,
warning
@ -92,15 +112,25 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
throw e;
}
}));
throwIfAnyFailed(results);
const { validItems: validRepos, warnings } = processPromiseResults(results);
allResults.push(...batchResults);
}
const { validItems: validRepos, warnings } = processPromiseResults(allResults);
allRepos = allRepos.concat(validRepos);
allWarnings = allWarnings.concat(warnings);
logger.debug(`Found ${validRepos.length} valid repositories in groups.`);
logger.debug(`These repositories will be downloaded: ${allRepos.map(repo => repo.path_with_namespace).join('\n')}`);
}
if (config.users) {
const results = await Promise.allSettled(config.users.map(async (user) => {
const batchSize = 10;
const allResults = [];
// Process users in batches of 10
for (let i = 0; i < config.users.length; i += batchSize) {
const batch = config.users.slice(i, i + batchSize);
logger.debug(`Processing batch ${i/batchSize + 1} of ${Math.ceil(config.users.length/batchSize)} (${batch.length} users)`);
const batchResults = await Promise.allSettled(batch.map(async (user) => {
try {
logger.debug(`Fetching project info for user ${user}...`);
const { durationMs, data } = await measure(async () => {
@ -121,7 +151,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
const status = e?.cause?.response?.status;
if (status === 404) {
const warning = `User ${user} not found or no access`;
logger.warn(warning);
logger.error(warning);
return {
type: 'warning' as const,
warning
@ -131,8 +161,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
}
}));
throwIfAnyFailed(results);
const { validItems: validRepos, warnings } = processPromiseResults(results);
allResults.push(...batchResults);
}
const { validItems: validRepos, warnings } = processPromiseResults(allResults);
allRepos = allRepos.concat(validRepos);
allWarnings = allWarnings.concat(warnings);
}
@ -168,7 +199,6 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
}
}));
throwIfAnyFailed(results);
const { validItems: validRepos, warnings } = processPromiseResults(results);
allRepos = allRepos.concat(validRepos);
allWarnings = allWarnings.concat(warnings);