mirror of
https://github.com/sourcebot-dev/sourcebot.git
synced 2025-12-12 04:15:30 +00:00
refactor gitlab all
This commit is contained in:
Alexander Braverman Masis
parent
384aa9ebe6
commit
3876b98df6
1 changed files with 99 additions and 69 deletions
|
|
@ -4,7 +4,7 @@ import { createLogger } from "@sourcebot/logger";
|
||||||
import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"
|
import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"
|
||||||
import { measure, fetchWithRetry } from "./utils.js";
|
import { measure, fetchWithRetry } from "./utils.js";
|
||||||
import { PrismaClient } from "@sourcebot/db";
|
import { PrismaClient } from "@sourcebot/db";
|
||||||
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
|
import { processPromiseResults } from "./connectionUtils.js";
|
||||||
import * as Sentry from "@sentry/node";
|
import * as Sentry from "@sentry/node";
|
||||||
import { env } from "./env.js";
|
import { env } from "./env.js";
|
||||||
import { getTokenFromConfig } from "@sourcebot/crypto";
|
import { getTokenFromConfig } from "@sourcebot/crypto";
|
||||||
|
|
@ -39,15 +39,27 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
|
||||||
if (config.all === true) {
|
if (config.all === true) {
|
||||||
if (hostname !== GITLAB_CLOUD_HOSTNAME) {
|
if (hostname !== GITLAB_CLOUD_HOSTNAME) {
|
||||||
try {
|
try {
|
||||||
logger.debug(`Fetching all projects visible in ${config.url}...`);
|
// Fetch all groups
|
||||||
const { durationMs, data: _projects } = await measure(async () => {
|
logger.debug(`Fetching all groups visible in ${config.url}...`);
|
||||||
const fetchFn = () => api.Projects.all({
|
const { durationMs: groupsDuration, data: _groups } = await measure(async () => {
|
||||||
perPage: 100,
|
const fetchFn = () => api.Groups.all({ perPage: 100, allAvailable: true });
|
||||||
});
|
return fetchWithRetry(fetchFn, `all groups in ${config.url}`, logger);
|
||||||
return fetchWithRetry(fetchFn, `all projects in ${config.url}`, logger);
|
|
||||||
});
|
});
|
||||||
logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`);
|
logger.debug(`Found ${_groups.length} groups in ${groupsDuration}ms.`);
|
||||||
allRepos = allRepos.concat(_projects);
|
|
||||||
|
config.groups = _groups.map(g => g.full_path);
|
||||||
|
|
||||||
|
logger.debug(`Found these groups: ${config.groups.join('\n')}`);
|
||||||
|
|
||||||
|
// Fetch all users - too much for sourcebot/gitlab
|
||||||
|
logger.debug(`Fetching all users visible in ${config.url}...`);
|
||||||
|
const { durationMs: usersDuration, data: _users } = await measure(async () => {
|
||||||
|
const fetchFn = () => api.Users.all({ perPage: 100, withoutProjects: false });
|
||||||
|
return fetchWithRetry(fetchFn, `all users in ${config.url}`, logger);
|
||||||
|
});
|
||||||
|
logger.debug(`Found ${_users.length} users in ${usersDuration}ms.`);
|
||||||
|
|
||||||
|
config.users = _users.map(u => u.username);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
Sentry.captureException(e);
|
Sentry.captureException(e);
|
||||||
logger.error(`Failed to fetch all projects visible in ${config.url}.`, e);
|
logger.error(`Failed to fetch all projects visible in ${config.url}.`, e);
|
||||||
|
|
@ -61,78 +73,97 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.groups) {
|
if (config.groups) {
|
||||||
const results = await Promise.allSettled(config.groups.map(async (group) => {
|
const batchSize = 10;
|
||||||
try {
|
const allResults = [];
|
||||||
logger.debug(`Fetching project info for group ${group}...`);
|
|
||||||
const { durationMs, data } = await measure(async () => {
|
// Process groups in batches of 10
|
||||||
const fetchFn = () => api.Groups.allProjects(group, {
|
for (let i = 0; i < config.groups.length; i += batchSize) {
|
||||||
perPage: 100,
|
const batch = config.groups.slice(i, i + batchSize);
|
||||||
includeSubgroups: true
|
logger.debug(`Processing batch ${i/batchSize + 1} of ${Math.ceil(config.groups.length/batchSize)} (${batch.length} groups)`);
|
||||||
|
|
||||||
|
const batchResults = await Promise.allSettled(batch.map(async (group) => {
|
||||||
|
try {
|
||||||
|
logger.debug(`Fetching project info for group ${group}...`);
|
||||||
|
const { durationMs, data } = await measure(async () => {
|
||||||
|
const fetchFn = () => api.Groups.allProjects(group, {
|
||||||
|
perPage: 100,
|
||||||
|
includeSubgroups: true,
|
||||||
|
});
|
||||||
|
return fetchWithRetry(fetchFn, `group ${group}`, logger);
|
||||||
});
|
});
|
||||||
return fetchWithRetry(fetchFn, `group ${group}`, logger);
|
logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`);
|
||||||
});
|
|
||||||
logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`);
|
|
||||||
return {
|
|
||||||
type: 'valid' as const,
|
|
||||||
data
|
|
||||||
};
|
|
||||||
} catch (e: any) {
|
|
||||||
Sentry.captureException(e);
|
|
||||||
logger.error(`Failed to fetch projects for group ${group}.`, e);
|
|
||||||
|
|
||||||
const status = e?.cause?.response?.status;
|
|
||||||
if (status === 404) {
|
|
||||||
const warning = `Group ${group} not found or no access`;
|
|
||||||
logger.warn(warning);
|
|
||||||
return {
|
return {
|
||||||
type: 'warning' as const,
|
type: 'valid' as const,
|
||||||
warning
|
data
|
||||||
};
|
};
|
||||||
}
|
} catch (e: any) {
|
||||||
throw e;
|
Sentry.captureException(e);
|
||||||
}
|
logger.error(`Failed to fetch projects for group ${group}.`, e);
|
||||||
}));
|
|
||||||
|
|
||||||
throwIfAnyFailed(results);
|
const status = e?.cause?.response?.status;
|
||||||
const { validItems: validRepos, warnings } = processPromiseResults(results);
|
if (status === 404) {
|
||||||
|
const warning = `Group ${group} not found or no access`;
|
||||||
|
logger.error(warning);
|
||||||
|
return {
|
||||||
|
type: 'warning' as const,
|
||||||
|
warning
|
||||||
|
};
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
allResults.push(...batchResults);
|
||||||
|
}
|
||||||
|
const { validItems: validRepos, warnings } = processPromiseResults(allResults);
|
||||||
allRepos = allRepos.concat(validRepos);
|
allRepos = allRepos.concat(validRepos);
|
||||||
allWarnings = allWarnings.concat(warnings);
|
allWarnings = allWarnings.concat(warnings);
|
||||||
|
logger.debug(`Found ${validRepos.length} valid repositories in groups.`);
|
||||||
|
logger.debug(`These repositories will be downloaded: ${allRepos.map(repo => repo.path_with_namespace).join('\n')}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.users) {
|
if (config.users) {
|
||||||
const results = await Promise.allSettled(config.users.map(async (user) => {
|
const batchSize = 10;
|
||||||
try {
|
const allResults = [];
|
||||||
logger.debug(`Fetching project info for user ${user}...`);
|
|
||||||
const { durationMs, data } = await measure(async () => {
|
// Process users in batches of 10
|
||||||
const fetchFn = () => api.Users.allProjects(user, {
|
for (let i = 0; i < config.users.length; i += batchSize) {
|
||||||
perPage: 100,
|
const batch = config.users.slice(i, i + batchSize);
|
||||||
|
logger.debug(`Processing batch ${i/batchSize + 1} of ${Math.ceil(config.users.length/batchSize)} (${batch.length} users)`);
|
||||||
|
|
||||||
|
const batchResults = await Promise.allSettled(batch.map(async (user) => {
|
||||||
|
try {
|
||||||
|
logger.debug(`Fetching project info for user ${user}...`);
|
||||||
|
const { durationMs, data } = await measure(async () => {
|
||||||
|
const fetchFn = () => api.Users.allProjects(user, {
|
||||||
|
perPage: 100,
|
||||||
|
});
|
||||||
|
return fetchWithRetry(fetchFn, `user ${user}`, logger);
|
||||||
});
|
});
|
||||||
return fetchWithRetry(fetchFn, `user ${user}`, logger);
|
logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`);
|
||||||
});
|
|
||||||
logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`);
|
|
||||||
return {
|
|
||||||
type: 'valid' as const,
|
|
||||||
data
|
|
||||||
};
|
|
||||||
} catch (e: any) {
|
|
||||||
Sentry.captureException(e);
|
|
||||||
logger.error(`Failed to fetch projects for user ${user}.`, e);
|
|
||||||
|
|
||||||
const status = e?.cause?.response?.status;
|
|
||||||
if (status === 404) {
|
|
||||||
const warning = `User ${user} not found or no access`;
|
|
||||||
logger.warn(warning);
|
|
||||||
return {
|
return {
|
||||||
type: 'warning' as const,
|
type: 'valid' as const,
|
||||||
warning
|
data
|
||||||
};
|
};
|
||||||
}
|
} catch (e: any) {
|
||||||
throw e;
|
Sentry.captureException(e);
|
||||||
}
|
logger.error(`Failed to fetch projects for user ${user}.`, e);
|
||||||
}));
|
|
||||||
|
|
||||||
throwIfAnyFailed(results);
|
const status = e?.cause?.response?.status;
|
||||||
const { validItems: validRepos, warnings } = processPromiseResults(results);
|
if (status === 404) {
|
||||||
|
const warning = `User ${user} not found or no access`;
|
||||||
|
logger.error(warning);
|
||||||
|
return {
|
||||||
|
type: 'warning' as const,
|
||||||
|
warning
|
||||||
|
};
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
allResults.push(...batchResults);
|
||||||
|
}
|
||||||
|
const { validItems: validRepos, warnings } = processPromiseResults(allResults);
|
||||||
allRepos = allRepos.concat(validRepos);
|
allRepos = allRepos.concat(validRepos);
|
||||||
allWarnings = allWarnings.concat(warnings);
|
allWarnings = allWarnings.concat(warnings);
|
||||||
}
|
}
|
||||||
|
|
@ -168,7 +199,6 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
|
|
||||||
throwIfAnyFailed(results);
|
|
||||||
const { validItems: validRepos, warnings } = processPromiseResults(results);
|
const { validItems: validRepos, warnings } = processPromiseResults(results);
|
||||||
allRepos = allRepos.concat(validRepos);
|
allRepos = allRepos.concat(validRepos);
|
||||||
allWarnings = allWarnings.concat(warnings);
|
allWarnings = allWarnings.concat(warnings);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue