import { Gitlab, ProjectSchema } from "@gitbeaker/rest"; import * as Sentry from "@sentry/node"; import { getTokenFromConfig } from "@sourcebot/shared"; import { createLogger } from "@sourcebot/shared"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; import { env } from "@sourcebot/shared"; import micromatch from "micromatch"; import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; import { fetchWithRetry, measure } from "./utils.js"; const logger = createLogger('gitlab'); export const GITLAB_CLOUD_HOSTNAME = "gitlab.com"; export const createGitLabFromPersonalAccessToken = async ({ token, url }: { token?: string, url?: string }) => { const isGitLabCloud = url ? new URL(url).hostname === GITLAB_CLOUD_HOSTNAME : false; return new Gitlab({ token, ...(isGitLabCloud ? {} : { host: url, }), queryTimeout: env.GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS * 1000, }); } export const createGitLabFromOAuthToken = async ({ oauthToken, url }: { oauthToken?: string, url?: string }) => { const isGitLabCloud = url ? new URL(url).hostname === GITLAB_CLOUD_HOSTNAME : false; return new Gitlab({ oauthToken, ...(isGitLabCloud ? {} : { host: url, }), queryTimeout: env.GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS * 1000, }); } export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) => { const hostname = config.url ? new URL(config.url).hostname : GITLAB_CLOUD_HOSTNAME; const token = config.token ? await getTokenFromConfig(config.token) : hostname === GITLAB_CLOUD_HOSTNAME ? env.FALLBACK_GITLAB_CLOUD_TOKEN : undefined; const api = await createGitLabFromPersonalAccessToken({ token, url: config.url, }); let allRepos: ProjectSchema[] = []; let allWarnings: string[] = []; if (config.all === true) { if (hostname !== GITLAB_CLOUD_HOSTNAME) { try { logger.debug(`Fetching all projects visible in ${config.url}...`); const { durationMs, data: _projects } = await measure(async () => { const fetchFn = () => api.Projects.all({ perPage: 100, }); return fetchWithRetry(fetchFn, `all projects in ${config.url}`, logger); }); logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`); allRepos = allRepos.concat(_projects); } catch (e) { Sentry.captureException(e); logger.error(`Failed to fetch all projects visible in ${config.url}.`, e); throw e; } } else { const warning = `Ignoring option all:true in config : host is ${GITLAB_CLOUD_HOSTNAME}`; logger.warn(warning); allWarnings = allWarnings.concat(warning); } } if (config.groups) { const results = await Promise.allSettled(config.groups.map(async (group) => { try { logger.debug(`Fetching project info for group ${group}...`); const { durationMs, data } = await measure(async () => { const fetchFn = () => api.Groups.allProjects(group, { perPage: 100, includeSubgroups: true }); return fetchWithRetry(fetchFn, `group ${group}`, logger); }); logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`); return { type: 'valid' as const, data }; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch projects for group ${group}.`, e); const status = e?.cause?.response?.status; if (status === 404) { const warning = `Group ${group} not found or no access`; logger.warn(warning); return { type: 'warning' as const, warning }; } throw e; } })); throwIfAnyFailed(results); const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); allWarnings = allWarnings.concat(warnings); } if (config.users) { const results = await Promise.allSettled(config.users.map(async (user) => { try { logger.debug(`Fetching project info for user ${user}...`); const { durationMs, data } = await measure(async () => { const fetchFn = () => api.Users.allProjects(user, { perPage: 100, }); return fetchWithRetry(fetchFn, `user ${user}`, logger); }); logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`); return { type: 'valid' as const, data }; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch projects for user ${user}.`, e); const status = e?.cause?.response?.status; if (status === 404) { const warning = `User ${user} not found or no access`; logger.warn(warning); return { type: 'warning' as const, warning }; } throw e; } })); throwIfAnyFailed(results); const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); allWarnings = allWarnings.concat(warnings); } if (config.projects) { const results = await Promise.allSettled(config.projects.map(async (project) => { try { logger.debug(`Fetching project info for project ${project}...`); const { durationMs, data } = await measure(async () => { const fetchFn = () => api.Projects.show(project); return fetchWithRetry(fetchFn, `project ${project}`, logger); }); logger.debug(`Found project ${project} in ${durationMs}ms.`); return { type: 'valid' as const, data: [data] }; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch project ${project}.`, e); const status = e?.cause?.response?.status; if (status === 404) { const warning = `Project ${project} not found or no access`; logger.warn(warning); return { type: 'warning' as const, warning }; } throw e; } })); throwIfAnyFailed(results); const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); allWarnings = allWarnings.concat(warnings); } let repos = allRepos .filter((project) => { const isExcluded = shouldExcludeProject({ project, include: { topics: config.topics, }, exclude: config.exclude }); return !isExcluded; }); logger.debug(`Found ${repos.length} total repositories.`); return { repos, warnings: allWarnings, }; } export const shouldExcludeProject = ({ project, include, exclude, }: { project: ProjectSchema, include?: { topics?: GitlabConnectionConfig['topics'], }, exclude?: GitlabConnectionConfig['exclude'], }) => { const projectName = project.path_with_namespace; let reason = ''; const shouldExclude = (() => { if (!!exclude?.archived && project.archived) { reason = `\`exclude.archived\` is true`; return true; } if (!!exclude?.forks && project.forked_from_project !== undefined) { reason = `\`exclude.forks\` is true`; return true; } if (exclude?.userOwnedProjects && project.namespace.kind === 'user') { reason = `\`exclude.userOwnedProjects\` is true`; return true; } if (exclude?.projects) { if (micromatch.isMatch(projectName, exclude.projects)) { reason = `\`exclude.projects\` contains ${projectName}`; return true; } } if (include?.topics) { const configTopics = include.topics.map(topic => topic.toLowerCase()); const projectTopics = project.topics ?? []; const matchingTopics = projectTopics.filter((topic) => micromatch.isMatch(topic, configTopics)); if (matchingTopics.length === 0) { reason = `\`include.topics\` does not match any of the following topics: ${configTopics.join(', ')}`; return true; } } if (exclude?.topics) { const configTopics = exclude.topics.map(topic => topic.toLowerCase()); const projectTopics = project.topics ?? []; const matchingTopics = projectTopics.filter((topic) => micromatch.isMatch(topic, configTopics)); if (matchingTopics.length > 0) { reason = `\`exclude.topics\` matches the following topics: ${matchingTopics.join(', ')}`; return true; } } })(); if (shouldExclude) { logger.debug(`Excluding project ${projectName}. Reason: ${reason}`); return true; } return false; } export const getProjectMembers = async (projectId: string, api: InstanceType) => { try { const fetchFn = () => api.ProjectMembers.all(projectId, { perPage: 100, includeInherited: true, }); const members = await fetchWithRetry(fetchFn, `project ${projectId}`, logger); return members as Array<{ id: number }>; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch members for project ${projectId}.`, error); throw error; } } export const getProjectsForAuthenticatedUser = async (visibility: 'private' | 'internal' | 'public' | 'all' = 'all', api: InstanceType) => { try { const fetchFn = () => api.Projects.all({ membership: true, ...(visibility !== 'all' ? { visibility, } : {}), perPage: 100, }); const response = await fetchWithRetry(fetchFn, `authenticated user`, logger); return response; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch projects for authenticated user.`, error); throw error; } } // Fetches OAuth scopes for the authenticated user. // @see: https://github.com/doorkeeper-gem/doorkeeper/wiki/API-endpoint-descriptions-and-examples#get----oauthtokeninfo // @see: https://docs.gitlab.com/api/oauth2/#retrieve-the-token-information export const getOAuthScopesForAuthenticatedUser = async (api: InstanceType) => { try { const response = await api.requester.get('/oauth/token/info'); console.log('response', response); if ( response && typeof response.body === 'object' && response.body !== null && 'scope' in response.body && Array.isArray(response.body.scope) ) { return response.body.scope; } throw new Error('/oauth/token_info response body is not in the expected format.'); } catch (error) { Sentry.captureException(error); logger.error('Failed to fetch OAuth scopes for authenticated user.', error); throw error; } }