From 26a7555f538abd23b37f6ba63301d141cb9b58b1 Mon Sep 17 00:00:00 2001 From: bkellam Date: Wed, 29 Oct 2025 23:26:17 -0700 Subject: [PATCH] gitlab permission syncing wip --- packages/backend/src/constants.ts | 1 + .../backend/src/ee/repoPermissionSyncer.ts | 28 ++++++++ .../backend/src/ee/userPermissionSyncer.ts | 44 +++++++++++- packages/backend/src/env.ts | 1 + packages/backend/src/gitlab.ts | 68 ++++++++++++++++--- packages/backend/src/repoCompileUtils.ts | 1 - packages/web/src/ee/features/sso/sso.ts | 11 ++- 7 files changed, 141 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts index d6db3bec..9ba858de 100644 --- a/packages/backend/src/constants.ts +++ b/packages/backend/src/constants.ts @@ -5,6 +5,7 @@ export const SINGLE_TENANT_ORG_ID = 1; export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [ 'github', + 'gitlab', ]; export const REPOS_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'repos'); diff --git a/packages/backend/src/ee/repoPermissionSyncer.ts b/packages/backend/src/ee/repoPermissionSyncer.ts index 1e7ec815..683e3659 100644 --- a/packages/backend/src/ee/repoPermissionSyncer.ts +++ b/packages/backend/src/ee/repoPermissionSyncer.ts @@ -7,6 +7,7 @@ import { Redis } from 'ioredis'; import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js"; import { env } from "../env.js"; import { createOctokitFromToken, getRepoCollaborators, GITHUB_CLOUD_HOSTNAME } from "../github.js"; +import { createGitLabFromPersonalAccessToken, getProjectMembers } from "../gitlab.js"; import { Settings } from "../types.js"; import { getAuthCredentialsForRepo } from "../utils.js"; @@ -194,6 +195,33 @@ export class RepoPermissionSyncer { }, }); + return accounts.map(account => account.userId); + } else if (repo.external_codeHostType === 'gitlab') { + const api = await createGitLabFromPersonalAccessToken({ + token: credentials.token, + url: credentials.hostUrl, + }); + + const projectId = repo.external_id; + if (!projectId) { + throw new Error(`Repo ${id} does not have an external_id`); + } + + const members = await getProjectMembers(projectId, api); + const gitlabUserIds = members.map(member => member.id.toString()); + + const accounts = await this.db.account.findMany({ + where: { + provider: 'gitlab', + providerAccountId: { + in: gitlabUserIds, + } + }, + select: { + userId: true, + }, + }); + return accounts.map(account => account.userId); } diff --git a/packages/backend/src/ee/userPermissionSyncer.ts b/packages/backend/src/ee/userPermissionSyncer.ts index 6ef77bcf..326ea91c 100644 --- a/packages/backend/src/ee/userPermissionSyncer.ts +++ b/packages/backend/src/ee/userPermissionSyncer.ts @@ -6,10 +6,13 @@ import { Redis } from "ioredis"; import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js"; import { env } from "../env.js"; import { createOctokitFromToken, getReposForAuthenticatedUser } from "../github.js"; +import { createGitLabFromOAuthToken, createGitLabFromPersonalAccessToken, getProjectsForAuthenticatedUser } from "../gitlab.js"; import { hasEntitlement } from "@sourcebot/shared"; import { Settings } from "../types.js"; -const logger = createLogger('user-permission-syncer'); +const LOG_TAG = 'user-permission-syncer'; +const logger = createLogger(LOG_TAG); +const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); const QUEUE_NAME = 'userPermissionSyncQueue'; @@ -132,6 +135,8 @@ export class UserPermissionSyncer { private async runJob(job: Job) { const id = job.data.jobId; + const logger = createJobLogger(id); + const { user } = await this.db.userPermissionSyncJob.update({ where: { id, @@ -183,6 +188,37 @@ export class UserPermissionSyncer { } }); + repos.forEach(repo => aggregatedRepoIds.add(repo.id)); + } else if (account.provider === 'gitlab') { + if (!account.access_token) { + throw new Error(`User '${user.email}' does not have a GitLab OAuth access token associated with their GitLab account.`); + } + + const api = await createGitLabFromOAuthToken({ + oauthToken: account.access_token, + url: env.AUTH_EE_GITLAB_BASE_URL, + }); + + // @note: we only care about the private and internal repos since we don't need to build a mapping + // for public repos. + // @see: packages/web/src/prisma.ts + const privateGitLabProjects = await getProjectsForAuthenticatedUser('private', api); + const internalGitLabProjects = await getProjectsForAuthenticatedUser('internal', api); + + const gitLabProjectIds = [ + ...privateGitLabProjects, + ...internalGitLabProjects, + ].map(project => project.id.toString()); + + const repos = await this.db.repo.findMany({ + where: { + external_codeHostType: 'gitlab', + external_id: { + in: gitLabProjectIds, + } + } + }); + repos.forEach(repo => aggregatedRepoIds.add(repo.id)); } } @@ -212,6 +248,8 @@ export class UserPermissionSyncer { } private async onJobCompleted(job: Job) { + const logger = createJobLogger(job.data.jobId); + const { user } = await this.db.userPermissionSyncJob.update({ where: { id: job.data.jobId, @@ -234,6 +272,8 @@ export class UserPermissionSyncer { } private async onJobFailed(job: Job | undefined, err: Error) { + const logger = createJobLogger(job?.data.jobId ?? 'unknown'); + Sentry.captureException(err, { tags: { jobId: job?.data.jobId, @@ -260,7 +300,7 @@ export class UserPermissionSyncer { logger.error(errorMessage(user.email ?? user.id)); } else { - logger.error(errorMessage('unknown user (id not found)')); + logger.error(errorMessage('unknown job (id not found)')); } } } \ No newline at end of file diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 841caf98..c3ea3679 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -56,6 +56,7 @@ export const env = createEnv({ EXPERIMENT_EE_PERMISSION_SYNC_ENABLED: booleanSchema.default('false'), AUTH_EE_GITHUB_BASE_URL: z.string().optional(), + AUTH_EE_GITLAB_BASE_URL: z.string().default("https://gitlab.com"), }, runtimeEnv: process.env, emptyStringAsUndefined: true, diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index e4954b34..ebef3313 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -12,6 +12,28 @@ import { getTokenFromConfig } from "@sourcebot/crypto"; const logger = createLogger('gitlab'); export const GITLAB_CLOUD_HOSTNAME = "gitlab.com"; +export const createGitLabFromPersonalAccessToken = async ({ token, url }: { token?: string, url?: string }) => { + const isGitLabCloud = url ? new URL(url).hostname === GITLAB_CLOUD_HOSTNAME : false; + return new Gitlab({ + token, + ...(isGitLabCloud ? {} : { + host: url, + }), + queryTimeout: env.GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS * 1000, + }); +} + +export const createGitLabFromOAuthToken = async ({ oauthToken, url }: { oauthToken?: string, url?: string }) => { + const isGitLabCloud = url ? new URL(url).hostname === GITLAB_CLOUD_HOSTNAME : false; + return new Gitlab({ + oauthToken, + ...(isGitLabCloud ? {} : { + host: url, + }), + queryTimeout: env.GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS * 1000, + }); +} + export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => { const hostname = config.url ? new URL(config.url).hostname : @@ -22,15 +44,10 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o hostname === GITLAB_CLOUD_HOSTNAME ? env.FALLBACK_GITLAB_CLOUD_TOKEN : undefined; - - const api = new Gitlab({ - ...(token ? { - token, - } : {}), - ...(config.url ? { - host: config.url, - } : {}), - queryTimeout: env.GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS * 1000, + + const api = await createGitLabFromPersonalAccessToken({ + token, + url: config.url, }); let allRepos: ProjectSchema[] = []; @@ -261,4 +278,37 @@ export const shouldExcludeProject = ({ } return false; +} + +export const getProjectMembers = async (projectId: string, api: InstanceType) => { + try { + const fetchFn = () => api.ProjectMembers.all(projectId, { + perPage: 100, + includeInherited: true, + }); + + const members = await fetchWithRetry(fetchFn, `project ${projectId}`, logger); + return members as Array<{ id: number }>; + } catch (error) { + Sentry.captureException(error); + logger.error(`Failed to fetch members for project ${projectId}.`, error); + throw error; + } +} + +export const getProjectsForAuthenticatedUser = async (visibility: 'private' | 'internal' | 'public' | 'all' = 'all', api: InstanceType) => { + try { + const fetchFn = () => api.Projects.all({ + membership: true, + ...(visibility !== 'all' ? { + visibility, + } : {}), + perPage: 100, + }); + return fetchWithRetry(fetchFn, `authenticated user`, logger) as Promise; + } catch (error) { + Sentry.captureException(error); + logger.error(`Failed to fetch projects for authenticated user.`, error); + throw error; + } } \ No newline at end of file diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index d78455e0..8e8b1f26 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -121,7 +121,6 @@ export const compileGitlabConfig = async ( const projectUrl = `${hostUrl}/${project.path_with_namespace}`; const cloneUrl = new URL(project.http_url_to_repo); const isFork = project.forked_from_project !== undefined; - // @todo: we will need to double check whether 'internal' should also be considered public or not. const isPublic = project.visibility === 'public'; const repoDisplayName = project.path_with_namespace; const repoName = path.join(repoNameRoot, repoDisplayName); diff --git a/packages/web/src/ee/features/sso/sso.ts b/packages/web/src/ee/features/sso/sso.ts index 287453d1..7accc065 100644 --- a/packages/web/src/ee/features/sso/sso.ts +++ b/packages/web/src/ee/features/sso/sso.ts @@ -51,7 +51,16 @@ export const getSSOProviders = (): Provider[] => { authorization: { url: `${env.AUTH_EE_GITLAB_BASE_URL}/oauth/authorize`, params: { - scope: "read_user", + scope: [ + "read_user", + // Permission syncing requires the `read_api` scope in order to fetch projects + // for the authenticated user and project members. + // @see: https://docs.gitlab.com/ee/api/projects.html#list-all-projects + ...(env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && hasEntitlement('permission-syncing') ? + ['read_api'] : + [] + ), + ].join(' '), }, }, token: {