From e5c8caadb819154fd1ed9b93615c965e1e2810ab Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 18 Sep 2025 23:56:39 -0700 Subject: [PATCH] user syncing + represent sync job status in a seperate table --- package.json | 1 + packages/backend/src/constants.ts | 4 + packages/backend/src/github.ts | 11 + packages/backend/src/index.ts | 19 +- ...ssionSyncer.ts => repoPermissionSyncer.ts} | 162 +++++++----- packages/backend/src/userPermissionSyncer.ts | 249 ++++++++++++++++++ .../migration.sql | 14 - .../migration.sql | 6 - .../migration.sql | 58 ++++ packages/db/prisma/schema.prisma | 57 +++- packages/web/src/ee/features/sso/sso.tsx | 14 +- 11 files changed, 488 insertions(+), 107 deletions(-) rename packages/backend/src/{permissionSyncer.ts => repoPermissionSyncer.ts} (60%) create mode 100644 packages/backend/src/userPermissionSyncer.ts delete mode 100644 packages/db/prisma/migrations/20250827010055_repo_to_user_join_table/migration.sql delete mode 100644 packages/db/prisma/migrations/20250918030650_add_permission_sync_tracking_to_repo_table/migration.sql create mode 100644 packages/db/prisma/migrations/20250919065312_add_permission_sync_tables/migration.sql diff --git a/package.json b/package.json index 7420579f..c5909e76 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "watch:mcp": "yarn workspace @sourcebot/mcp build:watch", "watch:schemas": "yarn workspace @sourcebot/schemas watch", "dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev", + "dev:prisma:generate": "yarn with-env yarn workspace @sourcebot/db prisma:generate", "dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio", "dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset", "dev:prisma:db:push": "yarn with-env yarn workspace @sourcebot/db prisma:db:push", diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts index c0d77f05..3329f3d8 100644 --- a/packages/backend/src/constants.ts +++ b/packages/backend/src/constants.ts @@ -17,3 +17,7 @@ export const DEFAULT_SETTINGS: Settings = { repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours enablePublicAccess: false // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead } + +export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [ + 'github', +]; \ No newline at end of file diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index 730c891c..a489c55c 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -129,6 +129,17 @@ export const getUserIdsWithReadAccessToRepo = async (owner: string, repo: string return collaborators.map(collaborator => collaborator.id.toString()); } +export const getReposThatAuthenticatedUserHasReadAccessTo = async (octokit: Octokit) => { + const fetchFn = () => octokit.paginate(octokit.repos.listForAuthenticatedUser, { + per_page: 100, + // @todo: do we need to set a visibility to private only? + // visibility: 'private' + }); + + const repos = await fetchWithRetry(fetchFn, `authenticated user`, logger); + return repos.map(repo => repo.id.toString()); +} + export const createOctokitFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient): Promise<{ octokit: Octokit, isAuthenticated: boolean }> => { const hostname = config.url ? new URL(config.url).hostname : diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index f8034034..0210b3ba 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -10,10 +10,11 @@ import path from 'path'; import { ConnectionManager } from './connectionManager.js'; import { DEFAULT_SETTINGS } from './constants.js'; import { env } from "./env.js"; -import { RepoPermissionSyncer } from './permissionSyncer.js'; +import { RepoPermissionSyncer } from './repoPermissionSyncer.js'; import { PromClient } from './promClient.js'; import { RepoManager } from './repoManager.js'; import { AppContext } from "./types.js"; +import { UserPermissionSyncer } from "./userPermissionSyncer.js"; const logger = createLogger('backend-entrypoint'); @@ -68,20 +69,25 @@ const settings = await getSettings(env.CONFIG_PATH); const connectionManager = new ConnectionManager(prisma, settings, redis); const repoManager = new RepoManager(prisma, settings, redis, promClient, context); -const permissionSyncer = new RepoPermissionSyncer(prisma, redis); +const repoPermissionSyncer = new RepoPermissionSyncer(prisma, redis); +const userPermissionSyncer = new UserPermissionSyncer(prisma, redis); await repoManager.validateIndexedReposHaveShards(); const connectionManagerInterval = connectionManager.startScheduler(); const repoManagerInterval = repoManager.startScheduler(); -const permissionSyncerInterval = env.EXPERIMENT_PERMISSION_SYNC_ENABLED === 'true' ? permissionSyncer.startScheduler() : null; +const repoPermissionSyncerInterval = env.EXPERIMENT_PERMISSION_SYNC_ENABLED === 'true' ? repoPermissionSyncer.startScheduler() : null; +const userPermissionSyncerInterval = env.EXPERIMENT_PERMISSION_SYNC_ENABLED === 'true' ? userPermissionSyncer.startScheduler() : null; const cleanup = async (signal: string) => { logger.info(`Recieved ${signal}, cleaning up...`); - if (permissionSyncerInterval) { - clearInterval(permissionSyncerInterval); + if (userPermissionSyncerInterval) { + clearInterval(userPermissionSyncerInterval); + } + if (repoPermissionSyncerInterval) { + clearInterval(repoPermissionSyncerInterval); } clearInterval(connectionManagerInterval); @@ -89,7 +95,8 @@ const cleanup = async (signal: string) => { connectionManager.dispose(); repoManager.dispose(); - permissionSyncer.dispose(); + repoPermissionSyncer.dispose(); + userPermissionSyncer.dispose(); await prisma.$disconnect(); await redis.quit(); diff --git a/packages/backend/src/permissionSyncer.ts b/packages/backend/src/repoPermissionSyncer.ts similarity index 60% rename from packages/backend/src/permissionSyncer.ts rename to packages/backend/src/repoPermissionSyncer.ts index baf85f05..b6d8be8d 100644 --- a/packages/backend/src/permissionSyncer.ts +++ b/packages/backend/src/repoPermissionSyncer.ts @@ -1,5 +1,5 @@ import * as Sentry from "@sentry/node"; -import { PrismaClient, Repo, RepoPermissionSyncStatus } from "@sourcebot/db"; +import { PrismaClient, Repo, RepoPermissionSyncJobStatus } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; @@ -10,16 +10,16 @@ import { Redis } from 'ioredis'; import { env } from "./env.js"; import { createOctokitFromConfig, getUserIdsWithReadAccessToRepo } from "./github.js"; import { RepoWithConnections } from "./types.js"; +import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "./constants.js"; type RepoPermissionSyncJob = { - repoId: number; + jobId: string; } const QUEUE_NAME = 'repoPermissionSyncQueue'; -const logger = createLogger('permission-syncer'); +const logger = createLogger('repo-permission-syncer'); -const SUPPORTED_CODE_HOST_TYPES = ['github']; export class RepoPermissionSyncer { private queue: Queue; @@ -46,6 +46,7 @@ export class RepoPermissionSyncer { return setInterval(async () => { // @todo: make this configurable const thresholdDate = new Date(Date.now() - 1000 * 60 * 60 * 24); + const repos = await this.db.repo.findMany({ // Repos need their permissions to be synced against the code host when... where: { @@ -53,40 +54,47 @@ export class RepoPermissionSyncer { AND: [ { external_codeHostType: { - in: SUPPORTED_CODE_HOST_TYPES, + in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, } }, - // and, they either require a sync (SYNC_NEEDED) or have been in a completed state (SYNCED or FAILED) - // for > some duration (default 24 hours) { OR: [ - { - permissionSyncStatus: RepoPermissionSyncStatus.SYNC_NEEDED - }, - { - AND: [ - { - OR: [ - { permissionSyncStatus: RepoPermissionSyncStatus.SYNCED }, - { permissionSyncStatus: RepoPermissionSyncStatus.FAILED }, - ] - }, - { - OR: [ - { permissionSyncJobLastCompletedAt: null }, - { permissionSyncJobLastCompletedAt: { lt: thresholdDate } } - ] - } - ] + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: thresholdDate } }, + ], + }, + { + NOT: { + permissionSyncJobs: { + some: { + OR: [ + // Don't schedule if there are active jobs + { + status: { + in: [ + RepoPermissionSyncJobStatus.PENDING, + RepoPermissionSyncJobStatus.IN_PROGRESS, + ], + } + }, + // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. + { + AND: [ + { status: RepoPermissionSyncJobStatus.FAILED }, + { completedAt: { gt: thresholdDate } }, + ] + } + ] + } } - ] + } }, ] } }); await this.schedulePermissionSync(repos); - }, 1000 * 30); + }, 1000 * 5); } public dispose() { @@ -96,15 +104,16 @@ export class RepoPermissionSyncer { private async schedulePermissionSync(repos: Repo[]) { await this.db.$transaction(async (tx) => { - await tx.repo.updateMany({ - where: { id: { in: repos.map(repo => repo.id) } }, - data: { permissionSyncStatus: RepoPermissionSyncStatus.IN_SYNC_QUEUE }, + const jobs = await tx.repoPermissionSyncJob.createManyAndReturn({ + data: repos.map(repo => ({ + repoId: repo.id, + })), }); - await this.queue.addBulk(repos.map(repo => ({ + await this.queue.addBulk(jobs.map((job) => ({ name: 'repoPermissionSyncJob', data: { - repoId: repo.id, + jobId: job.id, }, opts: { removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, @@ -115,21 +124,25 @@ export class RepoPermissionSyncer { } private async runJob(job: Job) { - const id = job.data.repoId; - const repo = await this.db.repo.update({ + const id = job.data.jobId; + const { repo } = await this.db.repoPermissionSyncJob.update({ where: { - id + id, }, data: { - permissionSyncStatus: RepoPermissionSyncStatus.SYNCING, + status: RepoPermissionSyncJobStatus.IN_PROGRESS, }, - include: { - connections: { + select: { + repo: { include: { - connection: true, - }, - }, - }, + connections: { + include: { + connection: true, + } + } + } + } + } }); if (!repo) { @@ -171,34 +184,43 @@ export class RepoPermissionSyncer { return []; })(); - await this.db.repo.update({ - where: { - id: repo.id, - }, - data: { - permittedUsers: { - deleteMany: {}, + await this.db.$transaction([ + this.db.repo.update({ + where: { + id: repo.id, + }, + data: { + permittedUsers: { + deleteMany: {}, + } } - } - }); - - await this.db.userToRepoPermission.createMany({ - data: userIds.map(userId => ({ - userId, - repoId: repo.id, - })), - }); + }), + this.db.userToRepoPermission.createMany({ + data: userIds.map(userId => ({ + userId, + repoId: repo.id, + })), + }) + ]); } private async onJobCompleted(job: Job) { - const repo = await this.db.repo.update({ + const { repo } = await this.db.repoPermissionSyncJob.update({ where: { - id: job.data.repoId, + id: job.data.jobId, }, data: { - permissionSyncStatus: RepoPermissionSyncStatus.SYNCED, - permissionSyncJobLastCompletedAt: new Date(), + status: RepoPermissionSyncJobStatus.COMPLETED, + repo: { + update: { + permissionSyncedAt: new Date(), + } + }, + completedAt: new Date(), }, + select: { + repo: true + } }); logger.info(`Permissions synced for repo ${repo.displayName ?? repo.name}`); @@ -207,21 +229,25 @@ export class RepoPermissionSyncer { private async onJobFailed(job: Job | undefined, err: Error) { Sentry.captureException(err, { tags: { - repoId: job?.data.repoId, + jobId: job?.data.jobId, queue: QUEUE_NAME, } }); - const errorMessage = (repoName: string) => `Repo permission sync job failed for repo ${repoName}: ${err}`; + const errorMessage = (repoName: string) => `Repo permission sync job failed for repo ${repoName}: ${err.message}`; if (job) { - const repo = await this.db.repo.update({ + const { repo } = await this.db.repoPermissionSyncJob.update({ where: { - id: job?.data.repoId, + id: job.data.jobId, }, data: { - permissionSyncStatus: RepoPermissionSyncStatus.FAILED, - permissionSyncJobLastCompletedAt: new Date(), + status: RepoPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: err.message, + }, + select: { + repo: true }, }); logger.error(errorMessage(repo.displayName ?? repo.name)); diff --git a/packages/backend/src/userPermissionSyncer.ts b/packages/backend/src/userPermissionSyncer.ts new file mode 100644 index 00000000..872b1f7a --- /dev/null +++ b/packages/backend/src/userPermissionSyncer.ts @@ -0,0 +1,249 @@ +import { Octokit } from "@octokit/rest"; +import * as Sentry from "@sentry/node"; +import { PrismaClient, User, UserPermissionSyncJobStatus } from "@sourcebot/db"; +import { createLogger } from "@sourcebot/logger"; +import { Job, Queue, Worker } from "bullmq"; +import { Redis } from "ioredis"; +import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "./constants.js"; +import { env } from "./env.js"; +import { getReposThatAuthenticatedUserHasReadAccessTo } from "./github.js"; + +const logger = createLogger('user-permission-syncer'); + +const QUEUE_NAME = 'userPermissionSyncQueue'; + +type UserPermissionSyncJob = { + jobId: string; +} + + +export class UserPermissionSyncer { + private queue: Queue; + private worker: Worker; + + constructor( + private db: PrismaClient, + redis: Redis, + ) { + this.queue = new Queue(QUEUE_NAME, { + connection: redis, + }); + this.worker = new Worker(QUEUE_NAME, this.runJob.bind(this), { + connection: redis, + concurrency: 1, + }); + this.worker.on('completed', this.onJobCompleted.bind(this)); + this.worker.on('failed', this.onJobFailed.bind(this)); + } + + public startScheduler() { + logger.debug('Starting scheduler'); + + return setInterval(async () => { + const thresholdDate = new Date(Date.now() - 1000 * 60 * 60 * 24); + + const users = await this.db.user.findMany({ + where: { + AND: [ + { + accounts: { + some: { + provider: { + in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES + } + } + } + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: thresholdDate } }, + ] + }, + { + NOT: { + permissionSyncJobs: { + some: { + OR: [ + // Don't schedule if there are active jobs + { + status: { + in: [ + UserPermissionSyncJobStatus.PENDING, + UserPermissionSyncJobStatus.IN_PROGRESS, + ], + } + }, + // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. + { + AND: [ + { status: UserPermissionSyncJobStatus.FAILED }, + { completedAt: { gt: thresholdDate } }, + ] + } + ] + } + } + } + }, + ] + } + }); + + await this.schedulePermissionSync(users); + }, 1000 * 5); + } + + public dispose() { + this.worker.close(); + this.queue.close(); + } + + private async schedulePermissionSync(users: User[]) { + await this.db.$transaction(async (tx) => { + const jobs = await tx.userPermissionSyncJob.createManyAndReturn({ + data: users.map(user => ({ + userId: user.id, + })), + }); + + await this.queue.addBulk(jobs.map((job) => ({ + name: 'userPermissionSyncJob', + data: { + jobId: job.id, + }, + opts: { + removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, + removeOnFail: env.REDIS_REMOVE_ON_FAIL, + } + }))) + }); + } + + private async runJob(job: Job) { + const id = job.data.jobId; + const { user } = await this.db.userPermissionSyncJob.update({ + where: { + id, + }, + data: { + status: UserPermissionSyncJobStatus.IN_PROGRESS, + }, + select: { + user: { + include: { + accounts: true, + } + } + } + }); + + if (!user) { + throw new Error(`User ${id} not found`); + } + + logger.info(`Syncing permissions for user ${user.email}...`); + + for (const account of user.accounts) { + const repoIds = await (async () => { + if (account.provider === 'github') { + // @todo: we will need to provide some mechanism for the user to provide a custom + // URL here. This will correspond to the host URL they are using for their GitHub + // instance. + const octokit = new Octokit({ + auth: account.access_token, + // baseUrl: /* todo */ + }); + + const repoIds = await getReposThatAuthenticatedUserHasReadAccessTo(octokit); + + const repos = await this.db.repo.findMany({ + where: { + external_codeHostType: 'github', + external_id: { + in: repoIds, + } + } + }); + + return repos.map(repo => repo.id); + } + + return []; + })(); + + + await this.db.$transaction([ + this.db.user.update({ + where: { + id: user.id, + }, + data: { + accessibleRepos: { + deleteMany: {}, + } + } + }), + this.db.userToRepoPermission.createMany({ + data: repoIds.map(repoId => ({ + userId: user.id, + repoId, + })) + }) + ]); + } + } + + private async onJobCompleted(job: Job) { + const { user } = await this.db.userPermissionSyncJob.update({ + where: { + id: job.data.jobId, + }, + data: { + status: UserPermissionSyncJobStatus.COMPLETED, + user: { + update: { + permissionSyncedAt: new Date(), + } + }, + completedAt: new Date(), + }, + select: { + user: true + } + }); + + logger.info(`Permissions synced for user ${user.email}`); + } + + private async onJobFailed(job: Job | undefined, err: Error) { + Sentry.captureException(err, { + tags: { + jobId: job?.data.jobId, + queue: QUEUE_NAME, + } + }); + + const errorMessage = (email: string) => `User permission sync job failed for user ${email}: ${err.message}`; + + if (job) { + const { user } = await this.db.userPermissionSyncJob.update({ + where: { + id: job.data.jobId, + }, + data: { + status: UserPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: err.message, + }, + select: { + user: true, + } + }); + + logger.error(errorMessage(user.email ?? user.id)); + } else { + logger.error(errorMessage('unknown user (id not found)')); + } + } +} \ No newline at end of file diff --git a/packages/db/prisma/migrations/20250827010055_repo_to_user_join_table/migration.sql b/packages/db/prisma/migrations/20250827010055_repo_to_user_join_table/migration.sql deleted file mode 100644 index 62a56d49..00000000 --- a/packages/db/prisma/migrations/20250827010055_repo_to_user_join_table/migration.sql +++ /dev/null @@ -1,14 +0,0 @@ --- CreateTable -CREATE TABLE "UserToRepoPermission" ( - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "repoId" INTEGER NOT NULL, - "userId" TEXT NOT NULL, - - CONSTRAINT "UserToRepoPermission_pkey" PRIMARY KEY ("repoId","userId") -); - --- AddForeignKey -ALTER TABLE "UserToRepoPermission" ADD CONSTRAINT "UserToRepoPermission_repoId_fkey" FOREIGN KEY ("repoId") REFERENCES "Repo"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "UserToRepoPermission" ADD CONSTRAINT "UserToRepoPermission_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20250918030650_add_permission_sync_tracking_to_repo_table/migration.sql b/packages/db/prisma/migrations/20250918030650_add_permission_sync_tracking_to_repo_table/migration.sql deleted file mode 100644 index ffbe376e..00000000 --- a/packages/db/prisma/migrations/20250918030650_add_permission_sync_tracking_to_repo_table/migration.sql +++ /dev/null @@ -1,6 +0,0 @@ --- CreateEnum -CREATE TYPE "RepoPermissionSyncStatus" AS ENUM ('SYNC_NEEDED', 'IN_SYNC_QUEUE', 'SYNCING', 'SYNCED', 'FAILED'); - --- AlterTable -ALTER TABLE "Repo" ADD COLUMN "permissionSyncJobLastCompletedAt" TIMESTAMP(3), -ADD COLUMN "permissionSyncStatus" "RepoPermissionSyncStatus" NOT NULL DEFAULT 'SYNC_NEEDED'; diff --git a/packages/db/prisma/migrations/20250919065312_add_permission_sync_tables/migration.sql b/packages/db/prisma/migrations/20250919065312_add_permission_sync_tables/migration.sql new file mode 100644 index 00000000..1a8f0597 --- /dev/null +++ b/packages/db/prisma/migrations/20250919065312_add_permission_sync_tables/migration.sql @@ -0,0 +1,58 @@ +-- CreateEnum +CREATE TYPE "RepoPermissionSyncJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'); + +-- CreateEnum +CREATE TYPE "UserPermissionSyncJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'); + +-- AlterTable +ALTER TABLE "Repo" ADD COLUMN "permissionSyncedAt" TIMESTAMP(3); + +-- AlterTable +ALTER TABLE "User" ADD COLUMN "permissionSyncedAt" TIMESTAMP(3); + +-- CreateTable +CREATE TABLE "RepoPermissionSyncJob" ( + "id" TEXT NOT NULL, + "status" "RepoPermissionSyncJobStatus" NOT NULL DEFAULT 'PENDING', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "completedAt" TIMESTAMP(3), + "errorMessage" TEXT, + "repoId" INTEGER NOT NULL, + + CONSTRAINT "RepoPermissionSyncJob_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "UserPermissionSyncJob" ( + "id" TEXT NOT NULL, + "status" "UserPermissionSyncJobStatus" NOT NULL DEFAULT 'PENDING', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "completedAt" TIMESTAMP(3), + "errorMessage" TEXT, + "userId" TEXT NOT NULL, + + CONSTRAINT "UserPermissionSyncJob_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "UserToRepoPermission" ( + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "repoId" INTEGER NOT NULL, + "userId" TEXT NOT NULL, + + CONSTRAINT "UserToRepoPermission_pkey" PRIMARY KEY ("repoId","userId") +); + +-- AddForeignKey +ALTER TABLE "RepoPermissionSyncJob" ADD CONSTRAINT "RepoPermissionSyncJob_repoId_fkey" FOREIGN KEY ("repoId") REFERENCES "Repo"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "UserPermissionSyncJob" ADD CONSTRAINT "UserPermissionSyncJob_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "UserToRepoPermission" ADD CONSTRAINT "UserToRepoPermission_repoId_fkey" FOREIGN KEY ("repoId") REFERENCES "Repo"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "UserToRepoPermission" ADD CONSTRAINT "UserToRepoPermission_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index feaaff02..db0ed906 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -30,14 +30,6 @@ enum ConnectionSyncStatus { FAILED } -enum RepoPermissionSyncStatus { - SYNC_NEEDED - IN_SYNC_QUEUE - SYNCING - SYNCED - FAILED -} - enum StripeSubscriptionStatus { ACTIVE INACTIVE @@ -67,9 +59,9 @@ model Repo { repoIndexingStatus RepoIndexingStatus @default(NEW) permittedUsers UserToRepoPermission[] - permissionSyncStatus RepoPermissionSyncStatus @default(SYNC_NEEDED) - /// When the repo permissions were last synced, either successfully or unsuccessfully. - permissionSyncJobLastCompletedAt DateTime? + permissionSyncJobs RepoPermissionSyncJob[] + /// When the permissions were last synced successfully. + permissionSyncedAt DateTime? // The id of the repo in the external service external_id String @@ -87,6 +79,26 @@ model Repo { @@index([orgId]) } +enum RepoPermissionSyncJobStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + +model RepoPermissionSyncJob { + id String @id @default(cuid()) + status RepoPermissionSyncJobStatus @default(PENDING) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + completedAt DateTime? + + errorMessage String? + + repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) + repoId Int +} + model SearchContext { id Int @id @default(autoincrement()) @@ -301,6 +313,29 @@ model User { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + + permissionSyncJobs UserPermissionSyncJob[] + permissionSyncedAt DateTime? +} + +enum UserPermissionSyncJobStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + +model UserPermissionSyncJob { + id String @id @default(cuid()) + status UserPermissionSyncJobStatus @default(PENDING) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + completedAt DateTime? + + errorMessage String? + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + userId String } model UserToRepoPermission { diff --git a/packages/web/src/ee/features/sso/sso.tsx b/packages/web/src/ee/features/sso/sso.tsx index 966f9c79..07332b63 100644 --- a/packages/web/src/ee/features/sso/sso.tsx +++ b/packages/web/src/ee/features/sso/sso.tsx @@ -27,7 +27,17 @@ export const getSSOProviders = (): Provider[] => { authorization: { url: `${baseUrl}/login/oauth/authorize`, params: { - scope: "read:user user:email", + scope: [ + 'read:user', + 'user:email', + // Permission syncing requires the `repo` in order to fetch repositories + // for the authenticated user. + // @see: https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repositories-for-the-authenticated-user + ...(env.EXPERIMENT_PERMISSION_SYNC_ENABLED === 'true' ? + ['repo'] : + [] + ), + ].join(' '), }, }, token: { @@ -103,7 +113,7 @@ export const getSSOProviders = (): Provider[] => { } const oauth2Client = new OAuth2Client(); - + const { pubkeys } = await oauth2Client.getIapPublicKeys(); const ticket = await oauth2Client.verifySignedJwtWithCertsAsync( iapAssertion,