ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-12 04:15:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Refactor user permission syncing to happen on the Account level

Browse source
This commit is contained in:
bkellam 2025-11-04 19:04:22 -08:00
parent 26ec7af7f0
commit f53ca10867
7 changed files with 218 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

105
packages/backend/src/ee/userPermissionSyncer.ts → packages/backend/src/ee/accountPermissionSyncer.ts
View file

@ -1,5 +1,5 @@
import * as Sentry from "@sentry/node";
import { PrismaClient, User, UserPermissionSyncJobStatus } from "@sourcebot/db";
import { PrismaClient, AccountPermissionSyncJobStatus, Account } from "@sourcebot/db";
import { createLogger } from "@sourcebot/logger";
import { Job, Queue, Worker } from "bullmq";
import { Redis } from "ioredis";
@ -14,16 +14,15 @@ const LOG_TAG = 'user-permission-syncer';
const logger = createLogger(LOG_TAG);
const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`);
const QUEUE_NAME = 'userPermissionSyncQueue';
const QUEUE_NAME = 'accountPermissionSyncQueue';
type UserPermissionSyncJob = {
type AccountPermissionSyncJob = {
jobId: string;
}
export class UserPermissionSyncer {
private queue: Queue<UserPermissionSyncJob>;
private worker: Worker<UserPermissionSyncJob>;
export class AccountPermissionSyncer {
private queue: Queue<AccountPermissionSyncJob>;
private worker: Worker<AccountPermissionSyncJob>;
private interval?: NodeJS.Timeout;
constructor(
@ -31,10 +30,10 @@ export class UserPermissionSyncer {
private settings: Settings,
redis: Redis,
) {
this.queue = new Queue<UserPermissionSyncJob>(QUEUE_NAME, {
this.queue = new Queue<AccountPermissionSyncJob>(QUEUE_NAME, {
connection: redis,
});
this.worker = new Worker<UserPermissionSyncJob>(QUEUE_NAME, this.runJob.bind(this), {
this.worker = new Worker<AccountPermissionSyncJob>(QUEUE_NAME, this.runJob.bind(this), {
connection: redis,
concurrency: 1,
});
@ -52,17 +51,13 @@ export class UserPermissionSyncer {
this.interval = setInterval(async () => {
const thresholdDate = new Date(Date.now() - this.settings.experiment_userDrivenPermissionSyncIntervalMs);
const users = await this.db.user.findMany({
const accounts = await this.db.account.findMany({
where: {
AND: [
{
accounts: {
some: {
provider: {
in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES
}
}
}
},
{
OR: [
@ -79,15 +74,15 @@ export class UserPermissionSyncer {
{
status: {
in: [
UserPermissionSyncJobStatus.PENDING,
UserPermissionSyncJobStatus.IN_PROGRESS,
AccountPermissionSyncJobStatus.PENDING,
AccountPermissionSyncJobStatus.IN_PROGRESS,
],
}
},
// Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition.
{
AND: [
{ status: UserPermissionSyncJobStatus.FAILED },
{ status: AccountPermissionSyncJobStatus.FAILED },
{ completedAt: { gt: thresholdDate } },
]
}
@ -100,7 +95,7 @@ export class UserPermissionSyncer {
}
});
await this.schedulePermissionSync(users);
await this.schedulePermissionSync(accounts);
}, 1000 * 5);
}
@ -112,18 +107,18 @@ export class UserPermissionSyncer {
await this.queue.close();
}
private async schedulePermissionSync(users: User[]) {
private async schedulePermissionSync(accounts: Account[]) {
// @note: we don't perform this in a transaction because
// we want to avoid the situation where a job is created and run
// prior to the transaction being committed.
const jobs = await this.db.userPermissionSyncJob.createManyAndReturn({
data: users.map(user => ({
userId: user.id,
const jobs = await this.db.accountPermissionSyncJob.createManyAndReturn({
data: accounts.map(account => ({
accountId: account.id,
})),
});
await this.queue.addBulk(jobs.map((job) => ({
name: 'userPermissionSyncJob',
name: 'accountPermissionSyncJob',
data: {
jobId: job.id,
},
@ -134,40 +129,35 @@ export class UserPermissionSyncer {
})))
}
private async runJob(job: Job<UserPermissionSyncJob>) {
private async runJob(job: Job<AccountPermissionSyncJob>) {
const id = job.data.jobId;
const logger = createJobLogger(id);
const { user } = await this.db.userPermissionSyncJob.update({
const { account } = await this.db.accountPermissionSyncJob.update({
where: {
id,
},
data: {
status: UserPermissionSyncJobStatus.IN_PROGRESS,
status: AccountPermissionSyncJobStatus.IN_PROGRESS,
},
select: {
user: {
account: {
include: {
accounts: true,
user: true,
}
}
}
});
if (!user) {
throw new Error(`User ${id} not found`);
}
logger.info(`Syncing permissions for user ${user.email}...`);
logger.info(`Syncing permissions for ${account.provider} account (id: ${account.id}) for user ${account.user.email}...`);
// Get a list of all repos that the user has access to from all connected accounts.
const repoIds = await (async () => {
const aggregatedRepoIds: Set<number> = new Set();
for (const account of user.accounts) {
if (account.provider === 'github') {
if (!account.access_token) {
throw new Error(`User '${user.email}' does not have an GitHub OAuth access token associated with their GitHub account.`);
throw new Error(`User '${account.user.email}' does not have an GitHub OAuth access token associated with their GitHub account.`);
}
const { octokit } = await createOctokitFromToken({
@ -192,7 +182,7 @@ export class UserPermissionSyncer {
repos.forEach(repo => aggregatedRepoIds.add(repo.id));
} else if (account.provider === 'gitlab') {
if (!account.access_token) {
throw new Error(`User '${user.email}' does not have a GitLab OAuth access token associated with their GitLab account.`);
throw new Error(`User '${account.user.email}' does not have a GitLab OAuth access token associated with their GitLab account.`);
}
const api = await createGitLabFromOAuthToken({
@ -222,15 +212,14 @@ export class UserPermissionSyncer {
repos.forEach(repo => aggregatedRepoIds.add(repo.id));
}
}
return Array.from(aggregatedRepoIds);
})();
await this.db.$transaction([
this.db.user.update({
this.db.account.update({
where: {
id: user.id,
id: account.id,
},
data: {
accessibleRepos: {
@ -238,9 +227,9 @@ export class UserPermissionSyncer {
}
}
}),
this.db.userToRepoPermission.createMany({
this.db.accountToRepoPermission.createMany({
data: repoIds.map(repoId => ({
userId: user.id,
accountId: account.id,
repoId,
})),
skipDuplicates: true,
@ -248,31 +237,35 @@ export class UserPermissionSyncer {
]);
}
private async onJobCompleted(job: Job<UserPermissionSyncJob>) {
private async onJobCompleted(job: Job<AccountPermissionSyncJob>) {
const logger = createJobLogger(job.data.jobId);
const { user } = await this.db.userPermissionSyncJob.update({
const { account } = await this.db.accountPermissionSyncJob.update({
where: {
id: job.data.jobId,
},
data: {
status: UserPermissionSyncJobStatus.COMPLETED,
user: {
status: AccountPermissionSyncJobStatus.COMPLETED,
account: {
update: {
permissionSyncedAt: new Date(),
}
},
},
completedAt: new Date(),
},
select: {
user: true
account: {
include: {
user: true,
}
}
}
});
logger.info(`Permissions synced for user ${user.email}`);
logger.info(`Permissions synced for ${account.provider} account (id: ${account.id}) for user ${account.user.email}`);
}
private async onJobFailed(job: Job<UserPermissionSyncJob> | undefined, err: Error) {
private async onJobFailed(job: Job<AccountPermissionSyncJob> | undefined, err: Error) {
const logger = createJobLogger(job?.data.jobId ?? 'unknown');
Sentry.captureException(err, {
@ -282,26 +275,30 @@ export class UserPermissionSyncer {
}
});
const errorMessage = (email: string) => `User permission sync job failed for user ${email}: ${err.message}`;
const errorMessage = (accountId: string, email: string) => `Account permission sync job failed for account (id: ${accountId}) for user ${email}: ${err.message}`;
if (job) {
const { user } = await this.db.userPermissionSyncJob.update({
const { account } = await this.db.accountPermissionSyncJob.update({
where: {
id: job.data.jobId,
},
data: {
status: UserPermissionSyncJobStatus.FAILED,
status: AccountPermissionSyncJobStatus.FAILED,
completedAt: new Date(),
errorMessage: err.message,
},
select: {
account: {
include: {
user: true,
}
}
}
});
logger.error(errorMessage(user.email ?? user.id));
logger.error(errorMessage(account.id, account.user.email ?? 'unknown user (email not found)'));
} else {
logger.error(errorMessage('unknown job (id not found)'));
logger.error(errorMessage('unknown account (id not found)', 'unknown user (id not found)'));
}
}
}

20
packages/backend/src/ee/repoPermissionSyncer.ts
View file

@ -168,7 +168,7 @@ export class RepoPermissionSyncer {
throw new Error(`No credentials found for repo ${id}`);
}
const userIds = await (async () => {
const accountIds = await (async () => {
if (repo.external_codeHostType === 'github') {
const isGitHubCloud = credentials.hostUrl ? new URL(credentials.hostUrl).hostname === GITHUB_CLOUD_HOSTNAME : false;
const { octokit } = await createOctokitFromToken({
@ -195,12 +195,9 @@ export class RepoPermissionSyncer {
in: githubUserIds,
}
},
select: {
userId: true,
},
});
return accounts.map(account => account.userId);
return accounts.map(account => account.id);
} else if (repo.external_codeHostType === 'gitlab') {
const api = await createGitLabFromPersonalAccessToken({
token: credentials.token,
@ -222,12 +219,9 @@ export class RepoPermissionSyncer {
in: gitlabUserIds,
}
},
select: {
userId: true,
},
});
return accounts.map(account => account.userId);
return accounts.map(account => account.id);
}
return [];
@ -239,14 +233,14 @@ export class RepoPermissionSyncer {
id: repo.id,
},
data: {
permittedUsers: {
permittedAccounts: {
deleteMany: {},
}
}
}),
this.db.userToRepoPermission.createMany({
data: userIds.map(userId => ({
userId,
this.db.accountToRepoPermission.createMany({
data: accountIds.map(accountId => ({
accountId,
repoId: repo.id,
})),
})

8
packages/backend/src/index.ts
View file

@ -11,7 +11,7 @@ import { ConnectionManager } from './connectionManager.js';
import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
import { GithubAppManager } from "./ee/githubAppManager.js";
import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js";
import { env } from "./env.js";
import { PromClient } from './promClient.js';
import { RepoIndexManager } from "./repoIndexManager.js";
@ -52,7 +52,7 @@ if (hasEntitlement('github-app')) {
const connectionManager = new ConnectionManager(prisma, settings, redis, promClient);
const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis);
const userPermissionSyncer = new UserPermissionSyncer(prisma, settings, redis);
const accountPermissionSyncer = new AccountPermissionSyncer(prisma, settings, redis);
const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient);
const configManager = new ConfigManager(prisma, connectionManager, env.CONFIG_PATH);
@ -65,7 +65,7 @@ if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && !hasEntitlement('per
}
else if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && hasEntitlement('permission-syncing')) {
repoPermissionSyncer.startScheduler();
userPermissionSyncer.startScheduler();
accountPermissionSyncer.startScheduler();
}
logger.info('Worker started.');
@ -81,7 +81,7 @@ const cleanup = async (signal: string) => {
repoIndexManager.dispose(),
connectionManager.dispose(),
repoPermissionSyncer.dispose(),
userPermissionSyncer.dispose(),
accountPermissionSyncer.dispose(),
promClient.dispose(),
configManager.dispose(),
]),

65
packages/db/prisma/migrations/20251105021913_move_permission_syncing_to_account_level/migration.sql Normal file
View file

@ -0,0 +1,65 @@
/*
Warnings:
- You are about to drop the column `permissionSyncedAt` on the `User` table. All the data in the column will be lost.
- You are about to drop the `UserPermissionSyncJob` table. If the table is not empty, all the data it contains will be lost.
- You are about to drop the `UserToRepoPermission` table. If the table is not empty, all the data it contains will be lost.
*/
-- CreateEnum
CREATE TYPE "AccountPermissionSyncJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED');
-- DropForeignKey
ALTER TABLE "UserPermissionSyncJob" DROP CONSTRAINT "UserPermissionSyncJob_userId_fkey";
-- DropForeignKey
ALTER TABLE "UserToRepoPermission" DROP CONSTRAINT "UserToRepoPermission_repoId_fkey";
-- DropForeignKey
ALTER TABLE "UserToRepoPermission" DROP CONSTRAINT "UserToRepoPermission_userId_fkey";
-- AlterTable
ALTER TABLE "Account" ADD COLUMN "permissionSyncedAt" TIMESTAMP(3);
-- AlterTable
ALTER TABLE "User" DROP COLUMN "permissionSyncedAt";
-- DropTable
DROP TABLE "UserPermissionSyncJob";
-- DropTable
DROP TABLE "UserToRepoPermission";
-- DropEnum
DROP TYPE "UserPermissionSyncJobStatus";
-- CreateTable
CREATE TABLE "AccountPermissionSyncJob" (
"id" TEXT NOT NULL,
"status" "AccountPermissionSyncJobStatus" NOT NULL DEFAULT 'PENDING',
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
"completedAt" TIMESTAMP(3),
"errorMessage" TEXT,
"accountId" TEXT NOT NULL,
CONSTRAINT "AccountPermissionSyncJob_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "AccountToRepoPermission" (
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"repoId" INTEGER NOT NULL,
"accountId" TEXT NOT NULL,
CONSTRAINT "AccountToRepoPermission_pkey" PRIMARY KEY ("repoId","accountId")
);
-- AddForeignKey
ALTER TABLE "AccountPermissionSyncJob" ADD CONSTRAINT "AccountPermissionSyncJob_accountId_fkey" FOREIGN KEY ("accountId") REFERENCES "Account"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "AccountToRepoPermission" ADD CONSTRAINT "AccountToRepoPermission_repoId_fkey" FOREIGN KEY ("repoId") REFERENCES "Repo"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "AccountToRepoPermission" ADD CONSTRAINT "AccountToRepoPermission_accountId_fkey" FOREIGN KEY ("accountId") REFERENCES "Account"("id") ON DELETE CASCADE ON UPDATE CASCADE;

29
packages/db/prisma/schema.prisma
View file

@ -59,7 +59,7 @@ model Repo {
connections RepoToConnection[]
imageUrl String?
permittedUsers UserToRepoPermission[]
permittedAccounts AccountToRepoPermission[]
permissionSyncJobs RepoPermissionSyncJob[]
permissionSyncedAt DateTime? /// When the permissions were last synced successfully.
@ -349,7 +349,6 @@ model User {
accounts Account[]
orgs UserToOrg[]
accountRequest AccountRequest?
accessibleRepos UserToRepoPermission[]
/// List of pending invites that the user has created
invites Invite[]
@ -361,40 +360,38 @@ model User {
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
permissionSyncJobs UserPermissionSyncJob[]
permissionSyncedAt DateTime?
}
enum UserPermissionSyncJobStatus {
enum AccountPermissionSyncJobStatus {
PENDING
IN_PROGRESS
COMPLETED
FAILED
}
model UserPermissionSyncJob {
model AccountPermissionSyncJob {
id String @id @default(cuid())
status UserPermissionSyncJobStatus @default(PENDING)
status AccountPermissionSyncJobStatus @default(PENDING)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
completedAt DateTime?
errorMessage String?
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
userId String
account Account @relation(fields: [accountId], references: [id], onDelete: Cascade)
accountId String
}
model UserToRepoPermission {
model AccountToRepoPermission {
createdAt DateTime @default(now())
repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade)
repoId Int
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
userId String
account Account @relation(fields: [accountId], references: [id], onDelete: Cascade)
accountId String
@@id([repoId, userId])
@@id([repoId, accountId])
}
// @see : https://authjs.dev/concepts/database-models#account
@ -412,6 +409,12 @@ model Account {
id_token String?
session_state String?
/// List of repos that this account has access to.
accessibleRepos AccountToRepoPermission[]
permissionSyncJobs AccountPermissionSyncJob[]
permissionSyncedAt DateTime?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt

18
packages/web/src/prisma.ts
View file

@ -20,7 +20,7 @@ if (env.NODE_ENV !== "production") globalForPrisma.prisma = prisma
* Creates a prisma client extension that scopes queries to striclty information
* a given user should be able to access.
*/
export const userScopedPrismaClientExtension = (userId?: string) => {
export const userScopedPrismaClientExtension = (accountIds?: string[]) => {
return Prisma.defineExtension(
(prisma) => {
return prisma.$extends({
@ -28,17 +28,21 @@ export const userScopedPrismaClientExtension = (userId?: string) => {
...(env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && hasEntitlement('permission-syncing') ? {
repo: {
async $allOperations({ args, query }) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const argsWithWhere = args as any;
const argsWithWhere = args as Record<string, unknown> & {
where?: Prisma.RepoWhereInput;
}
argsWithWhere.where = {
...(argsWithWhere.where || {}),
OR: [
// Only include repos that are permitted to the user
...(userId ? [
...(accountIds ? [
{
permittedUsers: {
permittedAccounts: {
some: {
userId,
accountId: {
in: accountIds,
}
}
}
},
@ -48,7 +52,7 @@ export const userScopedPrismaClientExtension = (userId?: string) => {
isPublic: true,
}
]
}
};
return query(args);
}

9
packages/web/src/withAuthV2.ts
View file

@ -88,7 +88,8 @@ export const getAuthContext = async (): Promise<OptionalAuthContext | ServiceErr
},
}) : null;
const prisma = __unsafePrisma.$extends(userScopedPrismaClientExtension(user?.id)) as PrismaClient;
const accountIds = user?.accounts.map(account => account.id);
const prisma = __unsafePrisma.$extends(userScopedPrismaClientExtension(accountIds)) as PrismaClient;
return {
user: user ?? undefined,
@ -106,6 +107,9 @@ export const getAuthenticatedUser = async () => {
const user = await __unsafePrisma.user.findUnique({
where: {
id: userId,
},
include: {
accounts: true,
}
});
@ -125,6 +129,9 @@ export const getAuthenticatedUser = async () => {
where: {
id: apiKey.createdById,
},
include: {
accounts: true,
}
});
if (!user) {