From c912a0b9a927e8c21b2d2aab7254c735840f0fe3 Mon Sep 17 00:00:00 2001 From: bkellam Date: Sat, 18 Oct 2025 14:30:46 -0700 Subject: [PATCH] db migration --- packages/backend/src/repoIndexManager.ts | 48 +++++++++---------- packages/backend/src/utils.ts | 2 +- .../migration.sql | 34 +++++++++++++ packages/db/prisma/schema.prisma | 26 +++------- packages/web/src/actions.ts | 8 ++-- .../components/navigationMenu/index.tsx | 8 ++-- packages/web/src/app/[domain]/repos/page.tsx | 4 +- 7 files changed, 75 insertions(+), 55 deletions(-) create mode 100644 packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts index e296220d..2f4e1031 100644 --- a/packages/backend/src/repoIndexManager.ts +++ b/packages/backend/src/repoIndexManager.ts @@ -1,5 +1,5 @@ import * as Sentry from '@sentry/node'; -import { PrismaClient, Repo, RepoJobStatus, RepoJobType } from "@sourcebot/db"; +import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { createLogger, Logger } from "@sourcebot/logger"; import { existsSync } from 'fs'; import { readdir, rm } from 'fs/promises'; @@ -97,7 +97,7 @@ export class RepoIndexManager { some: { AND: [ { - type: RepoJobType.INDEX, + type: RepoIndexingJobType.INDEX, }, { OR: [ @@ -108,8 +108,8 @@ export class RepoIndexManager { { status: { in: [ - RepoJobStatus.PENDING, - RepoJobStatus.IN_PROGRESS, + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, ] }, }, @@ -123,7 +123,7 @@ export class RepoIndexManager { // Don't schedule if there are recent failed jobs (within the threshold date). { AND: [ - { status: RepoJobStatus.FAILED }, + { status: RepoIndexingJobStatus.FAILED }, { completedAt: { gt: thresholdDate } }, ] } @@ -139,7 +139,7 @@ export class RepoIndexManager { }); if (reposToIndex.length > 0) { - await this.createJobs(reposToIndex, RepoJobType.INDEX); + await this.createJobs(reposToIndex, RepoIndexingJobType.INDEX); } } @@ -161,13 +161,13 @@ export class RepoIndexManager { some: { AND: [ { - type: RepoJobType.CLEANUP, + type: RepoIndexingJobType.CLEANUP, }, { status: { in: [ - RepoJobStatus.PENDING, - RepoJobStatus.IN_PROGRESS, + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, ] }, }, @@ -184,15 +184,15 @@ export class RepoIndexManager { }); if (reposToCleanup.length > 0) { - await this.createJobs(reposToCleanup, RepoJobType.CLEANUP); + await this.createJobs(reposToCleanup, RepoIndexingJobType.CLEANUP); } } - private async createJobs(repos: Repo[], type: RepoJobType) { + private async createJobs(repos: Repo[], type: RepoIndexingJobType) { // @note: we don't perform this in a transaction because // we want to avoid the situation where a job is created and run // prior to the transaction being committed. - const jobs = await this.db.repoJob.createManyAndReturn({ + const jobs = await this.db.repoIndexingJob.createManyAndReturn({ data: repos.map(repo => ({ type, repoId: repo.id, @@ -222,12 +222,12 @@ export class RepoIndexManager { logger.info(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`); - const { repo, type: jobType } = await this.db.repoJob.update({ + const { repo, type: jobType } = await this.db.repoIndexingJob.update({ where: { id, }, data: { - status: RepoJobStatus.IN_PROGRESS, + status: RepoIndexingJobStatus.IN_PROGRESS, }, select: { type: true, @@ -253,9 +253,9 @@ export class RepoIndexManager { process.on('SIGINT', signalHandler); try { - if (jobType === RepoJobType.INDEX) { + if (jobType === RepoIndexingJobType.INDEX) { await this.indexRepository(repo, logger, abortController.signal); - } else if (jobType === RepoJobType.CLEANUP) { + } else if (jobType === RepoIndexingJobType.CLEANUP) { await this.cleanupRepository(repo, logger); } } finally { @@ -370,15 +370,15 @@ export class RepoIndexManager { private onJobCompleted = async (job: Job) => groupmqLifecycleExceptionWrapper('onJobCompleted', logger, async () => { const logger = createJobLogger(job.data.jobId); - const jobData = await this.db.repoJob.update({ + const jobData = await this.db.repoIndexingJob.update({ where: { id: job.data.jobId }, data: { - status: RepoJobStatus.COMPLETED, + status: RepoIndexingJobStatus.COMPLETED, completedAt: new Date(), } }); - if (jobData.type === RepoJobType.INDEX) { + if (jobData.type === RepoIndexingJobType.INDEX) { const repo = await this.db.repo.update({ where: { id: jobData.repoId }, data: { @@ -388,7 +388,7 @@ export class RepoIndexManager { logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); } - else if (jobData.type === RepoJobType.CLEANUP) { + else if (jobData.type === RepoIndexingJobType.CLEANUP) { const repo = await this.db.repo.delete({ where: { id: jobData.repoId }, }); @@ -405,10 +405,10 @@ export class RepoIndexManager { const wasLastAttempt = attempt >= job.opts.attempts; if (wasLastAttempt) { - const { repo } = await this.db.repoJob.update({ + const { repo } = await this.db.repoIndexingJob.update({ where: { id: job.data.jobId }, data: { - status: RepoJobStatus.FAILED, + status: RepoIndexingJobStatus.FAILED, completedAt: new Date(), errorMessage: job.failedReason, }, @@ -428,10 +428,10 @@ export class RepoIndexManager { private onJobStalled = async (jobId: string) => groupmqLifecycleExceptionWrapper('onJobStalled', logger, async () => { const logger = createJobLogger(jobId); - const { repo } = await this.db.repoJob.update({ + const { repo } = await this.db.repoIndexingJob.update({ where: { id: jobId }, data: { - status: RepoJobStatus.FAILED, + status: RepoIndexingJobStatus.FAILED, completedAt: new Date(), errorMessage: 'Job stalled', }, diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 575e0cc9..aaaad4ea 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -246,7 +246,7 @@ const createGitCloneUrlWithToken = (cloneUrl: string, credentials: { username?: /** * Wraps groupmq worker lifecycle callbacks with exception handling. This prevents - * uncaught exceptions (e.g., like a RepoJob not existing in the DB) from crashing + * uncaught exceptions (e.g., like a RepoIndexingJob not existing in the DB) from crashing * the app. * @see: https://openpanel-dev.github.io/groupmq/api-worker/#events */ diff --git a/packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql b/packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql new file mode 100644 index 00000000..23fc47c3 --- /dev/null +++ b/packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql @@ -0,0 +1,34 @@ +/* + Warnings: + + - You are about to drop the column `repoIndexingStatus` on the `Repo` table. All the data in the column will be lost. + +*/ +-- CreateEnum +CREATE TYPE "RepoIndexingJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'); + +-- CreateEnum +CREATE TYPE "RepoIndexingJobType" AS ENUM ('INDEX', 'CLEANUP'); + +-- AlterTable +ALTER TABLE "Repo" DROP COLUMN "repoIndexingStatus"; + +-- DropEnum +DROP TYPE "RepoIndexingStatus"; + +-- CreateTable +CREATE TABLE "RepoIndexingJob" ( + "id" TEXT NOT NULL, + "type" "RepoIndexingJobType" NOT NULL, + "status" "RepoIndexingJobStatus" NOT NULL DEFAULT 'PENDING', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "completedAt" TIMESTAMP(3), + "errorMessage" TEXT, + "repoId" INTEGER NOT NULL, + + CONSTRAINT "RepoIndexingJob_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "RepoIndexingJob" ADD CONSTRAINT "RepoIndexingJob_repoId_fkey" FOREIGN KEY ("repoId") REFERENCES "Repo"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 16ed94f0..8952d0fc 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -10,17 +10,6 @@ datasource db { url = env("DATABASE_URL") } -enum RepoIndexingStatus { - NEW - IN_INDEX_QUEUE - INDEXING - INDEXED - FAILED - IN_GC_QUEUE - GARBAGE_COLLECTING - GARBAGE_COLLECTION_FAILED -} - enum ConnectionSyncStatus { SYNC_NEEDED IN_SYNC_QUEUE @@ -55,14 +44,11 @@ model Repo { connections RepoToConnection[] imageUrl String? - /// @deprecated status tracking is now done via the `jobs` table. - repoIndexingStatus RepoIndexingStatus @default(NEW) - permittedUsers UserToRepoPermission[] permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. - jobs RepoJob[] + jobs RepoIndexingJob[] indexedAt DateTime? /// When the repo was last indexed successfully. external_id String /// The id of the repo in the external service @@ -78,22 +64,22 @@ model Repo { @@index([orgId]) } -enum RepoJobStatus { +enum RepoIndexingJobStatus { PENDING IN_PROGRESS COMPLETED FAILED } -enum RepoJobType { +enum RepoIndexingJobType { INDEX CLEANUP } -model RepoJob { +model RepoIndexingJob { id String @id @default(cuid()) - type RepoJobType - status RepoJobStatus @default(PENDING) + type RepoIndexingJobType + status RepoIndexingJobStatus @default(PENDING) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt completedAt DateTime? diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 4f605d8e..f7ba284a 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -10,7 +10,7 @@ import { prisma } from "@/prisma"; import { render } from "@react-email/components"; import * as Sentry from '@sentry/nextjs'; import { encrypt, generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto"; -import { ApiKey, Org, OrgRole, Prisma, RepoJobStatus, RepoJobType, StripeSubscriptionStatus } from "@sourcebot/db"; +import { ApiKey, Org, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType, StripeSubscriptionStatus } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; @@ -586,11 +586,11 @@ export const getReposStats = async () => sew(() => orgId: org.id, jobs: { some: { - type: RepoJobType.INDEX, + type: RepoIndexingJobType.INDEX, status: { in: [ - RepoJobStatus.PENDING, - RepoJobStatus.IN_PROGRESS, + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, ] } }, diff --git a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx index 400481c2..54842731 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx @@ -10,7 +10,7 @@ import { env } from "@/env.mjs"; import { ServiceErrorException } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; import { DiscordLogoIcon, GitHubLogoIcon } from "@radix-ui/react-icons"; -import { RepoJobStatus, RepoJobType } from "@sourcebot/db"; +import { RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import Link from "next/link"; import { redirect } from "next/navigation"; import { OrgSelector } from "../orgSelector"; @@ -43,11 +43,11 @@ export const NavigationMenu = async ({ where: { jobs: { some: { - type: RepoJobType.INDEX, + type: RepoIndexingJobType.INDEX, status: { in: [ - RepoJobStatus.PENDING, - RepoJobStatus.IN_PROGRESS, + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, ] } }, diff --git a/packages/web/src/app/[domain]/repos/page.tsx b/packages/web/src/app/[domain]/repos/page.tsx index c5ed9689..e9dd4e43 100644 --- a/packages/web/src/app/[domain]/repos/page.tsx +++ b/packages/web/src/app/[domain]/repos/page.tsx @@ -1,5 +1,5 @@ import { env } from "@/env.mjs"; -import { RepoJob } from "@sourcebot/db"; +import { RepoIndexingJob } from "@sourcebot/db"; import { Header } from "../components/header"; import { RepoStatus } from "./columns"; import { RepositoryTable } from "./repositoryTable"; @@ -8,7 +8,7 @@ import { withOptionalAuthV2 } from "@/withAuthV2"; import { isServiceError } from "@/lib/utils"; import { ServiceErrorException } from "@/lib/serviceError"; -function getRepoStatus(repo: { indexedAt: Date | null, jobs: RepoJob[] }): RepoStatus { +function getRepoStatus(repo: { indexedAt: Date | null, jobs: RepoIndexingJob[] }): RepoStatus { const latestJob = repo.jobs[0]; if (latestJob?.status === 'PENDING' || latestJob?.status === 'IN_PROGRESS') {