From 023edd10001e421a3b33b8cda8da10ee3db2c51c Mon Sep 17 00:00:00 2001 From: bkellam Date: Sun, 23 Mar 2025 13:58:01 -0700 Subject: [PATCH] Improved database DX --- packages/db/README.md | 25 ++++++++- packages/db/package.json | 15 ++++-- packages/db/tools/runPrismaCommand.ts | 29 +++++++++++ packages/db/tools/scriptRunner.ts | 42 +++++++++++++++ .../scripts/migrate-duplicate-connections.ts | 52 +++++++++++++++++++ packages/db/tools/utils.ts | 9 ++++ yarn.lock | 19 ++++--- 7 files changed, 179 insertions(+), 12 deletions(-) create mode 100644 packages/db/tools/runPrismaCommand.ts create mode 100644 packages/db/tools/scriptRunner.ts create mode 100644 packages/db/tools/scripts/migrate-duplicate-connections.ts create mode 100644 packages/db/tools/utils.ts diff --git a/packages/db/README.md b/packages/db/README.md index d40f04e9..1bb1a3a1 100644 --- a/packages/db/README.md +++ b/packages/db/README.md @@ -1 +1,24 @@ -This package contains the database schema (prisma/schema.prisma), migrations (prisma/migrations) and the client library for interacting with the database. Before making edits to the schema, please read about prisma's [migration model](https://www.prisma.io/docs/orm/prisma-migrate/understanding-prisma-migrate/mental-model) to get an idea of how migrations work. \ No newline at end of file +# @sourcebot/db + +This package contains the database schema (prisma/schema.prisma), migrations (prisma/migrations) and the client library for interacting with the database. Before making edits to the schema, please read about prisma's [migration model](https://www.prisma.io/docs/orm/prisma-migrate/understanding-prisma-migrate/mental-model) to get an idea of how migrations work. + +## Tools + +This library contains a `/tools` directory with a collection of tooling needed for database management. Notable tools are: + +- `yarn tool:prisma` - runs the prisma CLI with an additional required param `--url`, the connection URL of the database you want the command to run against. This tool is geared towards running commands against non-dev database like staging or prod since 1) it allows you to quickly switch between environments, and 2) connection URLs do not need to be persisted in a `DATABASE_URL` environment variable. Examples: + +```sh +# Run prisma studio +yarn tool:prisma studio --url postgresql://username:password@url:5432/db_name + +# Rollback a migration +yarn tool:prisma migrate resolve --rolled-back "migration_name" --url postgresql://username:password@url:5432/db_name +``` + +- `yarn tool:run-script` - runs a script (located in the `/tools/scripts` directory) that performs some operations against the DB. This is useful for writing bespoke CRUD operations while still being type-safe and having all the perks of the prisma client lib. + +```sh +# Run `migrate-duplicate-connections.ts` +yarn tool:run-script --script migrate-duplicate-connections --url postgresql://username:password@url:5432/db_name +``` diff --git a/packages/db/package.json b/packages/db/package.json index 2381afa4..9886e83f 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -4,6 +4,9 @@ "main": "dist/index.js", "private": true, "scripts": { + "build": "yarn prisma:generate && tsc", + "postinstall": "yarn build", + "prisma:generate": "prisma generate", "prisma:generate:watch": "prisma generate --watch", "prisma:migrate:dev": "prisma migrate dev", @@ -11,14 +14,20 @@ "prisma:migrate:reset": "prisma migrate reset", "prisma:db:push": "prisma db push", "prisma:studio": "prisma studio", - "build": "yarn prisma:generate && tsc", - "postinstall": "yarn build" + + "tool:prisma": "tsx tools/runPrismaCommand.ts", + "tool:run-script": "tsx tools/scriptRunner.ts" }, "devDependencies": { + "@types/argparse": "^2.0.16", + "argparse": "^2.0.1", "prisma": "^6.2.1", + "tsx": "^4.19.1", "typescript": "^5.7.3" }, "dependencies": { - "@prisma/client": "6.2.1" + "@prisma/client": "6.2.1", + "@types/readline-sync": "^1.4.8", + "readline-sync": "^1.4.10" } } diff --git a/packages/db/tools/runPrismaCommand.ts b/packages/db/tools/runPrismaCommand.ts new file mode 100644 index 00000000..65b49f17 --- /dev/null +++ b/packages/db/tools/runPrismaCommand.ts @@ -0,0 +1,29 @@ +import { ArgumentParser } from "argparse"; +import { spawn } from 'child_process'; +import { confirmAction } from "./utils"; + +// This script is used to run a prisma command with a database URL. + +const parser = new ArgumentParser({ + add_help: false, +}); +parser.add_argument("--url", { required: true, help: "Database URL" }); + +// Parse known args to get the URL, but preserve the rest +const parsed = parser.parse_known_args(); +const args = parsed[0]; +const remainingArgs = parsed[1]; + +process.env.DATABASE_URL = args.url; + +confirmAction(`command: prisma ${remainingArgs.join(' ')}\nurl: ${args.url}\n\nContinue? [N/y]`); + +// Run prisma command with remaining arguments +const prisma = spawn('npx', ['prisma', ...remainingArgs], { + stdio: 'inherit', + env: process.env +}); + +prisma.on('exit', (code) => { + process.exit(code); +}); diff --git a/packages/db/tools/scriptRunner.ts b/packages/db/tools/scriptRunner.ts new file mode 100644 index 00000000..0b86058d --- /dev/null +++ b/packages/db/tools/scriptRunner.ts @@ -0,0 +1,42 @@ +import { PrismaClient } from "@sourcebot/db"; +import { ArgumentParser } from "argparse"; +import { migrateDuplicateConnections } from "./scripts/migrate-duplicate-connections"; +import { confirmAction } from "./utils"; + +export interface Script { + run: (prisma: PrismaClient) => Promise; +} + +export const scripts: Record = { + "migrate-duplicate-connections": migrateDuplicateConnections, +} + +const parser = new ArgumentParser(); +parser.add_argument("--url", { required: true, help: "Database URL" }); +parser.add_argument("--script", { required: true, help: "Script to run" }); +const args = parser.parse_args(); + +(async () => { + if (!(args.script in scripts)) { + console.log("Invalid script"); + process.exit(1); + } + + const selectedScript = scripts[args.script]; + + console.log("\nTo confirm:"); + console.log(`- Database URL: ${args.url}`); + console.log(`- Script: ${args.script}`); + + confirmAction(); + + const prisma = new PrismaClient({ + datasourceUrl: args.url, + }); + + await selectedScript.run(prisma); + + console.log("\nDone."); + process.exit(0); +})(); + diff --git a/packages/db/tools/scripts/migrate-duplicate-connections.ts b/packages/db/tools/scripts/migrate-duplicate-connections.ts new file mode 100644 index 00000000..7093e429 --- /dev/null +++ b/packages/db/tools/scripts/migrate-duplicate-connections.ts @@ -0,0 +1,52 @@ +import { Script } from "../scriptRunner"; +import { PrismaClient } from "../../dist"; +import { confirmAction } from "../utils"; + +// Handles duplicate connections by renaming them to be unique. +// @see: 20250320215449_unique_connection_name_constraint_within_org +export const migrateDuplicateConnections: Script = { + run: async (prisma: PrismaClient) => { + + // Find all duplicate connections based on name and orgId + const duplicates = (await prisma.connection.groupBy({ + by: ['name', 'orgId'], + _count: { + _all: true, + }, + })).filter(({ _count }) => _count._all > 1); + + console.log(`Found ${duplicates.reduce((acc, { _count }) => acc + _count._all, 0)} duplicate connections.`); + + confirmAction(); + + let migrated = 0; + + for (const duplicate of duplicates) { + const { name, orgId } = duplicate; + const connections = await prisma.connection.findMany({ + where: { + name, + orgId, + }, + orderBy: { + createdAt: 'asc', + }, + }); + + for (let i = 0; i < connections.length; i++) { + const connection = connections[i]; + const newName = `${name}-${i + 1}`; + + console.log(`Migrating connection with id ${connection.id} from name=${name} to name=${newName}`); + + await prisma.connection.update({ + where: { id: connection.id }, + data: { name: newName }, + }); + migrated++; + } + } + + console.log(`Migrated ${migrated} connections.`); + }, +}; diff --git a/packages/db/tools/utils.ts b/packages/db/tools/utils.ts new file mode 100644 index 00000000..dbd08b00 --- /dev/null +++ b/packages/db/tools/utils.ts @@ -0,0 +1,9 @@ +import readline from 'readline-sync'; + +export const confirmAction = (message: string = "Are you sure you want to proceed? [N/y]") => { + const response = readline.question(message).toLowerCase(); + if (response !== 'y') { + console.log("Aborted."); + process.exit(0); + } +} diff --git a/yarn.lock b/yarn.lock index c790806a..566992af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3813,6 +3813,11 @@ "@types/prop-types" "*" csstype "^3.0.2" +"@types/readline-sync@^1.4.8": + version "1.4.8" + resolved "https://registry.npmjs.org/@types/readline-sync/-/readline-sync-1.4.8.tgz#dc9767a93fc83825d90331f2549a2e90fc3255f0" + integrity sha512-BL7xOf0yKLA6baAX6MMOnYkoflUyj/c7y3pqMRfU0va7XlwHAOTOIo4x55P/qLfMsuaYdJJKubToLqRVmRtRZA== + "@types/send@*": version "0.17.4" resolved "https://registry.yarnpkg.com/@types/send/-/send-0.17.4.tgz#6619cd24e7270793702e4e6a4b958a9010cfc57a" @@ -8822,6 +8827,11 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +readline-sync@^1.4.10: + version "1.4.10" + resolved "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b" + integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw== + redis-errors@^1.0.0, redis-errors@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz" @@ -9574,14 +9584,7 @@ stringify-entities@^4.0.0: character-entities-html4 "^2.0.0" character-entities-legacy "^3.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==