From 244afeadb8875f8f72437425af469b04959dce8c Mon Sep 17 00:00:00 2001 From: msukkari Date: Thu, 10 Apr 2025 21:17:57 -0600 Subject: [PATCH] add back names in logs for repo and connection sync --- packages/backend/src/connectionManager.ts | 64 +++++++++++++---------- packages/backend/src/repoCompileUtils.ts | 13 +++++ packages/backend/src/repoManager.ts | 40 +++++++------- 3 files changed, 70 insertions(+), 47 deletions(-) diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index 1320e437..6985b2de 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -20,6 +20,7 @@ const QUEUE_NAME = 'connectionSyncQueue'; type JobPayload = { connectionId: number, + connectionName: string, orgId: number, config: ConnectionConfig, }; @@ -60,12 +61,13 @@ export class ConnectionManager implements IConnectionManager { await this.queue.add('connectionSyncJob', { connectionId: connection.id, + connectionName: connection.name, orgId: connection.orgId, config: connectionConfig, }); - this.logger.info(`Added job to queue for connection ${connection.id}`); + this.logger.info(`Added job to queue for connection ${connection.name} (id: ${connection.id})`); }).catch((err: unknown) => { - this.logger.error(`Failed to add job to queue for connection ${connection.id}: ${err}`); + this.logger.error(`Failed to add job to queue for connection ${connection.name} (id: ${connection.id}): ${err}`); }); } @@ -83,14 +85,18 @@ export class ConnectionManager implements IConnectionManager { // (or if the date isn't set for some reason). { AND: [ - { OR: [ - { syncStatus: ConnectionSyncStatus.SYNCED }, - { syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS }, - ]}, - { OR: [ - { syncedAt: null }, - { syncedAt: { lt: thresholdDate } }, - ]} + { + OR: [ + { syncStatus: ConnectionSyncStatus.SYNCED }, + { syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS }, + ] + }, + { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: thresholdDate } }, + ] + } ] } ] @@ -103,7 +109,7 @@ export class ConnectionManager implements IConnectionManager { } private async runSyncJob(job: Job): Promise { - const { config, orgId } = job.data; + const { config, orgId, connectionName } = job.data; // @note: We aren't actually doing anything with this atm. const abortController = new AbortController(); @@ -120,7 +126,7 @@ export class ConnectionManager implements IConnectionManager { Sentry.captureException(e); throw e; } - + // Reset the syncStatusMetadata to an empty object at the start of the sync job await this.db.connection.update({ where: { @@ -131,7 +137,7 @@ export class ConnectionManager implements IConnectionManager { syncStatusMetadata: {} } }) - + let result: { repoData: RepoData[], @@ -167,7 +173,7 @@ export class ConnectionManager implements IConnectionManager { } })(); } catch (err) { - this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId}: ${err}`); + this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId} (${connectionName}): ${err}`); Sentry.captureException(err); if (err instanceof BackendException) { @@ -191,7 +197,7 @@ export class ConnectionManager implements IConnectionManager { syncStatusMetadata: { notFound } } }); - + // Filter out any duplicates by external_id and external_codeHostUrl. repoData = repoData.filter((repo, index, self) => { return index === self.findIndex(r => @@ -218,7 +224,7 @@ export class ConnectionManager implements IConnectionManager { } }); const deleteDuration = performance.now() - deleteStart; - this.logger.info(`Deleted all RepoToConnection records for connection ${job.data.connectionId} in ${deleteDuration}ms`); + this.logger.info(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`); const totalUpsertStart = performance.now(); for (const repo of repoData) { @@ -235,10 +241,10 @@ export class ConnectionManager implements IConnectionManager { create: repo, }) const upsertDuration = performance.now() - upsertStart; - this.logger.info(`Upserted repo ${repo.external_id} in ${upsertDuration}ms`); + this.logger.info(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); } const totalUpsertDuration = performance.now() - totalUpsertStart; - this.logger.info(`Upserted ${repoData.length} repos in ${totalUpsertDuration}ms`); + this.logger.info(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`); }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); return { @@ -248,18 +254,20 @@ export class ConnectionManager implements IConnectionManager { private async onSyncJobCompleted(job: Job, result: JobResult) { - this.logger.info(`Connection sync job ${job.id} completed`); + this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`); const { connectionId } = job.data; let syncStatusMetadata: Record = (await this.db.connection.findUnique({ where: { id: connectionId }, select: { syncStatusMetadata: true } }))?.syncStatusMetadata as Record ?? {}; - const { notFound } = syncStatusMetadata as { notFound: { - users: string[], - orgs: string[], - repos: string[], - }}; + const { notFound } = syncStatusMetadata as { + notFound: { + users: string[], + orgs: string[], + repos: string[], + } + }; await this.db.connection.update({ where: { @@ -268,8 +276,8 @@ export class ConnectionManager implements IConnectionManager { data: { syncStatus: notFound.users.length > 0 || - notFound.orgs.length > 0 || - notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED, + notFound.orgs.length > 0 || + notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED, syncedAt: new Date() } }) @@ -281,7 +289,7 @@ export class ConnectionManager implements IConnectionManager { } private async onSyncJobFailed(job: Job | undefined, err: unknown) { - this.logger.info(`Connection sync job failed with error: ${err}`); + this.logger.info(`Connection sync job for connection ${job?.data.connectionName} (id: ${job?.data.connectionId}, jobId: ${job?.id}) failed with error: ${err}`); Sentry.captureException(err, { tags: { connectionid: job?.data.connectionId, @@ -312,7 +320,7 @@ export class ConnectionManager implements IConnectionManager { } } else { syncStatusMetadata = { - ...syncStatusMetadata, + ...syncStatusMetadata, error: 'UNKNOWN', } } diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 01e3e72c..b0469f97 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -6,12 +6,15 @@ import { getGerritReposFromConfig } from "./gerrit.js"; import { Prisma, PrismaClient } from '@sourcebot/db'; import { WithRequired } from "./types.js" import { marshalBool } from "./utils.js"; +import { createLogger } from './logger.js'; import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { RepoMetadata } from './types.js'; import path from 'path'; export type RepoData = WithRequired; +const logger = createLogger('RepoCompileUtils'); + export const compileGithubConfig = async ( config: GithubConnectionConfig, connectionId: number, @@ -39,6 +42,8 @@ export const compileGithubConfig = async ( const repoName = path.join(repoNameRoot, repoDisplayName); const cloneUrl = new URL(repo.clone_url!); + logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`); + const record: RepoData = { external_id: repo.id.toString(), external_codeHostType: 'github', @@ -110,6 +115,8 @@ export const compileGitlabConfig = async ( const repoDisplayName = project.path_with_namespace; const repoName = path.join(repoNameRoot, repoDisplayName); + logger.debug(`Found gitlab repo ${repoDisplayName} with webUrl: ${projectUrl}`); + const record: RepoData = { external_id: project.id.toString(), external_codeHostType: 'gitlab', @@ -177,6 +184,8 @@ export const compileGiteaConfig = async ( const repoDisplayName = repo.full_name!; const repoName = path.join(repoNameRoot, repoDisplayName); + logger.debug(`Found gitea repo ${repoDisplayName} with webUrl: ${repo.html_url}`); + const record: RepoData = { external_id: repo.id!.toString(), external_codeHostType: 'gitea', @@ -246,11 +255,15 @@ export const compileGerritConfig = async ( const webLink = project.web_links[0]; const webUrl = webLink.url; + logger.debug(`Found gerrit repo ${project.name} with webUrl: ${webUrl}`); + // Handle case where webUrl is just a gitiles path // https://github.com/GerritCodeReview/plugins_gitiles/blob/5ee7f57/src/main/java/com/googlesource/gerrit/plugins/gitiles/GitilesWeblinks.java#L50 if (webUrl.startsWith('/plugins/gitiles/')) { + logger.debug(`WebUrl is a gitiles path, joining with hostUrl: ${webUrl}`); return path.join(hostUrl, webUrl); } else { + logger.debug(`WebUrl is not a gitiles path, returning as is: ${webUrl}`); return webUrl; } })(); diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts index 9f77f004..ca5db265 100644 --- a/packages/backend/src/repoManager.ts +++ b/packages/backend/src/repoManager.ts @@ -140,10 +140,12 @@ export class RepoManager implements IRepoManager { { AND: [ { repoIndexingStatus: RepoIndexingStatus.INDEXED }, - { OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } }, - ]} + { + OR: [ + { indexedAt: null }, + { indexedAt: { lt: thresholdDate } }, + ] + } ] } ] @@ -201,27 +203,27 @@ export class RepoManager implements IRepoManager { const repoPath = getRepoPath(repo, this.ctx); const metadata = repoMetadataSchema.parse(repo.metadata); - + // If the repo was already in the indexing state, this job was likely killed and picked up again. As a result, // to ensure the repo state is valid, we delete the repo if it exists so we get a fresh clone if (repoAlreadyInIndexingState && existsSync(repoPath)) { this.logger.info(`Deleting repo directory ${repoPath} during sync because it was already in the indexing state`); - await promises.rm(repoPath, { recursive: true, force: true }); + await promises.rm(repoPath, { recursive: true, force: true }); } if (existsSync(repoPath)) { - this.logger.info(`Fetching ${repo.id}...`); + this.logger.info(`Fetching ${repo.displayName}...`); const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => { - this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) + this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) })); fetchDuration_s = durationMs / 1000; process.stdout.write('\n'); - this.logger.info(`Fetched ${repo.name} in ${fetchDuration_s}s`); + this.logger.info(`Fetched ${repo.displayName} in ${fetchDuration_s}s`); } else { - this.logger.info(`Cloning ${repo.id}...`); + this.logger.info(`Cloning ${repo.displayName}...`); const token = await this.getTokenForRepo(repo, this.db); const cloneUrl = new URL(repo.cloneUrl); @@ -240,12 +242,12 @@ export class RepoManager implements IRepoManager { } const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, ({ method, stage, progress }) => { - this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) + this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) })); cloneDuration_s = durationMs / 1000; process.stdout.write('\n'); - this.logger.info(`Cloned ${repo.id} in ${cloneDuration_s}s`); + this.logger.info(`Cloned ${repo.displayName} in ${cloneDuration_s}s`); } // Regardless of clone or fetch, always upsert the git config for the repo. @@ -255,10 +257,10 @@ export class RepoManager implements IRepoManager { await upsertGitConfig(repoPath, metadata.gitConfig); } - this.logger.info(`Indexing ${repo.id}...`); + this.logger.info(`Indexing ${repo.displayName}...`); const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx)); const indexDuration_s = durationMs / 1000; - this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`); + this.logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`); return { fetchDuration_s, @@ -268,7 +270,7 @@ export class RepoManager implements IRepoManager { } private async runIndexJob(job: Job) { - this.logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.id}`); + this.logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.displayName}`); const repo = job.data.repo as RepoWithConnections; // We have to use the existing repo object to get the repoIndexingStatus because the repo object @@ -332,7 +334,7 @@ export class RepoManager implements IRepoManager { } private async onIndexJobCompleted(job: Job) { - this.logger.info(`Repo index job ${job.id} completed`); + this.logger.info(`Repo index job for repo ${job.data.repo.displayName} (id: ${job.data.repo.id}, jobId: ${job.id}) completed`); this.promClient.activeRepoIndexingJobs.dec(); this.promClient.repoIndexingSuccessTotal.inc(); @@ -348,7 +350,7 @@ export class RepoManager implements IRepoManager { } private async onIndexJobFailed(job: Job | undefined, err: unknown) { - this.logger.info(`Repo index job failed (id: ${job?.id ?? 'unknown'}) with error: ${err}`); + this.logger.info(`Repo index job for repo ${job?.data.repo.displayName} (id: ${job?.data.repo.id}, jobId: ${job?.id}) failed with error: ${err}`); Sentry.captureException(err, { tags: { repoId: job?.data.repo.id, @@ -468,7 +470,7 @@ export class RepoManager implements IRepoManager { const repoPath = getRepoPath(repo, this.ctx); if (existsSync(repoPath)) { this.logger.info(`Deleting repo directory ${repoPath}`); - await promises.rm(repoPath, { recursive: true, force: true }); + await promises.rm(repoPath, { recursive: true, force: true }); } // delete shards @@ -542,7 +544,7 @@ export class RepoManager implements IRepoManager { }); }); } - + public async dispose() { this.indexWorker.close(); this.indexQueue.close();