ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-12 20:35:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

add back names in logs for repo and connection sync

Browse source
This commit is contained in:
msukkari 2025-04-10 21:17:57 -06:00
parent 1ab22e3624
commit 244afeadb8
3 changed files with 70 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
packages/backend/src/connectionManager.ts
View file

@ -20,6 +20,7 @@ const QUEUE_NAME = 'connectionSyncQueue';
type JobPayload = { type JobPayload = {
connectionId: number, connectionId: number,
connectionName: string,
orgId: number, orgId: number,
config: ConnectionConfig, config: ConnectionConfig,
}; };
@ -60,12 +61,13 @@ export class ConnectionManager implements IConnectionManager {
await this.queue.add('connectionSyncJob', { await this.queue.add('connectionSyncJob', {
connectionId: connection.id, connectionId: connection.id,
connectionName: connection.name,
orgId: connection.orgId, orgId: connection.orgId,
config: connectionConfig, config: connectionConfig,
}); });
this.logger.info(`Added job to queue for connection ${connection.id}`); this.logger.info(`Added job to queue for connection ${connection.name} (id: ${connection.id})`);
}).catch((err: unknown) => { }).catch((err: unknown) => {
this.logger.error(`Failed to add job to queue for connection ${connection.id}: ${err}`); this.logger.error(`Failed to add job to queue for connection ${connection.name} (id: ${connection.id}): ${err}`);
}); });
} }
@ -83,14 +85,18 @@ export class ConnectionManager implements IConnectionManager {
// (or if the date isn't set for some reason). // (or if the date isn't set for some reason).
{ {
AND: [ AND: [
{ OR: [ {
{ syncStatus: ConnectionSyncStatus.SYNCED }, OR: [
{ syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS }, { syncStatus: ConnectionSyncStatus.SYNCED },
]}, { syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS },
{ OR: [ ]
{ syncedAt: null }, },
{ syncedAt: { lt: thresholdDate } }, {
]} OR: [
{ syncedAt: null },
{ syncedAt: { lt: thresholdDate } },
]
}
] ]
} }
] ]
@ -103,7 +109,7 @@ export class ConnectionManager implements IConnectionManager {
} }
private async runSyncJob(job: Job<JobPayload>): Promise<JobResult> { private async runSyncJob(job: Job<JobPayload>): Promise<JobResult> {
const { config, orgId } = job.data; const { config, orgId, connectionName } = job.data;
// @note: We aren't actually doing anything with this atm. // @note: We aren't actually doing anything with this atm.
const abortController = new AbortController(); const abortController = new AbortController();
@ -167,7 +173,7 @@ export class ConnectionManager implements IConnectionManager {
} }
})(); })();
} catch (err) { } catch (err) {
this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId}: ${err}`); this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId} (${connectionName}): ${err}`);
Sentry.captureException(err); Sentry.captureException(err);
if (err instanceof BackendException) { if (err instanceof BackendException) {
@ -218,7 +224,7 @@ export class ConnectionManager implements IConnectionManager {
} }
}); });
const deleteDuration = performance.now() - deleteStart; const deleteDuration = performance.now() - deleteStart;
this.logger.info(`Deleted all RepoToConnection records for connection ${job.data.connectionId} in ${deleteDuration}ms`); this.logger.info(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`);
const totalUpsertStart = performance.now(); const totalUpsertStart = performance.now();
for (const repo of repoData) { for (const repo of repoData) {
@ -235,10 +241,10 @@ export class ConnectionManager implements IConnectionManager {
create: repo, create: repo,
}) })
const upsertDuration = performance.now() - upsertStart; const upsertDuration = performance.now() - upsertStart;
this.logger.info(`Upserted repo ${repo.external_id} in ${upsertDuration}ms`); this.logger.info(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`);
} }
const totalUpsertDuration = performance.now() - totalUpsertStart; const totalUpsertDuration = performance.now() - totalUpsertStart;
this.logger.info(`Upserted ${repoData.length} repos in ${totalUpsertDuration}ms`); this.logger.info(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`);
}, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS });
return { return {
@ -248,18 +254,20 @@ export class ConnectionManager implements IConnectionManager {
private async onSyncJobCompleted(job: Job<JobPayload>, result: JobResult) { private async onSyncJobCompleted(job: Job<JobPayload>, result: JobResult) {
this.logger.info(`Connection sync job ${job.id} completed`); this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`);
const { connectionId } = job.data; const { connectionId } = job.data;
let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({ let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
where: { id: connectionId }, where: { id: connectionId },
select: { syncStatusMetadata: true } select: { syncStatusMetadata: true }
}))?.syncStatusMetadata as Record<string, unknown> ?? {}; }))?.syncStatusMetadata as Record<string, unknown> ?? {};
const { notFound } = syncStatusMetadata as { notFound: { const { notFound } = syncStatusMetadata as {
users: string[], notFound: {
orgs: string[], users: string[],
repos: string[], orgs: string[],
}}; repos: string[],
}
};
await this.db.connection.update({ await this.db.connection.update({
where: { where: {
@ -268,8 +276,8 @@ export class ConnectionManager implements IConnectionManager {
data: { data: {
syncStatus: syncStatus:
notFound.users.length > 0 || notFound.users.length > 0 ||
notFound.orgs.length > 0 || notFound.orgs.length > 0 ||
notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED, notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
syncedAt: new Date() syncedAt: new Date()
} }
}) })
@ -281,7 +289,7 @@ export class ConnectionManager implements IConnectionManager {
} }
private async onSyncJobFailed(job: Job<JobPayload> | undefined, err: unknown) { private async onSyncJobFailed(job: Job<JobPayload> | undefined, err: unknown) {
this.logger.info(`Connection sync job failed with error: ${err}`); this.logger.info(`Connection sync job for connection ${job?.data.connectionName} (id: ${job?.data.connectionId}, jobId: ${job?.id}) failed with error: ${err}`);
Sentry.captureException(err, { Sentry.captureException(err, {
tags: { tags: {
connectionid: job?.data.connectionId, connectionid: job?.data.connectionId,

13
packages/backend/src/repoCompileUtils.ts
View file

@ -6,12 +6,15 @@ import { getGerritReposFromConfig } from "./gerrit.js";
import { Prisma, PrismaClient } from '@sourcebot/db'; import { Prisma, PrismaClient } from '@sourcebot/db';
import { WithRequired } from "./types.js" import { WithRequired } from "./types.js"
import { marshalBool } from "./utils.js"; import { marshalBool } from "./utils.js";
import { createLogger } from './logger.js';
import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { RepoMetadata } from './types.js'; import { RepoMetadata } from './types.js';
import path from 'path'; import path from 'path';
export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>; export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
const logger = createLogger('RepoCompileUtils');
export const compileGithubConfig = async ( export const compileGithubConfig = async (
config: GithubConnectionConfig, config: GithubConnectionConfig,
connectionId: number, connectionId: number,
@ -39,6 +42,8 @@ export const compileGithubConfig = async (
const repoName = path.join(repoNameRoot, repoDisplayName); const repoName = path.join(repoNameRoot, repoDisplayName);
const cloneUrl = new URL(repo.clone_url!); const cloneUrl = new URL(repo.clone_url!);
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
const record: RepoData = { const record: RepoData = {
external_id: repo.id.toString(), external_id: repo.id.toString(),
external_codeHostType: 'github', external_codeHostType: 'github',
@ -110,6 +115,8 @@ export const compileGitlabConfig = async (
const repoDisplayName = project.path_with_namespace; const repoDisplayName = project.path_with_namespace;
const repoName = path.join(repoNameRoot, repoDisplayName); const repoName = path.join(repoNameRoot, repoDisplayName);
logger.debug(`Found gitlab repo ${repoDisplayName} with webUrl: ${projectUrl}`);
const record: RepoData = { const record: RepoData = {
external_id: project.id.toString(), external_id: project.id.toString(),
external_codeHostType: 'gitlab', external_codeHostType: 'gitlab',
@ -177,6 +184,8 @@ export const compileGiteaConfig = async (
const repoDisplayName = repo.full_name!; const repoDisplayName = repo.full_name!;
const repoName = path.join(repoNameRoot, repoDisplayName); const repoName = path.join(repoNameRoot, repoDisplayName);
logger.debug(`Found gitea repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
const record: RepoData = { const record: RepoData = {
external_id: repo.id!.toString(), external_id: repo.id!.toString(),
external_codeHostType: 'gitea', external_codeHostType: 'gitea',
@ -246,11 +255,15 @@ export const compileGerritConfig = async (
const webLink = project.web_links[0]; const webLink = project.web_links[0];
const webUrl = webLink.url; const webUrl = webLink.url;
logger.debug(`Found gerrit repo ${project.name} with webUrl: ${webUrl}`);
// Handle case where webUrl is just a gitiles path // Handle case where webUrl is just a gitiles path
// https://github.com/GerritCodeReview/plugins_gitiles/blob/5ee7f57/src/main/java/com/googlesource/gerrit/plugins/gitiles/GitilesWeblinks.java#L50 // https://github.com/GerritCodeReview/plugins_gitiles/blob/5ee7f57/src/main/java/com/googlesource/gerrit/plugins/gitiles/GitilesWeblinks.java#L50
if (webUrl.startsWith('/plugins/gitiles/')) { if (webUrl.startsWith('/plugins/gitiles/')) {
logger.debug(`WebUrl is a gitiles path, joining with hostUrl: ${webUrl}`);
return path.join(hostUrl, webUrl); return path.join(hostUrl, webUrl);
} else { } else {
logger.debug(`WebUrl is not a gitiles path, returning as is: ${webUrl}`);
return webUrl; return webUrl;
} }
})(); })();

32
packages/backend/src/repoManager.ts
View file

@ -140,10 +140,12 @@ export class RepoManager implements IRepoManager {
{ {
AND: [ AND: [
{ repoIndexingStatus: RepoIndexingStatus.INDEXED }, { repoIndexingStatus: RepoIndexingStatus.INDEXED },
{ OR: [ {
{ indexedAt: null }, OR: [
{ indexedAt: { lt: thresholdDate } }, { indexedAt: null },
]} { indexedAt: { lt: thresholdDate } },
]
}
] ]
} }
] ]
@ -210,18 +212,18 @@ export class RepoManager implements IRepoManager {
} }
if (existsSync(repoPath)) { if (existsSync(repoPath)) {
this.logger.info(`Fetching ${repo.id}...`); this.logger.info(`Fetching ${repo.displayName}...`);
const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => { const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`)
})); }));
fetchDuration_s = durationMs / 1000; fetchDuration_s = durationMs / 1000;
process.stdout.write('\n'); process.stdout.write('\n');
this.logger.info(`Fetched ${repo.name} in ${fetchDuration_s}s`); this.logger.info(`Fetched ${repo.displayName} in ${fetchDuration_s}s`);
} else { } else {
this.logger.info(`Cloning ${repo.id}...`); this.logger.info(`Cloning ${repo.displayName}...`);
const token = await this.getTokenForRepo(repo, this.db); const token = await this.getTokenForRepo(repo, this.db);
const cloneUrl = new URL(repo.cloneUrl); const cloneUrl = new URL(repo.cloneUrl);
@ -240,12 +242,12 @@ export class RepoManager implements IRepoManager {
} }
const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, ({ method, stage, progress }) => { const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, ({ method, stage, progress }) => {
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`)
})); }));
cloneDuration_s = durationMs / 1000; cloneDuration_s = durationMs / 1000;
process.stdout.write('\n'); process.stdout.write('\n');
this.logger.info(`Cloned ${repo.id} in ${cloneDuration_s}s`); this.logger.info(`Cloned ${repo.displayName} in ${cloneDuration_s}s`);
} }
// Regardless of clone or fetch, always upsert the git config for the repo. // Regardless of clone or fetch, always upsert the git config for the repo.
@ -255,10 +257,10 @@ export class RepoManager implements IRepoManager {
await upsertGitConfig(repoPath, metadata.gitConfig); await upsertGitConfig(repoPath, metadata.gitConfig);
} }
this.logger.info(`Indexing ${repo.id}...`); this.logger.info(`Indexing ${repo.displayName}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx)); const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000; const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`); this.logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`);
return { return {
fetchDuration_s, fetchDuration_s,
@ -268,7 +270,7 @@ export class RepoManager implements IRepoManager {
} }
private async runIndexJob(job: Job<RepoIndexingPayload>) { private async runIndexJob(job: Job<RepoIndexingPayload>) {
this.logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.id}`); this.logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.displayName}`);
const repo = job.data.repo as RepoWithConnections; const repo = job.data.repo as RepoWithConnections;
// We have to use the existing repo object to get the repoIndexingStatus because the repo object // We have to use the existing repo object to get the repoIndexingStatus because the repo object
@ -332,7 +334,7 @@ export class RepoManager implements IRepoManager {
} }
private async onIndexJobCompleted(job: Job<RepoIndexingPayload>) { private async onIndexJobCompleted(job: Job<RepoIndexingPayload>) {
this.logger.info(`Repo index job ${job.id} completed`); this.logger.info(`Repo index job for repo ${job.data.repo.displayName} (id: ${job.data.repo.id}, jobId: ${job.id}) completed`);
this.promClient.activeRepoIndexingJobs.dec(); this.promClient.activeRepoIndexingJobs.dec();
this.promClient.repoIndexingSuccessTotal.inc(); this.promClient.repoIndexingSuccessTotal.inc();
@ -348,7 +350,7 @@ export class RepoManager implements IRepoManager {
} }
private async onIndexJobFailed(job: Job<RepoIndexingPayload> | undefined, err: unknown) { private async onIndexJobFailed(job: Job<RepoIndexingPayload> | undefined, err: unknown) {
this.logger.info(`Repo index job failed (id: ${job?.id ?? 'unknown'}) with error: ${err}`); this.logger.info(`Repo index job for repo ${job?.data.repo.displayName} (id: ${job?.data.repo.id}, jobId: ${job?.id}) failed with error: ${err}`);
Sentry.captureException(err, { Sentry.captureException(err, {
tags: { tags: {
repoId: job?.data.repo.id, repoId: job?.data.repo.id,