ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-12 12:25:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

add back names in logs for repo and connection sync

Browse source
This commit is contained in:
msukkari 2025-04-10 21:17:57 -06:00
parent 1ab22e3624
commit 244afeadb8
3 changed files with 70 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
packages/backend/src/connectionManager.ts
View file

@ -20,6 +20,7 @@ const QUEUE_NAME = 'connectionSyncQueue';
type JobPayload = {
connectionId: number,
connectionName: string,
orgId: number,
config: ConnectionConfig,
};
@ -60,12 +61,13 @@ export class ConnectionManager implements IConnectionManager {
await this.queue.add('connectionSyncJob', {
connectionId: connection.id,
connectionName: connection.name,
orgId: connection.orgId,
config: connectionConfig,
});
this.logger.info(`Added job to queue for connection ${connection.id}`);
this.logger.info(`Added job to queue for connection ${connection.name} (id: ${connection.id})`);
}).catch((err: unknown) => {
this.logger.error(`Failed to add job to queue for connection ${connection.id}: ${err}`);
this.logger.error(`Failed to add job to queue for connection ${connection.name} (id: ${connection.id}): ${err}`);
});
}
@ -83,14 +85,18 @@ export class ConnectionManager implements IConnectionManager {
// (or if the date isn't set for some reason).
{
AND: [
{ OR: [
{ syncStatus: ConnectionSyncStatus.SYNCED },
{ syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS },
]},
{ OR: [
{ syncedAt: null },
{ syncedAt: { lt: thresholdDate } },
]}
{
OR: [
{ syncStatus: ConnectionSyncStatus.SYNCED },
{ syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS },
]
},
{
OR: [
{ syncedAt: null },
{ syncedAt: { lt: thresholdDate } },
]
}
]
}
]
@ -103,7 +109,7 @@ export class ConnectionManager implements IConnectionManager {
}
private async runSyncJob(job: Job<JobPayload>): Promise<JobResult> {
const { config, orgId } = job.data;
const { config, orgId, connectionName } = job.data;
// @note: We aren't actually doing anything with this atm.
const abortController = new AbortController();
@ -167,7 +173,7 @@ export class ConnectionManager implements IConnectionManager {
}
})();
} catch (err) {
this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId}: ${err}`);
this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId} (${connectionName}): ${err}`);
Sentry.captureException(err);
if (err instanceof BackendException) {
@ -218,7 +224,7 @@ export class ConnectionManager implements IConnectionManager {
}
});
const deleteDuration = performance.now() - deleteStart;
this.logger.info(`Deleted all RepoToConnection records for connection ${job.data.connectionId} in ${deleteDuration}ms`);
this.logger.info(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`);
const totalUpsertStart = performance.now();
for (const repo of repoData) {
@ -235,10 +241,10 @@ export class ConnectionManager implements IConnectionManager {
create: repo,
})
const upsertDuration = performance.now() - upsertStart;
this.logger.info(`Upserted repo ${repo.external_id} in ${upsertDuration}ms`);
this.logger.info(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`);
}
const totalUpsertDuration = performance.now() - totalUpsertStart;
this.logger.info(`Upserted ${repoData.length} repos in ${totalUpsertDuration}ms`);
this.logger.info(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`);
}, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS });
return {
@ -248,18 +254,20 @@ export class ConnectionManager implements IConnectionManager {
private async onSyncJobCompleted(job: Job<JobPayload>, result: JobResult) {
this.logger.info(`Connection sync job ${job.id} completed`);
this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`);
const { connectionId } = job.data;
let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
where: { id: connectionId },
select: { syncStatusMetadata: true }
}))?.syncStatusMetadata as Record<string, unknown> ?? {};
const { notFound } = syncStatusMetadata as { notFound: {
users: string[],
orgs: string[],
repos: string[],
}};
const { notFound } = syncStatusMetadata as {
notFound: {
users: string[],
orgs: string[],
repos: string[],
}
};
await this.db.connection.update({
where: {
@ -268,8 +276,8 @@ export class ConnectionManager implements IConnectionManager {
data: {
syncStatus:
notFound.users.length > 0 ||
notFound.orgs.length > 0 ||
notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
notFound.orgs.length > 0 ||
notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
syncedAt: new Date()
}
})
@ -281,7 +289,7 @@ export class ConnectionManager implements IConnectionManager {
}
private async onSyncJobFailed(job: Job<JobPayload> | undefined, err: unknown) {
this.logger.info(`Connection sync job failed with error: ${err}`);
this.logger.info(`Connection sync job for connection ${job?.data.connectionName} (id: ${job?.data.connectionId}, jobId: ${job?.id}) failed with error: ${err}`);
Sentry.captureException(err, {
tags: {
connectionid: job?.data.connectionId,

13
packages/backend/src/repoCompileUtils.ts
View file

@ -6,12 +6,15 @@ import { getGerritReposFromConfig } from "./gerrit.js";
import { Prisma, PrismaClient } from '@sourcebot/db';
import { WithRequired } from "./types.js"
import { marshalBool } from "./utils.js";
import { createLogger } from './logger.js';
import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
import { RepoMetadata } from './types.js';
import path from 'path';
export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
const logger = createLogger('RepoCompileUtils');
export const compileGithubConfig = async (
config: GithubConnectionConfig,
connectionId: number,
@ -39,6 +42,8 @@ export const compileGithubConfig = async (
const repoName = path.join(repoNameRoot, repoDisplayName);
const cloneUrl = new URL(repo.clone_url!);
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
const record: RepoData = {
external_id: repo.id.toString(),
external_codeHostType: 'github',
@ -110,6 +115,8 @@ export const compileGitlabConfig = async (
const repoDisplayName = project.path_with_namespace;
const repoName = path.join(repoNameRoot, repoDisplayName);
logger.debug(`Found gitlab repo ${repoDisplayName} with webUrl: ${projectUrl}`);
const record: RepoData = {
external_id: project.id.toString(),
external_codeHostType: 'gitlab',
@ -177,6 +184,8 @@ export const compileGiteaConfig = async (
const repoDisplayName = repo.full_name!;
const repoName = path.join(repoNameRoot, repoDisplayName);
logger.debug(`Found gitea repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
const record: RepoData = {
external_id: repo.id!.toString(),
external_codeHostType: 'gitea',
@ -246,11 +255,15 @@ export const compileGerritConfig = async (
const webLink = project.web_links[0];
const webUrl = webLink.url;
logger.debug(`Found gerrit repo ${project.name} with webUrl: ${webUrl}`);
// Handle case where webUrl is just a gitiles path
// https://github.com/GerritCodeReview/plugins_gitiles/blob/5ee7f57/src/main/java/com/googlesource/gerrit/plugins/gitiles/GitilesWeblinks.java#L50
if (webUrl.startsWith('/plugins/gitiles/')) {
logger.debug(`WebUrl is a gitiles path, joining with hostUrl: ${webUrl}`);
return path.join(hostUrl, webUrl);
} else {
logger.debug(`WebUrl is not a gitiles path, returning as is: ${webUrl}`);
return webUrl;
}
})();

32
packages/backend/src/repoManager.ts
View file

@ -140,10 +140,12 @@ export class RepoManager implements IRepoManager {
{
AND: [
{ repoIndexingStatus: RepoIndexingStatus.INDEXED },
{ OR: [
{ indexedAt: null },
{ indexedAt: { lt: thresholdDate } },
]}
{
OR: [
{ indexedAt: null },
{ indexedAt: { lt: thresholdDate } },
]
}
]
}
]
@ -210,18 +212,18 @@ export class RepoManager implements IRepoManager {
}
if (existsSync(repoPath)) {
this.logger.info(`Fetching ${repo.id}...`);
this.logger.info(`Fetching ${repo.displayName}...`);
const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`)
}));
fetchDuration_s = durationMs / 1000;
process.stdout.write('\n');
this.logger.info(`Fetched ${repo.name} in ${fetchDuration_s}s`);
this.logger.info(`Fetched ${repo.displayName} in ${fetchDuration_s}s`);
} else {
this.logger.info(`Cloning ${repo.id}...`);
this.logger.info(`Cloning ${repo.displayName}...`);
const token = await this.getTokenForRepo(repo, this.db);
const cloneUrl = new URL(repo.cloneUrl);
@ -240,12 +242,12 @@ export class RepoManager implements IRepoManager {
}
const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, ({ method, stage, progress }) => {
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`)
}));
cloneDuration_s = durationMs / 1000;
process.stdout.write('\n');
this.logger.info(`Cloned ${repo.id} in ${cloneDuration_s}s`);
this.logger.info(`Cloned ${repo.displayName} in ${cloneDuration_s}s`);
}
// Regardless of clone or fetch, always upsert the git config for the repo.
@ -255,10 +257,10 @@ export class RepoManager implements IRepoManager {
await upsertGitConfig(repoPath, metadata.gitConfig);
}
this.logger.info(`Indexing ${repo.id}...`);
this.logger.info(`Indexing ${repo.displayName}...`);
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
const indexDuration_s = durationMs / 1000;
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);
this.logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`);
return {
fetchDuration_s,
@ -268,7 +270,7 @@ export class RepoManager implements IRepoManager {
}
private async runIndexJob(job: Job<RepoIndexingPayload>) {
this.logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.id}`);
this.logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.displayName}`);
const repo = job.data.repo as RepoWithConnections;
// We have to use the existing repo object to get the repoIndexingStatus because the repo object
@ -332,7 +334,7 @@ export class RepoManager implements IRepoManager {
}
private async onIndexJobCompleted(job: Job<RepoIndexingPayload>) {
this.logger.info(`Repo index job ${job.id} completed`);
this.logger.info(`Repo index job for repo ${job.data.repo.displayName} (id: ${job.data.repo.id}, jobId: ${job.id}) completed`);
this.promClient.activeRepoIndexingJobs.dec();
this.promClient.repoIndexingSuccessTotal.inc();
@ -348,7 +350,7 @@ export class RepoManager implements IRepoManager {
}
private async onIndexJobFailed(job: Job<RepoIndexingPayload> | undefined, err: unknown) {
this.logger.info(`Repo index job failed (id: ${job?.id ?? 'unknown'}) with error: ${err}`);
this.logger.info(`Repo index job for repo ${job?.data.repo.displayName} (id: ${job?.data.repo.id}, jobId: ${job?.id}) failed with error: ${err}`);
Sentry.captureException(err, {
tags: {
repoId: job?.data.repo.id,