mirror of
https://github.com/sourcebot-dev/sourcebot.git
synced 2025-12-12 04:15:30 +00:00
github app service auth
This commit is contained in:
parent
1360caa905
commit
1cd90071b3
13 changed files with 765 additions and 10 deletions
114
docs/snippets/schemas/v3/app.schema.mdx
Normal file
114
docs/snippets/schemas/v3/app.schema.mdx
Normal file
|
|
@ -0,0 +1,114 @@
|
||||||
|
{/* THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! */}
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "AppConfig",
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"type": "object",
|
||||||
|
"title": "GithubAppConfig",
|
||||||
|
"properties": {
|
||||||
|
"type": {
|
||||||
|
"const": "githubApp",
|
||||||
|
"description": "GitHub App Configuration"
|
||||||
|
},
|
||||||
|
"deploymentHostname": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "url",
|
||||||
|
"default": "github.com",
|
||||||
|
"description": "The hostname of the GitHub App deployment.",
|
||||||
|
"examples": [
|
||||||
|
"github.com",
|
||||||
|
"github.example.com"
|
||||||
|
],
|
||||||
|
"pattern": "^[^\\s/$.?#].[^\\s]*$"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The ID of the GitHub App."
|
||||||
|
},
|
||||||
|
"privateKey": {
|
||||||
|
"description": "The private key of the GitHub App.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"secret": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the secret that contains the token."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"secret"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"env": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"env"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"privateKeyPath": {
|
||||||
|
"description": "The path to the private key of the GitHub App.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"secret": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the secret that contains the token."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"secret"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"env": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"env"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"type",
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"required": [
|
||||||
|
"privateKey"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"required": [
|
||||||
|
"privateKeyPath"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
108
docs/snippets/schemas/v3/githubApp.schema.mdx
Normal file
108
docs/snippets/schemas/v3/githubApp.schema.mdx
Normal file
|
|
@ -0,0 +1,108 @@
|
||||||
|
{/* THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! */}
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"type": "object",
|
||||||
|
"title": "GithubAppConfig",
|
||||||
|
"properties": {
|
||||||
|
"type": {
|
||||||
|
"const": "githubApp",
|
||||||
|
"description": "GitHub App Configuration"
|
||||||
|
},
|
||||||
|
"deploymentHostname": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "url",
|
||||||
|
"default": "github.com",
|
||||||
|
"description": "The hostname of the GitHub App deployment.",
|
||||||
|
"examples": [
|
||||||
|
"github.com",
|
||||||
|
"github.example.com"
|
||||||
|
],
|
||||||
|
"pattern": "^[^\\s/$.?#].[^\\s]*$"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The ID of the GitHub App."
|
||||||
|
},
|
||||||
|
"privateKey": {
|
||||||
|
"description": "The private key of the GitHub App.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"secret": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the secret that contains the token."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"secret"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"env": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"env"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"privateKeyPath": {
|
||||||
|
"description": "The path to the private key of the GitHub App.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"secret": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the secret that contains the token."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"secret"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"env": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"env"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"type",
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"required": [
|
||||||
|
"privateKey"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"required": [
|
||||||
|
"privateKeyPath"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
```
|
||||||
133
packages/backend/src/ee/githubAppManager.ts
Normal file
133
packages/backend/src/ee/githubAppManager.ts
Normal file
|
|
@ -0,0 +1,133 @@
|
||||||
|
import { GithubAppConfig, SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
|
||||||
|
import { loadConfig } from "@sourcebot/shared";
|
||||||
|
import { env } from "../env.js";
|
||||||
|
import { createLogger } from "@sourcebot/logger";
|
||||||
|
import { getTokenFromConfig } from "../utils.js";
|
||||||
|
import { PrismaClient } from "@sourcebot/db";
|
||||||
|
import { App } from "@octokit/app";
|
||||||
|
|
||||||
|
const logger = createLogger('githubAppManager');
|
||||||
|
const GITHUB_DEFAULT_DEPLOYMENT_HOSTNAME = 'github.com';
|
||||||
|
|
||||||
|
type Installation = {
|
||||||
|
id: number;
|
||||||
|
appId: number;
|
||||||
|
account: {
|
||||||
|
login: string;
|
||||||
|
type: 'organization' | 'user';
|
||||||
|
};
|
||||||
|
createdAt: string;
|
||||||
|
expiresAt: string;
|
||||||
|
token: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export class GithubAppManager {
|
||||||
|
private static instance: GithubAppManager | null = null;
|
||||||
|
private octokitApps: Map<number, App>;
|
||||||
|
private installationMap: Map<string, Installation>;
|
||||||
|
private db: PrismaClient | null = null;
|
||||||
|
private initialized: boolean = false;
|
||||||
|
|
||||||
|
private constructor() {
|
||||||
|
this.octokitApps = new Map<number, App>();
|
||||||
|
this.installationMap = new Map<string, Installation>();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static getInstance(): GithubAppManager {
|
||||||
|
if (!GithubAppManager.instance) {
|
||||||
|
GithubAppManager.instance = new GithubAppManager();
|
||||||
|
}
|
||||||
|
return GithubAppManager.instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
private ensureInitialized(): void {
|
||||||
|
if (!this.initialized) {
|
||||||
|
throw new Error('GithubAppManager must be initialized before use. Call init() first.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async init(db: PrismaClient) {
|
||||||
|
this.db = db;
|
||||||
|
const config = await loadConfig(env.CONFIG_PATH!);
|
||||||
|
const githubApps = config.apps?.filter(app => app.type === 'githubApp') as GithubAppConfig[];
|
||||||
|
|
||||||
|
logger.info(`Found ${githubApps.length} GitHub apps in config`);
|
||||||
|
|
||||||
|
for (const app of githubApps) {
|
||||||
|
const deploymentHostname = app.deploymentHostname as string || GITHUB_DEFAULT_DEPLOYMENT_HOSTNAME;
|
||||||
|
|
||||||
|
// @todo: we should move SINGLE_TENANT_ORG_ID to shared package or just remove the need to pass this in
|
||||||
|
// when resolving tokens
|
||||||
|
const SINGLE_TENANT_ORG_ID = 1;
|
||||||
|
const privateKey = await getTokenFromConfig(app.privateKey, SINGLE_TENANT_ORG_ID, this.db!);
|
||||||
|
|
||||||
|
const octokitApp = new App({
|
||||||
|
appId: Number(app.id),
|
||||||
|
privateKey: privateKey,
|
||||||
|
});
|
||||||
|
this.octokitApps.set(Number(app.id), octokitApp);
|
||||||
|
|
||||||
|
const installations = await octokitApp.octokit.request("GET /app/installations");
|
||||||
|
logger.info(`Found ${installations.data.length} GitHub App installations for ${deploymentHostname}/${app.id}:`);
|
||||||
|
|
||||||
|
for (const installationData of installations.data) {
|
||||||
|
logger.info(`\tInstallation ID: ${installationData.id}, Account: ${installationData.account?.login}, Type: ${installationData.account?.type}`);
|
||||||
|
|
||||||
|
const owner = installationData.account?.login!;
|
||||||
|
const accountType = installationData.account?.type!.toLowerCase() as 'organization' | 'user';
|
||||||
|
const installationOctokit = await octokitApp.getInstallationOctokit(installationData.id);
|
||||||
|
const auth = await installationOctokit.auth({ type: "installation" }) as { expires_at: string, token: string };
|
||||||
|
|
||||||
|
const installation: Installation = {
|
||||||
|
id: installationData.id,
|
||||||
|
appId: Number(app.id),
|
||||||
|
account: {
|
||||||
|
login: owner,
|
||||||
|
type: accountType,
|
||||||
|
},
|
||||||
|
createdAt: installationData.created_at,
|
||||||
|
expiresAt: auth.expires_at,
|
||||||
|
token: auth.token
|
||||||
|
};
|
||||||
|
this.installationMap.set(this.generateMapKey(owner, deploymentHostname), installation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.initialized = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getInstallationToken(owner: string, deploymentHostname: string = GITHUB_DEFAULT_DEPLOYMENT_HOSTNAME): Promise<string> {
|
||||||
|
this.ensureInitialized();
|
||||||
|
|
||||||
|
const key = this.generateMapKey(owner, deploymentHostname);
|
||||||
|
const installation = this.installationMap.get(key) as Installation | undefined;
|
||||||
|
if (!installation) {
|
||||||
|
throw new Error(`GitHub App Installation not found for ${key}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (installation.expiresAt < new Date().toISOString()) {
|
||||||
|
const octokitApp = this.octokitApps.get(installation.appId) as App;
|
||||||
|
const installationOctokit = await octokitApp.getInstallationOctokit(installation.id);
|
||||||
|
const auth = await installationOctokit.auth({ type: "installation" }) as { expires_at: string, token: string };
|
||||||
|
|
||||||
|
const newInstallation: Installation = {
|
||||||
|
...installation,
|
||||||
|
expiresAt: auth.expires_at,
|
||||||
|
token: auth.token
|
||||||
|
};
|
||||||
|
this.installationMap.set(key, newInstallation);
|
||||||
|
|
||||||
|
return newInstallation.token;
|
||||||
|
} else {
|
||||||
|
return installation.token;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public appsConfigured() {
|
||||||
|
return this.octokitApps.size > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private generateMapKey(owner: string, deploymentHostname: string): string {
|
||||||
|
return `${deploymentHostname}/${owner}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -18,7 +18,6 @@ const QUEUE_NAME = 'repoPermissionSyncQueue';
|
||||||
|
|
||||||
const logger = createLogger('repo-permission-syncer');
|
const logger = createLogger('repo-permission-syncer');
|
||||||
|
|
||||||
|
|
||||||
export class RepoPermissionSyncer {
|
export class RepoPermissionSyncer {
|
||||||
private queue: Queue<RepoPermissionSyncJob>;
|
private queue: Queue<RepoPermissionSyncJob>;
|
||||||
private worker: Worker<RepoPermissionSyncJob>;
|
private worker: Worker<RepoPermissionSyncJob>;
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,8 @@ import { BackendException, BackendError } from "@sourcebot/error";
|
||||||
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
|
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
|
||||||
import * as Sentry from "@sentry/node";
|
import * as Sentry from "@sentry/node";
|
||||||
import { env } from "./env.js";
|
import { env } from "./env.js";
|
||||||
|
import { GithubAppManager } from "./ee/githubAppManager.js";
|
||||||
|
import { hasEntitlement } from "@sourcebot/shared";
|
||||||
|
|
||||||
const logger = createLogger('github');
|
const logger = createLogger('github');
|
||||||
const GITHUB_CLOUD_HOSTNAME = "github.com";
|
const GITHUB_CLOUD_HOSTNAME = "github.com";
|
||||||
|
|
@ -55,6 +57,40 @@ export const createOctokitFromToken = async ({ token, url }: { token?: string, u
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper function to get an authenticated Octokit instance using GitHub App if available,
|
||||||
|
* otherwise falls back to the provided octokit instance.
|
||||||
|
*/
|
||||||
|
const getOctokitWithGithubApp = async (
|
||||||
|
octokit: Octokit,
|
||||||
|
owner: string,
|
||||||
|
url: string | undefined,
|
||||||
|
context: string
|
||||||
|
): Promise<Octokit> => {
|
||||||
|
if (!hasEntitlement('github-app') || !GithubAppManager.getInstance().appsConfigured()) {
|
||||||
|
return octokit;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const hostname = url ? new URL(url).hostname : GITHUB_CLOUD_HOSTNAME;
|
||||||
|
const token = await GithubAppManager.getInstance().getInstallationToken(owner, hostname);
|
||||||
|
const { octokit: octokitFromToken, isAuthenticated } = await createOctokitFromToken({
|
||||||
|
token,
|
||||||
|
url,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (isAuthenticated) {
|
||||||
|
return octokitFromToken;
|
||||||
|
} else {
|
||||||
|
logger.error(`Failed to authenticate with GitHub App for ${context}. Falling back to legacy token resolution.`);
|
||||||
|
return octokit;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(`Error getting GitHub App token for ${context}. Falling back to legacy token resolution.`, error);
|
||||||
|
return octokit;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
|
export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
|
||||||
const hostname = config.url ?
|
const hostname = config.url ?
|
||||||
new URL(config.url).hostname :
|
new URL(config.url).hostname :
|
||||||
|
|
@ -107,19 +143,19 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o
|
||||||
};
|
};
|
||||||
|
|
||||||
if (config.orgs) {
|
if (config.orgs) {
|
||||||
const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, octokit, signal);
|
const { validRepos, notFoundOrgs } = await getReposForOrgs(config.orgs, octokit, signal, config.url);
|
||||||
allRepos = allRepos.concat(validRepos);
|
allRepos = allRepos.concat(validRepos);
|
||||||
notFound.orgs = notFoundOrgs;
|
notFound.orgs = notFoundOrgs;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.repos) {
|
if (config.repos) {
|
||||||
const { validRepos, notFoundRepos } = await getRepos(config.repos, octokit, signal);
|
const { validRepos, notFoundRepos } = await getRepos(config.repos, octokit, signal, config.url);
|
||||||
allRepos = allRepos.concat(validRepos);
|
allRepos = allRepos.concat(validRepos);
|
||||||
notFound.repos = notFoundRepos;
|
notFound.repos = notFoundRepos;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.users) {
|
if (config.users) {
|
||||||
const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, octokit, signal);
|
const { validRepos, notFoundUsers } = await getReposOwnedByUsers(config.users, octokit, signal, config.url);
|
||||||
allRepos = allRepos.concat(validRepos);
|
allRepos = allRepos.concat(validRepos);
|
||||||
notFound.users = notFoundUsers;
|
notFound.users = notFoundUsers;
|
||||||
}
|
}
|
||||||
|
|
@ -178,11 +214,12 @@ export const getReposForAuthenticatedUser = async (visibility: 'all' | 'private'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: AbortSignal) => {
|
const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: AbortSignal, url?: string) => {
|
||||||
const results = await Promise.allSettled(users.map(async (user) => {
|
const results = await Promise.allSettled(users.map(async (user) => {
|
||||||
try {
|
try {
|
||||||
logger.debug(`Fetching repository info for user ${user}...`);
|
logger.debug(`Fetching repository info for user ${user}...`);
|
||||||
|
|
||||||
|
const octokitToUse = await getOctokitWithGithubApp(octokit, user, url, `user ${user}`);
|
||||||
const { durationMs, data } = await measure(async () => {
|
const { durationMs, data } = await measure(async () => {
|
||||||
const fetchFn = async () => {
|
const fetchFn = async () => {
|
||||||
let query = `user:${user}`;
|
let query = `user:${user}`;
|
||||||
|
|
@ -194,7 +231,7 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A
|
||||||
// the username as a parameter.
|
// the username as a parameter.
|
||||||
// @see: https://github.com/orgs/community/discussions/24382#discussioncomment-3243958
|
// @see: https://github.com/orgs/community/discussions/24382#discussioncomment-3243958
|
||||||
// @see: https://api.github.com/search/repositories?q=user:USERNAME
|
// @see: https://api.github.com/search/repositories?q=user:USERNAME
|
||||||
const searchResults = await octokit.paginate(octokit.rest.search.repos, {
|
const searchResults = await octokitToUse.paginate(octokitToUse.rest.search.repos, {
|
||||||
q: query,
|
q: query,
|
||||||
per_page: 100,
|
per_page: 100,
|
||||||
request: {
|
request: {
|
||||||
|
|
@ -237,13 +274,14 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal) => {
|
const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal, url?: string) => {
|
||||||
const results = await Promise.allSettled(orgs.map(async (org) => {
|
const results = await Promise.allSettled(orgs.map(async (org) => {
|
||||||
try {
|
try {
|
||||||
logger.info(`Fetching repository info for org ${org}...`);
|
logger.info(`Fetching repository info for org ${org}...`);
|
||||||
|
|
||||||
|
const octokitToUse = await getOctokitWithGithubApp(octokit, org, url, `org ${org}`);
|
||||||
const { durationMs, data } = await measure(async () => {
|
const { durationMs, data } = await measure(async () => {
|
||||||
const fetchFn = () => octokit.paginate(octokit.repos.listForOrg, {
|
const fetchFn = () => octokitToUse.paginate(octokitToUse.repos.listForOrg, {
|
||||||
org: org,
|
org: org,
|
||||||
per_page: 100,
|
per_page: 100,
|
||||||
request: {
|
request: {
|
||||||
|
|
@ -283,14 +321,15 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal) => {
|
const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal, url?: string) => {
|
||||||
const results = await Promise.allSettled(repoList.map(async (repo) => {
|
const results = await Promise.allSettled(repoList.map(async (repo) => {
|
||||||
try {
|
try {
|
||||||
const [owner, repoName] = repo.split('/');
|
const [owner, repoName] = repo.split('/');
|
||||||
logger.info(`Fetching repository info for ${repo}...`);
|
logger.info(`Fetching repository info for ${repo}...`);
|
||||||
|
|
||||||
|
const octokitToUse = await getOctokitWithGithubApp(octokit, owner, url, `repo ${repo}`);
|
||||||
const { durationMs, data: result } = await measure(async () => {
|
const { durationMs, data: result } = await measure(async () => {
|
||||||
const fetchFn = () => octokit.repos.get({
|
const fetchFn = () => octokitToUse.repos.get({
|
||||||
owner,
|
owner,
|
||||||
repo: repoName,
|
repo: repoName,
|
||||||
request: {
|
request: {
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ import { PromClient } from './promClient.js';
|
||||||
import { RepoManager } from './repoManager.js';
|
import { RepoManager } from './repoManager.js';
|
||||||
import { AppContext } from "./types.js";
|
import { AppContext } from "./types.js";
|
||||||
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
|
import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js";
|
||||||
|
import { GithubAppManager } from "./ee/githubAppManager.js";
|
||||||
|
|
||||||
|
|
||||||
const logger = createLogger('backend-entrypoint');
|
const logger = createLogger('backend-entrypoint');
|
||||||
|
|
@ -67,6 +68,11 @@ const promClient = new PromClient();
|
||||||
|
|
||||||
const settings = await getSettings(env.CONFIG_PATH);
|
const settings = await getSettings(env.CONFIG_PATH);
|
||||||
|
|
||||||
|
|
||||||
|
if (hasEntitlement('github-app')) {
|
||||||
|
await GithubAppManager.getInstance().init(prisma);
|
||||||
|
}
|
||||||
|
|
||||||
const connectionManager = new ConnectionManager(prisma, settings, redis);
|
const connectionManager = new ConnectionManager(prisma, settings, redis);
|
||||||
const repoManager = new RepoManager(prisma, settings, redis, promClient, context);
|
const repoManager = new RepoManager(prisma, settings, redis, promClient, context);
|
||||||
const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis);
|
const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis);
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,8 @@ import { getTokenFromConfig as getTokenFromConfigBase } from "@sourcebot/crypto"
|
||||||
import { BackendException, BackendError } from "@sourcebot/error";
|
import { BackendException, BackendError } from "@sourcebot/error";
|
||||||
import * as Sentry from "@sentry/node";
|
import * as Sentry from "@sentry/node";
|
||||||
import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
|
import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
|
||||||
|
import { GithubAppManager } from "./ee/githubAppManager.js";
|
||||||
|
import { hasEntitlement } from "@sourcebot/shared";
|
||||||
|
|
||||||
export const measure = async <T>(cb: () => Promise<T>) => {
|
export const measure = async <T>(cb: () => Promise<T>) => {
|
||||||
const start = Date.now();
|
const start = Date.now();
|
||||||
|
|
@ -125,6 +127,28 @@ export const fetchWithRetry = async <T>(
|
||||||
// may have their own token. This method will just pick the first connection that has a token (if one exists) and uses that. This
|
// may have their own token. This method will just pick the first connection that has a token (if one exists) and uses that. This
|
||||||
// may technically cause syncing to fail if that connection's token just so happens to not have access to the repo it's referencing.
|
// may technically cause syncing to fail if that connection's token just so happens to not have access to the repo it's referencing.
|
||||||
export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: PrismaClient, logger?: Logger): Promise<RepoAuthCredentials | undefined> => {
|
export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, db: PrismaClient, logger?: Logger): Promise<RepoAuthCredentials | undefined> => {
|
||||||
|
// If we have github apps configured we assume that we must use them for github service auth
|
||||||
|
if (repo.external_codeHostType === 'github' && hasEntitlement('github-app') && GithubAppManager.getInstance().appsConfigured()) {
|
||||||
|
const org = repo.displayName?.split('/')[0];
|
||||||
|
const deploymentHostname = new URL(repo.external_codeHostUrl).hostname;
|
||||||
|
if (!org || !deploymentHostname) {
|
||||||
|
throw new Error(`Failed to fetch GitHub App for repo ${repo.displayName}:Invalid repo displayName (${repo.displayName}) or deployment hostname (${deploymentHostname})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const token = await GithubAppManager.getInstance().getInstallationToken(org, deploymentHostname);
|
||||||
|
return {
|
||||||
|
hostUrl: repo.external_codeHostUrl,
|
||||||
|
token,
|
||||||
|
cloneUrlWithToken: createGitCloneUrlWithToken(
|
||||||
|
repo.cloneUrl,
|
||||||
|
{
|
||||||
|
username: 'x-access-token',
|
||||||
|
password: token
|
||||||
|
}
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (const { connection } of repo.connections) {
|
for (const { connection } of repo.connections) {
|
||||||
if (connection.connectionType === 'github') {
|
if (connection.connectionType === 'github') {
|
||||||
const config = connection.config as unknown as GithubConnectionConfig;
|
const config = connection.config as unknown as GithubConnectionConfig;
|
||||||
|
|
|
||||||
113
packages/schemas/src/v3/app.schema.ts
Normal file
113
packages/schemas/src/v3/app.schema.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
|
||||||
|
const schema = {
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "AppConfig",
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"type": "object",
|
||||||
|
"title": "GithubAppConfig",
|
||||||
|
"properties": {
|
||||||
|
"type": {
|
||||||
|
"const": "githubApp",
|
||||||
|
"description": "GitHub App Configuration"
|
||||||
|
},
|
||||||
|
"deploymentHostname": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "url",
|
||||||
|
"default": "github.com",
|
||||||
|
"description": "The hostname of the GitHub App deployment.",
|
||||||
|
"examples": [
|
||||||
|
"github.com",
|
||||||
|
"github.example.com"
|
||||||
|
],
|
||||||
|
"pattern": "^[^\\s/$.?#].[^\\s]*$"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The ID of the GitHub App."
|
||||||
|
},
|
||||||
|
"privateKey": {
|
||||||
|
"description": "The private key of the GitHub App.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"secret": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the secret that contains the token."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"secret"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"env": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"env"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"privateKeyPath": {
|
||||||
|
"description": "The path to the private key of the GitHub App.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"secret": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the secret that contains the token."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"secret"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"env": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"env"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"type",
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"required": [
|
||||||
|
"privateKey"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"required": [
|
||||||
|
"privateKeyPath"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
} as const;
|
||||||
|
export { schema as appSchema };
|
||||||
6
packages/schemas/src/v3/app.type.ts
Normal file
6
packages/schemas/src/v3/app.type.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
|
||||||
|
|
||||||
|
export type AppConfig = GithubAppConfig;
|
||||||
|
export type GithubAppConfig = {
|
||||||
|
[k: string]: unknown;
|
||||||
|
};
|
||||||
107
packages/schemas/src/v3/githubApp.schema.ts
Normal file
107
packages/schemas/src/v3/githubApp.schema.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
||||||
|
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
|
||||||
|
const schema = {
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"type": "object",
|
||||||
|
"title": "GithubAppConfig",
|
||||||
|
"properties": {
|
||||||
|
"type": {
|
||||||
|
"const": "githubApp",
|
||||||
|
"description": "GitHub App Configuration"
|
||||||
|
},
|
||||||
|
"deploymentHostname": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "url",
|
||||||
|
"default": "github.com",
|
||||||
|
"description": "The hostname of the GitHub App deployment.",
|
||||||
|
"examples": [
|
||||||
|
"github.com",
|
||||||
|
"github.example.com"
|
||||||
|
],
|
||||||
|
"pattern": "^[^\\s/$.?#].[^\\s]*$"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The ID of the GitHub App."
|
||||||
|
},
|
||||||
|
"privateKey": {
|
||||||
|
"description": "The private key of the GitHub App.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"secret": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the secret that contains the token."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"secret"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"env": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"env"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"privateKeyPath": {
|
||||||
|
"description": "The path to the private key of the GitHub App.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"secret": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the secret that contains the token."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"secret"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"env": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"env"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"type",
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"required": [
|
||||||
|
"privateKey"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"required": [
|
||||||
|
"privateKeyPath"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
} as const;
|
||||||
|
export { schema as githubAppSchema };
|
||||||
50
packages/schemas/src/v3/githubApp.type.ts
Normal file
50
packages/schemas/src/v3/githubApp.type.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
|
||||||
|
|
||||||
|
export type GithubAppConfig = {
|
||||||
|
/**
|
||||||
|
* GitHub App Configuration
|
||||||
|
*/
|
||||||
|
type: "githubApp";
|
||||||
|
/**
|
||||||
|
* The hostname of the GitHub App deployment.
|
||||||
|
*/
|
||||||
|
deploymentHostname?: string;
|
||||||
|
/**
|
||||||
|
* The ID of the GitHub App.
|
||||||
|
*/
|
||||||
|
id: string;
|
||||||
|
/**
|
||||||
|
* The private key of the GitHub App.
|
||||||
|
*/
|
||||||
|
privateKey?:
|
||||||
|
| {
|
||||||
|
/**
|
||||||
|
* The name of the secret that contains the token.
|
||||||
|
*/
|
||||||
|
secret: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
/**
|
||||||
|
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
|
||||||
|
*/
|
||||||
|
env: string;
|
||||||
|
};
|
||||||
|
/**
|
||||||
|
* The path to the private key of the GitHub App.
|
||||||
|
*/
|
||||||
|
privateKeyPath?:
|
||||||
|
| {
|
||||||
|
/**
|
||||||
|
* The name of the secret that contains the token.
|
||||||
|
*/
|
||||||
|
secret: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
/**
|
||||||
|
* The name of the environment variable that contains the token. Only supported in declarative connection configs.
|
||||||
|
*/
|
||||||
|
env: string;
|
||||||
|
};
|
||||||
|
} & {
|
||||||
|
[k: string]: unknown;
|
||||||
|
};
|
||||||
9
schemas/v3/app.json
Normal file
9
schemas/v3/app.json
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"title": "AppConfig",
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"$ref": "./githubApp.json"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
47
schemas/v3/githubApp.json
Normal file
47
schemas/v3/githubApp.json
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"type": "object",
|
||||||
|
"title": "GithubAppConfig",
|
||||||
|
"properties": {
|
||||||
|
"type": {
|
||||||
|
"const": "githubApp",
|
||||||
|
"description": "GitHub App Configuration"
|
||||||
|
},
|
||||||
|
"deploymentHostname": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "url",
|
||||||
|
"default": "github.com",
|
||||||
|
"description": "The hostname of the GitHub App deployment.",
|
||||||
|
"examples": [
|
||||||
|
"github.com",
|
||||||
|
"github.example.com"
|
||||||
|
],
|
||||||
|
"pattern": "^[^\\s/$.?#].[^\\s]*$"
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The ID of the GitHub App."
|
||||||
|
},
|
||||||
|
"privateKey": {
|
||||||
|
"$ref": "./shared.json#/definitions/Token",
|
||||||
|
"description": "The private key of the GitHub App."
|
||||||
|
},
|
||||||
|
"privateKeyPath": {
|
||||||
|
"$ref": "./shared.json#/definitions/Token",
|
||||||
|
"description": "The path to the private key of the GitHub App."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"type",
|
||||||
|
"id"
|
||||||
|
],
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"required": ["privateKey"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"required": ["privateKeyPath"]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue