sourcebot/packages/backend/src/github.ts

import { Octokit } from "@octokit/rest";
import { GitHubConfig } from "./schemas/v2.js";
import { createLogger } from "./logger.js";
import { AppContext, GitRepository } from "./types.js";
import path from 'path';
import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from "./utils.js";
import micromatch from "micromatch";

const logger = createLogger("GitHub");

type OctokitRepository = {
    name: string,
    id: number,
    full_name: string,
    fork: boolean,
    private: boolean,
    html_url: string,
    clone_url?: string,
    stargazers_count?: number,
    watchers_count?: number,
    subscribers_count?: number,
    forks_count?: number,
    archived?: boolean,
}

export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: AbortSignal, ctx: AppContext) => {
    const token = config.token ? getTokenFromConfig(config.token, ctx) : undefined;

    const octokit = new Octokit({
        auth: token,
        ...(config.url ? {
            baseUrl: `${config.url}/api/v3`
        } : {}),
    });

    let allRepos: OctokitRepository[] = [];

    if (config.orgs) {
        const _repos = await getReposForOrgs(config.orgs, octokit, signal);
        allRepos = allRepos.concat(_repos);
    }

    if (config.repos) {
        const _repos = await getRepos(config.repos, octokit, signal);
        allRepos = allRepos.concat(_repos);
    }

    if (config.users) {
        const isAuthenticated = config.token !== undefined;
        const _repos = await getReposOwnedByUsers(config.users, isAuthenticated, octokit, signal);
        allRepos = allRepos.concat(_repos);
    }

    // Marshall results to our type
    let repos: GitRepository[] = allRepos
        .filter((repo) => {
            if (!repo.clone_url) {
                logger.warn(`Repository ${repo.name} missing property 'clone_url'. Excluding.`)
                return false;
            }
            return true;
        })
        .map((repo) => {
            const hostname = config.url ? new URL(config.url).hostname : 'github.com';
            const repoId = `${hostname}/${repo.full_name}`;
            const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));

            const cloneUrl = new URL(repo.clone_url!);
            if (token) {
                cloneUrl.username = token;
            }
            
            return {
                vcs: 'git',
                name: repo.full_name,
                id: repoId,
                cloneUrl: cloneUrl.toString(),
                path: repoPath,
                isStale: false,
                isFork: repo.fork,
                isArchived: !!repo.archived,
                gitConfigMetadata: {
                    'zoekt.web-url-type': 'github',
                    'zoekt.web-url': repo.html_url,
                    'zoekt.name': repoId,
                    'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
                    'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
                    'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
                    'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
                    'zoekt.archived': marshalBool(repo.archived),
                    'zoekt.fork': marshalBool(repo.fork),
                    'zoekt.public': marshalBool(repo.private === false)
                },
                branches: [],
                tags: [],
            } satisfies GitRepository;
        });

    if (config.exclude) {
        if (!!config.exclude.forks) {
            repos = excludeForkedRepos(repos, logger);
        }

        if (!!config.exclude.archived) {
            repos = excludeArchivedRepos(repos, logger);
        }

        if (config.exclude.repos) {
            repos = excludeReposByName(repos, config.exclude.repos, logger);
        }
    }

    logger.debug(`Found ${repos.length} total repositories.`);

    if (config.revisions) {
        if (config.revisions.branches) {
            const branchGlobs = config.revisions.branches;
            repos = await Promise.all(
                repos.map(async (repo) => {
                    const [owner, name] = repo.name.split('/');
                    let branches = (await getBranchesForRepo(owner, name, octokit, signal)).map(branch => branch.name);
                    branches = micromatch.match(branches, branchGlobs);

                    return {
                        ...repo,
                        branches,
                    };
                })
            )
        }

        if (config.revisions.tags) {
            const tagGlobs = config.revisions.tags;
            repos = await Promise.all(
                repos.map(async (repo) => {
                    const [owner, name] = repo.name.split('/');
                    let tags = (await getTagsForRepo(owner, name, octokit, signal)).map(tag => tag.name);
                    tags = micromatch.match(tags, tagGlobs);

                    return {
                        ...repo,
                        tags,
                    };
                })
            )
        }
    }

    return repos;
}

const getTagsForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
    logger.debug(`Fetching tags for repo ${owner}/${repo}...`);

    const { durationMs, data: tags } = await measure(() => octokit.paginate(octokit.repos.listTags, {
        owner,
        repo,
        per_page: 100,
        request: {
            signal
        }
    }));

    logger.debug(`Found ${tags.length} tags for repo ${owner}/${repo} in ${durationMs}ms`);
    return tags;
}

const getBranchesForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
    logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
    const { durationMs, data: branches } = await measure(() => octokit.paginate(octokit.repos.listBranches, {
        owner,
        repo,
        per_page: 100,
        request: {
            signal
        }
    }));
    logger.debug(`Found ${branches.length} branches for repo ${owner}/${repo} in ${durationMs}ms`);
    return branches;
}


const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, octokit: Octokit, signal: AbortSignal) => {
    // @todo : error handling
    const repos = (await Promise.all(users.map(async (user) => {
        logger.debug(`Fetching repository info for user ${user}...`);
        const start = Date.now();

        const result = await (() => {
            if (isAuthenticated) {
                return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
                    username: user,
                    visibility: 'all',
                    affiliation: 'owner',
                    per_page: 100,
                    request: {
                        signal,
                    },
                });
            } else {
                return octokit.paginate(octokit.repos.listForUser, {
                    username: user,
                    per_page: 100,
                    request: {
                        signal,
                    },
                });
            }
        })();

        const duration = Date.now() - start;
        logger.debug(`Found ${result.length} owned by user ${user} in ${duration}ms.`);

        return result;
    }))).flat();

    return repos;
}

const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal) => {
    const repos = (await Promise.all(orgs.map(async (org) => {
        logger.debug(`Fetching repository info for org ${org}...`);
        const start = Date.now();

        const result = await octokit.paginate(octokit.repos.listForOrg, {
            org: org,
            per_page: 100,
            request: {
                signal
            }
        });

        const duration = Date.now() - start;
        logger.debug(`Found ${result.length} in org ${org} in ${duration}ms.`);

        return result;
    }))).flat();

    return repos;
}

const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal) => {
    const repos = await Promise.all(repoList.map(async (repo) => {
        logger.debug(`Fetching repository info for ${repo}...`);
        const start = Date.now();

        const [owner, repoName] = repo.split('/');
        const result = await octokit.repos.get({
            owner,
            repo: repoName,
            request: {
                signal
            }
        });

        const duration = Date.now() - start;
        logger.debug(`Found info for repository ${repo} in ${duration}ms`);

        return result.data;
    }));

    return repos;
}