ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-12 12:25:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

fix: make gerrit repo fetching paginated (#114)

Browse source 
* fix: make gerrit repo fetching paginated

In some cases gerrit will limit the number of projects returned by
/projects endpoint, forcing the client to paginate their request to get
all projects.
This fulfills this requirement to get all projects

* Add some more metadata repo projects to ignore list

---------

Co-authored-by: Brendan Kellam <bshizzle1234@gmail.com>
This commit is contained in:
Konrad Staniszewski 2024-12-07 10:45:46 -08:00 committed by GitHub
parent 77198bd773
commit f71a83a941
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 30 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
packages/backend/src/gerrit.ts
View file

@ -23,7 +23,7 @@ interface GerritWebLink {
const logger = createLogger('Gerrit'); const logger = createLogger('Gerrit');
export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext) => { export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext): Promise<GitRepository[]> => {
const url = config.url.endsWith('/') ? config.url : `${config.url}/`; const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
const hostname = new URL(config.url).hostname; const hostname = new URL(config.url).hostname;
@ -35,6 +35,8 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
// exclude "All-Projects" and "All-Users" projects // exclude "All-Projects" and "All-Users" projects
delete projects['All-Projects']; delete projects['All-Projects'];
delete projects['All-Users']; delete projects['All-Users'];
delete projects['All-Avatars']
delete projects['All-Archived-Projects']
logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`); logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
@ -90,20 +92,35 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
}; };
const fetchAllProjects = async (url: string): Promise<GerritProjects> => { const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
const projectsEndpoint = `${url}projects/`; const projectsEndpoint = `${url}projects/`;
logger.debug(`Fetching projects from Gerrit at ${projectsEndpoint}...`); let allProjects: GerritProjects = {};
const response = await fetch(projectsEndpoint); let start = 0; // Start offset for pagination
let hasMoreProjects = true;
while (hasMoreProjects) {
const endpointWithParams = `${projectsEndpoint}?S=${start}`;
logger.debug(`Fetching projects from Gerrit at ${endpointWithParams}`);
const response = await fetch(endpointWithParams);
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`); throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`);
} }
const text = await response.text(); const text = await response.text();
const jsonText = text.replace(")]}'\n", ''); // Remove XSSI protection prefix
const data: GerritProjects = JSON.parse(jsonText);
// Gerrit prepends ")]}'\n" to prevent XSSI attacks; remove it // Merge the current batch of projects with allProjects
// https://gerrit-review.googlesource.com/Documentation/rest-api.html Object.assign(allProjects, data);
const jsonText = text.replace(")]}'\n", '');
const data = JSON.parse(jsonText); // Check if there are more projects to fetch
return data; hasMoreProjects = Object.values(data).some(
(project) => (project as any)._more_projects === true
);
// Update the offset based on the number of projects in the current response
start += Object.keys(data).length;
}
return allProjects;
}; };

1
packages/backend/src/schemas/v2.ts
View file

@ -192,7 +192,6 @@ export interface GerritConfig {
*/ */
projects?: string[]; projects?: string[];
}; };
revisions?: GitRevisions;
} }
export interface LocalConfig { export interface LocalConfig {
/** /**