ti.sil/sourcebot
1
0
Fork 0
mirror of https://github.com/sourcebot-dev/sourcebot.git synced 2025-12-12 04:15:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

fix: make gerrit repo fetching paginated (#114)

Browse source 
* fix: make gerrit repo fetching paginated

In some cases gerrit will limit the number of projects returned by
/projects endpoint, forcing the client to paginate their request to get
all projects.
This fulfills this requirement to get all projects

* Add some more metadata repo projects to ignore list

---------

Co-authored-by: Brendan Kellam <bshizzle1234@gmail.com>
This commit is contained in:
Konrad Staniszewski 2024-12-07 10:45:46 -08:00 committed by GitHub
parent 77198bd773
commit f71a83a941
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 30 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
packages/backend/src/gerrit.ts
View file

@ -23,7 +23,7 @@ interface GerritWebLink {
const logger = createLogger('Gerrit');
export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext) => {
export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext): Promise<GitRepository[]> => {
const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
const hostname = new URL(config.url).hostname;
@ -35,6 +35,8 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
// exclude "All-Projects" and "All-Users" projects
delete projects['All-Projects'];
delete projects['All-Users'];
delete projects['All-Avatars']
delete projects['All-Archived-Projects']
logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
@ -90,20 +92,35 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
};
const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
const projectsEndpoint = `${url}projects/`;
logger.debug(`Fetching projects from Gerrit at ${projectsEndpoint}...`);
const response = await fetch(projectsEndpoint);
let allProjects: GerritProjects = {};
let start = 0; // Start offset for pagination
let hasMoreProjects = true;
if (!response.ok) {
throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`);
while (hasMoreProjects) {
const endpointWithParams = `${projectsEndpoint}?S=${start}`;
logger.debug(`Fetching projects from Gerrit at ${endpointWithParams}`);
const response = await fetch(endpointWithParams);
if (!response.ok) {
throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`);
}
const text = await response.text();
const jsonText = text.replace(")]}'\n", ''); // Remove XSSI protection prefix
const data: GerritProjects = JSON.parse(jsonText);
// Merge the current batch of projects with allProjects
Object.assign(allProjects, data);
// Check if there are more projects to fetch
hasMoreProjects = Object.values(data).some(
(project) => (project as any)._more_projects === true
);
// Update the offset based on the number of projects in the current response
start += Object.keys(data).length;
}
const text = await response.text();
// Gerrit prepends ")]}'\n" to prevent XSSI attacks; remove it
// https://gerrit-review.googlesource.com/Documentation/rest-api.html
const jsonText = text.replace(")]}'\n", '');
const data = JSON.parse(jsonText);
return data;
return allProjects;
};

1
packages/backend/src/schemas/v2.ts
View file

@ -192,7 +192,6 @@ export interface GerritConfig {
*/
projects?: string[];
};
revisions?: GitRevisions;
}
export interface LocalConfig {
/**