experimenting with concurrent syncing using redis & bull

packages/backend/package.json

@@ -9,7 +9,8 @@
         "dev": "export PATH=\"$PWD/../../bin:$PATH\" && export CTAGS_COMMAND=ctags && node ./dist/index.js",
         "build": "yarn generate:types && tsc",
         "generate:types": "tsx tools/generateTypes.ts",
-        "test": "vitest --config ./vitest.config.ts"
+        "test": "vitest --config ./vitest.config.ts",
+        "playground": "tsx tools/playground.ts"
     },
     "devDependencies": {
         "@types/argparse": "^2.0.16",
@@ -25,6 +26,7 @@
         "@gitbeaker/rest": "^40.5.1",
         "@octokit/rest": "^21.0.2",
         "argparse": "^2.0.1",
+        "bull": "^4.16.5",
         "cross-fetch": "^4.0.0",
         "dotenv": "^16.4.5",
         "gitea-js": "^1.22.0",

packages/backend/tools/playground.ts

@@ -0,0 +1,83 @@
+import Bull from 'bull';
+import { simpleGit } from 'simple-git';
+import { createLogger } from "../src/logger.js";
+import { Repository } from "../src/types.js";
+import { existsSync } from 'fs';
+
+
+const logger = createLogger('git-service');
+
+const CONCURRENCY = 5;
+
+type CloneJob = {
+    cloneUrl: string;
+    outputPath: string;
+}
+
+interface GitService {
+    clone (url: string, path: string): void;
+    onCloneCompleted (callback: (cloneUrl: string) => void): void;
+
+    syncRepository (repo: Repository): void;
+}
+
+
+const createGitService = (): GitService => {
+    const queue = new Bull<CloneJob>('clone-queue');
+    const git = simpleGit({
+        progress: (event) => {
+            console.log(`git.${event.method} ${event.stage} stage ${event.progress}% complete`)
+        }
+    });
+
+    queue.process(CONCURRENCY, async ({ data }) => {
+        logger.debug(`Cloning ${data.cloneUrl} to ${data.outputPath}`);
+        await git.clone(
+            data.cloneUrl,
+            data.outputPath,
+            ['--bare']
+        );
+    });
+
+    return {
+        clone: async (url: string, path: string) => {
+            const job = await queue.add({
+                cloneUrl: url,
+                outputPath: path,
+            })
+            console.log(job);
+        },
+        onCloneCompleted: (callback) => {
+            queue.on('completed', (job) => {
+                callback(job.data.cloneUrl);
+            })
+            queue.on('failed', (job) => {
+                console.error(`Clone failed: ${job.data.cloneUrl}`);
+            });
+        },
+        
+        syncRepository: (repo) => {
+            if (existsSync(repo.path)) {
+                
+            }
+        }
+    }
+}
+
+// -----
+
+
+const gitService = createGitService();
+gitService.onCloneCompleted((url) => {
+    console.log(`Clone completed: ${url}`);
+});
+
+console.log('hello world');
+gitService.clone('https://github.com/sourcegraph/zoekt', '/tmp/zoekt');
+// gitService.clone('https://github.com/facebook/react', '/tmp/react');
+// gitService.clone('https://github.com/microsoft/typescript', '/tmp/typescript');
+// gitService.clone('https://github.com/nodejs/node', '/tmp/node');
+// gitService.clone('https://github.com/kubernetes/kubernetes', '/tmp/kubernetes');
+// gitService.clone('https://github.com/tensorflow/tensorflow', '/tmp/tensorflow');
+
+