connections qol improvements (#195)

* add client side polling to connections list

* properly fetch repo image url

* add client polling to connection management page, and add ability to sync failed connections

packages/backend/src/constants.ts

@@ -8,7 +8,7 @@ export const DEFAULT_SETTINGS: Settings = {
     autoDeleteStaleRepos: true,
     reindexIntervalMs: 1000 * 60,
     resyncConnectionPollingIntervalMs: 1000,
-    reindexRepoPollingInternvalMs: 1000,
+    reindexRepoPollingIntervalMs: 1000,
     indexConcurrencyMultiple: 3,
     configSyncConcurrencyMultiple: 3,
 }

packages/backend/src/repoCompileUtils.ts

@@ -30,6 +30,7 @@ export const compileGithubConfig = async (
             external_codeHostUrl: hostUrl,
             cloneUrl: cloneUrl.toString(),
             name: repoName,
+            imageUrl: repo.owner.avatar_url,
             isFork: repo.fork,
             isArchived: !!repo.archived,
             org: {
@@ -80,6 +81,7 @@ export const compileGitlabConfig = async (
             external_codeHostUrl: hostUrl,
             cloneUrl: cloneUrl.toString(),
             name: project.path_with_namespace,
+            imageUrl: project.avatar_url,
             isFork: isFork,
             isArchived: !!project.archived,
             org: {
@@ -118,7 +120,6 @@ export const compileGiteaConfig = async (
     const hostUrl = config.url ?? 'https://gitea.com';
 
     return giteaRepos.map((repo) => {
-        const repoUrl = `${hostUrl}/${repo.full_name}`;
         const cloneUrl = new URL(repo.clone_url!);
 
         const record: RepoData = {
@@ -127,6 +128,7 @@ export const compileGiteaConfig = async (
             external_codeHostUrl: hostUrl,
             cloneUrl: cloneUrl.toString(),
             name: repo.full_name!,
+            imageUrl: repo.owner?.avatar_url,
             isFork: repo.fork!,
             isArchived: !!repo.archived,
             org: {

packages/backend/src/repoManager.ts

@@ -49,7 +49,7 @@ export class RepoManager implements IRepoManager {
             this.fetchAndScheduleRepoIndexing();
             this.garbageCollectRepo();
 
-            await new Promise(resolve => setTimeout(resolve, this.settings.reindexRepoPollingInternvalMs));
+            await new Promise(resolve => setTimeout(resolve, this.settings.reindexRepoPollingIntervalMs));
         }
     }
 

packages/backend/src/types.ts

@@ -77,7 +77,7 @@ export type Settings = {
     /**
      * The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed.
      */
-    reindexRepoPollingInternvalMs: number;
+    reindexRepoPollingIntervalMs: number;
     /**
      * The multiple of the number of CPUs to use for indexing.
      */

packages/web/src/actions.ts

@@ -13,8 +13,8 @@ import { giteaSchema } from "@sourcebot/schemas/v3/gitea.schema";
 import { gerritSchema } from "@sourcebot/schemas/v3/gerrit.schema";
 import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
 import { encrypt } from "@sourcebot/crypto"
-import { getConnection } from "./data/connection";
-import { ConnectionSyncStatus, Prisma, Invite, OrgRole } from "@sourcebot/db";
+import { getConnection, getLinkedRepos } from "./data/connection";
+import { ConnectionSyncStatus, Prisma, Invite, OrgRole, Connection, Repo, Org } from "@sourcebot/db";
 import { headers } from "next/headers"
 import { getStripe } from "@/lib/stripe"
 import { getUser } from "@/data/user";
@@ -236,6 +236,41 @@ export const createConnection = async (name: string, type: string, connectionCon
             }
         }));
 
+export const getConnectionInfoAction = async (connectionId: number, domain: string): Promise<{ connection: Connection, linkedRepos: Repo[] } | ServiceError> =>
+    withAuth((session) =>
+        withOrgMembership(session, domain, async (orgId) => {
+            const connection = await getConnection(connectionId, orgId);
+            if (!connection) {
+                return notFound();
+            }
+
+            const linkedRepos = await getLinkedRepos(connectionId, orgId);
+
+            return {
+                connection,
+                linkedRepos: linkedRepos.map((repo) => repo.repo),
+            }
+        })
+    );
+
+export const getOrgFromDomainAction = async (domain: string): Promise<Org | ServiceError> =>
+    withAuth((session) =>
+        withOrgMembership(session, domain, async (orgId) => {
+            const org = await prisma.org.findUnique({
+                where: {
+                    id: orgId,
+                },
+            });
+
+            if (!org) {
+                return notFound();
+            }
+
+            return org;
+        })
+    );
+
+
 export const updateConnectionDisplayName = async (connectionId: number, name: string, domain: string): Promise<{ success: boolean } | ServiceError> =>
     withAuth((session) =>
         withOrgMembership(session, domain, async (orgId) => {
@@ -298,6 +333,36 @@ export const updateConnectionConfigAndScheduleSync = async (connectionId: number
             }
         }));
 
+export const flagConnectionForSync = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> =>
+    withAuth((session) =>
+        withOrgMembership(session, domain, async (orgId) => {
+            const connection = await getConnection(connectionId, orgId);
+            if (!connection || connection.orgId !== orgId) {
+                return notFound();
+            }
+
+            if (connection.syncStatus !== "FAILED") {
+                return {
+                    statusCode: StatusCodes.BAD_REQUEST,
+                    errorCode: ErrorCode.CONNECTION_NOT_FAILED,
+                    message: "Connection is not in a failed state. Cannot flag for sync.",
+                } satisfies ServiceError;
+            }
+
+            await prisma.connection.update({
+                where: { 
+                    id: connection.id,
+                },
+                data: {
+                    syncStatus: "SYNC_NEEDED",
+                }
+            });
+
+            return {
+                success: true,
+            }
+        }));
+
 export const deleteConnection = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> =>
     withAuth((session) =>
         withOrgMembership(session, domain, async (orgId) => {

packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx

@@ -23,6 +23,7 @@ export const RepoListItem = ({
             case RepoIndexingStatus.NEW:
                 return 'Waiting...';
             case RepoIndexingStatus.IN_INDEX_QUEUE:
+                return 'In index queue...';
             case RepoIndexingStatus.INDEXING:
                 return 'Indexing...';
             case RepoIndexingStatus.INDEXED:

packages/web/src/app/[domain]/connections/[id]/page.tsx

@@ -1,3 +1,5 @@
+"use client";
+
 import { NotFound } from "@/app/[domain]/components/notFound";
 import {
     Breadcrumb,
@@ -10,58 +12,108 @@ import {
 import { ScrollArea } from "@/components/ui/scroll-area";
 import { TabSwitcher } from "@/components/ui/tab-switcher";
 import { Tabs, TabsContent } from "@/components/ui/tabs";
-import { getConnection, getLinkedRepos } from "@/data/connection";
 import { ConnectionIcon } from "../components/connectionIcon";
 import { Header } from "../../components/header";
 import { ConfigSetting } from "./components/configSetting";
 import { DeleteConnectionSetting } from "./components/deleteConnectionSetting";
 import { DisplayNameSetting } from "./components/displayNameSetting";
 import { RepoListItem } from "./components/repoListItem";
-import { getOrgFromDomain } from "@/data/org";
-import { PageNotFound } from "../../components/pageNotFound";
+import { useParams, useSearchParams } from "next/navigation";
+import { useEffect, useState } from "react";
+import { Connection, Repo, Org } from "@sourcebot/db";
+import { getConnectionInfoAction, getOrgFromDomainAction, flagConnectionForSync  } from "@/actions";
+import { isServiceError } from "@/lib/utils";
+import { Button } from "@/components/ui/button";
+import { ReloadIcon } from "@radix-ui/react-icons";
+import { useToast } from "@/components/hooks/use-toast";
 
-interface ConnectionManagementPageProps {
-    params: {
-        id: string;
-        domain: string;
-    },
-    searchParams: {
-        tab?: string;
-    }
-}
+export default function ConnectionManagementPage() {
+    const params = useParams();
+    const searchParams = useSearchParams();
+    const { toast } = useToast();
+    const [org, setOrg] = useState<Org | null>(null);
+    const [connection, setConnection] = useState<Connection | null>(null);
+    const [linkedRepos, setLinkedRepos] = useState<Repo[]>([]);
+    const [loading, setLoading] = useState(true);
+    const [error, setError] = useState<string | null>(null);
 
-export default async function ConnectionManagementPage({
-    params,
-    searchParams,
-}: ConnectionManagementPageProps) {
-    const org = await getOrgFromDomain(params.domain);
-    if (!org) {
-        return <PageNotFound />
+    useEffect(() => {
+        const loadData = async () => {
+            try {
+                const orgResult = await getOrgFromDomainAction(params.domain as string);
+                if (isServiceError(orgResult)) {
+                    setError(orgResult.message);
+                    setLoading(false);
+                    return;
+                }
+                setOrg(orgResult);
+
+                const connectionId = Number(params.id);
+                if (isNaN(connectionId)) {
+                    setError("Invalid connection ID");
+                    setLoading(false);
+                    return;
+                }
+
+                const connectionInfoResult = await getConnectionInfoAction(connectionId, params.domain as string);
+                if (isServiceError(connectionInfoResult)) {
+                    setError(connectionInfoResult.message);
+                    setLoading(false);
+                    return;
+                }
+
+                connectionInfoResult.linkedRepos.sort((a, b) => {
+                    // Helper function to get priority of indexing status
+                    const getPriority = (status: string) => {
+                        switch (status) {
+                            case 'FAILED': return 0;
+                            case 'IN_INDEX_QUEUE':
+                            case 'INDEXING': return 1;
+                            case 'INDEXED': return 2;
+                            default: return 3;
+                        }
+                    };
+
+                    const priorityA = getPriority(a.repoIndexingStatus);
+                    const priorityB = getPriority(b.repoIndexingStatus);
+
+                    // First sort by priority
+                    if (priorityA !== priorityB) {
+                        return priorityA - priorityB;
+                    }
+
+                    // If same priority, sort by createdAt
+                    return new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime();
+                });
+
+                setConnection(connectionInfoResult.connection);
+                setLinkedRepos(connectionInfoResult.linkedRepos);
+                setLoading(false);
+            } catch (err) {
+                setError(err instanceof Error ? err.message : "An error occurred while loading the connection. If the problem persists, please contact us at team@sourcebot.dev");
+                setLoading(false);
+            }
+        };
+
+        loadData();
+        const intervalId = setInterval(loadData, 1000);
+        return () => clearInterval(intervalId);
+    }, [params.domain, params.id]);
+
+    if (loading) {
+        return <div>Loading...</div>;
     }
 
-    const connectionId = Number(params.id);
-    if (isNaN(connectionId)) {
+    if (error || !org || !connection) {
         return (
             <NotFound
                 className="flex w-full h-full items-center justify-center"
-                message="Connection not found"
+                message={error || "Not found"}
             />
-        )
+        );
     }
 
-    const connection = await getConnection(Number(params.id), org.id);
-    if (!connection) {
-        return (
-            <NotFound
-                className="flex w-full h-full items-center justify-center"
-                message="Connection not found"
-            />
-        )
-    }
-
-    const linkedRepos = await getLinkedRepos(connectionId, org.id);
-
-    const currentTab = searchParams.tab || "overview";
+    const currentTab = searchParams.get("tab") || "overview";
 
     return (
         <Tabs
@@ -116,7 +168,30 @@ export default async function ConnectionManagementPage({
                         </div>
                         <div className="rounded-lg border border-border p-4 bg-background">
                             <h2 className="text-sm font-medium text-muted-foreground">Status</h2>
-                            <p className="mt-2 text-sm">{connection.syncStatus}</p>
+                            <div className="flex items-center gap-2">
+                                <p className="mt-2 text-sm">{connection.syncStatus}</p>
+                                {connection.syncStatus === "FAILED" && (
+                                    <Button
+                                        variant="outline"
+                                        size="sm" 
+                                        className="mt-2 rounded-full"
+                                        onClick={async () => {
+                                            const result = await flagConnectionForSync(connection.id, params.domain as string);
+                                            if (isServiceError(result)) {
+                                                toast({
+                                                    description: `❌ Failed to flag connection for sync. Reason: ${result.message}`,
+                                                })
+                                            } else {
+                                                toast({
+                                                    description: "✅ Connection flagged for sync.",
+                                                })
+                                            }
+                                        }}
+                                    >
+                                        <ReloadIcon className="h-4 w-4" />
+                                    </Button>
+                                )}
+                            </div>
                         </div>
                     </div>
                 </div>
@@ -127,12 +202,12 @@ export default async function ConnectionManagementPage({
                     <div className="flex flex-col gap-4">
                         {linkedRepos
                             .sort((a, b) => {
-                                const aIndexedAt = a.repo.indexedAt ?? new Date();
-                                const bIndexedAt = b.repo.indexedAt ?? new Date();
+                                const aIndexedAt = a.indexedAt ?? new Date();
+                                const bIndexedAt = b.indexedAt ?? new Date();
 
                                 return bIndexedAt.getTime() - aIndexedAt.getTime();
                             })
-                            .map(({ repo }) => (
+                            .map((repo) => (
                                 <RepoListItem
                                     key={repo.id}
                                     imageUrl={repo.imageUrl ?? undefined}
@@ -162,6 +237,5 @@ export default async function ConnectionManagementPage({
                 />
             </TabsContent>
         </Tabs>
-
-    )
+    );
 }

packages/web/src/app/[domain]/connections/components/connectionList/index.tsx

@@ -1,47 +1,84 @@
+"use client";
+import { useDomain } from "@/hooks/useDomain";
 import { ConnectionListItem } from "./connectionListItem";
 import { cn } from "@/lib/utils";
+import { useEffect } from "react";
 import { InfoCircledIcon } from "@radix-ui/react-icons";
+import { useState } from "react";
 import { ConnectionSyncStatus } from "@sourcebot/db";
-
+import { getConnections } from "@/actions";
+import { isServiceError } from "@/lib/utils";
 
 interface ConnectionListProps {
-    connections: {
-        id: number,
-        name: string,
-        connectionType: string,
-        syncStatus: ConnectionSyncStatus,
-        updatedAt: Date,
-        syncedAt?: Date
-    }[];
     className?: string;
 }
 
 export const ConnectionList = ({
-    connections,
     className,
 }: ConnectionListProps) => {
+    const domain = useDomain();
+    const [connections, setConnections] = useState<{
+        id: number;
+        name: string;
+        connectionType: string;
+        syncStatus: ConnectionSyncStatus;
+        updatedAt: Date;
+        syncedAt?: Date;
+    }[]>([]);
+    const [loading, setLoading] = useState(true);
+    const [error, setError] = useState<string | null>(null);
+
+    useEffect(() => {   
+        const fetchConnections = async () => {
+            try {
+                const result = await getConnections(domain);
+                if (isServiceError(result)) {
+                    setError(result.message);
+                } else {
+                    setConnections(result);
+                }
+                setLoading(false);
+            } catch (err) {
+                setError(err instanceof Error ? err.message : 'An error occured while fetching connections. If the problem persists, please contact us at team@sourcebot.dev');
+                setLoading(false);
+            }
+        };
+
+        fetchConnections();
+        const intervalId = setInterval(fetchConnections, 1000);
+        return () => clearInterval(intervalId);
+    }, [domain]);
 
     return (
         <div className={cn("flex flex-col gap-4", className)}>
-            {connections.length > 0 ? connections
-                .sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime())
-                .map((connection) => (
-                    <ConnectionListItem
-                        key={connection.id}
-                        id={connection.id.toString()}
-                        name={connection.name}
-                        type={connection.connectionType}
-                        status={connection.syncStatus}
-                        editedAt={connection.updatedAt}
-                        syncedAt={connection.syncedAt ?? undefined}
-                    />
-                ))
-                : (
-                    <div className="flex flex-col items-center justify-center border rounded-md p-4 h-full">
-                        <InfoCircledIcon className="w-7 h-7" />
-                        <h2 className="mt-2 font-medium">No connections</h2>
-                    </div>
-                )}
+            {loading ? (
+                <div className="flex flex-col items-center justify-center border rounded-md p-4 h-full">
+                    <p>Loading connections...</p>
+                </div>
+            ) : error ? (
+                <div className="flex flex-col items-center justify-center border rounded-md p-4 h-full">
+                    <p>Error loading connections: {error}</p>
+                </div>
+            ) : connections.length > 0 ? (
+                connections
+                    .sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime())
+                    .map((connection) => (
+                        <ConnectionListItem
+                            key={connection.id}
+                            id={connection.id.toString()}
+                            name={connection.name}
+                            type={connection.connectionType}
+                            status={connection.syncStatus}
+                            editedAt={connection.updatedAt}
+                            syncedAt={connection.syncedAt ?? undefined}
+                        />
+                    ))
+            ) : (
+                <div className="flex flex-col items-center justify-center border rounded-md p-4 h-full">
+                    <InfoCircledIcon className="w-7 h-7" />
+                    <h2 className="mt-2 font-medium">No connections</h2>
+                </div>
+            )}
         </div>
     )
 }

packages/web/src/app/[domain]/connections/page.tsx

@@ -18,7 +18,6 @@ export default async function ConnectionsPage({ params: { domain } }: { params:
             </Header>
             <div className="flex flex-col md:flex-row gap-4">
                 <ConnectionList
-                    connections={connections}
                     className="md:w-3/4"
                 />
                 <NewConnectionCard

packages/web/src/lib/errorCodes.ts

@@ -14,4 +14,5 @@ export enum ErrorCode {
     MEMBER_NOT_FOUND = 'MEMBER_NOT_FOUND',
     INVALID_CREDENTIALS = 'INVALID_CREDENTIALS',
     MEMBER_NOT_OWNER = 'MEMBER_NOT_OWNER',
+    CONNECTION_NOT_FAILED = 'CONNECTION_NOT_FAILED',
 }