update perm syncing docs

* fix: add support for anyuid to Dockerfile

* changelog

---------

Co-authored-by: Cade Schlaefli <cade.schlaefli@mouser.com>
Co-authored-by: Brendan Kellam <bshizzle1234@gmail.com>

.env.development

@@ -6,8 +6,6 @@ DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
 ZOEKT_WEBSERVER_URL="http://localhost:6070"
 # The command to use for generating ctags.
 CTAGS_COMMAND=ctags
-# logging, strict
-SRC_TENANT_ENFORCEMENT_MODE=strict
 
 # Auth.JS
 # You can generate a new secret with:
@@ -23,7 +21,7 @@ AUTH_URL="http://localhost:3000"
 
 DATA_CACHE_DIR=${PWD}/.sourcebot  # Path to the sourcebot cache dir (ex. ~/sourcebot/.sourcebot)
 SOURCEBOT_PUBLIC_KEY_PATH=${PWD}/public.pem
-# CONFIG_PATH=${PWD}/config.json # Path to the sourcebot config file (if one exists)
+CONFIG_PATH=${PWD}/config.json # Path to the sourcebot config file (if one exists)
 
 # Email
 # EMAIL_FROM_ADDRESS="" # The from address for transactional emails.
@@ -31,7 +29,6 @@ SOURCEBOT_PUBLIC_KEY_PATH=${PWD}/public.pem
 
 # PostHog
 # POSTHOG_PAPIK=""
-# NEXT_PUBLIC_POSTHOG_PAPIK=""
 
 # Sentry
 # SENTRY_BACKEND_DSN=""

.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,4 @@
 contact_links:
   - name: 👾 Discord
-    url: https://discord.gg/GbXMEM5H
+    url: https://discord.gg/HDScTs3ptP
     about: Something else? Join the Discord!

.github/workflows/_gcp-deploy.yml

@@ -55,7 +55,6 @@ jobs:
             ${{ env.IMAGE_PATH }}:latest
           build-args: |
             NEXT_PUBLIC_SOURCEBOT_VERSION=${{ github.ref_name }}
-            NEXT_PUBLIC_POSTHOG_PAPIK=${{ vars.NEXT_PUBLIC_POSTHOG_PAPIK }}
             NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT=${{ vars.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT }}
             NEXT_PUBLIC_SENTRY_ENVIRONMENT=${{ vars.NEXT_PUBLIC_SENTRY_ENVIRONMENT }}
             NEXT_PUBLIC_SENTRY_WEBAPP_DSN=${{ vars.NEXT_PUBLIC_SENTRY_WEBAPP_DSN }}

.github/workflows/ghcr-publish.yml

@@ -77,7 +77,6 @@ jobs:
           outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true,annotation.org.opencontainers.image.description=Blazingly fast code search
           build-args: |
             NEXT_PUBLIC_SOURCEBOT_VERSION=${{ github.ref_name }}
-            NEXT_PUBLIC_POSTHOG_PAPIK=${{ vars.NEXT_PUBLIC_POSTHOG_PAPIK }}
             
       - name: Export digest
         run: |

CHANGELOG.md

@@ -7,6 +7,78 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- Fixed review agent so that it works with GHES instances [#611](https://github.com/sourcebot-dev/sourcebot/pull/611)
+
+### Added
+- Added support for arbitrary user IDs required for OpenShift. [#658](https://github.com/sourcebot-dev/sourcebot/pull/658) 
+
+### Updated
+- Improved error messages in file source api. [#665](https://github.com/sourcebot-dev/sourcebot/pull/665)
+
+## [4.10.2] - 2025-12-04
+
+### Fixed
+- Fixed issue where the disable telemetry flag was not being respected for web server telemetry. [#657](https://github.com/sourcebot-dev/sourcebot/pull/657)
+
+## [4.10.1] - 2025-12-03
+
+### Added
+- Added `ALWAYS_INDEX_FILE_PATTERNS` environment variable to allow specifying a comma seperated list of glob patterns matching file paths that should always be indexed, regardless of size or # of trigrams. [#631](https://github.com/sourcebot-dev/sourcebot/pull/631)
+- Added button to explore menu to toggle cross-repository search. [#647](https://github.com/sourcebot-dev/sourcebot/pull/647)
+- Added server side telemetry for search metrics. [#652](https://github.com/sourcebot-dev/sourcebot/pull/652)
+
+### Fixed
+- Fixed issue where single quotes could not be used in search queries. [#629](https://github.com/sourcebot-dev/sourcebot/pull/629)
+- Fixed issue where files with special characters would fail to load. [#636](https://github.com/sourcebot-dev/sourcebot/issues/636)
+- Fixed Ask performance issues. [#632](https://github.com/sourcebot-dev/sourcebot/pull/632)
+- Fixed regression where creating a new Ask thread when unauthenticated would result in a 404. [#641](https://github.com/sourcebot-dev/sourcebot/pull/641)
+- Updated react and next package versions to fix CVE 2025-55182. [#654](https://github.com/sourcebot-dev/sourcebot/pull/654)
+
+### Changed
+- Changed the default behaviour for code nav to scope references & definitions search to the current repository. [#647](https://github.com/sourcebot-dev/sourcebot/pull/647)
+
+## [4.10.0] - 2025-11-24
+
+### Added
+- Added support for streaming code search results. [#623](https://github.com/sourcebot-dev/sourcebot/pull/623)
+- Added buttons to toggle case sensitivity and regex patterns. [#623](https://github.com/sourcebot-dev/sourcebot/pull/623)
+- Added counts to members, requets, and invites tabs in the members settings. [#621](https://github.com/sourcebot-dev/sourcebot/pull/621)
+- [Sourcebot EE] Add support for Authentik as a identity provider. [#627](https://github.com/sourcebot-dev/sourcebot/pull/627)
+
+### Changed
+- Changed the default search behaviour to match patterns as substrings and **not** regular expressions. Regular expressions can be used by toggling the regex button in search bar. [#623](https://github.com/sourcebot-dev/sourcebot/pull/623)
+- Renamed `public` query prefix to `visibility`. Allowed values for `visibility` are `public`, `private`, and `any`. [#623](https://github.com/sourcebot-dev/sourcebot/pull/623)
+- Changed `archived` query prefix to accept values `yes`, `no`, and `only`. [#623](https://github.com/sourcebot-dev/sourcebot/pull/623)
+
+### Removed
+- Removed `case` query prefix. [#623](https://github.com/sourcebot-dev/sourcebot/pull/623)
+- Removed `branch` and `b` query prefixes. Please use `rev:` instead. [#623](https://github.com/sourcebot-dev/sourcebot/pull/623)
+- Removed `regex` query prefix. [#623](https://github.com/sourcebot-dev/sourcebot/pull/623)
+
+### Fixed
+- Fixed spurious infinite loads with explore panel, file tree, and file search command. [#617](https://github.com/sourcebot-dev/sourcebot/pull/617)
+- Wipe search context on init if entitlement no longer exists [#618](https://github.com/sourcebot-dev/sourcebot/pull/618)
+- Fixed Bitbucket repository exclusions not supporting glob patterns. [#620](https://github.com/sourcebot-dev/sourcebot/pull/620)
+- Fixed issue where the repo driven permission syncer was attempting to sync public repositories. [#624](https://github.com/sourcebot-dev/sourcebot/pull/624)
+- Fixed issue where worker would not shutdown while a permission sync job (repo or user) was in progress. [#624](https://github.com/sourcebot-dev/sourcebot/pull/624)
+
+## [4.9.2] - 2025-11-13
+
+### Changed
+- Bumped the default requested search result count from 5k to 10k after optimization pass. [#615](https://github.com/sourcebot-dev/sourcebot/pull/615)
+
+### Fixed
+- Fixed incorrect shutdown of PostHog SDK in the worker. [#609](https://github.com/sourcebot-dev/sourcebot/pull/609)
+- Fixed race condition in job schedulers. [#607](https://github.com/sourcebot-dev/sourcebot/pull/607)
+- Fixed connection sync jobs getting stuck in pending or in progress after restarting the worker. [#612](https://github.com/sourcebot-dev/sourcebot/pull/612)
+- Fixed issue where connections would always sync on startup, regardless if they changed or not. [#613](https://github.com/sourcebot-dev/sourcebot/pull/613)
+- Fixed performance bottleneck in search api. Result is a order of magnitutde improvement to average search time according to benchmarks. [#615](https://github.com/sourcebot-dev/sourcebot/pull/615) 
+
+### Added
+- Added force resync buttons for connections and repositories. [#610](https://github.com/sourcebot-dev/sourcebot/pull/610)
+- Added environment variable to configure default search result count. [#616](https://github.com/sourcebot-dev/sourcebot/pull/616)
+
 ## [4.9.1] - 2025-11-07
 
 ### Added

Dockerfile

@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 # ------ Global scope variables ------
 
 # Set of global build arguments.
@@ -8,11 +9,6 @@
 # @see: https://docs.docker.com/build/building/variables/#scoping
 
 ARG NEXT_PUBLIC_SOURCEBOT_VERSION
-# PAPIK = Project API Key
-# Note that this key does not need to be kept secret, so it's not
-# necessary to use Docker build secrets here.
-# @see: https://posthog.com/tutorials/api-capture-events#authenticating-with-the-project-api-key
-ARG NEXT_PUBLIC_POSTHOG_PAPIK
 ARG NEXT_PUBLIC_SENTRY_ENVIRONMENT
 ARG NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT
 ARG NEXT_PUBLIC_SENTRY_WEBAPP_DSN
@@ -43,10 +39,12 @@ COPY .yarn ./.yarn
 COPY ./packages/db ./packages/db
 COPY ./packages/schemas ./packages/schemas
 COPY ./packages/shared ./packages/shared
+COPY ./packages/queryLanguage ./packages/queryLanguage
 
 RUN yarn workspace @sourcebot/db install
 RUN yarn workspace @sourcebot/schemas install
 RUN yarn workspace @sourcebot/shared install
+RUN yarn workspace @sourcebot/query-language install
 # ------------------------------------
 
 # ------ Build Web ------
@@ -55,8 +53,6 @@ ENV SKIP_ENV_VALIDATION=1
 # -----------
 ARG NEXT_PUBLIC_SOURCEBOT_VERSION
 ENV NEXT_PUBLIC_SOURCEBOT_VERSION=$NEXT_PUBLIC_SOURCEBOT_VERSION
-ARG NEXT_PUBLIC_POSTHOG_PAPIK
-ENV NEXT_PUBLIC_POSTHOG_PAPIK=$NEXT_PUBLIC_POSTHOG_PAPIK
 ARG NEXT_PUBLIC_SENTRY_ENVIRONMENT
 ENV NEXT_PUBLIC_SENTRY_ENVIRONMENT=$NEXT_PUBLIC_SENTRY_ENVIRONMENT
 ARG NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT
@@ -92,6 +88,7 @@ COPY --from=shared-libs-builder /app/node_modules ./node_modules
 COPY --from=shared-libs-builder /app/packages/db ./packages/db
 COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
+COPY --from=shared-libs-builder /app/packages/queryLanguage ./packages/queryLanguage
 
 # Fixes arm64 timeouts
 RUN yarn workspace @sourcebot/web install
@@ -130,6 +127,7 @@ COPY --from=shared-libs-builder /app/node_modules ./node_modules
 COPY --from=shared-libs-builder /app/packages/db ./packages/db
 COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
+COPY --from=shared-libs-builder /app/packages/queryLanguage ./packages/queryLanguage
 RUN yarn workspace @sourcebot/backend install
 RUN yarn workspace @sourcebot/backend build
 
@@ -150,8 +148,6 @@ FROM node-alpine AS runner
 # -----------
 ARG NEXT_PUBLIC_SOURCEBOT_VERSION
 ENV NEXT_PUBLIC_SOURCEBOT_VERSION=$NEXT_PUBLIC_SOURCEBOT_VERSION
-ARG NEXT_PUBLIC_POSTHOG_PAPIK
-ENV NEXT_PUBLIC_POSTHOG_PAPIK=$NEXT_PUBLIC_POSTHOG_PAPIK
 ARG NEXT_PUBLIC_SENTRY_ENVIRONMENT
 ENV NEXT_PUBLIC_SENTRY_ENVIRONMENT=$NEXT_PUBLIC_SENTRY_ENVIRONMENT
 ARG NEXT_PUBLIC_SENTRY_WEBAPP_DSN
@@ -173,8 +169,13 @@ ENV DATA_DIR=/data
 ENV DATA_CACHE_DIR=$DATA_DIR/.sourcebot
 ENV DATABASE_DATA_DIR=$DATA_CACHE_DIR/db
 ENV REDIS_DATA_DIR=$DATA_CACHE_DIR/redis
-ENV SRC_TENANT_ENFORCEMENT_MODE=strict
 ENV SOURCEBOT_PUBLIC_KEY_PATH=/app/public.pem
+# PAPIK = Project API Key
+# Note that this key does not need to be kept secret, so it's not
+# necessary to use Docker build secrets here.
+# @see: https://posthog.com/tutorials/api-capture-events#authenticating-with-the-project-api-key
+# @note: this is also declared in the shared env.server.ts file.
+ENV POSTHOG_PAPIK=phc_lLPuFFi5LH6c94eFJcqvYVFwiJffVcV6HD8U4a1OnRW
 
 # Valid values are: debug, info, warn, error
 ENV SOURCEBOT_LOG_LEVEL=info
@@ -194,6 +195,7 @@ RUN addgroup -g $GID sourcebot && \
     adduser -D -u $UID -h /app -S sourcebot && \
     adduser sourcebot postgres && \
     adduser sourcebot redis && \
+    chown -R sourcebot /app && \
     adduser sourcebot node && \
     mkdir /var/log/sourcebot && \
     chown sourcebot /var/log/sourcebot
@@ -217,18 +219,23 @@ COPY --from=zoekt-builder \
 /cmd/zoekt-index \
 /usr/local/bin/
 
+RUN chown -R sourcebot:sourcebot /app
+
+# Copy zoekt proto files (needed for gRPC client at runtime)
+COPY --chown=sourcebot:sourcebot vendor/zoekt/grpc/protos /app/vendor/zoekt/grpc/protos
+
 # Copy all of the things
-COPY --from=web-builder /app/packages/web/public ./packages/web/public
+COPY --chown=sourcebot:sourcebot --from=web-builder /app/packages/web/public ./packages/web/public
-COPY --from=web-builder /app/packages/web/.next/standalone ./
+COPY --chown=sourcebot:sourcebot --from=web-builder /app/packages/web/.next/standalone ./
-COPY --from=web-builder /app/packages/web/.next/static ./packages/web/.next/static
+COPY --chown=sourcebot:sourcebot --from=web-builder /app/packages/web/.next/static ./packages/web/.next/static
 
-COPY --from=backend-builder /app/node_modules ./node_modules
+COPY --chown=sourcebot:sourcebot --from=backend-builder /app/node_modules ./node_modules
-COPY --from=backend-builder /app/packages/backend ./packages/backend
+COPY --chown=sourcebot:sourcebot --from=backend-builder /app/packages/backend ./packages/backend
 
-COPY --from=shared-libs-builder /app/node_modules ./node_modules
+COPY --chown=sourcebot:sourcebot --from=shared-libs-builder /app/packages/db ./packages/db
-COPY --from=shared-libs-builder /app/packages/db ./packages/db
+COPY --chown=sourcebot:sourcebot --from=shared-libs-builder /app/packages/schemas ./packages/schemas
-COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
+COPY --chown=sourcebot:sourcebot --from=shared-libs-builder /app/packages/shared ./packages/shared
-COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
+COPY --chown=sourcebot:sourcebot --from=shared-libs-builder /app/packages/queryLanguage ./packages/queryLanguage
 
 # Fixes git "dubious ownership" issues when the volume is mounted with different permissions to the container.
 RUN git config --global safe.directory "*"
@@ -239,9 +246,11 @@ RUN mkdir -p /run/postgresql && \
     chmod 775 /run/postgresql
 
 # Make app directory accessible to both root and sourcebot user
-RUN chown -R sourcebot:sourcebot /app
+RUN chown -R sourcebot /app \
+	&& chgrp -R 0  /app \
+	&& chmod -R g=u /app
 # Make data directory accessible to both root and sourcebot user
-RUN chown -R sourcebot:sourcebot /data
+RUN chown -R sourcebot /data
 
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 COPY prefix-output.sh ./prefix-output.sh

README.md

@@ -110,7 +110,7 @@ To configure Sourcebot (index your own repos, connect your LLMs, etc), check out
 
 > [!NOTE]
 > Sourcebot collects <a href="https://demo.sourcebot.dev/~/search?query=captureEvent%5C(%20repo%3Asourcebot">anonymous usage data</a> by default to help us improve the product. No sensitive data is collected, but if you'd like to disable this you can do so by setting the `SOURCEBOT_TELEMETRY_DISABLED` environment
-> variable to `true`. Please refer to our [telemetry docs](https://docs.sourcebot.dev/self-hosting/overview#telemetry) for more information.
+> variable to `true`. Please refer to our [telemetry docs](https://docs.sourcebot.dev/docs/overview#telemetry) for more information.
 
 # Build from source
 >[!NOTE]

docs/docs.json

@@ -144,7 +144,7 @@
         "socials": {
             "github": "https://github.com/sourcebot-dev/sourcebot",
             "twitter": "https://x.com/sourcebot_dev",
-            "discord": "https://discord.gg/GbXMEM5H",
+            "discord": "https://discord.gg/HDScTs3ptP",
             "linkedin": "https://www.linkedin.com/company/sourcebot"
         }
     },

docs/docs/configuration/auth/overview.mdx

@@ -25,4 +25,4 @@ Sourcebot's built-in authentication system gates your deployment, and allows adm
 # Troubleshooting
 
 - If you experience issues logging in, logging out, or accessing an organization you should have access to, try clearing your cookies & performing a full page refresh (`Cmd/Ctrl + Shift + R` on most browsers).
-- Still not working? Reach out to us on our [discord](https://discord.gg/GbXMEM5H) or [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose)
+- Still not working? Reach out to us on our [discord](https://discord.gg/HDScTs3ptP) or [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose)

docs/docs/configuration/environment-variables.mdx

@@ -34,6 +34,8 @@ The following environment variables allow you to configure your Sourcebot deploy
 | `SOURCEBOT_STRUCTURED_LOGGING_ENABLED` | `false` | <p>Enables/disable structured JSON logging. See [this doc](/docs/configuration/structured-logging) for more info.</p> |
 | `SOURCEBOT_STRUCTURED_LOGGING_FILE` | - | <p>Optional file to log to if structured logging is enabled</p> | 
 | `SOURCEBOT_TELEMETRY_DISABLED` | `false` | <p>Enables/disables telemetry collection in Sourcebot. See [this doc](/docs/overview.mdx#telemetry) for more info.</p> |
+| `DEFAULT_MAX_MATCH_COUNT` | `10000` | <p>The default maximum number of search results to return when using search in the web app.</p> |
+| `ALWAYS_INDEX_FILE_PATTERNS` | - | <p>A comma separated list of glob patterns matching file paths that should always be indexed, regardless of size or number of trigrams.</p> |
 
 ### Enterprise Environment Variables
 | Variable | Default | Description |

docs/docs/configuration/idp.mdx

@@ -366,3 +366,53 @@ A Microsoft Entra ID connection can be used for [authentication](/docs/configura
 </Steps>
 </Accordion>
 
+### Authentik
+
+[Auth.js Authentik Provider Docs](https://authjs.dev/getting-started/providers/authentik)
+
+An Authentik connection can be used for [authentication](/docs/configuration/auth).
+
+<Accordion title="instructions">
+<Steps>
+    <Step title="Create a OAuth2/OpenID Connect application">
+        To begin, you must create a OAuth2/OpenID Connect application in Authentik. For more information, see the [Authentik documentation](https://docs.goauthentik.io/add-secure-apps/applications/manage_apps/#create-an-application-and-provider-pair).
+
+        When configuring your application:
+        - Set the provider type to "OAuth2/OpenID Connect"
+        - Set the client type to "Confidential"
+        - Add `<sourcebot_url>/api/auth/callback/authentik` to the redirect URIs (ex. https://sourcebot.coolcorp.com/api/auth/callback/authentik)
+
+        After creating the application, open the application details to obtain the client id, client secret, and issuer URL (typically in the format `https://<authentik-domain>/application/o/<provider-slug>/`).
+    </Step>
+    <Step title="Define environment variables">
+        The client id, secret, and issuer URL are provided to Sourcebot via environment variables. These can be named whatever you like
+        (ex. `AUTHENTIK_IDENTITY_PROVIDER_CLIENT_ID`, `AUTHENTIK_IDENTITY_PROVIDER_CLIENT_SECRET`, and `AUTHENTIK_IDENTITY_PROVIDER_ISSUER`)
+    </Step>
+    <Step title="Define the identity provider config">
+        Create a `identityProvider` object in the [config file](/docs/configuration/config-file) with the following fields:
+
+       ```json wrap icon="code"
+        {
+            "$schema": "https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v3/index.json",
+            "identityProviders": [
+                {
+                    "provider": "authentik",
+                    "purpose": "sso",
+                    "clientId": {
+                        "env": "AUTHENTIK_IDENTITY_PROVIDER_CLIENT_ID"
+                    },
+                    "clientSecret": {
+                        "env": "AUTHENTIK_IDENTITY_PROVIDER_CLIENT_SECRET"
+                    },
+                    "issuer": {
+                        "env": "AUTHENTIK_IDENTITY_PROVIDER_ISSUER"
+                    }
+                }
+            ]
+        }
+        ```
+    </Step>
+</Steps>
+</Accordion>
+
+

docs/docs/connections/overview.mdx

@@ -69,6 +69,26 @@ To learn more about how to create a connection for a specific code host, check o
 
 <Note>Missing your code host? [Submit a feature request on GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new?template=feature_request.md).</Note>
 
+## Indexing Large Files
+
+By default, Sourcebot will skip indexing files that are larger than 2MB or have more than 20,000 trigrams. You can configure this by setting the `maxFileSize` and `maxTrigramCount` [settings](/docs/configuration/config-file#settings).
+
+These limits can be ignored for specific files by passing in a comma separated list of glob patterns matching file paths to the `ALWAYS_INDEX_FILE_PATTERNS` environment variable. For example:
+
+```bash
+# Always index all .sum and .lock files
+ALWAYS_INDEX_FILE_PATTERNS=**/*.sum,**/*.lock
+```
+
+Files that have been skipped are assigned the `skipped` language. You can view a list of all skipped files by using the following query:
+```
+lang:skipped
+```
+
+## Indexing Binary Files
+
+Binary files cannot be indexed by Sourcebot. See [#575](https://github.com/sourcebot-dev/sourcebot/issues/575) for more information.
+
 
 ## Schema reference
 ---

docs/docs/features/code-navigation.mdx

@@ -21,6 +21,7 @@ import LicenseKeyRequired from '/snippets/license-key-required.mdx'
 | **Go to definition** | Clicking the "go to definition" button in the popover or clicking the symbol name navigates to the symbol's definition. |
 | **Find references** | Clicking the "find all references" button in the popover lists all references in the explore panel. |
 | **Explore panel** | Lists all references and definitions for the symbol selected in the popover. |
+| **Cross-repository navigation** | You can search across all repositories by clicking the globe icon in the explore panel. By default, references and definitions are scoped to the repository where the symbol is being resolved. |
 
 ## How does it work?
 

docs/docs/features/permission-syncing.mdx

@@ -1,14 +1,11 @@
 ---
 title: "Permission syncing"
 sidebarTitle: "Permission syncing"
-tag: "experimental"
 ---
 
 import LicenseKeyRequired from '/snippets/license-key-required.mdx'
-import ExperimentalFeatureWarning from '/snippets/experimental-feature-warning.mdx'
 
 <LicenseKeyRequired />
-<ExperimentalFeatureWarning />
 
 # Overview
 

docs/docs/features/search/syntax-reference.mdx

@@ -4,32 +4,51 @@ title: Writing search queries
 
 Sourcebot uses a powerful regex-based query language that enabled precise code search within large codebases.
 
-
 ## Syntax reference guide
 
-Queries consist of space-separated regular expressions. Wrapping expressions in `""` combines them. By default, a file must have at least one match for each expression to be included.
+Queries consist of space-separated search patterns that are matched against file contents. A file must have at least one match for each expression to be included. Queries can optionally contain search filters to further refine the search results.
+
+## Keyword search (default)
+
+Keyword search matches search patterns exactly in file contents. Wrapping search patterns in `""` combines them as a single expression.
+
+| Example | Explanation |
+| :--- | :--- |
+| `foo` | Match files containing the keyword `foo` |
+| `foo bar` | Match files containing both `foo` **and** `bar` |
+| `"foo bar"` | Match files containing the phrase `foo bar` |
+| `"foo \"bar\""` | Match files containing `foo "bar"` exactly (escaped quotes) |
+
+## Regex search
+
+Toggle the regex button (`.*`) in the search bar to interpret search patterns as regular expressions.
 
 | Example | Explanation |
 | :--- | :--- |
 | `foo` | Match files with regex `/foo/` |
-| `foo bar` | Match files with regex `/foo/` **and** `/bar/` |
+| `foo.*bar` | Match files with regex `/foo.*bar/` (foo followed by any characters, then bar) |
-| `"foo bar"` | Match files with regex `/foo bar/` |
+| `^function\s+\w+` | Match files with regex `/^function\s+\w+/` (function at start of line, followed by whitespace and word characters) |
+| `"foo bar"` | Match files with regex `/foo bar/`. Quotes are not matched. |
 
-Multiple expressions can be or'd together with `or`, negated with `-`, or grouped with `()`.
+## Search filters
 
-| Example | Explanation |
+Search queries (keyword or regex) can include multiple search filters to further refine the search results. Some filters can be negated using the `-` prefix.
-| :--- | :--- |
-| `foo or bar` | Match files with regex `/foo/` **or** `/bar/` |
-| `foo -bar` | Match files with regex `/foo/` but **not** `/bar/` |
-| `foo (bar or baz)` | Match files with regex `/foo/` **and** either `/bar/` **or** `/baz/` |
-
-Expressions can be prefixed with certain keywords to modify search behavior. Some keywords can be negated using the `-` prefix.
 
 | Prefix | Description | Example |
 | :--- | :--- | :--- |
 | `file:` | Filter results from filepaths that match the regex. By default all files are searched. | `file:README` - Filter results to filepaths that match regex `/README/`<br/>`file:"my file"` - Filter results to filepaths that match regex `/my file/`<br/>`-file:test\.ts$` - Ignore results from filepaths match regex `/test\.ts$/` |
-| `repo:` | Filter results from repos that match the regex. By default all repos are searched. | `repo:linux` - Filter results to repos that match regex `/linux/`<br/>`-repo:^web/.*` - Ignore results from repos that match regex `/^web\/.*` |
+| `repo:` | Filter results from repos that match the regex. By default all repos are searched. | `repo:linux` - Filter results to repos that match regex `/linux/`<br/>`-repo:^web/.*` - Ignore results from repos that match regex `/^web\/.*/` |
 | `rev:` | Filter results from a specific branch or tag. By default **only** the default branch is searched. | `rev:beta` - Filter results to branches that match regex `/beta/` |
 | `lang:` | Filter results by language (as defined by [linguist](https://github.com/github-linguist/linguist/blob/main/lib/linguist/languages.yml)). By default all languages are searched. | `lang:TypeScript` - Filter results to TypeScript files<br/>`-lang:YAML` - Ignore results from YAML files |
 | `sym:` | Match symbol definitions created by [universal ctags](https://ctags.io/) at index time. | `sym:\bmain\b` - Filter results to symbols that match regex `/\bmain\b/` |
 | `context:` | Filter results to a predefined [search context](/docs/features/search/search-contexts). | `context:web` - Filter results to the web context<br/>`-context:pipelines` - Ignore results from the pipelines context |
+
+## Boolean operators & grouping
+
+By default, space-separated expressions are and'd together. Using the `or` keyword as well as parentheses `()` can be used to create more complex boolean logic. Parentheses can be negated using the `-` prefix.
+
+| Example | Explanation |
+| :--- | :--- |
+| `foo or bar` | Match files containing `foo` **or** `bar` |
+| `foo (bar or baz)` | Match files containing `foo` **and** either `bar` **or** `baz`. |
+| `-(foo) bar` | Match files containing `bar` **and not** `foo`. |

docs/docs/upgrade/v2-to-v3-guide.mdx

@@ -78,7 +78,7 @@ If your deployment is dependent on these features, please [reach out](https://gi
         After updating your configuration file, restart your Sourcebot deployment to pick up the new changes.
     </Step>
     <Step title="You're done!">
-        Congrats, you've successfully migrated to v3! Please let us know what you think of the new features by reaching out on our [discord](https://discord.gg/GbXMEM5H) or on [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose).
+        Congrats, you've successfully migrated to v3! Please let us know what you think of the new features by reaching out on our [discord](https://discord.gg/HDScTs3ptP) or on [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose).
     </Step>
 </Steps>
 
@@ -90,4 +90,4 @@ Some things to check:
 - Make sure you have a name for each `connection`, and that the name only contains letters, digits, hyphens, or underscores
 - Make sure each `connection` has a `type` field with a valid value (`gitlab`, `github`, `gitea`, `gerrit`)
 
-Having troubles migrating from v2 to v3? Reach out to us on [discord](https://discord.gg/GbXMEM5H) or [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose) and we'll try our best to help
+Having troubles migrating from v2 to v3? Reach out to us on [discord](https://discord.gg/HDScTs3ptP) or [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose) and we'll try our best to help

docs/docs/upgrade/v3-to-v4-guide.mdx

@@ -40,7 +40,7 @@ Please note that the following features are no longer supported in v4:
 
     </Step>
     <Step title="You're done!">
-        Congrats, you've successfully migrated to v4! Please let us know what you think of the new features by reaching out on our [discord](https://discord.gg/GbXMEM5H) or [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose)
+        Congrats, you've successfully migrated to v4! Please let us know what you think of the new features by reaching out on our [discord](https://discord.gg/HDScTs3ptP) or [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose)
     </Step>
 </Steps>
 
@@ -58,4 +58,4 @@ to finish upgrading to v4 in single-tenant mode.
 - If you're hitting issues with signing into your Sourcebot instance, make sure you're setting `AUTH_URL` correctly to your deployment domain (ex. `https://sourcebot.yourcompany.com`)
 
 
-Having troubles migrating from v3 to v4? Reach out to us on [discord](https://discord.gg/GbXMEM5H) or [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose) and we'll try our best to help
+Having troubles migrating from v3 to v4? Reach out to us on [discord](https://discord.gg/HDScTs3ptP) or [GitHub](https://github.com/sourcebot-dev/sourcebot/issues/new/choose) and we'll try our best to help

docs/snippets/schemas/v3/identityProvider.schema.mdx

@@ -647,6 +647,115 @@
         "purpose",
         "audience"
       ]
+    },
+    "AuthentikIdentityProviderConfig": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "provider": {
+          "const": "authentik"
+        },
+        "purpose": {
+          "const": "sso"
+        },
+        "clientId": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "clientSecret": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "issuer": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        }
+      },
+      "required": [
+        "provider",
+        "purpose",
+        "clientId",
+        "clientSecret",
+        "issuer"
+      ]
     }
   },
   "oneOf": [
@@ -1293,6 +1402,115 @@
         "purpose",
         "audience"
       ]
+    },
+    {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "provider": {
+          "const": "authentik"
+        },
+        "purpose": {
+          "const": "sso"
+        },
+        "clientId": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "clientSecret": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "issuer": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        }
+      },
+      "required": [
+        "provider",
+        "purpose",
+        "clientId",
+        "clientSecret",
+        "issuer"
+      ]
     }
   ]
 }

docs/snippets/schemas/v3/index.schema.mdx

@@ -5163,6 +5163,115 @@
               "purpose",
               "audience"
             ]
+          },
+          "AuthentikIdentityProviderConfig": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "provider": {
+                "const": "authentik"
+              },
+              "purpose": {
+                "const": "sso"
+              },
+              "clientId": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              },
+              "clientSecret": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              },
+              "issuer": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              }
+            },
+            "required": [
+              "provider",
+              "purpose",
+              "clientId",
+              "clientSecret",
+              "issuer"
+            ]
           }
         },
         "oneOf": [
@@ -5809,6 +5918,115 @@
               "purpose",
               "audience"
             ]
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "provider": {
+                "const": "authentik"
+              },
+              "purpose": {
+                "const": "sso"
+              },
+              "clientId": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              },
+              "clientSecret": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              },
+              "issuer": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              }
+            },
+            "required": [
+              "provider",
+              "purpose",
+              "clientId",
+              "clientSecret",
+              "issuer"
+            ]
           }
         ]
       }

entrypoint.sh

@@ -66,12 +66,6 @@ fi
 
 echo -e "\e[34m[Info] Sourcebot version: $NEXT_PUBLIC_SOURCEBOT_VERSION\e[0m"
 
-# If we don't have a PostHog key, then we need to disable telemetry.
-if [ -z "$NEXT_PUBLIC_POSTHOG_PAPIK" ]; then
-    echo -e "\e[33m[Warning] NEXT_PUBLIC_POSTHOG_PAPIK was not set. Setting SOURCEBOT_TELEMETRY_DISABLED.\e[0m"
-    export SOURCEBOT_TELEMETRY_DISABLED=true
-fi
-
 if [ -n "$SOURCEBOT_TELEMETRY_DISABLED" ]; then
     # Validate that SOURCEBOT_TELEMETRY_DISABLED is either "true" or "false"
     if [ "$SOURCEBOT_TELEMETRY_DISABLED" != "true" ] && [ "$SOURCEBOT_TELEMETRY_DISABLED" != "false" ]; then
@@ -159,7 +153,7 @@ if [ ! -f "$FIRST_RUN_FILE" ]; then
     # (if telemetry is enabled)
     if [ "$SOURCEBOT_TELEMETRY_DISABLED" = "false" ]; then
         if ! ( curl -L --output /dev/null --silent --fail --header "Content-Type: application/json" -d '{
-            "api_key": "'"$NEXT_PUBLIC_POSTHOG_PAPIK"'",
+            "api_key": "'"$POSTHOG_PAPIK"'",
             "event": "install",
             "distinct_id": "'"$SOURCEBOT_INSTALL_ID"'",
             "properties": {
@@ -179,7 +173,7 @@ else
 
         if [ "$SOURCEBOT_TELEMETRY_DISABLED" = "false" ]; then
             if ! ( curl -L --output /dev/null --silent --fail --header "Content-Type: application/json" -d '{
-                "api_key": "'"$NEXT_PUBLIC_POSTHOG_PAPIK"'",
+                "api_key": "'"$POSTHOG_PAPIK"'",
                 "event": "upgrade",
                 "distinct_id": "'"$SOURCEBOT_INSTALL_ID"'",
                 "properties": {

package.json

@@ -18,7 +18,7 @@
         "dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
         "dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset",
         "dev:prisma:db:push": "yarn with-env yarn workspace @sourcebot/db prisma:db:push",
-        "build:deps": "yarn workspaces foreach --recursive --topological --from '{@sourcebot/schemas,@sourcebot/db,@sourcebot/shared}' run build"
+        "build:deps": "yarn workspaces foreach --recursive --topological --from '{@sourcebot/schemas,@sourcebot/db,@sourcebot/shared,@sourcebot/query-language}' run build"
     },
     "devDependencies": {
         "concurrently": "^9.2.1",
@@ -27,6 +27,7 @@
     },
     "packageManager": "yarn@4.7.0",
     "resolutions": {
-        "prettier": "3.5.3"
+        "prettier": "3.5.3",
+        "@lezer/common": "1.3.0"
     }
 }

packages/backend/package.json

@@ -40,6 +40,8 @@
         "cross-fetch": "^4.0.0",
         "dotenv": "^16.4.5",
         "express": "^4.21.2",
+        "express-async-errors": "^3.1.1",
+        "fast-deep-equal": "^3.1.3",
         "git-url-parse": "^16.1.0",
         "gitea-js": "^1.22.0",
         "glob": "^11.0.0",

packages/backend/src/api.ts

@@ -0,0 +1,103 @@
+import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db';
+import { createLogger } from '@sourcebot/shared';
+import express, { Request, Response } from 'express';
+import 'express-async-errors';
+import * as http from "http";
+import z from 'zod';
+import { ConnectionManager } from './connectionManager.js';
+import { PromClient } from './promClient.js';
+import { RepoIndexManager } from './repoIndexManager.js';
+
+const logger = createLogger('api');
+const PORT = 3060;
+
+export class Api {
+    private server: http.Server;
+
+    constructor(
+        promClient: PromClient,
+        private prisma: PrismaClient,
+        private connectionManager: ConnectionManager,
+        private repoIndexManager: RepoIndexManager,
+    ) {
+        const app = express();
+        app.use(express.json());
+        app.use(express.urlencoded({ extended: true }));
+
+        // Prometheus metrics endpoint
+        app.use('/metrics', async (_req: Request, res: Response) => {
+            res.set('Content-Type', promClient.registry.contentType);
+            const metrics = await promClient.registry.metrics();
+            res.end(metrics);
+        });
+
+        app.post('/api/sync-connection', this.syncConnection.bind(this));
+        app.post('/api/index-repo', this.indexRepo.bind(this));
+
+        this.server = app.listen(PORT, () => {
+            logger.info(`API server is running on port ${PORT}`);
+        });
+    }
+
+    private async syncConnection(req: Request, res: Response) {
+        const schema = z.object({
+            connectionId: z.number(),
+        }).strict();
+
+        const parsed = schema.safeParse(req.body);
+        if (!parsed.success) {
+            res.status(400).json({ error: parsed.error.message });
+            return;
+        }
+
+        const { connectionId } = parsed.data;
+        const connection = await this.prisma.connection.findUnique({
+            where: {
+                id: connectionId,
+            }
+        });
+
+        if (!connection) {
+            res.status(404).json({ error: 'Connection not found' });
+            return;
+        }
+
+        const [jobId] = await this.connectionManager.createJobs([connection]);
+
+        res.status(200).json({ jobId });
+    }
+
+    private async indexRepo(req: Request, res: Response) {
+        const schema = z.object({
+            repoId: z.number(),
+        }).strict();
+
+        const parsed = schema.safeParse(req.body);
+        if (!parsed.success) {
+            res.status(400).json({ error: parsed.error.message });
+            return;
+        }
+
+        const { repoId } = parsed.data;
+        const repo = await this.prisma.repo.findUnique({
+            where: { id: repoId },
+        });
+
+        if (!repo) {
+            res.status(404).json({ error: 'Repo not found' });
+            return;
+        }
+
+        const [jobId] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX);
+        res.status(200).json({ jobId });
+    }
+
+    public async dispose() {
+        return new Promise<void>((resolve, reject) => {
+            this.server.close((err) => {
+                if (err) reject(err);
+                else resolve(undefined);
+            });
+        });
+    }
+}

packages/backend/src/bitbucket.ts

@@ -5,6 +5,7 @@ import type { ClientOptions, ClientPathsWithMethod } from "openapi-fetch";
 import { createLogger } from "@sourcebot/shared";
 import { measure, fetchWithRetry } from "./utils.js";
 import * as Sentry from "@sentry/node";
+import micromatch from "micromatch";
 import {
     SchemaRepository as CloudRepository,
 } from "@coderabbitai/bitbucket/cloud/openapi";
@@ -346,10 +347,15 @@ async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promi
 
 function cloudShouldExcludeRepo(repo: BitbucketRepository, config: BitbucketConnectionConfig): boolean {
     const cloudRepo = repo as CloudRepository;
+    let reason = '';
+    const repoName = cloudRepo.full_name!;
     
     const shouldExclude = (() => {
-        if (config.exclude?.repos && config.exclude.repos.includes(cloudRepo.full_name!)) {
+        if (config.exclude?.repos) {
-            return true;
+            if (micromatch.isMatch(repoName, config.exclude.repos)) {
+                reason = `\`exclude.repos\` contains ${repoName}`;
+                return true;
+            }
         }
 
         if (!!config.exclude?.archived) {
@@ -357,12 +363,15 @@ function cloudShouldExcludeRepo(repo: BitbucketRepository, config: BitbucketConn
         }
 
         if (!!config.exclude?.forks && cloudRepo.parent !== undefined) {
+            reason = `\`exclude.forks\` is true`;
             return true;
         }
+
+        return false;
     })();
 
     if (shouldExclude) {
-        logger.debug(`Excluding repo ${cloudRepo.full_name} because it matches the exclude pattern`);
+        logger.debug(`Excluding repo ${repoName}. Reason: ${reason}`);
         return true;
     }
     return false;
@@ -548,23 +557,32 @@ function serverShouldExcludeRepo(repo: BitbucketRepository, config: BitbucketCon
 
     const projectName = serverRepo.project!.key;
     const repoSlug = serverRepo.slug!;
+    const repoName = `${projectName}/${repoSlug}`;
+    let reason = '';
     
     const shouldExclude = (() => {
-        if (config.exclude?.repos && config.exclude.repos.includes(`${projectName}/${repoSlug}`)) {
+        if (config.exclude?.repos) {
-            return true;
+            if (micromatch.isMatch(repoName, config.exclude.repos)) {
+                reason = `\`exclude.repos\` contains ${repoName}`;
+                return true;
+            }
         }
 
         if (!!config.exclude?.archived && serverRepo.archived) {
+            reason = `\`exclude.archived\` is true`;
             return true;
         }
 
         if (!!config.exclude?.forks && serverRepo.origin !== undefined) {
+            reason = `\`exclude.forks\` is true`;
             return true;
         }
+
+        return false;
     })();
 
     if (shouldExclude) {
-        logger.debug(`Excluding repo ${projectName}/${repoSlug} because it matches the exclude pattern`);
+        logger.debug(`Excluding repo ${repoName}. Reason: ${reason}`);
         return true;
     }
     return false;

packages/backend/src/configManager.ts

@@ -6,6 +6,7 @@ import chokidar, { FSWatcher } from 'chokidar';
 import { ConnectionManager } from "./connectionManager.js";
 import { SINGLE_TENANT_ORG_ID } from "./constants.js";
 import { syncSearchContexts } from "./ee/syncSearchContexts.js";
+import isEqual from 'fast-deep-equal';
 
 const logger = createLogger('config-manager');
 
@@ -64,8 +65,8 @@ export class ConfigManager {
 
                 const existingConnectionConfig = existingConnection ? existingConnection.config as unknown as ConnectionConfig : undefined;
                 const connectionNeedsSyncing =
-                    !existingConnection ||
+                    !existingConnectionConfig ||
-                    (JSON.stringify(existingConnectionConfig) !== JSON.stringify(newConnectionConfig));
+                    !isEqual(existingConnectionConfig, newConnectionConfig);
 
                 // Either update the existing connection or create a new one.
                 const connection = existingConnection ?
@@ -93,8 +94,8 @@ export class ConfigManager {
                     });
 
                 if (connectionNeedsSyncing) {
-                    const [jobId] = await this.connectionManager.createJobs([connection]);
+                    logger.info(`Change detected for connection '${key}' (id: ${connection.id}). Creating sync job.`);
-                    logger.info(`Change detected for connection '${key}' (id: ${connection.id}). Created sync job ${jobId}.`);
+                    await this.connectionManager.createJobs([connection]);
                 }
             }
         }

packages/backend/src/connectionManager.ts

@@ -7,14 +7,16 @@ import { Job, Queue, ReservedJob, Worker } from "groupmq";
 import { Redis } from 'ioredis';
 import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js";
 import { Settings } from "./types.js";
-import { groupmqLifecycleExceptionWrapper } from "./utils.js";
+import { groupmqLifecycleExceptionWrapper, setIntervalAsync } from "./utils.js";
 import { syncSearchContexts } from "./ee/syncSearchContexts.js";
 import { captureEvent } from "./posthog.js";
 import { PromClient } from "./promClient.js";
+import { GROUPMQ_WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js";
 
 const LOG_TAG = 'connection-manager';
 const logger = createLogger(LOG_TAG);
 const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`);
+const QUEUE_NAME = 'connection-sync-queue';
 
 type JobPayload = {
     jobId: string,
@@ -30,19 +32,19 @@ type JobResult = {
 const JOB_TIMEOUT_MS = 1000 * 60 * 60 * 2; // 2 hour timeout
 
 export class ConnectionManager {
-    private worker: Worker;
+    private worker: Worker<JobPayload>;
     private queue: Queue<JobPayload>;
     private interval?: NodeJS.Timeout;
 
     constructor(
         private db: PrismaClient,
         private settings: Settings,
-        redis: Redis,
+        private redis: Redis,
         private promClient: PromClient,
     ) {
         this.queue = new Queue<JobPayload>({
             redis,
-            namespace: 'connection-sync-queue',
+            namespace: QUEUE_NAME,
             jobTimeoutMs: JOB_TIMEOUT_MS,
             maxAttempts: 3,
             logger: env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true',
@@ -62,11 +64,15 @@ export class ConnectionManager {
         this.worker.on('failed', this.onJobFailed.bind(this));
         this.worker.on('stalled', this.onJobStalled.bind(this));
         this.worker.on('error', this.onWorkerError.bind(this));
+        // graceful-timeout is triggered when a job is still processing after
+        // worker.close() is called and the timeout period has elapsed. In this case,
+        // we fail the job with no retry.
+        this.worker.on('graceful-timeout', this.onJobGracefulTimeout.bind(this));
     }
 
     public startScheduler() {
         logger.debug('Starting scheduler');
-        this.interval = setInterval(async () => {
+        this.interval = setIntervalAsync(async () => {
             const thresholdDate = new Date(Date.now() - this.settings.resyncConnectionIntervalMs);
             const timeoutDate = new Date(Date.now() - JOB_TIMEOUT_MS);
 
@@ -128,6 +134,7 @@ export class ConnectionManager {
         });
 
         for (const job of jobs) {
+            logger.info(`Scheduling job ${job.id} for connection ${job.connection.name} (id: ${job.connectionId})`);
             await this.queue.add({
                 groupId: `connection:${job.connectionId}`,
                 data: {
@@ -150,6 +157,22 @@ export class ConnectionManager {
         const logger = createJobLogger(jobId);
         logger.info(`Running connection sync job ${jobId} for connection ${connectionName} (id: ${job.data.connectionId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`);
 
+        const currentStatus = await this.db.connectionSyncJob.findUniqueOrThrow({
+            where: {
+                id: jobId,
+            },
+            select: {
+                status: true,
+            }
+        });
+
+        // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job
+        // is in an invalid state and should be skipped.
+        if (currentStatus.status !== ConnectionSyncJobStatus.PENDING && currentStatus.status !== ConnectionSyncJobStatus.IN_PROGRESS) {
+            throw new Error(`Job ${jobId} is not in a valid state. Expected: ${ConnectionSyncJobStatus.PENDING} or ${ConnectionSyncJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`);
+        }
+
+
         this.promClient.pendingConnectionSyncJobs.dec({ connection: connectionName });
         this.promClient.activeConnectionSyncJobs.inc({ connection: connectionName });
 
@@ -178,7 +201,7 @@ export class ConnectionManager {
         const result = await (async () => {
             switch (config.type) {
                 case 'github': {
-                    return await compileGithubConfig(config, job.data.connectionId, abortController);
+                    return await compileGithubConfig(config, job.data.connectionId, abortController.signal);
                 }
                 case 'gitlab': {
                     return await compileGitlabConfig(config, job.data.connectionId);
@@ -383,6 +406,33 @@ export class ConnectionManager {
             });
         });
 
+    private onJobGracefulTimeout = async (job: Job<JobPayload>) =>
+        groupmqLifecycleExceptionWrapper('onJobGracefulTimeout', logger, async () => {
+            const logger = createJobLogger(job.id);
+
+            const { connection } = await this.db.connectionSyncJob.update({
+                where: { id: job.id },
+                data: {
+                    status: ConnectionSyncJobStatus.FAILED,
+                    completedAt: new Date(),
+                    errorMessage: 'Job timed out',
+                },
+                select: {
+                    connection: true,
+                }
+            });
+
+            this.promClient.activeConnectionSyncJobs.dec({ connection: connection.name });
+            this.promClient.connectionSyncJobFailTotal.inc({ connection: connection.name });
+
+            logger.error(`Job ${job.id} timed out for connection ${connection.name} (id: ${connection.id})`);
+
+            captureEvent('backend_connection_sync_job_failed', {
+                connectionId: connection.id,
+                error: 'Job timed out',
+            });
+        });
+
     private async onWorkerError(error: Error) {
         Sentry.captureException(error);
         logger.error(`Connection syncer worker error.`, error);
@@ -392,8 +442,28 @@ export class ConnectionManager {
         if (this.interval) {
             clearInterval(this.interval);
         }
-        await this.worker.close();
+
-        await this.queue.close();
+        const inProgressJobs = this.worker.getCurrentJobs();
+        await this.worker.close(GROUPMQ_WORKER_STOP_GRACEFUL_TIMEOUT_MS);
+
+        // Manually release group locks for in progress jobs to prevent deadlocks.
+        // @see: https://github.com/Openpanel-dev/groupmq/issues/8
+        for (const { job } of inProgressJobs) {
+            const lockKey = `groupmq:${QUEUE_NAME}:lock:${job.groupId}`;
+            logger.debug(`Releasing group lock ${lockKey} for in progress job ${job.id}`);
+            try {
+                await this.redis.del(lockKey);
+            } catch (error) {
+                Sentry.captureException(error);
+                logger.error(`Failed to release group lock ${lockKey} for in progress job ${job.id}. Error: `, error);
+            }
+        }
+
+        // @note: As of groupmq v1.0.0, queue.close() will just close the underlying
+        // redis connection. Since we share the same redis client between, skip this
+        // step and close the redis client directly in index.ts.
+        // @see: https://github.com/Openpanel-dev/groupmq/blob/main/src/queue.ts#L1900
+        // await this.queue.close();
     }
 }
 

packages/backend/src/constants.ts

@@ -11,3 +11,23 @@ export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: CodeHostType[] = [
 
 export const REPOS_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'repos');
 export const INDEX_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'index');
+
+// Maximum time to wait for current job to finish
+export const GROUPMQ_WORKER_STOP_GRACEFUL_TIMEOUT_MS = 5 * 1000; // 5 seconds
+
+// List of shutdown signals
+export const SHUTDOWN_SIGNALS: string[] = [
+    'SIGHUP',
+    'SIGINT',
+    'SIGQUIT',
+    'SIGILL',
+    'SIGTRAP',
+    'SIGABRT',
+    'SIGBUS',
+    'SIGFPE',
+    'SIGSEGV',
+    'SIGUSR2',
+    'SIGTERM',
+    // @note: SIGKILL and SIGSTOP cannot have listeners installed.
+    // @see: https://nodejs.org/api/process.html#signal-events
+];

packages/backend/src/ee/accountPermissionSyncer.ts

@@ -7,6 +7,7 @@ import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js";
 import { createOctokitFromToken, getReposForAuthenticatedUser } from "../github.js";
 import { createGitLabFromOAuthToken, getProjectsForAuthenticatedUser } from "../gitlab.js";
 import { Settings } from "../types.js";
+import { setIntervalAsync } from "../utils.js";
 
 const LOG_TAG = 'user-permission-syncer';
 const logger = createLogger(LOG_TAG);
@@ -46,7 +47,7 @@ export class AccountPermissionSyncer {
 
         logger.debug('Starting scheduler');
 
-        this.interval = setInterval(async () => {
+        this.interval = setIntervalAsync(async () => {
             const thresholdDate = new Date(Date.now() - this.settings.experiment_userDrivenPermissionSyncIntervalMs);
 
             const accounts = await this.db.account.findMany({
@@ -101,7 +102,7 @@ export class AccountPermissionSyncer {
         if (this.interval) {
             clearInterval(this.interval);
         }
-        await this.worker.close();
+        await this.worker.close(/* force = */ true);
         await this.queue.close();
     }
 

packages/backend/src/ee/repoPermissionSyncer.ts

@@ -8,7 +8,7 @@ import { PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "../constants.js";
 import { createOctokitFromToken, getRepoCollaborators, GITHUB_CLOUD_HOSTNAME } from "../github.js";
 import { createGitLabFromPersonalAccessToken, getProjectMembers } from "../gitlab.js";
 import { Settings } from "../types.js";
-import { getAuthCredentialsForRepo } from "../utils.js";
+import { getAuthCredentialsForRepo, setIntervalAsync } from "../utils.js";
 
 type RepoPermissionSyncJob = {
     jobId: string;
@@ -48,26 +48,34 @@ export class RepoPermissionSyncer {
 
         logger.debug('Starting scheduler');
 
-        this.interval = setInterval(async () => {
+        this.interval = setIntervalAsync(async () => {
             // @todo: make this configurable
             const thresholdDate = new Date(Date.now() - this.settings.experiment_repoDrivenPermissionSyncIntervalMs);
 
             const repos = await this.db.repo.findMany({
                 // Repos need their permissions to be synced against the code host when...
                 where: {
-                    // They belong to a code host that supports permissions syncing
                     AND: [
+                        // They are not public. Public repositories are always visible to all users, therefore we don't
+                        // need to explicitly perform permission syncing for them.
+                        // @see: packages/web/src/prisma.ts
+                        {
+                            isPublic: false
+                        },
+                        // They belong to a code host that supports permissions syncing
                         {
                             external_codeHostType: {
                                 in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES,
                             }
                         },
+                        // They have not been synced within the threshold date.
                         {
                             OR: [
                                 { permissionSyncedAt: null },
                                 { permissionSyncedAt: { lt: thresholdDate } },
                             ],
                         },
+                        // There aren't any active or recently failed jobs.
                         {
                             NOT: {
                                 permissionSyncJobs: {
@@ -106,7 +114,7 @@ export class RepoPermissionSyncer {
         if (this.interval) {
             clearInterval(this.interval);
         }
-        await this.worker.close();
+        await this.worker.close(/* force = */ true);
         await this.queue.close();
     }
 

packages/backend/src/index.ts

@@ -1,17 +1,20 @@
 import "./instrument.js";
 
+import * as Sentry from "@sentry/node";
 import { PrismaClient } from "@sourcebot/db";
-import { createLogger } from "@sourcebot/shared";
+import { createLogger, env, getConfigSettings, getDBConnectionString, hasEntitlement } from "@sourcebot/shared";
-import { env, getConfigSettings, hasEntitlement, getDBConnectionString } from '@sourcebot/shared';
+import 'express-async-errors';
 import { existsSync } from 'fs';
 import { mkdir } from 'fs/promises';
 import { Redis } from 'ioredis';
+import { Api } from "./api.js";
 import { ConfigManager } from "./configManager.js";
 import { ConnectionManager } from './connectionManager.js';
-import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js';
+import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS } from './constants.js';
+import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js";
 import { GithubAppManager } from "./ee/githubAppManager.js";
 import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js';
-import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js";
+import { shutdownPosthog } from "./posthog.js";
 import { PromClient } from './promClient.js';
 import { RepoIndexManager } from "./repoIndexManager.js";
 
@@ -39,13 +42,14 @@ const prisma = new PrismaClient({
 const redis = new Redis(env.REDIS_URL, {
     maxRetriesPerRequest: null
 });
-redis.ping().then(() => {
+
+try {
+    await redis.ping();
     logger.info('Connected to redis');
-}).catch((err: unknown) => {
+} catch (err: unknown) {
-    logger.error('Failed to connect to redis');
+    logger.error('Failed to connect to redis. Error:', err);
-    logger.error(err);
     process.exit(1);
-});
+}
 
 const promClient = new PromClient();
 
@@ -73,46 +77,74 @@ else if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && hasEntitlement(
     accountPermissionSyncer.startScheduler();
 }
 
+const api = new Api(
+    promClient,
+    prisma,
+    connectionManager,
+    repoIndexManager,
+);
+
 logger.info('Worker started.');
 
-const cleanup = async (signal: string) => {
+const listenToShutdownSignals = () => {
-    logger.info(`Received ${signal}, cleaning up...`);
+    const signals = SHUTDOWN_SIGNALS;
 
-    const shutdownTimeout = 30000; // 30 seconds
+    let receivedSignal = false;
 
-    try {
+    const cleanup = async (signal: string) => {
-        await Promise.race([
+        try {
-            Promise.all([
+            if (receivedSignal) {
-                repoIndexManager.dispose(),
+                return;
-                connectionManager.dispose(),
+            }
-                repoPermissionSyncer.dispose(),
+            receivedSignal = true;
-                accountPermissionSyncer.dispose(),
+
-                promClient.dispose(),
+            logger.info(`Received ${signal}, cleaning up...`);
-                configManager.dispose(),
+
-            ]),
+            await repoIndexManager.dispose()
-            new Promise((_, reject) =>
+            await connectionManager.dispose()
-                setTimeout(() => reject(new Error('Shutdown timeout')), shutdownTimeout)
+            await repoPermissionSyncer.dispose()
-            )
+            await accountPermissionSyncer.dispose()
-        ]);
+            await configManager.dispose()
-        logger.info('All workers shut down gracefully');
+
-    } catch (error) {
+            await prisma.$disconnect();
-        logger.warn('Shutdown timeout or error, forcing exit:', error instanceof Error ? error.message : String(error));
+            await redis.quit();
+            await api.dispose();
+            await shutdownPosthog();
+            
+            logger.info('All workers shut down gracefully');
+            signals.forEach(sig => process.removeListener(sig, cleanup));
+            return 0;
+        } catch (error) {
+            Sentry.captureException(error);
+            logger.error('Error shutting down worker:', error);
+            return 1;
+        }
     }
 
-    await prisma.$disconnect();
+    signals.forEach(signal => {
-    await redis.quit();
+        process.on(signal, (err) => {
+            cleanup(err).then(code => {
+                process.exit(code);
+            });
+        });
+    });
+
+    // Register handlers for uncaught exceptions and unhandled rejections
+    process.on('uncaughtException', (err) => {
+        logger.error(`Uncaught exception: ${err.message}`);
+        cleanup('uncaughtException').then(() => {
+            process.exit(1);
+        });
+    });
+
+    process.on('unhandledRejection', (reason, promise) => {
+        logger.error(`Unhandled rejection at: ${promise}, reason: ${reason}`);
+        cleanup('unhandledRejection').then(() => {
+            process.exit(1);
+        });
+    });
+
+
 }
 
-process.on('SIGINT', () => cleanup('SIGINT').finally(() => process.exit(0)));
+listenToShutdownSignals();
-process.on('SIGTERM', () => cleanup('SIGTERM').finally(() => process.exit(0)));
-
-// Register handlers for uncaught exceptions and unhandled rejections
-process.on('uncaughtException', (err) => {
-    logger.error(`Uncaught exception: ${err.message}`);
-    cleanup('uncaughtException').finally(() => process.exit(1));
-});
-
-process.on('unhandledRejection', (reason, promise) => {
-    logger.error(`Unhandled rejection at: ${promise}, reason: ${reason}`);
-    cleanup('unhandledRejection').finally(() => process.exit(1));
-});

packages/backend/src/posthog.ts

@@ -5,9 +5,9 @@ import { PosthogEvent, PosthogEventMap } from './posthogEvents.js';
 
 let posthog: PostHog | undefined = undefined;
 
-if (clientEnv.NEXT_PUBLIC_POSTHOG_PAPIK) {
+if (env.POSTHOG_PAPIK) {
     posthog = new PostHog(
-        clientEnv.NEXT_PUBLIC_POSTHOG_PAPIK,
+        env.POSTHOG_PAPIK,
         {
             host: "https://us.i.posthog.com",
         }
@@ -29,4 +29,6 @@ export function captureEvent<E extends PosthogEvent>(event: E, properties: Posth
     });
 }
 
-await posthog?.shutdown();
+export async function shutdownPosthog() {
+    await posthog?.shutdown();
+}

packages/backend/src/promClient.ts

@@ -1,14 +1,6 @@
-import express, { Request, Response } from 'express';
-import { Server } from 'http';
 import client, { Registry, Counter, Gauge } from 'prom-client';
-import { createLogger } from "@sourcebot/shared";
-
-const logger = createLogger('prometheus-client');
-
 export class PromClient {
-    private registry: Registry;
+    public registry: Registry;
-    private app: express.Application;
-    private server: Server;
 
     public activeRepoIndexJobs: Gauge<string>;
     public pendingRepoIndexJobs: Gauge<string>;
@@ -22,8 +14,6 @@ export class PromClient {
     public connectionSyncJobFailTotal: Counter<string>;
     public connectionSyncJobSuccessTotal: Counter<string>;
 
-    public readonly PORT = 3060;
-
     constructor() {
         this.registry = new Registry();
 
@@ -100,26 +90,5 @@ export class PromClient {
         client.collectDefaultMetrics({
             register: this.registry,
         });
-
-        this.app = express();
-        this.app.get('/metrics', async (req: Request, res: Response) => {
-            res.set('Content-Type', this.registry.contentType);
-
-            const metrics = await this.registry.metrics();
-            res.end(metrics);
-        });
-
-        this.server = this.app.listen(this.PORT, () => {
-            logger.info(`Prometheus metrics server is running on port ${this.PORT}`);
-        });
-    }
-
-    async dispose() {
-        return new Promise<void>((resolve, reject) => {
-            this.server.close((err) => {
-                if (err) reject(err);
-                else resolve();
-            });
-        });
     }
 }

packages/backend/src/repoCompileUtils.ts

@@ -39,8 +39,8 @@ type CompileResult = {
 export const compileGithubConfig = async (
     config: GithubConnectionConfig,
     connectionId: number,
-    abortController: AbortController): Promise<CompileResult> => {
+    signal: AbortSignal): Promise<CompileResult> => {
-    const gitHubReposResult = await getGitHubReposFromConfig(config, abortController.signal);
+    const gitHubReposResult = await getGitHubReposFromConfig(config, signal);
     const gitHubRepos = gitHubReposResult.repos;
     const warnings = gitHubReposResult.warnings;
 

packages/backend/src/repoIndexManager.ts

@@ -7,12 +7,12 @@ import { readdir, rm } from 'fs/promises';
 import { Job, Queue, ReservedJob, Worker } from "groupmq";
 import { Redis } from 'ioredis';
 import micromatch from 'micromatch';
-import { INDEX_CACHE_DIR } from './constants.js';
+import { GROUPMQ_WORKER_STOP_GRACEFUL_TIMEOUT_MS, INDEX_CACHE_DIR } from './constants.js';
 import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
 import { captureEvent } from './posthog.js';
 import { PromClient } from './promClient.js';
 import { RepoWithConnections, Settings } from "./types.js";
-import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
+import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure, setIntervalAsync } from './utils.js';
 import { indexGitRepository } from './zoekt.js';
 
 const LOG_TAG = 'repo-index-manager';
@@ -45,7 +45,7 @@ export class RepoIndexManager {
     constructor(
         private db: PrismaClient,
         private settings: Settings,
-        redis: Redis,
+        private redis: Redis,
         private promClient: PromClient,
     ) {
         this.queue = new Queue<JobPayload>({
@@ -70,11 +70,15 @@ export class RepoIndexManager {
         this.worker.on('failed', this.onJobFailed.bind(this));
         this.worker.on('stalled', this.onJobStalled.bind(this));
         this.worker.on('error', this.onWorkerError.bind(this));
+        // graceful-timeout is triggered when a job is still processing after
+        // worker.close() is called and the timeout period has elapsed. In this case,
+        // we fail the job with no retry.
+        this.worker.on('graceful-timeout', this.onJobGracefulTimeout.bind(this));
     }
 
-    public async startScheduler() {
+    public startScheduler() {
         logger.debug('Starting scheduler');
-        this.interval = setInterval(async () => {
+        this.interval = setIntervalAsync(async () => {
             await this.scheduleIndexJobs();
             await this.scheduleCleanupJobs();
         }, this.settings.reindexRepoPollingIntervalMs);
@@ -192,7 +196,7 @@ export class RepoIndexManager {
         }
     }
 
-    private async createJobs(repos: Repo[], type: RepoIndexingJobType) {
+    public async createJobs(repos: Repo[], type: RepoIndexingJobType) {
         // @note: we don't perform this in a transaction because
         // we want to avoid the situation where a job is created and run
         // prior to the transaction being committed.
@@ -221,6 +225,8 @@ export class RepoIndexManager {
             const jobTypeLabel = getJobTypePrometheusLabel(type);
             this.promClient.pendingRepoIndexJobs.inc({ repo: job.repo.name, type: jobTypeLabel });
         }
+
+        return jobs.map(job => job.id);
     }
 
     private async runJob(job: ReservedJob<JobPayload>) {
@@ -228,6 +234,23 @@ export class RepoIndexManager {
         const logger = createJobLogger(id);
         logger.info(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`);
 
+        const currentStatus = await this.db.repoIndexingJob.findUniqueOrThrow({
+            where: {
+                id,
+            },
+            select: {
+                status: true,
+            }
+        });
+        
+        // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job
+        // is in an invalid state and should be skipped.
+        if (
+            currentStatus.status !== RepoIndexingJobStatus.PENDING &&
+            currentStatus.status !== RepoIndexingJobStatus.IN_PROGRESS
+        ) {
+            throw new Error(`Job ${id} is not in a valid state. Expected: ${RepoIndexingJobStatus.PENDING} or ${RepoIndexingJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`);
+        }
 
         const { repo, type: jobType } = await this.db.repoIndexingJob.update({
             where: {
@@ -538,6 +561,28 @@ export class RepoIndexManager {
             logger.error(`Job ${jobId} stalled for repo ${repo.name} (id: ${repo.id})`);
         });
 
+    private onJobGracefulTimeout = async (job: Job<JobPayload>) =>
+        groupmqLifecycleExceptionWrapper('onJobGracefulTimeout', logger, async () => {
+            const logger = createJobLogger(job.data.jobId);
+            const jobTypeLabel = getJobTypePrometheusLabel(job.data.type);
+
+            const { repo } = await this.db.repoIndexingJob.update({
+                where: { id: job.data.jobId },
+                data: {
+                    status: RepoIndexingJobStatus.FAILED,
+                    completedAt: new Date(),
+                    errorMessage: 'Job timed out',
+                },
+                select: { repo: true }
+            });
+
+            this.promClient.activeRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel });
+            this.promClient.repoIndexJobFailTotal.inc({ repo: job.data.repoName, type: jobTypeLabel });
+
+            logger.error(`Job ${job.data.jobId} timed out for repo ${repo.name} (id: ${repo.id}). Failing job.`);
+
+        });
+
     private async onWorkerError(error: Error) {
         Sentry.captureException(error);
         logger.error(`Index syncer worker error.`, error);
@@ -547,8 +592,20 @@ export class RepoIndexManager {
         if (this.interval) {
             clearInterval(this.interval);
         }
-        await this.worker.close();
+        const inProgressJobs = this.worker.getCurrentJobs();
-        await this.queue.close();
+        await this.worker.close(GROUPMQ_WORKER_STOP_GRACEFUL_TIMEOUT_MS);
+        // Manually release group locks for in progress jobs to prevent deadlocks.
+        // @see: https://github.com/Openpanel-dev/groupmq/issues/8
+        for (const { job } of inProgressJobs) {
+            const lockKey = `groupmq:repo-index-queue:lock:${job.groupId}`;
+            logger.debug(`Releasing group lock ${lockKey} for in progress job ${job.id}`);
+            await this.redis.del(lockKey);
+        }
+
+        // @note: As of groupmq v1.0.0, queue.close() will just close the underlying
+        // redis connection. Since we share the same redis client between, skip this
+        // step and close the redis client directly in index.ts.
+        // await this.queue.close();
     }
 }
 

packages/backend/src/utils.ts

@@ -268,3 +268,27 @@ export const groupmqLifecycleExceptionWrapper = async (name: string, logger: Log
     }
 }
 
+
+// setInterval wrapper that ensures async callbacks are not executed concurrently.
+// @see: https://mottaquikarim.github.io/dev/posts/setinterval-that-blocks-on-await/
+export const setIntervalAsync = (target: () => Promise<void>, pollingIntervalMs: number): NodeJS.Timeout => {
+    const setIntervalWithPromise = <T extends (...args: any[]) => Promise<any>>(
+        target: T
+    ): (...args: Parameters<T>) => Promise<void> => {
+        return async function (...args: Parameters<T>): Promise<void> {
+            if ((target as any).isRunning) return;
+    
+            (target as any).isRunning = true;
+            try {
+                await target(...args);
+            } finally {
+                (target as any).isRunning = false;
+            }
+        };
+    }
+
+    return setInterval(
+        setIntervalWithPromise(target),
+        pollingIntervalMs
+    );
+}

packages/backend/src/zoekt.ts

@@ -1,5 +1,5 @@
 import { Repo } from "@sourcebot/db";
-import { createLogger } from "@sourcebot/shared";
+import { createLogger, env } from "@sourcebot/shared";
 import { exec } from "child_process";
 import { INDEX_CACHE_DIR } from "./constants.js";
 import { Settings } from "./types.js";
@@ -11,6 +11,8 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, revisio
     const { path: repoPath } = getRepoPath(repo);
     const shardPrefix = getShardPrefix(repo.orgId, repo.id);
 
+    const largeFileGlobPatterns = env.ALWAYS_INDEX_FILE_PATTERNS?.split(',').map(pattern => pattern.trim()) ?? [];
+
     const command = [
         'zoekt-git-index',
         '-allow_missing_branches',
@@ -21,6 +23,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, revisio
         `-tenant_id ${repo.orgId}`,
         `-repo_id ${repo.id}`,
         `-shard_prefix ${shardPrefix}`,
+        ...largeFileGlobPatterns.map((pattern) => `-large_file ${pattern}`),
         repoPath
     ].join(' ');
 

packages/db/prisma/migrations/20251129063148_change_chat_created_by_to_optional/migration.sql

@@ -0,0 +1,5 @@
+-- First, remove the NOT NULL constraint on the createdById column.
+ALTER TABLE "Chat" ALTER COLUMN "createdById" DROP NOT NULL;
+
+-- Then, set all chats created by the guest user (id: 1) to have a NULL createdById.
+UPDATE "Chat" SET "createdById" = NULL WHERE "createdById" = '1';

packages/db/prisma/schema.prisma

@@ -437,8 +437,8 @@ model Chat {
 
   name String?
 
-  createdBy   User   @relation(fields: [createdById], references: [id], onDelete: Cascade)
+  createdBy   User?   @relation(fields: [createdById], references: [id], onDelete: Cascade)
-  createdById String
+  createdById String?
 
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt

packages/db/src/index.ts

@@ -1 +1,3 @@
+import type { User, Account } from ".prisma/client";
+export type UserWithAccounts = User & { accounts: Account[] };
 export * from ".prisma/client";

packages/mcp/CHANGELOG.md

@@ -7,10 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.0.11] - 2025-12-03
+
+### Changed
+- Updated API client to match the latest Sourcebot release. [#652](https://github.com/sourcebot-dev/sourcebot/pull/652)
+
+## [1.0.10] - 2025-11-24
+
+### Changed
+- Updated API client to match the latest Sourcebot release. [#555](https://github.com/sourcebot-dev/sourcebot/pull/555)
+
+## [1.0.9] - 2025-11-17
+
+### Added
+- Added pagination and filtering to `list_repos` tool to handle large repository lists efficiently and prevent oversized responses that waste token context. [#614](https://github.com/sourcebot-dev/sourcebot/pull/614)
+
+## [1.0.8] - 2025-11-10
+
+### Fixed
+- Fixed issue where search results exceeding token limits would be completely discarded instead of returning truncated content. [#604](https://github.com/sourcebot-dev/sourcebot/pull/604)
+
 ## [1.0.7] - 2025-10-28
+
+### Changed
 - Updated API client to match the latest Sourcebot release. [#555](https://github.com/sourcebot-dev/sourcebot/pull/555)
 
 ## [1.0.6] - 2025-09-26
+
+### Fixed
 - Fix `linkedConnections is required` schema error.
 
 ## [1.0.5] - 2025-09-15

packages/mcp/README.md

@@ -182,7 +182,18 @@ Fetches code that matches the provided regex pattern in `query`.
 
 ### list_repos
 
-Lists all repositories indexed by Sourcebot.
+Lists repositories indexed by Sourcebot with optional filtering and pagination.
+
+<details>
+<summary>Parameters</summary>
+
+| Name         | Required | Description                                                         |
+|:-------------|:---------|:--------------------------------------------------------------------|
+| `query`      | no       | Filter repositories by name (case-insensitive).                     |
+| `pageNumber` | no       | Page number (1-indexed, default: 1).                                |
+| `limit`      | no       | Number of repositories per page (default: 50).                      |
+
+</details>
 
 ### get_file_source
 

packages/mcp/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@sourcebot/mcp",
-    "version": "1.0.7",
+    "version": "1.0.11",
     "type": "module",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",

packages/mcp/src/client.ts

@@ -8,7 +8,6 @@ export const search = async (request: SearchRequest): Promise<SearchResponse | S
         method: 'POST',
         headers: {
             'Content-Type': 'application/json',
-            'X-Org-Domain': '~',
             ...(env.SOURCEBOT_API_KEY ? { 'X-Sourcebot-Api-Key': env.SOURCEBOT_API_KEY } : {})
         },
         body: JSON.stringify(request)
@@ -26,7 +25,6 @@ export const listRepos = async (): Promise<ListRepositoriesResponse | ServiceErr
         method: 'GET',
         headers: {
             'Content-Type': 'application/json',
-            'X-Org-Domain': '~',
             ...(env.SOURCEBOT_API_KEY ? { 'X-Sourcebot-Api-Key': env.SOURCEBOT_API_KEY } : {})
         },
     }).then(response => response.json());
@@ -43,7 +41,6 @@ export const getFileSource = async (request: FileSourceRequest): Promise<FileSou
         method: 'POST',
         headers: {
             'Content-Type': 'application/json',
-            'X-Org-Domain': '~',
             ...(env.SOURCEBOT_API_KEY ? { 'X-Sourcebot-Api-Key': env.SOURCEBOT_API_KEY } : {})
         },
         body: JSON.stringify(request)

packages/mcp/src/index.ts

@@ -7,6 +7,7 @@ import escapeStringRegexp from 'escape-string-regexp';
 import { z } from 'zod';
 import { listRepos, search, getFileSource } from './client.js';
 import { env, numberSchema } from './env.js';
+import { listReposRequestSchema } from './schemas.js';
 import { TextContent } from './types.js';
 import { isServiceError } from './utils.js';
 
@@ -69,16 +70,13 @@ server.tool(
             query += ` ( lang:${languages.join(' or lang:')} )`;
         }
 
-        if (caseSensitive) {
-            query += ` case:yes`;
-        } else {
-            query += ` case:no`;
-        }
-
         const response = await search({
             query,
             matches: env.DEFAULT_MATCHES,
             contextLines: env.DEFAULT_CONTEXT_LINES,
+            isRegexEnabled: true,
+            isCaseSensitivityEnabled: caseSensitive,
+            source: 'mcp'
         });
 
         if (isServiceError(response)) {
@@ -123,6 +121,22 @@ server.tool(
             const tokens = text.length / 4;
 
             if ((totalTokens + tokens) > maxTokens) {
+                // Calculate remaining token budget
+                const remainingTokens = maxTokens - totalTokens;
+
+                if (remainingTokens > 100) {  // Only truncate if meaningful space left
+                    // Truncate text to fit remaining tokens (tokens ≈ chars/4)
+                    const maxLength = Math.floor(remainingTokens * 4);
+                    const truncatedText = text.substring(0, maxLength) + "\n\n...[content truncated due to token limit]";
+
+                    content.push({
+                        type: "text",
+                        text: truncatedText,
+                    });
+
+                    totalTokens += remainingTokens;
+                }
+
                 isResponseTruncated = true;
                 break;
             }
@@ -149,8 +163,13 @@ server.tool(
 
 server.tool(
     "list_repos",
-    "Lists all repositories in the organization. If you receive an error that indicates that you're not authenticated, please inform the user to set the SOURCEBOT_API_KEY environment variable.",
+    "Lists repositories in the organization with optional filtering and pagination. If you receive an error that indicates that you're not authenticated, please inform the user to set the SOURCEBOT_API_KEY environment variable.",
-    async () => {
+    listReposRequestSchema.shape,
+    async ({ query, pageNumber = 1, limit = 50 }: {
+        query?: string;
+        pageNumber?: number;
+        limit?: number;
+    }) => {
         const response = await listRepos();
         if (isServiceError(response)) {
             return {
@@ -161,13 +180,45 @@ server.tool(
             };
         }
 
-        const content: TextContent[] = response.map(repo => {
+        // Apply query filter if provided
+        let filtered = response;
+        if (query) {
+            const lowerQuery = query.toLowerCase();
+            filtered = response.filter(repo =>
+                repo.repoName.toLowerCase().includes(lowerQuery) ||
+                repo.repoDisplayName?.toLowerCase().includes(lowerQuery)
+            );
+        }
+
+        // Sort alphabetically for consistent pagination
+        filtered.sort((a, b) => a.repoName.localeCompare(b.repoName));
+
+        // Apply pagination
+        const startIndex = (pageNumber - 1) * limit;
+        const endIndex = startIndex + limit;
+        const paginated = filtered.slice(startIndex, endIndex);
+
+        // Format output
+        const content: TextContent[] = paginated.map(repo => {
             return {
                 type: "text",
                 text: `id: ${repo.repoName}\nurl: ${repo.webUrl}`,
             }
         });
 
+        // Add pagination info
+        if (content.length === 0 && filtered.length > 0) {
+            content.push({
+                type: "text",
+                text: `No results on page ${pageNumber}. Total matching repositories: ${filtered.length}`,
+            });
+        } else if (filtered.length > endIndex) {
+            content.push({
+                type: "text",
+                text: `Showing ${paginated.length} repositories (page ${pageNumber}). Total matching: ${filtered.length}. Use pageNumber ${pageNumber + 1} to see more.`,
+            });
+        }
+
         return {
             content,
         };

packages/mcp/src/schemas.ts

@@ -21,15 +21,18 @@ export const symbolSchema = z.object({
     kind: z.string(),
 });
 
+export const searchOptionsSchema = z.object({
+    matches: z.number(),                              // The number of matches to return.
+    contextLines: z.number().optional(),              // The number of context lines to return.
+    whole: z.boolean().optional(),                    // Whether to return the whole file as part of the response.
+    isRegexEnabled: z.boolean().optional(),           // Whether to enable regular expression search.
+    isCaseSensitivityEnabled: z.boolean().optional(), // Whether to enable case sensitivity.
+});
+
 export const searchRequestSchema = z.object({
-    // The zoekt query to execute.
+    query: z.string(),                                // The zoekt query to execute.
-    query: z.string(),
+    source: z.string().optional(),                    // The source of the search request.
-    // The number of matches to return.
+    ...searchOptionsSchema.shape,
-    matches: z.number(),
-    // The number of context lines to return.
-    contextLines: z.number().optional(),
-    // Whether to return the whole file as part of the response.
-    whole: z.boolean().optional(),
 });
 
 export const repositoryInfoSchema = z.object({
@@ -109,7 +112,7 @@ export const searchStatsSchema = z.object({
     regexpsConsidered: z.number(),
 
     // FlushReason explains why results were flushed.
-    flushReason: z.number(),
+    flushReason: z.string(),
 });
 
 export const searchResponseSchema = z.object({
@@ -139,7 +142,6 @@ export const searchResponseSchema = z.object({
         content: z.string().optional(),
     })),
     repositoryInfo: z.array(repositoryInfoSchema),
-    isBranchFilteringEnabled: z.boolean(),
     isSearchExhaustive: z.boolean(),
 });
 
@@ -156,6 +158,25 @@ export const repositoryQuerySchema = z.object({
 
 export const listRepositoriesResponseSchema = repositoryQuerySchema.array();
 
+export const listReposRequestSchema = z.object({
+    query: z
+        .string()
+        .describe("Filter repositories by name or displayName (case-insensitive)")
+        .optional(),
+    pageNumber: z
+        .number()
+        .int()
+        .positive()
+        .describe("Page number (1-indexed, default: 1)")
+        .default(1),
+    limit: z
+        .number()
+        .int()
+        .positive()
+        .describe("Number of repositories per page (default: 50)")
+        .default(50),
+});
+
 export const fileSourceRequestSchema = z.object({
     fileName: z.string(),
     repository: z.string(),

packages/queryLanguage/.gitignore

@@ -0,0 +1,2 @@
+/node_modules/
+/dist

packages/queryLanguage/package.json

@@ -0,0 +1,20 @@
+{
+    "name": "@sourcebot/query-language",
+    "private": true,
+    "main": "dist/index.js",
+    "scripts": {
+        "build": "lezer-generator src/query.grammar -o src/parser --typeScript --names && tsc",
+        "test": "vitest",
+        "postinstall": "yarn build"
+    },
+    "devDependencies": {
+        "@lezer/generator": "^1.8.0",
+        "tsx": "^4.19.1",
+        "typescript": "^5.7.3",
+        "vitest": "^2.1.9"
+    },
+    "dependencies": {
+        "@lezer/common": "^1.3.0",
+        "@lezer/lr": "^1.4.3"
+    }
+}

packages/queryLanguage/src/index.ts

@@ -0,0 +1,7 @@
+import { parser } from "./parser";
+
+type Tree = ReturnType<typeof parser.parse>;
+type SyntaxNode = Tree['topNode'];
+export type { Tree, SyntaxNode };
+export * from "./parser";
+export * from "./parser.terms";

packages/queryLanguage/src/parser.terms.ts

@@ -0,0 +1,22 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+export const
+  negate = 23,
+  Program = 1,
+  OrExpr = 2,
+  AndExpr = 3,
+  NegateExpr = 4,
+  PrefixExpr = 5,
+  ArchivedExpr = 6,
+  RevisionExpr = 7,
+  ContentExpr = 8,
+  ContextExpr = 9,
+  FileExpr = 10,
+  ForkExpr = 11,
+  VisibilityExpr = 12,
+  RepoExpr = 13,
+  LangExpr = 14,
+  SymExpr = 15,
+  RepoSetExpr = 16,
+  ParenExpr = 17,
+  QuotedTerm = 18,
+  Term = 19

packages/queryLanguage/src/query.grammar

@@ -0,0 +1,102 @@
+@external tokens negateToken from "./tokens" { negate }
+
+@top Program { query }
+
+@precedence {
+  negate,
+  and,
+  or @left
+}
+
+query {
+  OrExpr |
+  AndExpr |
+  expr
+}
+
+OrExpr { andExpr (or andExpr)+ }
+
+AndExpr { expr expr+ }
+
+andExpr { AndExpr | expr }
+
+expr {
+  NegateExpr |
+  ParenExpr |
+  PrefixExpr |
+  QuotedTerm |
+  Term
+}
+
+NegateExpr { !negate negate (PrefixExpr | ParenExpr) }
+
+ParenExpr { "(" query? ")" }
+
+PrefixExpr {
+  ArchivedExpr |
+  RevisionExpr |
+  ContentExpr |
+  ContextExpr |
+  FileExpr |
+  ForkExpr |
+  VisibilityExpr |
+  RepoExpr |
+  LangExpr |
+  SymExpr |
+  RepoSetExpr
+}
+
+RevisionExpr { revisionKw value }
+ContentExpr { contentKw value }
+ContextExpr { contextKw value }
+FileExpr { fileKw value }
+RepoExpr { repoKw value }
+LangExpr { langKw value }
+SymExpr { symKw value }
+RepoSetExpr { reposetKw value }
+
+// Modifiers
+ArchivedExpr { archivedKw archivedValue }
+ForkExpr { forkKw forkValue }
+VisibilityExpr { visibilityKw visibilityValue }
+
+archivedValue { "yes" | "no" | "only" }
+forkValue { "yes" | "no" | "only" }
+visibilityValue { "public" | "private" | "any" }
+
+QuotedTerm { quotedString }
+Term { word }
+
+value { quotedString | word }
+
+@skip { space }
+
+@tokens {
+  archivedKw { "archived:" }
+  revisionKw { "rev:" }
+  contentKw { "content:" | "c:" }
+  contextKw { "context:" }
+  fileKw { "file:" | "f:" }
+  forkKw { "fork:" }
+  visibilityKw { "visibility:" }
+  repoKw { "repo:" | "r:" }
+  langKw { "lang:" }
+  symKw { "sym:" }
+  reposetKw { "reposet:" }
+  
+  or { "or" ![a-zA-Z0-9_] }
+  
+  quotedString { '"' (!["\\\n] | "\\" _)* '"' }
+  
+  word { (![ \t\n()]) (![ \t\n():] | ":" | "-")* }
+  
+  space { $[ \t\n]+ }
+  
+  @precedence { 
+    quotedString,
+    archivedKw, revisionKw, contentKw, contextKw, fileKw, 
+    forkKw, visibilityKw, repoKw, langKw, 
+    symKw, reposetKw, or,
+    word 
+  }
+}

packages/queryLanguage/src/tokens.ts

@@ -0,0 +1,59 @@
+import { ExternalTokenizer } from "@lezer/lr";
+import { negate } from "./parser.terms";
+
+// External tokenizer for negation
+// Only tokenizes `-` as negate when followed by a prefix keyword or `(`
+export const negateToken = new ExternalTokenizer((input) => {
+    if (input.next !== 45 /* '-' */) return; // Not a dash
+    
+    const startPos = input.pos;
+    
+    // Look ahead to see what follows the dash
+    input.advance();
+    
+    // Skip whitespace
+    let ch = input.next;
+    while (ch === 32 || ch === 9 || ch === 10) {
+        input.advance();
+        ch = input.next;
+    }
+    
+    // Check if followed by opening paren
+    if (ch === 40 /* '(' */) {
+        input.acceptToken(negate, -input.pos + startPos + 1); // Accept just the dash
+        return;
+    }
+    
+    // Check if followed by a prefix keyword (by checking for keyword followed by colon)
+    // Look ahead until we hit a delimiter or colon
+    const checkPos = input.pos;
+    let foundColon = false;
+    
+    // Look ahead until we hit a delimiter or colon
+    while (ch >= 0) {
+        if (ch === 58 /* ':' */) {
+            foundColon = true;
+            break;
+        }
+        // Hit a delimiter (whitespace, paren, or quote) - not a prefix keyword
+        if (ch === 32 || ch === 9 || ch === 10 || ch === 40 || ch === 41 || ch === 34) {
+            break;
+        }
+        input.advance();
+        ch = input.next;
+    }
+    
+    // Reset position
+    while (input.pos > checkPos) {
+        input.advance(-1);
+    }
+    
+    if (foundColon) {
+        // It's a prefix keyword, accept as negate
+        input.acceptToken(negate, -input.pos + startPos + 1);
+        return;
+    }
+    
+    // Otherwise, don't tokenize as negate (let word handle it)
+});
+

packages/queryLanguage/test/basic.txt

@@ -0,0 +1,72 @@
+# Single term
+
+hello
+
+==>
+
+Program(Term)
+
+# Multiple terms
+
+hello world
+
+==>
+
+Program(AndExpr(Term,Term))
+
+# Multiple terms with various characters
+
+console.log error_handler
+
+==>
+
+Program(AndExpr(Term,Term))
+
+# Term with underscores
+
+my_variable_name
+
+==>
+
+Program(Term)
+
+# Term with dots
+
+com.example.package
+
+==>
+
+Program(Term)
+
+# Term with numbers
+
+func123 test_456
+
+==>
+
+Program(AndExpr(Term,Term))
+
+# Regex pattern
+
+[a-z]+
+
+==>
+
+Program(Term)
+
+# Wildcard pattern
+
+test.*
+
+==>
+
+Program(Term)
+
+# Multiple regex patterns
+
+\w+ [0-9]+ \s*
+
+==>
+
+Program(AndExpr(Term,Term,Term))
+

packages/queryLanguage/test/grammar.test.ts

@@ -0,0 +1,21 @@
+import { parser } from "../src/parser";
+import { fileTests } from "@lezer/generator/dist/test";
+import { describe, it } from "vitest";
+import { fileURLToPath } from "url"
+import * as fs from "fs";
+import * as path from "path";
+
+const caseDir = path.dirname(fileURLToPath(import.meta.url))
+
+for (const file of fs.readdirSync(caseDir)) {
+    if (!/\.txt$/.test(file)) {
+        continue;
+    }
+
+    let name = /^[^\.]*/.exec(file)?.[0];
+    describe(name ?? "unknown", () => {
+        for (const { name, run } of fileTests(fs.readFileSync(path.join(caseDir, file), "utf8"), file)) {
+            it(name, () => run(parser));
+        }
+    });
+}

packages/queryLanguage/test/grouping.txt

@@ -0,0 +1,120 @@
+# Empty parentheses
+
+()
+
+==>
+
+Program(ParenExpr)
+
+# Simple grouping
+
+(test)
+
+==>
+
+Program(ParenExpr(Term))
+
+# Multiple terms in group
+
+(hello world)
+
+==>
+
+Program(ParenExpr(AndExpr(Term,Term)))
+
+# Nested parentheses
+
+((test))
+
+==>
+
+Program(ParenExpr(ParenExpr(Term)))
+
+# Multiple groups
+
+(first) (second)
+
+==>
+
+Program(AndExpr(ParenExpr(Term),ParenExpr(Term)))
+
+# Group with multiple terms
+
+(one two three)
+
+==>
+
+Program(ParenExpr(AndExpr(Term,Term,Term)))
+
+# Mixed grouped and ungrouped
+
+test (grouped) another
+
+==>
+
+Program(AndExpr(Term,ParenExpr(Term),Term))
+
+# Deeply nested
+
+(((nested)))
+
+==>
+
+Program(ParenExpr(ParenExpr(ParenExpr(Term))))
+
+# Multiple nested groups
+
+((a b) (c d))
+
+==>
+
+Program(ParenExpr(AndExpr(ParenExpr(AndExpr(Term,Term)),ParenExpr(AndExpr(Term,Term)))))
+
+# Group at start
+
+(start) middle end
+
+==>
+
+Program(AndExpr(ParenExpr(Term),Term,Term))
+
+# Group at end
+
+start middle (end)
+
+==>
+
+Program(AndExpr(Term,Term,ParenExpr(Term)))
+
+# Complex grouping pattern
+
+(a (b c) d)
+
+==>
+
+Program(ParenExpr(AndExpr(Term,ParenExpr(AndExpr(Term,Term)),Term)))
+
+# Sequential groups
+
+(a)(b)(c)
+
+==>
+
+Program(AndExpr(ParenExpr(Term),ParenExpr(Term),ParenExpr(Term)))
+
+# Group with regex
+
+([a-z]+)
+
+==>
+
+Program(ParenExpr(Term))
+
+# Group with dots
+
+(com.example.test)
+
+==>
+
+Program(ParenExpr(Term))
+

packages/queryLanguage/test/negation.txt

@@ -0,0 +1,255 @@
+# Literal dash term
+
+-test
+
+==>
+
+Program(Term)
+
+# Quoted dash term
+
+"-excluded"
+
+==>
+
+Program(QuotedTerm)
+
+# Dash in middle
+
+test-case
+
+==>
+
+Program(Term)
+
+# Multiple dash terms
+
+-one -two -three
+
+==>
+
+Program(AndExpr(Term,Term,Term))
+
+# Negate file prefix
+
+-file:test.js
+
+==>
+
+Program(NegateExpr(PrefixExpr(FileExpr)))
+
+# Negate repo prefix
+
+-repo:archived
+
+==>
+
+Program(NegateExpr(PrefixExpr(RepoExpr)))
+
+# Negate lang prefix
+
+-lang:python
+
+==>
+
+Program(NegateExpr(PrefixExpr(LangExpr)))
+
+# Negate content prefix
+
+-content:TODO
+
+==>
+
+Program(NegateExpr(PrefixExpr(ContentExpr)))
+
+# Negate revision prefix
+
+-rev:develop
+
+==>
+
+Program(NegateExpr(PrefixExpr(RevisionExpr)))
+
+# Negate archived prefix
+
+-archived:yes
+
+==>
+
+Program(NegateExpr(PrefixExpr(ArchivedExpr)))
+
+# Negate fork prefix
+
+-fork:yes
+
+==>
+
+Program(NegateExpr(PrefixExpr(ForkExpr)))
+
+# Negate visibility prefix
+
+-visibility:any
+
+==>
+
+Program(NegateExpr(PrefixExpr(VisibilityExpr)))
+
+# Negate context prefix
+
+-context:backend
+
+==>
+
+Program(NegateExpr(PrefixExpr(ContextExpr)))
+
+# Negate symbol prefix
+
+-sym:OldClass
+
+==>
+
+Program(NegateExpr(PrefixExpr(SymExpr)))
+
+# Negate parentheses
+
+-(test)
+
+==>
+
+Program(NegateExpr(ParenExpr(Term)))
+
+# Negate group with multiple terms
+
+-(test exclude)
+
+==>
+
+Program(NegateExpr(ParenExpr(AndExpr(Term,Term))))
+
+# Negate group with prefix
+
+-(file:test.js console.log)
+
+==>
+
+Program(NegateExpr(ParenExpr(AndExpr(PrefixExpr(FileExpr),Term))))
+
+# Prefix with negated term
+
+file:test.js -console
+
+==>
+
+Program(AndExpr(PrefixExpr(FileExpr),Term))
+
+# Multiple prefixes with negation
+
+file:test.js -lang:python
+
+==>
+
+Program(AndExpr(PrefixExpr(FileExpr),NegateExpr(PrefixExpr(LangExpr))))
+
+# Complex negation pattern
+
+function -file:test.js -lang:java
+
+==>
+
+Program(AndExpr(Term,NegateExpr(PrefixExpr(FileExpr)),NegateExpr(PrefixExpr(LangExpr))))
+
+# Negation inside parentheses
+
+(-file:test.js)
+
+==>
+
+Program(ParenExpr(NegateExpr(PrefixExpr(FileExpr))))
+
+# Multiple negations in group
+
+(-file:a.js -lang:python)
+
+==>
+
+Program(ParenExpr(AndExpr(NegateExpr(PrefixExpr(FileExpr)),NegateExpr(PrefixExpr(LangExpr)))))
+
+# Mixed in parentheses
+
+(include -file:test.js)
+
+==>
+
+Program(ParenExpr(AndExpr(Term,NegateExpr(PrefixExpr(FileExpr)))))
+
+# Negate nested group
+
+-((file:test.js))
+
+==>
+
+Program(NegateExpr(ParenExpr(ParenExpr(PrefixExpr(FileExpr)))))
+
+# Negate short form prefix
+
+-f:test.js
+
+==>
+
+Program(NegateExpr(PrefixExpr(FileExpr)))
+
+# Negate short form repo
+
+-r:myrepo
+
+==>
+
+Program(NegateExpr(PrefixExpr(RepoExpr)))
+
+# Negate short form content
+
+-c:console
+
+==>
+
+Program(NegateExpr(PrefixExpr(ContentExpr)))
+
+# Negate with prefix in quotes
+
+-file:"test file.js"
+
+==>
+
+Program(NegateExpr(PrefixExpr(FileExpr)))
+
+# Complex with multiple negated prefixes
+
+lang:typescript -file:*.test.ts -file:*.spec.ts
+
+==>
+
+Program(AndExpr(PrefixExpr(LangExpr),NegateExpr(PrefixExpr(FileExpr)),NegateExpr(PrefixExpr(FileExpr))))
+
+# Negated group with prefix
+
+-(file:test.js lang:python)
+
+==>
+
+Program(NegateExpr(ParenExpr(AndExpr(PrefixExpr(FileExpr),PrefixExpr(LangExpr)))))
+
+# Negate empty group
+
+-()
+
+==>
+
+Program(NegateExpr(ParenExpr))
+
+# Negate with space after dash
+
+- file:test.js
+
+==>
+
+Program(NegateExpr(PrefixExpr(FileExpr)))

packages/queryLanguage/test/operators.txt

@@ -0,0 +1,271 @@
+# Simple OR
+
+test or example
+
+==>
+
+Program(OrExpr(Term,Term))
+
+# Multiple OR
+
+one or two or three
+
+==>
+
+Program(OrExpr(Term,Term,Term))
+
+# OR with prefixes
+
+file:test.js or file:example.js
+
+==>
+
+Program(OrExpr(PrefixExpr(FileExpr),PrefixExpr(FileExpr)))
+
+# OR with negation
+
+test or -file:excluded.js
+
+==>
+
+Program(OrExpr(Term,NegateExpr(PrefixExpr(FileExpr))))
+
+# OR with quoted strings
+
+"first option" or "second option"
+
+==>
+
+Program(OrExpr(QuotedTerm,QuotedTerm))
+
+# OR with different prefixes
+
+lang:python or lang:javascript
+
+==>
+
+Program(OrExpr(PrefixExpr(LangExpr),PrefixExpr(LangExpr)))
+
+# Multiple terms with OR
+
+function test or class example
+
+==>
+
+Program(OrExpr(AndExpr(Term,Term),AndExpr(Term,Term)))
+
+# OR in parentheses
+
+(test or example)
+
+==>
+
+Program(ParenExpr(OrExpr(Term,Term)))
+
+# OR with parentheses outside
+
+(test) or (example)
+
+==>
+
+Program(OrExpr(ParenExpr(Term),ParenExpr(Term)))
+
+# Complex OR with grouping
+
+(file:*.js lang:javascript) or (file:*.ts lang:typescript)
+
+==>
+
+Program(OrExpr(ParenExpr(AndExpr(PrefixExpr(FileExpr),PrefixExpr(LangExpr))),ParenExpr(AndExpr(PrefixExpr(FileExpr),PrefixExpr(LangExpr)))))
+
+# OR with mixed content
+
+test or file:example.js
+
+==>
+
+Program(OrExpr(Term,PrefixExpr(FileExpr)))
+
+# Prefix OR term
+
+file:test.js or example
+
+==>
+
+Program(OrExpr(PrefixExpr(FileExpr),Term))
+
+# OR with short form prefixes
+
+f:test.js or r:myrepo
+
+==>
+
+Program(OrExpr(PrefixExpr(FileExpr),PrefixExpr(RepoExpr)))
+
+# OR with repo prefixes
+
+repo:project1 or repo:project2
+
+==>
+
+Program(OrExpr(PrefixExpr(RepoExpr),PrefixExpr(RepoExpr)))
+
+# OR with revision prefixes
+
+rev:main or rev:develop
+
+==>
+
+Program(OrExpr(PrefixExpr(RevisionExpr),PrefixExpr(RevisionExpr)))
+
+# OR with lang prefixes
+
+lang:rust or lang:go
+
+==>
+
+Program(OrExpr(PrefixExpr(LangExpr),PrefixExpr(LangExpr)))
+
+# OR with content
+
+content:TODO or content:FIXME
+
+==>
+
+Program(OrExpr(PrefixExpr(ContentExpr),PrefixExpr(ContentExpr)))
+
+# OR with negated terms
+
+-file:test.js or -file:spec.js
+
+==>
+
+Program(OrExpr(NegateExpr(PrefixExpr(FileExpr)),NegateExpr(PrefixExpr(FileExpr))))
+
+# OR in nested parentheses
+
+((a or b) or (c or d))
+
+==>
+
+Program(ParenExpr(OrExpr(ParenExpr(OrExpr(Term,Term)),ParenExpr(OrExpr(Term,Term)))))
+
+# Multiple OR with parentheses and implicit AND
+
+(a or b) and (c or d)
+
+==>
+
+Program(AndExpr(ParenExpr(OrExpr(Term,Term)),Term,ParenExpr(OrExpr(Term,Term))))
+
+# OR with wildcards
+
+*.test.js or *.spec.js
+
+==>
+
+Program(OrExpr(Term,Term))
+
+# OR with regex patterns
+
+[a-z]+ or [0-9]+
+
+==>
+
+Program(OrExpr(Term,Term))
+
+# OR with dots
+
+com.example.test or org.example.test
+
+==>
+
+Program(OrExpr(Term,Term))
+
+# OR with dashes
+
+test-one or test-two
+
+==>
+
+Program(OrExpr(Term,Term))
+
+# Word containing 'or'
+
+order
+
+==>
+
+Program(Term)
+
+# Word containing 'or' in middle
+
+before
+
+==>
+
+Program(Term)
+
+# OR at start
+
+or test
+
+==>
+
+Program(⚠,Term)
+
+# OR at end (or becomes term)
+
+test or
+
+==>
+
+Program(AndExpr(Term,Term))
+
+# Multiple consecutive OR
+
+test or or example
+
+==>
+
+Program(OrExpr(Term,⚠,Term))
+
+# OR with all prefix types
+
+file:*.js or repo:myrepo or lang:javascript
+
+==>
+
+Program(OrExpr(PrefixExpr(FileExpr),PrefixExpr(RepoExpr),PrefixExpr(LangExpr)))
+
+# Complex query with OR and negation
+
+(lang:python or lang:ruby) -file:test.py
+
+==>
+
+Program(AndExpr(ParenExpr(OrExpr(PrefixExpr(LangExpr),PrefixExpr(LangExpr))),NegateExpr(PrefixExpr(FileExpr))))
+
+# OR with quoted prefix values
+
+file:"test one.js" or file:"test two.js"
+
+==>
+
+Program(OrExpr(PrefixExpr(FileExpr),PrefixExpr(FileExpr)))
+
+# OR with empty parentheses
+
+() or ()
+
+==>
+
+Program(OrExpr(ParenExpr,ParenExpr))
+
+# OR with negated groups
+
+-(file:a.js) or -(file:b.js)
+
+==>
+
+Program(OrExpr(NegateExpr(ParenExpr(PrefixExpr(FileExpr))),NegateExpr(ParenExpr(PrefixExpr(FileExpr)))))

packages/queryLanguage/test/precedence.txt

@@ -0,0 +1,200 @@
+# OR has lowest precedence - implicit AND groups first
+
+a b or c d
+
+==>
+
+Program(OrExpr(AndExpr(Term,Term),AndExpr(Term,Term)))
+
+# Multiple OR operators are left-associative
+
+a or b or c
+
+==>
+
+Program(OrExpr(Term,Term,Term))
+
+# AND before OR
+
+file:test.js error or file:test.go panic
+
+==>
+
+Program(OrExpr(AndExpr(PrefixExpr(FileExpr),Term),AndExpr(PrefixExpr(FileExpr),Term)))
+
+# Negation binds tighter than AND
+
+-file:test.js error
+
+==>
+
+Program(AndExpr(NegateExpr(PrefixExpr(FileExpr)),Term))
+
+# Negation binds tighter than OR
+
+-file:a.js or file:b.js
+
+==>
+
+Program(OrExpr(NegateExpr(PrefixExpr(FileExpr)),PrefixExpr(FileExpr)))
+
+# Parentheses override precedence
+
+(a or b) c
+
+==>
+
+Program(AndExpr(ParenExpr(OrExpr(Term,Term)),Term))
+
+# Parentheses override - OR inside parens groups first
+
+a (b or c)
+
+==>
+
+Program(AndExpr(Term,ParenExpr(OrExpr(Term,Term))))
+
+# Complex: AND, OR, and negation
+
+a -b or c d
+
+==>
+
+Program(OrExpr(AndExpr(Term,Term),AndExpr(Term,Term)))
+
+# Negated group in OR expression
+
+-(a b) or c
+
+==>
+
+Program(OrExpr(NegateExpr(ParenExpr(AndExpr(Term,Term))),Term))
+
+# Multiple negations in OR
+
+-file:a.js or -file:b.js or file:c.js
+
+==>
+
+Program(OrExpr(NegateExpr(PrefixExpr(FileExpr)),NegateExpr(PrefixExpr(FileExpr)),PrefixExpr(FileExpr)))
+
+# Prefix binds to its value only
+
+file:a.js b.js
+
+==>
+
+Program(AndExpr(PrefixExpr(FileExpr),Term))
+
+# OR with prefixes and terms mixed
+
+repo:backend error or repo:frontend warning
+
+==>
+
+Program(OrExpr(AndExpr(PrefixExpr(RepoExpr),Term),AndExpr(PrefixExpr(RepoExpr),Term)))
+
+# Nested parentheses with OR
+
+((a or b) c) or d
+
+==>
+
+Program(OrExpr(ParenExpr(AndExpr(ParenExpr(OrExpr(Term,Term)),Term)),Term))
+
+# OR at different nesting levels
+
+(a or (b or c))
+
+==>
+
+Program(ParenExpr(OrExpr(Term,ParenExpr(OrExpr(Term,Term)))))
+
+# Implicit AND groups all adjacent terms before OR
+
+a b c or d e f
+
+==>
+
+Program(OrExpr(AndExpr(Term,Term,Term),AndExpr(Term,Term,Term)))
+
+# Mixed prefix and regular terms with OR
+
+lang:go func or lang:rust fn
+
+==>
+
+Program(OrExpr(AndExpr(PrefixExpr(LangExpr),Term),AndExpr(PrefixExpr(LangExpr),Term)))
+
+# Negation doesn't affect OR grouping
+
+a or -b or c
+
+==>
+
+Program(OrExpr(Term,Term,Term))
+
+# Parentheses can isolate OR from surrounding AND
+
+a (b or c) d
+
+==>
+
+Program(AndExpr(Term,ParenExpr(OrExpr(Term,Term)),Term))
+
+# Multiple parenthesized groups with AND
+
+(a or b) (c or d)
+
+==>
+
+Program(AndExpr(ParenExpr(OrExpr(Term,Term)),ParenExpr(OrExpr(Term,Term))))
+
+# Quoted strings are atomic - no precedence inside
+
+"a or b"
+
+==>
+
+Program(QuotedTerm)
+
+# Prefix with OR value doesn't split
+
+file:"a.js or b.js"
+
+==>
+
+Program(PrefixExpr(FileExpr))
+
+# Negated prefix in complex expression
+
+-file:test.js lang:go error or warning
+
+==>
+
+Program(OrExpr(AndExpr(NegateExpr(PrefixExpr(FileExpr)),PrefixExpr(LangExpr),Term),Term))
+
+# OR followed by parenthesized AND
+
+a or (b c)
+
+==>
+
+Program(OrExpr(Term,ParenExpr(AndExpr(Term,Term))))
+
+# Empty parens don't affect precedence
+
+() or a b
+
+==>
+
+Program(OrExpr(ParenExpr,AndExpr(Term,Term)))
+
+# Negation of empty group
+
+-() a
+
+==>
+
+Program(AndExpr(NegateExpr(ParenExpr),Term))
+

packages/queryLanguage/test/prefixes.txt

@@ -0,0 +1,336 @@
+# File prefix
+
+file:README.md
+
+==>
+
+Program(PrefixExpr(FileExpr))
+
+# File prefix short form
+
+f:index.ts
+
+==>
+
+Program(PrefixExpr(FileExpr))
+
+# Repo prefix
+
+repo:myproject
+
+==>
+
+Program(PrefixExpr(RepoExpr))
+
+# Repo prefix short form
+
+r:github.com/user/repo
+
+==>
+
+Program(PrefixExpr(RepoExpr))
+
+# Content prefix
+
+content:function
+
+==>
+
+Program(PrefixExpr(ContentExpr))
+
+# Content prefix short form
+
+c:console.log
+
+==>
+
+Program(PrefixExpr(ContentExpr))
+
+# Revision prefix
+
+rev:main
+
+==>
+
+Program(PrefixExpr(RevisionExpr))
+
+# Lang prefix
+
+lang:typescript
+
+==>
+
+Program(PrefixExpr(LangExpr))
+
+# Archived prefix - no
+
+archived:no
+
+==>
+
+Program(PrefixExpr(ArchivedExpr))
+
+# Archived prefix - only
+
+archived:only
+
+==>
+
+Program(PrefixExpr(ArchivedExpr))
+
+# Fork prefix - yes
+
+fork:yes
+
+==>
+
+Program(PrefixExpr(ForkExpr))
+
+# Fork prefix - only
+
+fork:only
+
+==>
+
+Program(PrefixExpr(ForkExpr))
+
+# Visibility prefix - public
+
+visibility:public
+
+==>
+
+Program(PrefixExpr(VisibilityExpr))
+
+# Context prefix
+
+context:web
+
+==>
+
+Program(PrefixExpr(ContextExpr))
+
+# Symbol prefix
+
+sym:MyClass
+
+==>
+
+Program(PrefixExpr(SymExpr))
+
+# RepoSet prefix
+
+reposet:repo1,repo2
+
+==>
+
+Program(PrefixExpr(RepoSetExpr))
+
+# File with wildcard
+
+file:*.ts
+
+==>
+
+Program(PrefixExpr(FileExpr))
+
+# File with path
+
+file:src/components/Button.tsx
+
+==>
+
+Program(PrefixExpr(FileExpr))
+
+# Repo with full URL
+
+repo:github.com/org/project
+
+==>
+
+Program(PrefixExpr(RepoExpr))
+
+# Multiple prefixes
+
+file:test.js repo:myproject
+
+==>
+
+Program(AndExpr(PrefixExpr(FileExpr),PrefixExpr(RepoExpr)))
+
+# Prefix with term
+
+file:test.js console.log
+
+==>
+
+Program(AndExpr(PrefixExpr(FileExpr),Term))
+
+# Term then prefix
+
+console.log file:handler.ts
+
+==>
+
+Program(AndExpr(Term,PrefixExpr(FileExpr)))
+
+# Multiple prefixes and terms
+
+lang:typescript function file:handler.ts
+
+==>
+
+Program(AndExpr(PrefixExpr(LangExpr),Term,PrefixExpr(FileExpr)))
+
+# Prefix with regex pattern
+
+file:[a-z]+\.test\.js
+
+==>
+
+Program(PrefixExpr(FileExpr))
+
+# Content with spaces in value (no quotes)
+
+content:hello
+
+==>
+
+Program(PrefixExpr(ContentExpr))
+
+# Revision with slashes
+
+rev:feature/new-feature
+
+==>
+
+Program(PrefixExpr(RevisionExpr))
+
+# RepoSet with multiple repos
+
+reposet:repo1,repo2,repo3
+
+==>
+
+Program(PrefixExpr(RepoSetExpr))
+
+# Symbol with dots
+
+sym:package.Class.method
+
+==>
+
+Program(PrefixExpr(SymExpr))
+
+# Lang with various languages
+
+lang:python
+
+==>
+
+Program(PrefixExpr(LangExpr))
+
+# Archived prefix - yes
+
+archived:yes
+
+==>
+
+Program(PrefixExpr(ArchivedExpr))
+
+# Archived prefix - invalid value (error case)
+
+archived:invalid
+
+==>
+
+Program(AndExpr(PrefixExpr(ArchivedExpr(⚠)),Term))
+
+# Fork prefix - no
+
+fork:no
+
+==>
+
+Program(PrefixExpr(ForkExpr))
+
+# Fork prefix - invalid value (error case)
+
+fork:invalid
+
+==>
+
+Program(AndExpr(PrefixExpr(ForkExpr(⚠)),Term))
+
+# Visibility prefix - private
+
+visibility:private
+
+==>
+
+Program(PrefixExpr(VisibilityExpr))
+
+# Visibility prefix - any
+
+visibility:any
+
+==>
+
+Program(PrefixExpr(VisibilityExpr))
+
+# Visibility prefix - invalid value (error case)
+
+visibility:invalid
+
+==>
+
+Program(AndExpr(PrefixExpr(VisibilityExpr(⚠)),Term))
+
+# File with dashes
+
+file:my-component.tsx
+
+==>
+
+Program(PrefixExpr(FileExpr))
+
+# Repo with numbers
+
+repo:project123
+
+==>
+
+Program(PrefixExpr(RepoExpr))
+
+# Content with special chars
+
+content:@Component
+
+==>
+
+Program(PrefixExpr(ContentExpr))
+
+# Context with underscores
+
+context:data_engineering
+
+==>
+
+Program(PrefixExpr(ContextExpr))
+
+# Prefix in parentheses
+
+(file:test.js)
+
+==>
+
+Program(ParenExpr(PrefixExpr(FileExpr)))
+
+# Multiple prefixes in group
+
+(file:*.ts lang:typescript)
+
+==>
+
+Program(ParenExpr(AndExpr(PrefixExpr(FileExpr),PrefixExpr(LangExpr))))
+

packages/queryLanguage/test/quoted.txt

@@ -0,0 +1,503 @@
+# Simple quoted string
+
+"hello"
+
+==>
+
+Program(QuotedTerm)
+
+# Leading unclosed quote
+
+"hello
+
+==>
+
+Program(Term)
+
+# Trailing unclosed quote
+
+hello"
+
+==>
+
+Program(Term)
+
+# Quoted string with spaces
+
+"hello world"
+
+==>
+
+Program(QuotedTerm)
+
+# Multiple words in quotes
+
+"this is a search term"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with escaped quote
+
+"hello \"world\""
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with escaped backslash
+
+"path\\to\\file"
+
+==>
+
+Program(QuotedTerm)
+
+# Double backslash
+
+"test\\\\path"
+
+==>
+
+Program(QuotedTerm)
+
+# Multiple escaped quotes
+
+"\"quoted\" \"words\""
+
+==>
+
+Program(QuotedTerm)
+
+# Mixed escaped characters
+
+"test\\nvalue\"quoted"
+
+==>
+
+Program(QuotedTerm)
+
+# Empty quoted string
+
+""
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with only spaces
+
+"   "
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string in file prefix
+
+file:"my file.txt"
+
+==>
+
+Program(PrefixExpr(FileExpr))
+
+# Quoted string in repo prefix
+
+repo:"github.com/user/repo name"
+
+==>
+
+Program(PrefixExpr(RepoExpr))
+
+# Quoted string in content prefix
+
+content:"console.log"
+
+==>
+
+Program(PrefixExpr(ContentExpr))
+
+# Quoted string in revision prefix
+
+rev:"feature/my feature"
+
+==>
+
+Program(PrefixExpr(RevisionExpr))
+
+# Multiple quoted strings
+
+"first string" "second string"
+
+==>
+
+Program(AndExpr(QuotedTerm,QuotedTerm))
+
+# Quoted and unquoted mixed
+
+unquoted "quoted string" another
+
+==>
+
+Program(AndExpr(Term,QuotedTerm,Term))
+
+# Quoted string with parentheses inside
+
+"(test)"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with brackets
+
+"[a-z]+"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with special chars
+
+"test@example.com"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with colons
+
+"key:value"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with dashes
+
+"test-case-example"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with dots
+
+"com.example.package"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with regex pattern
+
+"\\w+\\s*=\\s*\\d+"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with forward slashes
+
+"path/to/file"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with underscores
+
+"my_variable_name"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with numbers
+
+"test123"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with mixed case
+
+"CamelCaseTest"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted prefix value with spaces
+
+file:"test file.js"
+
+==>
+
+Program(PrefixExpr(FileExpr))
+
+# Multiple prefixes with quoted values
+
+file:"my file.txt" repo:"my repo"
+
+==>
+
+Program(AndExpr(PrefixExpr(FileExpr),PrefixExpr(RepoExpr)))
+
+# Quoted string in parentheses
+
+("quoted term")
+
+==>
+
+Program(ParenExpr(QuotedTerm))
+
+# Multiple quoted in parentheses
+
+("first" "second")
+
+==>
+
+Program(ParenExpr(AndExpr(QuotedTerm,QuotedTerm)))
+
+# Quoted with escaped newline
+
+"line1\\nline2"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted with tab character
+
+"value\\ttab"
+
+==>
+
+Program(QuotedTerm)
+
+# Lang prefix with quoted value
+
+lang:"objective-c"
+
+==>
+
+Program(PrefixExpr(LangExpr))
+
+# Sym prefix with quoted value
+
+sym:"My Class"
+
+==>
+
+Program(PrefixExpr(SymExpr))
+
+# Content with quoted phrase
+
+content:"TODO: fix this"
+
+==>
+
+Program(PrefixExpr(ContentExpr))
+
+# Quoted string with at symbol
+
+"@decorator"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with hash
+
+"#define"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with dollar sign
+
+"$variable"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with percent
+
+"100%"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with ampersand
+
+"foo&bar"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with asterisk
+
+"test*"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with plus
+
+"a+b"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with equals
+
+"a=b"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with angle brackets
+
+"<template>"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with pipe
+
+"a|b"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with tilde
+
+"~/.config"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with backtick
+
+"`code`"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with question mark
+
+"what?"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with exclamation
+
+"important!"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with semicolon
+
+"stmt;"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted string with comma
+
+"a,b,c"
+
+==>
+
+Program(QuotedTerm)
+
+# Multiple quotes in content
+
+content:"function \"test\" {"
+
+==>
+
+Program(PrefixExpr(ContentExpr))
+
+# Quoted prefix keyword becomes literal
+
+"repo:hello"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted file prefix as literal
+
+"file:test.js"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted lang prefix as literal
+
+"lang:python"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted partial prefix
+
+"repo:"
+
+==>
+
+Program(QuotedTerm)
+
+# Mix of quoted prefix and real prefix
+
+"repo:test" file:actual.js
+
+==>
+
+Program(AndExpr(QuotedTerm,PrefixExpr(FileExpr)))
+
+# Quoted short form prefix
+
+"f:test"
+
+==>
+
+Program(QuotedTerm)
+
+# Quoted revision prefix
+
+"rev:main"
+
+==>
+
+Program(QuotedTerm)
+
+# Quotes can be used within words
+
+name\s*=\s*"projectmanagementlugapi lang:HCL
+
+==>
+
+Program(AndExpr(Term, PrefixExpr(LangExpr)))

packages/queryLanguage/tsconfig.json

@@ -0,0 +1,23 @@
+{
+    "compilerOptions": {
+      "target": "ES2022",
+      "module": "Node16",
+      "moduleResolution": "Node16",
+      "lib": ["ES2023"],
+      "outDir": "dist",
+      "rootDir": "src",
+      "declaration": true,
+      "declarationMap": true,
+      "sourceMap": true,
+      "strict": true,
+      "noImplicitAny": true,
+      "strictNullChecks": true,
+      "esModuleInterop": true,
+      "forceConsistentCasingInFileNames": true,
+      "skipLibCheck": true,
+      "isolatedModules": true,
+      "resolveJsonModule": true
+    },
+    "include": ["src/index.ts"],
+    "exclude": ["node_modules", "dist"]
+  } 

packages/queryLanguage/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+    test: {
+        environment: 'node',
+        watch: false,
+    }
+});

packages/schemas/src/v3/identityProvider.schema.ts

@@ -646,6 +646,115 @@ const schema = {
         "purpose",
         "audience"
       ]
+    },
+    "AuthentikIdentityProviderConfig": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "provider": {
+          "const": "authentik"
+        },
+        "purpose": {
+          "const": "sso"
+        },
+        "clientId": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "clientSecret": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "issuer": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        }
+      },
+      "required": [
+        "provider",
+        "purpose",
+        "clientId",
+        "clientSecret",
+        "issuer"
+      ]
     }
   },
   "oneOf": [
@@ -1292,6 +1401,115 @@ const schema = {
         "purpose",
         "audience"
       ]
+    },
+    {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "provider": {
+          "const": "authentik"
+        },
+        "purpose": {
+          "const": "sso"
+        },
+        "clientId": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "clientSecret": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "issuer": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "env": {
+                  "type": "string",
+                  "description": "The name of the environment variable that contains the token."
+                }
+              },
+              "required": [
+                "env"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "googleCloudSecret": {
+                  "type": "string",
+                  "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                }
+              },
+              "required": [
+                "googleCloudSecret"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        }
+      },
+      "required": [
+        "provider",
+        "purpose",
+        "clientId",
+        "clientSecret",
+        "issuer"
+      ]
     }
   ]
 } as const;

packages/schemas/src/v3/identityProvider.type.ts

@@ -7,7 +7,8 @@ export type IdentityProviderConfig =
   | OktaIdentityProviderConfig
   | KeycloakIdentityProviderConfig
   | MicrosoftEntraIDIdentityProviderConfig
-  | GCPIAPIdentityProviderConfig;
+  | GCPIAPIdentityProviderConfig
+  | AuthentikIdentityProviderConfig;
 
 export interface GitHubIdentityProviderConfig {
   provider: "github";
@@ -255,3 +256,46 @@ export interface GCPIAPIdentityProviderConfig {
         googleCloudSecret: string;
       };
 }
+export interface AuthentikIdentityProviderConfig {
+  provider: "authentik";
+  purpose: "sso";
+  clientId:
+    | {
+        /**
+         * The name of the environment variable that contains the token.
+         */
+        env: string;
+      }
+    | {
+        /**
+         * The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets
+         */
+        googleCloudSecret: string;
+      };
+  clientSecret:
+    | {
+        /**
+         * The name of the environment variable that contains the token.
+         */
+        env: string;
+      }
+    | {
+        /**
+         * The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets
+         */
+        googleCloudSecret: string;
+      };
+  issuer:
+    | {
+        /**
+         * The name of the environment variable that contains the token.
+         */
+        env: string;
+      }
+    | {
+        /**
+         * The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets
+         */
+        googleCloudSecret: string;
+      };
+}

packages/schemas/src/v3/index.schema.ts

@@ -5162,6 +5162,115 @@ const schema = {
               "purpose",
               "audience"
             ]
+          },
+          "AuthentikIdentityProviderConfig": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "provider": {
+                "const": "authentik"
+              },
+              "purpose": {
+                "const": "sso"
+              },
+              "clientId": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              },
+              "clientSecret": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              },
+              "issuer": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              }
+            },
+            "required": [
+              "provider",
+              "purpose",
+              "clientId",
+              "clientSecret",
+              "issuer"
+            ]
           }
         },
         "oneOf": [
@@ -5808,6 +5917,115 @@ const schema = {
               "purpose",
               "audience"
             ]
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "provider": {
+                "const": "authentik"
+              },
+              "purpose": {
+                "const": "sso"
+              },
+              "clientId": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              },
+              "clientSecret": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              },
+              "issuer": {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "env": {
+                        "type": "string",
+                        "description": "The name of the environment variable that contains the token."
+                      }
+                    },
+                    "required": [
+                      "env"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "googleCloudSecret": {
+                        "type": "string",
+                        "description": "The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets"
+                      }
+                    },
+                    "required": [
+                      "googleCloudSecret"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
+              }
+            },
+            "required": [
+              "provider",
+              "purpose",
+              "clientId",
+              "clientSecret",
+              "issuer"
+            ]
           }
         ]
       }

packages/schemas/src/v3/index.type.ts

@@ -33,7 +33,8 @@ export type IdentityProviderConfig =
   | OktaIdentityProviderConfig
   | KeycloakIdentityProviderConfig
   | MicrosoftEntraIDIdentityProviderConfig
-  | GCPIAPIdentityProviderConfig;
+  | GCPIAPIdentityProviderConfig
+  | AuthentikIdentityProviderConfig;
 
 export interface SourcebotConfig {
   $schema?: string;
@@ -1401,3 +1402,46 @@ export interface GCPIAPIdentityProviderConfig {
         googleCloudSecret: string;
       };
 }
+export interface AuthentikIdentityProviderConfig {
+  provider: "authentik";
+  purpose: "sso";
+  clientId:
+    | {
+        /**
+         * The name of the environment variable that contains the token.
+         */
+        env: string;
+      }
+    | {
+        /**
+         * The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets
+         */
+        googleCloudSecret: string;
+      };
+  clientSecret:
+    | {
+        /**
+         * The name of the environment variable that contains the token.
+         */
+        env: string;
+      }
+    | {
+        /**
+         * The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets
+         */
+        googleCloudSecret: string;
+      };
+  issuer:
+    | {
+        /**
+         * The name of the environment variable that contains the token.
+         */
+        env: string;
+      }
+    | {
+        /**
+         * The resource name of a Google Cloud secret. Must be in the format `projects/<project-id>/secrets/<secret-name>/versions/<version-id>`. See https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets
+         */
+        googleCloudSecret: string;
+      };
+}

packages/shared/src/env.client.ts

@@ -7,7 +7,6 @@ export const env = createEnv({
     client: {
         NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(),
         NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"),
-        NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(),
         NEXT_PUBLIC_SENTRY_BACKEND_DSN: z.string().optional(),
         NEXT_PUBLIC_SENTRY_ENVIRONMENT: z.string().optional(),
         NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: z.string().optional(),
@@ -16,7 +15,6 @@ export const env = createEnv({
     runtimeEnvStrict: {
         NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT,
         NEXT_PUBLIC_SOURCEBOT_VERSION: process.env.NEXT_PUBLIC_SOURCEBOT_VERSION,
-        NEXT_PUBLIC_POSTHOG_PAPIK: process.env.NEXT_PUBLIC_POSTHOG_PAPIK,
         NEXT_PUBLIC_SENTRY_BACKEND_DSN: process.env.NEXT_PUBLIC_SENTRY_BACKEND_DSN,
         NEXT_PUBLIC_SENTRY_ENVIRONMENT: process.env.NEXT_PUBLIC_SENTRY_ENVIRONMENT,
         NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY,

packages/shared/src/env.server.ts

@@ -120,6 +120,8 @@ export const env = createEnv({
         CONFIG_MAX_REPOS_NO_TOKEN: numberSchema.default(Number.MAX_SAFE_INTEGER),
         NODE_ENV: z.enum(["development", "test", "production"]),
         SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default('false'),
+        // @note: this is also declared in the Dockerfile.
+        POSTHOG_PAPIK: z.string().default("phc_lLPuFFi5LH6c94eFJcqvYVFwiJffVcV6HD8U4a1OnRW"),
 
         // Database variables
         // Either DATABASE_URL or DATABASE_HOST, DATABASE_USERNAME, DATABASE_PASSWORD, and DATABASE_NAME must be set.
@@ -216,6 +218,12 @@ export const env = createEnv({
         SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"),
         SOURCEBOT_STRUCTURED_LOGGING_ENABLED: booleanSchema.default("false"),
         SOURCEBOT_STRUCTURED_LOGGING_FILE: z.string().optional(),
+
+        // Configure the default maximum number of search results to return by default.
+        DEFAULT_MAX_MATCH_COUNT: numberSchema.default(10_000),
+
+        // A comma separated list of glob patterns that shwould always be indexed regardless of their size.
+        ALWAYS_INDEX_FILE_PATTERNS: z.string().optional(),
     },
     runtimeEnv,
     emptyStringAsUndefined: true,

packages/web/.eslintignore

@@ -1,3 +1,4 @@
 # shadcn components
 src/components/
 next-env.d.ts
+src/proto/**

packages/web/package.json

@@ -8,6 +8,7 @@
     "start": "next start",
     "lint": "cross-env SKIP_ENV_VALIDATION=1 eslint .",
     "test": "cross-env SKIP_ENV_VALIDATION=1 vitest",
+    "generate:protos": "proto-loader-gen-types --includeComments --longs=Number --enums=String --defaults --oneofs --grpcLib=@grpc/grpc-js --keepCase --includeDirs=../../vendor/zoekt/grpc/protos --outDir=src/proto zoekt/webserver/v1/webserver.proto zoekt/webserver/v1/query.proto",
     "dev:emails": "email dev --dir ./src/emails",
     "stripe:listen": "stripe listen --forward-to http://localhost:3000/api/stripe"
   },
@@ -52,6 +53,8 @@
     "@codemirror/state": "^6.4.1",
     "@codemirror/view": "^6.33.0",
     "@floating-ui/react": "^0.27.2",
+    "@grpc/grpc-js": "^1.14.1",
+    "@grpc/proto-loader": "^0.8.0",
     "@hookform/resolvers": "^3.9.0",
     "@iconify/react": "^5.1.0",
     "@iizukak/codemirror-lang-wgsl": "^0.3.0",
@@ -91,6 +94,7 @@
     "@shopify/lang-jsonc": "^1.0.0",
     "@sourcebot/codemirror-lang-tcl": "^1.0.12",
     "@sourcebot/db": "workspace:*",
+    "@sourcebot/query-language": "workspace:*",
     "@sourcebot/schemas": "workspace:*",
     "@sourcebot/shared": "workspace:*",
     "@ssddanbrown/codemirror-lang-twig": "^1.0.0",
@@ -133,6 +137,7 @@
     "embla-carousel-auto-scroll": "^8.3.0",
     "embla-carousel-react": "^8.3.0",
     "escape-string-regexp": "^5.0.0",
+    "fast-deep-equal": "^3.1.3",
     "fuse.js": "^7.0.0",
     "google-auth-library": "^10.1.0",
     "graphql": "^16.9.0",
@@ -142,7 +147,7 @@
     "langfuse-vercel": "^3.38.4",
     "lucide-react": "^0.517.0",
     "micromatch": "^4.0.8",
-    "next": "15.5.0",
+    "next": "^15.5.7",
     "next-auth": "^5.0.0-beta.30",
     "next-navigation-guard": "^0.2.0",
     "next-themes": "^0.3.0",
@@ -151,11 +156,12 @@
     "openai": "^4.98.0",
     "parse-diff": "^0.11.1",
     "posthog-js": "^1.161.5",
+    "posthog-node": "^5.15.0",
     "pretty-bytes": "^6.1.1",
     "psl": "^1.15.0",
-    "react": "19.1.1",
+    "react": "^19.2.1",
     "react-device-detect": "^2.2.3",
-    "react-dom": "19.1.1",
+    "react-dom": "^19.2.1",
     "react-hook-form": "^7.53.0",
     "react-hotkeys-hook": "^4.5.1",
     "react-icons": "^5.3.0",
@@ -191,8 +197,8 @@
     "@types/node": "^20",
     "@types/nodemailer": "^6.4.17",
     "@types/psl": "^1.1.3",
-    "@types/react": "19.1.10",
+    "@types/react": "19.2.1",
-    "@types/react-dom": "19.1.7",
+    "@types/react-dom": "19.2.1",
     "@typescript-eslint/eslint-plugin": "^8.40.0",
     "@typescript-eslint/parser": "^8.40.0",
     "cross-env": "^7.0.3",
@@ -212,7 +218,7 @@
     "vitest-mock-extended": "^3.1.0"
   },
   "resolutions": {
-    "@types/react": "19.1.10",
+    "@types/react": "19.2.1",
-    "@types/react-dom": "19.1.7"
+    "@types/react-dom": "19.2.1"
   }
 }

packages/web/public/authentik.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" id="Layer_1" x="0" y="0" version="1.1" viewBox="-0.03 59.9 512.03 392.1"><style>.st0{fill:#fd4b2d}</style><path d="M279.9 141h17.9v51.2h-17.9zm46.6-2.2h17.9v40h-17.9zM65.3 197.3c-24 0-46 13.2-57.4 34.3h30.4c13.5-11.6 33-15 47.1 0h32.2c-12.6-17.1-31.4-34.3-52.3-34.3" class="st0"/><path d="M108.7 262.4C66.8 350-6.6 275.3 38.3 231.5H7.9C-15.9 273 17 329 65.3 327.8c37.4 0 68.2-55.5 68.2-65.3 0-4.3-6-17.6-16-31H85.4c10.7 9.7 20 23.7 23.3 30.9m1.1-2.6" class="st0"/><path d="M512 140.3v231.3c0 44.3-36.1 80.4-80.4 80.4h-34.1v-78.8h-163V452h-34.1c-44.4 0-80.4-36.1-80.4-80.4v-72.8h258.4v-139H253.6V238H119.9v-97.6c0-3.1.2-6.2.5-9.2.4-3.7 1.1-7.3 2-10.8.3-1.1.6-2.3 1-3.4.1-.3.2-.6.3-.8.2-.6.4-1.1.5-1.7.2-.5.4-1.1.6-1.7s.5-1.2.7-1.8.5-1.2.8-1.8c2-4.7 4.4-9.3 7.3-13.6l.1-.1c.7-1.1 1.5-2.1 2.3-3.2.7-.9 1.3-1.7 2-2.6.8-.9 1.6-1.9 2.4-2.8s1.6-1.8 2.4-2.6l.1-.1c.4-.5.9-.9 1.4-1.4 3-2.9 6.2-5.6 9.6-8 .9-.7 1.9-1.3 2.8-1.9 1.1-.7 2.2-1.4 3.3-2 2.1-1.2 4.2-2.4 6.5-3.4.7-.3 1.4-.7 2.1-1 3.1-1.3 6.2-2.5 9.4-3.4 1.2-.4 2.5-.7 3.7-1 .6-.2 1.2-.3 1.8-.4 3.6-.8 7.2-1.3 10.9-1.6l1.6-.1h.8c1.2-.1 2.4-.1 3.7-.1h231.3c1.2 0 2.5 0 3.7.1h.8l1.6.1c3.7.3 7.3.8 10.9 1.6.6.1 1.2.3 1.8.4 1.3.3 2.5.6 3.7 1 3.2.9 6.3 2.1 9.4 3.4.7.3 1.4.6 2.1 1 2.2 1 4.4 2.2 6.5 3.4 1.1.7 2.2 1.3 3.3 2 1 .6 1.9 1.3 2.8 1.9 3.9 2.8 7.6 6 11 9.4.8.8 1.7 1.7 2.4 2.6.8.9 1.6 1.9 2.4 2.8.7.8 1.3 1.7 2 2.6.8 1.1 1.5 2.1 2.3 3.2l.1.1c2.9 4.3 5.3 8.8 7.3 13.6.2.6.5 1.2.8 1.8.2.6.5 1.2.7 1.8.2.5.4 1.1.6 1.7s.4 1.1.5 1.7c.1.3.2.6.3.8.3 1.1.7 2.3 1 3.4.9 3.6 1.6 7.2 2 10.8 0 3.1.2 6.1.2 9.2" class="st0"/><path d="M498.3 95.5H133.5c14.9-22.2 40-35.6 66.7-35.6h231.3c26.9 0 51.9 13.4 66.8 35.6m13.2 35.6H120.4c1.4-12.8 6-25 13.1-35.6h364.8c7.2 10.6 11.7 22.9 13.2 35.6m.5 9.2v26.4H378.3v-6.9H253.6v6.9H119.9v-26.4c0-3.1.2-6.2.5-9.2h391.1c.3 3.1.5 6.1.5 9.2M119.9 166.7h133.7v35.6H119.9zm258.4 0H512v35.6H378.3zm-258.4 35.6h133.7v35.6H119.9zm258.4 0H512v35.6H378.3z" class="st0"/></svg>

packages/web/src/actions.ts

@@ -38,8 +38,8 @@ const auditService = getAuditService();
 /**
  * "Service Error Wrapper".
  * 
- * Captures any thrown exceptions and converts them to a unexpected
+ * Captures any thrown exceptions, logs them to the console and Sentry,
- * service error. Also logs them with Sentry.
+ * and returns a generic unexpected service error.
  */
 export const sew = async <T>(fn: () => Promise<T>): Promise<T | ServiceError> => {
     try {
@@ -48,8 +48,8 @@ export const sew = async <T>(fn: () => Promise<T>): Promise<T | ServiceError> =>
         Sentry.captureException(e);
         logger.error(e);
 
-        if (e instanceof Error) {
+        if (e instanceof ServiceErrorException) {
-            return unexpectedError(e.message);
+            return e.serviceError;
         }
 
         return unexpectedError(`An unexpected error occurred. Please try again later.`);

packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx

@@ -1,10 +1,10 @@
 import { getRepoInfoByName } from "@/actions";
 import { PathHeader } from "@/app/[domain]/components/pathHeader";
 import { Separator } from "@/components/ui/separator";
-import { getFileSource } from "@/features/search/fileSourceApi";
 import { cn, getCodeHostInfoForRepo, isServiceError } from "@/lib/utils";
 import Image from "next/image";
 import { PureCodePreviewPanel } from "./pureCodePreviewPanel";
+import { getFileSource } from "@/features/search/fileSourceApi";
 
 interface CodePreviewPanelProps {
     path: string;
@@ -22,8 +22,12 @@ export const CodePreviewPanel = async ({ path, repoName, revisionName }: CodePre
         getRepoInfoByName(repoName),
     ]);
 
-    if (isServiceError(fileSourceResponse) || isServiceError(repoInfoResponse)) {
+    if (isServiceError(fileSourceResponse)) {
-        return <div>Error loading file source</div>
+        return <div>Error loading file source: {fileSourceResponse.message}</div>
+    }
+
+    if (isServiceError(repoInfoResponse)) {
+        return <div>Error loading repo info: {repoInfoResponse.message}</div>
     }
 
     const codeHostInfo = getCodeHostInfoForRepo({

packages/web/src/app/[domain]/browse/[...path]/components/pureCodePreviewPanel.tsx

@@ -3,7 +3,6 @@
 import { ScrollArea } from "@/components/ui/scroll-area";
 import { SymbolHoverPopup } from "@/ee/features/codeNav/components/symbolHoverPopup";
 import { symbolHoverTargetsExtension } from "@/ee/features/codeNav/components/symbolHoverPopup/symbolHoverTargetsExtension";
-import { SymbolDefinition } from "@/ee/features/codeNav/components/symbolHoverPopup/useHoveredOverSymbolInfo";
 import { useHasEntitlement } from "@/features/entitlements/useHasEntitlement";
 import { useCodeMirrorLanguageExtension } from "@/hooks/useCodeMirrorLanguageExtension";
 import { useCodeMirrorTheme } from "@/hooks/useCodeMirrorTheme";
@@ -11,15 +10,10 @@ import { useKeymapExtension } from "@/hooks/useKeymapExtension";
 import { useNonEmptyQueryParam } from "@/hooks/useNonEmptyQueryParam";
 import { search } from "@codemirror/search";
 import CodeMirror, { EditorSelection, EditorView, ReactCodeMirrorRef, SelectionRange, ViewUpdate } from "@uiw/react-codemirror";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { useEffect, useMemo, useState } from "react";
 import { EditorContextMenu } from "../../../components/editorContextMenu";
-import { useBrowseNavigation } from "../../hooks/useBrowseNavigation";
 import { BrowseHighlightRange, HIGHLIGHT_RANGE_QUERY_PARAM } from "../../hooks/utils";
-import { useBrowseState } from "../../hooks/useBrowseState";
 import { rangeHighlightingExtension } from "./rangeHighlightingExtension";
-import useCaptureEvent from "@/hooks/useCaptureEvent";
-import { createAuditAction } from "@/ee/features/audit/actions";
-import { useDomain } from "@/hooks/useDomain";
 
 interface PureCodePreviewPanelProps {
     path: string;
@@ -41,10 +35,6 @@ export const PureCodePreviewPanel = ({
     const [currentSelection, setCurrentSelection] = useState<SelectionRange>();
     const keymapExtension = useKeymapExtension(editorRef?.view);
     const hasCodeNavEntitlement = useHasEntitlement("code-nav");
-    const { updateBrowseState } = useBrowseState();
-    const { navigateToPath } = useBrowseNavigation();
-    const domain = useDomain();
-    const captureEvent = useCaptureEvent();
 
     const highlightRangeQuery = useNonEmptyQueryParam(HIGHLIGHT_RANGE_QUERY_PARAM);
     const highlightRange = useMemo((): BrowseHighlightRange | undefined => {
@@ -90,7 +80,6 @@ export const PureCodePreviewPanel = ({
                 }
             }
         }
-
     }, [highlightRangeQuery]);
 
     const extensions = useMemo(() => {
@@ -118,90 +107,31 @@ export const PureCodePreviewPanel = ({
 
     // Scroll the highlighted range into view.
     useEffect(() => {
-        if (!highlightRange || !editorRef || !editorRef.state) {
+        if (!highlightRange || !editorRef || !editorRef.state || !editorRef.view) {
             return;
         }
 
         const doc = editorRef.state.doc;
         const { start, end } = highlightRange;
-        const selection = EditorSelection.range(
+
-            doc.line(start.lineNumber).from,
+        const from = doc.line(start.lineNumber).from;
-            doc.line(end.lineNumber).from,
+        const to = doc.line(end.lineNumber).to;
-        );
+        const selection = EditorSelection.range(from, to);
+
+        // When the selection is in view, we don't want to perform any scrolling
+        // as it could be jarring for the user. If it is not in view, scroll to the
+        // center of the viewport.
+        const viewport = editorRef.view.viewport;
+        const isInView = from >= viewport.from && to <= viewport.to;
+        const scrollStrategy = isInView ? "nearest" : "center";
         
         editorRef.view?.dispatch({
             effects: [
-                EditorView.scrollIntoView(selection, { y: "center" }),
+                EditorView.scrollIntoView(selection, { y: scrollStrategy }),
             ]
         });
     }, [editorRef, highlightRange]);
 
-    const onFindReferences = useCallback((symbolName: string) => {
-        captureEvent('wa_find_references_pressed', {
-            source: 'browse',
-        });
-        createAuditAction({
-            action: "user.performed_find_references",
-            metadata: {
-                message: symbolName,
-            },
-        }, domain)
-
-        updateBrowseState({
-            selectedSymbolInfo: {
-                repoName,
-                symbolName,
-                revisionName,
-                language,
-            },
-            isBottomPanelCollapsed: false,
-            activeExploreMenuTab: "references",
-        })
-    }, [captureEvent, updateBrowseState, repoName, revisionName, language, domain]);
-
-
-    // If we resolve multiple matches, instead of navigating to the first match, we should
-    // instead popup the bottom sheet with the list of matches.
-    const onGotoDefinition = useCallback((symbolName: string, symbolDefinitions: SymbolDefinition[]) => {
-        captureEvent('wa_goto_definition_pressed', {
-            source: 'browse',
-        });
-        createAuditAction({
-            action: "user.performed_goto_definition",
-            metadata: {
-                message: symbolName,
-            },
-        }, domain)
-
-        if (symbolDefinitions.length === 0) {
-            return;
-        }
-
-        if (symbolDefinitions.length === 1) {
-            const symbolDefinition = symbolDefinitions[0];
-            const { fileName, repoName } = symbolDefinition;
-
-            navigateToPath({
-                repoName,
-                revisionName,
-                path: fileName,
-                pathType: 'blob',
-                highlightRange: symbolDefinition.range,
-            })
-        } else {
-            updateBrowseState({
-                selectedSymbolInfo: {
-                    symbolName,
-                    repoName,
-                    revisionName,
-                    language,
-                },
-                activeExploreMenuTab: "definitions",
-                isBottomPanelCollapsed: false,
-            })
-        }
-    }, [captureEvent, navigateToPath, revisionName, updateBrowseState, repoName, language, domain]);
-
     const theme = useCodeMirrorTheme();
 
     return (
@@ -225,11 +155,12 @@ export const PureCodePreviewPanel = ({
                 )}
                 {editorRef && hasCodeNavEntitlement && (
                     <SymbolHoverPopup
+                        source="preview"
                         editorRef={editorRef}
                         revisionName={revisionName}
                         language={language}
-                        onFindReferences={onFindReferences}
+                        fileName={path}
-                        onGotoDefinition={onGotoDefinition}
+                        repoName={repoName}
                     />
                 )}
             </CodeMirror>

packages/web/src/app/[domain]/browse/[...path]/components/pureTreePreviewPanel.tsx

@@ -1,12 +1,12 @@
 'use client';
 
 import { useRef } from "react";
-import { FileTreeItem } from "@/features/fileTree/actions";
 import { FileTreeItemComponent } from "@/features/fileTree/components/fileTreeItemComponent";
 import { getBrowsePath } from "../../hooks/utils";
 import { ScrollArea } from "@/components/ui/scroll-area";
 import { useBrowseParams } from "../../hooks/useBrowseParams";
 import { useDomain } from "@/hooks/useDomain";
+import { FileTreeItem } from "@/features/fileTree/types";
 
 interface PureTreePreviewPanelProps {
     items: FileTreeItem[];

packages/web/src/app/[domain]/browse/[...path]/components/treePreviewPanel.tsx

@@ -2,7 +2,7 @@
 import { Separator } from "@/components/ui/separator";
 import { getRepoInfoByName } from "@/actions";
 import { PathHeader } from "@/app/[domain]/components/pathHeader";
-import { getFolderContents } from "@/features/fileTree/actions";
+import { getFolderContents } from "@/features/fileTree/api";
 import { isServiceError } from "@/lib/utils";
 import { PureTreePreviewPanel } from "./pureTreePreviewPanel";
 

packages/web/src/app/[domain]/browse/components/fileSearchCommandDialog.tsx

@@ -5,7 +5,6 @@ import { useState, useRef, useMemo, useEffect, useCallback } from "react";
 import { useHotkeys } from "react-hotkeys-hook";
 import { useQuery } from "@tanstack/react-query";
 import { unwrapServiceError } from "@/lib/utils";
-import { FileTreeItem, getFiles } from "@/features/fileTree/actions";
 import { Dialog, DialogContent, DialogDescription, DialogTitle } from "@/components/ui/dialog";
 import { useBrowseNavigation } from "../hooks/useBrowseNavigation";
 import { useBrowseState } from "../hooks/useBrowseState";
@@ -13,6 +12,8 @@ import { useBrowseParams } from "../hooks/useBrowseParams";
 import { FileTreeItemIcon } from "@/features/fileTree/components/fileTreeItemIcon";
 import { useLocalStorage } from "usehooks-ts";
 import { Skeleton } from "@/components/ui/skeleton";
+import { FileTreeItem } from "@/features/fileTree/types";
+import { getFiles } from "@/app/api/(client)/client";
 
 const MAX_RESULTS = 100;
 

packages/web/src/app/[domain]/browse/layout.tsx

@@ -10,6 +10,7 @@ import { useBrowseParams } from "./hooks/useBrowseParams";
 import { FileSearchCommandDialog } from "./components/fileSearchCommandDialog";
 import { useDomain } from "@/hooks/useDomain";
 import { SearchBar } from "../components/searchBar";
+import escapeStringRegexp from "escape-string-regexp";
 
 interface LayoutProps {
     children: React.ReactNode;
@@ -29,7 +30,9 @@ export default function Layout({
                 >
                     <SearchBar
                         size="sm"
-                        defaultQuery={`repo:${repoName}${revisionName ? ` rev:${revisionName}` : ''} `}
+                        defaults={{
+                            query: `repo:^${escapeStringRegexp(repoName)}$${revisionName ? ` rev:${revisionName}` : ''} `,
+                        }}
                         className="w-full"
                     />
                 </TopBar>

packages/web/src/app/[domain]/chat/[id]/page.tsx

@@ -24,9 +24,9 @@ export default async function Page(props: PageProps) {
     const languageModels = await getConfiguredLanguageModelsInfo();
     const repos = await getRepos();
     const searchContexts = await getSearchContexts(params.domain);
-    const chatInfo = await getChatInfo({ chatId: params.id }, params.domain);
+    const chatInfo = await getChatInfo({ chatId: params.id });
     const session = await auth();
-    const chatHistory = session ? await getUserChatHistory(params.domain) : [];
+    const chatHistory = session ? await getUserChatHistory() : [];
 
     if (isServiceError(chatHistory)) {
         throw new ServiceErrorException(chatHistory);

packages/web/src/app/[domain]/chat/components/chatName.tsx

@@ -4,7 +4,6 @@ import { useToast } from "@/components/hooks/use-toast";
 import { Badge } from "@/components/ui/badge";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
 import { updateChatName } from "@/features/chat/actions";
-import { useDomain } from "@/hooks/useDomain";
 import { isServiceError } from "@/lib/utils";
 import { GlobeIcon } from "@radix-ui/react-icons";
 import { ChatVisibility } from "@sourcebot/db";
@@ -23,7 +22,6 @@ interface ChatNameProps {
 export const ChatName = ({ name, visibility, id, isReadonly }: ChatNameProps) => {
     const [isRenameDialogOpen, setIsRenameDialogOpen] = useState(false);
     const { toast } = useToast();
-    const domain = useDomain();
     const router = useRouter();
 
     const onRenameChat = useCallback(async (name: string) => {
@@ -31,7 +29,7 @@ export const ChatName = ({ name, visibility, id, isReadonly }: ChatNameProps) =>
         const response = await updateChatName({
             chatId: id,
             name: name,
-        }, domain);
+        });
 
         if (isServiceError(response)) {
             toast({
@@ -43,7 +41,7 @@ export const ChatName = ({ name, visibility, id, isReadonly }: ChatNameProps) =>
             });
             router.refresh();
         }
-    }, [id, domain, toast, router]);
+    }, [id, toast, router]);
 
     return (
         <>

packages/web/src/app/[domain]/chat/components/chatSidePanel.tsx

@@ -9,7 +9,6 @@ import { ScrollArea } from "@/components/ui/scroll-area";
 import { Separator } from "@/components/ui/separator";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
 import { deleteChat, updateChatName } from "@/features/chat/actions";
-import { useDomain } from "@/hooks/useDomain";
 import { cn, isServiceError } from "@/lib/utils";
 import { CirclePlusIcon, EllipsisIcon, PencilIcon, TrashIcon } from "lucide-react";
 import { useRouter } from "next/navigation";
@@ -23,6 +22,7 @@ import { useChatId } from "../useChatId";
 import { RenameChatDialog } from "./renameChatDialog";
 import { DeleteChatDialog } from "./deleteChatDialog";
 import Link from "next/link";
+import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants";
 
 interface ChatSidePanelProps {
     order: number;
@@ -41,7 +41,6 @@ export const ChatSidePanel = ({
     isAuthenticated,
     isCollapsedInitially,
 }: ChatSidePanelProps) => {
-    const domain = useDomain();
     const [isCollapsed, setIsCollapsed] = useState(isCollapsedInitially);
     const sidePanelRef = useRef<ImperativePanelHandle>(null);
     const router = useRouter();
@@ -72,7 +71,7 @@ export const ChatSidePanel = ({
         const response = await updateChatName({
             chatId,
             name: name,
-        }, domain);
+        });
 
         if (isServiceError(response)) {
             toast({
@@ -84,14 +83,14 @@ export const ChatSidePanel = ({
             });
             router.refresh();
         }
-    }, [router, toast, domain]);
+    }, [router, toast]);
 
     const onDeleteChat = useCallback(async (chatIdToDelete: string) => {
         if (!chatIdToDelete) {
             return;
         }
 
-        const response = await deleteChat({ chatId: chatIdToDelete }, domain);
+        const response = await deleteChat({ chatId: chatIdToDelete });
 
         if (isServiceError(response)) {
             toast({
@@ -104,12 +103,12 @@ export const ChatSidePanel = ({
 
             // If we just deleted the current chat, navigate to new chat
             if (chatIdToDelete === chatId) {
-                router.push(`/${domain}/chat`);
+                router.push(`/${SINGLE_TENANT_ORG_DOMAIN}/chat`);
             }
 
             router.refresh();
         }
-    }, [chatId, router, toast, domain]);
+    }, [chatId, router, toast]);
 
     return (
         <>
@@ -131,7 +130,7 @@ export const ChatSidePanel = ({
                             size="sm"
                             className="w-full"
                             onClick={() => {
-                                router.push(`/${domain}/chat`);
+                                router.push(`/${SINGLE_TENANT_ORG_DOMAIN}/chat`);
                             }}
                         >
                             <CirclePlusIcon className="w-4 h-4 mr-1" />
@@ -145,7 +144,7 @@ export const ChatSidePanel = ({
                                 <div className="flex flex-col">
                                     <p className="text-sm text-muted-foreground mb-4">
                                         <Link
-                                            href={`/login?callbackUrl=${encodeURIComponent(`/${domain}/chat`)}`}
+                                            href={`/login?callbackUrl=${encodeURIComponent(`/${SINGLE_TENANT_ORG_DOMAIN}/chat`)}`}
                                             className="text-sm text-link hover:underline cursor-pointer"
                                         >
                                             Sign in
@@ -163,7 +162,7 @@ export const ChatSidePanel = ({
                                         chat.id === chatId && "bg-muted"
                                     )}
                                     onClick={() => {
-                                        router.push(`/${domain}/chat/${chat.id}`);
+                                        router.push(`/${SINGLE_TENANT_ORG_DOMAIN}/chat/${chat.id}`);
                                     }}
                                 >
                                     <span className="text-sm truncate">{chat.name ?? 'Untitled chat'}</span>

packages/web/src/app/[domain]/components/lightweightCodeHighlighter.tsx

@@ -6,7 +6,7 @@ import { memo, useEffect, useMemo, useState } from 'react'
 import { useCodeMirrorHighlighter } from '@/hooks/useCodeMirrorHighlighter'
 import tailwind from '@/tailwind'
 import { measure } from '@/lib/utils'
-import { SourceRange } from '@/features/search/types'
+import { SourceRange } from '@/features/search'
 
 // Define a plain text language
 const plainTextLanguage = StreamLanguage.define({

packages/web/src/app/[domain]/components/navigationMenu/index.tsx

@@ -1,4 +1,4 @@
-import { getConnectionStats, getRepos, getReposStats } from "@/actions";
+import { getConnectionStats, getCurrentUserRole, getOrgAccountRequests, getRepos, getReposStats } from "@/actions";
 import { SourcebotLogo } from "@/app/components/sourcebotLogo";
 import { auth } from "@/auth";
 import { Button } from "@/components/ui/button";
@@ -10,7 +10,7 @@ import { env } from "@sourcebot/shared";
 import { ServiceErrorException } from "@/lib/serviceError";
 import { isServiceError } from "@/lib/utils";
 import { DiscordLogoIcon, GitHubLogoIcon } from "@radix-ui/react-icons";
-import { RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
+import { OrgRole, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
 import Link from "next/link";
 import { redirect } from "next/navigation";
 import { OrgSelector } from "../orgSelector";
@@ -20,7 +20,7 @@ import { NavigationItems } from "./navigationItems";
 import { ProgressIndicator } from "./progressIndicator";
 import { TrialIndicator } from "./trialIndicator";
 
-const SOURCEBOT_DISCORD_URL = "https://discord.gg/GbXMEM5H";
+const SOURCEBOT_DISCORD_URL = "https://discord.gg/HDScTs3ptP";
 const SOURCEBOT_GITHUB_URL = "https://github.com/sourcebot-dev/sourcebot";
 
 interface NavigationMenuProps {
@@ -39,11 +39,32 @@ export const NavigationMenu = async ({
         throw new ServiceErrorException(repoStats);
     }
 
-    const connectionStats = isAuthenticated ? await getConnectionStats() : null;
+    const role = isAuthenticated ? await getCurrentUserRole(domain) : null;
-    if (isServiceError(connectionStats)) {
+    if (isServiceError(role)) {
-        throw new ServiceErrorException(connectionStats);
+        throw new ServiceErrorException(role);
     }
 
+    const stats = await (async () => {
+        if (!isAuthenticated || role !== OrgRole.OWNER) {
+            return null;
+        }
+
+        const joinRequests = await getOrgAccountRequests(domain);
+        if (isServiceError(joinRequests)) {
+            throw new ServiceErrorException(joinRequests);
+        }
+
+        const connectionStats = await getConnectionStats();
+        if (isServiceError(connectionStats)) {
+            throw new ServiceErrorException(connectionStats);
+        }
+
+        return {
+            numJoinRequests: joinRequests.length,
+            connectionStats,
+        };
+    })();
+
     const sampleRepos = await getRepos({
         where: {
             jobs: {
@@ -100,9 +121,10 @@ export const NavigationMenu = async ({
                             numberOfRepos={numberOfRepos}
                             isReposButtonNotificationDotVisible={numberOfReposWithFirstTimeIndexingJobsInProgress > 0}
                             isSettingsButtonNotificationDotVisible={
-                                connectionStats ?
+                                stats ? (
-                                    connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0 :
+                                    stats.connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0 ||
-                                    false
+                                    stats.numJoinRequests > 0
+                                ) : false
                             }
                             isAuthenticated={isAuthenticated}
                         />

packages/web/src/app/[domain]/components/pathHeader.tsx

@@ -233,7 +233,7 @@ export const PathHeader = ({
                     }}
                 >
                     <span className="mr-0.5">@</span>
-                    {`${branchDisplayName}`}
+                    {`${branchDisplayName.replace(/^refs\/(heads|tags)\//, '')}`}
                 </p>
             )}
             <span>·</span>

packages/web/src/app/[domain]/components/searchBar/constants.ts

@@ -16,57 +16,53 @@ export enum SearchPrefix {
     sym = "sym:",
     content = "content:",
     archived = "archived:",
-    case = "case:",
     fork = "fork:",
-    public = "public:",
+    visibility = "visibility:",
     context = "context:",
 }
 
-export const publicModeSuggestions: Suggestion[] = [
+export const visibilityModeSuggestions: Suggestion[] = [
     {
-        value: "yes",
+        value: "public",
         description: "Only include results from public repositories."
     },
     {
-        value: "no",
+        value: "private",
         description: "Only include results from private repositories."
     },
+    {
+        value: "any",
+        description: "Include results from both public and private repositories (default)."
+    },
 ];
 
 export const forkModeSuggestions: Suggestion[] = [
     {
         value: "yes",
+        description: "Include results from forked repositories (default)."
+    },
+    {
+        value: "no",
+        description: "Exclude results from forked repositories."
+    },
+    {
+        value: "only",
         description: "Only include results from forked repositories."
-    },
+    }
-    {
-        value: "no",
-        description: "Only include results from non-forked repositories."
-    },
-];
-
-export const caseModeSuggestions: Suggestion[] = [
-    {
-        value: "auto",
-        description: "Search patterns are case-insensitive if all characters are lowercase, and case sensitive otherwise (default)."
-    },
-    {
-        value: "yes",
-        description: "Case sensitive search."
-    },
-    {
-        value: "no",
-        description: "Case insensitive search."
-    },
 ];
 
 export const archivedModeSuggestions: Suggestion[] = [
     {
         value: "yes",
-        description: "Only include results in archived repositories."
+        description: "Include results from archived repositories (default)."
     },
     {
         value: "no",
-        description: "Only include results in non-archived repositories."
+        description: "Exclude results from archived repositories."
     },
+    {
+        value: "only",
+        description: "Only include results from archived repositories."
+    }
 ];
 

packages/web/src/app/[domain]/components/searchBar/searchBar.tsx

@@ -42,14 +42,18 @@ import { Separator } from "@/components/ui/separator";
 import { Tooltip, TooltipTrigger, TooltipContent } from "@/components/ui/tooltip";
 import { Toggle } from "@/components/ui/toggle";
 import { useDomain } from "@/hooks/useDomain";
-import { KeyboardShortcutHint } from "@/app/components/keyboardShortcutHint";
 import { createAuditAction } from "@/ee/features/audit/actions";
 import tailwind from "@/tailwind";
+import { CaseSensitiveIcon, RegexIcon } from "lucide-react";
 
 interface SearchBarProps {
     className?: string;
     size?: "default" | "sm";
-    defaultQuery?: string;
+    defaults?: {
+        isRegexEnabled?: boolean;
+        isCaseSensitivityEnabled?: boolean;
+        query?: string;
+    }
     autoFocus?: boolean;
 }
 
@@ -91,8 +95,12 @@ const searchBarContainerVariants = cva(
 export const SearchBar = ({
     className,
     size,
-    defaultQuery,
     autoFocus,
+    defaults: {
+        isRegexEnabled: defaultIsRegexEnabled = false,
+        isCaseSensitivityEnabled: defaultIsCaseSensitivityEnabled = false,
+        query: defaultQuery = "",
+    } = {}
 }: SearchBarProps) => {
     const router = useRouter();
     const domain = useDomain();
@@ -102,11 +110,13 @@ export const SearchBar = ({
     const [isSuggestionsEnabled, setIsSuggestionsEnabled] = useState(false);
     const [isSuggestionsBoxFocused, setIsSuggestionsBoxFocused] = useState(false);
     const [isHistorySearchEnabled, setIsHistorySearchEnabled] = useState(false);
+    const [isRegexEnabled, setIsRegexEnabled] = useState(defaultIsRegexEnabled);
+    const [isCaseSensitivityEnabled, setIsCaseSensitivityEnabled] = useState(defaultIsCaseSensitivityEnabled);
 
     const focusEditor = useCallback(() => editorRef.current?.view?.focus(), []);
     const focusSuggestionsBox = useCallback(() => suggestionBoxRef.current?.focus(), []);
 
-    const [_query, setQuery] = useState(defaultQuery ?? "");
+    const [_query, setQuery] = useState(defaultQuery);
     const query = useMemo(() => {
         // Replace any newlines with spaces to handle
         // copy & pasting text with newlines.
@@ -211,13 +221,15 @@ export const SearchBar = ({
             metadata: {
                 message: query,
             },
-        }, domain)
+        })
 
         const url = createPathWithQueryParams(`/${domain}/search`,
             [SearchQueryParams.query, query],
+            [SearchQueryParams.isRegexEnabled, isRegexEnabled ? "true" : null],
+            [SearchQueryParams.isCaseSensitivityEnabled, isCaseSensitivityEnabled ? "true" : null],
         );
         router.push(url);
-    }, [domain, router]);
+    }, [domain, router, isRegexEnabled, isCaseSensitivityEnabled]);
 
     return (
         <div
@@ -275,18 +287,40 @@ export const SearchBar = ({
                 indentWithTab={false}
                 autoFocus={autoFocus ?? false}
             />
-            <Tooltip
+            <div className="flex flex-row items-center gap-1 ml-1">
-                delayDuration={100}
+                <Tooltip>
-            >
+                    <TooltipTrigger asChild>
-                <TooltipTrigger asChild>
+                        <span>
-                    <div>
+                            <Toggle
-                        <KeyboardShortcutHint shortcut="/" />
+                                className="h-7 w-7 min-w-7 p-0 cursor-pointer"
-                    </div>
+                                pressed={isCaseSensitivityEnabled}
-                </TooltipTrigger>
+                                onPressedChange={setIsCaseSensitivityEnabled}
-                <TooltipContent side="bottom" className="flex flex-row items-center gap-2">
+                            >
-                    Focus search bar
+                                <CaseSensitiveIcon className="w-4 h-4" />
-                </TooltipContent>
+                            </Toggle>
-            </Tooltip>
+                        </span>
+                    </TooltipTrigger>
+                    <TooltipContent side="bottom" className="flex flex-row items-center gap-2">
+                        {isCaseSensitivityEnabled ? "Disable" : "Enable"} case sensitivity
+                    </TooltipContent>
+                </Tooltip>
+                <Tooltip>
+                    <TooltipTrigger asChild>
+                        <span>
+                            <Toggle
+                                className="h-7 w-7 min-w-7 p-0 cursor-pointer"
+                                pressed={isRegexEnabled}
+                                onPressedChange={setIsRegexEnabled}
+                            >
+                                <RegexIcon className="w-4 h-4" />
+                            </Toggle>
+                        </span>
+                    </TooltipTrigger>
+                    <TooltipContent side="bottom" className="flex flex-row items-center gap-2">
+                        {isRegexEnabled ? "Disable" : "Enable"} regular expressions
+                    </TooltipContent>
+                </Tooltip>
+            </div>
             <SearchSuggestionsBox
                 ref={suggestionBoxRef}
                 query={query}

packages/web/src/app/[domain]/components/searchBar/searchSuggestionsBox.tsx

@@ -7,9 +7,8 @@ import Fuse from "fuse.js";
 import { forwardRef, Ref, useEffect, useMemo, useState } from "react";
 import {
     archivedModeSuggestions,
-    caseModeSuggestions,
     forkModeSuggestions,
-    publicModeSuggestions,
+    visibilityModeSuggestions,
 } from "./constants";
 import { IconType } from "react-icons/lib";
 import { VscFile, VscFilter, VscRepo, VscSymbolMisc } from "react-icons/vsc";
@@ -32,9 +31,8 @@ export type SuggestionMode =
     "archived" |
     "file" |
     "language" |
-    "case" |
     "fork" |
-    "public" |
+    "visibility" |
     "revision" |
     "symbol" |
     "content" |
@@ -137,9 +135,9 @@ const SearchSuggestionsBox = forwardRef(({
             DefaultIcon?: IconType
         } => {
             switch (suggestionMode) {
-                case "public":
+                case "visibility":
                     return {
-                        list: publicModeSuggestions,
+                        list: visibilityModeSuggestions,
                         onSuggestionClicked: createOnSuggestionClickedHandler(),
                     }
                 case "fork":
@@ -147,11 +145,6 @@ const SearchSuggestionsBox = forwardRef(({
                         list: forkModeSuggestions,
                         onSuggestionClicked: createOnSuggestionClickedHandler(),
                     }
-                case "case":
-                    return {
-                        list: caseModeSuggestions,
-                        onSuggestionClicked: createOnSuggestionClickedHandler(),
-                    }
                 case "archived":
                     return {
                         list: archivedModeSuggestions,
@@ -183,7 +176,7 @@ const SearchSuggestionsBox = forwardRef(({
                 case "file":
                     return {
                         list: fileSuggestions,
-                        onSuggestionClicked: createOnSuggestionClickedHandler(),
+                        onSuggestionClicked: createOnSuggestionClickedHandler({ regexEscaped: true }),
                         isClientSideSearchEnabled: false,
                         DefaultIcon: VscFile,
                     }

packages/web/src/app/[domain]/components/searchBar/useRefineModeSuggestions.ts

@@ -26,7 +26,7 @@ export const useRefineModeSuggestions = () => {
                 },
             ] : []),
             {
-                value: SearchPrefix.public,
+                value: SearchPrefix.visibility,
                 description: "Filter on repository visibility."
             },
             {
@@ -86,10 +86,6 @@ export const useRefineModeSuggestions = () => {
                 value: SearchPrefix.archived,
                 description: "Include results from archived repositories.",
             },
-            {
-                value: SearchPrefix.case,
-                description: "Control case-sensitivity of search patterns."
-            },
             {
                 value: SearchPrefix.fork,
                 description: "Include only results from forked repositories."

packages/web/src/app/[domain]/components/searchBar/useSuggestionModeMappings.ts

@@ -70,12 +70,6 @@ export const useSuggestionModeMappings = () => {
                     SearchPrefix.archived
                 ]
             },
-            {
-                suggestionMode: "case",
-                prefixes: [
-                    SearchPrefix.case
-                ]
-            },
             {
                 suggestionMode: "fork",
                 prefixes: [
@@ -83,9 +77,9 @@ export const useSuggestionModeMappings = () => {
                 ]
             },
             {
-                suggestionMode: "public",
+                suggestionMode: "visibility",
                 prefixes: [
-                    SearchPrefix.public
+                    SearchPrefix.visibility
                 ]
             },
             ...(isSearchContextsEnabled ? [

packages/web/src/app/[domain]/components/searchBar/useSuggestionsData.ts

@@ -5,7 +5,7 @@ import { Suggestion, SuggestionMode } from "./searchSuggestionsBox";
 import { getRepos, search } from "@/app/api/(client)/client";
 import { getSearchContexts } from "@/actions";
 import { useMemo } from "react";
-import { SearchSymbol } from "@/features/search/types";
+import { SearchSymbol } from "@/features/search";
 import { languageMetadataMap } from "@/lib/languageMetadata";
 import {
     VscSymbolClass,
@@ -55,7 +55,8 @@ export const useSuggestionsData = ({
             query: `file:${suggestionQuery}`,
             matches: 15,
             contextLines: 1,
-        }, domain),
+            source: 'search-bar-file-suggestions'
+        }),
         select: (data): Suggestion[] => {
             if (isServiceError(data)) {
                 return [];
@@ -75,7 +76,8 @@ export const useSuggestionsData = ({
             query: `sym:${suggestionQuery.length > 0 ? suggestionQuery : ".*"}`,
             matches: 15,
             contextLines: 1,
-        }, domain),
+            source: 'search-bar-symbol-suggestions'
+        }),
         select: (data): Suggestion[] => {
             if (isServiceError(data)) {
                 return [];

packages/web/src/app/[domain]/components/searchBar/zoektLanguageExtension.ts

@@ -47,7 +47,7 @@ export const zoekt = () => {
 
             // Check for prefixes first
             // If these match, we return 'keyword'
-            if (stream.match(/(archived:|branch:|b:|rev:|c:|case:|content:|f:|file:|fork:|public:|r:|repo:|regex:|lang:|sym:|t:|type:|context:)/)) {
+            if (stream.match(/(archived:|rev:|content:|f:|file:|fork:|visibility:|r:|repo:|regex:|lang:|sym:|t:|type:|context:)/)) {
                 return t.keyword.toString();
             }
 

packages/web/src/app/[domain]/components/syntaxReferenceGuide.tsx

@@ -15,6 +15,7 @@ import { useCallback, useRef } from "react";
 import { useHotkeys } from "react-hotkeys-hook";
 import { useSyntaxGuide } from "./syntaxGuideProvider";
 import { CodeSnippet } from "@/app/components/codeSnippet";
+import { ExternalLinkIcon, RegexIcon } from "lucide-react";
 
 const LINGUIST_LINK = "https://github.com/github-linguist/linguist/blob/main/lib/linguist/languages.yml";
 const CTAGS_LINK = "https://ctags.io/";
@@ -61,70 +62,92 @@ export const SyntaxReferenceGuide = () => {
             onOpenChange={handleOpenChange}
         >
             <DialogContent
-                className="max-h-[80vh] max-w-[700px] overflow-scroll"
+                className="max-h-[80vh] max-w-[700px] overflow-scroll gap-2"
             >
                 <DialogHeader>
-                    <DialogTitle>Syntax Reference Guide</DialogTitle>
+                    <DialogTitle>Syntax Reference Guide <Link href="https://docs.sourcebot.dev/docs/features/search/syntax-reference"><ExternalLinkIcon className="inline w-4 h-4 ml-1 mb-1 text-muted-foreground cursor-pointer" /></Link></DialogTitle>
                     <DialogDescription className="text-sm text-foreground">
-                        Queries consist of space-seperated regular expressions. Wrapping expressions in <CodeSnippet>{`""`}</CodeSnippet> combines them. By default, a file must have at least one match for each expression to be included.
+                        Queries consist of space-separated search patterns that are matched against file contents. A file must have at least one match for each expression to be included. Queries can optionally contain search filters to further refine the search results.
                     </DialogDescription>
                 </DialogHeader>
-                <Table>
-                    <TableHeader>
-                        <TableRow>
-                            <TableHead className="py-2">Example</TableHead>
-                            <TableHead className="py-2">Explanation</TableHead>
-                        </TableRow>
-                    </TableHeader>
-                    <TableBody>
-                        <TableRow>
-                            <TableCell className="py-2"><CodeSnippet>foo</CodeSnippet></TableCell>
-                            <TableCell className="py-2">Match files with regex <CodeSnippet>/foo/</CodeSnippet></TableCell>
-                        </TableRow>
-                        <TableRow>
-                            <TableCell className="py-2"><CodeSnippet>foo bar</CodeSnippet></TableCell>
-                            <TableCell className="py-2">Match files with regex <CodeSnippet>/foo/</CodeSnippet> <b>and</b> <CodeSnippet>/bar/</CodeSnippet></TableCell>
-                        </TableRow>
-                        <TableRow>
-                            <TableCell className="py-2"><CodeSnippet>{`"foo bar"`}</CodeSnippet></TableCell>
-                            <TableCell className="py-2">Match files with regex <CodeSnippet>/foo bar/</CodeSnippet></TableCell>
-                        </TableRow>
-                    </TableBody>
-                </Table>
 
-                <Separator className="my-2"/>
+                <div>
-                <p className="text-sm">
+                    <h3 className="text-lg font-semibold mt-4 mb-0">Keyword search (default)</h3>
-                    {`Multiple expressions can be or'd together with `}<CodeSnippet>or</CodeSnippet>, negated with <CodeSnippet>-</CodeSnippet>, or grouped with <CodeSnippet>()</CodeSnippet>.
+                    <p className="text-sm mb-2 mt-0">
-                </p>
+                        Keyword search matches search patterns exactly in file contents. Wrapping search patterns in <CodeSnippet>{`""`}</CodeSnippet> combines them as a single expression.
-                <Table>
+                    </p>
-                    <TableHeader>
+                    <Table>
-                        <TableRow>
+                        <TableHeader>
-                            <TableHead className="py-2">Example</TableHead>
+                            <TableRow>
-                            <TableHead className="py-2">Explanation</TableHead>
+                                <TableHead className="py-2">Example</TableHead>
-                        </TableRow>
+                                <TableHead className="py-2">Explanation</TableHead>
-                    </TableHeader>
+                            </TableRow>
-                    <TableBody>
+                        </TableHeader>
-                        <TableRow>
+                        <TableBody>
-                            <TableCell className="py-2"><CodeSnippet>foo <Highlight>or</Highlight> bar</CodeSnippet></TableCell>
+                            <TableRow>
-                            <TableCell className="py-2">Match files with regex <CodeSnippet>/foo/</CodeSnippet> <b>or</b> <CodeSnippet>/bar/</CodeSnippet></TableCell>
+                                <TableCell className="py-2"><CodeSnippet>foo</CodeSnippet></TableCell>
-                        </TableRow>
+                                <TableCell className="py-2">Match files containing the keyword <CodeSnippet>foo</CodeSnippet></TableCell>
-                        <TableRow>
+                            </TableRow>
-                            <TableCell className="py-2"><CodeSnippet>foo -bar</CodeSnippet></TableCell>
+                            <TableRow>
-                            <TableCell className="py-2">Match files with regex <CodeSnippet>/foo/</CodeSnippet> but <b>not</b> <CodeSnippet>/bar/</CodeSnippet></TableCell>
+                                <TableCell className="py-2"><CodeSnippet>foo bar</CodeSnippet></TableCell>
-                        </TableRow>
+                                <TableCell className="py-2">Match files containing both <CodeSnippet>foo</CodeSnippet> <b>and</b> <CodeSnippet>bar</CodeSnippet></TableCell>
-                        <TableRow>
+                            </TableRow>
-                            <TableCell className="py-2"><CodeSnippet>foo (bar <Highlight>or</Highlight> baz)</CodeSnippet></TableCell>
+                            <TableRow>
-                            <TableCell className="py-2">Match files with regex <CodeSnippet>/foo/</CodeSnippet> <b>and</b> either <CodeSnippet>/bar/</CodeSnippet> <b>or</b> <CodeSnippet>/baz/</CodeSnippet></TableCell>
+                                <TableCell className="py-2"><CodeSnippet>{`"foo bar"`}</CodeSnippet></TableCell>
-                        </TableRow>
+                                <TableCell className="py-2">Match files containing the phrase <CodeSnippet>foo bar</CodeSnippet></TableCell>
-                    </TableBody>
+                            </TableRow>
-                </Table>
+                            <TableRow>
+                                <TableCell className="py-2"><CodeSnippet>{'"foo \\"bar\\""'}</CodeSnippet></TableCell>
+                                <TableCell className="py-2">Match files containing <CodeSnippet>foo &quot;bar&quot;</CodeSnippet> exactly (escaped quotes)</TableCell>
+                            </TableRow>
+                        </TableBody>
+                    </Table>
+                </div>
 
-                <Separator className="my-2"/>
+                <Separator className="my-4"/>
-                <p className="text-sm">
-                    Expressions can be prefixed with certain keywords to modify search behavior. Some keywords can be negated using the <CodeSnippet>-</CodeSnippet> prefix.
-                </p>
 
-                <Table>
+                <div>
+                    <h3 className="text-lg font-semibold mt-4 mb-0">Regex search</h3>
+                    <p className="text-sm mb-2 mt-0">
+                        Toggle the <RegexIcon className="inline w-4 h-4 align-middle mx-0.5 border rounded px-0.5 py-0.5" /> button to interpret search patterns as regular expressions.
+                    </p>
+                    <Table>
+                        <TableHeader>
+                            <TableRow>
+                                <TableHead className="py-2">Example</TableHead>
+                                <TableHead className="py-2">Explanation</TableHead>
+                            </TableRow>
+                        </TableHeader>
+                        <TableBody>
+                            <TableRow>
+                                <TableCell className="py-2"><CodeSnippet>foo</CodeSnippet></TableCell>
+                                <TableCell className="py-2">Match files with regex <CodeSnippet>/foo/</CodeSnippet></TableCell>
+                            </TableRow>
+                            <TableRow>
+                                <TableCell className="py-2"><CodeSnippet>foo.*bar</CodeSnippet></TableCell>
+                                <TableCell className="py-2">Match files with regex <CodeSnippet>/foo.*bar/</CodeSnippet> (foo followed by any characters, then bar)</TableCell>
+                            </TableRow>
+                            <TableRow>
+                                <TableCell className="py-2"><CodeSnippet>{`^function\\s+\\w+`}</CodeSnippet></TableCell>
+                                <TableCell className="py-2">Match files with regex <CodeSnippet>/^function\s+\w+/</CodeSnippet> (function at start of line, followed by whitespace and word characters)</TableCell>
+                            </TableRow>
+                            <TableRow>
+                                <TableCell className="py-2"><CodeSnippet>{`"foo bar"`}</CodeSnippet></TableCell>
+                                <TableCell className="py-2">Match files with regex <CodeSnippet>/foo bar/</CodeSnippet>. Quotes are not matched.</TableCell>
+                            </TableRow>
+                        </TableBody>
+                    </Table>
+                </div>
+
+                <Separator className="my-4"/>
+
+                <div>
+                    <h3 className="text-lg font-semibold mt-4 mb-0">Search filters</h3>
+                    <p className="text-sm mb-2 mt-0">
+                        Search queries (keyword or regex) can include multiple search filters to further refine the search results. Some filters can be negated using the <CodeSnippet>-</CodeSnippet> prefix.
+                    </p>
+
+                    <Table>
                     <TableHeader>
                         <TableRow>
                             <TableHead className="py-2">Prefix</TableHead>
@@ -219,7 +242,39 @@ export const SyntaxReferenceGuide = () => {
                             </TableCell>
                         </TableRow>
                     </TableBody>
-                </Table>
+                    </Table>
+                </div>
+
+                <Separator className="my-4"/>
+
+                <div>
+                    <h3 className="text-lg font-semibold mt-4 mb-0">Boolean operators &amp; grouping</h3>
+                    <p className="text-sm mb-2 mt-0">
+                        By default, space-seperated expressions are and&apos;d together. Using the <CodeSnippet>or</CodeSnippet> keyword as well as parantheses <CodeSnippet>()</CodeSnippet> can be used to create more complex boolean logic. Parantheses can be negated using the <CodeSnippet>-</CodeSnippet> prefix.
+                    </p>
+                    <Table>
+                        <TableHeader>
+                            <TableRow>
+                                <TableHead className="py-2">Example</TableHead>
+                                <TableHead className="py-2">Explanation</TableHead>
+                            </TableRow>
+                        </TableHeader>
+                        <TableBody>
+                            <TableRow>
+                                <TableCell className="py-2"><CodeSnippet>foo <Highlight>or</Highlight> bar</CodeSnippet></TableCell>
+                                <TableCell className="py-2">Match files containing <CodeSnippet>foo</CodeSnippet> <b>or</b> <CodeSnippet>bar</CodeSnippet></TableCell>
+                            </TableRow>
+                            <TableRow>
+                                <TableCell className="py-2"><CodeSnippet>foo (bar <Highlight>or</Highlight> baz)</CodeSnippet></TableCell>
+                                <TableCell className="py-2">Match files containing <CodeSnippet>foo</CodeSnippet> <b>and</b> either <CodeSnippet>bar</CodeSnippet> <b>or</b> <CodeSnippet>baz</CodeSnippet>.</TableCell>
+                            </TableRow>
+                            <TableRow>
+                                <TableCell className="py-2"><CodeSnippet>-(foo) bar</CodeSnippet></TableCell>
+                                <TableCell className="py-2">Match files containing <CodeSnippet>bar</CodeSnippet> <b>and not</b> <CodeSnippet>foo</CodeSnippet>.</TableCell>
+                            </TableRow>
+                        </TableBody>
+                    </Table>
+                </div>
             </DialogContent>
         </Dialog>
     )

packages/web/src/app/[domain]/repos/[id]/page.tsx

@@ -1,4 +1,4 @@
-import { sew } from "@/actions"
+import { getCurrentUserRole, sew } from "@/actions"
 import { Badge } from "@/components/ui/badge"
 import { Button } from "@/components/ui/button"
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"
@@ -19,6 +19,7 @@ import { BackButton } from "../../components/backButton"
 import { DisplayDate } from "../../components/DisplayDate"
 import { RepoBranchesTable } from "../components/repoBranchesTable"
 import { RepoJobsTable } from "../components/repoJobsTable"
+import { OrgRole } from "@sourcebot/db"
 
 export default async function RepoDetailPage({ params }: { params: Promise<{ id: string }> }) {
     const { id } = await params
@@ -51,6 +52,11 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
 
     const repoMetadata = repoMetadataSchema.parse(repo.metadata);
 
+    const userRole = await getCurrentUserRole(SINGLE_TENANT_ORG_DOMAIN);
+    if (isServiceError(userRole)) {
+        throw new ServiceErrorException(userRole);
+    }
+
     return (
         <>
             <div className="mb-6">
@@ -172,7 +178,11 @@ export default async function RepoDetailPage({ params }: { params: Promise<{ id:
                 </CardHeader>
                 <CardContent>
                     <Suspense fallback={<Skeleton className="h-96 w-full" />}>
-                        <RepoJobsTable data={repo.jobs} />
+                        <RepoJobsTable
+                            data={repo.jobs}
+                            repoId={repo.id}
+                            isIndexButtonVisible={userRole === OrgRole.OWNER}
+                        />
                     </Suspense>
                 </CardContent>
             </Card>

packages/web/src/app/[domain]/repos/components/repoJobsTable.tsx

@@ -18,7 +18,7 @@ import {
     useReactTable,
 } from "@tanstack/react-table"
 import { cva } from "class-variance-authority"
-import { AlertCircle, ArrowUpDown, RefreshCwIcon } from "lucide-react"
+import { AlertCircle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react"
 import * as React from "react"
 import { CopyIconButton } from "../../components/copyIconButton"
 import { useMemo } from "react"
@@ -26,6 +26,9 @@ import { LightweightCodeHighlighter } from "../../components/lightweightCodeHigh
 import { useRouter } from "next/navigation"
 import { useToast } from "@/components/hooks/use-toast"
 import { DisplayDate } from "../../components/DisplayDate"
+import { LoadingButton } from "@/components/ui/loading-button"
+import { indexRepo } from "@/features/workerApi/actions"
+import { isServiceError } from "@/lib/utils"
 
 // @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS
 
@@ -129,7 +132,7 @@ export const columns: ColumnDef<RepoIndexingJob>[] = [
                 </Button>
             )
         },
-        cell: ({ row }) => <DisplayDate date={row.getValue("createdAt") as Date} className="ml-3"/>,
+        cell: ({ row }) => <DisplayDate date={row.getValue("createdAt") as Date} className="ml-3" />,
     },
     {
         accessorKey: "completedAt",
@@ -147,7 +150,7 @@ export const columns: ColumnDef<RepoIndexingJob>[] = [
                 return "-";
             }
 
-            return <DisplayDate date={completedAt} className="ml-3"/>
+            return <DisplayDate date={completedAt} className="ml-3" />
         },
     },
     {
@@ -176,13 +179,41 @@ export const columns: ColumnDef<RepoIndexingJob>[] = [
     },
 ]
 
-export const RepoJobsTable = ({ data }: { data: RepoIndexingJob[] }) => {
+export const RepoJobsTable = ({
+    data,
+    repoId,
+    isIndexButtonVisible,
+}: {
+    data: RepoIndexingJob[],
+    repoId: number,
+    isIndexButtonVisible: boolean,
+}) => {
     const [sorting, setSorting] = React.useState<SortingState>([{ id: "createdAt", desc: true }])
     const [columnFilters, setColumnFilters] = React.useState<ColumnFiltersState>([])
     const [columnVisibility, setColumnVisibility] = React.useState<VisibilityState>({})
     const router = useRouter();
     const { toast } = useToast();
 
+    const [isIndexSubmitting, setIsIndexSubmitting] = React.useState(false);
+    const onIndexButtonClick = React.useCallback(async () => {
+        setIsIndexSubmitting(true);
+        const response = await indexRepo(repoId);
+
+        if (!isServiceError(response)) {
+            const { jobId } = response;
+            toast({
+                description: `✅ Repository sync triggered successfully. Job ID: ${jobId}`,
+            })
+            router.refresh();
+        } else {
+            toast({
+                description: `❌ Failed to index repository. ${response.message}`,
+            });
+        }
+
+        setIsIndexSubmitting(false);
+    }, [repoId, router, toast]);
+
     const table = useReactTable({
         data,
         columns,
@@ -247,19 +278,31 @@ export const RepoJobsTable = ({ data }: { data: RepoIndexingJob[] }) => {
                     </SelectContent>
                 </Select>
 
-                <Button
+                <div className="ml-auto flex items-center gap-2">
-                    variant="outline"
+                    <Button
-                    className="ml-auto"
+                        variant="outline"
-                    onClick={() => {
+                        onClick={() => {
-                        router.refresh();
+                            router.refresh();
-                        toast({
+                            toast({
-                            description: "Page refreshed",
+                                description: "Page refreshed",
-                        });
+                            });
-                    }}
+                        }}
-                >
+                    >
-                    <RefreshCwIcon className="w-3 h-3" />
+                        <RefreshCwIcon className="w-3 h-3" />
-                    Refresh
+                        Refresh
-                </Button>
+                    </Button>
+
+                    {isIndexButtonVisible && (
+                        <LoadingButton
+                            onClick={onIndexButtonClick}
+                            loading={isIndexSubmitting}
+                            variant="outline"
+                        >
+                            <PlusCircleIcon className="w-3 h-3" />
+                            Trigger sync
+                        </LoadingButton>
+                    )}
+                </div>
             </div>
 
             <div className="rounded-md border">

packages/web/src/app/[domain]/search/components/codePreviewPanel/codePreview.tsx

@@ -1,12 +1,16 @@
 'use client';
 
+import { useBrowseNavigation } from "@/app/[domain]/browse/hooks/useBrowseNavigation";
 import { EditorContextMenu } from "@/app/[domain]/components/editorContextMenu";
 import { Button } from "@/components/ui/button";
 import { ScrollArea } from "@/components/ui/scroll-area";
-import { SearchResultChunk } from "@/features/search/types";
+import { SymbolHoverPopup } from "@/ee/features/codeNav/components/symbolHoverPopup";
+import { symbolHoverTargetsExtension } from "@/ee/features/codeNav/components/symbolHoverPopup/symbolHoverTargetsExtension";
+import { useHasEntitlement } from "@/features/entitlements/useHasEntitlement";
+import { SearchResultChunk } from "@/features/search";
+import { useCodeMirrorLanguageExtension } from "@/hooks/useCodeMirrorLanguageExtension";
 import { useCodeMirrorTheme } from "@/hooks/useCodeMirrorTheme";
 import { useKeymapExtension } from "@/hooks/useKeymapExtension";
-import { useCodeMirrorLanguageExtension } from "@/hooks/useCodeMirrorLanguageExtension";
 import { gutterWidthExtension } from "@/lib/extensions/gutterWidthExtension";
 import { highlightRanges, searchResultHighlightExtension } from "@/lib/extensions/searchResultHighlightExtension";
 import { search } from "@codemirror/search";
@@ -16,15 +20,6 @@ import { Scrollbar } from "@radix-ui/react-scroll-area";
 import CodeMirror, { ReactCodeMirrorRef, SelectionRange } from '@uiw/react-codemirror';
 import { ArrowDown, ArrowUp } from "lucide-react";
 import { Dispatch, SetStateAction, useCallback, useEffect, useMemo, useState } from "react";
-import { useBrowseNavigation } from "@/app/[domain]/browse/hooks/useBrowseNavigation";
-import { SymbolHoverPopup } from "@/ee/features/codeNav/components/symbolHoverPopup";
-import { symbolHoverTargetsExtension } from "@/ee/features/codeNav/components/symbolHoverPopup/symbolHoverTargetsExtension";
-import { useHasEntitlement } from "@/features/entitlements/useHasEntitlement";
-import { SymbolDefinition } from "@/ee/features/codeNav/components/symbolHoverPopup/useHoveredOverSymbolInfo";
-import { createAuditAction } from "@/ee/features/audit/actions";
-import { useDomain } from "@/hooks/useDomain";
-
-import useCaptureEvent from "@/hooks/useCaptureEvent";
 
 export interface CodePreviewFile {
     content: string;
@@ -53,7 +48,6 @@ export const CodePreview = ({
     const [editorRef, setEditorRef] = useState<ReactCodeMirrorRef | null>(null);
     const { navigateToPath } = useBrowseNavigation();
     const hasCodeNavEntitlement = useHasEntitlement("code-nav");
-    const domain = useDomain();
 
     const [gutterWidth, setGutterWidth] = useState(0);
     const theme = useCodeMirrorTheme();
@@ -62,8 +56,6 @@ export const CodePreview = ({
     const languageExtension = useCodeMirrorLanguageExtension(file?.language ?? '', editorRef?.view);
     const [currentSelection, setCurrentSelection] = useState<SelectionRange>();
 
-    const captureEvent = useCaptureEvent();
-
     const extensions = useMemo(() => {
         return [
             keymapExtension,
@@ -118,81 +110,6 @@ export const CodePreview = ({
         onSelectedMatchIndexChange((prev) => prev + 1);
     }, [onSelectedMatchIndexChange]);
 
-    const onGotoDefinition = useCallback((symbolName: string, symbolDefinitions: SymbolDefinition[]) => {
-        captureEvent('wa_goto_definition_pressed', {
-            source: 'preview',
-        });
-        createAuditAction({
-            action: "user.performed_goto_definition",
-            metadata: {
-                message: symbolName,
-            },
-        }, domain)
-
-        if (symbolDefinitions.length === 0) {
-            return;
-        }
-
-        if (symbolDefinitions.length === 1) {
-            const symbolDefinition = symbolDefinitions[0];
-            const { fileName, repoName } = symbolDefinition;
-
-            navigateToPath({
-                repoName,
-                revisionName: file.revision,
-                path: fileName,
-                pathType: 'blob',
-                highlightRange: symbolDefinition.range,
-            })
-        } else {
-            navigateToPath({
-                repoName,
-                revisionName: file.revision,
-                path: file.filepath,
-                pathType: 'blob',
-                setBrowseState: {
-                    selectedSymbolInfo: {
-                        symbolName,
-                        repoName,
-                        revisionName: file.revision,
-                        language: file.language,
-                    },
-                    activeExploreMenuTab: "definitions",
-                    isBottomPanelCollapsed: false,
-                }
-            });
-        }
-    }, [captureEvent, file.filepath, file.language, file.revision, navigateToPath, repoName, domain]);
-    
-    const onFindReferences = useCallback((symbolName: string) => {
-        captureEvent('wa_find_references_pressed', {
-            source: 'preview',
-        });
-        createAuditAction({
-            action: "user.performed_find_references",
-            metadata: {
-                message: symbolName,
-            },
-        }, domain)
-
-        navigateToPath({
-            repoName,
-            revisionName: file.revision,
-            path: file.filepath,
-            pathType: 'blob',
-            setBrowseState: {
-                selectedSymbolInfo: {
-                    repoName,
-                    symbolName,
-                    revisionName: file.revision,
-                    language: file.language,
-                },
-                activeExploreMenuTab: "references",
-                isBottomPanelCollapsed: false,
-            }
-        })
-    }, [captureEvent, file.filepath, file.language, file.revision, navigateToPath, repoName, domain]);
-
     return (
         <div className="flex flex-col h-full">
             <div className="flex flex-row bg-accent items-center justify-between pr-3 py-0.5 mt-7">
@@ -289,11 +206,12 @@ export const CodePreview = ({
 
                     {editorRef && hasCodeNavEntitlement && (
                         <SymbolHoverPopup
+                            source="preview"
                             editorRef={editorRef}
                             language={file.language}
                             revisionName={file.revision}
-                            onFindReferences={onFindReferences}
+                            fileName={file.filepath}
-                            onGotoDefinition={onGotoDefinition}
+                            repoName={repoName}
                         />
                     )}
                 </CodeMirror>

packages/web/src/app/[domain]/search/components/codePreviewPanel/index.tsx

@@ -2,11 +2,11 @@
 
 import { useQuery } from "@tanstack/react-query";
 import { CodePreview } from "./codePreview";
-import { SearchResultFile } from "@/features/search/types";
+import { SearchResultFile } from "@/features/search";
 import { SymbolIcon } from "@radix-ui/react-icons";
 import { SetStateAction, Dispatch, useMemo } from "react";
-import { getFileSource } from "@/features/search/fileSourceApi";
 import { unwrapServiceError } from "@/lib/utils";
+import { getFileSource } from "@/app/api/(client)/client";
 
 interface CodePreviewPanelProps {
     previewedFile: SearchResultFile;

packages/web/src/app/[domain]/search/components/filterPanel/filter.tsx

@@ -5,6 +5,7 @@ import { compareEntries, Entry } from "./entry";
 import { Input } from "@/components/ui/input";
 import Fuse from "fuse.js";
 import { cn } from "@/lib/utils"
+import { Skeleton } from "@/components/ui/skeleton";
 
 interface FilterProps {
     title: string,
@@ -12,6 +13,7 @@ interface FilterProps {
     entries: Entry[],
     onEntryClicked: (key: string) => void,
     className?: string,
+    isStreaming: boolean,
 }
 
 export const Filter = ({
@@ -20,6 +22,7 @@ export const Filter = ({
     entries,
     onEntryClicked,
     className,
+    isStreaming,
 }: FilterProps) => {
     const [searchFilter, setSearchFilter] = useState<string>("");
 
@@ -43,27 +46,34 @@ export const Filter = ({
             className
         )}>
             <h2 className="text-sm font-semibold">{title}</h2>
-            <div className="pr-1">
+            {(isStreaming && entries.length === 0) ? (
-                <Input
+                <Skeleton className="h-12 w-full" />
-                    placeholder={searchPlaceholder}
+            ) : (
-                    className="h-8"
+                <>
-                    onChange={(event) => setSearchFilter(event.target.value)}
+                    <div className="pr-1">
-                />
+                        <Input
-            </div>
+                            placeholder={searchPlaceholder}
-
+                            className="h-8"
-            <div
+                            onChange={(event) => setSearchFilter(event.target.value)}
-                className="flex flex-col gap-0.5 text-sm overflow-scroll no-scrollbar"
-            >
-                {filteredEntries
-                    .sort((entryA, entryB) => compareEntries(entryB, entryA))
-                    .map((entry) => (
-                        <Entry
-                            key={entry.key}
-                            entry={entry}
-                            onClicked={() => onEntryClicked(entry.key)}
                         />
-                    ))}
+                    </div>
-            </div>
+
+                    <div
+                        className="flex flex-col gap-0.5 text-sm overflow-scroll no-scrollbar"
+                    >
+                        {filteredEntries
+                            .sort((entryA, entryB) => compareEntries(entryB, entryA))
+                            .map((entry) => (
+                                <Entry
+                                    key={entry.key}
+                                    entry={entry}
+                                    onClicked={() => onEntryClicked(entry.key)}
+                                />
+                            ))}
+                    </div>
+                </>
+            )}
+
         </div>
     )
 }

packages/web/src/app/[domain]/search/components/filterPanel/index.tsx

@@ -1,7 +1,7 @@
 'use client';
 
 import { FileIcon } from "@/components/ui/fileIcon";
-import { RepositoryInfo, SearchResultFile } from "@/features/search/types";
+import { RepositoryInfo, SearchResultFile } from "@/features/search";
 import { cn, getCodeHostInfoForRepo } from "@/lib/utils";
 import { LaptopIcon } from "@radix-ui/react-icons";
 import Image from "next/image";
@@ -15,6 +15,8 @@ import { useGetSelectedFromQuery } from "./useGetSelectedFromQuery";
 interface FilePanelProps {
     matches: SearchResultFile[];
     repoInfo: Record<number, RepositoryInfo>;
+    onFilterChange?: () => void;
+    isStreaming: boolean;
 }
 
 /**
@@ -31,10 +33,14 @@ interface FilePanelProps {
  * 
  * @param matches - Array of search result files to filter
  * @param repoInfo - Information about repositories including their display names and icons
+ * @param onFilterChange - Optional callback that is called whenever a filter is applied or removed
+ * @param isStreaming - Whether the search is streaming
  */
 export const FilterPanel = ({
     matches,
     repoInfo,
+    onFilterChange,
+    isStreaming,
 }: FilePanelProps) => {
     const router = useRouter();
     const searchParams = useSearchParams();
@@ -148,9 +154,11 @@ export const FilterPanel = ({
 
                     if (newParams.toString() !== searchParams.toString()) {
                         router.replace(`?${newParams.toString()}`, { scroll: false });
+                        onFilterChange?.();
                     }
                 }}
                 className="max-h-[50%]"
+                isStreaming={isStreaming}
             />
             <Filter
                 title="Filter By Language"
@@ -170,9 +178,11 @@ export const FilterPanel = ({
 
                     if (newParams.toString() !== searchParams.toString()) {
                         router.replace(`?${newParams.toString()}`, { scroll: false });
+                        onFilterChange?.();
                     }
                 }}
                 className="overflow-auto"
+                isStreaming={isStreaming}
             />
         </div>
     )

packages/web/src/app/[domain]/search/components/filterPanel/useFilterMatches.ts

@@ -1,6 +1,6 @@
 'use client';
 
-import { SearchResultFile } from "@/features/search/types";
+import { SearchResultFile } from "@/features/search";
 import {  useMemo } from "react";
 import { useGetSelectedFromQuery } from "./useGetSelectedFromQuery";
 

packages/web/src/app/[domain]/search/components/searchResultsPage.tsx

@@ -11,76 +11,76 @@ import {
 } from "@/components/ui/resizable";
 import { Separator } from "@/components/ui/separator";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
-import { RepositoryInfo, SearchResultFile, SearchStats } from "@/features/search/types";
+import { RepositoryInfo, SearchResultFile, SearchStats } from "@/features/search";
 import useCaptureEvent from "@/hooks/useCaptureEvent";
 import { useDomain } from "@/hooks/useDomain";
 import { useNonEmptyQueryParam } from "@/hooks/useNonEmptyQueryParam";
 import { useSearchHistory } from "@/hooks/useSearchHistory";
 import { SearchQueryParams } from "@/lib/types";
-import { createPathWithQueryParams, measure, unwrapServiceError } from "@/lib/utils";
+import { createPathWithQueryParams } from "@/lib/utils";
-import { InfoCircledIcon, SymbolIcon } from "@radix-ui/react-icons";
+import { InfoCircledIcon } from "@radix-ui/react-icons";
-import { useQuery } from "@tanstack/react-query";
 import { useLocalStorage } from "@uidotdev/usehooks";
-import { AlertTriangleIcon, BugIcon, FilterIcon } from "lucide-react";
+import { AlertTriangleIcon, BugIcon, FilterIcon, RefreshCwIcon } from "lucide-react";
 import { useRouter } from "next/navigation";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { useHotkeys } from "react-hotkeys-hook";
 import { ImperativePanelHandle } from "react-resizable-panels";
-import { search } from "../../../api/(client)/client";
 import { CopyIconButton } from "../../components/copyIconButton";
 import { SearchBar } from "../../components/searchBar";
 import { TopBar } from "../../components/topBar";
+import { useStreamedSearch } from "../useStreamedSearch";
 import { CodePreviewPanel } from "./codePreviewPanel";
 import { FilterPanel } from "./filterPanel";
 import { useFilteredMatches } from "./filterPanel/useFilterMatches";
-import { SearchResultsPanel } from "./searchResultsPanel";
+import { SearchResultsPanel, SearchResultsPanelHandle } from "./searchResultsPanel";
-
+import { ServiceErrorException } from "@/lib/serviceError";
-const DEFAULT_MAX_MATCH_COUNT = 5000;
 
 interface SearchResultsPageProps {
     searchQuery: string;
+    defaultMaxMatchCount: number;
+    isRegexEnabled: boolean;
+    isCaseSensitivityEnabled: boolean;
 }
 
 export const SearchResultsPage = ({
     searchQuery,
+    defaultMaxMatchCount,
+    isRegexEnabled,
+    isCaseSensitivityEnabled,
 }: SearchResultsPageProps) => {
     const router = useRouter();
     const { setSearchHistory } = useSearchHistory();
-    const captureEvent = useCaptureEvent();
     const domain = useDomain();
     const { toast } = useToast();
+    const captureEvent = useCaptureEvent();
 
     // Encodes the number of matches to return in the search response.
-    const _maxMatchCount = parseInt(useNonEmptyQueryParam(SearchQueryParams.matches) ?? `${DEFAULT_MAX_MATCH_COUNT}`);
+    const _maxMatchCount = parseInt(useNonEmptyQueryParam(SearchQueryParams.matches) ?? `${defaultMaxMatchCount}`);
-    const maxMatchCount = isNaN(_maxMatchCount) ? DEFAULT_MAX_MATCH_COUNT : _maxMatchCount;
+    const maxMatchCount = isNaN(_maxMatchCount) ? defaultMaxMatchCount : _maxMatchCount;
 
     const {
-        data: searchResponse,
+        error,
-        isPending: isSearchPending,
+        files,
-        isFetching: isFetching,
+        repoInfo,
-        error
+        timeToSearchCompletionMs,
-    } = useQuery({
+        timeToFirstSearchResultMs,
-        queryKey: ["search", searchQuery, maxMatchCount],
+        isStreaming,
-        queryFn: () => measure(() => unwrapServiceError(search({
+        numMatches,
-            query: searchQuery,
+        isExhaustive,
-            matches: maxMatchCount,
+        stats,
-            contextLines: 3,
+    } = useStreamedSearch({
-            whole: false,
+        query: searchQuery,
-        }, domain)), "client.search"),
+        matches: maxMatchCount,
-        select: ({ data, durationMs }) => ({
+        contextLines: 3,
-            ...data,
+        whole: false,
-            totalClientSearchDurationMs: durationMs,
+        isRegexEnabled,
-        }),
+        isCaseSensitivityEnabled,
-        enabled: searchQuery.length > 0,
-        refetchOnWindowFocus: false,
-        retry: false,
-        staleTime: 0,
     });
 
     useEffect(() => {
         if (error) {
             toast({
-                description: `❌ Search failed. Reason: ${error.message}`,
+                description: `❌ Search failed. Reason: ${error instanceof ServiceErrorException ? error.serviceError.message : error.message}`,
             });
         }
     }, [error, toast]);
@@ -103,38 +103,51 @@ export const SearchResultsPage = ({
     }, [searchQuery, setSearchHistory]);
 
     useEffect(() => {
-        if (!searchResponse) {
+        if (isStreaming || !stats) {
             return;
         }
 
-        const fileLanguages = searchResponse.files?.map(file => file.language) || [];
+        const fileLanguages = files.map(file => file.language) || [];
+
+        console.debug('timeToFirstSearchResultMs:', timeToFirstSearchResultMs);
+        console.debug('timeToSearchCompletionMs:', timeToSearchCompletionMs);
 
         captureEvent("search_finished", {
-            durationMs: searchResponse.totalClientSearchDurationMs,
+            durationMs: timeToSearchCompletionMs,
-            fileCount: searchResponse.stats.fileCount,
+            timeToSearchCompletionMs,
-            matchCount: searchResponse.stats.totalMatchCount,
+            timeToFirstSearchResultMs,
-            actualMatchCount: searchResponse.stats.actualMatchCount,
+            fileCount: stats.fileCount,
-            filesSkipped: searchResponse.stats.filesSkipped,
+            matchCount: stats.totalMatchCount,
-            contentBytesLoaded: searchResponse.stats.contentBytesLoaded,
+            actualMatchCount: stats.actualMatchCount,
-            indexBytesLoaded: searchResponse.stats.indexBytesLoaded,
+            filesSkipped: stats.filesSkipped,
-            crashes: searchResponse.stats.crashes,
+            contentBytesLoaded: stats.contentBytesLoaded,
-            shardFilesConsidered: searchResponse.stats.shardFilesConsidered,
+            indexBytesLoaded: stats.indexBytesLoaded,
-            filesConsidered: searchResponse.stats.filesConsidered,
+            crashes: stats.crashes,
-            filesLoaded: searchResponse.stats.filesLoaded,
+            shardFilesConsidered: stats.shardFilesConsidered,
-            shardsScanned: searchResponse.stats.shardsScanned,
+            filesConsidered: stats.filesConsidered,
-            shardsSkipped: searchResponse.stats.shardsSkipped,
+            filesLoaded: stats.filesLoaded,
-            shardsSkippedFilter: searchResponse.stats.shardsSkippedFilter,
+            shardsScanned: stats.shardsScanned,
-            ngramMatches: searchResponse.stats.ngramMatches,
+            shardsSkipped: stats.shardsSkipped,
-            ngramLookups: searchResponse.stats.ngramLookups,
+            shardsSkippedFilter: stats.shardsSkippedFilter,
-            wait: searchResponse.stats.wait,
+            ngramMatches: stats.ngramMatches,
-            matchTreeConstruction: searchResponse.stats.matchTreeConstruction,
+            ngramLookups: stats.ngramLookups,
-            matchTreeSearch: searchResponse.stats.matchTreeSearch,
+            wait: stats.wait,
-            regexpsConsidered: searchResponse.stats.regexpsConsidered,
+            matchTreeConstruction: stats.matchTreeConstruction,
-            flushReason: searchResponse.stats.flushReason,
+            matchTreeSearch: stats.matchTreeSearch,
+            regexpsConsidered: stats.regexpsConsidered,
+            flushReason: stats.flushReason,
             fileLanguages,
+            isSearchExhaustive: isExhaustive,
         });
-    }, [captureEvent, searchQuery, searchResponse]);
+    }, [
-
+        captureEvent,
+        files,
+        isStreaming,
+        isExhaustive,
+        stats,
+        timeToSearchCompletionMs,
+        timeToFirstSearchResultMs,
+    ]);
 
     const onLoadMoreResults = useCallback(() => {
         const url = createPathWithQueryParams(`/${domain}/search`,
@@ -144,6 +157,13 @@ export const SearchResultsPage = ({
         router.push(url);
     }, [maxMatchCount, router, searchQuery, domain]);
 
+    // Look for any files that are not on the default branch.
+    const isBranchFilteringEnabled = useMemo(() => {
+        return files.some((file) => {
+            return file.branches?.some((branch) => branch !== 'HEAD') ?? false;
+        });
+    }, [files]);
+
     return (
         <div className="flex flex-col h-screen overflow-clip">
             {/* TopBar */}
@@ -152,32 +172,32 @@ export const SearchResultsPage = ({
             >
                 <SearchBar
                     size="sm"
-                    defaultQuery={searchQuery}
+                    defaults={{
+                        isRegexEnabled,
+                        isCaseSensitivityEnabled,
+                        query: searchQuery,
+                    }}
                     className="w-full"
                 />
             </TopBar>
 
-            {(isSearchPending || isFetching) ? (
+            {error ? (
-                <div className="flex flex-col items-center justify-center h-full gap-2">
-                    <SymbolIcon className="h-6 w-6 animate-spin" />
-                    <p className="font-semibold text-center">Searching...</p>
-                </div>
-            ) : error ? (
                 <div className="flex flex-col items-center justify-center h-full gap-2">
                     <AlertTriangleIcon className="h-6 w-6" />
                     <p className="font-semibold text-center">Failed to search</p>
-                    <p className="text-sm text-center">{error.message}</p>
+                    <p className="text-sm text-center">{error instanceof ServiceErrorException ? error.serviceError.message : error.message}</p>
                 </div>
             ) : (
                 <PanelGroup
-                    fileMatches={searchResponse.files}
+                    fileMatches={files}
-                    isMoreResultsButtonVisible={searchResponse.isSearchExhaustive === false}
                     onLoadMoreResults={onLoadMoreResults}
-                    isBranchFilteringEnabled={searchResponse.isBranchFilteringEnabled}
+                    numMatches={numMatches}
-                    repoInfo={searchResponse.repositoryInfo}
+                    repoInfo={repoInfo}
-                    searchDurationMs={searchResponse.totalClientSearchDurationMs}
+                    searchDurationMs={timeToSearchCompletionMs}
-                    numMatches={searchResponse.stats.actualMatchCount}
+                    isStreaming={isStreaming}
-                    searchStats={searchResponse.stats}
+                    searchStats={stats}
+                    isMoreResultsButtonVisible={!isExhaustive}
+                    isBranchFilteringEnabled={isBranchFilteringEnabled}
                 />
             )}
         </div>
@@ -186,10 +206,11 @@ export const SearchResultsPage = ({
 
 interface PanelGroupProps {
     fileMatches: SearchResultFile[];
-    isMoreResultsButtonVisible?: boolean;
     onLoadMoreResults: () => void;
+    isStreaming: boolean;
+    isMoreResultsButtonVisible?: boolean;
     isBranchFilteringEnabled: boolean;
-    repoInfo: RepositoryInfo[];
+    repoInfo: Record<number, RepositoryInfo>;
     searchDurationMs: number;
     numMatches: number;
     searchStats?: SearchStats;
@@ -198,9 +219,10 @@ interface PanelGroupProps {
 const PanelGroup = ({
     fileMatches,
     isMoreResultsButtonVisible,
+    isStreaming,
     onLoadMoreResults,
     isBranchFilteringEnabled,
-    repoInfo: _repoInfo,
+    repoInfo,
     searchDurationMs: _searchDurationMs,
     numMatches,
     searchStats,
@@ -208,6 +230,7 @@ const PanelGroup = ({
     const [previewedFile, setPreviewedFile] = useState<SearchResultFile | undefined>(undefined);
     const filteredFileMatches = useFilteredMatches(fileMatches);
     const filterPanelRef = useRef<ImperativePanelHandle>(null);
+    const searchResultsPanelRef = useRef<SearchResultsPanelHandle>(null);
     const [selectedMatchIndex, setSelectedMatchIndex] = useState(0);
 
     const [isFilterPanelCollapsed, setIsFilterPanelCollapsed] = useLocalStorage('isFilterPanelCollapsed', false);
@@ -228,13 +251,6 @@ const PanelGroup = ({
         return Math.round(_searchDurationMs);
     }, [_searchDurationMs]);
 
-    const repoInfo = useMemo(() => {
-        return _repoInfo.reduce((acc, repo) => {
-            acc[repo.id] = repo;
-            return acc;
-        }, {} as Record<number, RepositoryInfo>);
-    }, [_repoInfo]);
-
     return (
         <ResizablePanelGroup
             direction="horizontal"
@@ -255,6 +271,10 @@ const PanelGroup = ({
                 <FilterPanel
                     matches={fileMatches}
                     repoInfo={repoInfo}
+                    isStreaming={isStreaming}
+                    onFilterChange={() => {
+                        searchResultsPanelRef.current?.resetScroll();
+                    }}
                 />
             </ResizablePanel>
             {isFilterPanelCollapsed && (
@@ -291,45 +311,58 @@ const PanelGroup = ({
                 order={2}
             >
                 <div className="py-1 px-2 flex flex-row items-center">
-                    <Tooltip>
+                    {isStreaming ? (
-                        <TooltipTrigger asChild>
+                        <>
-                            <InfoCircledIcon className="w-4 h-4 mr-2" />
+                            <RefreshCwIcon className="h-4 w-4 animate-spin mr-2" />
-                        </TooltipTrigger>
+                            <p className="text-sm font-medium mr-1">Searching...</p>
-                        <TooltipContent side="right" className="flex flex-col items-start gap-2 p-4">
+                            {numMatches > 0 && (
-                            <div className="flex flex-row items-center w-full">
+                                <p className="text-sm font-medium">{`Found ${numMatches} matches in ${fileMatches.length} ${fileMatches.length > 1 ? 'files' : 'file'}`}</p>
-                                <BugIcon className="w-4 h-4 mr-1.5" />
+                            )}
-                                <p className="text-md font-medium">Search stats for nerds</p>
+                        </>
-                                <CopyIconButton
+                    ) : (
-                                    onCopy={() => {
+                        <>
-                                        navigator.clipboard.writeText(JSON.stringify(searchStats, null, 2));
+                            <Tooltip>
-                                        return true;
+                                <TooltipTrigger asChild>
-                                    }}
+                                    <InfoCircledIcon className="w-4 h-4 mr-2" />
-                                    className="ml-auto"
+                                </TooltipTrigger>
-                                />
+                                <TooltipContent side="right" className="flex flex-col items-start gap-2 p-4">
-                            </div>
+                                    <div className="flex flex-row items-center w-full">
-                            <CodeSnippet renderNewlines>
+                                        <BugIcon className="w-4 h-4 mr-1.5" />
-                                {JSON.stringify(searchStats, null, 2)}
+                                        <p className="text-md font-medium">Search stats for nerds</p>
-                            </CodeSnippet>
+                                        <CopyIconButton
-                        </TooltipContent>
+                                            onCopy={() => {
-                    </Tooltip>
+                                                navigator.clipboard.writeText(JSON.stringify(searchStats, null, 2));
-                    {
+                                                return true;
-                        fileMatches.length > 0 ? (
+                                            }}
-                            <p className="text-sm font-medium">{`[${searchDurationMs} ms] Found ${numMatches} matches in ${fileMatches.length} ${fileMatches.length > 1 ? 'files' : 'file'}`}</p>
+                                            className="ml-auto"
-                        ) : (
+                                        />
-                            <p className="text-sm font-medium">No results</p>
+                                    </div>
-                        )
+                                    <CodeSnippet renderNewlines>
-                    }
+                                        {JSON.stringify(searchStats, null, 2)}
-                    {isMoreResultsButtonVisible && (
+                                    </CodeSnippet>
-                        <div
+                                </TooltipContent>
-                            className="cursor-pointer text-blue-500 text-sm hover:underline ml-4"
+                            </Tooltip>
-                            onClick={onLoadMoreResults}
+                            {
-                        >
+                                fileMatches.length > 0 ? (
-                            (load more)
+                                    <p className="text-sm font-medium">{`[${searchDurationMs} ms] Found ${numMatches} matches in ${fileMatches.length} ${fileMatches.length > 1 ? 'files' : 'file'}`}</p>
-                        </div>
+                                ) : (
+                                    <p className="text-sm font-medium">No results</p>
+                                )
+                            }
+                            {isMoreResultsButtonVisible && (
+                                <div
+                                    className="cursor-pointer text-blue-500 text-sm hover:underline ml-4"
+                                    onClick={onLoadMoreResults}
+                                >
+                                    (load more)
+                                </div>
+                            )}
+                        </>
                     )}
                 </div>
                 {filteredFileMatches.length > 0 ? (
                     <SearchResultsPanel
+                        ref={searchResultsPanelRef}
                         fileMatches={filteredFileMatches}
                         onOpenFilePreview={(fileMatch, matchIndex) => {
                             setSelectedMatchIndex(matchIndex ?? 0);
@@ -340,6 +373,11 @@ const PanelGroup = ({
                         isBranchFilteringEnabled={isBranchFilteringEnabled}
                         repoInfo={repoInfo}
                     />
+                ) : isStreaming ? (
+                    <div className="flex flex-col items-center justify-center h-full gap-2">
+                        <RefreshCwIcon className="h-6 w-6 animate-spin" />
+                        <p className="font-semibold text-center">Searching...</p>
+                    </div>
                 ) : (
                     <div className="flex flex-col items-center justify-center h-full">
                         <p className="text-sm text-muted-foreground">No results found</p>