further wip on query language

packages/queryLanguage/src/index.ts

@@ -4,3 +4,4 @@ type Tree = ReturnType<typeof parser.parse>;
 type SyntaxNode = Tree['topNode'];
 export type { Tree, SyntaxNode };
 export * from "./parser";
+export * from "./parser.terms";

packages/queryLanguage/src/parser.terms.ts

@@ -1,10 +1,20 @@
 // This file was generated by lezer-generator. You probably shouldn't edit it.
 export const
-  negate = 24,
+  negate = 21,
   Program = 1,
   OrExpr = 2,
   AndExpr = 3,
   NegateExpr = 4,
   PrefixExpr = 5,
-  ParenExpr = 19,
+  ArchivedExpr = 6,
-  Term = 20
+  RevisionExpr = 7,
+  ContentExpr = 8,
+  FileExpr = 9,
+  ForkExpr = 10,
+  VisibilityExpr = 11,
+  RepoExpr = 12,
+  LangExpr = 13,
+  SymExpr = 14,
+  RepoSetExpr = 15,
+  ParenExpr = 16,
+  Term = 17

packages/queryLanguage/src/query.grammar

@@ -32,21 +32,31 @@ NegateExpr { !negate negate (PrefixExpr | ParenExpr) }
 ParenExpr { "(" query ")" }
 
 PrefixExpr {
-  ArchivedExpr { archivedKw value } |
+  ArchivedExpr |
-  BranchExpr { branchKw value } |
+  RevisionExpr |
-  ContentExpr { contentKw value } |
+  ContentExpr |
-  CaseExpr { caseKw value } |
+  FileExpr |
-  FileExpr { fileKw value } |
+  ForkExpr |
-  ForkExpr { forkKw value } |
+  VisibilityExpr |
-  PublicExpr { publicKw value } |
+  RepoExpr |
-  RepoExpr { repoKw value } |
+  LangExpr |
-  RegexExpr { regexKw value } |
+  SymExpr |
-  LangExpr { langKw value } |
+  RepoSetExpr
-  SymExpr { symKw value } |
-  TypeExpr { typeKw value } |
-  RepoSetExpr { reposetKw value }
 }
 
+RevisionExpr { revisionKw value }
+ContentExpr { contentKw value }
+FileExpr { fileKw value }
+RepoExpr { repoKw value }
+LangExpr { langKw value }
+SymExpr { symKw value }
+RepoSetExpr { reposetKw value }
+
+// Modifiers
+ArchivedExpr { archivedKw value }
+ForkExpr { forkKw value }
+VisibilityExpr { visibilityKw value }
+
 
 Term { quotedString | word }
 
@@ -56,17 +66,14 @@ value { quotedString | word }
 
 @tokens {
   archivedKw { "archived:" }
-  branchKw { "branch:" | "b:" }
+  revisionKw { "rev:" }
   contentKw { "content:" | "c:" }
-  caseKw { "case:" }
   fileKw { "file:" | "f:" }
   forkKw { "fork:" }
-  publicKw { "public:" }
+  visibilityKw { "visibility:" }
   repoKw { "repo:" | "r:" }
-  regexKw { "regex:" }
   langKw { "lang:" }
   symKw { "sym:" }
-  typeKw { "type:" | "t:" }
   reposetKw { "reposet:" }
   
   or { "or" ![a-zA-Z0-9_] }
@@ -81,9 +88,9 @@ value { quotedString | word }
   
   @precedence { 
     quotedString,
-    archivedKw, branchKw, contentKw, caseKw, fileKw, 
+    archivedKw, revisionKw, contentKw, fileKw, 
-    forkKw, publicKw, repoKw, regexKw, langKw, 
+    forkKw, visibilityKw, repoKw, langKw, 
-    symKw, typeKw, reposetKw, or,
+    symKw, reposetKw, or,
     word 
   }
 }

packages/queryLanguage/src/tokens.ts

@@ -3,7 +3,7 @@ import { negate } from "./parser.terms";
 
 // External tokenizer for negation
 // Only tokenizes `-` as negate when followed by a prefix keyword or `(`
-export const negateToken = new ExternalTokenizer((input, stack) => {
+export const negateToken = new ExternalTokenizer((input) => {
     if (input.next !== 45 /* '-' */) return; // Not a dash
     
     const startPos = input.pos;
@@ -25,24 +25,22 @@ export const negateToken = new ExternalTokenizer((input, stack) => {
     }
     
     // Check if followed by a prefix keyword (by checking for keyword followed by colon)
-    // We need to look ahead to find the colon
+    // Look ahead until we hit a delimiter or colon
     const checkPos = input.pos;
     let foundColon = false;
-    let charCount = 0;
     
-    // Look ahead up to 10 characters to find a colon
+    // Look ahead until we hit a delimiter or colon
-    while (charCount < 10 && ch >= 0) {
+    while (ch >= 0) {
         if (ch === 58 /* ':' */) {
             foundColon = true;
             break;
         }
+        // Hit a delimiter (whitespace, paren, or quote) - not a prefix keyword
         if (ch === 32 || ch === 9 || ch === 10 || ch === 40 || ch === 41 || ch === 34) {
-            // Hit whitespace, paren, or quote - not a prefix
             break;
         }
         input.advance();
         ch = input.next;
-        charCount++;
     }
     
     // Reset position

packages/queryLanguage/test/negation.txt

@@ -62,21 +62,13 @@ Program(NegateExpr(PrefixExpr(LangExpr)))
 
 Program(NegateExpr(PrefixExpr(ContentExpr)))
 
-# Negate branch prefix
+# Negate revision prefix
 
--branch:develop
+-rev:develop
 
 ==>
 
-Program(NegateExpr(PrefixExpr(BranchExpr)))
+Program(NegateExpr(PrefixExpr(RevisionExpr)))
-
-# Negate case prefix
-
--case:yes
-
-==>
-
-Program(NegateExpr(PrefixExpr(CaseExpr)))
 
 # Negate archived prefix
 
@@ -94,13 +86,13 @@ Program(NegateExpr(PrefixExpr(ArchivedExpr)))
 
 Program(NegateExpr(PrefixExpr(ForkExpr)))
 
-# Negate public prefix
+# Negate visibility prefix
 
--public:no
+-visibility:any
 
 ==>
 
-Program(NegateExpr(PrefixExpr(PublicExpr)))
+Program(NegateExpr(PrefixExpr(VisibilityExpr)))
 
 # Negate symbol prefix
 
@@ -110,22 +102,6 @@ Program(NegateExpr(PrefixExpr(PublicExpr)))
 
 Program(NegateExpr(PrefixExpr(SymExpr)))
 
-# Negate type prefix
-
--type:repo
-
-==>
-
-Program(NegateExpr(PrefixExpr(TypeExpr)))
-
-# Negate regex prefix
-
--regex:test.*
-
-==>
-
-Program(NegateExpr(PrefixExpr(RegexExpr)))
-
 # Negate parentheses
 
 -(test)
@@ -222,14 +198,6 @@ Program(NegateExpr(PrefixExpr(FileExpr)))
 
 Program(NegateExpr(PrefixExpr(RepoExpr)))
 
-# Negate short form branch
-
--b:main
-
-==>
-
-Program(NegateExpr(PrefixExpr(BranchExpr)))
-
 # Negate short form content
 
 -c:console
@@ -238,14 +206,6 @@ Program(NegateExpr(PrefixExpr(BranchExpr)))
 
 Program(NegateExpr(PrefixExpr(ContentExpr)))
 
-# Negate short form type
-
--t:file
-
-==>
-
-Program(NegateExpr(PrefixExpr(TypeExpr)))
-
 # Negate with prefix in quotes
 
 -file:"test file.js"

packages/queryLanguage/test/operators.txt

@@ -110,13 +110,13 @@ repo:project1 or repo:project2
 
 Program(OrExpr(PrefixExpr(RepoExpr),PrefixExpr(RepoExpr)))
 
-# OR with branch prefixes
+# OR with revision prefixes
 
-branch:main or branch:develop
+rev:main or rev:develop
 
 ==>
 
-Program(OrExpr(PrefixExpr(BranchExpr),PrefixExpr(BranchExpr)))
+Program(OrExpr(PrefixExpr(RevisionExpr),PrefixExpr(RevisionExpr)))
 
 # OR with lang prefixes
 

packages/queryLanguage/test/prefixes.txt

@@ -46,21 +46,13 @@ c:console.log
 
 Program(PrefixExpr(ContentExpr))
 
-# Branch prefix
+# Revision prefix
 
-branch:main
+rev:main
 
 ==>
 
-Program(PrefixExpr(BranchExpr))
+Program(PrefixExpr(RevisionExpr))
-
-# Branch prefix short form
-
-b:develop
-
-==>
-
-Program(PrefixExpr(BranchExpr))
 
 # Lang prefix
 
@@ -70,14 +62,6 @@ lang:typescript
 
 Program(PrefixExpr(LangExpr))
 
-# Case prefix
-
-case:yes
-
-==>
-
-Program(PrefixExpr(CaseExpr))
-
 # Archived prefix
 
 archived:no
@@ -94,13 +78,13 @@ fork:yes
 
 Program(PrefixExpr(ForkExpr))
 
-# Public prefix
+# Visibility prefix - public
 
-public:yes
+visibility:public
 
 ==>
 
-Program(PrefixExpr(PublicExpr))
+Program(PrefixExpr(VisibilityExpr))
 
 # Symbol prefix
 
@@ -110,30 +94,6 @@ sym:MyClass
 
 Program(PrefixExpr(SymExpr))
 
-# Type prefix
-
-type:file
-
-==>
-
-Program(PrefixExpr(TypeExpr))
-
-# Type prefix short form
-
-t:repo
-
-==>
-
-Program(PrefixExpr(TypeExpr))
-
-# Regex prefix
-
-regex:test.*
-
-==>
-
-Program(PrefixExpr(RegexExpr))
-
 # RepoSet prefix
 
 reposet:repo1,repo2
@@ -214,21 +174,13 @@ content:hello
 
 Program(PrefixExpr(ContentExpr))
 
-# Branch with slashes
+# Revision with slashes
 
-branch:feature/new-feature
+rev:feature/new-feature
 
 ==>
 
-Program(PrefixExpr(BranchExpr))
+Program(PrefixExpr(RevisionExpr))
-
-# Case values
-
-case:auto
-
-==>
-
-Program(PrefixExpr(CaseExpr))
 
 # RepoSet with multiple repos
 
@@ -246,14 +198,6 @@ sym:package.Class.method
 
 Program(PrefixExpr(SymExpr))
 
-# Type variations
-
-type:filename
-
-==>
-
-Program(PrefixExpr(TypeExpr))
-
 # Lang with various languages
 
 lang:python
@@ -278,21 +222,13 @@ fork:no
 
 Program(PrefixExpr(ForkExpr))
 
-# Public values
+# Visibility prefix - private
 
-public:no
+visibility:private
 
 ==>
 
-Program(PrefixExpr(PublicExpr))
+Program(PrefixExpr(VisibilityExpr))
-
-# Regex with complex pattern
-
-regex:\w+\s*=\s*\d+
-
-==>
-
-Program(PrefixExpr(RegexExpr))
 
 # File with dashes
 

packages/queryLanguage/test/quoted.txt

@@ -102,13 +102,13 @@ content:"console.log"
 
 Program(PrefixExpr(ContentExpr))
 
-# Quoted string in branch prefix
+# Quoted string in revision prefix
 
-branch:"feature/my feature"
+rev:"feature/my feature"
 
 ==>
 
-Program(PrefixExpr(BranchExpr))
+Program(PrefixExpr(RevisionExpr))
 
 # Multiple quoted strings
 
@@ -286,22 +286,6 @@ content:"TODO: fix this"
 
 Program(PrefixExpr(ContentExpr))
 
-# Regex prefix with quoted pattern
-
-regex:"func\\s+\\w+"
-
-==>
-
-Program(PrefixExpr(RegexExpr))
-
-# Case prefix with quoted value
-
-case:"yes"
-
-==>
-
-Program(PrefixExpr(CaseExpr))
-
 # Quoted string with at symbol
 
 "@decorator"
@@ -486,9 +470,9 @@ Program(AndExpr(Term,PrefixExpr(FileExpr)))
 
 Program(Term)
 
-# Quoted branch prefix
+# Quoted revision prefix
 
-"branch:main"
+"rev:main"
 
 ==>
 

packages/web/src/app/api/(server)/stream_search/route.ts

@@ -16,7 +16,7 @@ import { PrismaClient, Repo } from '@sourcebot/db';
 import { createLogger, env } from '@sourcebot/shared';
 import { NextRequest } from 'next/server';
 import * as path from 'path';
-import { parser } from '@sourcebot/query-language';
+import { parser as _parser } from '@sourcebot/query-language';
 import { transformToZoektQuery } from './transformer';
 
 const logger = createLogger('streamSearchApi');
@@ -69,13 +69,38 @@ export const POST = async (request: NextRequest) => {
 
         const { query, matches, contextLines, whole } = parsed.data;
 
+        const isCaseSensitivityEnabled = false;
+        const isRegexEnabled = false;
+
+        const parser = _parser.configure({
+            strict: true,
+        })
+
         const tree = parser.parse(query);
-        const zoektQuery = transformToZoektQuery(tree, query);
+        const zoektQuery = transformToZoektQuery({
+            tree,
+            input: query,
+            isCaseSensitivityEnabled,
+            isRegexEnabled,
+        });
 
         console.log(JSON.stringify(zoektQuery, null, 2));
 
         const searchRequest: SearchRequest = {
-            query: zoektQuery,
+            query: {
+                and: {
+                    children: [
+                        zoektQuery,
+                        // {
+                        //     raw_config: {
+                        //         flags: [
+                        //             'FLAG_NO_FORKS',
+                        //         ]
+                        //     }
+                        // }
+                    ]
+                }
+            },
             // query: {
             //     and: {
             //         // @todo: we should use repo_ids to filter out repositories that the user

packages/web/src/app/api/(server)/stream_search/transformer.ts

@@ -1,43 +1,68 @@
 import { Tree, SyntaxNode } from "@sourcebot/query-language";
 import { Q } from '@/proto/zoekt/webserver/v1/Q';
+import {
+    Program,
+    AndExpr,
+    OrExpr,
+    NegateExpr,
+    ParenExpr,
+    PrefixExpr,
+    Term,
+    FileExpr,
+    RepoExpr,
+    RevisionExpr,
+    ContentExpr,
+    LangExpr,
+    SymExpr,
+    ArchivedExpr,
+    ForkExpr,
+    VisibilityExpr,
+    RepoSetExpr
+} from '@sourcebot/query-language';
 
 /**
  * Transform a Lezer parse tree into a Zoekt gRPC query
  */
-export function transformToZoektQuery(tree: Tree, input: string): Q {
+export const transformToZoektQuery = ({
-    return transformNode(tree.topNode, input);
+    tree,
-}
+    input,
+    isCaseSensitivityEnabled,
+    isRegexEnabled,
+}: {
+    tree: Tree;
+    input: string;
+    isCaseSensitivityEnabled: boolean;
+    isRegexEnabled: boolean;
+}): Q => {
 
-function transformNode(node: SyntaxNode, input: string): Q {
+    const transformNode = (node: SyntaxNode): Q => {
-    const nodeName = node.type.name;
+        switch (node.type.id) {
-
+            case Program: {
-    switch (nodeName) {
-        case "Program": {
                 // Program wraps the actual query - transform its child
                 const child = node.firstChild;
                 if (!child) {
                     // Empty query - match nothing
                     return { const: false, query: "const" };
                 }
-            return transformNode(child, input);
+                return transformNode(child);
             }
-        case "AndExpr":
+            case AndExpr:
                 return {
                     and: {
-                    children: getChildren(node).map(c => transformNode(c, input))
+                        children: getChildren(node).map(c => transformNode(c))
                     },
                     query: "and"
                 }
 
-        case "OrExpr":
+            case OrExpr:
                 return {
                     or: {
-                    children: getChildren(node).map(c => transformNode(c, input))
+                        children: getChildren(node).map(c => transformNode(c))
                     },
                     query: "or"
                 };
 
-        case "NegateExpr": {
+            case NegateExpr: {
                 // Find the child after the negate token
                 const negateChild = node.getChild("PrefixExpr") || node.getChild("ParenExpr");
                 if (!negateChild) {
@@ -45,30 +70,38 @@ function transformNode(node: SyntaxNode, input: string): Q {
                 }
                 return {
                     not: {
-                    child: transformNode(negateChild, input)
+                        child: transformNode(negateChild)
                     },
                     query: "not"
                 };
             }
-        case "ParenExpr": {
+            case ParenExpr: {
                 // Parentheses just group - transform the inner query
                 const innerQuery = node.getChild("query") || node.firstChild;
                 if (!innerQuery) {
                     return { const: false, query: "const" };
                 }
-            return transformNode(innerQuery, input);
+                return transformNode(innerQuery);
             }
-        case "PrefixExpr":
+            case PrefixExpr:
                 // PrefixExpr contains specific prefix types
-            return transformPrefixExpr(node, input);
+                return transformPrefixExpr(node);
 
-        case "Term": {
+            case Term: {
-            // Plain search term - becomes substring search in content
+                const termText = input.substring(node.from, node.to).replace(/^"|"$/g, '');
-            const termText = input.substring(node.from, node.to);
+
-            return {
+                return isRegexEnabled ? {
+                    regexp: {
+                        regexp: termText,
+                        case_sensitive: isCaseSensitivityEnabled,
+                        file_name: false,
+                        content: true
+                    },
+                    query: "regexp"
+                } : {
                     substring: {
-                    pattern: termText.replace(/^"|"$/g, ''), // Remove quotes if present
+                        pattern: termText,
-                    case_sensitive: false,
+                        case_sensitive: isCaseSensitivityEnabled,
                         file_name: false,
                         content: true
                     },
@@ -76,43 +109,43 @@ function transformNode(node: SyntaxNode, input: string): Q {
                 };
             }
             default:
-            console.warn(`Unhandled node type: ${nodeName}`);
+                console.warn(`Unhandled node type: ${node.type.name} (id: ${node.type.id})`);
                 return { const: true, query: "const" };
         }
-}
+    }
 
-function transformPrefixExpr(node: SyntaxNode, input: string): Q {
+    const transformPrefixExpr = (node: SyntaxNode): Q => {
         // Find which specific prefix type this is
         const prefixNode = node.firstChild;
         if (!prefixNode) {
             throw new Error("PrefixExpr has no child");
         }
 
-    const prefixType = prefixNode.type.name;
+        const prefixTypeId = prefixNode.type.id;
 
         // Extract the full text (e.g., "file:test.js") and split on the colon
         const fullText = input.substring(prefixNode.from, prefixNode.to);
         const colonIndex = fullText.indexOf(':');
         if (colonIndex === -1) {
-        throw new Error(`${prefixType} missing colon`);
+            throw new Error(`${prefixNode.type.name} missing colon`);
         }
 
         // Get the value part after the colon and remove quotes if present
         const value = fullText.substring(colonIndex + 1).replace(/^"|"$/g, '');
 
-    switch (prefixType) {
+        switch (prefixTypeId) {
-        case "FileExpr":
+            case FileExpr:
                 return {
                     substring: {
                         pattern: value,
-                    case_sensitive: false,
+                        case_sensitive: isCaseSensitivityEnabled,
                         file_name: true,
                         content: false
                     },
                     query: "substring"
                 };
 
-        case "RepoExpr":
+            case RepoExpr:
                 return {
                     repo: {
                         regexp: value
@@ -120,7 +153,7 @@ function transformPrefixExpr(node: SyntaxNode, input: string): Q {
                     query: "repo"
                 };
 
-        case "BranchExpr":
+            case RevisionExpr:
                 return {
                     branch: {
                         pattern: value,
@@ -129,32 +162,18 @@ function transformPrefixExpr(node: SyntaxNode, input: string): Q {
                     query: "branch"
                 };
 
-        case "ContentExpr":
+            case ContentExpr:
                 return {
                     substring: {
                         pattern: value,
-                    case_sensitive: false,
+                        case_sensitive: isCaseSensitivityEnabled,
                         file_name: false,
                         content: true
                     },
                     query: "substring"
                 };
 
-        case "CaseExpr": {
+            case LangExpr:
-            // case:yes/no wraps the next term with case sensitivity
-            const caseValue = value.toLowerCase();
-            const isCaseSensitive = caseValue === "yes" || caseValue === "true";
-            return {
-                substring: {
-                    pattern: value,
-                    case_sensitive: isCaseSensitive,
-                    file_name: false,
-                    content: true
-                },
-                query: "substring"
-            };
-        }
-        case "LangExpr":
                 return {
                     language: {
                         language: value
@@ -162,14 +181,14 @@ function transformPrefixExpr(node: SyntaxNode, input: string): Q {
                     query: "language"
                 };
 
-        case "SymExpr":
+            case SymExpr:
                 // Symbol search wraps a pattern
                 return {
                     symbol: {
                         expr: {
                             substring: {
                                 pattern: value,
-                            case_sensitive: false,
+                                case_sensitive: isCaseSensitivityEnabled,
                                 file_name: false,
                                 content: true
                             },
@@ -178,28 +197,54 @@ function transformPrefixExpr(node: SyntaxNode, input: string): Q {
                     },
                     query: "symbol"
                 };
-        case "RegexExpr":
+
+            case VisibilityExpr: {
+                const visibilityValue = value.toLowerCase();
+                const flags: ('FLAG_ONLY_PUBLIC' | 'FLAG_ONLY_PRIVATE')[] = [];
+
+                if (visibilityValue === 'public') {
+                    flags.push('FLAG_ONLY_PUBLIC');
+                } else if (visibilityValue === 'private') {
+                    flags.push('FLAG_ONLY_PRIVATE');
+                }
+                // 'any' means no filter
+
                 return {
-                regexp: {
+                    raw_config: {
-                    regexp: value,
+                        flags
-                    case_sensitive: false,
-                    file_name: false,
-                    content: true
                     },
-                query: "regexp"
+                    query: "raw_config"
                 };
+            }
 
             // @todo: handle this
-        case "ArchivedExpr":
+            case ArchivedExpr: {
-        case "ForkExpr":
+                const archivedValue = value.toLowerCase();
-        case "PublicExpr":
+                const flags: ('FLAG_ONLY_ARCHIVED' | 'FLAG_NO_ARCHIVED')[] = [];
+
+                if (archivedValue === 'yes') {
+                    // 'yes' means include archived repositories (default)
+                } else if (archivedValue === 'no') {
+                    flags.push('FLAG_NO_ARCHIVED');
+                } else if (archivedValue === 'only') {
+                    flags.push('FLAG_ONLY_ARCHIVED');
+                }
+
+                return {
+                    raw_config: {
+                        flags
+                    },
+                    query: "raw_config"
+                };
+            }
+            case ForkExpr:
                 // These are repo metadata filters
                 // They need to be handled via repo filters in Zoekt
                 // For now, return a const query (you might need custom handling)
-            console.warn(`${prefixType} not yet implemented`);
+                console.warn(`${prefixNode.type.name} not yet implemented`);
                 return { const: true, query: "const" };
 
-        case "RepoSetExpr": {
+            case RepoSetExpr: {
                 return {
                     repo_set: {
                         set: value.split(',').reduce((acc, s) => {
@@ -211,11 +256,14 @@ function transformPrefixExpr(node: SyntaxNode, input: string): Q {
                 };
             }
             default:
-            throw new Error(`Unknown prefix type: ${prefixType}`);
+                throw new Error(`Unknown prefix type: ${prefixNode.type.name} (id: ${prefixTypeId})`);
         }
+    }
+
+    return transformNode(tree.topNode);
 }
 
-function getChildren(node: SyntaxNode): SyntaxNode[] {
+const getChildren = (node: SyntaxNode): SyntaxNode[] => {
     const children: SyntaxNode[] = [];
     let child = node.firstChild;
     while (child) {