diff --git a/CHANGELOG.md b/CHANGELOG.md index 36c37812..c471137b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added fallback to default the Node.JS AWS SDK's `fromNodeProviderChain` when no credentials are provided for a bedrock config. [#513](https://github.com/sourcebot-dev/sourcebot/pull/513) +- Added support for Azure Devops support. [#514](https://github.com/sourcebot-dev/sourcebot/pull/514) ### Fixed - Fixed "At least one project, user, or group must be specified" for GitLab configs with `all` in web configurator. [#512](https://github.com/sourcebot-dev/sourcebot/pull/512) diff --git a/docs/docs.json b/docs/docs.json index 703b0a10..4710e6d0 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -68,6 +68,8 @@ "docs/connections/gitlab", "docs/connections/bitbucket-cloud", "docs/connections/bitbucket-data-center", + "docs/connections/ado-cloud", + "docs/connections/ado-server", "docs/connections/gitea", "docs/connections/gerrit", "docs/connections/generic-git-host", diff --git a/docs/docs/connections/ado-cloud.mdx b/docs/docs/connections/ado-cloud.mdx new file mode 100644 index 00000000..d4e51bd6 --- /dev/null +++ b/docs/docs/connections/ado-cloud.mdx @@ -0,0 +1,141 @@ +--- +title: Linking code from Azure Devops Cloud +sidebarTitle: Azure Devops Cloud +icon: https://www.svgrepo.com/show/448307/azure-devops.svg +--- + +import AzureDevopsSchema from '/snippets/schemas/v3/azuredevops.schema.mdx' + +If you're not familiar with Sourcebot [connections](/docs/connections/overview), please read that overview first. + +## Examples + + + + ```json + { + "type": "azuredevops", + "repos": [ + "organizationName/projectName/repoName", + "organizationName/projectName/repoName2 + ] + } + ``` + + + ```json + { + "type": "azuredevops", + "orgs": [ + "organizationName", + "organizationName2 + ] + } + ``` + + + ```json + { + "type": "azuredevops", + "projects": [ + "organizationName/projectName", + "organizationName/projectName2" + ] + } + ``` + + + ```json + { + "type": "azuredevops", + // Include all repos in my-org... + "orgs": [ + "my-org" + ], + // ...except: + "exclude": { + // repos that are disabled + "disabled": true, + // repos that match these glob patterns + "repos": [ + "reposToExclude*" + ], + // projects that match these glob patterns + "projects": [ + "projectstoExclude*" + ] + // repos less than the defined min OR larger than the defined max + "size": { + // repos that are less than 1MB (in bytes)... + "min": 1048576, + // or repos greater than 100MB (in bytes) + "max": 104857600 + } + } + } + ``` + + + +## Authenticating with Azure Devops Cloud + +Azure Devops Cloud requires you to provide a PAT in order to index your repositories. To learn how to create PAT, check out the [Azure Devops docs](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows). +Sourcebot needs the `Read` access for the `Code` scope in order to find and clone your repos. + +Next, provide the access token via the `token` property, either as an environment variable or a secret: + + + + + 1. Add the `token` property to your connection config: + ```json + { + "type": "azuredevops", + "token": { + // note: this env var can be named anything. It + // doesn't need to be `ADO_TOKEN`. + "env": "ADO_TOKEN" + } + // .. rest of config .. + } + ``` + + 2. Pass this environment variable each time you run Sourcebot: + ```bash + docker run \ + -e ADO_TOKEN= \ + /* additional args */ \ + ghcr.io/sourcebot-dev/sourcebot:latest + ``` + + + + Secrets are only supported when [authentication](/docs/configuration/auth/overview) is enabled. + + 1. Navigate to **Secrets** in settings and create a new secret with your PAT: + + ![](/images/secrets_list.png) + + 2. Add the `token` property to your connection config: + + ```json + { + "type": "azuredevops", + "token": { + "secret": "mysecret" + } + // .. rest of config .. + } + ``` + + + + +## Schema reference + + +[schemas/v3/azuredevops.json](https://github.com/sourcebot-dev/sourcebot/blob/main/schemas/v3/azuredevops.json) + + + + \ No newline at end of file diff --git a/docs/docs/connections/ado-server.mdx b/docs/docs/connections/ado-server.mdx new file mode 100644 index 00000000..b2586056 --- /dev/null +++ b/docs/docs/connections/ado-server.mdx @@ -0,0 +1,154 @@ +--- +title: Linking code from Azure Devops Server +sidebarTitle: Azure Devops Server +icon: https://www.svgrepo.com/show/448307/azure-devops.svg +--- + +import AzureDevopsSchema from '/snippets/schemas/v3/azuredevops.schema.mdx' + +If you're not familiar with Sourcebot [connections](/docs/connections/overview), please read that overview first. + +## Examples + + + + This is required if you're using an older version of ADO Server which has `/tfs` in the repo paths. + ```json + { + "type": "azuredevops", + "useTfsPath": true + "repos": [ + "organizationName/projectName/repoName", + "organizationName/projectName/repoName2 + ] + } + ``` + + + ```json + { + "type": "azuredevops", + "repos": [ + "organizationName/projectName/repoName", + "organizationName/projectName/repoName2 + ] + } + ``` + + + ```json + { + "type": "azuredevops", + "orgs": [ + "collectionName", + "collectionName2" + ] + } + ``` + + + ```json + { + "type": "azuredevops", + "projects": [ + "collectionName/projectName", + "collectionName/projectName2" + ] + } + ``` + + + ```json + { + "type": "azuredevops", + // Include all repos in my-org... + "orgs": [ + "my-org" + ], + // ...except: + "exclude": { + // repos that are disabled + "disabled": true, + // repos that match these glob patterns + "repos": [ + "reposToExclude*" + ], + // projects that match these glob patterns + "projects": [ + "projectstoExclude*" + ] + // repos less than the defined min OR larger than the defined max + "size": { + // repos that are less than 1MB (in bytes)... + "min": 1048576, + // or repos greater than 100MB (in bytes) + "max": 104857600 + } + } + } + ``` + + + +## Authenticating with Azure Devops Server + +Azure Devops Server requires you to provide a PAT in order to index your repositories. To learn how to create PAT, check out the [Azure Devops docs](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows). +Sourcebot needs the `Read` access for the `Code` scope in order to find and clone your repos. + +Next, provide the access token via the `token` property, either as an environment variable or a secret: + + + + + 1. Add the `token` property to your connection config: + ```json + { + "type": "azuredevops", + "token": { + // note: this env var can be named anything. It + // doesn't need to be `ADO_TOKEN`. + "env": "ADO_TOKEN" + } + // .. rest of config .. + } + ``` + + 2. Pass this environment variable each time you run Sourcebot: + ```bash + docker run \ + -e ADO_TOKEN= \ + /* additional args */ \ + ghcr.io/sourcebot-dev/sourcebot:latest + ``` + + + + Secrets are only supported when [authentication](/docs/configuration/auth/overview) is enabled. + + 1. Navigate to **Secrets** in settings and create a new secret with your PAT: + + ![](/images/secrets_list.png) + + 2. Add the `token` property to your connection config: + + ```json + { + "type": "azuredevops", + "token": { + "secret": "mysecret" + } + // .. rest of config .. + } + ``` + + + + +## Schema reference + + +[schemas/v3/azuredevops.json](https://github.com/sourcebot-dev/sourcebot/blob/main/schemas/v3/azuredevops.json) + + + + \ No newline at end of file diff --git a/docs/docs/connections/gitea.mdx b/docs/docs/connections/gitea.mdx index 2318b166..1d439c76 100644 --- a/docs/docs/connections/gitea.mdx +++ b/docs/docs/connections/gitea.mdx @@ -85,7 +85,6 @@ Next, provide the access token via the `token` property, either as an environmen - Environment variables are only supported in a [declarative config](/docs/configuration/declarative-config) and cannot be used in the web UI. 1. Add the `token` property to your connection config: ```json diff --git a/docs/docs/connections/github.mdx b/docs/docs/connections/github.mdx index 0abf13b8..c6e5f5c6 100644 --- a/docs/docs/connections/github.mdx +++ b/docs/docs/connections/github.mdx @@ -132,7 +132,6 @@ Next, provide the access token via the `token` property, either as an environmen - Environment variables are only supported in a [declarative config](/docs/configuration/declarative-config) and cannot be used in the web UI. 1. Add the `token` property to your connection config: ```json diff --git a/docs/docs/connections/gitlab.mdx b/docs/docs/connections/gitlab.mdx index 173c05f8..2680064b 100644 --- a/docs/docs/connections/gitlab.mdx +++ b/docs/docs/connections/gitlab.mdx @@ -120,7 +120,6 @@ Next, provide the PAT via the `token` property, either as an environment variabl - Environment variables are only supported in a [declarative config](/docs/configuration/declarative-config) and cannot be used in the web UI. 1. Add the `token` property to your connection config: ```json diff --git a/docs/images/ado.svg b/docs/images/ado.svg new file mode 100644 index 00000000..e5668e9b --- /dev/null +++ b/docs/images/ado.svg @@ -0,0 +1,3 @@ + + + diff --git a/docs/snippets/bitbucket-app-password.mdx b/docs/snippets/bitbucket-app-password.mdx index 1f52e79c..0a5fdac3 100644 --- a/docs/snippets/bitbucket-app-password.mdx +++ b/docs/snippets/bitbucket-app-password.mdx @@ -1,6 +1,5 @@ - Environment variables are only supported in a [declarative config](/docs/configuration/declarative-config) and cannot be used in the web UI. 1. Add the `token` and `user` (username associated with the app password you created) properties to your connection config: ```json diff --git a/docs/snippets/bitbucket-token.mdx b/docs/snippets/bitbucket-token.mdx index 8b7e1db6..94197e01 100644 --- a/docs/snippets/bitbucket-token.mdx +++ b/docs/snippets/bitbucket-token.mdx @@ -1,6 +1,5 @@ - Environment variables are only supported in a [declarative config](/docs/configuration/declarative-config) and cannot be used in the web UI. 1. Add the `token` property to your connection config: ```json diff --git a/docs/snippets/platform-support.mdx b/docs/snippets/platform-support.mdx index 38266b2d..1687d5db 100644 --- a/docs/snippets/platform-support.mdx +++ b/docs/snippets/platform-support.mdx @@ -3,6 +3,43 @@ + {/* Mintlify has a bug where linking to a file for the logo renders it with a white background, so we have to embed it directly */} + + + + } + /> + + + + } + /> diff --git a/docs/snippets/schemas/v3/azuredevops.schema.mdx b/docs/snippets/schemas/v3/azuredevops.schema.mdx new file mode 100644 index 00000000..a36132ff --- /dev/null +++ b/docs/snippets/schemas/v3/azuredevops.schema.mdx @@ -0,0 +1,206 @@ +{/* THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! */} +```json +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "AzureDevOpsConnectionConfig", + "properties": { + "type": { + "const": "azuredevops", + "description": "Azure DevOps Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + { + "secret": "SECRET_KEY" + } + ], + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://dev.azure.com", + "description": "The URL of the Azure DevOps host. For Azure DevOps Cloud, use https://dev.azure.com. For Azure DevOps Server, use your server URL.", + "examples": [ + "https://dev.azure.com", + "https://azuredevops.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "deploymentType": { + "type": "string", + "enum": [ + "cloud", + "server" + ], + "default": "cloud", + "description": "The type of Azure DevOps deployment" + }, + "useTfsPath": { + "type": "boolean", + "default": false, + "description": "Use legacy TFS path format (/tfs) in API URLs. Required for older TFS installations (TFS 2018 and earlier). When true, API URLs will include /tfs in the path (e.g., https://server/tfs/collection/_apis/...)." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org" + ] + ], + "description": "List of organizations to sync with. For Cloud, this is the organization name. For Server, this is the collection name. All projects and repositories visible to the provided `token` will be synced, unless explicitly defined in the `exclude` property." + }, + "projects": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project", + "my-collection/my-project" + ] + ], + "description": "List of specific projects to sync with. Expected to be formatted as '{orgName}/{projectName}' for Cloud or '{collectionName}/{projectName}' for Server." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project/my-repo" + ] + ], + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{projectName}/{repoName}'." + }, + "exclude": { + "type": "object", + "properties": { + "disabled": { + "type": "boolean", + "default": false, + "description": "Exclude disabled repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of repositories to exclude from syncing. Glob patterns are supported." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of projects to exclude from syncing. Glob patterns are supported." + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their size.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "token" + ], + "additionalProperties": false +} +``` diff --git a/docs/snippets/schemas/v3/connection.schema.mdx b/docs/snippets/schemas/v3/connection.schema.mdx index 631fc17f..3203cadd 100644 --- a/docs/snippets/schemas/v3/connection.schema.mdx +++ b/docs/snippets/schemas/v3/connection.schema.mdx @@ -869,6 +869,209 @@ }, "additionalProperties": false }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "AzureDevOpsConnectionConfig", + "properties": { + "type": { + "const": "azuredevops", + "description": "Azure DevOps Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + { + "secret": "SECRET_KEY" + } + ], + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://dev.azure.com", + "description": "The URL of the Azure DevOps host. For Azure DevOps Cloud, use https://dev.azure.com. For Azure DevOps Server, use your server URL.", + "examples": [ + "https://dev.azure.com", + "https://azuredevops.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "deploymentType": { + "type": "string", + "enum": [ + "cloud", + "server" + ], + "default": "cloud", + "description": "The type of Azure DevOps deployment" + }, + "useTfsPath": { + "type": "boolean", + "default": false, + "description": "Use legacy TFS path format (/tfs) in API URLs. Required for older TFS installations (TFS 2018 and earlier). When true, API URLs will include /tfs in the path (e.g., https://server/tfs/collection/_apis/...)." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org" + ] + ], + "description": "List of organizations to sync with. For Cloud, this is the organization name. For Server, this is the collection name. All projects and repositories visible to the provided `token` will be synced, unless explicitly defined in the `exclude` property." + }, + "projects": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project", + "my-collection/my-project" + ] + ], + "description": "List of specific projects to sync with. Expected to be formatted as '{orgName}/{projectName}' for Cloud or '{collectionName}/{projectName}' for Server." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project/my-repo" + ] + ], + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{projectName}/{repoName}'." + }, + "exclude": { + "type": "object", + "properties": { + "disabled": { + "type": "boolean", + "default": false, + "description": "Exclude disabled repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of repositories to exclude from syncing. Glob patterns are supported." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of projects to exclude from syncing. Glob patterns are supported." + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their size.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "token" + ], + "additionalProperties": false + }, { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 1a513eba..8f3a4e6f 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -1132,6 +1132,209 @@ }, "additionalProperties": false }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "AzureDevOpsConnectionConfig", + "properties": { + "type": { + "const": "azuredevops", + "description": "Azure DevOps Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + { + "secret": "SECRET_KEY" + } + ], + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://dev.azure.com", + "description": "The URL of the Azure DevOps host. For Azure DevOps Cloud, use https://dev.azure.com. For Azure DevOps Server, use your server URL.", + "examples": [ + "https://dev.azure.com", + "https://azuredevops.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "deploymentType": { + "type": "string", + "enum": [ + "cloud", + "server" + ], + "default": "cloud", + "description": "The type of Azure DevOps deployment" + }, + "useTfsPath": { + "type": "boolean", + "default": false, + "description": "Use legacy TFS path format (/tfs) in API URLs. Required for older TFS installations (TFS 2018 and earlier). When true, API URLs will include /tfs in the path (e.g., https://server/tfs/collection/_apis/...)." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org" + ] + ], + "description": "List of organizations to sync with. For Cloud, this is the organization name. For Server, this is the collection name. All projects and repositories visible to the provided `token` will be synced, unless explicitly defined in the `exclude` property." + }, + "projects": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project", + "my-collection/my-project" + ] + ], + "description": "List of specific projects to sync with. Expected to be formatted as '{orgName}/{projectName}' for Cloud or '{collectionName}/{projectName}' for Server." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project/my-repo" + ] + ], + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{projectName}/{repoName}'." + }, + "exclude": { + "type": "object", + "properties": { + "disabled": { + "type": "boolean", + "default": false, + "description": "Exclude disabled repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of repositories to exclude from syncing. Glob patterns are supported." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of projects to exclude from syncing. Glob patterns are supported." + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their size.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "token" + ], + "additionalProperties": false + }, { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", diff --git a/packages/backend/package.json b/packages/backend/package.json index 17527800..dade7893 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -37,6 +37,7 @@ "@t3-oss/env-core": "^0.12.0", "@types/express": "^5.0.0", "argparse": "^2.0.1", + "azure-devops-node-api": "^15.1.1", "bullmq": "^5.34.10", "cross-fetch": "^4.0.0", "dotenv": "^16.4.5", diff --git a/packages/backend/src/azuredevops.ts b/packages/backend/src/azuredevops.ts new file mode 100644 index 00000000..a06b9c09 --- /dev/null +++ b/packages/backend/src/azuredevops.ts @@ -0,0 +1,348 @@ +import { AzureDevOpsConnectionConfig } from "@sourcebot/schemas/v3/azuredevops.type"; +import { createLogger } from "@sourcebot/logger"; +import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js"; +import micromatch from "micromatch"; +import { PrismaClient } from "@sourcebot/db"; +import { BackendException, BackendError } from "@sourcebot/error"; +import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; +import * as Sentry from "@sentry/node"; +import * as azdev from "azure-devops-node-api"; +import { GitRepository } from "azure-devops-node-api/interfaces/GitInterfaces.js"; + +const logger = createLogger('azuredevops'); +const AZUREDEVOPS_CLOUD_HOSTNAME = "dev.azure.com"; + + +function buildOrgUrl(baseUrl: string, org: string, useTfsPath: boolean): string { + const tfsSegment = useTfsPath ? '/tfs' : ''; + return `${baseUrl}${tfsSegment}/${org}`; +} + +function createAzureDevOpsConnection( + orgUrl: string, + token: string, +): azdev.WebApi { + const authHandler = azdev.getPersonalAccessTokenHandler(token); + return new azdev.WebApi(orgUrl, authHandler); +} + +export const getAzureDevOpsReposFromConfig = async ( + config: AzureDevOpsConnectionConfig, + orgId: number, + db: PrismaClient +) => { + const baseUrl = config.url || `https://${AZUREDEVOPS_CLOUD_HOSTNAME}`; + + const token = config.token ? + await getTokenFromConfig(config.token, orgId, db, logger) : + undefined; + + if (!token) { + const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, { + message: 'Azure DevOps requires a Personal Access Token', + }); + Sentry.captureException(e); + throw e; + } + + const useTfsPath = config.useTfsPath || false; + let allRepos: GitRepository[] = []; + let notFound: { + users: string[], + orgs: string[], + repos: string[], + } = { + users: [], + orgs: [], + repos: [], + }; + + if (config.orgs) { + const { validRepos, notFoundOrgs } = await getReposForOrganizations( + config.orgs, + baseUrl, + token, + useTfsPath + ); + allRepos = allRepos.concat(validRepos); + notFound.orgs = notFoundOrgs; + } + + if (config.projects) { + const { validRepos, notFoundProjects } = await getReposForProjects( + config.projects, + baseUrl, + token, + useTfsPath + ); + allRepos = allRepos.concat(validRepos); + notFound.repos = notFound.repos.concat(notFoundProjects); + } + + if (config.repos) { + const { validRepos, notFoundRepos } = await getRepos( + config.repos, + baseUrl, + token, + useTfsPath + ); + allRepos = allRepos.concat(validRepos); + notFound.repos = notFound.repos.concat(notFoundRepos); + } + + let repos = allRepos + .filter((repo) => { + const isExcluded = shouldExcludeRepo({ + repo, + exclude: config.exclude, + }); + + return !isExcluded; + }); + + logger.debug(`Found ${repos.length} total repositories.`); + + return { + validRepos: repos, + notFound, + }; +}; + +export const shouldExcludeRepo = ({ + repo, + exclude +}: { + repo: GitRepository, + exclude?: AzureDevOpsConnectionConfig['exclude'] +}) => { + let reason = ''; + const repoName = `${repo.project!.name}/${repo.name}`; + + const shouldExclude = (() => { + if (!repo.remoteUrl) { + reason = 'remoteUrl is undefined'; + return true; + } + + if (!!exclude?.disabled && repo.isDisabled) { + reason = `\`exclude.disabled\` is true`; + return true; + } + + if (exclude?.repos) { + if (micromatch.isMatch(repoName, exclude.repos)) { + reason = `\`exclude.repos\` contains ${repoName}`; + return true; + } + } + + if (exclude?.projects) { + if (micromatch.isMatch(repo.project!.name!, exclude.projects)) { + reason = `\`exclude.projects\` contains ${repo.project!.name}`; + return true; + } + } + + const repoSizeInBytes = repo.size || 0; + if (exclude?.size && repoSizeInBytes) { + const min = exclude.size.min; + const max = exclude.size.max; + + if (min && repoSizeInBytes < min) { + reason = `repo is less than \`exclude.size.min\`=${min} bytes.`; + return true; + } + + if (max && repoSizeInBytes > max) { + reason = `repo is greater than \`exclude.size.max\`=${max} bytes.`; + return true; + } + } + + return false; + })(); + + if (shouldExclude) { + logger.debug(`Excluding repo ${repoName}. Reason: ${reason}`); + return true; + } + + return false; +}; + +async function getReposForOrganizations( + organizations: string[], + baseUrl: string, + token: string, + useTfsPath: boolean +) { + const results = await Promise.allSettled(organizations.map(async (org) => { + try { + logger.debug(`Fetching repositories for organization ${org}...`); + + const { durationMs, data } = await measure(async () => { + const fetchFn = async () => { + const orgUrl = buildOrgUrl(baseUrl, org, useTfsPath); + const connection = createAzureDevOpsConnection(orgUrl, token); // useTfsPath already handled in orgUrl + + const coreApi = await connection.getCoreApi(); + const gitApi = await connection.getGitApi(); + + const projects = await coreApi.getProjects(); + const allRepos: GitRepository[] = []; + for (const project of projects) { + if (!project.id) { + logger.warn(`Encountered project in org ${org} with no id: ${project.name}`); + continue; + } + + try { + const repos = await gitApi.getRepositories(project.id); + allRepos.push(...repos); + } catch (error) { + logger.warn(`Failed to fetch repositories for project ${project.name}: ${error}`); + } + } + + return allRepos; + }; + + return fetchWithRetry(fetchFn, `organization ${org}`, logger); + }); + + logger.debug(`Found ${data.length} repositories in organization ${org} in ${durationMs}ms.`); + return { + type: 'valid' as const, + data + }; + } catch (error) { + Sentry.captureException(error); + logger.error(`Failed to fetch repositories for organization ${org}.`, error); + + // Check if it's a 404-like error (organization not found) + if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { + logger.error(`Organization ${org} not found or no access`); + return { + type: 'notFound' as const, + value: org + }; + } + throw error; + } + })); + + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults(results); + + return { + validRepos, + notFoundOrgs, + }; +} + +async function getReposForProjects( + projects: string[], + baseUrl: string, + token: string, + useTfsPath: boolean +) { + const results = await Promise.allSettled(projects.map(async (project) => { + try { + const [org, projectName] = project.split('/'); + logger.debug(`Fetching repositories for project ${project}...`); + + const { durationMs, data } = await measure(async () => { + const fetchFn = async () => { + const orgUrl = buildOrgUrl(baseUrl, org, useTfsPath); + const connection = createAzureDevOpsConnection(orgUrl, token); + const gitApi = await connection.getGitApi(); + + const repos = await gitApi.getRepositories(projectName); + return repos; + }; + + return fetchWithRetry(fetchFn, `project ${project}`, logger); + }); + + logger.debug(`Found ${data.length} repositories in project ${project} in ${durationMs}ms.`); + return { + type: 'valid' as const, + data + }; + } catch (error) { + Sentry.captureException(error); + logger.error(`Failed to fetch repositories for project ${project}.`, error); + + if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { + logger.error(`Project ${project} not found or no access`); + return { + type: 'notFound' as const, + value: project + }; + } + throw error; + } + })); + + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults(results); + + return { + validRepos, + notFoundProjects, + }; +} + +async function getRepos( + repoList: string[], + baseUrl: string, + token: string, + useTfsPath: boolean +) { + const results = await Promise.allSettled(repoList.map(async (repo) => { + try { + const [org, projectName, repoName] = repo.split('/'); + logger.info(`Fetching repository info for ${repo}...`); + + const { durationMs, data: result } = await measure(async () => { + const fetchFn = async () => { + const orgUrl = buildOrgUrl(baseUrl, org, useTfsPath); + const connection = createAzureDevOpsConnection(orgUrl, token); + const gitApi = await connection.getGitApi(); + + const repo = await gitApi.getRepository(repoName, projectName); + return repo; + }; + + return fetchWithRetry(fetchFn, repo, logger); + }); + + logger.info(`Found info for repository ${repo} in ${durationMs}ms`); + return { + type: 'valid' as const, + data: [result] + }; + + } catch (error) { + Sentry.captureException(error); + logger.error(`Failed to fetch repository ${repo}.`, error); + + if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { + logger.error(`Repository ${repo} not found or no access`); + return { + type: 'notFound' as const, + value: repo + }; + } + throw error; + } + })); + + throwIfAnyFailed(results); + const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults(results); + + return { + validRepos, + notFoundRepos, + }; +} \ No newline at end of file diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index f025bdf7..5cf119b6 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -4,7 +4,7 @@ import { Settings } from "./types.js"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { createLogger } from "@sourcebot/logger"; import { Redis } from 'ioredis'; -import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig, compileBitbucketConfig, compileGenericGitHostConfig } from "./repoCompileUtils.js"; +import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig, compileGerritConfig, compileBitbucketConfig, compileAzureDevOpsConfig, compileGenericGitHostConfig } from "./repoCompileUtils.js"; import { BackendError, BackendException } from "@sourcebot/error"; import { captureEvent } from "./posthog.js"; import { env } from "./env.js"; @@ -177,6 +177,9 @@ export class ConnectionManager implements IConnectionManager { case 'bitbucket': { return await compileBitbucketConfig(config, job.data.connectionId, orgId, this.db); } + case 'azuredevops': { + return await compileAzureDevOpsConfig(config, job.data.connectionId, orgId, this.db, abortController); + } case 'git': { return await compileGenericGitHostConfig(config, job.data.connectionId, orgId); } diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index ad26ed58..ab162ffe 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -4,13 +4,15 @@ import { getGitLabReposFromConfig } from "./gitlab.js"; import { getGiteaReposFromConfig } from "./gitea.js"; import { getGerritReposFromConfig } from "./gerrit.js"; import { BitbucketRepository, getBitbucketReposFromConfig } from "./bitbucket.js"; +import { getAzureDevOpsReposFromConfig } from "./azuredevops.js"; import { SchemaRestRepository as BitbucketServerRepository } from "@coderabbitai/bitbucket/server/openapi"; import { SchemaRepository as BitbucketCloudRepository } from "@coderabbitai/bitbucket/cloud/openapi"; import { Prisma, PrismaClient } from '@sourcebot/db'; import { WithRequired } from "./types.js" import { marshalBool } from "./utils.js"; import { createLogger } from '@sourcebot/logger'; -import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; +import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; +import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js"; import { RepoMetadata } from './types.js'; import path from 'path'; import { glob } from 'glob'; @@ -544,6 +546,87 @@ export const compileGenericGitHostConfig_file = async ( } } +export const compileAzureDevOpsConfig = async ( + config: AzureDevOpsConnectionConfig, + connectionId: number, + orgId: number, + db: PrismaClient, + abortController: AbortController) => { + + const azureDevOpsReposResult = await getAzureDevOpsReposFromConfig(config, orgId, db); + const azureDevOpsRepos = azureDevOpsReposResult.validRepos; + const notFound = azureDevOpsReposResult.notFound; + + const hostUrl = config.url ?? 'https://dev.azure.com'; + const repoNameRoot = new URL(hostUrl) + .toString() + .replace(/^https?:\/\//, ''); + + const repos = azureDevOpsRepos.map((repo) => { + if (!repo.project) { + throw new Error(`No project found for repository ${repo.name}`); + } + + const repoDisplayName = `${repo.project.name}/${repo.name}`; + const repoName = path.join(repoNameRoot, repoDisplayName); + + if (!repo.remoteUrl) { + throw new Error(`No remoteUrl found for repository ${repoDisplayName}`); + } + if (!repo.id) { + throw new Error(`No id found for repository ${repoDisplayName}`); + } + + // Construct web URL for the repository + const webUrl = repo.webUrl || `${hostUrl}/${repo.project.name}/_git/${repo.name}`; + + logger.debug(`Found Azure DevOps repo ${repoDisplayName} with webUrl: ${webUrl}`); + + const record: RepoData = { + external_id: repo.id.toString(), + external_codeHostType: 'azuredevops', + external_codeHostUrl: hostUrl, + cloneUrl: webUrl, + webUrl: webUrl, + name: repoName, + displayName: repoDisplayName, + imageUrl: null, + isFork: !!repo.isFork, + isArchived: false, + org: { + connect: { + id: orgId, + }, + }, + connections: { + create: { + connectionId: connectionId, + } + }, + metadata: { + gitConfig: { + 'zoekt.web-url-type': 'azuredevops', + 'zoekt.web-url': webUrl, + 'zoekt.name': repoName, + 'zoekt.archived': marshalBool(false), + 'zoekt.fork': marshalBool(!!repo.isFork), + 'zoekt.public': marshalBool(repo.project.visibility === ProjectVisibility.Public), + 'zoekt.display-name': repoDisplayName, + }, + branches: config.revisions?.branches ?? undefined, + tags: config.revisions?.tags ?? undefined, + } satisfies RepoMetadata, + }; + + return record; + }) + + return { + repoData: repos, + notFound, + }; +} + export const compileGenericGitHostConfig_url = async ( config: GenericGitHostConnectionConfig, orgId: number, diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts index d8e091ec..0a58c2fe 100644 --- a/packages/backend/src/repoManager.ts +++ b/packages/backend/src/repoManager.ts @@ -2,7 +2,7 @@ import { Job, Queue, Worker } from 'bullmq'; import { Redis } from 'ioredis'; import { createLogger } from "@sourcebot/logger"; import { Connection, PrismaClient, Repo, RepoToConnection, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db"; -import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; +import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { AppContext, Settings, repoMetadataSchema } from "./types.js"; import { getRepoPath, getTokenFromConfig, measure, getShardPrefix } from "./utils.js"; import { cloneRepository, fetchRepository, unsetGitConfig, upsertGitConfig } from "./git.js"; @@ -186,9 +186,7 @@ export class RepoManager implements IRepoManager { password: token, } } - } - - else if (connection.connectionType === 'gitlab') { + } else if (connection.connectionType === 'gitlab') { const config = connection.config as unknown as GitlabConnectionConfig; if (config.token) { const token = await getTokenFromConfig(config.token, connection.orgId, db, logger); @@ -197,9 +195,7 @@ export class RepoManager implements IRepoManager { password: token, } } - } - - else if (connection.connectionType === 'gitea') { + } else if (connection.connectionType === 'gitea') { const config = connection.config as unknown as GiteaConnectionConfig; if (config.token) { const token = await getTokenFromConfig(config.token, connection.orgId, db, logger); @@ -207,9 +203,7 @@ export class RepoManager implements IRepoManager { password: token, } } - } - - else if (connection.connectionType === 'bitbucket') { + } else if (connection.connectionType === 'bitbucket') { const config = connection.config as unknown as BitbucketConnectionConfig; if (config.token) { const token = await getTokenFromConfig(config.token, connection.orgId, db, logger); @@ -219,6 +213,14 @@ export class RepoManager implements IRepoManager { password: token, } } + } else if (connection.connectionType === 'azuredevops') { + const config = connection.config as unknown as AzureDevOpsConnectionConfig; + if (config.token) { + const token = await getTokenFromConfig(config.token, connection.orgId, db, logger); + return { + password: token, + } + } } } diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 20454ddf..4b75430c 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -42,10 +42,11 @@ enum ChatVisibility { model Repo { id Int @id @default(autoincrement()) - name String - displayName String? + name String // Full repo name, including the vcs hostname (ex. github.com/sourcebot-dev/sourcebot) + displayName String? // Display name of the repo for UI (ex. sourcebot-dev/sourcebot) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + /// When the repo was last indexed successfully. indexedAt DateTime? isFork Boolean diff --git a/packages/schemas/src/v3/azuredevops.schema.ts b/packages/schemas/src/v3/azuredevops.schema.ts new file mode 100644 index 00000000..02bc989d --- /dev/null +++ b/packages/schemas/src/v3/azuredevops.schema.ts @@ -0,0 +1,205 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "AzureDevOpsConnectionConfig", + "properties": { + "type": { + "const": "azuredevops", + "description": "Azure DevOps Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + { + "secret": "SECRET_KEY" + } + ], + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://dev.azure.com", + "description": "The URL of the Azure DevOps host. For Azure DevOps Cloud, use https://dev.azure.com. For Azure DevOps Server, use your server URL.", + "examples": [ + "https://dev.azure.com", + "https://azuredevops.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "deploymentType": { + "type": "string", + "enum": [ + "cloud", + "server" + ], + "default": "cloud", + "description": "The type of Azure DevOps deployment" + }, + "useTfsPath": { + "type": "boolean", + "default": false, + "description": "Use legacy TFS path format (/tfs) in API URLs. Required for older TFS installations (TFS 2018 and earlier). When true, API URLs will include /tfs in the path (e.g., https://server/tfs/collection/_apis/...)." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org" + ] + ], + "description": "List of organizations to sync with. For Cloud, this is the organization name. For Server, this is the collection name. All projects and repositories visible to the provided `token` will be synced, unless explicitly defined in the `exclude` property." + }, + "projects": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project", + "my-collection/my-project" + ] + ], + "description": "List of specific projects to sync with. Expected to be formatted as '{orgName}/{projectName}' for Cloud or '{collectionName}/{projectName}' for Server." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project/my-repo" + ] + ], + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{projectName}/{repoName}'." + }, + "exclude": { + "type": "object", + "properties": { + "disabled": { + "type": "boolean", + "default": false, + "description": "Exclude disabled repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of repositories to exclude from syncing. Glob patterns are supported." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of projects to exclude from syncing. Glob patterns are supported." + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their size.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "token" + ], + "additionalProperties": false +} as const; +export { schema as azuredevopsSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v3/azuredevops.type.ts b/packages/schemas/src/v3/azuredevops.type.ts new file mode 100644 index 00000000..eec963bc --- /dev/null +++ b/packages/schemas/src/v3/azuredevops.type.ts @@ -0,0 +1,89 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +export interface AzureDevOpsConnectionConfig { + /** + * Azure DevOps Configuration + */ + type: "azuredevops"; + /** + * A Personal Access Token (PAT). + */ + token: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; + /** + * The URL of the Azure DevOps host. For Azure DevOps Cloud, use https://dev.azure.com. For Azure DevOps Server, use your server URL. + */ + url?: string; + /** + * The type of Azure DevOps deployment + */ + deploymentType?: "cloud" | "server"; + /** + * Use legacy TFS path format (/tfs) in API URLs. Required for older TFS installations (TFS 2018 and earlier). When true, API URLs will include /tfs in the path (e.g., https://server/tfs/collection/_apis/...). + */ + useTfsPath?: boolean; + /** + * List of organizations to sync with. For Cloud, this is the organization name. For Server, this is the collection name. All projects and repositories visible to the provided `token` will be synced, unless explicitly defined in the `exclude` property. + */ + orgs?: string[]; + /** + * List of specific projects to sync with. Expected to be formatted as '{orgName}/{projectName}' for Cloud or '{collectionName}/{projectName}' for Server. + */ + projects?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{projectName}/{repoName}'. + */ + repos?: string[]; + exclude?: { + /** + * Exclude disabled repositories from syncing. + */ + disabled?: boolean; + /** + * List of repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + /** + * List of projects to exclude from syncing. Glob patterns are supported. + */ + projects?: string[]; + /** + * Exclude repositories based on their size. + */ + size?: { + /** + * Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing. + */ + min?: number; + /** + * Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing. + */ + max?: number; + }; + }; + revisions?: GitRevisions; +} +/** + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. + */ +export interface GitRevisions { + /** + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored. + */ + branches?: string[]; + /** + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored. + */ + tags?: string[]; +} diff --git a/packages/schemas/src/v3/connection.schema.ts b/packages/schemas/src/v3/connection.schema.ts index 63357752..cb867694 100644 --- a/packages/schemas/src/v3/connection.schema.ts +++ b/packages/schemas/src/v3/connection.schema.ts @@ -868,6 +868,209 @@ const schema = { }, "additionalProperties": false }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "AzureDevOpsConnectionConfig", + "properties": { + "type": { + "const": "azuredevops", + "description": "Azure DevOps Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + { + "secret": "SECRET_KEY" + } + ], + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://dev.azure.com", + "description": "The URL of the Azure DevOps host. For Azure DevOps Cloud, use https://dev.azure.com. For Azure DevOps Server, use your server URL.", + "examples": [ + "https://dev.azure.com", + "https://azuredevops.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "deploymentType": { + "type": "string", + "enum": [ + "cloud", + "server" + ], + "default": "cloud", + "description": "The type of Azure DevOps deployment" + }, + "useTfsPath": { + "type": "boolean", + "default": false, + "description": "Use legacy TFS path format (/tfs) in API URLs. Required for older TFS installations (TFS 2018 and earlier). When true, API URLs will include /tfs in the path (e.g., https://server/tfs/collection/_apis/...)." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org" + ] + ], + "description": "List of organizations to sync with. For Cloud, this is the organization name. For Server, this is the collection name. All projects and repositories visible to the provided `token` will be synced, unless explicitly defined in the `exclude` property." + }, + "projects": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project", + "my-collection/my-project" + ] + ], + "description": "List of specific projects to sync with. Expected to be formatted as '{orgName}/{projectName}' for Cloud or '{collectionName}/{projectName}' for Server." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project/my-repo" + ] + ], + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{projectName}/{repoName}'." + }, + "exclude": { + "type": "object", + "properties": { + "disabled": { + "type": "boolean", + "default": false, + "description": "Exclude disabled repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of repositories to exclude from syncing. Glob patterns are supported." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of projects to exclude from syncing. Glob patterns are supported." + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their size.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "token" + ], + "additionalProperties": false + }, { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", diff --git a/packages/schemas/src/v3/connection.type.ts b/packages/schemas/src/v3/connection.type.ts index cba5800e..7922ef20 100644 --- a/packages/schemas/src/v3/connection.type.ts +++ b/packages/schemas/src/v3/connection.type.ts @@ -6,6 +6,7 @@ export type ConnectionConfig = | GiteaConnectionConfig | GerritConnectionConfig | BitbucketConnectionConfig + | AzureDevOpsConnectionConfig | GenericGitHostConnectionConfig; export interface GithubConnectionConfig { @@ -311,6 +312,80 @@ export interface BitbucketConnectionConfig { }; revisions?: GitRevisions; } +export interface AzureDevOpsConnectionConfig { + /** + * Azure DevOps Configuration + */ + type: "azuredevops"; + /** + * A Personal Access Token (PAT). + */ + token: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; + /** + * The URL of the Azure DevOps host. For Azure DevOps Cloud, use https://dev.azure.com. For Azure DevOps Server, use your server URL. + */ + url?: string; + /** + * The type of Azure DevOps deployment + */ + deploymentType?: "cloud" | "server"; + /** + * Use legacy TFS path format (/tfs) in API URLs. Required for older TFS installations (TFS 2018 and earlier). When true, API URLs will include /tfs in the path (e.g., https://server/tfs/collection/_apis/...). + */ + useTfsPath?: boolean; + /** + * List of organizations to sync with. For Cloud, this is the organization name. For Server, this is the collection name. All projects and repositories visible to the provided `token` will be synced, unless explicitly defined in the `exclude` property. + */ + orgs?: string[]; + /** + * List of specific projects to sync with. Expected to be formatted as '{orgName}/{projectName}' for Cloud or '{collectionName}/{projectName}' for Server. + */ + projects?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{projectName}/{repoName}'. + */ + repos?: string[]; + exclude?: { + /** + * Exclude disabled repositories from syncing. + */ + disabled?: boolean; + /** + * List of repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + /** + * List of projects to exclude from syncing. Glob patterns are supported. + */ + projects?: string[]; + /** + * Exclude repositories based on their size. + */ + size?: { + /** + * Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing. + */ + min?: number; + /** + * Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing. + */ + max?: number; + }; + }; + revisions?: GitRevisions; +} export interface GenericGitHostConnectionConfig { /** * Generic Git host configuration diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index eb97e78d..02b9f361 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -1131,6 +1131,209 @@ const schema = { }, "additionalProperties": false }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "AzureDevOpsConnectionConfig", + "properties": { + "type": { + "const": "azuredevops", + "description": "Azure DevOps Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + { + "secret": "SECRET_KEY" + } + ], + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://dev.azure.com", + "description": "The URL of the Azure DevOps host. For Azure DevOps Cloud, use https://dev.azure.com. For Azure DevOps Server, use your server URL.", + "examples": [ + "https://dev.azure.com", + "https://azuredevops.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "deploymentType": { + "type": "string", + "enum": [ + "cloud", + "server" + ], + "default": "cloud", + "description": "The type of Azure DevOps deployment" + }, + "useTfsPath": { + "type": "boolean", + "default": false, + "description": "Use legacy TFS path format (/tfs) in API URLs. Required for older TFS installations (TFS 2018 and earlier). When true, API URLs will include /tfs in the path (e.g., https://server/tfs/collection/_apis/...)." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org" + ] + ], + "description": "List of organizations to sync with. For Cloud, this is the organization name. For Server, this is the collection name. All projects and repositories visible to the provided `token` will be synced, unless explicitly defined in the `exclude` property." + }, + "projects": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project", + "my-collection/my-project" + ] + ], + "description": "List of specific projects to sync with. Expected to be formatted as '{orgName}/{projectName}' for Cloud or '{collectionName}/{projectName}' for Server." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org/my-project/my-repo" + ] + ], + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{projectName}/{repoName}'." + }, + "exclude": { + "type": "object", + "properties": { + "disabled": { + "type": "boolean", + "default": false, + "description": "Exclude disabled repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of repositories to exclude from syncing. Glob patterns are supported." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of projects to exclude from syncing. Glob patterns are supported." + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their size.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "token" + ], + "additionalProperties": false + }, { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 8f346bfa..68417ba7 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -10,6 +10,7 @@ export type ConnectionConfig = | GiteaConnectionConfig | GerritConnectionConfig | BitbucketConnectionConfig + | AzureDevOpsConnectionConfig | GenericGitHostConnectionConfig; export type LanguageModel = | AmazonBedrockLanguageModel @@ -436,6 +437,80 @@ export interface BitbucketConnectionConfig { }; revisions?: GitRevisions; } +export interface AzureDevOpsConnectionConfig { + /** + * Azure DevOps Configuration + */ + type: "azuredevops"; + /** + * A Personal Access Token (PAT). + */ + token: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; + /** + * The URL of the Azure DevOps host. For Azure DevOps Cloud, use https://dev.azure.com. For Azure DevOps Server, use your server URL. + */ + url?: string; + /** + * The type of Azure DevOps deployment + */ + deploymentType?: "cloud" | "server"; + /** + * Use legacy TFS path format (/tfs) in API URLs. Required for older TFS installations (TFS 2018 and earlier). When true, API URLs will include /tfs in the path (e.g., https://server/tfs/collection/_apis/...). + */ + useTfsPath?: boolean; + /** + * List of organizations to sync with. For Cloud, this is the organization name. For Server, this is the collection name. All projects and repositories visible to the provided `token` will be synced, unless explicitly defined in the `exclude` property. + */ + orgs?: string[]; + /** + * List of specific projects to sync with. Expected to be formatted as '{orgName}/{projectName}' for Cloud or '{collectionName}/{projectName}' for Server. + */ + projects?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{projectName}/{repoName}'. + */ + repos?: string[]; + exclude?: { + /** + * Exclude disabled repositories from syncing. + */ + disabled?: boolean; + /** + * List of repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + /** + * List of projects to exclude from syncing. Glob patterns are supported. + */ + projects?: string[]; + /** + * Exclude repositories based on their size. + */ + size?: { + /** + * Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing. + */ + min?: number; + /** + * Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing. + */ + max?: number; + }; + }; + revisions?: GitRevisions; +} export interface GenericGitHostConnectionConfig { /** * Generic Git host configuration diff --git a/packages/web/public/azuredevops.svg b/packages/web/public/azuredevops.svg new file mode 100644 index 00000000..3d4a462f --- /dev/null +++ b/packages/web/public/azuredevops.svg @@ -0,0 +1 @@ +Icon-devops-261 \ No newline at end of file diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 80094989..25e62a7f 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -14,6 +14,7 @@ import { gerritSchema } from "@sourcebot/schemas/v3/gerrit.schema"; import { giteaSchema } from "@sourcebot/schemas/v3/gitea.schema"; import { githubSchema } from "@sourcebot/schemas/v3/github.schema"; import { gitlabSchema } from "@sourcebot/schemas/v3/gitlab.schema"; +import { azuredevopsSchema } from "@sourcebot/schemas/v3/azuredevops.schema"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; @@ -2187,6 +2188,8 @@ const parseConnectionConfig = (config: string) => { return gerritSchema; case 'bitbucket': return bitbucketSchema; + case 'azuredevops': + return azuredevopsSchema; case 'git': return genericGitHostSchema; } @@ -2221,7 +2224,8 @@ const parseConnectionConfig = (config: string) => { switch (connectionType) { case "gitea": case "github": - case "bitbucket": { + case "bitbucket": + case "azuredevops": { return { numRepos: parsedConfig.repos?.length, hasToken: !!parsedConfig.token, diff --git a/packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx b/packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx index 3330514f..317e13c3 100644 --- a/packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx +++ b/packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx @@ -34,6 +34,11 @@ export const CodePreviewPanel = async ({ path, repoName, revisionName, domain }: webUrl: repoInfoResponse.webUrl, }); + // @todo: this is a hack to support linking to files for ADO. ADO doesn't support web urls with HEAD so we replace it with main. THis + // will break if the default branch is not main. + const fileWebUrl = repoInfoResponse.codeHostType === "azuredevops" && fileSourceResponse.webUrl ? + fileSourceResponse.webUrl.replace("version=GBHEAD", "version=GBmain") : fileSourceResponse.webUrl; + return ( <>
@@ -47,9 +52,11 @@ export const CodePreviewPanel = async ({ path, repoName, revisionName, domain }: }} branchDisplayName={revisionName} /> - {(fileSourceResponse.webUrl && codeHostInfo) && ( + + {(fileWebUrl && codeHostInfo) && ( +