feat: add api to serve files under repositories (#851)

* feat: add api to serve files under repositories

* resolve comments

* resolve comments

* Update repositories.rs

* Update lib.rs

* resolve comment

---------

Co-authored-by: Meng Zhang <[email protected]>

crates/tabby-common/src/config.rs

@@ -50,13 +50,17 @@ impl RepositoryConfig {
             let path = self.git_url.strip_prefix("file://").unwrap();
             path.into()
         } else {
-            repositories_dir().join(filenamify(&self.git_url))
+            repositories_dir().join(self.name())
         }
     }
 
     pub fn is_local_dir(&self) -> bool {
         self.git_url.starts_with("file://")
     }
+
+    pub fn name(&self) -> String {
+        filenamify(&self.git_url)
+    }
 }
 
 #[derive(Serialize, Deserialize)]
@@ -96,4 +100,12 @@ mod tests {
         };
         assert!(!repo.is_local_dir());
     }
+
+    #[test]
+    fn test_repository_config_name() {
+        let repo = RepositoryConfig {
+            git_url: "https://github.com/TabbyML/tabby.git".to_owned(),
+        };
+        assert_eq!(repo.name(), "https_github.com_TabbyML_tabby.git");
+    }
 }

crates/tabby-common/src/lib.rs

@@ -45,7 +45,7 @@ impl SourceFile {
     }
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Tag {
     pub range: Range<usize>,
     pub name_range: Range<usize>,

ee/tabby-webserver/Cargo.toml

@@ -29,6 +29,8 @@ thiserror.workspace = true
 tokio = { workspace = true, features = ["fs"] }
 tokio-rusqlite = "0.4.0"
 tokio-tungstenite = "0.20.1"
+tower = { version = "0.4", features = ["util"] }
+tower-http = { version = "0.4.0", features = ["fs", "trace"] }
 tracing.workspace = true
 unicase = "2.7.0"
 

ee/tabby-webserver/docs/api_spec.md

@@ -0,0 +1,143 @@
+# API Specs
+
+## Repository api: `/repositories`
+
+### Resolve
+
+Get file or directory content from local repositories
+
+**URL:** `/repositories/{name}/resolve/{path}`
+
+**Method:** `GET`
+
+**Request examples:**
+
+- Get directory content
+
+```shell
+curl --request GET \
+  --url http://localhost:8080/repositories/https_github.com_TabbyML_tabby.git/resolve/
+
+curl --request GET \
+  --url http://localhost:9090/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/
+```
+
+- Get file content
+
+```shell
+curl --request GET \
+  --url http://localhost:8080/repositories/https_github.com_TabbyML_tabby.git/resolve/package.json
+
+curl --request GET \
+  --url http://localhost:9090/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/src/api.rs
+```
+
+**Response examples:**
+
+- All directory query will return a list of string, with each string represents an entry under that directory. The `Content-Type` for directory query is `application/vnd.directory+json`.
+
+For `/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/`, the response is:
+
+```json
+{
+  "entries": [
+    "ee/tabby-webserver/src",
+    "ee/tabby-webserver/ui",
+    "ee/tabby-webserver/examples",
+    "ee/tabby-webserver/Cargo.toml",
+    "ee/tabby-webserver/graphql"
+  ]
+}
+```
+
+- The file query will return file content, the `Content-Type` will be guessed from the file extension.
+
+For request `/repositories/https_github.com_TabbyML_tabby.git/resolve/package.json`, the content type is `application/json`, and the response is:
+
+```json
+{
+  "private": true,
+  "workspaces": [
+    "clients/tabby-agent",
+    "clients/vscode",
+    "clients/vim",
+    "clients/intellij"
+  ],
+  "engines": {
+    "node": ">=18"
+  }
+}
+```
+
+For request `/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/src/api.rs`, the content type is `text/x-rust`, and the response is:
+
+```text
+use async_trait::async_trait;
+use juniper::{GraphQLEnum, GraphQLObject};
+use serde::{Deserialize, Serialize};
+use tabby_common::api::{
+    code::{CodeSearch, CodeSearchError, SearchResponse},
+    event::RawEventLogger,
+};
+use thiserror::Error;
+use tokio_tungstenite::connect_async;
+
+use crate::websocket::WebSocketTransport;
+
+#[derive(GraphQLEnum, Serialize, Deserialize, Clone, Debug)]
+pub enum WorkerKind {
+    Completion,
+    Chat,
+}
+
+......omit......
+```
+
+### Meta
+
+Get dataset entry for each indexed file in the repository
+
+**URL:** `/repositories/{name}/meta/{path}`
+
+**Method:** `GET`
+
+**Request example:**
+
+```shell
+curl --request GET \
+  --url http://localhost:9090/repositories/https_github.com_TabbyML_tabby.git/meta/ee/tabby-webserver/src/lib.rs
+```
+
+**Response example:**
+
+The `Content-Type` for successful response is always `application/json`.
+
+```json
+{
+  "git_url": "https://github.com/TabbyML/tabby.git",
+  "filepath": "ee/tabby-webserver/src/lib.rs",
+  "language": "rust",
+  "max_line_length": 88,
+  "avg_line_length": 26.340782,
+  "alphanum_fraction": 0.56416017,
+  "tags": [
+    {
+      "range": {
+        "start": 0,
+        "end": 12
+      },
+      "name_range": {
+        "start": 8,
+        "end": 11
+      },
+      "line_range": {
+        "start": 0,
+        "end": 12
+      },
+      "is_definition": true,
+      "syntax_type_name": "module"
+    },
+    ......omit......
+  ]
+}
+```

ee/tabby-webserver/src/lib.rs

@@ -11,6 +11,7 @@ use tracing::{error, warn};
 use websocket::WebSocketTransport;
 
 mod db;
+mod repositories;
 mod server;
 mod ui;
 mod websocket;
@@ -49,7 +50,8 @@ pub async fn attach_webserver(
         )
         .route("/graphql", routing::get(playground("/graphql", None)))
         .layer(Extension(schema))
-        .route("/hub", routing::get(ws_handler).with_state(ctx));
+        .route("/hub", routing::get(ws_handler).with_state(ctx))
+        .nest("/repositories", repositories::routes());
 
     let ui = ui
         .route("/graphiql", routing::get(graphiql("/graphql", None)))

ee/tabby-webserver/src/repositories.rs

@@ -0,0 +1,56 @@
+mod resolve;
+
+use anyhow::Result;
+use axum::{extract::Path, http::StatusCode, response::Response, routing, Json, Router};
+use tabby_common::path::repositories_dir;
+use tracing::{instrument, warn};
+
+use crate::{
+    repositories,
+    repositories::resolve::{resolve_dir, resolve_file, resolve_meta, Meta, ResolveParams},
+};
+
+pub fn routes() -> Router {
+    Router::new()
+        .route("/:name/resolve/", routing::get(repositories::resolve))
+        .route("/:name/resolve/*path", routing::get(repositories::resolve))
+        .route("/:name/meta/", routing::get(repositories::meta))
+        .route("/:name/meta/*path", routing::get(repositories::meta))
+}
+
+#[instrument(skip(repo))]
+async fn resolve(Path(repo): Path<ResolveParams>) -> Result<Response, StatusCode> {
+    let root = repositories_dir().join(repo.name_str());
+    let full_path = root.join(repo.path_str());
+    let is_dir = tokio::fs::metadata(full_path.clone())
+        .await
+        .map(|m| m.is_dir())
+        .unwrap_or(false);
+
+    if is_dir {
+        return match resolve_dir(root, full_path.clone()).await {
+            Ok(resp) => Ok(resp),
+            Err(err) => {
+                warn!("failed to resolve_dir <{:?}>: {}", full_path, err);
+                Err(StatusCode::INTERNAL_SERVER_ERROR)
+            }
+        };
+    }
+
+    match resolve_file(root, &repo).await {
+        Ok(resp) => Ok(resp),
+        Err(err) => {
+            warn!("failed to resolve_file <{:?}>: {}", full_path, err);
+            Err(StatusCode::INTERNAL_SERVER_ERROR)
+        }
+    }
+}
+
+#[instrument(skip(repo))]
+async fn meta(Path(repo): Path<ResolveParams>) -> Result<Json<Meta>, StatusCode> {
+    let key = repo.dataset_key();
+    if let Some(resp) = resolve_meta(&key) {
+        return Ok(Json(resp));
+    }
+    Err(StatusCode::NOT_FOUND)
+}