From 9bb15cb72a1ab632cc93216cab2de0db0f8aace3 Mon Sep 17 00:00:00 2001 From: darknight Date: Mon, 20 Nov 2023 21:23:41 +0800 Subject: [PATCH 1/6] feat: add api to serve files under repositories --- Cargo.lock | 8 ++ crates/tabby-common/src/config.rs | 14 +- crates/tabby-common/src/lib.rs | 7 +- ee/tabby-webserver/Cargo.toml | 2 + ee/tabby-webserver/docs/api_spec.md | 143 ++++++++++++++++++++ ee/tabby-webserver/src/lib.rs | 16 ++- ee/tabby-webserver/src/repositories.rs | 49 +++++++ ee/tabby-webserver/src/repositories/repo.rs | 122 +++++++++++++++++ 8 files changed, 356 insertions(+), 5 deletions(-) create mode 100644 ee/tabby-webserver/docs/api_spec.md create mode 100644 ee/tabby-webserver/src/repositories.rs create mode 100644 ee/tabby-webserver/src/repositories/repo.rs diff --git a/Cargo.lock b/Cargo.lock index b23d509adfc..8e4dc9bf8ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4704,6 +4704,8 @@ dependencies = [ "tokio", "tokio-rusqlite", "tokio-tungstenite", + "tower", + "tower-http 0.4.0", "tracing", "unicase", "uuid 1.4.1", @@ -5315,10 +5317,16 @@ dependencies = [ "http", "http-body", "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", "pin-project-lite", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] diff --git a/crates/tabby-common/src/config.rs b/crates/tabby-common/src/config.rs index e344f5324e7..edabae1c5a9 100644 --- a/crates/tabby-common/src/config.rs +++ b/crates/tabby-common/src/config.rs @@ -50,7 +50,7 @@ impl RepositoryConfig { let path = self.git_url.strip_prefix("file://").unwrap(); path.into() } else { - repositories_dir().join(filenamify(&self.git_url)) + repositories_dir().join(to_filename(&self.git_url)) } } @@ -73,9 +73,13 @@ impl Default for ServerConfig { } } +pub fn to_filename>(s: S) -> String { + filenamify(s) +} + #[cfg(test)] mod tests { - use super::{Config, RepositoryConfig}; + use super::{to_filename, Config, RepositoryConfig}; #[test] fn it_parses_empty_config() { @@ -96,4 +100,10 @@ mod tests { }; assert!(!repo.is_local_dir()); } + + #[test] + fn test_to_filename() { + let url = "https://github.com/TabbyML/tabby.git".to_string(); + assert_eq!(to_filename(url), "https_github.com_TabbyML_tabby.git"); + } } diff --git a/crates/tabby-common/src/lib.rs b/crates/tabby-common/src/lib.rs index ed90ff47683..8e0141cfc83 100644 --- a/crates/tabby-common/src/lib.rs +++ b/crates/tabby-common/src/lib.rs @@ -13,14 +13,17 @@ use std::{ path::PathBuf, }; +pub use config::to_filename; use path::dataset_dir; use serde::{Deserialize, Serialize}; use serde_jsonlines::JsonLinesReader; -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct SourceFile { pub git_url: String, pub filepath: String, + #[serde(skip_serializing_if = "String::is_empty")] + #[serde(default)] pub content: String, pub language: String, pub max_line_length: usize, @@ -45,7 +48,7 @@ impl SourceFile { } } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct Tag { pub range: Range, pub name_range: Range, diff --git a/ee/tabby-webserver/Cargo.toml b/ee/tabby-webserver/Cargo.toml index fdb4ecf1b71..71a05e9ec37 100644 --- a/ee/tabby-webserver/Cargo.toml +++ b/ee/tabby-webserver/Cargo.toml @@ -29,6 +29,8 @@ thiserror.workspace = true tokio = { workspace = true, features = ["fs"] } tokio-rusqlite = "0.4.0" tokio-tungstenite = "0.20.1" +tower = { version = "0.4", features = ["util"] } +tower-http = { version = "0.4.0", features = ["fs", "trace"] } tracing.workspace = true unicase = "2.7.0" diff --git a/ee/tabby-webserver/docs/api_spec.md b/ee/tabby-webserver/docs/api_spec.md new file mode 100644 index 00000000000..531a327fc22 --- /dev/null +++ b/ee/tabby-webserver/docs/api_spec.md @@ -0,0 +1,143 @@ +# API Specs + +## Repository api: `/repositories` + +### Resolve + +Get file or directory content from local repositories + +**URL:** `/repositories/{name}/resolve/{path}` + +**Method:** `GET` + +**Request examples:** + +- Get directory content + +```shell +curl --request GET \ + --url http://localhost:8080/repositories/https_github.com_TabbyML_tabby.git/resolve/ + +curl --request GET \ + --url http://localhost:9090/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/ +``` + +- Get file content + +```shell +curl --request GET \ + --url http://localhost:8080/repositories/https_github.com_TabbyML_tabby.git/resolve/package.json + +curl --request GET \ + --url http://localhost:9090/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/src/api.rs +``` + +**Response examples:** + +- All directory query will return a list of string, with each string represents an entry under that directory. The `Content-Type` for directory query is `application/vnd.directory+json`. + +For `/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/`, the response is: + +```json +{ + "entries": [ + "ee/tabby-webserver/src", + "ee/tabby-webserver/ui", + "ee/tabby-webserver/examples", + "ee/tabby-webserver/Cargo.toml", + "ee/tabby-webserver/graphql" + ] +} +``` + +- The file query will return file content, the `Content-Type` will be guessed from the file extension. + +For request `/repositories/https_github.com_TabbyML_tabby.git/resolve/package.json`, the content type is `application/json`, and the response is: + +```json +{ + "private": true, + "workspaces": [ + "clients/tabby-agent", + "clients/vscode", + "clients/vim", + "clients/intellij" + ], + "engines": { + "node": ">=18" + } +} +``` + +For request `/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/src/api.rs`, the content type is `text/x-rust`, and the response is: + +```text +use async_trait::async_trait; +use juniper::{GraphQLEnum, GraphQLObject}; +use serde::{Deserialize, Serialize}; +use tabby_common::api::{ + code::{CodeSearch, CodeSearchError, SearchResponse}, + event::RawEventLogger, +}; +use thiserror::Error; +use tokio_tungstenite::connect_async; + +use crate::websocket::WebSocketTransport; + +#[derive(GraphQLEnum, Serialize, Deserialize, Clone, Debug)] +pub enum WorkerKind { + Completion, + Chat, +} + +......omit...... +``` + +### Meta + +Get dataset entry for each indexed file in the repository + +**URL:** `/repositories/{name}/meta/{path}` + +**Method:** `GET` + +**Request example:** + +```shell +curl --request GET \ + --url http://localhost:9090/repositories/https_github.com_TabbyML_tabby.git/meta/ee/tabby-webserver/src/lib.rs +``` + +**Response example:** + +The `Content-Type` for successful response is always `application/json`. + +```json +{ + "git_url": "https://github.com/TabbyML/tabby.git", + "filepath": "ee/tabby-webserver/src/lib.rs", + "language": "rust", + "max_line_length": 88, + "avg_line_length": 26.340782, + "alphanum_fraction": 0.56416017, + "tags": [ + { + "range": { + "start": 0, + "end": 12 + }, + "name_range": { + "start": 8, + "end": 11 + }, + "line_range": { + "start": 0, + "end": 12 + }, + "is_definition": true, + "syntax_type_name": "module" + }, + ......omit...... + ] +} +``` diff --git a/ee/tabby-webserver/src/lib.rs b/ee/tabby-webserver/src/lib.rs index e93f3382a9e..0d809ad7250 100644 --- a/ee/tabby-webserver/src/lib.rs +++ b/ee/tabby-webserver/src/lib.rs @@ -11,6 +11,7 @@ use tracing::{error, warn}; use websocket::WebSocketTransport; mod db; +mod repositories; mod server; mod ui; mod websocket; @@ -31,6 +32,8 @@ use schema::Schema; use server::ServerContext; use tarpc::server::{BaseChannel, Channel}; +use crate::repositories::repo::load_dataset; + pub async fn attach_webserver( api: Router, ui: Router, @@ -49,7 +52,8 @@ pub async fn attach_webserver( ) .route("/graphql", routing::get(playground("/graphql", None))) .layer(Extension(schema)) - .route("/hub", routing::get(ws_handler).with_state(ctx)); + .route("/hub", routing::get(ws_handler).with_state(ctx)) + .nest("/repositories", repositories_routers().await); let ui = ui .route("/graphiql", routing::get(graphiql("/graphql", None))) @@ -58,6 +62,16 @@ pub async fn attach_webserver( (api, ui) } +async fn repositories_routers() -> Router { + load_dataset().await.unwrap(); + + Router::new() + .route("/:name/resolve/", routing::get(repositories::resolve)) + .route("/:name/resolve/*path", routing::get(repositories::resolve)) + .route("/:name/meta/", routing::get(repositories::meta)) + .route("/:name/meta/*path", routing::get(repositories::meta)) +} + async fn distributed_tabby_layer( State(ws): State>, request: Request, diff --git a/ee/tabby-webserver/src/repositories.rs b/ee/tabby-webserver/src/repositories.rs new file mode 100644 index 00000000000..dca71662473 --- /dev/null +++ b/ee/tabby-webserver/src/repositories.rs @@ -0,0 +1,49 @@ +pub(crate) mod repo; + +use anyhow::Result; +use axum::{extract::Path, http::StatusCode, response::Response, Json}; +use tabby_common::{path::repositories_dir, SourceFile}; +use tracing::{debug, instrument, warn}; + +use crate::repositories::repo::{resolve_dir, resolve_file, Repository, DATASET}; + +#[instrument(skip(repo))] +pub async fn resolve(Path(repo): Path) -> Result { + debug!("repo: {:?}", repo); + let root = repositories_dir().join(repo.name_str()); + let full_path = root.join(repo.path_str()); + let is_dir = tokio::fs::metadata(full_path.clone()) + .await + .map(|m| m.is_dir()) + .unwrap_or(false); + + if is_dir { + return match resolve_dir(root, full_path).await { + Ok(resp) => Ok(resp), + Err(err) => { + warn!("{}", err); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } + }; + } + + match resolve_file(root, &repo).await { + Ok(resp) => Ok(resp), + Err(err) => { + warn!("{}", err); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } + } +} + +#[instrument(skip(repo))] +pub async fn meta(Path(repo): Path) -> Result, StatusCode> { + debug!("repo: {:?}", repo); + let key = repo.dataset_key(); + if let Some(dataset) = DATASET.get() { + if let Some(file) = dataset.get(&key) { + return Ok(Json(file.clone())); + } + } + Err(StatusCode::NOT_FOUND) +} diff --git a/ee/tabby-webserver/src/repositories/repo.rs b/ee/tabby-webserver/src/repositories/repo.rs new file mode 100644 index 00000000000..bfe5eef32ed --- /dev/null +++ b/ee/tabby-webserver/src/repositories/repo.rs @@ -0,0 +1,122 @@ +use std::{collections::HashMap, path::PathBuf, str::FromStr}; + +use anyhow::Result; +use axum::{ + body::boxed, + http::{header, Request, Uri}, + response::{IntoResponse, Response}, + Json, +}; +use hyper::Body; +use serde::{Deserialize, Serialize}; +use tabby_common::{to_filename, SourceFile}; +use tokio::sync::OnceCell; +use tower::ServiceExt; +use tower_http::services::ServeDir; +use tracing::error; + +pub(crate) static DATASET: OnceCell> = OnceCell::const_new(); + +const MIME_VENDOR: &str = "application/vnd.directory+json"; + +#[derive(Hash, PartialEq, Eq, Debug)] +pub struct DatasetKey { + local_name: String, + rel_path: String, +} + +#[derive(Deserialize, Debug)] +pub struct Repository { + name: String, + path: Option, +} + +impl Repository { + pub fn dataset_key(&self) -> DatasetKey { + DatasetKey { + local_name: self.name.clone(), + rel_path: self.path_str().to_string(), + } + } + + pub fn name_str(&self) -> &str { + self.name.as_str() + } + + pub fn path_str(&self) -> &str { + self.path.as_deref().unwrap_or("") + } +} + +#[derive(Serialize)] +struct ListDir { + entries: Vec, +} + +/// Load dataset +pub async fn load_dataset() -> Result<()> { + // `SourceFile::all()` depends on `std::io`, so it's blocking. + // We need to spawn a blocking task dedicated for such scenario. + let dataset = tokio::task::spawn_blocking(|| { + let mut dataset = HashMap::new(); + let iter = match SourceFile::all() { + Ok(all) => all, + Err(err) => { + error!("load dataset: {}", err); + return dataset; + } + }; + for mut file in iter { + let key = DatasetKey { + local_name: to_filename(file.git_url.as_str()), + rel_path: file.filepath.clone(), + }; + // exclude content from response data + file.content = "".to_string(); + dataset.insert(key, file); + } + dataset + }) + .await?; + + DATASET.set(dataset)?; + Ok(()) +} + +/// Resolve a directory +pub async fn resolve_dir(root: PathBuf, full_path: PathBuf) -> Result { + let mut read_dir = tokio::fs::read_dir(full_path).await?; + let mut entries = vec![]; + + while let Some(entry) = read_dir.next_entry().await? { + let path = entry + .path() + .strip_prefix(&root)? + .to_str() + .unwrap() + .to_string(); + entries.push(path); + } + + let body = Json(ListDir { entries }).into_response(); + let resp = Response::builder() + .header(header::CONTENT_TYPE, MIME_VENDOR) + .body(body.into_body())?; + + Ok(resp) +} + +/// Resolve a file +pub async fn resolve_file(root: PathBuf, repo: &Repository) -> Result { + let uri = if !repo.path_str().starts_with('/') { + let path = format!("/{}", repo.path_str()); + Uri::from_str(path.as_str())? + } else { + Uri::from_str(repo.path_str())? + }; + + let req = Request::builder().uri(uri).body(Body::empty()).unwrap(); + let resp = ServeDir::new(root).oneshot(req).await?; + + Ok(resp.map(boxed)) +} From 2d7666b41f8bd9ad56ffc23d18dc444e911983db Mon Sep 17 00:00:00 2001 From: darknight Date: Thu, 23 Nov 2023 13:45:12 +0800 Subject: [PATCH 2/6] resolve comments --- crates/tabby-common/src/config.rs | 20 ++-- crates/tabby-common/src/lib.rs | 5 +- ee/tabby-webserver/src/lib.rs | 14 +-- ee/tabby-webserver/src/repositories.rs | 33 ++++--- .../src/repositories/{repo.rs => resolve.rs} | 95 +++++++++++++------ 5 files changed, 99 insertions(+), 68 deletions(-) rename ee/tabby-webserver/src/repositories/{repo.rs => resolve.rs} (51%) diff --git a/crates/tabby-common/src/config.rs b/crates/tabby-common/src/config.rs index edabae1c5a9..006fcbfccff 100644 --- a/crates/tabby-common/src/config.rs +++ b/crates/tabby-common/src/config.rs @@ -50,13 +50,17 @@ impl RepositoryConfig { let path = self.git_url.strip_prefix("file://").unwrap(); path.into() } else { - repositories_dir().join(to_filename(&self.git_url)) + repositories_dir().join(self.name()) } } pub fn is_local_dir(&self) -> bool { self.git_url.starts_with("file://") } + + pub fn name(&self) -> String { + filenamify(&self.git_url) + } } #[derive(Serialize, Deserialize)] @@ -73,13 +77,9 @@ impl Default for ServerConfig { } } -pub fn to_filename>(s: S) -> String { - filenamify(s) -} - #[cfg(test)] mod tests { - use super::{to_filename, Config, RepositoryConfig}; + use super::{Config, RepositoryConfig}; #[test] fn it_parses_empty_config() { @@ -102,8 +102,10 @@ mod tests { } #[test] - fn test_to_filename() { - let url = "https://github.com/TabbyML/tabby.git".to_string(); - assert_eq!(to_filename(url), "https_github.com_TabbyML_tabby.git"); + fn test_repository_config_name() { + let repo = RepositoryConfig { + git_url: "https://github.com/TabbyML/tabby.git".to_owned(), + }; + assert_eq!(repo.name(), "https_github.com_TabbyML_tabby.git"); } } diff --git a/crates/tabby-common/src/lib.rs b/crates/tabby-common/src/lib.rs index 8e0141cfc83..8966adf8959 100644 --- a/crates/tabby-common/src/lib.rs +++ b/crates/tabby-common/src/lib.rs @@ -13,17 +13,14 @@ use std::{ path::PathBuf, }; -pub use config::to_filename; use path::dataset_dir; use serde::{Deserialize, Serialize}; use serde_jsonlines::JsonLinesReader; -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize)] pub struct SourceFile { pub git_url: String, pub filepath: String, - #[serde(skip_serializing_if = "String::is_empty")] - #[serde(default)] pub content: String, pub language: String, pub max_line_length: usize, diff --git a/ee/tabby-webserver/src/lib.rs b/ee/tabby-webserver/src/lib.rs index 0d809ad7250..f887a3c828f 100644 --- a/ee/tabby-webserver/src/lib.rs +++ b/ee/tabby-webserver/src/lib.rs @@ -32,8 +32,6 @@ use schema::Schema; use server::ServerContext; use tarpc::server::{BaseChannel, Channel}; -use crate::repositories::repo::load_dataset; - pub async fn attach_webserver( api: Router, ui: Router, @@ -53,7 +51,7 @@ pub async fn attach_webserver( .route("/graphql", routing::get(playground("/graphql", None))) .layer(Extension(schema)) .route("/hub", routing::get(ws_handler).with_state(ctx)) - .nest("/repositories", repositories_routers().await); + .nest("/repositories", repositories::routers()); let ui = ui .route("/graphiql", routing::get(graphiql("/graphql", None))) @@ -62,16 +60,6 @@ pub async fn attach_webserver( (api, ui) } -async fn repositories_routers() -> Router { - load_dataset().await.unwrap(); - - Router::new() - .route("/:name/resolve/", routing::get(repositories::resolve)) - .route("/:name/resolve/*path", routing::get(repositories::resolve)) - .route("/:name/meta/", routing::get(repositories::meta)) - .route("/:name/meta/*path", routing::get(repositories::meta)) -} - async fn distributed_tabby_layer( State(ws): State>, request: Request, diff --git a/ee/tabby-webserver/src/repositories.rs b/ee/tabby-webserver/src/repositories.rs index dca71662473..cd0612137f5 100644 --- a/ee/tabby-webserver/src/repositories.rs +++ b/ee/tabby-webserver/src/repositories.rs @@ -1,15 +1,25 @@ -pub(crate) mod repo; +mod resolve; use anyhow::Result; -use axum::{extract::Path, http::StatusCode, response::Response, Json}; -use tabby_common::{path::repositories_dir, SourceFile}; -use tracing::{debug, instrument, warn}; +use axum::{extract::Path, http::StatusCode, response::Response, routing, Json, Router}; +use tabby_common::path::repositories_dir; +use tracing::{instrument, warn}; -use crate::repositories::repo::{resolve_dir, resolve_file, Repository, DATASET}; +use crate::{ + repositories, + repositories::resolve::{resolve_dir, resolve_file, resolve_meta, Meta, Repository}, +}; + +pub fn routers() -> Router { + Router::new() + .route("/:name/resolve/", routing::get(repositories::resolve)) + .route("/:name/resolve/*path", routing::get(repositories::resolve)) + .route("/:name/meta/", routing::get(repositories::meta)) + .route("/:name/meta/*path", routing::get(repositories::meta)) +} #[instrument(skip(repo))] -pub async fn resolve(Path(repo): Path) -> Result { - debug!("repo: {:?}", repo); +async fn resolve(Path(repo): Path) -> Result { let root = repositories_dir().join(repo.name_str()); let full_path = root.join(repo.path_str()); let is_dir = tokio::fs::metadata(full_path.clone()) @@ -37,13 +47,10 @@ pub async fn resolve(Path(repo): Path) -> Result) -> Result, StatusCode> { - debug!("repo: {:?}", repo); +async fn meta(Path(repo): Path) -> Result, StatusCode> { let key = repo.dataset_key(); - if let Some(dataset) = DATASET.get() { - if let Some(file) = dataset.get(&key) { - return Ok(Json(file.clone())); - } + if let Some(resp) = resolve_meta(&key) { + return Ok(Json(resp)); } Err(StatusCode::NOT_FOUND) } diff --git a/ee/tabby-webserver/src/repositories/repo.rs b/ee/tabby-webserver/src/repositories/resolve.rs similarity index 51% rename from ee/tabby-webserver/src/repositories/repo.rs rename to ee/tabby-webserver/src/repositories/resolve.rs index bfe5eef32ed..503f38b919a 100644 --- a/ee/tabby-webserver/src/repositories/repo.rs +++ b/ee/tabby-webserver/src/repositories/resolve.rs @@ -8,16 +8,18 @@ use axum::{ Json, }; use hyper::Body; +use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; -use tabby_common::{to_filename, SourceFile}; -use tokio::sync::OnceCell; +use tabby_common::{config::Config, SourceFile, Tag}; use tower::ServiceExt; use tower_http::services::ServeDir; use tracing::error; -pub(crate) static DATASET: OnceCell> = OnceCell::const_new(); +lazy_static! { + static ref META: HashMap = load_meta(); +} -const MIME_VENDOR: &str = "application/vnd.directory+json"; +const DIRECTORY_MIME_TYPE: &str = "application/vnd.directory+json"; #[derive(Hash, PartialEq, Eq, Debug)] pub struct DatasetKey { @@ -53,34 +55,62 @@ struct ListDir { entries: Vec, } -/// Load dataset -pub async fn load_dataset() -> Result<()> { - // `SourceFile::all()` depends on `std::io`, so it's blocking. - // We need to spawn a blocking task dedicated for such scenario. - let dataset = tokio::task::spawn_blocking(|| { - let mut dataset = HashMap::new(); - let iter = match SourceFile::all() { - Ok(all) => all, - Err(err) => { - error!("load dataset: {}", err); - return dataset; - } - }; - for mut file in iter { +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Meta { + git_url: String, + filepath: String, + language: String, + max_line_length: usize, + avg_line_length: f32, + alphanum_fraction: f32, + tags: Vec, +} + +impl From for Meta { + fn from(file: SourceFile) -> Self { + Self { + git_url: file.git_url, + filepath: file.filepath, + language: file.language, + max_line_length: file.max_line_length, + avg_line_length: file.avg_line_length, + alphanum_fraction: file.alphanum_fraction, + tags: file.tags, + } + } +} + +/// TODO: implement auto reloading logic in future (so changes produced by tabby-scheduler command will be loaded) +fn load_meta() -> HashMap { + let mut dataset = HashMap::new(); + let repo_conf = match Config::load() { + Ok(config) => config + .repositories + .into_iter() + .map(|repo| (repo.git_url.clone(), repo)) + .collect::>(), + Err(err) => { + error!("load config: {}", err); + return dataset; + } + }; + let iter = match SourceFile::all() { + Ok(all) => all, + Err(err) => { + error!("load dataset: {}", err); + return dataset; + } + }; + for file in iter { + if let Some(name) = repo_conf.get(&file.git_url).map(|repo| repo.name()) { let key = DatasetKey { - local_name: to_filename(file.git_url.as_str()), + local_name: name, rel_path: file.filepath.clone(), }; - // exclude content from response data - file.content = "".to_string(); - dataset.insert(key, file); + dataset.insert(key, file.into()); } - dataset - }) - .await?; - - DATASET.set(dataset)?; - Ok(()) + } + dataset } /// Resolve a directory @@ -100,7 +130,7 @@ pub async fn resolve_dir(root: PathBuf, full_path: PathBuf) -> Result let body = Json(ListDir { entries }).into_response(); let resp = Response::builder() - .header(header::CONTENT_TYPE, MIME_VENDOR) + .header(header::CONTENT_TYPE, DIRECTORY_MIME_TYPE) .body(body.into_body())?; Ok(resp) @@ -120,3 +150,10 @@ pub async fn resolve_file(root: PathBuf, repo: &Repository) -> Result Ok(resp.map(boxed)) } + +pub fn resolve_meta(key: &DatasetKey) -> Option { + if let Some(meta) = META.get(key) { + return Some(meta.clone()); + } + None +} From 94437496eb7ed34c8454a0e847da4ecefc5398e1 Mon Sep 17 00:00:00 2001 From: darknight Date: Thu, 23 Nov 2023 21:31:09 +0800 Subject: [PATCH 3/6] resolve comments --- ee/tabby-webserver/src/repositories.rs | 12 ++++++------ ee/tabby-webserver/src/repositories/resolve.rs | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ee/tabby-webserver/src/repositories.rs b/ee/tabby-webserver/src/repositories.rs index cd0612137f5..2b8de2cc83e 100644 --- a/ee/tabby-webserver/src/repositories.rs +++ b/ee/tabby-webserver/src/repositories.rs @@ -7,7 +7,7 @@ use tracing::{instrument, warn}; use crate::{ repositories, - repositories::resolve::{resolve_dir, resolve_file, resolve_meta, Meta, Repository}, + repositories::resolve::{resolve_dir, resolve_file, resolve_meta, Meta, ResolveParams}, }; pub fn routers() -> Router { @@ -19,7 +19,7 @@ pub fn routers() -> Router { } #[instrument(skip(repo))] -async fn resolve(Path(repo): Path) -> Result { +async fn resolve(Path(repo): Path) -> Result { let root = repositories_dir().join(repo.name_str()); let full_path = root.join(repo.path_str()); let is_dir = tokio::fs::metadata(full_path.clone()) @@ -28,10 +28,10 @@ async fn resolve(Path(repo): Path) -> Result { .unwrap_or(false); if is_dir { - return match resolve_dir(root, full_path).await { + return match resolve_dir(root, full_path.clone()).await { Ok(resp) => Ok(resp), Err(err) => { - warn!("{}", err); + warn!("failed to resolve_dir <{:?}>: {}", full_path, err); Err(StatusCode::INTERNAL_SERVER_ERROR) } }; @@ -40,14 +40,14 @@ async fn resolve(Path(repo): Path) -> Result { match resolve_file(root, &repo).await { Ok(resp) => Ok(resp), Err(err) => { - warn!("{}", err); + warn!("failed to resolve_file <{:?}>: {}", full_path, err); Err(StatusCode::INTERNAL_SERVER_ERROR) } } } #[instrument(skip(repo))] -async fn meta(Path(repo): Path) -> Result, StatusCode> { +async fn meta(Path(repo): Path) -> Result, StatusCode> { let key = repo.dataset_key(); if let Some(resp) = resolve_meta(&key) { return Ok(Json(resp)); diff --git a/ee/tabby-webserver/src/repositories/resolve.rs b/ee/tabby-webserver/src/repositories/resolve.rs index 503f38b919a..e7623345b17 100644 --- a/ee/tabby-webserver/src/repositories/resolve.rs +++ b/ee/tabby-webserver/src/repositories/resolve.rs @@ -28,12 +28,12 @@ pub struct DatasetKey { } #[derive(Deserialize, Debug)] -pub struct Repository { +pub struct ResolveParams { name: String, path: Option, } -impl Repository { +impl ResolveParams { pub fn dataset_key(&self) -> DatasetKey { DatasetKey { local_name: self.name.clone(), @@ -137,7 +137,7 @@ pub async fn resolve_dir(root: PathBuf, full_path: PathBuf) -> Result } /// Resolve a file -pub async fn resolve_file(root: PathBuf, repo: &Repository) -> Result { +pub async fn resolve_file(root: PathBuf, repo: &ResolveParams) -> Result { let uri = if !repo.path_str().starts_with('/') { let path = format!("/{}", repo.path_str()); Uri::from_str(path.as_str())? From 71a45a1f96aa695f2bdcf09500bcdef6dbe15c37 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Thu, 23 Nov 2023 21:33:46 +0800 Subject: [PATCH 4/6] Update repositories.rs --- ee/tabby-webserver/src/repositories.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/tabby-webserver/src/repositories.rs b/ee/tabby-webserver/src/repositories.rs index 2b8de2cc83e..b9d3ca2c9e4 100644 --- a/ee/tabby-webserver/src/repositories.rs +++ b/ee/tabby-webserver/src/repositories.rs @@ -10,7 +10,7 @@ use crate::{ repositories::resolve::{resolve_dir, resolve_file, resolve_meta, Meta, ResolveParams}, }; -pub fn routers() -> Router { +pub fn routes() -> Router { Router::new() .route("/:name/resolve/", routing::get(repositories::resolve)) .route("/:name/resolve/*path", routing::get(repositories::resolve)) From bb0f7ac13236dbb240367c9f79a84be18c7b972f Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Thu, 23 Nov 2023 21:34:08 +0800 Subject: [PATCH 5/6] Update lib.rs --- ee/tabby-webserver/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/tabby-webserver/src/lib.rs b/ee/tabby-webserver/src/lib.rs index f887a3c828f..8a5ad9e6c8b 100644 --- a/ee/tabby-webserver/src/lib.rs +++ b/ee/tabby-webserver/src/lib.rs @@ -51,7 +51,7 @@ pub async fn attach_webserver( .route("/graphql", routing::get(playground("/graphql", None))) .layer(Extension(schema)) .route("/hub", routing::get(ws_handler).with_state(ctx)) - .nest("/repositories", repositories::routers()); + .nest("/repositories", repositories::routes()); let ui = ui .route("/graphiql", routing::get(graphiql("/graphql", None))) From c17ca456d8d17de0777796ae50a77d3403f6c101 Mon Sep 17 00:00:00 2001 From: darknight Date: Fri, 24 Nov 2023 09:53:49 +0800 Subject: [PATCH 6/6] resolve comment --- ee/tabby-webserver/src/repositories/resolve.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ee/tabby-webserver/src/repositories/resolve.rs b/ee/tabby-webserver/src/repositories/resolve.rs index e7623345b17..b550f479ced 100644 --- a/ee/tabby-webserver/src/repositories/resolve.rs +++ b/ee/tabby-webserver/src/repositories/resolve.rs @@ -13,7 +13,6 @@ use serde::{Deserialize, Serialize}; use tabby_common::{config::Config, SourceFile, Tag}; use tower::ServiceExt; use tower_http::services::ServeDir; -use tracing::error; lazy_static! { static ref META: HashMap = load_meta(); @@ -89,15 +88,13 @@ fn load_meta() -> HashMap { .into_iter() .map(|repo| (repo.git_url.clone(), repo)) .collect::>(), - Err(err) => { - error!("load config: {}", err); + Err(_) => { return dataset; } }; let iter = match SourceFile::all() { Ok(all) => all, - Err(err) => { - error!("load dataset: {}", err); + Err(_) => { return dataset; } };