chore: cleanup and refactor

Use xtask for scraping job, cleanup dependencies.

.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run --package xtask --"

Cargo.toml

@@ -1,10 +1,19 @@
+[workspace.package]
+version = "0.1.0"
+edition = "2021"
+authors = ["Matouš Dzivjak <[email protected]>"]
+categories = ["wiki", "knowledge-bage"]
+repository = "https://github.com/matoous/mwp"
+homepage = "https://github.com/matoous/mwp"
+
 [workspace]
 resolver = "2"
 members = [
   "mwp-web",
   "mwp-scraper",
   "mwp-content",
   "mwp-search",
+  "xtask",
 ]
 
 default-members = [
@@ -14,12 +23,8 @@ default-members = [
 [profile.release]
 lto = "thin"
 
-[workspace.package]
-name = "mwp"
-version = "0.1.0"
-edition = "2021"
-authors = ["Matouš Dzivjak <[email protected]>"]
-categories = ["wiki", "knowledge-bage"]
-repository = "https://github.com/matoous/mwp"
-homepage = "https://github.com/matoous/mwp"
-rust-version = "1.70"
+[workspace.dependencies]
+tantivy = { version = "0.22.0", features = ["mmap"] }
+url = { version = "2.5.4", features = ["serde"] }
+time = "0.3.37"
+rusqlite = { version = "0.31.0", features = ["time", "url", "bundled"]}

mwp-scraper/Cargo.toml

@@ -8,20 +8,15 @@ repository.workspace = true
 
 [dependencies]
 html-escape = "0.2.13"
-lazy_static = "1.5.0"
 lol_html = "1.2.1"
-pulldown-cmark = "0.12.2"
+lazy_static = "1.5.0"
 regex = "1.11.1"
 reqwest = "0.12.9"
-serde = "1.0.217"
-serde_json = "1.0.134"
-sled = "0.34.7"
-tantivy = { version = "0.22.0", features = ["mmap"] }
-time = "0.3.37"
-tokio = { version = "1.42.0", features= ["full"]}
-url = { version = "2.5.4", features = ["serde"] }
-walkdir = "2.5.0"
-rusqlite = { version = "0.31.0", features = ["time", "url", "bundled"]}
+
+rusqlite = { workspace = true }
+tantivy = { workspace = true }
+url = { workspace = true }
+time = { workspace = true }
 
 mwp-content = { path="../mwp-content" }
 mwp-search = { path="../mwp-search" }

mwp-scraper/src/main.rs → mwp-scraper/src/lib.rs

@@ -19,8 +19,7 @@ pub async fn scrape(link: &Url) -> Result<DomParserResult, Box<dyn std::error::E
     Ok(rewriter.wrap())
 }
 
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+pub async fn scrape_all() -> Result<(), Box<dyn std::error::Error>> {
     let conn = Connection::open("./db.db3")?;
     conn.execute(
         r#"

mwp-search/Cargo.toml

@@ -6,7 +6,8 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tantivy = { version = "0.22.0", features = ["mmap"] }
+tantivy = { workspace = true }
+url = { workspace = true }
+time = { workspace = true }
+
 mwp-content = { path="../mwp-content" }
-time = "0.3.37"
-url = "2.5.4"

mwp-web/Cargo.toml

@@ -13,15 +13,17 @@ path = "src/main.rs"
 [dependencies]
 actix-files = "0.6.6"
 actix-web = "4.9.0"
+static-files = "0.2.4"
+actix-web-static-files = "4.0.1"
+
 env_logger = "0.11.6"
 maud = { version = "0.26.0", features = ["actix-web"] }
 serde = "1.0.217"
 serde_json = "1.0.134"
-tantivy = "0.22.0"
-rusqlite = { version = "0.31.0", features = ["time", "url", "bundled"]}
 clap = { version = "4.5.13", features = ["derive"]}
-static-files = "0.2.4"
-actix-web-static-files = "4.0.1"
+
+rusqlite = { workspace = true }
+tantivy = { workspace = true }
 
 mwp-content = { path="../mwp-content" }
 mwp-search = { path="../mwp-search" }

xtask/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "xtask"
+version = "0.0.1"
+edition = "2021"
+publish = false
+
+[dependencies]
+clap = { version = "4.5.23", features = [
+  "derive",
+  "string",
+  "env",
+  "wrap_help",
+] }
+tokio = { version = "1.42.0", features= ["full"]}
+
+mwp-scraper = { path="../mwp-scraper" }