diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2a37bd6..9f96bd5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,6 +6,10 @@ on: - main merge_group: +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + jobs: check: name: Check @@ -71,3 +75,34 @@ jobs: run: cargo doc --no-deps --workspace --document-private-items env: RUSTDOCFLAGS: -D warnings + + build: + name: Build and push + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/Cargo.lock b/Cargo.lock index c5bac67..dffee99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1492,6 +1492,8 @@ dependencies = [ "grass", "maud", "mwp-content", + "mwp-search", + "rusqlite", "serde", "serde_json", "tantivy", diff --git a/Dockerfile b/Dockerfile index f588582..60f176d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,10 @@ -FROM rust:1.70 as builder +FROM rust:1.70-slim-bookworm as builder # deps WORKDIR /app RUN USER=root cargo init -COPY Cargo.toml Cargo.lock ./ -RUN cargo build --release - -# app -ADD . ./ -RUN rm ./target/release/deps/mwp* -RUN cargo build --release +COPY . ./ +RUN cargo build --release --target mwp-web --bin mwp-web # runtime FROM debian:buster-slim @@ -24,8 +19,8 @@ RUN apt update \ ENV TZ=Etc/UTC -COPY --from=builder /app/target/release/mwp mwp +COPY --from=builder /app/target/release/mwp-web mwp-web EXPOSE 4444 -CMD ["/app/mwp"] +CMD ["/app/mwp-web"] diff --git a/mwp-content/src/lib.rs b/mwp-content/src/lib.rs index c0550aa..be1e63c 100644 --- a/mwp-content/src/lib.rs +++ b/mwp-content/src/lib.rs @@ -73,7 +73,6 @@ impl Page { let mut link_title = String::new(); let mut links: Vec = Vec::new(); let mut open_tags: Vec = Vec::new(); - let mut got_resources: bool = false; #[allow(clippy::unnecessary_filter_map)] let parser = Parser::new_ext(&content, options).filter_map(|event| match event.clone() { @@ -93,25 +92,10 @@ impl Page { link_title = text.to_string(); } - // once we detect "Resources" section of the markdown we ignore the rest of the document - if text.as_ref() == "Resources" { - got_resources = true; - return Some(event); - } - - if got_resources { - return None; - } - Some(event) } Event::Start(tag) => { open_tags.push(tag); - - if got_resources { - return None; - } - Some(event) } Event::End(tag) => { @@ -137,10 +121,7 @@ impl Page { Some(event) } - event => match got_resources { - true => None, - false => Some(event), - }, + event => Some(event), }); let mut html_output = String::new(); diff --git a/mwp-search/src/lib.rs b/mwp-search/src/lib.rs index 47577f3..dbbde9a 100644 --- a/mwp-search/src/lib.rs +++ b/mwp-search/src/lib.rs @@ -32,25 +32,21 @@ impl Doc { } impl SearchIndex { - pub fn new(dir: &str) -> Result> { - let index = if let Ok(index) = Index::open_in_dir(dir) { - index - } else { - let mut schema_builder = Schema::builder(); - schema_builder.add_text_field("url", STRING | STORED); - schema_builder.add_text_field("title", TEXT | STORED); - schema_builder.add_text_field("body", TEXT); - schema_builder.add_text_field("domain", STRING | STORED); - schema_builder.add_text_field("tags", STRING | STORED | FAST); - let schema = schema_builder.build(); - Index::builder() - .schema(schema) - .settings(tantivy::IndexSettings { - docstore_blocksize: 32_000_000, - ..tantivy::IndexSettings::default() - }) - .create_in_dir(dir)? - }; + pub fn new() -> Result> { + let mut schema_builder = Schema::builder(); + schema_builder.add_text_field("url", STRING | STORED); + schema_builder.add_text_field("title", TEXT | STORED); + schema_builder.add_text_field("body", TEXT); + schema_builder.add_text_field("domain", STRING | STORED); + schema_builder.add_text_field("tags", STRING | STORED | FAST); + let schema = schema_builder.build(); + let index = Index::builder() + .schema(schema) + .settings(tantivy::IndexSettings { + docstore_blocksize: 32_000_000, + ..tantivy::IndexSettings::default() + }) + .create_in_ram()?; Ok(SearchIndex { index }) } diff --git a/mwp-web/Cargo.toml b/mwp-web/Cargo.toml index dd34060..8eef757 100644 --- a/mwp-web/Cargo.toml +++ b/mwp-web/Cargo.toml @@ -14,8 +14,10 @@ maud = { version = "0.26.0", features = ["actix-web"] } serde = "1.0.196" serde_json = "1.0.113" tantivy = "0.21.1" +rusqlite = { version = "0.30.0", features = ["time", "url"]} mwp-content = { path="../mwp-content" } +mwp-search = { path="../mwp-search" } [build-dependencies] grass = "0.13.2" diff --git a/mwp-web/src/main.rs b/mwp-web/src/main.rs index 2452043..16b7084 100644 --- a/mwp-web/src/main.rs +++ b/mwp-web/src/main.rs @@ -6,6 +6,8 @@ use actix_web::{ }; use maud::{html, Markup, PreEscaped}; use mwp_content::Content; +use mwp_search::{Doc, SearchIndex}; +use rusqlite::Connection; use serde::Deserialize; use tantivy::{ query::{AllQuery, QueryParser, TermQuery}, @@ -161,9 +163,6 @@ async fn content_page( } } article { (PreEscaped(html)) } - .links { - (listing(searcher, schema, result.docs)) - } } )) } @@ -185,13 +184,38 @@ impl Guard for ContentGuard { async fn main() -> std::io::Result<()> { env_logger::init_from_env(env_logger::Env::new().default_filter_or("info")); - let index_path = "./index"; - let index = Index::open_in_dir(index_path).unwrap(); + let index = SearchIndex::new().unwrap(); + + let conn = Connection::open("./my_db.db3").unwrap(); + let mut stmt = conn + .prepare("SELECT title, url, domain, body, tags, created_at, scraped_at FROM links") + .unwrap(); + let docs_iter = stmt + .query_map([], |row| { + Ok(Doc { + title: row.get(0)?, + url: row.get(1)?, + domain: row.get(2)?, + body: row.get(3)?, + tags: row.get::>(4).map(|res| { + res.map(|s| s.split(';').map(|s| s.into()).collect::>()) + })?, + created_at: row.get(5)?, + scraped_at: row.get(6)?, + }) + }) + .unwrap(); + + let mut builder = index.builder(); + for doc in docs_iter { + builder.add(doc.unwrap()).unwrap(); + } + builder.commit(); let content = Content::from_dir("../wiki").await; HttpServer::new(move || { App::new() - .app_data(web::Data::new(index.clone())) + .app_data(web::Data::new(index.index.clone())) .app_data(web::Data::new(content.clone())) .service(tag_page) .service(search_page) diff --git a/src/main.rs b/src/main.rs index 71e2b9e..6436b5b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,5 @@ use mwp_content::Link; -use mwp_search::{Doc, SearchIndex}; +use mwp_search::Doc; use rusqlite::Connection; use time::OffsetDateTime; @@ -21,34 +21,7 @@ async fn main() -> Result<(), Box> { (), // empty list of parameters. )?; - let index = SearchIndex::new("./index")?; - let content = mwp_content::Content::from_dir("../wiki").await; - index.add_content(content.values())?; - - let mut stmt = - conn.prepare("SELECT title, url, domain, body, tags, created_at, scraped_at FROM links")?; - let docs_iter = stmt.query_map([], |row| { - Ok(Doc { - title: row.get(0)?, - url: row.get(1)?, - domain: row.get(2)?, - body: row.get(3)?, - tags: row - .get::>(4) - .map(|res| res.map(|s| s.split(';').map(|s| s.into()).collect::>()))?, - created_at: row.get(5)?, - scraped_at: row.get(6)?, - }) - })?; - - let mut builder = index.builder(); - for doc in docs_iter { - builder.add(doc.unwrap())?; - } - builder.commit(); - - return Ok(()); // only needed before links from content are migrated to bookmarking system let links = content