Merge pull request #93 from terraphim/spring_clean_config

Spring clean config

Cargo.toml

@@ -1,4 +1,4 @@
 [workspace]
 resolver = "2"
 members = ["crates/*", "terraphim_server", "desktop/src-tauri"]
-default-members = ["terraphim_server"]
+default-members = ["terraphim_server"]

Earthfile

@@ -246,8 +246,7 @@ docker-scratch:
     ENTRYPOINT ["./terraphim_server"]
     SAVE IMAGE aks/terraphim_server:scratch
 
-
-docs-pages:
+docs-deps:
   FROM +install-native 
   RUN cargo install mdbook
   RUN cargo install mdbook-epub
@@ -256,9 +255,15 @@ docs-pages:
   RUN cargo install --git https://github.com/typst/typst typst-cli 
   RUN cargo install --git https://github.com/terraphim/mdbook-typst.git
   RUN cargo install mdbook-mermaid
+  RUN cargo install mdbook-alerts
   RUN apt update && apt install libvips-dev -y
   RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash
   RUN bash -c "source $HOME/.nvm/nvm.sh && nvm install 20 && npm install -g netlify-cli"
+
+
+
+docs-pages:
+  FROM +docs-deps
   COPY --keep-ts docs /docs
   WORKDIR /docs
   RUN mdbook --version

Layerfile

@@ -15,7 +15,7 @@ RUN rustup component add rustfmt
 RUN cargo install ripgrep
 WORKDIR /code
 COPY . .
-RUN ./desktop/scripts/yarn_and_build.sh
+RUN ./scripts/yarn_and_build.sh
 ENV TERRAPHIM_SERVER_HOSTNAME=0.0.0.0:8000
 RUN cargo build --release
 RUN git clone --depth 1 https://github.com/terraphim/INCOSE-Systems-Engineering-Handbook.git /tmp/system_operator/

crates/terraphim_config/src/lib.rs

@@ -18,7 +18,6 @@ use ulid::Ulid;
 pub type Result<T> = std::result::Result<T, TerraphimConfigError>;
 
 use opendal::Result as OpendalResult;
-use url::Url;
 
 type PersistenceResult<T> = std::result::Result<T, terraphim_persistence::Error>;
 
@@ -66,9 +65,7 @@ pub struct Role {
     /// The relevance function used to rank search results
     pub relevance_function: RelevanceFunction,
     pub theme: String,
-    #[serde(rename = "serverUrl")]
-    pub server_url: Url,
-    pub kg: KnowledgeGraph,
+    pub kg: Option<KnowledgeGraph>,
     pub haystacks: Vec<Haystack>,
     #[serde(flatten)]
     pub extra: AHashMap<String, Value>,
@@ -282,12 +279,16 @@ impl ConfigState {
         let mut roles = AHashMap::new();
         for (name, role) in &config.roles {
             let role_name = name.to_lowercase();
-            let automata_url = role.kg.automata_path.clone();
-            log::info!("Loading Role `{}` - URL: {}", role_name, automata_url);
-
-            let thesaurus = load_thesaurus(&automata_url).await?;
-            let rolegraph = RoleGraph::new(role_name.clone(), thesaurus).await?;
-            roles.insert(role_name, RoleGraphSync::from(rolegraph));
+            log::info!("Creating role {}", role_name);
+            if role.kg.is_some() {
+                let automata_url = role.kg.as_ref().unwrap().automata_path.clone();
+                log::info!("Loading Role `{}` - URL: {}", role_name, automata_url);
+                let thesaurus = load_thesaurus(&automata_url).await?;
+                let rolegraph = RoleGraph::new(role_name.clone(), thesaurus).await?;
+                roles.insert(role_name, RoleGraphSync::from(rolegraph));
+            } else {
+                log::info!("Skipping KG due to None settings for role {}", role_name);
+            }
         }
 
         Ok(ConfigState {
@@ -319,22 +320,21 @@ impl ConfigState {
         Ok(())
     }
 
-    /// Search documents in rolegraph index, which match the search query
+    /// Search documents in rolegraph index using matching Knowledge Graph
+    /// If knowledge graph isn't defined for the role, RoleGraph isn't build for the role
     pub async fn search_indexed_documents(
         &self,
         search_query: &SearchQuery,
+        role: &Role,
     ) -> Vec<IndexedDocument> {
         log::debug!("search_documents: {:?}", search_query);
-        let current_config_state = self.config.lock().await.clone();
-        let default_role = current_config_state.default_role.clone();
 
-        // if role is not provided, use the default role from the config
-        let role = search_query.role.clone().unwrap_or(default_role);
         log::debug!("Role for search_documents: {:#?}", role);
 
-        let role_name = role.to_lowercase();
+        let role_name = role.name.to_lowercase();
+        log::debug!("Role name for searching {role_name}");
+        log::debug!("All roles defined  {:?}", self.roles.clone().into_keys());
         let role = self.roles.get(&role_name).unwrap().lock().await;
-
         let documents = role
             .query_graph(
                 &search_query.search_term,
@@ -410,14 +410,7 @@ mod tests {
                     name: "Default".to_string(),
                     relevance_function: RelevanceFunction::TitleScorer,
                     theme: "spacelab".to_string(),
-                    server_url: Url::parse("http://localhost:8000/documents/search").unwrap(),
-                    kg: KnowledgeGraph {
-                        automata_path: AutomataPath::local_example(),
-                        input_type: KnowledgeGraphInputType::Markdown,
-                        path: PathBuf::from("~/pkm"),
-                        public: true,
-                        publish: true,
-                    },
+                    kg: None,
                     haystacks: vec![Haystack {
                         path: PathBuf::from("localsearch"),
                         service: ServiceType::Ripgrep,
@@ -432,14 +425,7 @@ mod tests {
                     name: "Engineer".to_string(),
                     relevance_function: RelevanceFunction::TitleScorer,
                     theme: "lumen".to_string(),
-                    server_url: Url::parse("http://localhost:8000/documents/search").unwrap(),
-                    kg: KnowledgeGraph {
-                        automata_path: AutomataPath::local_example(),
-                        input_type: KnowledgeGraphInputType::Markdown,
-                        path: PathBuf::from("~/pkm"),
-                        public: true,
-                        publish: true,
-                    },
+                    kg: None,
                     haystacks: vec![Haystack {
                         path: PathBuf::from("localsearch"),
                         service: ServiceType::Ripgrep,
@@ -454,14 +440,13 @@ mod tests {
                     name: "System Operator".to_string(),
                     relevance_function: RelevanceFunction::TitleScorer,
                     theme: "superhero".to_string(),
-                    server_url: Url::parse("http://localhost:8000/documents/search").unwrap(),
-                    kg: KnowledgeGraph {
+                    kg: Some(KnowledgeGraph {
                         automata_path: AutomataPath::local_example(),
                         input_type: KnowledgeGraphInputType::Markdown,
                         path: PathBuf::from("~/pkm"),
                         public: true,
                         publish: true,
-                    },
+                    }),
                     haystacks: vec![Haystack {
                         path: PathBuf::from("/tmp/system_operator/pages/"),
                         service: ServiceType::Ripgrep,
@@ -500,14 +485,13 @@ mod tests {
             name: "Father".to_string(),
             relevance_function: RelevanceFunction::TitleScorer,
             theme: "lumen".to_string(),
-            server_url: Url::parse("http://localhost:8080").unwrap(),
-            kg: KnowledgeGraph {
+            kg: Some(KnowledgeGraph {
                 automata_path: AutomataPath::local_example(),
                 input_type: KnowledgeGraphInputType::Markdown,
                 path: PathBuf::from("~/pkm"),
                 public: true,
                 publish: true,
-            },
+            }),
             haystacks: vec![Haystack {
                 path: PathBuf::from("localsearch"),
                 service: ServiceType::Ripgrep,

crates/terraphim_middleware/src/thesaurus/mod.rs

@@ -70,7 +70,7 @@ pub async fn build_thesaurus_from_haystack(
         tokio::fs::write(&thesaurus_path, thesaurus_json).await?;
         log::debug!("Thesaurus written to {:#?}", thesaurus_path);
 
-        role.kg.automata_path = AutomataPath::Local(thesaurus_path);
+        role.kg.as_mut().unwrap().automata_path = AutomataPath::Local(thesaurus_path);
     }
     Ok(())
 }

crates/terraphim_rolegraph/src/lib.rs

@@ -197,7 +197,6 @@ impl RoleGraph {
 
         let mut ranked_documents = results.into_iter().collect::<Vec<_>>();
         ranked_documents.sort_by_key(|(_, doc)| std::cmp::Reverse(doc.rank));
-        ranked_documents.sort_by_key(|(_, doc)| std::cmp::Reverse(doc.id.clone()));
 
         let documents: Vec<_> = ranked_documents
             .into_iter()

crates/terraphim_service/src/lib.rs

@@ -74,24 +74,28 @@ impl<'a> TerraphimService {
         match role.relevance_function {
             RelevanceFunction::TitleScorer => {
                 log::debug!("Searching haystack with title scorer");
-                let indexed_docs: Vec<IndexedDocument> = self
-                    .config_state
-                    .search_indexed_documents(search_query)
-                    .await;
 
-                let documents = index.get_documents(indexed_docs);
-                // Sort the documents by relevance
+                let documents = index.get_all_documents();
+
                 log::debug!("Sorting documents by relevance");
+                // Sort the documents by relevance
                 let documents = score::sort_documents(search_query, documents);
-
-                Ok(documents)
+                let total_length = documents.len();
+                let mut docs_ranked = Vec::new();
+                for (idx, doc) in documents.iter().enumerate() {
+                    let document: &mut terraphim_types::Document = &mut doc.clone();
+                    let rank = terraphim_types::Rank::new((total_length - idx).try_into().unwrap());
+                    document.rank = Some(rank);
+                    docs_ranked.push(document.clone());
+                }
+                Ok(docs_ranked)
             }
             RelevanceFunction::TerraphimGraph => {
                 self.build_thesaurus(search_query).await?;
 
                 let scored_index_docs: Vec<IndexedDocument> = self
                     .config_state
-                    .search_indexed_documents(search_query)
+                    .search_indexed_documents(search_query, &role)
                     .await;
 
                 // Apply to ripgrep vector of document output

crates/terraphim_service/src/score/mod.rs

@@ -28,9 +28,9 @@ pub fn sort_documents(search_query: &SearchQuery, documents: Vec<Document>) -> V
 
     // Score the documents
     let mut results = scorer.score(&query, documents).unwrap();
-
+    //FIXME: results.score and rank solve the same purpose. Results score shall be normalized into rank and mapped to IndexedDocument
     results.rescore(|doc| query.similarity.similarity(&query.name, &doc.title));
-
+    log::debug!("Rescore results {:#?}", results);
     results
         .into_vec()
         .iter()

crates/terraphim_types/src/lib.rs

@@ -439,6 +439,11 @@ impl Index {
         }
         documents
     }
+    /// Returns all documents from the index for scorer without graph embeddings
+    pub fn get_all_documents(&self) -> Vec<Document> {
+        let documents: Vec<Document> = self.values().cloned().collect::<Vec<Document>>();
+        documents
+    }
 
     /// Get a document from the index (if it exists in the index)
     pub fn get_document(&self, doc: &IndexedDocument) -> Option<Document> {

desktop/scripts/yarn_and_build.sh

@@ -1,7 +1,9 @@
-!/usr/bin/env bash
+current_dir=${PWD}
+echo "Current folder ${current_dir}"
 # this is to install and build front end inside bionic
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
 bash -c "source $HOME/.nvm/nvm.sh && nvm install 16.15.1"
 bash -c "source $HOME/.nvm/nvm.sh && npm install -g yarn"
-bash -c "source $HOME/.nvm/nvm.sh && cd /code/desktop && yarn && yarn build"
-cp -Rv /code/desktop/dist /code/terraphim_server/
+bash -c "source $HOME/.nvm/nvm.sh && cd ${current_dir} && yarn && yarn build"
+echo "${current_dir}/dist ${current_dir}/../terraphim_server/"
+cp -Rv ${current_dir}/dist ${current_dir}/../terraphim_server/

desktop/src-tauri/src/config.rs

@@ -1,7 +1,6 @@
 use std::path::PathBuf;
 
 use ahash::AHashMap;
-use tauri::Url;
 use terraphim_automata::AutomataPath;
 use terraphim_config::{
     Config, ConfigBuilder, Haystack, KnowledgeGraph, Role, ServiceType, TerraphimConfigError,
@@ -11,7 +10,8 @@ use terraphim_types::{KnowledgeGraphInputType, RelevanceFunction};
 /// The path to the default haystack directory
 // TODO: Replace this with a file-based config loader based on `twelf` in the
 // future
-const DEFAULT_HAYSTACK_PATH: &str = "../../docs/";
+// const DEFAULT_HAYSTACK_PATH: &str = "docs/src/";
+const DEFAULT_HAYSTACK_PATH: &str = "terraphim_server/fixtures";
 
 /// Load the default config
 ///
@@ -22,7 +22,10 @@ pub(crate) fn load_config() -> Result<Config, TerraphimConfigError> {
 
     // Create the path to the default haystack directory
     // by concating the current directory with the default haystack path
-    let docs_path = std::env::current_dir()?.join(DEFAULT_HAYSTACK_PATH);
+    let mut docs_path = std::env::current_dir().unwrap();
+    docs_path.pop();
+    docs_path.pop();
+    docs_path = docs_path.join(DEFAULT_HAYSTACK_PATH);
     println!("Docs path: {:?}", docs_path);
 
     ConfigBuilder::new()
@@ -34,14 +37,7 @@ pub(crate) fn load_config() -> Result<Config, TerraphimConfigError> {
                 name: "Default".to_string(),
                 relevance_function: RelevanceFunction::TitleScorer,
                 theme: "spacelab".to_string(),
-                server_url: Url::parse("http://localhost:8000/documents/search").unwrap(),
-                kg: KnowledgeGraph {
-                    automata_path: automata_path.clone(),
-                    input_type: KnowledgeGraphInputType::Markdown,
-                    path: PathBuf::from("~/pkm"),
-                    public: true,
-                    publish: true,
-                },
+                kg: None,
                 haystacks: vec![Haystack {
                     path: docs_path.clone(),
                     service: ServiceType::Ripgrep,
@@ -56,14 +52,7 @@ pub(crate) fn load_config() -> Result<Config, TerraphimConfigError> {
                 name: "Engineer".to_string(),
                 relevance_function: RelevanceFunction::TitleScorer,
                 theme: "lumen".to_string(),
-                server_url: Url::parse("http://localhost:8000/documents/search").unwrap(),
-                kg: KnowledgeGraph {
-                    automata_path: automata_path.clone(),
-                    input_type: KnowledgeGraphInputType::Markdown,
-                    path: PathBuf::from("~/pkm"),
-                    public: true,
-                    publish: true,
-                },
+                kg: None,
                 haystacks: vec![Haystack {
                     path: docs_path.clone(),
                     service: ServiceType::Ripgrep,
@@ -78,14 +67,13 @@ pub(crate) fn load_config() -> Result<Config, TerraphimConfigError> {
                 name: "System Operator".to_string(),
                 relevance_function: RelevanceFunction::TitleScorer,
                 theme: "superhero".to_string(),
-                server_url: Url::parse("http://localhost:8000/documents/search").unwrap(),
-                kg: KnowledgeGraph {
+                kg: Some(KnowledgeGraph {
                     automata_path,
                     input_type: KnowledgeGraphInputType::Markdown,
                     path: PathBuf::from("~/pkm"),
                     public: true,
                     publish: true,
-                },
+                }),
                 haystacks: vec![Haystack {
                     path: docs_path,
                     service: ServiceType::Ripgrep,

desktop/src/lib/Search/ResultItem.svelte

@@ -28,15 +28,21 @@
     <div class="media-content">
       <div class="content">
         <div class="level-right">
+          {#if document.tags}
           <Taglist>
-            {#if "tags" in document}
               {#each document.tags as tag}
+              <!-- FIXME: link shall be config parameter for KG -->
                 <a
                   href="https://terraphim.github.io/terraphim-project/#/page/{tag}"
                   target="_blank"><Tag rounded>{tag}</Tag></a
                 >
               {/each}
-            {/if}
+          </Taglist>
+          {/if}
+        </div>
+          <div class="level-right">
+          <Taglist>
+            <Tag rounded>Rank {document.rank}</Tag>
           </Taglist>
         </div>
         <div transition:fade>

desktop/src/lib/Search/Search.svelte

@@ -88,8 +88,8 @@
 {#if error}
   <p class="error">{error}</p>
 {:else if results.length}
-  {#each results as result_item}
-    <ResultItem document={result_item} />
+  {#each results as item}
+    <ResultItem document={item} />
   {/each}
 {:else}
   <section class="section">