feat(db/filter): handle instance tree for 'is' claim

TODO.md

@@ -244,3 +244,17 @@ First attempt at test coverage.
 
 - support for description in a second opening bracket couple {}
 - @Entry {/* claims */}{/* free markdown description with recursive entries */}
+
+TODO: fix the bug when parsing markdown (multiple entries), when multiples entries are next to the next one (are tightly packed together without whitespace)
+
+Target for v0.1 : being able to fully parse my personal main notebook.
+for that we need to :
+- fix all the little parsing bugs
+- being able to skip the entries that have errors
+- have better reporting of errors, easier to understand and to spot
+- support for custom entry intro
+- support for nested entries in the description and in the claim value
+
+for v0.2:
+- have basic LSP that can be used to jump to definition of reference.
+- have

src/lib/cli/list.rs

@@ -15,10 +15,7 @@ pub fn get_list(mut output: impl FeedbackChannel, context: &NotebookContext) ->
 
     // output a summary of what inside the notebook
     for entry_container in &notebook.entries {
-        let rel_path = entry_container.source_file(&notebook)
+        let rel_path = &entry_container.source_file(&notebook).path;
-            .path
-            .strip_prefix(&context.sources_path())
-            .expect("Cannot strip out base path from source file");
         output.push_answer(
             &format!("{:?} - {:?} - {:?}",
                 rel_path,

src/lib/database/search.rs

@@ -3,6 +3,8 @@ use textdistance::str::damerau_levenshtein;
 use anyhow::{anyhow, Context, Result};
 use crate::database::parse_extractor::get_pure_claims;
 
+use super::{filter::filter_instance_of, models::{EntryValue, Property, Reference, ResolvedReference}};
+
 // the best score is zero
 pub fn search_entry_by_label<'a>(notebook: &'a Notebook, label_query: &str) -> Vec<(usize, &'a EntryContainer)> {
     let mut results: Vec<(usize, &EntryContainer)> = vec![];
@@ -21,6 +23,7 @@ pub fn search_complex<'a>(notebook: &'a Notebook, complex_query: &str) -> Result
     // we reparse complex query as entries claims, but we will need something to later associate
     // for now we just do a AND on all the claims
     // TODO: parse more complex query languages with closes and expressions
+    // TODO: make use filter_instance_of when you have "is"
 
     let parsed_claims = crate::pdel_parser::parse_claims_query(&complex_query)
         .map_err(|_e| anyhow!("Could not parse claims query"))?;
@@ -31,13 +34,49 @@ pub fn search_complex<'a>(notebook: &'a Notebook, complex_query: &str) -> Result
     // link
     let _link_count = link_claims(&notebook.labels_index, &mut query_claims);
 
+    let mut query_results: Vec<&EntryContainer> = vec![];
+    
     // basic claims AND filter
-    Ok(notebook.entries.iter().filter(|entry_container| {
+    let base_results = notebook.entries.iter().filter(|entry_container| {
         query_claims.iter().all(|qc| {
            entry_container.pure_entry.claims
                 .iter().any(|ec| {
                     qc.property == ec.property && qc.value == ec.value
                 })
         })
-    }).collect())
+    });
+    for r in base_results {
+        if query_results.iter().any(|qr| qr.pure_entry.id == r.pure_entry.id) {
+            continue;
+        }
+        query_results.push(r);
+    }
+    dbg!(&query_results);
+    
+    for qc in &query_claims {
+        // additional resolution to handle instance_of prop
+        if qc.property == Property::Custom("is".into()) {
+            let target_id = match &qc.value {
+                EntryValue::Reference(r) => {
+                    match r {
+                        Reference::Unresolved(_u) => {
+                            return Err(anyhow!("Found unresolved references in query claims"));
+                        },
+                        Reference::Resolved(resolved) => match resolved {
+                            ResolvedReference::InternalReference(id) => id,
+                            _ => { continue; }
+                        }
+                    }
+                },
+                _ => { continue; }
+            };
+            for r in filter_instance_of(notebook, &target_id) {
+                if query_results.iter().any(|qr| qr.pure_entry.id == r.pure_entry.id) {
+                    continue;
+                }
+                query_results.push(r);
+            }
+        }
+    }
+    Ok(query_results)
 }

src/lib/database/test_filter.rs

@@ -1,51 +1,30 @@
-use std::path::PathBuf;
+use std::assert_matches::assert_matches;
-use uuid::Uuid;
 
-use crate::{database::ids::Id, pdel_parser::parse_entry};
+use crate::database::ids::Id;
 
-use super::{models::{Entry, EntryContainer, Notebook, SourceFile}, search::search_complex};
+use super::{models::{Manifest, Notebook}, search::search_complex};
-
-fn build_test_notebook() -> Notebook {
-    // TODO: we need to have an handy fonction to get a notebook from a virtual notebook files ad-hoc.
-    // or we can skip the take from the filesystem and take from Pure Entry directly
-    // and so we can let the core inner lib manage the backlink build and all index build
-    let sf = SourceFile {
-        id: Uuid::new_v4(),
-        path: PathBuf::from("./hello"),
-        expansions: vec![]
-    };
-    let entry_code = r#"
-        @Entry {
-            id: "AAAAAAAAAA",
-            is: [Bike],
-            mass: 15, 
-            name: "Le super vélo"
-        }
-    "#;
-    let parsed_entry = parse_entry(entry_code, 0).unwrap();
-    let pure_entry = Entry::from_parsed_entry_with_id(&parsed_entry).unwrap();
-    let ec = EntryContainer {
-        parsed_entry, 
-        pure_entry,
-        source_file_id: sf.id,
-    };
-
-    Notebook::build(
-        super::models::Manifest {
-            name: "hello".to_string(),
-            description: "hello".to_string(),
-        },
-        vec![sf],
-        vec![ec]
-    ).unwrap()
-}
 
 #[test]
-fn test_filter_by_claims() {
+fn test_filter_by_single_claim() {
-    let notebook = build_test_notebook();
+    let entries_code = r#"
-    assert_eq!(notebook.entries.len(), 1);
+        @Entry {
+            id: "AAAAAAAAAA",
+            foo: "bar"
+        }
+        @Entry {
+            id: "AAAAAAAAAB",
+            foo: "another_value"
+        }
+    "#;
+    let notebook_res = Notebook::build_virtual(
+        Manifest::example(),
+        entries_code
+    );
+    assert_matches!(notebook_res, Ok(_));
+    let notebook = notebook_res.unwrap();
+    assert_eq!(notebook.entries.len(), 2);
 
-    let search_res = search_complex(&notebook, "{ is: [Bike] }");
+    let search_res = search_complex(&notebook, r#"{ foo: "bar" }"#);
     let found_entries = search_res.unwrap();
 
     assert_eq!(found_entries.len(), 1);
@@ -57,9 +36,27 @@ fn test_filter_by_claims() {
 
 #[test]
 fn test_filter_by_claims_and() {
-    let notebook = build_test_notebook();
+    let entries_code = r#"
-
+        @Entry {
-    assert_eq!(notebook.entries.len(), 1);
+            id: "AAAAAAAAAA",
+            name: "Bike"
+        }
+        @Entry {
+            id: "AAAAAAAAAC",
+            is: [Bike],
+            mass: 15
+        }
+        @Entry {
+            id: "AAAAAAAAAB",
+            foo: "another_value"
+        }
+    "#;
+    let notebook_res = Notebook::build_virtual(
+        Manifest::example(),
+        entries_code
+    );
+    assert_matches!(notebook_res, Ok(_));
+    let notebook = notebook_res.unwrap();
 
     let search_res = search_complex(&notebook, "{ is: [Bike], mass: 15 }");
     let found_entries = search_res.unwrap();
@@ -67,6 +64,62 @@ fn test_filter_by_claims_and() {
     assert_eq!(found_entries.len(), 1);
     assert_eq!(
         found_entries.get(0).unwrap().pure_entry.id,
-        Id::from_repr("AAAAAAAAAA").unwrap()
+        Id::from_repr("AAAAAAAAAC").unwrap()
     )
 }
+
+#[test]
+fn test_filter_by_claims_includes_tree_subs() {
+    // we define Event as the root of our Tree,
+    // The filter by Event must all leaf children that inherit from Event
+    let entries_code = r#"
+        @Entry {
+            id: "AAAAAAAAA2",
+            name: "Event"
+        }
+        @Entry {
+            id: "AAAAAAAAAB",
+            sub: [Event],
+            name: "Proclamation"
+        }
+        @Entry {
+            id: "AAAAAAAAAC",
+            sub: [Event],
+            name: "Inauguration"
+        }
+        @Entry {
+            id: "AAAAAAAAAD",
+            sub: [Event],
+            name: "Accident"
+        }
+        @Entry {
+            id: "AAAAAAAAAE",
+            sub: [Accident],
+            name: "Traffic Accident"
+            description: "This is the item that represent the topic of a traffic accident"
+        }
+        @Entry {
+            id: "AAAAAAAAAF",
+            is: [Accident],
+            name: "Some precise accident"
+        }
+    "#;
+    let notebook_res = Notebook::build_virtual(
+        Manifest::example(),
+        entries_code
+    );
+    assert_matches!(notebook_res, Ok(_));
+    let notebook = notebook_res.unwrap();
+    assert_eq!(notebook.entries.len(), 6);
+
+    // for ec in &notebook.entries {
+    //     dbg!(&ec.pure_entry);
+    // }
+
+    let search_res = search_complex(&notebook, "{ is: [Event] }");
+    let found_entries = search_res.unwrap();
+
+    // dbg!(&found_entries);
+    assert_eq!(found_entries.len(), 1);
+}
+

src/lib/indexer/manifest.rs

@@ -29,4 +29,11 @@ impl Manifest {
 
         Ok(data)
     }
+       
+    pub fn example() -> Manifest {
+        Manifest {
+            name: "Example Notebook".to_string(),
+            description: "Notebook description".to_string(),
+        }
+    }
 }

src/lib/indexer/mod.rs

@@ -1,8 +1,8 @@
 use crate::database::ids::Id;
 use crate::database::labels::LabelEntryIndex;
 use crate::database::graph::BackLinksIndex;
-use crate::database::models::{filter_claims_by_property, Entry, EntryContainer, EntryValue, Manifest, Notebook, Property, SourceFile};
+use crate::database::models::{self, filter_claims_by_property, Entry, EntryContainer, EntryValue, Manifest, Notebook, Property, SourceFile};
-use crate::database::parse_extractor::get_pure_claims;
+use crate::database::parse_extractor::{get_pure_claims, ExtractErr};
 use crate::pdel_parser::preprocess::{apply_expansions, preprocess_code, AdaptableRange, Expansion};
 use crate::pdel_parser::{markdown::parse_markdown, parse_wrapper, PEntry, ParseOutput};
 use crate::unwrap_or_return;
@@ -14,7 +14,7 @@ use uuid::Uuid;
 use bincode::serialize;
 use anyhow::{anyhow, Result, Context};
 use notify::{Config, RecommendedWatcher, RecursiveMode, Watcher};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::io;
 use std::fs;
 // import the write trait (not directly used)
@@ -41,6 +41,7 @@ pub enum IndexingErr {
     CannotProcessCodeToAddId,
     CannotWriteFileToAddId(io::Error),
     ExtractErr(anyhow::Error),
+    RExtractErr(ExtractErr),
     ManifestErr(ScanManifestErr)
 }
 
@@ -247,7 +248,7 @@ fn save_db_in_fs(notebook: &Notebook, database_path: &Path) -> Result<(), Indexi
     let mut file = match fs::File::create(database_path) {
         Ok(res) => res,
         Err(e) => {
-            dbg!(e);
+            
             return Err(IndexingErr::IoErr);
         }
     };
@@ -363,5 +364,38 @@ impl Notebook {
             backlinks
         })
     }
+
+    /// build virtual notebook from one single markdown string
+    /// for now does not support macros
+    pub fn build_virtual(
+        manifest: Manifest,
+        markdown_code: &str
+    )-> Result<Notebook, IndexingErr> {
+        let markdown_val = parse_markdown(&markdown_code, 0).map_err(
+            |_e| IndexingErr::ParseErr
+        )?;
+        let sf = SourceFile {
+            id: Uuid::new_v4(),
+            path: PathBuf::from("./all_in_one_virtual.md"),
+            expansions: vec![]
+        };
+
+        let mut entries_containers: Vec<EntryContainer> = vec![];
+
+        for parsed_entry in markdown_val.tree.p {
+            entries_containers.push(EntryContainer {
+                pure_entry: Entry::from_parsed_entry_with_id(&parsed_entry)
+                    .map_err(|e| IndexingErr::RExtractErr(e))?,
+                parsed_entry,
+                source_file_id: sf.id
+            })
+        }
+
+        Notebook::build(
+            manifest,
+            vec![sf],
+            entries_containers
+        )
+    }
 }