popequer/popequer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Activity

feat(db/filter): handle instance tree for 'is' claim

Browse source
This commit is contained in:
Matthieu Bessat 2024-02-23 23:33:05 +01:00
parent c0727cb671
commit 5040b404f5
6 changed files with 201 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
TODO.md
View file

@ -244,3 +244,17 @@ First attempt at test coverage.
- support for description in a second opening bracket couple {}
- @Entry {/* claims */}{/* free markdown description with recursive entries */}
TODO: fix the bug when parsing markdown (multiple entries), when multiples entries are next to the next one (are tightly packed together without whitespace)
Target for v0.1 : being able to fully parse my personal main notebook.
for that we need to :
- fix all the little parsing bugs
- being able to skip the entries that have errors
- have better reporting of errors, easier to understand and to spot
- support for custom entry intro
- support for nested entries in the description and in the claim value
for v0.2:
- have basic LSP that can be used to jump to definition of reference.
- have

5
src/lib/cli/list.rs
View file

@ -15,10 +15,7 @@ pub fn get_list(mut output: impl FeedbackChannel, context: &NotebookContext) ->
// output a summary of what inside the notebook
for entry_container in &notebook.entries {
let rel_path = entry_container.source_file(&notebook)
.path
.strip_prefix(&context.sources_path())
.expect("Cannot strip out base path from source file");
let rel_path = &entry_container.source_file(&notebook).path;
output.push_answer(
&format!("{:?} - {:?} - {:?}",
rel_path,

43
src/lib/database/search.rs
View file

@ -3,6 +3,8 @@ use textdistance::str::damerau_levenshtein;
use anyhow::{anyhow, Context, Result};
use crate::database::parse_extractor::get_pure_claims;
use super::{filter::filter_instance_of, models::{EntryValue, Property, Reference, ResolvedReference}};
// the best score is zero
pub fn search_entry_by_label<'a>(notebook: &'a Notebook, label_query: &str) -> Vec<(usize, &'a EntryContainer)> {
let mut results: Vec<(usize, &EntryContainer)> = vec![];
@ -21,6 +23,7 @@ pub fn search_complex<'a>(notebook: &'a Notebook, complex_query: &str) -> Result
// we reparse complex query as entries claims, but we will need something to later associate
// for now we just do a AND on all the claims
// TODO: parse more complex query languages with closes and expressions
// TODO: make use filter_instance_of when you have "is"
let parsed_claims = crate::pdel_parser::parse_claims_query(&complex_query)
.map_err(|_e| anyhow!("Could not parse claims query"))?;
@ -31,13 +34,49 @@ pub fn search_complex<'a>(notebook: &'a Notebook, complex_query: &str) -> Result
// link
let _link_count = link_claims(&notebook.labels_index, &mut query_claims);
let mut query_results: Vec<&EntryContainer> = vec![];
// basic claims AND filter
Ok(notebook.entries.iter().filter(|entry_container| {
let base_results = notebook.entries.iter().filter(|entry_container| {
query_claims.iter().all(|qc| {
entry_container.pure_entry.claims
.iter().any(|ec| {
qc.property == ec.property && qc.value == ec.value
})
})
}).collect())
});
for r in base_results {
if query_results.iter().any(|qr| qr.pure_entry.id == r.pure_entry.id) {
continue;
}
query_results.push(r);
}
dbg!(&query_results);
for qc in &query_claims {
// additional resolution to handle instance_of prop
if qc.property == Property::Custom("is".into()) {
let target_id = match &qc.value {
EntryValue::Reference(r) => {
match r {
Reference::Unresolved(_u) => {
return Err(anyhow!("Found unresolved references in query claims"));
},
Reference::Resolved(resolved) => match resolved {
ResolvedReference::InternalReference(id) => id,
_ => { continue; }
}
}
},
_ => { continue; }
};
for r in filter_instance_of(notebook, &target_id) {
if query_results.iter().any(|qr| qr.pure_entry.id == r.pure_entry.id) {
continue;
}
query_results.push(r);
}
}
}
Ok(query_results)
}

147
src/lib/database/test_filter.rs
View file

@ -1,51 +1,30 @@
use std::path::PathBuf;
use uuid::Uuid;
use std::assert_matches::assert_matches;
use crate::{database::ids::Id, pdel_parser::parse_entry};
use crate::database::ids::Id;
use super::{models::{Entry, EntryContainer, Notebook, SourceFile}, search::search_complex};
fn build_test_notebook() -> Notebook {
// TODO: we need to have an handy fonction to get a notebook from a virtual notebook files ad-hoc.
// or we can skip the take from the filesystem and take from Pure Entry directly
// and so we can let the core inner lib manage the backlink build and all index build
let sf = SourceFile {
id: Uuid::new_v4(),
path: PathBuf::from("./hello"),
expansions: vec![]
};
let entry_code = r#"
@Entry {
id: "AAAAAAAAAA",
is: [Bike],
mass: 15,
name: "Le super vélo"
}
"#;
let parsed_entry = parse_entry(entry_code, 0).unwrap();
let pure_entry = Entry::from_parsed_entry_with_id(&parsed_entry).unwrap();
let ec = EntryContainer {
parsed_entry,
pure_entry,
source_file_id: sf.id,
};
Notebook::build(
super::models::Manifest {
name: "hello".to_string(),
description: "hello".to_string(),
},
vec![sf],
vec![ec]
).unwrap()
}
use super::{models::{Manifest, Notebook}, search::search_complex};
#[test]
fn test_filter_by_claims() {
let notebook = build_test_notebook();
assert_eq!(notebook.entries.len(), 1);
fn test_filter_by_single_claim() {
let entries_code = r#"
@Entry {
id: "AAAAAAAAAA",
foo: "bar"
}
@Entry {
id: "AAAAAAAAAB",
foo: "another_value"
}
"#;
let notebook_res = Notebook::build_virtual(
Manifest::example(),
entries_code
);
assert_matches!(notebook_res, Ok(_));
let notebook = notebook_res.unwrap();
assert_eq!(notebook.entries.len(), 2);
let search_res = search_complex(&notebook, "{ is: [Bike] }");
let search_res = search_complex(&notebook, r#"{ foo: "bar" }"#);
let found_entries = search_res.unwrap();
assert_eq!(found_entries.len(), 1);
@ -57,9 +36,27 @@ fn test_filter_by_claims() {
#[test]
fn test_filter_by_claims_and() {
let notebook = build_test_notebook();
assert_eq!(notebook.entries.len(), 1);
let entries_code = r#"
@Entry {
id: "AAAAAAAAAA",
name: "Bike"
}
@Entry {
id: "AAAAAAAAAC",
is: [Bike],
mass: 15
}
@Entry {
id: "AAAAAAAAAB",
foo: "another_value"
}
"#;
let notebook_res = Notebook::build_virtual(
Manifest::example(),
entries_code
);
assert_matches!(notebook_res, Ok(_));
let notebook = notebook_res.unwrap();
let search_res = search_complex(&notebook, "{ is: [Bike], mass: 15 }");
let found_entries = search_res.unwrap();
@ -67,6 +64,62 @@ fn test_filter_by_claims_and() {
assert_eq!(found_entries.len(), 1);
assert_eq!(
found_entries.get(0).unwrap().pure_entry.id,
Id::from_repr("AAAAAAAAAA").unwrap()
Id::from_repr("AAAAAAAAAC").unwrap()
)
}
#[test]
fn test_filter_by_claims_includes_tree_subs() {
// we define Event as the root of our Tree,
// The filter by Event must all leaf children that inherit from Event
let entries_code = r#"
@Entry {
id: "AAAAAAAAA2",
name: "Event"
}
@Entry {
id: "AAAAAAAAAB",
sub: [Event],
name: "Proclamation"
}
@Entry {
id: "AAAAAAAAAC",
sub: [Event],
name: "Inauguration"
}
@Entry {
id: "AAAAAAAAAD",
sub: [Event],
name: "Accident"
}
@Entry {
id: "AAAAAAAAAE",
sub: [Accident],
name: "Traffic Accident"
description: "This is the item that represent the topic of a traffic accident"
}
@Entry {
id: "AAAAAAAAAF",
is: [Accident],
name: "Some precise accident"
}
"#;
let notebook_res = Notebook::build_virtual(
Manifest::example(),
entries_code
);
assert_matches!(notebook_res, Ok(_));
let notebook = notebook_res.unwrap();
assert_eq!(notebook.entries.len(), 6);
// for ec in &notebook.entries {
// dbg!(&ec.pure_entry);
// }
let search_res = search_complex(&notebook, "{ is: [Event] }");
let found_entries = search_res.unwrap();
// dbg!(&found_entries);
assert_eq!(found_entries.len(), 1);
}

7
src/lib/indexer/manifest.rs
View file

@ -29,4 +29,11 @@ impl Manifest {
Ok(data)
}
pub fn example() -> Manifest {
Manifest {
name: "Example Notebook".to_string(),
description: "Notebook description".to_string(),
}
}
}

42
src/lib/indexer/mod.rs
View file

@ -1,8 +1,8 @@
use crate::database::ids::Id;
use crate::database::labels::LabelEntryIndex;
use crate::database::graph::BackLinksIndex;
use crate::database::models::{filter_claims_by_property, Entry, EntryContainer, EntryValue, Manifest, Notebook, Property, SourceFile};
use crate::database::parse_extractor::get_pure_claims;
use crate::database::models::{self, filter_claims_by_property, Entry, EntryContainer, EntryValue, Manifest, Notebook, Property, SourceFile};
use crate::database::parse_extractor::{get_pure_claims, ExtractErr};
use crate::pdel_parser::preprocess::{apply_expansions, preprocess_code, AdaptableRange, Expansion};
use crate::pdel_parser::{markdown::parse_markdown, parse_wrapper, PEntry, ParseOutput};
use crate::unwrap_or_return;
@ -14,7 +14,7 @@ use uuid::Uuid;
use bincode::serialize;
use anyhow::{anyhow, Result, Context};
use notify::{Config, RecommendedWatcher, RecursiveMode, Watcher};
use std::path::Path;
use std::path::{Path, PathBuf};
use std::io;
use std::fs;
// import the write trait (not directly used)
@ -41,6 +41,7 @@ pub enum IndexingErr {
CannotProcessCodeToAddId,
CannotWriteFileToAddId(io::Error),
ExtractErr(anyhow::Error),
RExtractErr(ExtractErr),
ManifestErr(ScanManifestErr)
}
@ -247,7 +248,7 @@ fn save_db_in_fs(notebook: &Notebook, database_path: &Path) -> Result<(), Indexi
let mut file = match fs::File::create(database_path) {
Ok(res) => res,
Err(e) => {
dbg!(e);
return Err(IndexingErr::IoErr);
}
};
@ -363,5 +364,38 @@ impl Notebook {
backlinks
})
}
/// build virtual notebook from one single markdown string
/// for now does not support macros
pub fn build_virtual(
manifest: Manifest,
markdown_code: &str
)-> Result<Notebook, IndexingErr> {
let markdown_val = parse_markdown(&markdown_code, 0).map_err(
|_e| IndexingErr::ParseErr
)?;
let sf = SourceFile {
id: Uuid::new_v4(),
path: PathBuf::from("./all_in_one_virtual.md"),
expansions: vec![]
};
let mut entries_containers: Vec<EntryContainer> = vec![];
for parsed_entry in markdown_val.tree.p {
entries_containers.push(EntryContainer {
pure_entry: Entry::from_parsed_entry_with_id(&parsed_entry)
.map_err(|e| IndexingErr::RExtractErr(e))?,
parsed_entry,
source_file_id: sf.id
})
}
Notebook::build(
manifest,
vec![sf],
entries_containers
)
}
}