add tag indexing with suggestions

.gitignore

@@ -2,3 +2,4 @@
 **/*.rs.bk
 .idea/
 data/data.json
+data/index.json

data/repository.yaml

@@ -9,3 +9,5 @@ model:
     alive:
       type: "bool"
       default: true
+    tags:
+      type: "free_tags"

src/main.rs

@@ -14,13 +14,11 @@ use rocket_contrib::templates::Template;
 
 mod store;
 
-use crate::store::form::{render_card_fields, render_empty_fields, RenderedCard, RenderedField};
-use crate::store::model::FieldKind;
+use crate::store::form::{render_card_fields, render_empty_fields, RenderedCard, RenderedField, MapFromForm, collect_card_form};
 use crate::store::Store;
-use indexmap::map::IndexMap;
 use parking_lot::RwLock;
 
-use rocket::request::{Form, FormItems, FromForm};
+use rocket::request::Form;
 use rocket::response::Redirect;
 use rocket::State;
 use std::env;
@@ -33,7 +31,7 @@ pub struct ListContext<'a> {
     pub pages: usize,
 }
 
-const per_page: usize = 20;
+const PER_PAGE: usize = 20; // TODO configurable
 
 fn find_page_with_card(store: &Store, card_id: usize) -> Option<usize> {
     if let Some((n, _)) = store
@@ -43,22 +41,18 @@ fn find_page_with_card(store: &Store, card_id: usize) -> Option<usize> {
         .enumerate()
         .find(|(_n, (id, _card))| **id == card_id)
     {
-        Some(n / per_page)
+        Some(n / PER_PAGE)
     } else {
         None
     }
 }
 
-#[get("/?<page>&<card>")]
-fn route_index(store: State<RwLock<Store>>, page: Option<usize>, card: Option<usize>) -> Template {
+#[get("/?<page>")]
+fn route_index(store: State<RwLock<Store>>, page: Option<usize>) -> Template {
     let rg = store.read();
 
     let mut page = page.unwrap_or_default();
-    let n_pages = (rg.data.cards.len() as f64 / per_page as f64).ceil() as usize;
-
-    if let Some(card_id) = card {
-        page = find_page_with_card(&rg, card_id).unwrap_or(page);
-    }
+    let n_pages = (rg.data.cards.len() as f64 / PER_PAGE as f64).ceil() as usize;
 
     if page >= n_pages {
         page = n_pages - 1;
@@ -72,8 +66,8 @@ fn route_index(store: State<RwLock<Store>>, page: Option<usize>, card: Option<us
             .data
             .cards
             .iter()
-            .skip(page * per_page)
-            .take(per_page)
+            .skip(page * PER_PAGE)
+            .take(PER_PAGE)
             .filter_map(|(id, card)| {
                 if let Value::Object(ref map) = card {
                     Some(RenderedCard {
@@ -90,123 +84,6 @@ fn route_index(store: State<RwLock<Store>>, page: Option<usize>, card: Option<us
     Template::render("index", context)
 }
 
-#[derive(Default)]
-struct MapFromForm {
-    pub data: IndexMap<String, String>,
-}
-
-impl<'a> FromForm<'a> for MapFromForm {
-    type Error = ();
-
-    fn from_form(items: &mut FormItems, _strict: bool) -> Result<Self, Self::Error> {
-        let mut new = MapFromForm::default();
-        items.for_each(|item| {
-            let (k, v) = item.key_value_decoded();
-            new.data.insert(k, v);
-        });
-        Ok(new)
-    }
-}
-
-fn collect_card_form(store: &Store, mut form: MapFromForm) -> IndexMap<String, Value> {
-    let mut card = IndexMap::new();
-
-    for (k, field) in &store.model.fields {
-        let mut value: Option<Value> = None;
-        if let Some(input) = form.data.remove(k) {
-            value = Some(match &field.kind {
-                FieldKind::Text | FieldKind::String | FieldKind::FreeEnum { .. } => {
-                    Value::String(input)
-                }
-                FieldKind::Bool { .. } => serde_json::to_value(true).unwrap(),
-                FieldKind::Int { min, max, default } => {
-                    let mut val: i64 = if input.is_empty() {
-                        *default
-                    } else {
-                        input.parse().expect("Error parse number")
-                    };
-
-                    if let Some(min) = min {
-                        val = val.max(*min);
-                    }
-
-                    if let Some(max) = max {
-                        val = val.min(*max);
-                    }
-
-                    serde_json::to_value(val).unwrap()
-                }
-                FieldKind::Float { min, max, default } => {
-                    let mut val: f64 = if input.is_empty() {
-                        *default
-                    } else {
-                        input.parse().expect("Error parse number")
-                    };
-
-                    if let Some(min) = min {
-                        val = val.min(*min);
-                    }
-
-                    if let Some(max) = max {
-                        val = val.max(*max);
-                    }
-
-                    serde_json::to_value(val).unwrap()
-                }
-                FieldKind::Enum { options, default } => {
-                    if options.contains(&input) {
-                        Value::String(input)
-                    } else {
-                        let val = default.as_ref().map(ToOwned::to_owned).unwrap_or_else(|| {
-                            options
-                                .first()
-                                .expect("fixed enum must have values")
-                                .to_owned()
-                        });
-
-                        Value::String(val)
-                    }
-                }
-                FieldKind::Tags { options } => {
-                    let tags: Vec<String> = input
-                        .split(' ')
-                        .map(ToOwned::to_owned)
-                        .filter_map(|tag| {
-                            if options.contains(&tag) {
-                                Some(tag)
-                            } else {
-                                None
-                            }
-                        })
-                        .collect();
-
-                    serde_json::to_value(tags).unwrap()
-                }
-                FieldKind::FreeTags { .. } => {
-                    let tags: Vec<String> = input
-                        .split(' ')
-                        .map(str::trim)
-                        .filter(|s| !s.is_empty())
-                        .map(ToOwned::to_owned)
-                        .collect();
-
-                    serde_json::to_value(tags).unwrap()
-                }
-            });
-        } else {
-            if let FieldKind::Bool { .. } = field.kind {
-                value = Some(Value::Bool(false));
-            }
-        }
-
-        if let Some(v) = value {
-            card.insert(k.to_owned(), v);
-        }
-    }
-
-    card
-}
-
 #[derive(Serialize)]
 struct AddCardContext<'a> {
     pub fields: Vec<RenderedField<'a>>,
@@ -231,12 +108,13 @@ fn route_add(store: State<RwLock<Store>>) -> Template {
 
 #[post("/add", data = "<form>")]
 fn route_add_save(form: Form<MapFromForm>, store: State<RwLock<Store>>) -> Redirect {
-    let mut rg = store.write();
+    let mut wg = store.write();
 
-    let card = collect_card_form(&rg, form.into_inner());
-    let id = rg.add_card(card);
+    let card = collect_card_form(&wg, form.into_inner());
+    let id = wg.add_card(card);
 
-    Redirect::found(uri!(route_index: page=_, card=id))
+    let page = find_page_with_card(&wg, id).unwrap_or(0);
+    Redirect::found(uri!(route_index: page))
 }
 
 #[get("/edit/<id>")]
@@ -261,23 +139,32 @@ fn route_edit_save(
     form: Form<MapFromForm>,
     store: State<RwLock<Store>>,
 ) -> Option<Redirect> {
-    let mut rg = store.write();
+    let mut wg = store.write();
 
-    let card = collect_card_form(&rg, form.into_inner());
-    rg.update_card(id, card);
+    let card = collect_card_form(&wg, form.into_inner());
+    wg.update_card(id, card);
 
-    Some(Redirect::found(uri!(route_index: page=_, card=id)))
+    let page = find_page_with_card(&wg, id).unwrap_or(0);
+    Some(Redirect::found(uri!(route_index: page)))
 }
 
 #[get("/delete/<id>")]
 fn route_delete(id: usize, store: State<RwLock<Store>>) -> Redirect {
-    let mut rg = store.write();
+    let mut wg = store.write();
 
-    let page = find_page_with_card(&rg, id).unwrap_or(0);
+    // must find page before deleting
+    let page = find_page_with_card(&wg, id).unwrap_or(0);
+    wg.delete_card(id);
 
-    rg.delete_card(id);
+    Redirect::found(uri!(route_index: page))
+}
 
-    Redirect::found(uri!(route_index: page=page, card=_))
+#[get("/maintenance/reindex")]
+fn route_reindex(store: State<RwLock<Store>>) -> Redirect {
+    let mut wg = store.write();
+    wg.rebuild_indexes();
+    wg.persist();
+    Redirect::found(uri!(route_index: _))
 }
 
 fn main() {
@@ -298,6 +185,7 @@ fn main() {
                 route_edit,
                 route_edit_save,
                 route_delete,
+                route_reindex,
             ],
         )
         .launch();

src/store/form.rs

@@ -2,12 +2,15 @@ use crate::store::model::FieldKind;
 use crate::store::{model, Indexes, Store};
 use serde_json::Value;
 use std::borrow::Cow;
+use std::collections::BTreeSet;
 
 use lazy_static::lazy_static;
+use indexmap::map::IndexMap;
+use rocket::request::{FormItems, FromForm};
 
 lazy_static! {
     /// This is an example for using doc comment attributes
-    static ref EMPTY_VEC: Vec<String> = vec![];
+    static ref EMPTY_SET: BTreeSet<String> = Default::default();
 }
 
 #[derive(Serialize, Debug, Default)]
@@ -20,7 +23,7 @@ pub struct RenderedField<'a> {
     pub max: String,
     pub all_tags_json: String,
     pub tags_json: String,
-    pub options: Option<&'a Vec<String>>,
+    pub options: Option<Vec<&'a String>>,
     pub value: Cow<'a, str>,
     pub checked: bool,
 }
@@ -98,7 +101,7 @@ impl<'a> RenderedField<'a> {
             }
             FieldKind::Enum { options, default } => {
                 rendered.kind = "select";
-                rendered.options = Some(options);
+                rendered.options = Some(options.iter().collect());
 
                 if let Some(Value::String(s)) = value {
                     rendered.value = Cow::Borrowed(&s.as_str());
@@ -109,21 +112,21 @@ impl<'a> RenderedField<'a> {
             FieldKind::FreeEnum { enum_group } => {
                 rendered.kind = "free_select";
                 let group = enum_group.as_ref().unwrap_or(key);
-                let options = index.free_enums.get(group).unwrap_or(&EMPTY_VEC);
-                rendered.options = Some(options);
+                let options = index.free_enums.get(group).unwrap_or(&EMPTY_SET);
+                rendered.options = Some(options.iter().collect());
             }
             FieldKind::Tags { options } => {
                 rendered.kind = "tags";
-                rendered.options = Some(options);
+                rendered.options = Some(options.iter().collect());
 
                 rendered.all_tags_json = serde_json::to_string(options).unwrap();
             }
             FieldKind::FreeTags { tag_group } => {
                 rendered.kind = "free_tags";
                 let group = tag_group.as_ref().unwrap_or(key);
-                let options = index.free_tags.get(group).unwrap_or(&EMPTY_VEC);
+                let options = index.free_tags.get(group).unwrap_or(&EMPTY_SET);
 
-                rendered.options = Some(options);
+                rendered.options = Some(options.iter().collect());
                 rendered.all_tags_json = "[]".into();
             }
         }
@@ -190,3 +193,125 @@ pub fn render_card_fields<'a>(
         })
         .collect()
 }
+
+
+#[derive(Default)]
+pub struct MapFromForm {
+    pub data: IndexMap<String, String>,
+}
+
+impl<'a> FromForm<'a> for MapFromForm {
+    type Error = ();
+
+    fn from_form(items: &mut FormItems, _strict: bool) -> Result<Self, Self::Error> {
+        let mut new = MapFromForm::default();
+        items.for_each(|item| {
+            let (k, v) = item.key_value_decoded();
+            new.data.insert(k, v);
+        });
+        Ok(new)
+    }
+}
+
+pub fn collect_card_form(store: &Store, mut form: MapFromForm) -> IndexMap<String, Value> {
+    let mut card = IndexMap::new();
+
+    for (k, field) in &store.model.fields {
+        let mut value: Option<Value> = None;
+        if let Some(input) = form.data.remove(k) {
+            value = Some(match &field.kind {
+                FieldKind::Text | FieldKind::String | FieldKind::FreeEnum { .. } => {
+                    Value::String(input)
+                }
+                FieldKind::Bool { .. } => serde_json::to_value(true).unwrap(),
+                FieldKind::Int { min, max, default } => {
+                    let mut val: i64 = if input.is_empty() {
+                        *default
+                    } else {
+                        input.parse().expect("Error parse number")
+                    };
+
+                    if let Some(min) = min {
+                        val = val.max(*min);
+                    }
+
+                    if let Some(max) = max {
+                        val = val.min(*max);
+                    }
+
+                    serde_json::to_value(val).unwrap()
+                }
+                FieldKind::Float { min, max, default } => {
+                    let mut val: f64 = if input.is_empty() {
+                        *default
+                    } else {
+                        input.parse().expect("Error parse number")
+                    };
+
+                    if let Some(min) = min {
+                        val = val.min(*min);
+                    }
+
+                    if let Some(max) = max {
+                        val = val.max(*max);
+                    }
+
+                    serde_json::to_value(val).unwrap()
+                }
+                FieldKind::Enum { options, default } => {
+                    if options.contains(&input) {
+                        Value::String(input)
+                    } else {
+                        let val = default.as_ref().map(ToOwned::to_owned).unwrap_or_else(|| {
+                            options
+                                .first()
+                                .expect("fixed enum must have values")
+                                .to_owned()
+                        });
+
+                        Value::String(val)
+                    }
+                }
+                FieldKind::Tags { options } => {
+                    let mut tags: Vec<String> = input
+                        .split(' ')
+                        .map(ToOwned::to_owned)
+                        .filter_map(|tag| {
+                            if options.contains(&tag) {
+                                Some(tag)
+                            } else {
+                                None
+                            }
+                        })
+                        .collect();
+
+                    tags.sort();
+
+                    serde_json::to_value(tags).unwrap()
+                }
+                FieldKind::FreeTags { .. } => {
+                    let mut tags: Vec<String> = input
+                        .split(' ')
+                        .map(str::trim)
+                        .filter(|s| !s.is_empty())
+                        .map(ToOwned::to_owned)
+                        .collect();
+
+                    tags.sort();
+
+                    serde_json::to_value(tags).unwrap()
+                }
+            });
+        } else {
+            if let FieldKind::Bool { .. } = field.kind {
+                value = Some(Value::Bool(false));
+            }
+        }
+
+        if let Some(v) = value {
+            card.insert(k.to_owned(), v);
+        }
+    }
+
+    card
+}

src/store/mod.rs

@@ -1,4 +1,4 @@
-use crate::store::model::Model;
+use crate::store::model::{Model, FieldKind};
 use indexmap::map::IndexMap;
 use serde::Serialize;
 use serde_json::Value;
@@ -6,7 +6,8 @@ use std::collections::HashMap;
 use std::fs::{File, OpenOptions};
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
+use json_dotpath::DotPaths;
 
 pub mod form;
 pub mod model;
@@ -16,6 +17,7 @@ pub mod model;
 pub struct Store {
     path: PathBuf,
     pub model: Model,
+    freeform_fields : FreeformFieldsOfInterest,
     pub data: Cards,
     pub index: Indexes,
 }
@@ -23,8 +25,8 @@ pub struct Store {
 /// Indexes loaded from the indexes file
 #[derive(Serialize, Deserialize, Debug, Default)]
 pub struct Indexes {
-    pub free_enums: HashMap<String, Vec<String>>,
-    pub free_tags: HashMap<String, Vec<String>>,
+    pub free_enums: HashMap<String, BTreeSet<String>>,
+    pub free_tags: HashMap<String, BTreeSet<String>>,
 }
 
 /// Struct loaded from the repositroy config file
@@ -57,23 +59,30 @@ impl Store {
 
         Store {
             path: path.as_ref().into(),
+            freeform_fields: Self::get_fields_for_freeform_indexes(&repository_config.model),
             model: repository_config.model,
             data: serde_json::from_str(&items).expect("Error parsing data file."),
             index: serde_json::from_str(&indexes).unwrap_or_default(),
         }
     }
 
+    /// Handle a data change.
+    /// If a card was modified, selectively update the tags index
+    fn on_change(&mut self, changed_card : Option<usize>) {
+        if let Some(id) = changed_card {
+            // this needs to be so ugly because of lifetimes - we need a mutable reference to the index
+            Self::index_card(&mut self.index, &self.freeform_fields, self.data.cards.get(&id).unwrap());
+        }
+
+        self.persist();
+    }
+
     pub fn persist(&mut self) {
-        let mut file = OpenOptions::new()
-            .write(true)
-            .create(true)
-            .truncate(true)
-            .open(self.path.join(REPO_DATA_FILE))
-            .expect("Error opening data file for writing.");
+        let data_json = serde_json::to_string_pretty(&self.data).expect("Error serialize data");
+        write_file(self.path.join(REPO_DATA_FILE), data_json.as_bytes());
 
-        let serialized = serde_json::to_string_pretty(&self.data).expect("Error serialize.");
-        file.write(serialized.as_bytes())
-            .expect("Error write data file");
+        let index_json = serde_json::to_string_pretty(&self.index).expect("Error serialize index");
+        write_file(self.path.join(REPO_INDEX_FILE), index_json.as_bytes());
     }
 
     pub fn add_card(&mut self, values: IndexMap<String, Value>) -> usize {
@@ -82,7 +91,7 @@ impl Store {
             let id = self.data.counter;
             self.data.counter += 1;
             self.data.cards.insert(id, p);
-            self.persist();
+            self.on_change(Some(id));
             id
         } else {
             panic!("Packing did not produce a map.");
@@ -102,14 +111,97 @@ impl Store {
             panic!("Packing did not produce a map.");
         }
 
-        self.persist()
+        self.on_change(Some(id))
     }
 
     pub fn delete_card(&mut self, id: usize) {
         self.data.cards.remove(&id);
 
-        self.persist()
+        self.on_change(None)
+    }
+
+    /// Get a list of free_tags and free_enum fields
+    ///
+    /// Returns (free_tags, free_enums), where both members are vecs of (field_key, index_group)
+    fn get_fields_for_freeform_indexes(model : &Model) -> FreeformFieldsOfInterest {
+        // tuples (key, group)
+        let mut free_enum_fields : Vec<KeyAndGroup> = vec![];
+        let mut free_tag_fields : Vec<KeyAndGroup> = vec![];
+
+        for (key, field) in &model.fields {
+            match &field.kind {
+                FieldKind::FreeEnum { enum_group } => {
+                    let enum_group = enum_group.as_ref().unwrap_or(key);
+                    free_enum_fields.push(KeyAndGroup(key.to_string(), enum_group.to_string()));
+                },
+                FieldKind::FreeTags { tag_group } => {
+                    let tag_group = tag_group.as_ref().unwrap_or(key);
+                    free_tag_fields.push(KeyAndGroup(key.to_string(), tag_group.to_string()));
+                },
+                _ => {},
+            }
+        }
+
+        FreeformFieldsOfInterest {
+            free_tag_fields,
+            free_enum_fields,
+        }
+    }
+
+    /// This is an associated function to split the lifetimes
+    fn index_card<'a>(index : &mut Indexes, freeform_fields : &FreeformFieldsOfInterest, card : &'a Value) {
+        for KeyAndGroup(key, group) in &freeform_fields.free_enum_fields {
+            if !index.free_enums.contains_key(key.as_str()) {
+                index.free_enums.insert(key.to_string(), Default::default());
+            }
+
+            let group = index.free_enums.get_mut(group.as_str()).unwrap();
+
+            if let Some(value) = card.dot_get::<String>(&key) {
+                if !value.is_empty() {
+                    group.insert(value.to_string());
+                }
+            }
+        }
+
+        for KeyAndGroup(key, group) in &freeform_fields.free_tag_fields {
+            if !index.free_tags.contains_key(key.as_str()) {
+                index.free_tags.insert(key.to_string(), Default::default());
+            }
+
+            let group = index.free_tags.get_mut(group.as_str()).unwrap();
+
+            group.extend(card.dot_get_or_default::<Vec<String>>(&key));
+        }
+    }
+
+    pub fn rebuild_indexes(&mut self) {
+        self.index.free_enums.clear();
+        self.index.free_tags.clear();
+
+        for (_, card) in &self.data.cards {
+            Self::index_card(&mut self.index, &self.freeform_fields, card);
+        }
+    }
+}
+
+#[derive(Debug)]
+struct KeyAndGroup(String, String);
+
+#[derive(Debug)]
+struct FreeformFieldsOfInterest {
+    pub free_tag_fields: Vec<KeyAndGroup>,
+    pub free_enum_fields: Vec<KeyAndGroup>,
 }
+
+fn write_file(path : impl AsRef<Path>, bytes : &[u8]) {
+    let mut file = OpenOptions::new()
+        .write(true).create(true).truncate(true)
+        .open(path)
+        .expect("Error opening data file for writing.");
+
+    file.write(bytes)
+        .expect("Error write data file");
 }
 
 fn load_file(path: impl AsRef<Path>) -> String {

templates/_form_macros.html.tera

@@ -115,6 +115,8 @@
 			allowedTags: {% if not free %}{{field.all_tags_json | safe}}{% else %}[]{% endif %},
 			onTagAdd: onchange,
 			onTagRemove: onchange,
+			saveOnBlur: true,
+			clearOnBlur: false,
 		});
 
 		document.querySelector('#tag-input-{{field.key}} .taggle_input')

templates/index.html.tera

@@ -7,6 +7,7 @@
 
 {% block nav -%}
 	<a href="/add">Add</a>
+	<a href="/maintenance/reindex" onclick="return confirm('Unused learned tags and options will be forgotten. Proceed?')">Re-index</a>
 {%- endblock %}
 
 {% block content -%}
@@ -24,7 +25,7 @@
 			<tr>
 				<td class="actions">
 					<a href="/edit/{{card.id}}">Edit</a>
-					<a href="/delete/{{card.id}}" onclick="return confirm('Confirm delete')">Delete</a>
+					<a href="/delete/{{card.id}}" onclick="return confirm('Delete card?')">Delete</a>
 				</td>
 			{%- for field in card.fields %}
 				<td>