add persistence

Cargo.lock

@@ -448,6 +448,16 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
+[[package]]
+name = "bincode"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d"
+dependencies = [
+ "byteorder",
+ "serde",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -2040,18 +2050,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146"
+checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1"
+checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2559,6 +2569,7 @@ name = "yopa"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "bincode",
  "itertools",
  "lazy_static",
  "log",
@@ -2576,6 +2587,7 @@ dependencies = [
  "actix-session",
  "actix-web",
  "actix-web-static-files",
+ "anyhow",
  "heck",
  "include_dir",
  "itertools",
@@ -2588,6 +2600,7 @@ dependencies = [
  "serde_json",
  "simple-logging",
  "tera",
+ "thiserror",
  "tokio",
  "yopa",
 ]

yopa-web/Cargo.toml

@@ -8,7 +8,7 @@ build = "build.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-yopa = { path = "../yopa", features = [] }
+yopa = { path = "../yopa", features = ["uuid-ids"] }
 serde = "1"
 serde_json = "1"
 heck = "0.3.2"
@@ -24,6 +24,8 @@ once_cell = "1.5.2"
 rand = "0.8.3"
 itertools = "0.10.0"
 json_dotpath = "1.0.3"
+anyhow = "1.0.38"
+thiserror = "1.0.24"
 
 tokio = { version="0.2.6", features=["full"] }
 

yopa-web/resources/src/utils.js

@@ -23,8 +23,8 @@ export function objCopy(object) {
 }
 
 export function castId(id) {
-  // TODO no-op after switching to UUIDs
+  return id.toString();
-  return +id;
+  //return +id;
 }
 
 // like _.isEmpty, but less stupid

yopa-web/src/main.rs

@@ -2,6 +2,8 @@
 extern crate actix_web;
 #[macro_use]
 extern crate log;
+#[macro_use]
+extern crate thiserror;
 
 use std::collections::HashMap;
 use std::ops::Deref;
@@ -106,7 +108,8 @@ async fn main() -> std::io::Result<()> {
     // Ensure the lazy ref is initialized early (to catch template bugs ASAP)
     let _ = TERA.deref();
 
-    let yopa_store: YopaStoreWrapper = init_yopa();
+    let yopa_store: YopaStoreWrapper = init_yopa()
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
 
     let mut session_key = [0u8; 32];
     rand::thread_rng().fill(&mut session_key);
@@ -164,133 +167,9 @@ async fn main() -> std::io::Result<()> {
     .await
 }
 
-fn init_yopa() -> YopaStoreWrapper {
+fn init_yopa() -> anyhow::Result<YopaStoreWrapper> {
-    let mut store = Storage::new();
+    let mut store = Storage::new_bincode(std::env::current_dir()?.join("yopa-store.dat"));
-
+    store.load()?;
-    /*
-
-    // Seed the store with some dummy data for view development
-    use yopa::model;
-    use yopa::DataType;
-
-    let id_recipe = store
-        .define_object(model::ObjectModel {
-            id: Default::default(),
-            name: "Recipe".to_string(),
-        })
-        .unwrap();
-
-    let id_book = store
-        .define_object(model::ObjectModel {
-            id: Default::default(),
-            name: "Book".to_string(),
-        })
-        .unwrap();
-
-    let _id_ing = store
-        .define_object(model::ObjectModel {
-            id: Default::default(),
-            name: "Ingredient".to_string(),
-        })
-        .unwrap();
-
-    let val_descr = store
-        .define_property(model::PropertyModel {
-            id: Default::default(),
-            object: id_recipe,
-            name: "description".to_string(),
-            optional: true,
-            multiple: true,
-            data_type: DataType::String,
-            default: TypedValue::String("".into()),
-        })
-        .unwrap();
-
-    store
-        .define_property(model::PropertyModel {
-            id: Default::default(),
-            object: id_book,
-            name: "author".to_string(),
-            optional: true,
-            multiple: true,
-            data_type: DataType::String,
-            default: TypedValue::String("Pepa Novák".into()),
-        })
-        .unwrap();
-
-    let rel_book_id = store
-        .define_relation(model::RelationModel {
-            id: Default::default(),
-            object: id_recipe,
-            name: "book reference".to_string(),
-            reciprocal_name: "recipes".to_string(),
-            optional: true,
-            multiple: true,
-            related: id_book,
-        })
-        .unwrap();
-
-    let page = store
-        .define_property(model::PropertyModel {
-            id: Default::default(),
-            object: rel_book_id,
-            name: "page".to_string(),
-            optional: true,
-            multiple: false,
-            data_type: DataType::Integer,
-            default: TypedValue::Integer(0),
-        })
-        .unwrap();
-
-    store
-        .define_relation(model::RelationModel {
-            id: Default::default(),
-            object: id_recipe,
-            name: "related recipe".to_string(),
-            reciprocal_name: "related recipe".to_string(),
-            optional: true,
-            multiple: true,
-            related: id_recipe,
-        })
-        .unwrap();
-
-    let book1 = store
-        .insert_object(InsertObj {
-            model: id_book,
-            name: "Book 1".to_string(),
-            values: vec![],
-            relations: vec![],
-        })
-        .unwrap();
-
-    store
-        .insert_object(InsertObj {
-            model: id_book,
-            name: "Book 2".to_string(),
-            values: vec![],
-            relations: vec![],
-        })
-        .unwrap();
-
-    store
-        .insert_object(InsertObj {
-            model: id_recipe,
-            name: "Recipe1".to_string(),
-            values: vec![InsertValue {
-                model: val_descr,
-                value: TypedValue::String("Bla bla bla".into()),
-            }],
-            relations: vec![InsertRel {
-                model: rel_book_id,
-                related: book1,
-                values: vec![InsertValue {
-                    model: page,
-                    value: TypedValue::Integer(15),
-                }],
-            }],
-        })
-        .unwrap();
-    */
 
-    web::Data::new(tokio::sync::RwLock::new(store))
+    Ok(web::Data::new(tokio::sync::RwLock::new(store)))
 }

yopa-web/src/routes/models/object.rs

@@ -1,5 +1,5 @@
 use actix_session::Session;
-use actix_web::{web, Responder};
+use actix_web::{web, Responder, HttpResponse};
 use serde::{Deserialize, Serialize};
 
 use yopa::model::{ObjectModel, PropertyModel};
@@ -8,7 +8,7 @@ use yopa::ID;
 use crate::routes::models::relation::RelationModelDisplay;
 use crate::session_ext::SessionExt;
 use crate::tera_ext::TeraExt;
-use crate::utils::redirect;
+use crate::utils::{redirect, StorageErrorIntoResponseError};
 use crate::TERA;
 
 #[derive(Serialize, Debug)]
@@ -45,7 +45,7 @@ pub(crate) async fn create(
     form: web::Form<ObjectModelForm>,
     store: crate::YopaStoreWrapper,
     session: Session,
-) -> actix_web::Result<impl Responder> {
+) -> actix_web::Result<HttpResponse> {
     let mut wg = store.write().await;
     let form = form.into_inner();
     match wg.define_object(ObjectModel {
@@ -53,6 +53,7 @@ pub(crate) async fn create(
         name: form.name.clone(),
     }) {
         Ok(_id) => {
+            wg.persist().err_to_500()?;
             debug!("Object created, redirecting to root");
             session.flash_success(format!("Object model \"{}\" created.", form.name));
             redirect("/models")
@@ -108,6 +109,7 @@ pub(crate) async fn update(
         name: form.name.clone(),
     }) {
         Ok(_id) => {
+            wg.persist().err_to_500()?;
             debug!("Object updated, redirecting to root");
             session.flash_success(format!("Object model \"{}\" updated.", form.name));
             redirect("/models")
@@ -133,6 +135,7 @@ pub(crate) async fn delete(
             .map_err(|e| actix_web::error::ErrorBadRequest(e))?,
     ) {
         Ok(om) => {
+            wg.persist().err_to_500()?;
             debug!("Object model deleted, redirecting to root");
             session.flash_success(format!("Object model \"{}\" deleted.", om.name));
             redirect("/models")

yopa-web/src/routes/models/property.rs

@@ -8,7 +8,7 @@ use yopa::{DataType, TypedValue, ID};
 use crate::routes::models::relation::ObjectOrRelationModelDisplay;
 use crate::session_ext::SessionExt;
 use crate::tera_ext::TeraExt;
-use crate::utils::{redirect, ParseOrBadReq};
+use crate::utils::{redirect, ParseOrBadReq, StorageErrorIntoResponseError};
 use crate::TERA;
 
 #[get("/model/property/create/{object_id}")]
@@ -148,6 +148,7 @@ pub(crate) async fn create(
         default,
     }) {
         Ok(_id) => {
+            wg.persist().err_to_500()?;
             debug!("Property created, redirecting to root");
             session.flash_success(format!("Property model \"{}\" created.", form.name));
             redirect("/models")
@@ -170,6 +171,7 @@ pub(crate) async fn delete(
     let mut wg = store.write().await;
     match wg.undefine_property(id.parse_or_bad_request()?) {
         Ok(rm) => {
+            wg.persist().err_to_500()?;
             debug!("Property deleted, redirecting to root");
             session.flash_success(format!("Property \"{}\" deleted.", rm.name));
             redirect("/models")
@@ -256,7 +258,8 @@ pub(crate) async fn update(
         default,
     }) {
         Ok(_id) => {
-            debug!("Relation updated, redirecting to root");
+            wg.persist().err_to_500()?;
+            debug!("Property updated, redirecting to root");
             session.flash_success(format!("Property \"{}\" updated.", form.name));
             redirect("/models")
         }
@@ -264,7 +267,7 @@ pub(crate) async fn update(
             warn!("Error updating model: {}", e);
             session.flash_error(e.to_string());
             session.set("old", form).unwrap();
-            redirect(format!("/model/relation/update/{}", id))
+            redirect(format!("/model/property/update/{}", id))
         }
     }
 }

yopa-web/src/routes/models/relation.rs

@@ -7,7 +7,7 @@ use yopa::ID;
 
 use crate::session_ext::SessionExt;
 use crate::tera_ext::TeraExt;
-use crate::utils::{redirect, ParseOrBadReq};
+use crate::utils::{redirect, ParseOrBadReq, StorageErrorIntoResponseError};
 use crate::TERA;
 
 #[derive(Serialize, Debug)]
@@ -90,6 +90,7 @@ pub(crate) async fn create(
         related: form.related,
     }) {
         Ok(_id) => {
+            wg.persist().err_to_500()?;
             debug!("Relation created, redirecting to root");
             session.flash_success(format!("Relation model \"{}\" created.", form.name));
             redirect("/models")
@@ -118,6 +119,7 @@ pub(crate) async fn delete(
     let mut wg = store.write().await;
     match wg.undefine_relation(id.parse_or_bad_request()?) {
         Ok(rm) => {
+            wg.persist().err_to_500()?;
             debug!("Relation deleted, redirecting to root");
             session.flash_success(format!("Relation model \"{}\" deleted.", rm.name));
             redirect("/models")
@@ -189,6 +191,7 @@ pub(crate) async fn update(
         related: Default::default(), // dummy
     }) {
         Ok(_id) => {
+            wg.persist().err_to_500()?;
             debug!("Relation updated, redirecting to root");
             session.flash_success(format!("Relation model \"{}\" updated.", form.name));
             redirect("/models")

yopa-web/src/routes/objects.rs

@@ -3,7 +3,7 @@ use actix_session::Session;
 use actix_web::{web, HttpResponse, Responder};
 
 use crate::tera_ext::TeraExt;
-use crate::utils::redirect;
+use crate::utils::{redirect, StorageErrorIntoResponseError};
 use crate::TERA;
 use serde::Serialize;
 use yopa::data::Object;
@@ -110,6 +110,7 @@ pub(crate) async fn create(
 
     match wg.insert_object(form) {
         Ok(_id) => {
+            wg.persist().err_to_500()?;
             debug!("Object created, redirecting to root");
             session.flash_success(format!("{} \"{}\" created.", model_name, name));
             Ok(HttpResponse::Ok().finish())
@@ -512,6 +513,7 @@ pub(crate) async fn update(
 
     match wg.update_object(form) {
         Ok(_id) => {
+            wg.persist().err_to_500()?;
             debug!("Object created, redirecting to root");
             session.flash_success(format!("{} \"{}\" updated.", model_name, name));
             Ok(HttpResponse::Ok().finish())
@@ -532,6 +534,7 @@ pub(crate) async fn delete(
     let mut wg = store.write().await;
     match wg.delete_object(*id) {
         Ok(obj) => {
+            wg.persist().err_to_500()?;
             debug!("Object deleted, redirecting to root");
             session.flash_success(format!("Object \"{}\" deleted.", obj.name));
             redirect("/")

yopa-web/src/utils.rs

@@ -1,7 +1,8 @@
 use actix_web::http::header::IntoHeaderValue;
-use actix_web::HttpResponse;
+use actix_web::{HttpResponse, ResponseError, Error};
 use std::fmt::{Debug, Display};
 use std::str::FromStr;
+use yopa::StorageError;
 
 pub fn redirect(path: impl IntoHeaderValue) -> actix_web::Result<HttpResponse> {
     Ok(HttpResponse::SeeOther()
@@ -38,3 +39,23 @@ impl ParseOrBadReq for String {
         self.as_str().parse_or_bad_request()
     }
 }
+
+#[derive(Debug, Error)]
+pub enum ActixErrorWrapper {
+    #[error(transparent)]
+    Storage(#[from] StorageError),
+}
+
+impl ResponseError for ActixErrorWrapper {
+
+}
+
+pub trait StorageErrorIntoResponseError<T> {
+    fn err_to_500(self) -> Result<T, ActixErrorWrapper>;
+}
+
+impl<T> StorageErrorIntoResponseError<T> for Result<T, StorageError> {
+    fn err_to_500(self) -> Result<T, ActixErrorWrapper> {
+        self.map_err(|e| e.into())
+    }
+}

yopa/Cargo.toml

@@ -20,6 +20,7 @@ anyhow = "1.0.38"
 thiserror = "1.0.23"
 itertools = "0.10.0"
 lazy_static = "1.4.0"
+bincode = "1.3.1"
 
 [features]
 default = []

yopa/src/lib.rs

@@ -22,6 +22,9 @@ use crate::data::Object;
 use crate::update::{UpdateObj, UpsertValue};
 pub use data::TypedValue;
 pub use model::DataType;
+use std::path::{PathBuf, Path};
+use std::fs::OpenOptions;
+use std::io::{BufReader, BufWriter};
 
 mod cool;
 pub mod data;
@@ -34,7 +37,7 @@ mod serde_map_as_list;
 #[cfg(test)]
 mod tests;
 
-/// Stupid storage with no persistence
+/// Stupid storage with naive inefficient file persistence
 #[derive(Debug, Default, Serialize, Deserialize, Clone)]
 pub struct Storage {
     #[serde(with = "serde_map_as_list")]
@@ -50,14 +53,46 @@ pub struct Storage {
     relations: HashMap<ID, data::Relation>,
     #[serde(with = "serde_map_as_list")]
     values: HashMap<ID, data::Value>,
+
+    #[serde(skip)]
+    opts: StoreOpts,
+}
+
+#[derive(Debug, Clone)]
+pub struct StoreOpts {
+    file : Option<PathBuf>,
+    file_format: FileEncoding
+}
+
+impl Default for StoreOpts {
+    fn default() -> Self {
+        Self {
+            file: None,
+            file_format: FileEncoding::JSON
+        }
+    }
+}
+
+#[derive(Debug,Clone,Copy)]
+pub enum FileEncoding {
+    JSON,
+    BINCODE,
 }
 
-#[derive(Debug, Error)]
+#[derive(Debug,Error)]
 pub enum StorageError {
     #[error("Referenced {0} does not exist")]
     NotExist(Cow<'static, str>),
     #[error("{0}")]
     ConstraintViolation(Cow<'static, str>),
+    #[error("Persistence not configured!")]
+    NotPersistent,
+    #[error(transparent)]
+    IO(#[from] std::io::Error),
+    #[error(transparent)]
+    SerdeJson(#[from] serde_json::Error),
+    #[error(transparent)]
+    Bincode(#[from] bincode::Error),
 }
 
 impl Storage {
@@ -66,6 +101,90 @@ impl Storage {
         Self::default()
     }
 
+    pub fn new_json(file : impl AsRef<Path>) -> Self {
+        let mut s = Self::new();
+        s.opts.file_format = FileEncoding::JSON;
+        s.opts.file = Some(file.as_ref().to_path_buf());
+        s
+    }
+
+    pub fn new_bincode(file : impl AsRef<Path>) -> Self {
+        let mut s = Self::new();
+        s.opts.file_format = FileEncoding::BINCODE;
+        s.opts.file = Some(file.as_ref().to_path_buf());
+        s
+    }
+
+    pub fn set_file(&mut self, file : impl AsRef<Path>, format : FileEncoding) {
+        self.opts.file_format = format;
+        self.opts.file = Some(file.as_ref().to_path_buf());
+    }
+
+    pub fn unset_file(&mut self) {
+        self.opts.file = None;
+    }
+
+    pub fn load(&mut self) -> Result<(), StorageError> {
+        match &self.opts.file {
+            None => {
+                return Err(StorageError::NotPersistent);
+            }
+            Some(path) => {
+                debug!("Load from: {}", path.display());
+
+                if !path.exists() {
+                    warn!("File does not exist, skip load.");
+                    return Ok(())
+                }
+
+                let f = OpenOptions::new().read(true).open(&path)?;
+
+                let reader = BufReader::new(f);
+
+                let parsed : Self = match self.opts.file_format {
+                    FileEncoding::JSON => {
+                        serde_json::from_reader(reader)?
+                    }
+                    FileEncoding::BINCODE => {
+                        bincode::deserialize_from(reader)?
+                    }
+                };
+
+                let opts = std::mem::replace(&mut self.opts, StoreOpts::default());
+                *self = parsed;
+                self.opts = opts;
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn persist(&mut self) -> Result<(), StorageError> {
+        match &self.opts.file {
+            None => {
+                warn!("Store is not persistent!");
+                //return Err(StorageError::NotPersistent);
+            }
+            Some(path) => {
+                debug!("Persist to: {}", path.display());
+
+                let f = OpenOptions::new().write(true).create(true).truncate(true).open(&path)?;
+                let writer = BufWriter::new(f);
+
+                match self.opts.file_format {
+                    FileEncoding::JSON => {
+                        serde_json::to_writer(writer, self)?;
+                    }
+                    FileEncoding::BINCODE => {
+                        bincode::serialize_into(writer, self)?
+                    }
+                };
+            }
+        }
+
+        Ok(())
+    }
+
     /// Define a data object
     pub fn define_object(&mut self, mut tpl: model::ObjectModel) -> Result<ID, StorageError> {
         if tpl.name.is_empty() {