#[macro_use] extern crate serde_json; #[macro_use] extern crate log; use std::borrow::Cow; use std::collections::{HashMap}; use itertools::Itertools; use serde::{Deserialize, Serialize}; use thiserror::Error; use cool::{KVVecToKeysOrValues, map_drain_filter}; pub use id::ID; use id::next_id; use insert::InsertObj; use insert::InsertValue; use model::ObjectModel; use crate::model::{PropertyModel, RelationModel}; pub use data::{TypedValue}; pub use model::{DataType}; use crate::data::{Object}; use crate::update::{UpdateObj, UpsertValue}; pub mod model; pub mod data; pub mod insert; pub mod update; pub mod id; mod cool; #[cfg(test)] mod tests; mod serde_map_as_list; /// Stupid storage with no persistence #[derive(Debug, Default, Serialize, Deserialize, Clone)] pub struct Storage { #[serde(with = "serde_map_as_list")] obj_models: HashMap, #[serde(with = "serde_map_as_list")] rel_models: HashMap, #[serde(with = "serde_map_as_list")] prop_models: HashMap, #[serde(with = "serde_map_as_list")] objects: HashMap, #[serde(with = "serde_map_as_list")] relations: HashMap, #[serde(with = "serde_map_as_list")] values: HashMap, } #[derive(Debug, Error)] pub enum StorageError { #[error("Referenced {0} does not exist")] NotExist(Cow<'static, str>), #[error("{0}")] ConstraintViolation(Cow<'static, str>), } impl Storage { /// Create empty store pub fn new() -> Self { Self::default() } /// Define a data object pub fn define_object(&mut self, mut tpl: model::ObjectModel) -> Result { if tpl.name.is_empty() { return Err(StorageError::ConstraintViolation("Name must not be empty".into())); } if self.obj_models.iter().find(|(_, t)| t.name == tpl.name).is_some() { return Err(StorageError::ConstraintViolation(format!("Object model with the name \"{}\" already exists", tpl.name).into())); } debug!("Define object model \"{}\"", tpl.name); let id = next_id(); tpl.id = id; self.obj_models.insert(id, tpl); Ok(id) } /// Define a relation between two data objects pub fn define_relation(&mut self, mut rel: model::RelationModel) -> Result { if rel.name.is_empty() || rel.reciprocal_name.is_empty() { return Err(StorageError::ConstraintViolation("Names must not be empty".into())); } if !self.obj_models.contains_key(&rel.object) { return Err(StorageError::NotExist(format!("Source object model {}", rel.object).into())); } if !self.obj_models.contains_key(&rel.related) { return Err(StorageError::NotExist(format!("Related object model {}", rel.related).into())); } if let Some((_, colliding)) = self.rel_models.iter().find(|(_, other)| { (other.name == rel.name && other.object == rel.object) // Exact match || (other.name == rel.reciprocal_name && other.object == rel.related) // Our reciprocal name collides with related's own relation name || (other.reciprocal_name == rel.name && other.related == rel.object) // Our name name collides with a reciprocal name on the other relation || (other.reciprocal_name == rel.reciprocal_name && other.related == rel.related) // Reciprocal names collide for the same destination }) { return Err(StorageError::ConstraintViolation( format!("Name collision (\"{}\" / \"{}\") with existing relation (\"{}\" / \"{}\")", rel.name, rel.reciprocal_name, colliding.name, colliding.reciprocal_name ).into())); } debug!("Define relation model \"{}\" from {} to {}, reciprocal name \"{}\"", rel.name, self.describe_model(rel.object), self.describe_model(rel.related), rel.reciprocal_name); let id = next_id(); rel.id = id; self.rel_models.insert(id, rel); Ok(id) } /// Define a property attached to an object or a relation pub fn define_property(&mut self, mut prop: model::PropertyModel) -> Result { if prop.name.is_empty() { return Err(StorageError::ConstraintViolation("Name must not be empty".into())); } if !self.obj_models.contains_key(&prop.object) { // Maybe it's attached to a relation? if !self.rel_models.contains_key(&prop.object) { return Err(StorageError::NotExist(format!("Object or relation model {}", prop.object).into())); } } if self.prop_models.iter().find(|(_, t)| t.object == prop.object && t.name == prop.name).is_some() { return Err(StorageError::ConstraintViolation( format!("Property with the name \"{}\" already exists on model {}", prop.name, self.describe_model(prop.object)).into())); } // Ensure the default type is compatible prop.default = match prop.default.clone().cast_to(prop.data_type) { Ok(v) => v, Err(_) => return Err(StorageError::NotExist(format!("default value {:?} has invalid type", prop.default).into())) }; debug!("Define property model \"{}\" of {}", prop.name, self.describe_model(prop.object)); let id = next_id(); prop.id = id; self.prop_models.insert(id, prop); Ok(id) } /// Delete an object definition and associated data pub fn undefine_object(&mut self, id: ID) -> Result { return if let Some(t) = self.obj_models.remove(&id) { debug!("Undefine object model \"{}\"", t.name); // Remove relation templates let removed_relation_ids = map_drain_filter(&mut self.rel_models, |_k, v| v.object == id || v.related == id) .keys(); debug!("Undefined {} relation models", removed_relation_ids.len()); // Remove related property templates let removed_prop_ids = map_drain_filter(&mut self.prop_models, |_k, v| v.object == id || removed_relation_ids.contains(&v.object)) .keys(); debug!("Undefined {} property models", removed_prop_ids.len()); // Remove objects let removed_objects = map_drain_filter(&mut self.objects, |_k, v| v.model == id); debug!("Deleted {} objects", removed_objects.len()); // Remove property values let removed_values = map_drain_filter(&mut self.values, |_k, v| removed_prop_ids.contains(&v.model)); debug!("Deleted {} object or relation values", removed_values.len()); // Remove relations let removed_relations = map_drain_filter(&mut self.relations, |_k, v| removed_relation_ids.contains(&v.model)); debug!("Deleted {} object relations", removed_relations.len()); // Related object remain untouched, so there can be a problem with orphans. This is up to the application to deal with. Ok(t) } else { Err(StorageError::NotExist(format!("object model {}", id).into())) }; } /// Delete a relation definition and associated data pub fn undefine_relation(&mut self, id: ID) -> Result { return if let Some(t) = self.rel_models.remove(&id) { debug!("Undefine relation model \"{}\"", t.name); // Remove relations let removed = map_drain_filter(&mut self.relations, |_k, v| v.model == id); debug!("Deleted {} object relations", removed.len()); // Remove related property templates let removed_prop_tpl_ids = map_drain_filter(&mut self.prop_models, |_k, v| v.object == id).keys(); debug!("Undefined {} relation property models", removed_prop_tpl_ids.len()); let removed_values = map_drain_filter(&mut self.values, |_k, v| removed_prop_tpl_ids.contains(&v.model)); debug!("Deleted {} relation values", removed_values.len()); // Related object remain untouched, so there can be a problem with orphans. This is up to the application to deal with. Ok(t) } else { Err(StorageError::NotExist(format!("relation model {}", id).into())) }; } /// Delete a property definition and associated data pub fn undefine_property(&mut self, id: ID) -> Result { return if let Some(t) = self.prop_models.remove(&id) { debug!("Undefine property model \"{}\"", t.name); // Remove values let removed_values = map_drain_filter(&mut self.values, |_k, v| v.model == id); debug!("Deleted {} values", removed_values.len()); Ok(t) } else { Err(StorageError::NotExist(format!("property model {}", id).into())) }; } pub fn describe_model(&self, id: ID) -> String { if let Some(x) = self.obj_models.get(&id) { x.to_string() } else if let Some(x) = self.rel_models.get(&id) { x.to_string() } else if let Some(x) = self.prop_models.get(&id) { x.to_string() } else { id.to_string() } } pub fn get_model_name(&self, id: ID) -> &str { if let Some(x) = self.obj_models.get(&id) { &x.name } else if let Some(x) = self.rel_models.get(&id) { &x.name } else if let Some(x) = self.prop_models.get(&id) { &x.name } else { "???" } } // DATA /// Insert object with relations, validating the data model constraints pub fn insert_object(&mut self, insobj: InsertObj) -> Result { let obj_model_id = insobj.model; debug!("Insert object {:?}", insobj); let obj_model = match self.obj_models.get(&obj_model_id) { Some(m) => m, None => return Err(StorageError::NotExist(format!("object model {}", obj_model_id).into())) }; // validate unique name if self.objects.iter().find(|(_, o)| o.model == obj_model_id && o.name == insobj.name).is_some() { return Err(StorageError::ConstraintViolation( format!("{} named \"{}\" already exists", self.get_model_name(obj_model_id), insobj.name).into())); } let object_id = next_id(); let object = data::Object { id: object_id, model: obj_model_id, name: insobj.name }; let find_values_to_insert = |values: Vec, parent_id : ID, parent_model_id: ID| -> Result, StorageError> { let mut values_by_id = values.into_iter().into_group_map_by(|iv| iv.model); let mut values_to_insert = vec![]; for (id, prop) in self.prop_models.iter().filter(|(_id, p)| p.object == parent_model_id) { if let Some(values) = values_by_id.remove(id) { if values.len() > 1 && !prop.multiple { return Err(StorageError::ConstraintViolation(format!("{} of {} cannot have multiple values", prop, self.describe_model(parent_model_id)).into())); } for val_instance in values { values_to_insert.push(data::Value { id: next_id(), object: parent_id, model: prop.id, value: val_instance.value.cast_to(prop.data_type) .map_err(|v| StorageError::ConstraintViolation(format!("{} cannot accept value {:?}", prop, v).into()))?, }); } } else { if !prop.optional { values_to_insert.push(data::Value { id: next_id(), object: parent_id, model: prop.id, value: prop.default.clone(), }); } } } Ok(values_to_insert) }; let mut values_to_insert = find_values_to_insert(insobj.values, object_id, obj_model_id)?; // And now ..... relations! let mut relations_by_id = insobj.relations.into_iter().into_group_map_by(|ir| ir.model); let mut relations_to_insert = vec![]; for (relation_model_id, relation_model) in self.rel_models.iter().filter(|(_id, r)| r.object == obj_model_id) { if let Some(instances) = relations_by_id.remove(relation_model_id) { if instances.len() > 1 && !relation_model.multiple { return Err(StorageError::ConstraintViolation(format!("{} of {} cannot be set multiply", relation_model, obj_model).into())); } for rel_instance in instances { if let Some(related) = self.objects.get(&rel_instance.related) { if related.model != relation_model.related { return Err(StorageError::ConstraintViolation( format!("{} of {} requires object of type {}, got {}", relation_model, obj_model, self.describe_model(relation_model.related), self.describe_model(related.model)).into())); } } let relation_id = next_id(); // Relations can have properties values_to_insert.extend(find_values_to_insert(rel_instance.values, relation_id, *relation_model_id)?); relations_to_insert.push(data::Relation { id: relation_id, object: object_id, model: rel_instance.model, related: rel_instance.related, }); } } else { if !relation_model.optional { return Err(StorageError::ConstraintViolation(format!("{} is required for {}", relation_model, obj_model).into())); } } } self.objects.insert(object_id, object); debug!("Add {} new object relations", relations_to_insert.len()); for rel in relations_to_insert { self.relations.insert(rel.id, rel); } debug!("Add {} new values", values_to_insert.len()); for value in values_to_insert { self.values.insert(value.id, value); } Ok(object_id) } // Reading pub fn get_object_models(&self) -> impl Iterator { self.obj_models.values() } pub fn get_object_model(&self, id : ID) -> Option<&ObjectModel> { self.obj_models.get(&id) } pub fn get_relation_model(&self, id : ID) -> Option<&RelationModel> { self.rel_models.get(&id) } pub fn get_property_model(&self, id : ID) -> Option<&PropertyModel> { self.prop_models.get(&id) } pub fn get_grouped_prop_models(&self) -> HashMap> { self.prop_models.values() .into_group_map_by(|model| model.object) } pub fn get_grouped_prop_models_for_parents(&self, parents: Vec) -> HashMap> { self.prop_models.values() .filter(|p| parents.contains(&p.object)) .into_group_map_by(|model| model.object) } pub fn get_relations_for_object(&self, object_id: ID) -> impl Iterator { self.relations.values() .filter(move |rel| rel.object == object_id) } pub fn get_reciprocal_relations_for_object(&self, object_id: ID) -> impl Iterator { self.relations.values() .filter(move |rel| rel.related == object_id) } pub fn get_values_for_object(&self, object_id: ID) -> impl Iterator { self.values.values() .filter(move |prop| prop.object == object_id) } pub fn get_grouped_values_for_objects(&self, parents: Vec) -> HashMap> { self.values.values() .filter(move |prop| parents.contains(&prop.object)) .into_group_map_by(|model| model.object) } pub fn get_relation_models_for_object_model(&self, model_id: ID) -> impl Iterator { self.rel_models.values() .filter(move |model| model.object == model_id) } pub fn get_property_models_for_parents(&self, parents: Vec) -> impl Iterator { self.prop_models.values() .filter(move |model| parents.contains(&model.object)) } pub fn get_objects_of_type(&self, model_ids : Vec) -> impl Iterator { self.objects.values() .filter(move |object| model_ids.contains(&object.model)) } pub fn get_grouped_objects(&self) -> HashMap> { self.objects.values() .into_group_map_by(|object| object.model) } pub fn get_grouped_relation_models(&self) -> HashMap> { self.rel_models.values() .into_group_map_by(|model| model.object) } pub fn get_grouped_reciprocal_relation_models(&self) -> HashMap> { self.rel_models.values() .into_group_map_by(|model| model.related) } pub fn get_object(&self, id : ID) -> Option<&Object> { self.objects.get(&id) } // Updates pub fn update_object(&mut self, updobj: UpdateObj) -> Result<(), StorageError> { let old_object = self.objects.get(&updobj.id) .ok_or_else(|| StorageError::ConstraintViolation(format!("Object does not exist").into()))?; let updated_object_id = old_object.id; let updated_object_model_id = old_object.model; debug!("Update object {:?}", updobj); let obj_model = match self.obj_models.get(&updated_object_model_id) { Some(m) => m, None => return Err(StorageError::NotExist(format!("object model {}", updated_object_model_id).into())) }; // validate unique name if self.objects.iter().find(|(_, o)| o.model == updated_object_model_id && o.name == updobj.name && o.id != updated_object_id).is_some() { return Err(StorageError::ConstraintViolation( format!("{} named \"{}\" already exists", self.get_model_name(updated_object_model_id), updobj.name).into())); } // Update the object after everything else is checked let find_values_to_change = |values: Vec, parent_id : ID, parent_model_id: ID| -> Result<( // Insert (can overwrite existing, the ID will not change) Vec, // Delete Vec ), StorageError> { let mut values_by_model = values.into_iter().into_group_map_by(|iv| iv.model); let mut values_to_insert = vec![]; let mut ids_to_delete = vec![]; let mut existing_values_by_id = self.values.values() .filter(|v| v.object == parent_id) .into_group_map_by(|v| v.model); for (prop_model_id, prop) in self.prop_models.iter().filter(|(_id, p)| p.object == parent_model_id) { if let Some(values) = values_by_model.remove(prop_model_id) { if values.len() > 1 && !prop.multiple { return Err(StorageError::ConstraintViolation(format!("{} of {} cannot have multiple values", prop, self.describe_model(parent_model_id)).into())); } let updated_ids = values.iter().filter_map(|v| v.id).collect_vec(); ids_to_delete.extend(existing_values_by_id.remove(&prop.id).unwrap_or_default().into_iter() .filter(|v| !updated_ids.contains(&v.id)).map(|v| v.id)); for val_instance in values { values_to_insert.push(data::Value { id: val_instance.id.unwrap_or_else(|| next_id()), object: parent_id, model: prop.id, value: val_instance.value }); } } else { if !prop.optional { warn!("Attempt to remove non-optional prop, do nothing"); } else { if let Some(existing) = existing_values_by_id.remove(&prop.id) { ids_to_delete.extend(existing.into_iter().map(|v| v.id)); } } } } Ok((values_to_insert, ids_to_delete)) }; let (mut values_to_insert, mut value_ids_to_delete) = find_values_to_change(updobj.values, updated_object_id, updated_object_model_id)?; // And now ..... relations! let mut relations_by_model = updobj.relations.into_iter().into_group_map_by(|ir| ir.model); let mut relations_to_insert = vec![]; let mut relations_to_delete = vec![]; let mut existing_relations_by_id = self.relations.values() .filter(|v| v.object == updated_object_id) .into_group_map_by(|v| v.model); let rel_models_by_id = self.rel_models.iter().filter(|(_id, r)| r.object == updated_object_model_id); for (relation_model_id, relation_model) in rel_models_by_id { let mut updated_ids = vec![]; if let Some(instances) = relations_by_model.remove(relation_model_id) { if instances.len() > 1 && !relation_model.multiple { return Err(StorageError::ConstraintViolation(format!("{} of {} cannot be set multiply", relation_model, obj_model).into())); } for rel_instance in instances { if let Some(related) = self.objects.get(&rel_instance.related) { if related.model != relation_model.related { return Err(StorageError::ConstraintViolation( format!("{} of {} requires object of type {}, got {}", relation_model, obj_model, self.describe_model(relation_model.related), self.describe_model(related.model)).into())); } } let relation_id = rel_instance.id.unwrap_or_else(|| next_id()); // Relations can have properties let (ins, del) = find_values_to_change(rel_instance.values, relation_id, *relation_model_id)?; values_to_insert.extend(ins); value_ids_to_delete.extend(del); relations_to_insert.push(data::Relation { id: relation_id, object: updated_object_id, model: rel_instance.model, related: rel_instance.related, }); updated_ids.push(relation_id); } } else { if !relation_model.optional { return Err(StorageError::ConstraintViolation(format!("{} is required for {}", relation_model, obj_model).into())); } } relations_to_delete.extend(existing_relations_by_id.remove(&relation_model_id).unwrap_or_default().into_iter() .filter(|rel| !updated_ids.contains(&rel.id)).map(|rel| rel.id)); } let obj_mut = self.objects.get_mut(&updated_object_id).unwrap(); obj_mut.name = updobj.name; debug!("Add {} new object relations", relations_to_insert.len()); for rel in relations_to_insert { self.relations.insert(rel.id, rel); } debug!("Add {} new values", values_to_insert.len()); for value in values_to_insert { self.values.insert(value.id, value); } debug!("Deleted {} values", value_ids_to_delete.len()); for id in value_ids_to_delete { self.values.remove(&id); } debug!("Deleted {} object relations", relations_to_delete.len()); for id in relations_to_delete { self.relations.remove(&id); } Ok(()) } pub fn update_object_model(&mut self, model : ObjectModel) -> Result<(), StorageError> { if model.name.is_empty() { return Err(StorageError::ConstraintViolation(format!("Model name must not be empty.").into())); } if !self.obj_models.contains_key(&model.id) { return Err(StorageError::NotExist(format!("Object model ID {} does not exist.", model.id).into())); } if let Some(conflict) = self.obj_models.values().find(|m| m.id != model.id && m.name == model.name) { return Err(StorageError::ConstraintViolation(format!("Object {} already has the name {}", conflict.id, model.name).into())); } self.obj_models.insert(model.id, model); Ok(()) } pub fn update_relation_model(&mut self, mut rel : RelationModel) -> Result<(), StorageError> { if rel.name.is_empty() || rel.reciprocal_name.is_empty() { return Err(StorageError::ConstraintViolation(format!("Relation names must not be empty.").into())); } // Object and Related can't be changed, so we re-fill them from the existing model if let Some(existing) = self.rel_models.get(&rel.id) { rel.object = existing.object; rel.related = existing.related; } else { return Err(StorageError::NotExist(format!("Relation model ID {} does not exist.", rel.id).into())); } // Difficult checks ... // yes this is stupid and inefficient and slow and if let Some((_, colliding)) = self.rel_models.iter().find(|(_, other)| { (other.name == rel.name && other.object == rel.object && rel.id != other.id) // Exact match || (other.name == rel.reciprocal_name && other.object == rel.related && rel.id != other.id) // Our reciprocal name collides with related's own relation name || (other.reciprocal_name == rel.name && other.related == rel.object && rel.id != other.id) // Our name name collides with a reciprocal name on the other relation || (other.reciprocal_name == rel.reciprocal_name && other.related == rel.related && rel.id != other.id) // Reciprocal names collide for the same destination }) { return Err(StorageError::ConstraintViolation( format!("name collision (\"{}\" / \"{}\") with existing relation (\"{}\" / \"{}\")", rel.name, rel.reciprocal_name, colliding.name, colliding.reciprocal_name ).into())); } self.rel_models.insert(rel.id, rel); Ok(()) } pub fn update_property_model(&mut self, mut prop: PropertyModel) -> Result<(), StorageError> { if prop.name.is_empty() { return Err(StorageError::ConstraintViolation(format!("Property name must not be empty.").into())); } // Object can't be changed, so we re-fill them from the existing model if let Some(existing) = self.prop_models.get(&prop.id) { prop.object = existing.object; } else { return Err(StorageError::NotExist(format!("Property model ID {} does not exist.", prop.id).into())); } if self.prop_models.iter().find(|(_, t)| t.object == prop.object && t.name == prop.name && t.id != prop.id).is_some() { return Err(StorageError::ConstraintViolation( format!("property with the name \"{}\" already exists on {}", prop.name, self.describe_model(prop.object)).into())); } // Ensure the default type is compatible prop.default = match prop.default.clone().cast_to(prop.data_type) { Ok(v) => v, Err(_) => return Err(StorageError::NotExist(format!("default value {:?} has invalid type", prop.default).into())) }; self.prop_models.insert(prop.id, prop); Ok(()) } /// Delete an object and associated data pub fn delete_object(&mut self, id: ID) -> Result { return if let Some(t) = self.objects.remove(&id) { debug!("Delete object \"{}\"", t.name); // Remove relation templates let removed_relation_ids = map_drain_filter(&mut self.relations, |_k, v| v.object == id || v.related == id) .keys(); debug!("Deleted {} object relations", removed_relation_ids.len()); // Remove values let removed_values = map_drain_filter(&mut self.values, |_k, v| removed_relation_ids.contains(&v.object) || v.object == id); debug!("Deleted {} object values", removed_values.len()); Ok(t) } else { Err(StorageError::NotExist(format!("object {}", id).into())) }; } }