yopa/yopa/src/lib.rs

#[macro_use]
extern crate log;
#[cfg(test)]
#[macro_use]
extern crate serde_json;

use std::borrow::Cow;
use std::collections::HashMap;
use std::fs::OpenOptions;
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::{Path, PathBuf};

use itertools::Itertools;
use serde::{Deserialize, Serialize};
use thiserror::Error;

use cool::{map_drain_filter, KVVecToKeysOrValues};
pub use data::TypedValue;
pub use id::ID;
use insert::InsertObj;
use insert::InsertValue;
pub use model::DataType;
use model::ObjectModel;

use crate::cool::IsNoneOrElse;
use crate::data::Object;
use crate::helpers::{GroupByModel, GroupByParent};
use crate::model::{PropertyModel, RelationModel};
use crate::update::{UpdateObj, UpsertValue};

mod cool;
pub mod data;
pub mod id;
pub mod insert;
pub mod model;
pub mod update;

pub mod helpers;

mod serde_atomic_id;
mod serde_map_as_list;
#[cfg(test)]
mod tests;

pub const VERSION: &'static str = env!("CARGO_PKG_VERSION");

pub const YOPA_MAGIC: &[u8; 4] = b"YOPA";
pub const BINARY_FORMAT: u16 = 2;

/// Stupid storage with naive inefficient file persistence
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct Storage {
    #[serde(with = "serde_map_as_list")]
    obj_models: HashMap<ID, model::ObjectModel>,
    #[serde(with = "serde_map_as_list")]
    rel_models: HashMap<ID, model::RelationModel>,
    #[serde(with = "serde_map_as_list")]
    prop_models: HashMap<ID, model::PropertyModel>,

    #[serde(with = "serde_map_as_list")]
    objects: HashMap<ID, data::Object>,
    #[serde(with = "serde_map_as_list")]
    relations: HashMap<ID, data::Relation>,
    #[serde(with = "serde_map_as_list")]
    values: HashMap<ID, data::Value>,

    #[cfg(not(feature = "uuid-ids"))]
    #[serde(with = "serde_atomic_id")]
    next_id: atomic::Atomic<ID>,

    #[serde(skip)]
    opts: StoreOpts,
}

#[derive(Debug, Clone)]
pub struct StoreOpts {
    file: Option<PathBuf>,
    file_format: FileEncoding,
}

impl Default for StoreOpts {
    fn default() -> Self {
        Self {
            file: None,
            file_format: FileEncoding::JSON,
        }
    }
}

#[derive(Debug, Clone, Copy)]
pub enum FileEncoding {
    JSON,
    BINCODE,
}

#[derive(Debug, Error)]
pub enum StorageError {
    #[error("Referenced {0} does not exist")]
    NotExist(Cow<'static, str>),
    #[error("{0}")]
    ConstraintViolation(Cow<'static, str>),
    #[error("{0}")]
    Invalid(Cow<'static, str>),
    #[error("Persistence not configured!")]
    NotPersistent,
    #[error("Bad magic! Not a binary Yopa file")]
    BadMagic,
    #[error("Binary format {0} is not compatible with this version of Yopa")]
    NotCompatible(u16),
    #[error(transparent)]
    IO(#[from] std::io::Error),
    #[error(transparent)]
    SerdeJson(#[from] serde_json::Error),
    #[error(transparent)]
    Bincode(#[from] bincode::Error),
}

impl Storage {
    /// Create empty store
    pub fn new() -> Self {
        Self::default()
    }

    /// Create a new instance backed by a JSON file. The file is created if needed.
    /// The backing file can be read or edited by any text editor.
    pub fn new_json(file: impl AsRef<Path>) -> Result<Self, StorageError> {
        let mut s = Self::new();
        s.opts.file_format = FileEncoding::JSON;
        s.opts.file = Some(file.as_ref().to_path_buf());
        s.load()?;
        Ok(s)
    }

    /// Create a new instance backed by a BinCode file. The file is created if needed.
    /// This format has higher data density and faster save/load times,
    /// but cannot be easily read outside yopa.
    pub fn new_bincode(file: impl AsRef<Path>) -> Result<Self, StorageError> {
        let mut s = Self::new();
        s.opts.file_format = FileEncoding::BINCODE;
        s.opts.file = Some(file.as_ref().to_path_buf());
        s.load()?;
        Ok(s)
    }

    #[cfg(not(feature = "uuid-ids"))]
    fn next_id(&self) -> ID {
        self.next_id.fetch_add(1, atomic::Ordering::Relaxed)
    }

    #[cfg(feature = "uuid-ids")]
    fn next_id(&self) -> ID {
        uuid::Uuid::new_v4()
    }

    /// Set backing file and its encoding
    pub fn set_file(&mut self, file: impl AsRef<Path>, format: FileEncoding) {
        self.opts.file_format = format;
        self.opts.file = Some(file.as_ref().to_path_buf());
    }

    /// Unset backing file
    pub fn unset_file(&mut self) {
        self.opts.file = None;
    }

    /// Manually load from the backing file.
    pub fn load(&mut self) -> Result<(), StorageError> {
        match &self.opts.file {
            None => {
                return Err(StorageError::NotPersistent);
            }
            Some(path) => {
                debug!("Load from: {}", path.display());

                if !path.exists() {
                    warn!("File does not exist, skip load.");
                    return Ok(());
                }

                let f = OpenOptions::new().read(true).open(&path)?;

                let mut reader = BufReader::new(f);

                let parsed: Self = match self.opts.file_format {
                    FileEncoding::JSON => serde_json::from_reader(reader)?,
                    FileEncoding::BINCODE => {
                        let mut magic: [u8; 6] = [0; 6];
                        reader.read_exact(&mut magic)?;

                        if &magic[0..4] != YOPA_MAGIC {
                            return Err(StorageError::BadMagic);
                        }

                        let version = u16::from_le_bytes([magic[4], magic[5]]);
                        if version != BINARY_FORMAT {
                            return Err(StorageError::NotCompatible(version));
                        }

                        bincode::deserialize_from(reader)?
                    }
                };

                let opts = std::mem::replace(&mut self.opts, StoreOpts::default());
                *self = parsed;
                self.opts = opts;
            }
        }

        Ok(())
    }

    /// Persist to the backing file.
    /// This must be called after each transaction that should be saved.
    pub fn persist(&mut self) -> Result<(), StorageError> {
        match &self.opts.file {
            None => {
                warn!("Store is not persistent!");
                //return Err(StorageError::NotPersistent);
            }
            Some(path) => {
                debug!("Persist to: {}", path.display());

                let f = OpenOptions::new()
                    .write(true)
                    .create(true)
                    .truncate(true)
                    .open(&path)?;
                let mut writer = BufWriter::new(f);

                match self.opts.file_format {
                    FileEncoding::JSON => {
                        serde_json::to_writer(writer, self)?;
                    }
                    FileEncoding::BINCODE => {
                        writer.write_all(YOPA_MAGIC)?;
                        writer.write_all(&BINARY_FORMAT.to_le_bytes())?;
                        bincode::serialize_into(writer, self)?
                    }
                };
            }
        }

        Ok(())
    }

    /// Get textual description of a model (of any kind), suitable for error messages or logging
    pub fn describe_model(&self, id: ID) -> String {
        if let Some(x) = self.obj_models.get(&id) {
            x.to_string()
        } else if let Some(x) = self.rel_models.get(&id) {
            x.to_string()
        } else if let Some(x) = self.prop_models.get(&id) {
            x.to_string()
        } else {
            id.to_string()
        }
    }

    /// Get object name, or the ID as string if no name is configured
    pub fn get_object_name_by_id<'a>(&'a self, id: ID) -> Cow<'a, str> {
        if let Some(o) = self.get_object(id) {
            return self.get_object_name(o);
        }

        return id.to_string().into();
    }

    pub fn get_object_name<'b, 'a: 'b>(&'a self, object: &'a Object) -> Cow<'b, str> {
        if let Some(ObjectModel {
            name_property: Some(name_property),
            ..
        }) = self.get_object_model(object.model)
        {
            if let Some(v) = self
                .values
                .values()
                .find(|v| v.object == object.id && &v.model == name_property)
            {
                return v.value.to_cow();
            }
        }

        return object.id.to_string().into();
    }

    /// Get model name. Accepts ID of object, relation or property models.
    pub fn get_model_name(&self, id: ID) -> &str {
        if let Some(x) = self.obj_models.get(&id) {
            &x.name
        } else if let Some(x) = self.rel_models.get(&id) {
            &x.name
        } else if let Some(x) = self.prop_models.get(&id) {
            &x.name
        } else {
            "???"
        }
    }

    //region Model queries

    /// Iterate all object models
    pub fn get_object_models(&self) -> impl Iterator<Item = &ObjectModel> {
        self.obj_models.values()
    }

    /// Iterate object models with given IDs
    pub fn get_object_models_by_ids(&self, ids: Vec<ID>) -> impl Iterator<Item = &ObjectModel> {
        self.obj_models
            .values()
            .filter(move |m| ids.contains(&m.id))
    }

    /// Get an object model by ID
    pub fn get_object_model(&self, model_id: ID) -> Option<&ObjectModel> {
        self.obj_models.get(&model_id)
    }

    /// Get a relation model by ID
    pub fn get_relation_model(&self, model_id: ID) -> Option<&RelationModel> {
        self.rel_models.get(&model_id)
    }

    /// Get a property model by ID
    pub fn get_property_model(&self, model_id: ID) -> Option<&PropertyModel> {
        self.prop_models.get(&model_id)
    }

    /// Get all property models grouped by the parent object ID.
    pub fn get_grouped_prop_models(&self) -> HashMap<ID, Vec<&PropertyModel>> {
        self.prop_models.values().group_by_parent()
    }

    /// Get property models belonging to a group of parent IDs,
    /// grouped by the parent ID.
    pub fn get_grouped_prop_models_for_parents(
        &self,
        parent_model_ids: Vec<ID>,
    ) -> HashMap<ID, Vec<&PropertyModel>> {
        self.prop_models
            .values()
            .filter(|p| parent_model_ids.contains(&p.object))
            .group_by_parent()
    }

    /// Iterate relation models attached to an object model
    pub fn get_relation_models_for_object_model(
        &self,
        model_id: ID,
    ) -> impl Iterator<Item = &RelationModel> {
        self.rel_models
            .values()
            .filter(move |model| model.object == model_id)
    }

    /// Iterate property models attached to an object or relation model
    pub fn get_property_models_for_parents(
        &self,
        parents: Vec<ID>,
    ) -> impl Iterator<Item = &PropertyModel> {
        self.prop_models
            .values()
            .filter(move |model| parents.contains(&model.object))
    }

    /// Find properties for a parent ID
    pub fn get_property_models_for_parent(
        &self,
        parent: ID,
    ) -> impl Iterator<Item = &PropertyModel> {
        self.prop_models
            .values()
            .filter(move |model| model.object == parent)
    }

    /// Get all relation models, grouped by their source object model ID
    pub fn get_grouped_relation_models(&self) -> HashMap<ID, Vec<&RelationModel>> {
        self.rel_models.values().group_by_parent()
    }

    /// Get reciprocal relation models, grouped by their destination model ID
    pub fn get_grouped_reciprocal_relation_models(&self) -> HashMap<ID, Vec<&RelationModel>> {
        self.rel_models
            .values()
            .into_group_map_by(|model| model.related)
    }

    //endregion Model queries

    //region Data queries

    /// Get all relation for an object
    pub fn get_relations_for_object(&self, object_id: ID) -> impl Iterator<Item = &data::Relation> {
        self.relations
            .values()
            .filter(move |rel| rel.object == object_id)
    }

    /// Get all reciprocal relation for an object
    pub fn get_reciprocal_relations_for_object(
        &self,
        object_id: ID,
    ) -> impl Iterator<Item = &data::Relation> {
        self.relations
            .values()
            .filter(move |rel| rel.related == object_id)
    }

    /// Get values attached to a parent object or relation
    pub fn get_values_for_object(&self, object_id: ID) -> impl Iterator<Item = &data::Value> {
        self.values
            .values()
            .filter(move |prop| prop.object == object_id)
    }

    /// Get values belonging to a list of objects or relations,
    /// grouped by the parent entity ID.
    pub fn get_grouped_values_for_objects(
        &self,
        parents: Vec<ID>,
    ) -> HashMap<ID, Vec<&data::Value>> {
        self.values
            .values()
            .filter(move |prop| parents.contains(&prop.object))
            .group_by_parent()
    }

    /// Get all objects belonging to a model.
    ///
    /// Use `get_objects_of_types` to specify more than one model
    pub fn get_objects_of_type(&self, model_id: ID) -> impl Iterator<Item = &data::Object> {
        self.objects
            .values()
            .filter(move |object| object.model == model_id)
    }

    /// Get all objects belonging to one of several models
    pub fn get_objects_of_types(&self, model_ids: Vec<ID>) -> impl Iterator<Item = &data::Object> {
        self.objects
            .values()
            .filter(move |object| model_ids.contains(&object.model))
    }

    /// Get all objects, grouped by their model ID
    pub fn get_grouped_objects(&self) -> HashMap<ID, Vec<&data::Object>> {
        self.objects.values().group_by_model()
    }

    /// Get object by ID
    pub fn get_object(&self, id: ID) -> Option<&data::Object> {
        self.objects.get(&id)
    }

    /// Get value by ID
    pub fn get_value(&self, id: ID) -> Option<&data::Value> {
        self.values.get(&id)
    }

    /// Get relation by ID
    pub fn get_relation(&self, id: ID) -> Option<&data::Relation> {
        self.relations.get(&id)
    }

    //endregion Data queries

    //region Model editing

    /// Define an object model
    pub fn define_object(&mut self, mut tpl: model::ObjectModel) -> Result<ID, StorageError> {
        if tpl.name.is_empty() {
            return Err(StorageError::ConstraintViolation(
                "Name must not be empty".into(),
            ));
        }

        if self
            .obj_models
            .iter()
            .find(|(_, t)| t.name == tpl.name)
            .is_some()
        {
            return Err(StorageError::ConstraintViolation(
                format!("Object model with the name \"{}\" already exists", tpl.name).into(),
            ));
        }

        debug!("Define object model \"{}\"", tpl.name);
        let id = self.next_id();
        tpl.id = id;
        self.obj_models.insert(id, tpl);
        Ok(id)
    }

    /// Define an object relation model
    pub fn define_relation(&mut self, mut rel: model::RelationModel) -> Result<ID, StorageError> {
        if rel.name.is_empty() || rel.reciprocal_name.is_empty() {
            return Err(StorageError::ConstraintViolation(
                "Names must not be empty".into(),
            ));
        }

        if !self.obj_models.contains_key(&rel.object) {
            return Err(StorageError::NotExist(
                format!("Source object model {}", rel.object).into(),
            ));
        }
        if !self.obj_models.contains_key(&rel.related) {
            return Err(StorageError::NotExist(
                format!("Related object model {}", rel.related).into(),
            ));
        }

        if let Some((_, colliding)) = self.rel_models.iter().find(|(_, other)| {
            (other.name == rel.name && other.object == rel.object) // Exact match
                || (other.name == rel.reciprocal_name && other.object == rel.related) // Our reciprocal name collides with related's own relation name
                || (other.reciprocal_name == rel.name && other.related == rel.object) // Our name name collides with a reciprocal name on the other relation
                || (other.reciprocal_name == rel.reciprocal_name && other.related == rel.related)
            // Reciprocal names collide for the same destination
        }) {
            return Err(StorageError::ConstraintViolation(
                format!(
                    "Name collision (\"{}\" / \"{}\") with existing relation (\"{}\" / \"{}\")",
                    rel.name, rel.reciprocal_name, colliding.name, colliding.reciprocal_name
                )
                .into(),
            ));
        }

        debug!(
            "Define relation model \"{}\" from {} to {}, reciprocal name \"{}\"",
            rel.name,
            self.describe_model(rel.object),
            self.describe_model(rel.related),
            rel.reciprocal_name
        );

        let id = self.next_id();
        rel.id = id;
        self.rel_models.insert(id, rel);
        Ok(id)
    }

    /// Define a property model, attached to an object or a relation
    pub fn define_property(&mut self, mut prop: model::PropertyModel) -> Result<ID, StorageError> {
        if prop.name.is_empty() {
            return Err(StorageError::ConstraintViolation(
                "Name must not be empty".into(),
            ));
        }

        if !self.obj_models.contains_key(&prop.object) {
            // Maybe it's attached to a relation?
            if !self.rel_models.contains_key(&prop.object) {
                return Err(StorageError::NotExist(
                    format!("Object or relation model {}", prop.object).into(),
                ));
            }
        }

        if prop.unique && prop.multiple {
            return Err(StorageError::Invalid(
                "Multi-value properties cannot have the \"unique\" constraint".into(),
            ));
        }

        if self
            .prop_models
            .iter()
            .find(|(_, t)| t.object == prop.object && t.name == prop.name)
            .is_some()
        {
            return Err(StorageError::ConstraintViolation(
                format!(
                    "Property with the name \"{}\" already exists on model {}",
                    prop.name,
                    self.describe_model(prop.object)
                )
                .into(),
            ));
        }

        // Ensure the default type is compatible
        prop.default = match prop.default.clone().cast_to(prop.data_type) {
            Ok(v) => v,
            Err(_) => {
                return Err(StorageError::NotExist(
                    format!("default value {:?} has invalid type", prop.default).into(),
                ));
            }
        };

        debug!(
            "Define property model \"{}\" of {}",
            prop.name,
            self.describe_model(prop.object)
        );
        let id = self.next_id();
        prop.id = id;
        self.prop_models.insert(id, prop);
        Ok(id)
    }

    /// Undefine an object model, its properties and relations. Deletes all associated data.
    pub fn undefine_object(&mut self, id: ID) -> Result<ObjectModel, StorageError> {
        return if let Some(model) = self.obj_models.remove(&id) {
            debug!("Undefine object model \"{}\"", model.name);
            // Remove relation templates
            let removed_relation_ids = map_drain_filter(&mut self.rel_models, |_k, v| {
                v.object == id || v.related == id
            })
            .keys();
            debug!("Undefined {} relation models", removed_relation_ids.len());

            // Remove related property templates
            let removed_prop_ids = map_drain_filter(&mut self.prop_models, |_k, v| {
                v.object == id || removed_relation_ids.contains(&v.object)
            })
            .keys();
            debug!("Undefined {} property models", removed_prop_ids.len());

            // Remove objects
            let removed_objects = map_drain_filter(&mut self.objects, |_k, v| v.model == id);
            debug!("Deleted {} objects", removed_objects.len());

            // Remove property values
            let removed_values = map_drain_filter(&mut self.values, |_k, v| {
                removed_prop_ids.contains(&v.model)
            });
            debug!("Deleted {} object or relation values", removed_values.len());

            // Remove relations
            let removed_relations = map_drain_filter(&mut self.relations, |_k, v| {
                removed_relation_ids.contains(&v.model)
            });
            debug!("Deleted {} object relations", removed_relations.len());

            // Related object remain untouched, so there can be a problem with orphans. This is up to the application to deal with.

            Ok(model)
        } else {
            Err(StorageError::NotExist(
                format!("object model {}", id).into(),
            ))
        };
    }

    /// Undefine a relation model and its properties. Deletes all associated data.
    pub fn undefine_relation(&mut self, id: ID) -> Result<model::RelationModel, StorageError> {
        return if let Some(model) = self.rel_models.remove(&id) {
            debug!("Undefine relation model \"{}\"", model.name);

            // Remove relations
            let removed = map_drain_filter(&mut self.relations, |_k, v| v.model == id);
            debug!("Deleted {} object relations", removed.len());

            // Remove related property templates
            let removed_prop_tpl_ids =
                map_drain_filter(&mut self.prop_models, |_k, v| v.object == id).keys();
            debug!(
                "Undefined {} relation property models",
                removed_prop_tpl_ids.len()
            );

            let removed_values = map_drain_filter(&mut self.values, |_k, v| {
                removed_prop_tpl_ids.contains(&v.model)
            });
            debug!("Deleted {} relation values", removed_values.len());

            // Related object remain untouched, so there can be a problem with orphans. This is up to the application to deal with.

            Ok(model)
        } else {
            Err(StorageError::NotExist(
                format!("relation model {}", id).into(),
            ))
        };
    }

    /// Undefine a property model and delete associated data
    pub fn undefine_property(&mut self, id: ID) -> Result<model::PropertyModel, StorageError> {
        return if let Some(t) = self.prop_models.remove(&id) {
            debug!("Undefine property model \"{}\"", t.name);

            // Remove values
            let removed_values = map_drain_filter(&mut self.values, |_k, v| v.model == id);
            debug!("Deleted {} values", removed_values.len());
            Ok(t)
        } else {
            Err(StorageError::NotExist(
                format!("property model {}", id).into(),
            ))
        };
    }

    /// Update an object model, matched by its ID
    pub fn update_object_model(&mut self, model: ObjectModel) -> Result<(), StorageError> {
        if model.name.is_empty() {
            return Err(StorageError::ConstraintViolation(
                format!("Model name must not be empty.").into(),
            ));
        }

        if !self.obj_models.contains_key(&model.id) {
            return Err(StorageError::NotExist(
                format!("Object model ID {} does not exist.", model.id).into(),
            ));
        }

        if let Some(conflict) = self
            .obj_models
            .values()
            .find(|m| m.id != model.id && m.name == model.name)
        {
            return Err(StorageError::ConstraintViolation(
                format!("Object {} already has the name {}", conflict.id, model.name).into(),
            ));
        }

        self.obj_models.insert(model.id, model);
        Ok(())
    }

    /// Update a relation model, matched by its ID
    pub fn update_relation_model(&mut self, mut rel: RelationModel) -> Result<(), StorageError> {
        if rel.name.is_empty() || rel.reciprocal_name.is_empty() {
            return Err(StorageError::ConstraintViolation(
                format!("Relation names must not be empty.").into(),
            ));
        }

        // Object and Related can't be changed, so we re-fill them from the existing model
        if let Some(existing) = self.rel_models.get(&rel.id) {
            rel.object = existing.object;
            rel.related = existing.related;
        } else {
            return Err(StorageError::NotExist(
                format!("Relation model ID {} does not exist.", rel.id).into(),
            ));
        }

        // Difficult checks ...

        // yes this is stupid and inefficient and slow and
        if let Some((_, colliding)) = self.rel_models.iter().find(|(_, other)| {
            (other.name == rel.name && other.object == rel.object && rel.id != other.id) // Exact match
                || (other.name == rel.reciprocal_name && other.object == rel.related && rel.id != other.id) // Our reciprocal name collides with related's own relation name
                || (other.reciprocal_name == rel.name && other.related == rel.object && rel.id != other.id) // Our name name collides with a reciprocal name on the other relation
                || (other.reciprocal_name == rel.reciprocal_name && other.related == rel.related && rel.id != other.id) // Reciprocal names collide for the same destination
        }) {
            return Err(StorageError::ConstraintViolation(
                format!("name collision (\"{}\" / \"{}\") with existing relation (\"{}\" / \"{}\")",
                        rel.name, rel.reciprocal_name,
                        colliding.name, colliding.reciprocal_name
                ).into()));
        }

        self.rel_models.insert(rel.id, rel);
        Ok(())
    }

    /// Update a property model, matched by its ID
    pub fn update_property_model(&mut self, mut prop: PropertyModel) -> Result<(), StorageError> {
        if prop.name.is_empty() {
            return Err(StorageError::ConstraintViolation(
                format!("Property name must not be empty.").into(),
            ));
        }

        if prop.unique && prop.multiple {
            return Err(StorageError::Invalid(
                "Multi-value properties cannot have the \"unique\" constraint".into(),
            ));
        }

        // Object can't be changed, so we re-fill them from the existing model
        if let Some(existing) = self.prop_models.get(&prop.id) {
            prop.object = existing.object;
        } else {
            return Err(StorageError::NotExist(
                format!("Property model ID {} does not exist.", prop.id).into(),
            ));
        }

        if self
            .prop_models
            .iter()
            .find(|(_, t)| t.object == prop.object && t.name == prop.name && t.id != prop.id)
            .is_some()
        {
            return Err(StorageError::ConstraintViolation(
                format!(
                    "property with the name \"{}\" already exists on {}",
                    prop.name,
                    self.describe_model(prop.object)
                )
                .into(),
            ));
        }

        // Ensure the default type is compatible
        prop.default = match prop.default.clone().cast_to(prop.data_type) {
            Ok(v) => v,
            Err(_) => {
                return Err(StorageError::NotExist(
                    format!("default value {:?} has invalid type", prop.default).into(),
                ));
            }
        };

        self.prop_models.insert(prop.id, prop);
        Ok(())
    }

    //endregion Model editing

    //region Data editing

    /// Insert a data object with properties and relations, validating the data model constraints
    pub fn insert_object(&mut self, insobj: InsertObj) -> Result<ID, StorageError> {
        let obj_model_id = insobj.model;
        debug!("Insert object {:?}", insobj);

        let obj_model = match self.obj_models.get(&obj_model_id) {
            Some(m) => m,
            None => {
                return Err(StorageError::NotExist(
                    format!("object model {}", obj_model_id).into(),
                ));
            }
        };

        let object_id = self.next_id();
        let object = data::Object {
            id: object_id,
            model: obj_model_id,
        };

        let find_values_to_insert = |values: Vec<InsertValue>,
                                     parent_id: ID,
                                     parent_model_id: ID|
         -> Result<Vec<data::Value>, StorageError> {
            let mut values_by_id = values.group_by_model();
            let mut values_to_insert = vec![];

            for (prop_model_id, prop) in self
                .prop_models
                .iter()
                .filter(|(_id, p)| p.object == parent_model_id)
            {
                if let Some(values) = values_by_id.remove(prop_model_id) {
                    if values.len() > 1 && !prop.multiple {
                        return Err(StorageError::ConstraintViolation(
                            format!(
                                "{} of {} cannot have multiple values",
                                prop,
                                self.describe_model(parent_model_id)
                            )
                            .into(),
                        ));
                    }

                    if prop.unique {
                        // we know the length is at least 1. Unique should not be allowed together
                        // with "multiple", but if it is set so, only the first value will
                        // be checked.
                        let first = &values[0];

                        // validate unique name
                        if self
                            .values
                            .iter()
                            .find(|(_, o)| {
                                &o.model == prop_model_id
                                    && o.object == parent_id
                                    && o.value == first.value
                            })
                            .is_some()
                        {
                            return Err(StorageError::ConstraintViolation(
                                format!(
                                    "Value {} already used for unique property \"{}\"",
                                    first.value,
                                    self.get_model_name(*prop_model_id),
                                )
                                .into(),
                            ));
                        }
                    }

                    for val_instance in values {
                        values_to_insert.push(data::Value {
                            id: self.next_id(),
                            object: parent_id,
                            model: prop.id,
                            value: val_instance.value.cast_to(prop.data_type).map_err(|v| {
                                StorageError::ConstraintViolation(
                                    format!("{} cannot accept value {:?}", prop, v).into(),
                                )
                            })?,
                        });
                    }
                } else {
                    if !prop.optional {
                        values_to_insert.push(data::Value {
                            id: self.next_id(),
                            object: parent_id,
                            model: prop.id,
                            value: prop.default.clone(),
                        });
                    }
                }
            }

            Ok(values_to_insert)
        };

        let mut values_to_insert = find_values_to_insert(insobj.values, object_id, obj_model_id)?;

        // And now ..... relations!
        let mut relations_by_id = insobj.relations.group_by_model();
        let mut relations_to_insert = vec![];

        for (relation_model_id, relation_model) in self
            .rel_models
            .iter()
            .filter(|(_id, r)| r.object == obj_model_id)
        {
            if let Some(instances) = relations_by_id.remove(relation_model_id) {
                if instances.len() > 1 && !relation_model.multiple {
                    return Err(StorageError::ConstraintViolation(
                        format!("{} of {} cannot be set multiply", relation_model, obj_model)
                            .into(),
                    ));
                }

                for rel_instance in instances {
                    if let Some(related) = self.objects.get(&rel_instance.related) {
                        if related.model != relation_model.related {
                            return Err(StorageError::ConstraintViolation(
                                format!(
                                    "{} of {} requires object of type {}, got {}",
                                    relation_model,
                                    obj_model,
                                    self.describe_model(relation_model.related),
                                    self.describe_model(related.model)
                                )
                                .into(),
                            ));
                        }
                    }

                    let relation_id = self.next_id();

                    // Relations can have properties
                    values_to_insert.extend(find_values_to_insert(
                        rel_instance.values,
                        relation_id,
                        *relation_model_id,
                    )?);

                    relations_to_insert.push(data::Relation {
                        id: relation_id,
                        object: object_id,
                        model: rel_instance.model,
                        related: rel_instance.related,
                    });
                }
            } else {
                if !relation_model.optional {
                    return Err(StorageError::ConstraintViolation(
                        format!("{} is required for {}", relation_model, obj_model).into(),
                    ));
                }
            }
        }

        self.objects.insert(object_id, object);

        debug!("Add {} new object relations", relations_to_insert.len());
        for rel in relations_to_insert {
            self.relations.insert(rel.id, rel);
        }

        debug!("Add {} new values", values_to_insert.len());
        for value in values_to_insert {
            self.values.insert(value.id, value);
        }

        Ok(object_id)
    }

    /// Update an existing object and its values, relations and their values.
    ///
    /// The existing data is synchronized to match the new data, that is, relations and values
    /// not specified will be deleted, these with matching IDs will be
    /// overwritten, and ones without an ID will be created and assigned a unique ID.
    pub fn update_object(&mut self, updobj: UpdateObj) -> Result<(), StorageError> {
        let old_object = self.objects.get(&updobj.id).ok_or_else(|| {
            StorageError::ConstraintViolation(format!("Object does not exist").into())
        })?;

        let updated_object_id = old_object.id;
        let updated_object_model_id = old_object.model;
        debug!("Update object {:?}", updobj);

        let obj_model = match self.obj_models.get(&updated_object_model_id) {
            Some(m) => m,
            None => {
                return Err(StorageError::NotExist(
                    format!("object model {}", updated_object_model_id).into(),
                ));
            }
        };

        // Update the object after everything else is checked

        let find_values_to_change = |values: Vec<UpsertValue>,
                                     parent_id: ID,
                                     parent_model_id: ID|
         -> Result<
            (
                // Insert (can overwrite existing, the ID will not change)
                Vec<data::Value>,
                // Delete
                Vec<ID>,
            ),
            StorageError,
        > {
            let mut values_by_model = values.group_by_model();
            let mut values_to_insert = vec![];
            let mut ids_to_delete = vec![];

            let mut existing_values_by_id = self
                .values
                .values()
                .filter(|v| v.object == parent_id)
                .group_by_model();

            for (prop_model_id, prop) in self
                .prop_models
                .iter()
                .filter(|(_id, p)| p.object == parent_model_id)
            {
                if let Some(values) = values_by_model.remove(prop_model_id) {
                    if values.len() > 1 && !prop.multiple {
                        return Err(StorageError::ConstraintViolation(
                            format!(
                                "{} of {} cannot have multiple values",
                                prop,
                                self.describe_model(parent_model_id)
                            )
                            .into(),
                        ));
                    }

                    if prop.unique {
                        // we know the length is at least 1. Unique should not be allowed together
                        // with "multiple", but if it is set so, only the first value will
                        // be checked.
                        let first = &values[0];

                        // validate unique name
                        if self
                            .values
                            .iter()
                            .find(|(_, o)| {
                                &o.model == prop_model_id
                                    && o.object == parent_id
                                    && o.value == first.value
                                    && (first.id.is_none_or_else(|id| &o.id != id))
                            })
                            .is_some()
                        {
                            return Err(StorageError::ConstraintViolation(
                                format!(
                                    "Value {} already used for unique property \"{}\"",
                                    first.value,
                                    self.get_model_name(*prop_model_id),
                                )
                                .into(),
                            ));
                        }
                    }

                    let updated_ids = values.iter().filter_map(|v| v.id).collect_vec();

                    ids_to_delete.extend(
                        existing_values_by_id
                            .remove(&prop.id)
                            .unwrap_or_default()
                            .into_iter()
                            .filter(|v| !updated_ids.contains(&v.id))
                            .map(|v| v.id),
                    );

                    for val_instance in values {
                        values_to_insert.push(data::Value {
                            id: val_instance.id.unwrap_or_else(|| self.next_id()),
                            object: parent_id,
                            model: prop.id,
                            value: val_instance.value,
                        });
                    }
                } else {
                    if !prop.optional {
                        warn!("Attempt to remove non-optional prop, do nothing");
                    } else {
                        if let Some(existing) = existing_values_by_id.remove(&prop.id) {
                            ids_to_delete.extend(existing.into_iter().map(|v| v.id));
                        }
                    }
                }
            }

            Ok((values_to_insert, ids_to_delete))
        };

        let (mut values_to_insert, mut value_ids_to_delete) =
            find_values_to_change(updobj.values, updated_object_id, updated_object_model_id)?;

        // And now ..... relations!
        let mut relations_by_model = updobj.relations.group_by_model();
        let mut relations_to_insert = vec![];
        let mut relations_to_delete = vec![];

        let mut existing_relations_by_id = self
            .relations
            .values()
            .filter(|v| v.object == updated_object_id)
            .group_by_model();

        let rel_models_by_id = self
            .rel_models
            .iter()
            .filter(|(_id, r)| r.object == updated_object_model_id);

        for (relation_model_id, relation_model) in rel_models_by_id {
            let mut updated_ids = vec![];

            if let Some(instances) = relations_by_model.remove(relation_model_id) {
                if instances.len() > 1 && !relation_model.multiple {
                    return Err(StorageError::ConstraintViolation(
                        format!("{} of {} cannot be set multiply", relation_model, obj_model)
                            .into(),
                    ));
                }

                for rel_instance in instances {
                    if let Some(related) = self.objects.get(&rel_instance.related) {
                        if related.model != relation_model.related {
                            return Err(StorageError::ConstraintViolation(
                                format!(
                                    "{} of {} requires object of type {}, got {}",
                                    relation_model,
                                    obj_model,
                                    self.describe_model(relation_model.related),
                                    self.describe_model(related.model)
                                )
                                .into(),
                            ));
                        }
                    }

                    let relation_id = rel_instance.id.unwrap_or_else(|| self.next_id());

                    // Relations can have properties
                    let (ins, del) = find_values_to_change(
                        rel_instance.values,
                        relation_id,
                        *relation_model_id,
                    )?;
                    values_to_insert.extend(ins);
                    value_ids_to_delete.extend(del);

                    relations_to_insert.push(data::Relation {
                        id: relation_id,
                        object: updated_object_id,
                        model: rel_instance.model,
                        related: rel_instance.related,
                    });

                    updated_ids.push(relation_id);
                }
            } else {
                if !relation_model.optional {
                    return Err(StorageError::ConstraintViolation(
                        format!("{} is required for {}", relation_model, obj_model).into(),
                    ));
                }
            }

            relations_to_delete.extend(
                existing_relations_by_id
                    .remove(&relation_model_id)
                    .unwrap_or_default()
                    .into_iter()
                    .filter(|rel| !updated_ids.contains(&rel.id))
                    .map(|rel| rel.id),
            );
        }

        debug!("Add {} new object relations", relations_to_insert.len());
        for rel in relations_to_insert {
            self.relations.insert(rel.id, rel);
        }

        debug!("Add {} new values", values_to_insert.len());
        for value in values_to_insert {
            self.values.insert(value.id, value);
        }

        debug!("Deleted {} values", value_ids_to_delete.len());
        for id in value_ids_to_delete {
            self.values.remove(&id);
        }

        debug!("Deleted {} object relations", relations_to_delete.len());
        for id in relations_to_delete {
            self.relations.remove(&id);
        }

        Ok(())
    }

    /// Delete an object and associated data
    pub fn delete_object(&mut self, id: ID) -> Result<data::Object, StorageError> {
        let name = self.get_object_name_by_id(id).to_string();
        return if let Some(t) = self.objects.remove(&id) {
            debug!("Delete object \"{}\"", name);
            // Remove relation templates
            let removed_relation_ids = map_drain_filter(&mut self.relations, |_k, v| {
                v.object == id || v.related == id
            })
            .keys();
            debug!("Deleted {} object relations", removed_relation_ids.len());

            // Remove values
            let removed_values = map_drain_filter(&mut self.values, |_k, v| {
                removed_relation_ids.contains(&v.object) || v.object == id
            });
            debug!("Deleted {} object values", removed_values.len());

            Ok(t)
        } else {
            Err(StorageError::NotExist(format!("object {}", id).into()))
        };
    }

    //endregion Data editing
}