group config files refactored to groups.d and subfolders, WIP common config file

README.md

@@ -36,7 +36,7 @@ You can also run the program using Cargo, that is handy for development: `cargo
 3. **Make sure you auth as the correct user!** 
 4. Paste the Oauth2 token you got into the terminal, hit enter.
 
-The program now ends. The credentials are saved in a file `groups.json`.
+The program now ends. The credentials are saved in the directory `groups.d/account@server/`, which is created if missing.
 
 You can repeat this for any number of groups.
 
@@ -44,53 +44,82 @@ In case you need to re-authenticate an existing group, do the same but use `-A`
 
 ### Editing config
 
-**Do not edit the config while the group service is running, it will overwrite your changes!**
+**Do not edit the config while the group service is running, it may overwrite your changes!**
+
+Each group is stored as a sub-directory in `groups.d`. The sub-directories are normally named after their accounts, 
+but this is not required.
+
+The group's config and state is split into three files:
+
+- `config.json` - immutable config, never changed beside when you run the -A command to reauth the group.
+  This is where the auth token and the `enabled` flag are stored.
+- `control.json` - settings and state that changes at runtime or can be set using slash commands.
+  This file is overwritten by the group service when needed.
+- `state.json` - frequently changing state data. The last-seen status/notification timestamps are kept here.
+  State is split from Control to reduce the risk of damaging the control file. Timestamps can be updated multiple times
+  per minute.
+
+The JSON files are easily editable, you can e.g. add yourself as an admin (use the e-mail format, e.g. `piggo@piggo.space`). 
+
+When adding hashtags, note that *they must be entered as lowercase*!
 
-The JSON file is easily editable, you can e.g. add yourself as an admin (use the e-mail format, e.g. `piggo@piggo.space`). 
 The file format is quite self-explanatory.
 
+#### config.json
+
 ```json
 {
-  "groups": {
+  "enabled": true,
-    "group@myserver.xyz": {
+  "acct": "group@myserver.xyz",
-      "enabled": true,
+  "appdata": {
-      "acct": "group@myserver.xyz",
+    "base": "https://myserver.xyz",
-      "appdata": {
+    "client_id": "...",
-        "base": "https://myserver.xyz",
+    "client_secret": "...",
-        "client_id": "...",
+    "redirect": "urn:ietf:wg:oauth:2.0:oob",
-        "client_secret": "...",
+    "token": "..."
-        "redirect": "urn:ietf:wg:oauth:2.0:oob",
-        "token": "..."
-      },
-      "group_tags": [
-        "grouptest"
-      ],
-      "admin_users": [
-        "admin@myserver.xyz"
-      ],
-      "member_only": false,
-      "member_users": [],
-      "banned_users": [],
-      "banned_servers": [
-        "bad-stuff-here.cc"
-      ],
-      "last_notif_ts": 1630011219000,
-      "last_status_ts": 1630011362000
-    }
   }
 }
 ```
 
-- `group_tags` - group hashtags (without the `#`). The group reblogs anything with these hashtags if the author is a member.
+#### control.json
+
+```json
+{
+  "group_tags": [
+    "grouptest"
+  ],
+  "admin_users": [
+    "admin@myserver.xyz"
+  ],
+  "member_only": false,
+  "member_users": [],
+  "banned_users": [],
+  "banned_servers": [
+    "bad-stuff-here.cc"
+  ]
+}
+```
+
+- `group_tags` - group hashtags (without the `#`), lowercase! The group reblogs anything with these hashtags if the author is a member.
 - `member_users` - group members, used to track whose hashtags should be reblogged; in member-only groups, this is also a user whitelist.
 - `banned_users` - can't post or interact with the group service
 - `banned_servers` - work like an instance block
 
+#### state.json
+
+```json
+{
+  "last_notif_ts": 1630011219000,
+  "last_status_ts": 1630011362000
+}
+```
+
 ### Running
 
-To run the group service, simply run it with no arguments. It will read what to do from `groups.json`.
+To run the group service, simply run it with no arguments. It will find groups in `groups.d` and start the service threads for you.
 
-Note that the file must be writable, it is updated at run-time.
+Note that the control and status files must be writable, they are updated at run-time. Config files can have restricted permissions
+to avoid accidental overwrite.
 
 An example systemd service file is included in the repository as well. Make sure to set up the system user/group and file permissions according to your needs. You can use targets in the included `Makefile` to manage the systemd service and look at logs.
 
@@ -116,7 +145,7 @@ These won't be shared:
 - `ducks suck`
 - `@group #ducks /i` (anything with the "ignore" command is ignored)
 - `@group /remove #ducks` (admin command, even if it includes a group hashtag)
-- `@otheruser tell me about ducks` (in a thread)
+- `@otheruser tell me about #ducks` (in a thread)
 - `@otheruser @group tell me about ducks` (in a thread)
 
 ### Commands

src/group_handler/mod.rs

@@ -15,7 +15,7 @@ use handle_mention::ProcessMention;
 
 use crate::error::GroupError;
 use crate::store::ConfigStore;
-use crate::store::data::GroupConfig;
+use crate::store::data::{CommonConfig, GroupConfig};
 use crate::utils::{LogError, normalize_acct, VisExt};
 use crate::command::StatusCommand;
 use elefren::entities::account::Account;
@@ -25,25 +25,22 @@ mod handle_mention;
 /// This is one group's config store capable of persistence
 #[derive(Debug)]
 pub struct GroupHandle {
-    pub(crate) group_account: Account,
+    pub group_account: Account,
-    pub(crate) client: FediClient,
+    pub client: FediClient,
-    pub(crate) config: GroupConfig,
+    pub config: GroupConfig,
-    pub(crate) store: Arc<ConfigStore>,
+    pub common_config: Arc<CommonConfig>,
 }
 
+// TODO move other options to common_config!
+
 // const DELAY_BEFORE_ACTION: Duration = Duration::from_millis(250);
 const DELAY_REOPEN_STREAM: Duration = Duration::from_millis(500);
-const MAX_CATCHUP_NOTIFS: usize = 30;
-// also statuses
-const MAX_CATCHUP_STATUSES: usize = 50;
 // higher because we can expect a lot of non-hashtag statuses here
 const PERIODIC_SAVE: Duration = Duration::from_secs(60);
 const SOCKET_ALIVE_TIMEOUT: Duration = Duration::from_secs(30);
 const SOCKET_RETIRE_TIME: Duration = Duration::from_secs(120);
 const PING_INTERVAL: Duration = Duration::from_secs(15); // must be < periodic save!
 
-
-
 macro_rules! grp_debug {
     ($self:ident, $f:expr) => {
         ::log::debug!(concat!("(@{}) ", $f), $self.config.get_acct());
@@ -94,16 +91,12 @@ macro_rules! grp_error {
 impl GroupHandle {
     pub async fn save(&mut self) -> Result<(), GroupError> {
         grp_debug!(self, "Saving group config & status");
-        self.store.set_group_config(self.config.clone()).await?;
+        self.config.save(false).await?;
-        grp_trace!(self, "Saved");
-        self.config.clear_dirty_status();
         Ok(())
     }
 
     pub async fn save_if_needed(&mut self) -> Result<(), GroupError> {
-        if self.config.is_dirty() {
+        self.config.save_if_needed(false).await?;
-            self.save().await?;
-        }
         Ok(())
     }
 
@@ -361,13 +354,9 @@ impl GroupHandle {
                             account: s.account.clone(),
                             status: Some(s)
                         }).await;
-                    } else {
+                    } else if private {
-                        if !private {
+                        grp_debug!(self, "mention in private without commands, discard, this is nothing");
-                            grp_debug!(self, "Detected mention status, handle as notif");
+                        return Ok(());
-                        } else {
-                            grp_debug!(self, "mention in private without commands, discard, this is nothing");
-                            return Ok(());
-                        }
                     }
                 }
             }
@@ -433,7 +422,7 @@ impl GroupHandle {
             grp_debug!(self, "Inspecting notif {}", NotificationDisplay(&n));
             notifs_to_handle.push(n);
             num += 1;
-            if num > MAX_CATCHUP_NOTIFS {
+            if num > self.common_config.max_catchup_notifs {
                 grp_warn!(self, "Too many notifs missed to catch up!");
                 break;
             }
@@ -486,7 +475,7 @@ impl GroupHandle {
 
             statuses_to_handle.push(s);
             num += 1;
-            if num > MAX_CATCHUP_STATUSES {
+            if num > self.common_config.max_catchup_statuses {
                 grp_warn!(self, "Too many statuses missed to catch up!");
                 break;
             }

src/main.rs

@@ -36,7 +36,7 @@ async fn main() -> anyhow::Result<()> {
                 .short("c")
                 .long("config")
                 .takes_value(true)
-                .help("set custom storage file, defaults to groups.json"),
+                .help("set custom config directory, defaults to groups.d"),
         )
         .arg(
             Arg::with_name("auth")
@@ -75,9 +75,8 @@ async fn main() -> anyhow::Result<()> {
         .filter_module("reqwest", LevelFilter::Warn)
         .init();
 
-    let store = store::ConfigStore::new(StoreOptions {
+    let store = store::ConfigStore::load_from_fs(StoreOptions {
-        store_path: args.value_of("config").unwrap_or("groups.json").to_string(),
+        store_dir: args.value_of("config").unwrap_or(".").to_string(),
-        save_pretty: true,
     })
     .await?;
 
@@ -112,7 +111,7 @@ async fn main() -> anyhow::Result<()> {
     }
 
     // Start
-    let groups = store.spawn_groups().await;
+    let groups = store.spawn_groups().await?;
 
     let mut handles = vec![];
     for mut g in groups {

src/store/data.rs

@@ -1,33 +1,29 @@
 use std::collections::{HashMap, HashSet};
+use std::path::{Path, PathBuf};
 
 use elefren::AppData;
 
 use crate::error::GroupError;
 
-/// This is the inner data struct holding the config
+#[derive(Debug, Clone, Serialize, Deserialize)]
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(default)]
-pub(crate) struct Config {
+pub struct CommonConfig {
-    pub(crate) groups: HashMap<String, GroupConfig>,
+    pub max_catchup_notifs: usize,
+    pub max_catchup_statuses: usize,
 }
 
-impl Config {
+impl Default for CommonConfig {
-    // pub(crate) fn iter_groups(&self) -> impl Iterator<Item = &GroupConfig> {
+    fn default() -> Self {
-    //     self.groups.values()
+        Self {
-    // }
+            max_catchup_notifs: 30,
-
+            max_catchup_statuses: 50,
-    pub(crate) fn get_group_config(&self, acct: &str) -> Option<&GroupConfig> {
+        }
-        self.groups.get(acct)
-    }
-
-    pub(crate) fn set_group_config(&mut self, grp: GroupConfig) {
-        self.groups.insert(grp.acct.clone(), grp);
     }
 }
 
-/// This is the inner data struct holding a group's config
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default)]
-pub(crate) struct GroupConfig {
+struct FixedConfig {
     enabled: bool,
     /// Group actor's acct
     acct: String,
@@ -35,6 +31,50 @@ pub(crate) struct GroupConfig {
     appdata: AppData,
     /// Server's character limit
     character_limit: usize,
+    #[serde(skip)]
+    _dirty: bool,
+    #[serde(skip)]
+    _path: PathBuf,
+}
+
+macro_rules! impl_change_tracking {
+    ($struc:ident) => {
+        impl $struc {
+            pub(crate) fn mark_dirty(&mut self) {
+                self._dirty = true;
+            }
+
+            pub(crate) fn is_dirty(&self) -> bool {
+                self._dirty
+            }
+
+            pub(crate) fn clear_dirty_status(&mut self) {
+                self._dirty = false;
+            }
+
+            pub(crate) async fn save_if_needed(&mut self) -> Result<(), GroupError> {
+                if self._dirty {
+                    self.save().await?;
+                }
+                Ok(())
+            }
+
+            pub(crate) async fn save(&mut self) -> Result<(), GroupError> {
+                tokio::fs::write(&self._path, serde_json::to_string_pretty(&self)?.as_bytes()).await?;
+                self._dirty = false;
+                Ok(())
+            }
+        }
+    };
+}
+
+impl_change_tracking!(FixedConfig);
+impl_change_tracking!(MutableConfig);
+impl_change_tracking!(StateConfig);
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+struct MutableConfig {
     /// Hashtags the group will auto-boost from it's members
     group_tags: HashSet<String>,
     /// List of admin account "acct" names, e.g. piggo@piggo.space
@@ -49,15 +89,38 @@ pub(crate) struct GroupConfig {
     member_only: bool,
     /// Banned domain names, e.g. kiwifarms.cc
     banned_servers: HashSet<String>,
+    #[serde(skip)]
+    _dirty: bool,
+    #[serde(skip)]
+    _path: PathBuf,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+struct StateConfig {
     /// Last seen notification timestamp (millis)
     last_notif_ts: u64,
     /// Last seen status timestamp (millis)
     last_status_ts: u64,
     #[serde(skip)]
-    dirty: bool,
+    _dirty: bool,
+    #[serde(skip)]
+    _path: PathBuf,
 }
 
-impl Default for GroupConfig {
+/// This is the inner data struct holding a group's config
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+pub struct GroupConfig {
+    /// Fixed config that we only read
+    config: FixedConfig,
+    /// Mutable config we can write
+    control: MutableConfig,
+    /// State config with timestamps and transient data that is changed frequently
+    state: StateConfig,
+}
+
+impl Default for FixedConfig {
     fn default() -> Self {
         Self {
             enabled: true,
@@ -70,6 +133,15 @@ impl Default for GroupConfig {
                 token: Default::default(),
             },
             character_limit: 5000,
+            _dirty: false,
+            _path: PathBuf::default(),
+        }
+    }
+}
+
+impl Default for MutableConfig {
+    fn default() -> Self {
+        Self {
             group_tags: Default::default(),
             admin_users: Default::default(),
             member_users: Default::default(),
@@ -77,96 +149,279 @@ impl Default for GroupConfig {
             optout_users: Default::default(),
             member_only: false,
             banned_servers: Default::default(),
+            _dirty: false,
+            _path: PathBuf::default(),
+        }
+    }
+}
+
+impl Default for StateConfig {
+    fn default() -> Self {
+        Self {
             last_notif_ts: 0,
             last_status_ts: 0,
-            dirty: false,
+            _dirty: false,
+            _path: PathBuf::default(),
         }
     }
 }
 
-impl GroupConfig {
+impl Default for GroupConfig {
-    pub(crate) fn new(acct: String, appdata: AppData) -> Self {
+    fn default() -> Self {
         Self {
-            acct,
+            config: Default::default(),
-            appdata,
+            control: Default::default(),
+            state: Default::default(),
+        }
+    }
+}
+
+async fn load_or_create_control_file(control_path : impl AsRef<Path>) -> Result<MutableConfig, GroupError> {
+    let control_path = control_path.as_ref();
+    let mut dirty = false;
+    let mut control : MutableConfig = if control_path.is_file() {
+        let f = tokio::fs::read(&control_path).await?;
+        let mut control: MutableConfig = serde_json::from_slice(&f)?;
+        control._path = control_path.to_owned();
+        control
+    } else {
+        debug!("control file missing, creating empty");
+        dirty = true;
+        MutableConfig {
+            _path: control_path.to_owned(),
             ..Default::default()
         }
+    };
+    if dirty {
+        control.save().await?;
+        // tokio::fs::write(&control._path, serde_json::to_string(&control)?.as_bytes()).await?;
     }
+    Ok(control)
+}
 
-    pub(crate) fn get_character_limit(&self) -> usize {
+async fn load_or_create_state_file(state_path : impl AsRef<Path>) -> Result<StateConfig, GroupError> {
-        self.character_limit
+    let state_path = state_path.as_ref();
+    let mut dirty = false;
+    let mut state : StateConfig = if state_path.is_file() {
+        let f = tokio::fs::read(&state_path).await?;
+        let mut control: StateConfig = serde_json::from_slice(&f)?;
+        control._path = state_path.to_owned();
+        control
+    } else {
+        debug!("state file missing, creating empty");
+        dirty = true;
+        StateConfig {
+            _path: state_path.to_owned(),
+            ..Default::default()
+        }
+    };
+    if dirty {
+        state.save().await?;
+        // tokio::fs::write(&state._path, serde_json::to_string(&state)?.as_bytes()).await?;
     }
+    Ok(state)
+}
 
-    pub(crate) fn is_enabled(&self) -> bool {
+impl GroupConfig {
-        self.enabled
+    pub(crate) fn is_dirty(&self) -> bool {
+        self.config.is_dirty()
+            || self.control.is_dirty()
+            || self.state.is_dirty()
+    }
+
+    /// Save only what changed
+    pub(crate) async fn save_if_needed(&mut self, danger_allow_overwriting_config: bool) -> Result<(), GroupError> {
+        if danger_allow_overwriting_config {
+            self.config.save_if_needed().await?;
+        }
+        self.control.save_if_needed().await?;
+        self.state.save_if_needed().await?;
+        Ok(())
+    }
+
+    /// Save all unconditionally
+    pub(crate) async fn save(&mut self, danger_allow_overwriting_config: bool) -> Result<(), GroupError> {
+        if danger_allow_overwriting_config {
+            self.config.save().await?;
+        }
+        self.control.save().await?;
+        self.state.save().await?;
+        Ok(())
     }
 
-    /*
+    /// (re)init using new authorization
-    pub(crate) fn set_enabled(&mut self, ena: bool) {
+    pub(crate) async fn from_appdata(acct: String, appdata: AppData, group_dir: PathBuf) -> Result<Self, GroupError> {
-        self.enabled = ena;
+        if !group_dir.is_dir() {
-        self.mark_dirty();
+            debug!("Creating group directory");
+            tokio::fs::create_dir_all(&group_dir).await?;
+        }
+
+        let config_path = group_dir.join("config.json");
+        let control_path = group_dir.join("control.json");
+        let state_path = group_dir.join("state.json");
+
+        // try to reuse content of the files, if present
+
+        /* config */
+        let mut dirty = false;
+        let mut config: FixedConfig = if config_path.is_file() {
+            let f = tokio::fs::read(&config_path).await?;
+            let mut config: FixedConfig = serde_json::from_slice(&f)?;
+            config._path = config_path;
+            if config.appdata != appdata {
+                config.appdata = appdata;
+                dirty = true;
+            }
+            if config.acct != acct {
+                config.acct = acct.clone();
+                dirty = true;
+            }
+            config
+        } else {
+            dirty = true;
+            FixedConfig {
+                acct: acct.clone(),
+                appdata,
+                _path: config_path,
+                ..Default::default()
+            }
+        };
+        if dirty {
+            debug!("config file for {} changed, creating/updating", acct);
+            config.save().await?;
+        }
+
+        /* control */
+        let control = load_or_create_control_file(control_path).await?;
+
+        /* state */
+        let state = load_or_create_state_file(state_path).await?;
+
+        let g = GroupConfig {
+            config,
+            control,
+            state
+        };
+        g.warn_of_bad_config();
+        Ok(g)
+    }
+
+    pub(crate) async fn from_dir(group_dir: PathBuf) -> Result<Self, GroupError> {
+        let config_path = group_dir.join("config.json");
+        let control_path = group_dir.join("control.json");
+        let state_path = group_dir.join("state.json");
+
+        // try to reuse content of the files, if present
+
+        /* config */
+        let f = tokio::fs::read(&config_path).await?;
+        let mut config: FixedConfig = serde_json::from_slice(&f)?;
+        config._path = config_path;
+
+        /* control */
+        let control = load_or_create_control_file(control_path).await?;
+
+        /* state */
+        let state = load_or_create_state_file(state_path).await?;
+
+        let g = GroupConfig {
+            config,
+            control,
+            state
+        };
+        g.warn_of_bad_config();
+        Ok(g)
+    }
+
+    fn warn_of_bad_config(&self) {
+        for t in &self.control.group_tags {
+            if &t.to_lowercase() != t {
+                warn!("Group {} hashtag \"{}\" is not lowercase, it won't work!", self.config.acct, t);
+            }
+        }
+
+        for u in self.control.admin_users.iter()
+            .chain(self.control.member_users.iter())
+            .chain(self.control.banned_users.iter())
+            .chain(self.control.optout_users.iter())
+        {
+            if &u.to_lowercase() != u {
+                warn!("Group {} config contains a user with non-lowercase name \"{}\", it won't work!", self.config.acct, u);
+            }
+
+            if u.starts_with('@') || u.chars().filter(|c| *c == '@').count() != 1 {
+                warn!("Group {} config contains an invalid user name: {}", self.config.acct, u);
+            }
+        }
+    }
+
+    pub(crate) fn get_character_limit(&self) -> usize {
+        self.config.character_limit
+    }
+
+    pub(crate) fn is_enabled(&self) -> bool {
+        self.config.enabled
     }
-    */
 
     pub(crate) fn get_appdata(&self) -> &AppData {
-        &self.appdata
+        &self.config.appdata
     }
 
     pub(crate) fn set_appdata(&mut self, appdata: AppData) {
-        if self.appdata != appdata {
+        if self.config.appdata != appdata {
-            self.mark_dirty();
+            self.config.mark_dirty();
         }
-        self.appdata = appdata;
+        self.config.appdata = appdata;
     }
 
-    pub(crate) fn get_admins(&self) -> impl Iterator<Item = &String> {
+    pub(crate) fn get_admins(&self) -> impl Iterator<Item=&String> {
-        self.admin_users.iter()
+        self.control.admin_users.iter()
     }
 
-    pub(crate) fn get_members(&self) -> impl Iterator<Item = &String> {
+    pub(crate) fn get_members(&self) -> impl Iterator<Item=&String> {
-        self.member_users.iter()
+        self.control.member_users.iter()
     }
 
-    pub(crate) fn get_tags(&self) -> impl Iterator<Item = &String> {
+    pub(crate) fn get_tags(&self) -> impl Iterator<Item=&String> {
-        self.group_tags.iter()
+        self.control.group_tags.iter()
     }
 
     pub(crate) fn set_last_notif(&mut self, ts: u64) {
-        if self.last_notif_ts != ts {
+        if self.state.last_notif_ts != ts {
-            self.mark_dirty();
+            self.state.mark_dirty();
         }
-        self.last_notif_ts = self.last_notif_ts.max(ts);
+        self.state.last_notif_ts = self.state.last_notif_ts.max(ts);
     }
 
     pub(crate) fn get_last_notif(&self) -> u64 {
-        self.last_notif_ts
+        self.state.last_notif_ts
     }
 
     pub(crate) fn set_last_status(&mut self, ts: u64) {
-        if self.last_status_ts != ts {
+        if self.state.last_status_ts != ts {
-            self.mark_dirty();
+            self.state.mark_dirty();
         }
-        self.last_status_ts = self.last_status_ts.max(ts);
+        self.state.last_status_ts = self.state.last_status_ts.max(ts);
     }
 
     pub(crate) fn get_last_status(&self) -> u64 {
-        self.last_status_ts
+        self.state.last_status_ts
     }
 
     pub(crate) fn get_acct(&self) -> &str {
-        &self.acct
+        &self.config.acct
     }
 
     pub(crate) fn is_optout(&self, acct: &str) -> bool {
-        self.optout_users.contains(acct)
+        self.control.optout_users.contains(acct)
     }
 
     pub(crate) fn is_admin(&self, acct: &str) -> bool {
-        self.admin_users.contains(acct)
+        self.control.admin_users.contains(acct)
     }
 
     pub(crate) fn is_member(&self, acct: &str) -> bool {
-        self.member_users.contains(acct)
+        self.control.member_users.contains(acct)
     }
 
     pub(crate) fn is_member_or_admin(&self, acct: &str) -> bool {
@@ -175,11 +430,11 @@ impl GroupConfig {
     }
 
     pub(crate) fn is_banned(&self, acct: &str) -> bool {
-        self.banned_users.contains(acct) || self.is_users_server_banned(acct)
+        self.control.banned_users.contains(acct) || self.is_users_server_banned(acct)
     }
 
     pub(crate) fn is_server_banned(&self, server: &str) -> bool {
-        self.banned_servers.contains(server)
+        self.control.banned_servers.contains(server)
     }
 
     /// Check if the user's server is banned
@@ -201,12 +456,12 @@ impl GroupConfig {
             if self.is_banned(acct) {
                 return Err(GroupError::UserIsBanned);
             }
-            self.admin_users.insert(acct.to_owned())
+            self.control.admin_users.insert(acct.to_owned())
         } else {
-            self.admin_users.remove(acct)
+            self.control.admin_users.remove(acct)
         };
         if change {
-            self.mark_dirty();
+            self.control.mark_dirty();
         }
         Ok(())
     }
@@ -216,24 +471,24 @@ impl GroupConfig {
             if self.is_banned(acct) {
                 return Err(GroupError::UserIsBanned);
             }
-            self.member_users.insert(acct.to_owned())
+            self.control.member_users.insert(acct.to_owned())
         } else {
-            self.member_users.remove(acct)
+            self.control.member_users.remove(acct)
         };
         if change {
-            self.mark_dirty();
+            self.control.mark_dirty();
         }
         Ok(())
     }
 
     pub(crate) fn set_optout(&mut self, acct: &str, optout: bool) {
         let change = if optout {
-            self.optout_users.insert(acct.to_owned())
+            self.control.optout_users.insert(acct.to_owned())
         } else {
-            self.optout_users.remove(acct)
+            self.control.optout_users.remove(acct)
         };
         if change {
-            self.mark_dirty();
+            self.control.mark_dirty();
         }
     }
 
@@ -244,72 +499,60 @@ impl GroupConfig {
                 return Err(GroupError::UserIsAdmin);
             }
             // Banned user is also kicked
-            change |= self.member_users.remove(acct);
+            change |= self.control.member_users.remove(acct);
-            change |= self.banned_users.insert(acct.to_owned());
+            change |= self.control.banned_users.insert(acct.to_owned());
         } else {
-            change |= self.banned_users.remove(acct);
+            change |= self.control.banned_users.remove(acct);
         }
         if change {
-            self.mark_dirty();
+            self.control.mark_dirty();
         }
         Ok(())
     }
 
     pub(crate) fn ban_server(&mut self, server: &str, ban: bool) -> Result<(), GroupError> {
         let changed = if ban {
-            for acct in &self.admin_users {
+            for acct in &self.control.admin_users {
                 let acct_server = acct_to_server(acct);
                 if acct_server == server {
                     return Err(GroupError::AdminsOnServer);
                 }
             }
-            self.banned_servers.insert(server.to_owned())
+            self.control.banned_servers.insert(server.to_owned())
         } else {
-            self.banned_servers.remove(server)
+            self.control.banned_servers.remove(server)
         };
         if changed {
-            self.mark_dirty();
+            self.control.mark_dirty();
         }
         Ok(())
     }
 
     pub(crate) fn add_tag(&mut self, tag: &str) {
-        if self.group_tags.insert(tag.to_string()) {
+        if self.control.group_tags.insert(tag.to_string()) {
-            self.mark_dirty();
+            self.control.mark_dirty();
         }
     }
 
     pub(crate) fn remove_tag(&mut self, tag: &str) {
-        if self.group_tags.remove(tag) {
+        if self.control.group_tags.remove(tag) {
-            self.mark_dirty();
+            self.control.mark_dirty();
         }
     }
 
     pub(crate) fn is_tag_followed(&self, tag: &str) -> bool {
-        self.group_tags.contains(tag)
+        self.control.group_tags.contains(tag)
     }
 
     pub(crate) fn set_member_only(&mut self, member_only: bool) {
-        if self.member_only != member_only {
+        if self.control.member_only != member_only {
-            self.mark_dirty();
+            self.control.mark_dirty();
         }
-        self.member_only = member_only;
+        self.control.member_only = member_only;
     }
 
     pub(crate) fn is_member_only(&self) -> bool {
-        self.member_only
+        self.control.member_only
-    }
-
-    pub(crate) fn mark_dirty(&mut self) {
-        self.dirty = true;
-    }
-
-    pub(crate) fn is_dirty(&self) -> bool {
-        self.dirty
-    }
-
-    pub(crate) fn clear_dirty_status(&mut self) {
-        self.dirty = false;
     }
 }
 

src/store/mod.rs

@@ -5,19 +5,19 @@ use elefren::{scopes, FediClient, Registration, Scopes};
 use futures::StreamExt;
 use tokio::sync::RwLock;
 
-use data::{Config, GroupConfig};
+use data::{GroupConfig};
 
 use crate::error::GroupError;
 use crate::group_handler::GroupHandle;
 use std::time::Duration;
+use crate::store::data::CommonConfig;
 
 pub(crate) mod data;
 
 #[derive(Debug, Default)]
 pub struct ConfigStore {
     store_path: PathBuf,
-    save_pretty: bool,
+    config: Arc<CommonConfig>,
-    data: tokio::sync::RwLock<Config>,
 }
 
 #[derive(Debug)]
@@ -28,29 +28,49 @@ pub struct NewGroupOptions {
 
 #[derive(Debug)]
 pub struct StoreOptions {
-    pub store_path: String,
+    pub store_dir: String,
-    pub save_pretty: bool,
 }
 
 impl ConfigStore {
     /// Create a new instance of the store.
     /// If a path is given, it will try to load the content from a file.
-    pub async fn new(options: StoreOptions) -> Result<Arc<Self>, GroupError> {
+    pub async fn load_from_fs(options: StoreOptions) -> Result<Arc<Self>, GroupError> {
-        let path: &Path = options.store_path.as_ref();
+        let given_path: &Path = options.store_dir.as_ref();
+
+        let mut common_file : Option<PathBuf> = None;
+        let base_dir : PathBuf;
+        if given_path.is_file() {
+            if given_path.extension().unwrap_or_default().to_string_lossy() == "json" {
+                // this is a groups.json file
+                common_file = Some(given_path.to_owned());
+                base_dir = given_path.parent().ok_or_else(|| GroupError::BadConfig("no parent dir".into()))?.to_owned();
+            } else {
+                return Err(GroupError::BadConfig("bad config file, should be JSON".into()));
+            }
+        } else if given_path.is_dir() {
+            let cf = given_path.join("groups.json");
+            if cf.is_file() {
+                common_file = Some(cf);
+            }
+            base_dir = given_path.to_owned();
+        } else {
+            return Err(GroupError::BadConfig("bad config file/dir".into()));
+        }
 
-        let config = if path.is_file() {
+        if !base_dir.is_dir() {
-            let f = tokio::fs::read(path).await?;
+            return Err(GroupError::BadConfig("base dir does not exist".into()));
+        }
+
+        let config : CommonConfig  = if let Some(cf) = &common_file {
+            let f = tokio::fs::read(&cf).await?;
             serde_json::from_slice(&f)?
         } else {
-            let empty = Config::default();
+            CommonConfig::default()
-            tokio::fs::write(path, serde_json::to_string(&empty)?.as_bytes()).await?;
-            empty
         };
 
         Ok(Arc::new(Self {
-            store_path: path.to_owned(),
+            store_path: base_dir.to_owned(),
-            save_pretty: options.save_pretty,
+            config: Arc::new(config),
-            data: RwLock::new(config),
         }))
     }
 
@@ -67,10 +87,11 @@ impl ConfigStore {
         let client = elefren::helpers::cli::authenticate(registration).await?;
         let appdata = client.data.clone();
 
-        let data = GroupConfig::new(opts.acct.clone(), appdata);
+        let group_dir = self.store_path.join(&opts.acct);
+
+        let data = GroupConfig::from_appdata(opts.acct.clone(), appdata, group_dir).await?;
 
         // save & persist
-        self.set_group_config(data.clone()).await?;
 
         let group_account = match client.verify_credentials().await {
             Ok(account) => {
@@ -90,15 +111,16 @@ impl ConfigStore {
             group_account,
             client,
             config: data,
-            store: self.clone(),
+            common_config: self.config.clone(),
         })
     }
 
     /// Re-auth an existing group
     pub async fn reauth_group(self: &Arc<Self>, acct: &str) -> Result<GroupHandle, GroupError> {
-        let groups = self.data.read().await;
+
-        let mut config = groups.get_group_config(acct).ok_or(GroupError::GroupNotExist)?.clone();
+        let group_dir = self.store_path.join(&acct);
-        drop(groups);
+
+        let mut config = GroupConfig::from_dir(group_dir).await?;
 
         println!("--- Re-authenticating bot user @{} ---", acct);
         let registration = Registration::new(config.get_appdata().base.to_string())
@@ -114,7 +136,7 @@ impl ConfigStore {
         let appdata = client.data.clone();
 
         config.set_appdata(appdata);
-        self.set_group_config(config.clone()).await?;
+        config.save_if_needed(true).await?;
 
         let group_account = match client.verify_credentials().await {
             Ok(account) => {
@@ -134,95 +156,70 @@ impl ConfigStore {
             group_account,
             client,
             config,
-            store: self.clone(),
+            common_config: self.config.clone(),
         })
     }
 
     /// Spawn existing group using saved creds
-    pub async fn spawn_groups(self: Arc<Self>) -> Vec<GroupHandle> {
+    pub async fn spawn_groups(self: Arc<Self>) -> Result<Vec<GroupHandle>, GroupError> {
-        let groups = self.data.read().await.clone();
+        let dirs = std::fs::read_dir(&self.store_path.join("groups.d"))?;
-        let groups_iter = groups.groups.into_values();
 
         // Connect in parallel
-        futures::stream::iter(groups_iter)
+        Ok(futures::stream::iter(dirs)
-            .map(|gc| async {
+            .map(|entry_maybe : Result<std::fs::DirEntry, std::io::Error>| async {
-                if !gc.is_enabled() {
+                match entry_maybe {
-                    debug!("Group @{} is DISABLED", gc.get_acct());
+                    Ok(entry) => {
-                    return None;
+                        let mut gc = GroupConfig::from_dir(entry.path())
-                }
+                            .await.ok()?;
 
-                debug!("Connecting to @{}", gc.get_acct());
+                        if !gc.is_enabled() {
+                            debug!("Group @{} is DISABLED", gc.get_acct());
+                            return None;
+                        }
 
-                let client = FediClient::from(gc.get_appdata().clone());
+                        debug!("Connecting to @{}", gc.get_acct());
 
-                let my_account = match client.verify_credentials().await {
+                        let client = FediClient::from(gc.get_appdata().clone());
-                    Ok(account) => {
+
-                        info!(
+                        let my_account = match client.verify_credentials().await {
+                            Ok(account) => {
+                                info!(
                             "Group account verified: @{}, \"{}\"",
                             account.acct, account.display_name
                         );
-                        account
+                                account
+                            }
+                            Err(e) => {
+                                error!("Group @{} auth error: {}", gc.get_acct(), e);
+                                return None;
+                            }
+                        };
+
+                        Some(GroupHandle {
+                            group_account: my_account,
+                            client,
+                            config: gc,
+                            common_config: self.config.clone(),
+                        })
                     }
                     Err(e) => {
-                        error!("Group @{} auth error: {}", gc.get_acct(), e);
+                        error!("{}", e);
-                        return None;
+                        None
                     }
-                };
+                }
-
-                Some(GroupHandle {
-                    group_account: my_account,
-                    client,
-                    config: gc,
-                    store: self.clone(),
-                })
             })
             .buffer_unordered(8)
             .collect::<Vec<_>>()
             .await
             .into_iter()
             .flatten()
-            .collect()
+            .collect())
     }
 
     pub async fn group_exists(&self, acct : &str) -> bool {
-        self.data.read().await.groups.contains_key(acct)
+        self.store_path.join(acct)
-    }
+            .join("config.json")
-
+            .is_file()
-    /*
-    pub(crate) async fn get_group_config(&self, group: &str) -> Option<GroupConfig> {
-        let c = self.data.read().await;
-        c.get_group_config(group).cloned()
-    }
-    */
-
-    //noinspection RsSelfConvention
-    /// Set group config to the store. The store then saved.
-    pub(crate) async fn set_group_config(&self, config: GroupConfig) -> Result<(), GroupError> {
-        trace!("Locking mutex");
-        if let Ok(mut data) = tokio::time::timeout(Duration::from_secs(1), self.data.write()).await {
-            trace!("Locked");
-            data.set_group_config(config);
-            trace!("Writing file");
-            self.persist(&data).await?;
-        } else {
-            error!("DEADLOCK? Timeout waiting for data RW Lock in settings store");
-        }
-        Ok(())
-    }
-
-    /// Persist the store
-    async fn persist(&self, data: &Config) -> Result<(), GroupError> {
-        tokio::fs::write(
-            &self.store_path,
-            if self.save_pretty {
-                serde_json::to_string_pretty(&data)
-            } else {
-                serde_json::to_string(&data)
-            }?
-            .as_bytes(),
-        )
-        .await?;
-        Ok(())
     }
 }