group config files refactored to groups.d and subfolders, WIP common config file

This commit is contained in:
2021-10-05 00:44:39 +02:00
parent 3a4f0ef153
commit 7ea6225ae9
5 changed files with 509 additions and 252 deletions
+345 -102
View File
@@ -1,33 +1,29 @@
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use elefren::AppData;
use crate::error::GroupError;
/// This is the inner data struct holding the config
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub(crate) struct Config {
pub(crate) groups: HashMap<String, GroupConfig>,
}
impl Config {
// pub(crate) fn iter_groups(&self) -> impl Iterator<Item = &GroupConfig> {
// self.groups.values()
// }
pub(crate) fn get_group_config(&self, acct: &str) -> Option<&GroupConfig> {
self.groups.get(acct)
}
pub(crate) fn set_group_config(&mut self, grp: GroupConfig) {
self.groups.insert(grp.acct.clone(), grp);
}
}
/// This is the inner data struct holding a group's config
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub(crate) struct GroupConfig {
pub struct CommonConfig {
pub max_catchup_notifs: usize,
pub max_catchup_statuses: usize,
}
impl Default for CommonConfig {
fn default() -> Self {
Self {
max_catchup_notifs: 30,
max_catchup_statuses: 50,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
struct FixedConfig {
enabled: bool,
/// Group actor's acct
acct: String,
@@ -35,6 +31,50 @@ pub(crate) struct GroupConfig {
appdata: AppData,
/// Server's character limit
character_limit: usize,
#[serde(skip)]
_dirty: bool,
#[serde(skip)]
_path: PathBuf,
}
macro_rules! impl_change_tracking {
($struc:ident) => {
impl $struc {
pub(crate) fn mark_dirty(&mut self) {
self._dirty = true;
}
pub(crate) fn is_dirty(&self) -> bool {
self._dirty
}
pub(crate) fn clear_dirty_status(&mut self) {
self._dirty = false;
}
pub(crate) async fn save_if_needed(&mut self) -> Result<(), GroupError> {
if self._dirty {
self.save().await?;
}
Ok(())
}
pub(crate) async fn save(&mut self) -> Result<(), GroupError> {
tokio::fs::write(&self._path, serde_json::to_string_pretty(&self)?.as_bytes()).await?;
self._dirty = false;
Ok(())
}
}
};
}
impl_change_tracking!(FixedConfig);
impl_change_tracking!(MutableConfig);
impl_change_tracking!(StateConfig);
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
struct MutableConfig {
/// Hashtags the group will auto-boost from it's members
group_tags: HashSet<String>,
/// List of admin account "acct" names, e.g. piggo@piggo.space
@@ -49,15 +89,38 @@ pub(crate) struct GroupConfig {
member_only: bool,
/// Banned domain names, e.g. kiwifarms.cc
banned_servers: HashSet<String>,
#[serde(skip)]
_dirty: bool,
#[serde(skip)]
_path: PathBuf,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
struct StateConfig {
/// Last seen notification timestamp (millis)
last_notif_ts: u64,
/// Last seen status timestamp (millis)
last_status_ts: u64,
#[serde(skip)]
dirty: bool,
_dirty: bool,
#[serde(skip)]
_path: PathBuf,
}
impl Default for GroupConfig {
/// This is the inner data struct holding a group's config
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct GroupConfig {
/// Fixed config that we only read
config: FixedConfig,
/// Mutable config we can write
control: MutableConfig,
/// State config with timestamps and transient data that is changed frequently
state: StateConfig,
}
impl Default for FixedConfig {
fn default() -> Self {
Self {
enabled: true,
@@ -70,6 +133,15 @@ impl Default for GroupConfig {
token: Default::default(),
},
character_limit: 5000,
_dirty: false,
_path: PathBuf::default(),
}
}
}
impl Default for MutableConfig {
fn default() -> Self {
Self {
group_tags: Default::default(),
admin_users: Default::default(),
member_users: Default::default(),
@@ -77,96 +149,279 @@ impl Default for GroupConfig {
optout_users: Default::default(),
member_only: false,
banned_servers: Default::default(),
last_notif_ts: 0,
last_status_ts: 0,
dirty: false,
_dirty: false,
_path: PathBuf::default(),
}
}
}
impl GroupConfig {
pub(crate) fn new(acct: String, appdata: AppData) -> Self {
impl Default for StateConfig {
fn default() -> Self {
Self {
acct,
appdata,
last_notif_ts: 0,
last_status_ts: 0,
_dirty: false,
_path: PathBuf::default(),
}
}
}
impl Default for GroupConfig {
fn default() -> Self {
Self {
config: Default::default(),
control: Default::default(),
state: Default::default(),
}
}
}
async fn load_or_create_control_file(control_path : impl AsRef<Path>) -> Result<MutableConfig, GroupError> {
let control_path = control_path.as_ref();
let mut dirty = false;
let mut control : MutableConfig = if control_path.is_file() {
let f = tokio::fs::read(&control_path).await?;
let mut control: MutableConfig = serde_json::from_slice(&f)?;
control._path = control_path.to_owned();
control
} else {
debug!("control file missing, creating empty");
dirty = true;
MutableConfig {
_path: control_path.to_owned(),
..Default::default()
}
};
if dirty {
control.save().await?;
// tokio::fs::write(&control._path, serde_json::to_string(&control)?.as_bytes()).await?;
}
Ok(control)
}
async fn load_or_create_state_file(state_path : impl AsRef<Path>) -> Result<StateConfig, GroupError> {
let state_path = state_path.as_ref();
let mut dirty = false;
let mut state : StateConfig = if state_path.is_file() {
let f = tokio::fs::read(&state_path).await?;
let mut control: StateConfig = serde_json::from_slice(&f)?;
control._path = state_path.to_owned();
control
} else {
debug!("state file missing, creating empty");
dirty = true;
StateConfig {
_path: state_path.to_owned(),
..Default::default()
}
};
if dirty {
state.save().await?;
// tokio::fs::write(&state._path, serde_json::to_string(&state)?.as_bytes()).await?;
}
Ok(state)
}
impl GroupConfig {
pub(crate) fn is_dirty(&self) -> bool {
self.config.is_dirty()
|| self.control.is_dirty()
|| self.state.is_dirty()
}
/// Save only what changed
pub(crate) async fn save_if_needed(&mut self, danger_allow_overwriting_config: bool) -> Result<(), GroupError> {
if danger_allow_overwriting_config {
self.config.save_if_needed().await?;
}
self.control.save_if_needed().await?;
self.state.save_if_needed().await?;
Ok(())
}
/// Save all unconditionally
pub(crate) async fn save(&mut self, danger_allow_overwriting_config: bool) -> Result<(), GroupError> {
if danger_allow_overwriting_config {
self.config.save().await?;
}
self.control.save().await?;
self.state.save().await?;
Ok(())
}
/// (re)init using new authorization
pub(crate) async fn from_appdata(acct: String, appdata: AppData, group_dir: PathBuf) -> Result<Self, GroupError> {
if !group_dir.is_dir() {
debug!("Creating group directory");
tokio::fs::create_dir_all(&group_dir).await?;
}
let config_path = group_dir.join("config.json");
let control_path = group_dir.join("control.json");
let state_path = group_dir.join("state.json");
// try to reuse content of the files, if present
/* config */
let mut dirty = false;
let mut config: FixedConfig = if config_path.is_file() {
let f = tokio::fs::read(&config_path).await?;
let mut config: FixedConfig = serde_json::from_slice(&f)?;
config._path = config_path;
if config.appdata != appdata {
config.appdata = appdata;
dirty = true;
}
if config.acct != acct {
config.acct = acct.clone();
dirty = true;
}
config
} else {
dirty = true;
FixedConfig {
acct: acct.clone(),
appdata,
_path: config_path,
..Default::default()
}
};
if dirty {
debug!("config file for {} changed, creating/updating", acct);
config.save().await?;
}
/* control */
let control = load_or_create_control_file(control_path).await?;
/* state */
let state = load_or_create_state_file(state_path).await?;
let g = GroupConfig {
config,
control,
state
};
g.warn_of_bad_config();
Ok(g)
}
pub(crate) async fn from_dir(group_dir: PathBuf) -> Result<Self, GroupError> {
let config_path = group_dir.join("config.json");
let control_path = group_dir.join("control.json");
let state_path = group_dir.join("state.json");
// try to reuse content of the files, if present
/* config */
let f = tokio::fs::read(&config_path).await?;
let mut config: FixedConfig = serde_json::from_slice(&f)?;
config._path = config_path;
/* control */
let control = load_or_create_control_file(control_path).await?;
/* state */
let state = load_or_create_state_file(state_path).await?;
let g = GroupConfig {
config,
control,
state
};
g.warn_of_bad_config();
Ok(g)
}
fn warn_of_bad_config(&self) {
for t in &self.control.group_tags {
if &t.to_lowercase() != t {
warn!("Group {} hashtag \"{}\" is not lowercase, it won't work!", self.config.acct, t);
}
}
for u in self.control.admin_users.iter()
.chain(self.control.member_users.iter())
.chain(self.control.banned_users.iter())
.chain(self.control.optout_users.iter())
{
if &u.to_lowercase() != u {
warn!("Group {} config contains a user with non-lowercase name \"{}\", it won't work!", self.config.acct, u);
}
if u.starts_with('@') || u.chars().filter(|c| *c == '@').count() != 1 {
warn!("Group {} config contains an invalid user name: {}", self.config.acct, u);
}
}
}
pub(crate) fn get_character_limit(&self) -> usize {
self.character_limit
self.config.character_limit
}
pub(crate) fn is_enabled(&self) -> bool {
self.enabled
self.config.enabled
}
/*
pub(crate) fn set_enabled(&mut self, ena: bool) {
self.enabled = ena;
self.mark_dirty();
}
*/
pub(crate) fn get_appdata(&self) -> &AppData {
&self.appdata
&self.config.appdata
}
pub(crate) fn set_appdata(&mut self, appdata: AppData) {
if self.appdata != appdata {
self.mark_dirty();
if self.config.appdata != appdata {
self.config.mark_dirty();
}
self.appdata = appdata;
self.config.appdata = appdata;
}
pub(crate) fn get_admins(&self) -> impl Iterator<Item = &String> {
self.admin_users.iter()
pub(crate) fn get_admins(&self) -> impl Iterator<Item=&String> {
self.control.admin_users.iter()
}
pub(crate) fn get_members(&self) -> impl Iterator<Item = &String> {
self.member_users.iter()
pub(crate) fn get_members(&self) -> impl Iterator<Item=&String> {
self.control.member_users.iter()
}
pub(crate) fn get_tags(&self) -> impl Iterator<Item = &String> {
self.group_tags.iter()
pub(crate) fn get_tags(&self) -> impl Iterator<Item=&String> {
self.control.group_tags.iter()
}
pub(crate) fn set_last_notif(&mut self, ts: u64) {
if self.last_notif_ts != ts {
self.mark_dirty();
if self.state.last_notif_ts != ts {
self.state.mark_dirty();
}
self.last_notif_ts = self.last_notif_ts.max(ts);
self.state.last_notif_ts = self.state.last_notif_ts.max(ts);
}
pub(crate) fn get_last_notif(&self) -> u64 {
self.last_notif_ts
self.state.last_notif_ts
}
pub(crate) fn set_last_status(&mut self, ts: u64) {
if self.last_status_ts != ts {
self.mark_dirty();
if self.state.last_status_ts != ts {
self.state.mark_dirty();
}
self.last_status_ts = self.last_status_ts.max(ts);
self.state.last_status_ts = self.state.last_status_ts.max(ts);
}
pub(crate) fn get_last_status(&self) -> u64 {
self.last_status_ts
self.state.last_status_ts
}
pub(crate) fn get_acct(&self) -> &str {
&self.acct
&self.config.acct
}
pub(crate) fn is_optout(&self, acct: &str) -> bool {
self.optout_users.contains(acct)
self.control.optout_users.contains(acct)
}
pub(crate) fn is_admin(&self, acct: &str) -> bool {
self.admin_users.contains(acct)
self.control.admin_users.contains(acct)
}
pub(crate) fn is_member(&self, acct: &str) -> bool {
self.member_users.contains(acct)
self.control.member_users.contains(acct)
}
pub(crate) fn is_member_or_admin(&self, acct: &str) -> bool {
@@ -175,11 +430,11 @@ impl GroupConfig {
}
pub(crate) fn is_banned(&self, acct: &str) -> bool {
self.banned_users.contains(acct) || self.is_users_server_banned(acct)
self.control.banned_users.contains(acct) || self.is_users_server_banned(acct)
}
pub(crate) fn is_server_banned(&self, server: &str) -> bool {
self.banned_servers.contains(server)
self.control.banned_servers.contains(server)
}
/// Check if the user's server is banned
@@ -201,12 +456,12 @@ impl GroupConfig {
if self.is_banned(acct) {
return Err(GroupError::UserIsBanned);
}
self.admin_users.insert(acct.to_owned())
self.control.admin_users.insert(acct.to_owned())
} else {
self.admin_users.remove(acct)
self.control.admin_users.remove(acct)
};
if change {
self.mark_dirty();
self.control.mark_dirty();
}
Ok(())
}
@@ -216,24 +471,24 @@ impl GroupConfig {
if self.is_banned(acct) {
return Err(GroupError::UserIsBanned);
}
self.member_users.insert(acct.to_owned())
self.control.member_users.insert(acct.to_owned())
} else {
self.member_users.remove(acct)
self.control.member_users.remove(acct)
};
if change {
self.mark_dirty();
self.control.mark_dirty();
}
Ok(())
}
pub(crate) fn set_optout(&mut self, acct: &str, optout: bool) {
let change = if optout {
self.optout_users.insert(acct.to_owned())
self.control.optout_users.insert(acct.to_owned())
} else {
self.optout_users.remove(acct)
self.control.optout_users.remove(acct)
};
if change {
self.mark_dirty();
self.control.mark_dirty();
}
}
@@ -244,72 +499,60 @@ impl GroupConfig {
return Err(GroupError::UserIsAdmin);
}
// Banned user is also kicked
change |= self.member_users.remove(acct);
change |= self.banned_users.insert(acct.to_owned());
change |= self.control.member_users.remove(acct);
change |= self.control.banned_users.insert(acct.to_owned());
} else {
change |= self.banned_users.remove(acct);
change |= self.control.banned_users.remove(acct);
}
if change {
self.mark_dirty();
self.control.mark_dirty();
}
Ok(())
}
pub(crate) fn ban_server(&mut self, server: &str, ban: bool) -> Result<(), GroupError> {
let changed = if ban {
for acct in &self.admin_users {
for acct in &self.control.admin_users {
let acct_server = acct_to_server(acct);
if acct_server == server {
return Err(GroupError::AdminsOnServer);
}
}
self.banned_servers.insert(server.to_owned())
self.control.banned_servers.insert(server.to_owned())
} else {
self.banned_servers.remove(server)
self.control.banned_servers.remove(server)
};
if changed {
self.mark_dirty();
self.control.mark_dirty();
}
Ok(())
}
pub(crate) fn add_tag(&mut self, tag: &str) {
if self.group_tags.insert(tag.to_string()) {
self.mark_dirty();
if self.control.group_tags.insert(tag.to_string()) {
self.control.mark_dirty();
}
}
pub(crate) fn remove_tag(&mut self, tag: &str) {
if self.group_tags.remove(tag) {
self.mark_dirty();
if self.control.group_tags.remove(tag) {
self.control.mark_dirty();
}
}
pub(crate) fn is_tag_followed(&self, tag: &str) -> bool {
self.group_tags.contains(tag)
self.control.group_tags.contains(tag)
}
pub(crate) fn set_member_only(&mut self, member_only: bool) {
if self.member_only != member_only {
self.mark_dirty();
if self.control.member_only != member_only {
self.control.mark_dirty();
}
self.member_only = member_only;
self.control.member_only = member_only;
}
pub(crate) fn is_member_only(&self) -> bool {
self.member_only
}
pub(crate) fn mark_dirty(&mut self) {
self.dirty = true;
}
pub(crate) fn is_dirty(&self) -> bool {
self.dirty
}
pub(crate) fn clear_dirty_status(&mut self) {
self.dirty = false;
self.control.member_only
}
}
+84 -87
View File
@@ -5,19 +5,19 @@ use elefren::{scopes, FediClient, Registration, Scopes};
use futures::StreamExt;
use tokio::sync::RwLock;
use data::{Config, GroupConfig};
use data::{GroupConfig};
use crate::error::GroupError;
use crate::group_handler::GroupHandle;
use std::time::Duration;
use crate::store::data::CommonConfig;
pub(crate) mod data;
#[derive(Debug, Default)]
pub struct ConfigStore {
store_path: PathBuf,
save_pretty: bool,
data: tokio::sync::RwLock<Config>,
config: Arc<CommonConfig>,
}
#[derive(Debug)]
@@ -28,29 +28,49 @@ pub struct NewGroupOptions {
#[derive(Debug)]
pub struct StoreOptions {
pub store_path: String,
pub save_pretty: bool,
pub store_dir: String,
}
impl ConfigStore {
/// Create a new instance of the store.
/// If a path is given, it will try to load the content from a file.
pub async fn new(options: StoreOptions) -> Result<Arc<Self>, GroupError> {
let path: &Path = options.store_path.as_ref();
pub async fn load_from_fs(options: StoreOptions) -> Result<Arc<Self>, GroupError> {
let given_path: &Path = options.store_dir.as_ref();
let config = if path.is_file() {
let f = tokio::fs::read(path).await?;
let mut common_file : Option<PathBuf> = None;
let base_dir : PathBuf;
if given_path.is_file() {
if given_path.extension().unwrap_or_default().to_string_lossy() == "json" {
// this is a groups.json file
common_file = Some(given_path.to_owned());
base_dir = given_path.parent().ok_or_else(|| GroupError::BadConfig("no parent dir".into()))?.to_owned();
} else {
return Err(GroupError::BadConfig("bad config file, should be JSON".into()));
}
} else if given_path.is_dir() {
let cf = given_path.join("groups.json");
if cf.is_file() {
common_file = Some(cf);
}
base_dir = given_path.to_owned();
} else {
return Err(GroupError::BadConfig("bad config file/dir".into()));
}
if !base_dir.is_dir() {
return Err(GroupError::BadConfig("base dir does not exist".into()));
}
let config : CommonConfig = if let Some(cf) = &common_file {
let f = tokio::fs::read(&cf).await?;
serde_json::from_slice(&f)?
} else {
let empty = Config::default();
tokio::fs::write(path, serde_json::to_string(&empty)?.as_bytes()).await?;
empty
CommonConfig::default()
};
Ok(Arc::new(Self {
store_path: path.to_owned(),
save_pretty: options.save_pretty,
data: RwLock::new(config),
store_path: base_dir.to_owned(),
config: Arc::new(config),
}))
}
@@ -67,10 +87,11 @@ impl ConfigStore {
let client = elefren::helpers::cli::authenticate(registration).await?;
let appdata = client.data.clone();
let data = GroupConfig::new(opts.acct.clone(), appdata);
let group_dir = self.store_path.join(&opts.acct);
let data = GroupConfig::from_appdata(opts.acct.clone(), appdata, group_dir).await?;
// save & persist
self.set_group_config(data.clone()).await?;
let group_account = match client.verify_credentials().await {
Ok(account) => {
@@ -90,15 +111,16 @@ impl ConfigStore {
group_account,
client,
config: data,
store: self.clone(),
common_config: self.config.clone(),
})
}
/// Re-auth an existing group
pub async fn reauth_group(self: &Arc<Self>, acct: &str) -> Result<GroupHandle, GroupError> {
let groups = self.data.read().await;
let mut config = groups.get_group_config(acct).ok_or(GroupError::GroupNotExist)?.clone();
drop(groups);
let group_dir = self.store_path.join(&acct);
let mut config = GroupConfig::from_dir(group_dir).await?;
println!("--- Re-authenticating bot user @{} ---", acct);
let registration = Registration::new(config.get_appdata().base.to_string())
@@ -114,7 +136,7 @@ impl ConfigStore {
let appdata = client.data.clone();
config.set_appdata(appdata);
self.set_group_config(config.clone()).await?;
config.save_if_needed(true).await?;
let group_account = match client.verify_credentials().await {
Ok(account) => {
@@ -134,95 +156,70 @@ impl ConfigStore {
group_account,
client,
config,
store: self.clone(),
common_config: self.config.clone(),
})
}
/// Spawn existing group using saved creds
pub async fn spawn_groups(self: Arc<Self>) -> Vec<GroupHandle> {
let groups = self.data.read().await.clone();
let groups_iter = groups.groups.into_values();
pub async fn spawn_groups(self: Arc<Self>) -> Result<Vec<GroupHandle>, GroupError> {
let dirs = std::fs::read_dir(&self.store_path.join("groups.d"))?;
// Connect in parallel
futures::stream::iter(groups_iter)
.map(|gc| async {
if !gc.is_enabled() {
debug!("Group @{} is DISABLED", gc.get_acct());
return None;
}
Ok(futures::stream::iter(dirs)
.map(|entry_maybe : Result<std::fs::DirEntry, std::io::Error>| async {
match entry_maybe {
Ok(entry) => {
let mut gc = GroupConfig::from_dir(entry.path())
.await.ok()?;
debug!("Connecting to @{}", gc.get_acct());
if !gc.is_enabled() {
debug!("Group @{} is DISABLED", gc.get_acct());
return None;
}
let client = FediClient::from(gc.get_appdata().clone());
debug!("Connecting to @{}", gc.get_acct());
let my_account = match client.verify_credentials().await {
Ok(account) => {
info!(
let client = FediClient::from(gc.get_appdata().clone());
let my_account = match client.verify_credentials().await {
Ok(account) => {
info!(
"Group account verified: @{}, \"{}\"",
account.acct, account.display_name
);
account
account
}
Err(e) => {
error!("Group @{} auth error: {}", gc.get_acct(), e);
return None;
}
};
Some(GroupHandle {
group_account: my_account,
client,
config: gc,
common_config: self.config.clone(),
})
}
Err(e) => {
error!("Group @{} auth error: {}", gc.get_acct(), e);
return None;
error!("{}", e);
None
}
};
Some(GroupHandle {
group_account: my_account,
client,
config: gc,
store: self.clone(),
})
}
})
.buffer_unordered(8)
.collect::<Vec<_>>()
.await
.into_iter()
.flatten()
.collect()
.collect())
}
pub async fn group_exists(&self, acct : &str) -> bool {
self.data.read().await.groups.contains_key(acct)
}
/*
pub(crate) async fn get_group_config(&self, group: &str) -> Option<GroupConfig> {
let c = self.data.read().await;
c.get_group_config(group).cloned()
}
*/
//noinspection RsSelfConvention
/// Set group config to the store. The store then saved.
pub(crate) async fn set_group_config(&self, config: GroupConfig) -> Result<(), GroupError> {
trace!("Locking mutex");
if let Ok(mut data) = tokio::time::timeout(Duration::from_secs(1), self.data.write()).await {
trace!("Locked");
data.set_group_config(config);
trace!("Writing file");
self.persist(&data).await?;
} else {
error!("DEADLOCK? Timeout waiting for data RW Lock in settings store");
}
Ok(())
}
/// Persist the store
async fn persist(&self, data: &Config) -> Result<(), GroupError> {
tokio::fs::write(
&self.store_path,
if self.save_pretty {
serde_json::to_string_pretty(&data)
} else {
serde_json::to_string(&data)
}?
.as_bytes(),
)
.await?;
Ok(())
self.store_path.join(acct)
.join("config.json")
.is_file()
}
}