MightyPork/group-actor
1
1
Fork 1
Code Issues 6 Pull Requests Releases Wiki Activity

improvements, more config, add -q, readme

Browse Source
This commit is contained in:
MightyPork
2021-10-05 10:39:10 +02:00
parent 7ea6225ae9
commit de3fd4e729
14 changed files with 560 additions and 427 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/store/common_config.rs
+39
View File
@@ -0,0 +1,39 @@
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct CommonConfig {
/// Max number of missed notifs to process after connect
pub max_catchup_notifs: usize,
/// Max number of missed statuses to process after connect
pub max_catchup_statuses: usize,
/// Delay between fetched pages when catching up
pub delay_fetch_page_s: f64,
/// Delay after sending a status, making a follow or some other action.
/// Set if there are Throttled errors and you need to slow the service down.
pub delay_after_post_s: f64,
/// Delay before trying to re-connect after the server closed the socket
pub delay_reopen_closed_s: f64,
/// Delay before trying to re-connect after an error
pub delay_reopen_error_s: f64,
/// Timeout for a notification/timeline socket to be considered alive.
/// If nothing arrives in this interval, reopen it. Some servers have a buggy socket
/// implementation where it stays open but no longer works.
pub socket_alive_timeout_s: f64,
/// Time after which a socket is always closed, even if seemingly alive.
/// This is a work-around for servers that stop sending notifs after a while.
pub socket_retire_time_s: f64,
}
impl Default for CommonConfig {
fn default() -> Self {
Self {
max_catchup_notifs: 30,
max_catchup_statuses: 50,
delay_fetch_page_s: 0.25,
delay_after_post_s: 0.0,
delay_reopen_closed_s: 0.5,
delay_reopen_error_s: 5.0,
socket_alive_timeout_s: 30.0,
socket_retire_time_s: 120.0,
}
}
}
src/store/data.rs → src/store/group_config.rs
+85 -88
View File
@@ -1,4 +1,4 @@
use std::collections::{HashMap, HashSet};
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use elefren::AppData;
@@ -6,23 +6,7 @@ use elefren::AppData;
use crate::error::GroupError;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
pub struct CommonConfig {
pub max_catchup_notifs: usize,
pub max_catchup_statuses: usize,
}
impl Default for CommonConfig {
fn default() -> Self {
Self {
max_catchup_notifs: 30,
max_catchup_statuses: 50,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
#[serde(default, deny_unknown_fields)]
struct FixedConfig {
enabled: bool,
/// Group actor's acct
@@ -37,43 +21,8 @@ struct FixedConfig {
_path: PathBuf,
}
macro_rules! impl_change_tracking {
($struc:ident) => {
impl $struc {
pub(crate) fn mark_dirty(&mut self) {
self._dirty = true;
}
pub(crate) fn is_dirty(&self) -> bool {
self._dirty
}
pub(crate) fn clear_dirty_status(&mut self) {
self._dirty = false;
}
pub(crate) async fn save_if_needed(&mut self) -> Result<(), GroupError> {
if self._dirty {
self.save().await?;
}
Ok(())
}
pub(crate) async fn save(&mut self) -> Result<(), GroupError> {
tokio::fs::write(&self._path, serde_json::to_string_pretty(&self)?.as_bytes()).await?;
self._dirty = false;
Ok(())
}
}
};
}
impl_change_tracking!(FixedConfig);
impl_change_tracking!(MutableConfig);
impl_change_tracking!(StateConfig);
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
#[serde(default, deny_unknown_fields)]
struct MutableConfig {
/// Hashtags the group will auto-boost from it's members
group_tags: HashSet<String>,
@@ -96,7 +45,7 @@ struct MutableConfig {
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
#[serde(default, deny_unknown_fields)]
struct StateConfig {
/// Last seen notification timestamp (millis)
last_notif_ts: u64,
@@ -109,8 +58,7 @@ struct StateConfig {
}
/// This is the inner data struct holding a group's config
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default)]
#[derive(Debug, Clone)]
pub struct GroupConfig {
/// Fixed config that we only read
config: FixedConfig,
@@ -166,6 +114,43 @@ impl Default for StateConfig {
}
}
macro_rules! impl_change_tracking {
($struc:ident) => {
impl $struc {
pub(crate) fn mark_dirty(&mut self) {
self._dirty = true;
}
pub(crate) fn is_dirty(&self) -> bool {
self._dirty
}
pub(crate) fn clear_dirty_status(&mut self) {
self._dirty = false;
}
pub(crate) async fn save_if_needed(&mut self) -> Result<bool, GroupError> {
if self.is_dirty() {
self.save().await?;
Ok(true)
} else {
Ok(false)
}
}
pub(crate) async fn save(&mut self) -> Result<(), GroupError> {
tokio::fs::write(&self._path, serde_json::to_string_pretty(&self)?.as_bytes()).await?;
self.clear_dirty_status();
Ok(())
}
}
};
}
impl_change_tracking!(FixedConfig);
impl_change_tracking!(MutableConfig);
impl_change_tracking!(StateConfig);
impl Default for GroupConfig {
fn default() -> Self {
Self {
@@ -176,10 +161,10 @@ impl Default for GroupConfig {
}
}
async fn load_or_create_control_file(control_path : impl AsRef<Path>) -> Result<MutableConfig, GroupError> {
async fn load_or_create_control_file(control_path: impl AsRef<Path>) -> Result<MutableConfig, GroupError> {
let control_path = control_path.as_ref();
let mut dirty = false;
let mut control : MutableConfig = if control_path.is_file() {
let mut control: MutableConfig = if control_path.is_file() {
let f = tokio::fs::read(&control_path).await?;
let mut control: MutableConfig = serde_json::from_slice(&f)?;
control._path = control_path.to_owned();
@@ -194,15 +179,14 @@ async fn load_or_create_control_file(control_path : impl AsRef<Path>) -> Result<
};
if dirty {
control.save().await?;
// tokio::fs::write(&control._path, serde_json::to_string(&control)?.as_bytes()).await?;
}
Ok(control)
}
async fn load_or_create_state_file(state_path : impl AsRef<Path>) -> Result<StateConfig, GroupError> {
async fn load_or_create_state_file(state_path: impl AsRef<Path>) -> Result<StateConfig, GroupError> {
let state_path = state_path.as_ref();
let mut dirty = false;
let mut state : StateConfig = if state_path.is_file() {
let mut state: StateConfig = if state_path.is_file() {
let f = tokio::fs::read(&state_path).await?;
let mut control: StateConfig = serde_json::from_slice(&f)?;
control._path = state_path.to_owned();
@@ -217,29 +201,42 @@ async fn load_or_create_state_file(state_path : impl AsRef<Path>) -> Result<Stat
};
if dirty {
state.save().await?;
// tokio::fs::write(&state._path, serde_json::to_string(&state)?.as_bytes()).await?;
}
Ok(state)
}
impl GroupConfig {
pub(crate) fn is_dirty(&self) -> bool {
self.config.is_dirty()
|| self.control.is_dirty()
|| self.state.is_dirty()
self.config.is_dirty() || self.control.is_dirty() || self.state.is_dirty()
}
/// Save only what changed
pub(crate) async fn save_if_needed(&mut self, danger_allow_overwriting_config: bool) -> Result<(), GroupError> {
#[allow(clippy::collapsible_if)]
if danger_allow_overwriting_config {
self.config.save_if_needed().await?;
if self.config.save_if_needed().await? {
debug!(
"Written {} config file {}",
self.config.acct,
self.config._path.display()
);
}
}
if self.control.save_if_needed().await? {
debug!(
"Written {} control file {}",
self.config.acct,
self.control._path.display()
);
}
if self.state.save_if_needed().await? {
debug!("Written {} state file {}", self.config.acct, self.state._path.display());
}
self.control.save_if_needed().await?;
self.state.save_if_needed().await?;
Ok(())
}
/// Save all unconditionally
#[allow(unused)]
pub(crate) async fn save(&mut self, danger_allow_overwriting_config: bool) -> Result<(), GroupError> {
if danger_allow_overwriting_config {
self.config.save().await?;
@@ -297,11 +294,7 @@ impl GroupConfig {
/* state */
let state = load_or_create_state_file(state_path).await?;
let g = GroupConfig {
config,
control,
state
};
let g = GroupConfig { config, control, state };
g.warn_of_bad_config();
Ok(g)
}
@@ -324,11 +317,7 @@ impl GroupConfig {
/* state */
let state = load_or_create_state_file(state_path).await?;
let g = GroupConfig {
config,
control,
state
};
let g = GroupConfig { config, control, state };
g.warn_of_bad_config();
Ok(g)
}
@@ -336,17 +325,26 @@ impl GroupConfig {
fn warn_of_bad_config(&self) {
for t in &self.control.group_tags {
if &t.to_lowercase() != t {
warn!("Group {} hashtag \"{}\" is not lowercase, it won't work!", self.config.acct, t);
warn!(
"Group {} hashtag \"{}\" is not lowercase, it won't work!",
self.config.acct, t
);
}
}
for u in self.control.admin_users.iter()
for u in self
.control
.admin_users
.iter()
.chain(self.control.member_users.iter())
.chain(self.control.banned_users.iter())
.chain(self.control.optout_users.iter())
{
if &u.to_lowercase() != u {
warn!("Group {} config contains a user with non-lowercase name \"{}\", it won't work!", self.config.acct, u);
warn!(
"Group {} config contains a user with non-lowercase name \"{}\", it won't work!",
self.config.acct, u
);
}
if u.starts_with('@') || u.chars().filter(|c| *c == '@').count() != 1 {
@@ -374,15 +372,15 @@ impl GroupConfig {
self.config.appdata = appdata;
}
pub(crate) fn get_admins(&self) -> impl Iterator<Item=&String> {
pub(crate) fn get_admins(&self) -> impl Iterator<Item = &String> {
self.control.admin_users.iter()
}
pub(crate) fn get_members(&self) -> impl Iterator<Item=&String> {
pub(crate) fn get_members(&self) -> impl Iterator<Item = &String> {
self.control.member_users.iter()
}
pub(crate) fn get_tags(&self) -> impl Iterator<Item=&String> {
pub(crate) fn get_tags(&self) -> impl Iterator<Item = &String> {
self.control.group_tags.iter()
}
@@ -425,8 +423,7 @@ impl GroupConfig {
}
pub(crate) fn is_member_or_admin(&self, acct: &str) -> bool {
self.is_member(acct)
|| self.is_admin(acct)
self.is_member(acct) || self.is_admin(acct)
}
pub(crate) fn is_banned(&self, acct: &str) -> bool {
@@ -563,7 +560,7 @@ fn acct_to_server(acct: &str) -> &str {
#[cfg(test)]
mod tests {
use crate::error::GroupError;
use crate::store::data::{acct_to_server, GroupConfig};
use crate::store::group_config::{acct_to_server, GroupConfig};
#[test]
fn test_acct_to_server() {
src/store/mod.rs
+37 -27
View File
@@ -3,20 +3,19 @@ use std::sync::Arc;
use elefren::{scopes, FediClient, Registration, Scopes};
use futures::StreamExt;
use tokio::sync::RwLock;
use data::{GroupConfig};
use crate::error::GroupError;
use crate::group_handler::GroupHandle;
use std::time::Duration;
use crate::store::data::CommonConfig;
pub(crate) mod data;
pub mod common_config;
pub mod group_config;
pub use common_config::CommonConfig;
pub use group_config::GroupConfig;
#[derive(Debug, Default)]
pub struct ConfigStore {
store_path: PathBuf,
groups_path: PathBuf,
config: Arc<CommonConfig>,
}
@@ -37,13 +36,16 @@ impl ConfigStore {
pub async fn load_from_fs(options: StoreOptions) -> Result<Arc<Self>, GroupError> {
let given_path: &Path = options.store_dir.as_ref();
let mut common_file : Option<PathBuf> = None;
let base_dir : PathBuf;
let mut common_file: Option<PathBuf> = None;
let base_dir: PathBuf;
if given_path.is_file() {
if given_path.extension().unwrap_or_default().to_string_lossy() == "json" {
// this is a groups.json file
common_file = Some(given_path.to_owned());
base_dir = given_path.parent().ok_or_else(|| GroupError::BadConfig("no parent dir".into()))?.to_owned();
base_dir = given_path
.parent()
.ok_or_else(|| GroupError::BadConfig("no parent dir".into()))?
.to_owned();
} else {
return Err(GroupError::BadConfig("bad config file, should be JSON".into()));
}
@@ -61,15 +63,26 @@ impl ConfigStore {
return Err(GroupError::BadConfig("base dir does not exist".into()));
}
let config : CommonConfig = if let Some(cf) = &common_file {
let config: CommonConfig = if let Some(cf) = &common_file {
debug!("Loading common config from {}", cf.display());
let f = tokio::fs::read(&cf).await?;
serde_json::from_slice(&f)?
} else {
debug!("No common config file, using defaults");
CommonConfig::default()
};
debug!("Using common config:\n{:#?}", config);
let groups_path = base_dir.join("groups.d");
if !groups_path.exists() {
debug!("Creating groups directory");
tokio::fs::create_dir_all(&groups_path).await?;
}
Ok(Arc::new(Self {
store_path: base_dir.to_owned(),
groups_path,
config: Arc::new(config),
}))
}
@@ -87,7 +100,7 @@ impl ConfigStore {
let client = elefren::helpers::cli::authenticate(registration).await?;
let appdata = client.data.clone();
let group_dir = self.store_path.join(&opts.acct);
let group_dir = self.groups_path.join(&opts.acct);
let data = GroupConfig::from_appdata(opts.acct.clone(), appdata, group_dir).await?;
@@ -111,14 +124,13 @@ impl ConfigStore {
group_account,
client,
config: data,
common_config: self.config.clone(),
cc: self.config.clone(),
})
}
/// Re-auth an existing group
pub async fn reauth_group(self: &Arc<Self>, acct: &str) -> Result<GroupHandle, GroupError> {
let group_dir = self.store_path.join(&acct);
let group_dir = self.groups_path.join(&acct);
let mut config = GroupConfig::from_dir(group_dir).await?;
@@ -156,21 +168,21 @@ impl ConfigStore {
group_account,
client,
config,
common_config: self.config.clone(),
cc: self.config.clone(),
})
}
/// Spawn existing group using saved creds
pub async fn spawn_groups(self: Arc<Self>) -> Result<Vec<GroupHandle>, GroupError> {
let dirs = std::fs::read_dir(&self.store_path.join("groups.d"))?;
info!("Starting group services for groups in {}", self.groups_path.display());
let dirs = std::fs::read_dir(&self.groups_path)?;
// Connect in parallel
Ok(futures::stream::iter(dirs)
.map(|entry_maybe : Result<std::fs::DirEntry, std::io::Error>| async {
.map(|entry_maybe: Result<std::fs::DirEntry, std::io::Error>| async {
match entry_maybe {
Ok(entry) => {
let mut gc = GroupConfig::from_dir(entry.path())
.await.ok()?;
let gc = GroupConfig::from_dir(entry.path()).await.ok()?;
if !gc.is_enabled() {
debug!("Group @{} is DISABLED", gc.get_acct());
@@ -184,9 +196,9 @@ impl ConfigStore {
let my_account = match client.verify_credentials().await {
Ok(account) => {
info!(
"Group account verified: @{}, \"{}\"",
account.acct, account.display_name
);
"Group account verified: @{}, \"{}\"",
account.acct, account.display_name
);
account
}
Err(e) => {
@@ -199,7 +211,7 @@ impl ConfigStore {
group_account: my_account,
client,
config: gc,
common_config: self.config.clone(),
cc: self.config.clone(),
})
}
Err(e) => {
@@ -216,10 +228,8 @@ impl ConfigStore {
.collect())
}
pub async fn group_exists(&self, acct : &str) -> bool {
self.store_path.join(acct)
.join("config.json")
.is_file()
pub async fn group_exists(&self, acct: &str) -> bool {
self.store_path.join(acct).join("config.json").is_file()
}
}