wip translation system

master
Ondřej Hruška 3 years ago
parent 881411ebd3
commit bd47a004bf
Signed by untrusted user: MightyPork
GPG Key ID: 2C5FD5035250423D
  1. 10
      README.md
  2. 18
      locales/en.json
  3. 62
      src/group_handler/handle_mention.rs
  4. 29
      src/group_handler/mod.rs
  5. 19
      src/main.rs
  6. 6
      src/store/common_config.rs
  7. 7
      src/store/group_config.rs
  8. 86
      src/store/mod.rs
  9. 81
      src/tr.rs

@ -36,7 +36,7 @@ You can also run the program using Cargo, that is handy for development: `cargo
3. **Make sure you auth as the correct user!**
4. Paste the Oauth2 token you got into the terminal, hit enter.
The program now ends. The credentials are saved in the directory `groups.d/account@server/`, which is created if missing.
The program now ends. The credentials are saved in the directory `groups/account@server/`, which is created if missing.
You can repeat this for any number of groups.
@ -49,7 +49,7 @@ In case you need to re-authenticate an existing group, do the same but use `-A`
A typical setup could look like this:
```
├── groups.d
├── groups
│ ├── betty@piggo.space
│ │ ├── config.json
│ │ ├── control.json
@ -97,8 +97,8 @@ There is one shared config file: `groups.json`
#### Per-group config
Each group is stored as a sub-directory of `groups.d/`. The sub-directories are normally named after their accounts,
but this is not required. For example, `groups.d/betty@piggo.space/`.
Each group is stored as a sub-directory of `groups/`. The sub-directories are normally named after their accounts,
but this is not required. For example, `groups/betty@piggo.space/`.
The group's config and state is split into three files in a way that minimizes the risk of data loss.
@ -180,7 +180,7 @@ Internal use, millisecond timestamps of the last-seen status and notification.
### Running
To run the group service, simply run it with no arguments.
It will read the `groups.json` file (if present), find groups in `groups.d/` and start the services for you.
It will read the `groups.json` file (if present), find groups in `groups/` and start the services for you.
Note that the control and status files must be writable, they are updated at run-time.
Config files can have limited permissions to avoid accidental overwrite.

@ -0,0 +1,18 @@
{
"welcome_public": "@{user} Welcome to the group! The group user will now follow you back to complete the sign-up.\nTo share a post, @ the group user or use a group hashtag.\n\nUse /help for more info.",
"welcome_member_only": "@{user} Welcome to the group! This group has posting restricted to members.\nIf you'd like to join, please ask one of the group admins:\n{admins}",
"welcome_join_cmd": "Welcome to the group! The group user will now follow you to complete the sign-up. Make sure you follow back to receive shared posts!\n\nUse /help for more info.",
"welcome_closed": "Sorry, this group is closed to new sign-ups.\nPlease ask one of the group admins to add you:",
"user_list_member": "- {user}",
"user_list_admin": "- {user} [admin]",
"help_admin_commands": "\n**Admin commands:**\n`/ping` - check the group works\n`/add user` - add a member (user@domain)\n`/remove user` - remove a member\n`/add #hashtag` - add a group hashtag\n`/remove #hashtag` - remove a group hashtag\n`/undo` - un-boost a replied-to post, delete an announcement\n`/ban x` - ban a user or server\n`/unban x` - lift a ban\n`/admin user` - grant admin rights\n`/deadmin user` - revoke admin rights\n`/closegroup` - make member-only\n`/opengroup` - make public-access\n`/announce x` - make a public announcement",
"cmd_leave_resp": "You're no longer a group member. Unfollow the group user to stop receiving group messages.",
"member_list_heading": "Group members:",
"admin_list_heading": "Group admins:",
"tag_list_heading": "Group tags:",
"tag_list_entry": "- {tag}",
"cmd_close_resp": "Group changed to member-only",
"cmd_close_resp_noaction": "No action, group is member-only already",
"cmd_open_resp": "Group changed to open-access",
"cmd_open_resp_noaction": "No action, group is open-access already",
}

@ -12,6 +12,7 @@ use crate::error::GroupError;
use crate::group_handler::GroupHandle;
use crate::store::group_config::GroupConfig;
use crate::store::CommonConfig;
use crate::tr::{EMPTY_TRANSLATION_TABLE, TranslationTable};
use crate::utils;
use crate::utils::{normalize_acct, LogError};
@ -33,6 +34,11 @@ pub struct ProcessMention<'a> {
}
impl<'a> ProcessMention<'a> {
fn tr(&self) -> &TranslationTable {
self.cc.tr.get(self.config.get_locale())
.unwrap_or(&EMPTY_TRANSLATION_TABLE)
}
async fn lookup_acct_id(&self, acct: &str, followed: bool) -> Result<Option<String>, GroupError> {
debug!("Looking up user ID by acct: {}", acct);
@ -85,9 +91,9 @@ impl<'a> ProcessMention<'a> {
members.dedup();
for m in members {
self.replies.push(if admins.contains(&m) {
format!("- {} [admin]", m)
crate::tr!(self, "user_list_admin", user=m)
} else {
format!("- {}", m)
crate::tr!(self, "user_list_member", user=m)
});
}
}
@ -610,9 +616,9 @@ impl<'a> ProcessMention<'a> {
if self.is_admin {
if self.config.is_member_only() {
self.config.set_member_only(false);
self.add_reply("Group changed to open-access");
self.add_reply(crate::tr!(self, "cmd_open_resp"));
} else {
self.add_reply("No action, group is open-access already");
self.add_reply(crate::tr!(self, "cmd_open_resp_noaction"));
}
} else {
warn!("Ignore cmd, user not admin");
@ -623,9 +629,9 @@ impl<'a> ProcessMention<'a> {
if self.is_admin {
if !self.config.is_member_only() {
self.config.set_member_only(true);
self.add_reply("Group changed to member-only");
self.add_reply(crate::tr!(self, "cmd_close_resp"));
} else {
self.add_reply("No action, group is member-only already");
self.add_reply(crate::tr!(self, "cmd_close_resp_noaction"));
}
} else {
warn!("Ignore cmd, user not admin");
@ -675,44 +681,28 @@ impl<'a> ProcessMention<'a> {
// XXX when used on instance with small character limit, this won't fit!
if self.is_admin {
self.add_reply(
"\n\
**Admin commands:**\n\
`/ping` - check the group works\n\
`/add user` - add a member (user@domain)\n\
`/remove user` - remove a member\n\
`/add #hashtag` - add a group hashtag\n\
`/remove #hashtag` - remove a group hashtag\n\
`/undo` - un-boost a replied-to post, delete an announcement\n\
`/ban x` - ban a user or server\n\
`/unban x` - lift a ban\n\
`/admin user` - grant admin rights\n\
`/deadmin user` - revoke admin rights\n\
`/closegroup` - make member-only\n\
`/opengroup` - make public-access\n\
`/announce x` - make a public announcement",
);
self.add_reply(crate::tr!(self, "help_admin_commands"));
}
}
async fn cmd_list_members(&mut self) {
self.want_markdown = true;
if self.is_admin {
self.add_reply("Group members:");
self.add_reply(crate::tr!(self, "member_list_heading"));
self.append_member_list_to_reply();
} else {
self.add_reply("Group admins:");
self.add_reply(crate::tr!(self, "admin_list_heading"));
self.append_admin_list_to_reply();
}
}
async fn cmd_list_tags(&mut self) {
self.add_reply("Group tags:");
self.add_reply(crate::tr!(self, "tag_list_heading"));
self.want_markdown = true;
let mut tags = self.config.get_tags().collect::<Vec<_>>();
tags.sort();
for t in tags {
self.replies.push(format!("- {}", t).to_string());
self.replies.push(crate::tr!(self, "tag_list_entry", tag=t));
}
}
@ -720,9 +710,7 @@ impl<'a> ProcessMention<'a> {
if self.config.is_member_or_admin(&self.status_acct) {
// admin can leave but that's a bad idea
let _ = self.config.set_member(&self.status_acct, false);
self.add_reply(
"You're no longer a group member. Unfollow the group user to stop receiving group messages.",
);
self.add_reply(crate::tr!(self, "cmd_leave_resp"));
}
self.unfollow_user_by_id(&self.status_user_id)
@ -740,12 +728,7 @@ impl<'a> ProcessMention<'a> {
// Not a member yet
if self.config.is_member_only() {
// No you can't
self.add_reply(
"\
Sorry, this group is closed to new sign-ups.\n\
Please ask one of the group admins to add you:",
);
self.add_reply(crate::tr!(self, "welcome_closed"));
self.append_admin_list_to_reply();
} else {
// Open access, try to follow back
@ -753,12 +736,7 @@ impl<'a> ProcessMention<'a> {
// This only fails if the user is banned, but that is filtered above
let _ = self.config.set_member(&self.status_acct, true);
self.add_reply(
"\
Welcome to the group! The group user will now follow you to complete the sign-up. \
Make sure you follow back to receive shared posts!\n\n\
Use /help for more info.",
);
self.add_reply(crate::tr!(self, "welcome_join_cmd"));
}
}
}

@ -1,4 +1,5 @@
use std::collections::VecDeque;
use std::ops::Deref;
use std::sync::Arc;
use std::time::{Duration, Instant};
@ -19,6 +20,7 @@ use crate::command::StatusCommand;
use crate::error::GroupError;
use crate::store::CommonConfig;
use crate::store::GroupConfig;
use crate::tr::{EMPTY_TRANSLATION_TABLE, TranslationTable};
use crate::utils::{normalize_acct, LogError, VisExt};
mod handle_mention;
@ -46,14 +48,6 @@ impl Default for GroupInternal {
}
}
// TODO move other options to common_config!
// // const DELAY_BEFORE_ACTION: Duration = Duration::from_millis(250);
// const DELAY_REOPEN_STREAM: Duration = Duration::from_millis(500);
// // higher because we can expect a lot of non-hashtag statuses here
// const SOCKET_ALIVE_TIMEOUT: Duration = Duration::from_secs(30);
// const SOCKET_RETIRE_TIME: Duration = Duration::from_secs(120);
macro_rules! grp_debug {
($self:ident, $f:expr) => {
::log::debug!(concat!("(@{}) ", $f), $self.config.get_acct());
@ -531,6 +525,11 @@ impl GroupHandle {
res
}
fn tr(&self) -> &TranslationTable {
self.cc.tr.get(self.config.get_locale())
.unwrap_or(&EMPTY_TRANSLATION_TABLE)
}
async fn handle_new_follow(&mut self, notif_acct: &str, notif_user_id: &str) {
let mut follow_back = false;
let text = if self.config.is_member_only() {
@ -539,24 +538,16 @@ impl GroupHandle {
let mut admins = self.config.get_admins().cloned().collect::<Vec<_>>();
admins.sort();
format!(
"\
@{user} Welcome to the group! This group has posting restricted to members. \
If you'd like to join, please ask one of the group admins:\n\
{admins}",
crate::tr!(self, "welcome_member_only",
user = notif_acct,
admins = admins.join(", ")
admins = &admins.join(", ")
)
} else {
follow_back = true;
self.config.set_member(notif_acct, true).log_error("Fail add a member");
format!(
"\
@{user} Welcome to the group! The group user will now follow you back to complete the sign-up. \
To share a post, @ the group user or use a group hashtag.\n\n\
Use /help for more info.",
crate::tr!(self, "welcome_public",
user = notif_acct
)
};

@ -10,10 +10,12 @@ extern crate serde;
#[macro_use]
extern crate thiserror;
use std::sync::Arc;
use clap::Arg;
use log::LevelFilter;
use crate::store::{NewGroupOptions, StoreOptions};
use crate::tr::TranslationTable;
use crate::utils::acct_to_server;
mod command;
@ -22,6 +24,9 @@ mod group_handler;
mod store;
mod utils;
#[macro_use]
mod tr;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let args = clap::App::new("groups")
@ -90,7 +95,7 @@ async fn main() -> anyhow::Result<()> {
.filter_module("mio", LevelFilter::Warn)
.init();
let store = store::ConfigStore::load_from_fs(StoreOptions {
let mut store = store::ConfigStore::load_from_fs(StoreOptions {
store_dir: args.value_of("config").unwrap_or(".").to_string(),
})
.await?;
@ -104,14 +109,14 @@ async fn main() -> anyhow::Result<()> {
}
if let Some(server) = acct_to_server(acct) {
let g = store
store
.auth_new_group(NewGroupOptions {
server: format!("https://{}", server),
acct: acct.to_string(),
})
.await?;
eprintln!("New group @{} added to config!", g.config.get_acct());
eprintln!("New group added to config!");
return Ok(());
} else {
anyhow::bail!("--auth handle must be username@server, got: \"{}\"", handle);
@ -120,13 +125,17 @@ async fn main() -> anyhow::Result<()> {
if let Some(acct) = args.value_of("reauth") {
let acct = acct.trim_start_matches('@');
let _ = store.reauth_group(acct).await?;
store.reauth_group(acct).await?;
eprintln!("Group @{} re-authed!", acct);
return Ok(());
}
store.find_locales().await;
return Ok(());
// Start
let groups = store.spawn_groups().await?;
let groups = Arc::new(store).spawn_groups().await?;
let mut handles = vec![];
for mut g in groups {

@ -1,3 +1,6 @@
use std::collections::HashMap;
use crate::tr::TranslationTable;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct CommonConfig {
@ -21,6 +24,8 @@ pub struct CommonConfig {
/// Time after which a socket is always closed, even if seemingly alive.
/// This is a work-around for servers that stop sending notifs after a while.
pub socket_retire_time_s: f64,
#[serde(skip)]
pub tr : HashMap<String, TranslationTable>,
}
impl Default for CommonConfig {
@ -34,6 +39,7 @@ impl Default for CommonConfig {
delay_reopen_error_s: 5.0,
socket_alive_timeout_s: 30.0,
socket_retire_time_s: 120.0,
tr: Default::default(),
}
}
}

@ -13,6 +13,8 @@ struct FixedConfig {
acct: String,
/// elefren data
appdata: AppData,
/// configured locale to use
locale: String,
/// Server's character limit
character_limit: usize,
#[serde(skip)]
@ -73,6 +75,7 @@ impl Default for FixedConfig {
Self {
enabled: true,
acct: "".to_string(),
locale: "en".to_string(),
appdata: AppData {
base: Default::default(),
client_id: Default::default(),
@ -365,6 +368,10 @@ impl GroupConfig {
&self.config.appdata
}
pub(crate) fn get_locale(&self) -> &str {
&self.config.locale
}
pub(crate) fn set_appdata(&mut self, appdata: AppData) {
if self.config.appdata != appdata {
self.config.mark_dirty();

@ -11,12 +11,14 @@ pub mod common_config;
pub mod group_config;
pub use common_config::CommonConfig;
pub use group_config::GroupConfig;
use crate::tr::TranslationTable;
#[derive(Debug, Default)]
pub struct ConfigStore {
store_path: PathBuf,
groups_path: PathBuf,
config: Arc<CommonConfig>,
locales_path: PathBuf,
config: CommonConfig,
}
#[derive(Debug)]
@ -33,7 +35,7 @@ pub struct StoreOptions {
impl ConfigStore {
/// Create a new instance of the store.
/// If a path is given, it will try to load the content from a file.
pub async fn load_from_fs(options: StoreOptions) -> Result<Arc<Self>, GroupError> {
pub async fn load_from_fs(options: StoreOptions) -> Result<Self, GroupError> {
let given_path: &Path = options.store_dir.as_ref();
let mut common_file: Option<PathBuf> = None;
@ -74,21 +76,24 @@ impl ConfigStore {
debug!("Using common config:\n{:#?}", config);
let groups_path = base_dir.join("groups.d");
let groups_path = base_dir.join("groups");
if !groups_path.exists() {
debug!("Creating groups directory");
tokio::fs::create_dir_all(&groups_path).await?;
}
Ok(Arc::new(Self {
let locales_path = base_dir.join("locales");
Ok(Self {
store_path: base_dir.to_owned(),
groups_path,
config: Arc::new(config),
}))
locales_path,
config,
})
}
/// Spawn a new group
pub async fn auth_new_group(self: &Arc<Self>, opts: NewGroupOptions) -> Result<GroupHandle, GroupError> {
pub async fn auth_new_group(&self, opts: NewGroupOptions) -> Result<(), GroupError> {
let registration = Registration::new(&opts.server)
.client_name("group-actor")
.force_login(true)
@ -106,13 +111,12 @@ impl ConfigStore {
// save & persist
let group_account = match client.verify_credentials().await {
match client.verify_credentials().await {
Ok(account) => {
info!(
"Group account verified: @{}, \"{}\"",
account.acct, account.display_name
);
account
}
Err(e) => {
error!("Group @{} auth error: {}", opts.acct, e);
@ -120,17 +124,11 @@ impl ConfigStore {
}
};
Ok(GroupHandle {
group_account,
client,
config: data,
cc: self.config.clone(),
internal: GroupInternal::default(),
})
Ok(())
}
/// Re-auth an existing group
pub async fn reauth_group(self: &Arc<Self>, acct: &str) -> Result<GroupHandle, GroupError> {
pub async fn reauth_group(&self, acct: &str) -> Result<(), GroupError> {
let group_dir = self.groups_path.join(&acct);
let mut config = GroupConfig::from_dir(group_dir).await?;
@ -165,20 +163,56 @@ impl ConfigStore {
}
};
Ok(GroupHandle {
group_account,
client,
config,
cc: self.config.clone(),
internal: GroupInternal::default(),
})
Ok(())
}
pub async fn find_locales(&mut self) {
if !self.locales_path.is_dir() {
debug!("No locales path set!");
return;
}
let entries = match std::fs::read_dir(&self.locales_path) {
Ok(ee) => ee,
Err(e) => {
warn!("Error listing locales");
return;
}
};
for e in entries {
if let Ok(e) = e {
let path = e.path();
if path.is_file() && path.extension().unwrap_or_default().to_string_lossy() == "json" {
let filename = path.file_name().unwrap_or_default().to_string_lossy();
debug!("Loading locale {}", filename);
match tokio::fs::read(&path).await {
Ok(f) => {
if let Ok(tr) = serde_json::from_slice::<TranslationTable>(&f) {
let locale_name = path.file_stem().unwrap_or_default().to_string_lossy().to_string();
debug!("Loaded locale: {}", locale_name);
self.config.tr.insert(locale_name, tr);
} else {
error!("Failed to parse locale file {}", path.display());
}
},
Err(e) => {
error!("Failed to read locale file {}: {}", path.display(), e);
}
}
}
}
}
}
/// Spawn existing group using saved creds
pub async fn spawn_groups(self: Arc<Self>) -> Result<Vec<GroupHandle>, GroupError> {
pub async fn spawn_groups(self) -> Result<Vec<GroupHandle>, GroupError> {
info!("Starting group services for groups in {}", self.groups_path.display());
let dirs = std::fs::read_dir(&self.groups_path)?;
let config = Arc::new(self.config);
// Connect in parallel
Ok(futures::stream::iter(dirs)
.map(|entry_maybe: Result<std::fs::DirEntry, std::io::Error>| async {
@ -213,7 +247,7 @@ impl ConfigStore {
group_account: my_account,
client,
config: gc,
cc: self.config.clone(),
cc: config.clone(),
internal: GroupInternal::default(),
})
}

@ -0,0 +1,81 @@
//! magic for custom translations and strings
use std::collections::HashMap;
use once_cell::sync::Lazy;
#[derive(Debug,Clone,Serialize,Deserialize,Default)]
pub struct TranslationTable {
#[serde(flatten)]
entries: Option<HashMap<String, String>>,
}
pub const EMPTY_TRANSLATION_TABLE : TranslationTable = TranslationTable {
entries: None,
};
impl TranslationTable {
pub fn new() -> Self {
Self::default()
}
pub fn add_translation(&mut self, key : impl ToString, subs : impl ToString) {
if self.entries.is_none() {
self.entries = Some(Default::default());
}
self.entries.as_mut().unwrap().insert(key.to_string(), subs.to_string());
}
pub fn subs(&self, key : &str, substitutions: &[&str]) -> String {
if let Some(ee) = &self.entries {
match ee.get(key) {
Some(s) => {
// TODO optimize
let mut s = s.clone();
for pair in substitutions.chunks(2) {
if pair.len() != 2 {
continue;
}
s = s.replace(&format!("{{{}}}", pair[0]), pair[1]);
}
s
}
None => key.to_owned()
}
} else {
key.to_owned()
}
}
}
#[cfg(test)]
mod tests {
use crate::tr::TranslationTable;
#[test]
fn deser_tr_table() {
let tr : TranslationTable = serde_json::from_str(r#"{"foo":"bar"}"#).unwrap();
assert_eq!("bar", tr.subs("foo", &[]));
assert_eq!("xxx", tr.subs("xxx", &[]));
}
#[test]
fn subs() {
let mut tr = TranslationTable::new();
tr.add_translation("hello_user", "Hello, {user}!");
assert_eq!("Hello, James!", tr.subs("hello_user", &["user", "James"]));
}
}
#[macro_export]
macro_rules! tr {
($tr_haver:expr, $key:literal) => {
$tr_haver.tr().subs($key, &[])
};
($tr_haver:expr, $key:literal, $($k:tt=$value:expr),*) => {
$tr_haver.tr().subs($key, &[
$(stringify!($k), $value),*
])
};
}
Loading…
Cancel
Save