diff --git a/.gitignore b/.gitignore index 765220b..495a94c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target .idea/ postit.json +postit.db diff --git a/Cargo.lock b/Cargo.lock index 78ea7d8..d63b7da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -576,52 +576,6 @@ dependencies = [ "memchr 1.0.2", ] -[[package]] -name = "num" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg 1.0.0", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" -dependencies = [ - "autocfg 1.0.0", - "num-traits", -] - -[[package]] -name = "num-derive" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8b15b261814f992e33760b1fca9fe8b693d8a65299f20c9901688636cfb746" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "num-integer" version = "0.1.42" @@ -632,29 +586,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb0800a0291891dd9f4fe7bd9c19384f98f7fbe0cd0f39a2c6b88b9868bbc00" -dependencies = [ - "autocfg 1.0.0", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" -dependencies = [ - "autocfg 1.0.0", - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.11" @@ -811,9 +742,8 @@ dependencies = [ "chrono", "clappconfig", "flate2", + "lazy_static", "log 0.4.8", - "num", - "num-derive", "parking_lot", "rand 0.7.3", "rouille", diff --git a/Cargo.toml b/Cargo.toml index 13667dc..804918a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ name = "postit" version = "0.1.0" authors = ["Ondřej Hruška "] edition = "2018" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -21,5 +22,4 @@ bincode = "1.2.1" flate2 = "1.0.14" anyhow = "1.0.28" tree_magic = { version = "0.2.3", default_features = false, features = ["staticmime"] } -num = "0.2.1" -num-derive = "0.3.0" +lazy_static = "1.4.0" diff --git a/README.md b/README.md new file mode 100644 index 0000000..7fea6dc --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# PostIt file sharing server + +PostIt is designed to work as a temporary public storage for text (and other) +files uploaded to it by software that need a publicly reachable page without +hosting its own server or even having a public IP. + +The primary use case is to share diagnostic and contextual information +produced by Fediverse bots (think an interactive game where the game board +is rendered to an image or text file on demand). There are sure to be many +other uses I didn't think of. + +The uploaded files have a lifetime of 10 minutes, which can be shortened or +extended up to 1 hour (or more, as configured). + +## Uploading a file + +To upload a file, send a POST request to the running PostIt server. + +```none +$ curl -X POST --data-binary @RICKROLL.txt 0.0.0.0:7745 -i +HTTP/1.1 200 OK +X-Secret: 5273d775746e393b +X-Expire: 599 +Content-Length: 16 + +421d082ef85827ea +``` + +Take note of the `X-Secret` header, you will need it to update or delete the file. + +If you only want to share the file, this is all you need. Grab the file ID from the response body +and share it. The URL is `/`, e.g. + +```none +$ curl -X GET 0.0.0.0:7745/421d082ef85827ea -i +HTTP/1.1 200 OK +Content-Type: text/plain; charset=utf8 +X-Expire: 459 +Content-Length: 688 + +File content here... +``` + +### Content type + +The server attempts to auto-detect the file's `Content-Type`. The fallback is `text/plain`. +If you wish to set a custom type, use the `Content-Type` header when uploading the file, e.g. + +``` +$ curl -X POST --data-binary @RICKROLL.txt 0.0.0.0:7745 -i -H 'Content-Type: application/json' +``` + +### Expiration time + +To customize the expiration time, use the header `X-Expire: `, or a GET argument `?expire=` e.g. + +``` +$ curl -X POST --data-binary @RICKROLL.txt 0.0.0.0:7745 -i -H 'X-Expire: 60' +``` + +## Updating a file + +A file you uploaded can be deleted or modified using the **secret token** obtained in respose to its upload. +Send the token as the `X-Secret` header, or GET argument `?secret=....` + +File is updated by sending a `PUT` request to the file's URL. + +The `PUT` request can change file expiration (`X-Expire: ` or GET arg `expire`), +update its `Content-Type`, or replace its content. + +Note that sending `PUT` with empty body will *not* clear the file, in that case the file content is +not changed at all. This can be used to extend file's expiration without changing it in any other way +(by sending the `X-Expire` header). + +## Deleting a file + +The `DELETE` verb, unsurprisingly, deletes a file. As with `PUT`, the secret token is required. + +. diff --git a/postit.db b/postit.db deleted file mode 100644 index 156c822..0000000 Binary files a/postit.db and /dev/null differ diff --git a/src/config.rs b/src/config.rs index f03d461..cf6a0f4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -40,7 +40,7 @@ pub(crate) struct Config { pub(crate) compression: bool, /// Persistence file - pub(crate) persist_file : String, + pub(crate) persist_file: String, } impl Default for Config { diff --git a/src/main.rs b/src/main.rs index f451c87..15f47bc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,36 +2,38 @@ extern crate serde_derive; #[macro_use] extern crate log; -#[macro_use] -extern crate num_derive; use crate::config::Config; +use crate::well_known_mime::Mime; +use chrono::{DateTime, Utc}; use clappconfig::{anyhow, AppConfig}; use parking_lot::Mutex; use rand::rngs::OsRng; use rand::Rng; use rouille::{Request, Response, ResponseBody}; + use std::borrow::Cow; use std::collections::HashMap; + +use std::fs::OpenOptions; use std::hash::{Hash, Hasher}; use std::io::Read; use std::time::Duration; -use chrono::{Utc, DateTime}; -use std::fs::OpenOptions; -use std::fmt::Display; -use serde::export::Formatter; -use std::fmt; -use serde::{Serialize, Serializer, Deserialize, Deserializer}; -use serde::de::{DeserializeOwned, Visitor}; -use crate::well_known_mime::Mime; +use siphasher::sip::SipHasher; mod config; mod well_known_mime; -const HDR_EXPIRES : &str = "X-Expires"; -const HDR_SECRET : &str = "X-Secret"; +/// Header to set expiry (seconds) +const HDR_EXPIRY: &str = "X-Expire"; +/// Header to pass secret token for update/delete +const HDR_SECRET: &str = "X-Secret"; +/// GET param to pass secret token (as a substitute for header) +const GET_EXPIRY: &str = "expire"; +/// GET param to pass secret token (as a substitute for header) +const GET_SECRET: &str = "secret"; -const FAVICON : &[u8] = include_bytes!("favicon.ico"); +const FAVICON: &[u8] = include_bytes!("favicon.ico"); /// Post ID (represented as a 16-digit hex string) type PostId = u64; @@ -41,7 +43,7 @@ type Secret = u64; type DataHash = u64; /// Post stored in the repository -#[derive(Debug,Serialize,Deserialize)] +#[derive(Debug, Serialize, Deserialize)] struct Post { /// Content-Type mime: Mime, @@ -59,6 +61,17 @@ impl Post { pub fn is_expired(&self) -> bool { self.expires < Utc::now() } + + /// Get remaining lifetime + pub fn time_remains(&self) -> Duration { + let seconds_remains = self.expires.signed_duration_since(Utc::now()) + .num_seconds(); + if seconds_remains < 0 { + Duration::from_secs(0) + } else { + Duration::from_secs(seconds_remains as u64) + } + } } fn main() -> anyhow::Result<()> { @@ -72,6 +85,7 @@ fn main() -> anyhow::Result<()> { error!("Load failed: {}", e); } } + store.gc_expired_posts(); store }); @@ -82,7 +96,7 @@ fn main() -> anyhow::Result<()> { info!("{} {}", method, req.raw_url()); if req.url() == "/favicon.ico" { - return Response::from_data("image/vnd.microsoft.icon", FAVICON); + return decorate_response(Response::from_data("image/vnd.microsoft.icon", FAVICON)); } store_w.gc_expired_posts_if_needed(); @@ -104,14 +118,21 @@ fn main() -> anyhow::Result<()> { warn!("Error resp: {}", resp.status_code); } - resp + decorate_response(resp) }); } +fn decorate_response(resp : Response) -> Response { + resp.without_header("Server") + .with_additional_header("Server", "postit.rs") + .with_additional_header("Access-Control-Allow-Origin", "*") + .with_additional_header("X-Version", env!("CARGO_PKG_VERSION")) +} + type PostsMap = HashMap; type DataMap = HashMap)>; -#[derive(Debug,Serialize,Deserialize)] +#[derive(Debug, Serialize, Deserialize)] struct Repository { #[serde(skip)] config: Config, @@ -175,7 +196,7 @@ impl Repository { .read(true) .open(&self.config.persist_file)?; - let result : Repository = if self.config.compression { + let result: Repository = if self.config.compression { let flate = flate2::read::DeflateDecoder::new(file); bincode::deserialize_from(flate)? } else { @@ -193,7 +214,7 @@ impl Repository { fn serve_delete(&mut self, req: &Request) -> Response { let post_id = match self.request_to_post_id(req, true) { Ok(Some(pid)) => pid, - Ok(None) => return error_with_text(400, "Post ID required."), + Ok(None) => return error_with_text(400, "File ID required."), Err(resp) => return resp, }; @@ -207,13 +228,16 @@ impl Repository { /// POST inserts a new record /// PUT updates a record fn serve_post_put(&mut self, req: &Request) -> Response { + let is_post = req.method() == "POST"; + let is_put = req.method() == "PUT"; + // Post ID is empty for POST, set for PUT let post_id = match self.request_to_post_id(req, true) { Ok(pid) => { - if req.method() == "PUT" && pid.is_none() { + if is_put && pid.is_none() { warn!("PUT without ID!"); return error_with_text(400, "PUT requires a file ID!"); - } else if req.method() == "POST" && pid.is_some() { + } else if is_post && pid.is_some() { warn!("POST with ID!"); return error_with_text(400, "Use PUT to update a file!"); } @@ -232,11 +256,17 @@ impl Repository { body.take(self.config.max_file_size as u64 + 1) .read_to_end(&mut data) .unwrap(); - if data.len() > self.config.max_file_size { + + if is_post && data.len() == 0 { + warn!("Empty body!"); + return error_with_text(400, "Empty body!"); + } else if data.len() > self.config.max_file_size { + warn!("Upload too large!"); return empty_error(413); } } else { - return error_with_text(400, "Empty body!"); + // Should not be possible + panic!("Req data None!"); } // Convert "application/x-www-form-urlencoded" to text/plain (CURL uses this) @@ -247,7 +277,13 @@ impl Repository { Some(v) => Some(v), }; - let expiry = match req.header(HDR_EXPIRES) { + let expiry = req.get_param(GET_EXPIRY); + let mut expiry_s = expiry.as_ref().map(|s| s.as_str()); + if expiry_s.is_none() { + expiry_s = req.header(HDR_EXPIRY); + } + + let expiry = match expiry_s { Some(text) => match text.parse() { Ok(v) => { let dur = Duration::from_secs(v); @@ -266,31 +302,38 @@ impl Repository { Err(_) => { return error_with_text( 400, - "Malformed \"X-Expires\", use relative time in seconds.", + "Malformed expiration, use relative time in seconds.", ); } }, None => None, }; - if let Some(id) = post_id { + let the_id; + + let resp = if let Some(id) = post_id { // UPDATE self.update(id, data, mime, expiry); + the_id = id; Response::text("Updated OK.") } else { // INSERT - let (id, token) = self.insert(data, mime, expiry.unwrap_or(self.config.default_expiry)); + the_id = self.insert(data, mime, expiry.unwrap_or(self.config.default_expiry)); + Response::text(format!("{:016x}", the_id)) + }; - Response::text(format!("{:016x}", id)) - .with_additional_header("X-Secret", format!("{:016x}", token)) - } + let post = self.posts.get(&the_id).unwrap(); + + resp + .with_additional_header(HDR_SECRET, format!("{:016x}", post.secret)) + .with_additional_header(HDR_EXPIRY, post.time_remains().as_secs().to_string()) } /// Serve a GET or HEAD request fn serve_get_head(&mut self, req: &Request) -> Response { let post_id = match self.request_to_post_id(req, false) { Ok(Some(pid)) => pid, - Ok(None) => return error_with_text(400, "Post ID required."), + Ok(None) => return error_with_text(400, "File ID required."), Err(resp) => return resp, }; @@ -305,12 +348,25 @@ impl Repository { return error_with_text(500, "File data lost."); } + let seconds_remains = post.expires.signed_duration_since(Utc::now()) + .num_seconds(); + Response { status_code: 200, - headers: vec![( - "Content-Type".into(), - format!("{}; charset=utf8", post.mime).into(), - )], + headers: vec![ + ( + "Content-Type".into(), + format!("{}; charset=utf8", post.mime).into(), + ), + ( + "Cache-Control".into(), + format!("public, max-age={}", seconds_remains).into() + ), + ( + HDR_EXPIRY.into(), + seconds_remains.to_string().into() + ) + ], data: if req.method() == "HEAD" { ResponseBody::empty() } else { @@ -361,30 +417,38 @@ impl Repository { None => { warn!("ID {} does not exist!", id); return Err(error_with_text(404, "No file with this ID!")); - }, + } Some(post) => { if post.is_expired() { warn!("Access of expired file {}!", id); return Err(error_with_text(404, "No file with this ID!")); } - let secret: u64 = match req.header(HDR_SECRET).map(|v| u64::from_str_radix(v, 16)) { - Some(Ok(bytes)) => bytes, - None => { - warn!("Missing secret token!"); - return Err(error_with_text(400, "Secret token required!")); - } - Some(Err(e)) => { - warn!("Token parse error: {:?}", e); - return Err(error_with_text(400, "Bad secret token format!")); - }, - }; + let secret = req.get_param(GET_SECRET); + let mut secret_str = secret.as_ref().map(|s| s.as_str()); + + if secret_str.is_none() { + secret_str = req.header(HDR_SECRET); + } + + let secret: u64 = + match secret_str.map(|v| u64::from_str_radix(v, 16)) { + Some(Ok(bytes)) => bytes, + None => { + warn!("Missing secret token!"); + return Err(error_with_text(400, "Secret token required!")); + } + Some(Err(e)) => { + warn!("Token parse error: {:?}", e); + return Err(error_with_text(400, "Bad secret token format!")); + } + }; if post.secret != secret { warn!("Secret token mismatch"); return Err(error_with_text(401, "Invalid secret token!")); } - }, + } } } @@ -394,7 +458,12 @@ impl Repository { /// Drop expired posts, if cleaning is due fn gc_expired_posts_if_needed(&mut self) { - if Utc::now().signed_duration_since(self.last_gc_time).to_std().unwrap_or_default() > self.config.expired_gc_interval { + if Utc::now() + .signed_duration_since(self.last_gc_time) + .to_std() + .unwrap_or_default() + > self.config.expired_gc_interval + { self.gc_expired_posts(); self.last_gc_time = Utc::now(); } @@ -425,7 +494,7 @@ impl Repository { /// Get hash of a byte vector (for deduplication) fn hash_data(data: &Vec) -> DataHash { - let mut hasher = siphasher::sip::SipHasher::new(); + let mut hasher = SipHasher::new(); data.hash(&mut hasher); hasher.finish() } @@ -462,7 +531,7 @@ impl Repository { } /// Insert a post - fn insert(&mut self, data: Vec, mime: Option<&str>, expires: Duration) -> (PostId, Secret) { + fn insert(&mut self, data: Vec, mime: Option<&str>, expires: Duration) -> PostId { info!( "Insert post with data of len {} bytes, mime {}, expiry {:?}", data.len(), @@ -473,12 +542,8 @@ impl Repository { let hash = Self::hash_data(&data); let mime = match mime { - None => { - Mime::from(tree_magic::from_u8(&data)) - }, - Some(explicit) => { - Mime::from(explicit) - }, + None => Mime::from(tree_magic::from_u8(&data)), + Some(explicit) => Mime::from(explicit), }; Self::store_data_or_increment_rc(&mut self.data, hash, data); @@ -492,7 +557,7 @@ impl Repository { let secret = OsRng.gen(); - debug!("Post ID = #{:016x}", post_id); + debug!("File ID = #{:016x} (http://{}:{}/{:016x})", post_id, self.config.host, self.config.port, post_id); debug!("Data hash = #{:016x}, mime {}", hash, mime); debug!("Secret = #{:016x}", secret); @@ -508,7 +573,7 @@ impl Repository { self.dirty = true; - (post_id, secret) + post_id } /// Update a post by ID @@ -523,18 +588,21 @@ impl Repository { .unwrap_or("unchanged".into()) ); - let hash = Self::hash_data(&data); let post = self.posts.get_mut(&id).unwrap(); // post existence was checked before - if hash != post.hash { - debug!("Data hash = #{:016x} (content changed)", hash); + if !data.is_empty() { + let hash = Self::hash_data(&data); - Self::drop_data_or_decrement_rc(&mut self.data, post.hash); - Self::store_data_or_increment_rc(&mut self.data, hash, data); - post.hash = hash; - self.dirty = true; - } else { - debug!("Data hash = #{:016x} (no change)", hash); + if hash != post.hash { + debug!("Data hash = #{:016x} (content changed)", hash); + + Self::drop_data_or_decrement_rc(&mut self.data, post.hash); + Self::store_data_or_increment_rc(&mut self.data, hash, data); + post.hash = hash; + self.dirty = true; + } else { + debug!("Data hash = #{:016x} (no change)", hash); + } } if let Some(mime) = mime { @@ -566,22 +634,25 @@ impl Repository { /// Serialize chrono unix timestamp as seconds mod serde_chrono_datetime_as_unix { + use chrono::{DateTime, NaiveDateTime, Utc}; use serde::{self, Deserialize, Deserializer, Serializer}; - use chrono::{DateTime, Utc, NaiveDateTime}; pub fn serialize(value: &DateTime, se: S) -> Result - where - S: Serializer, + where + S: Serializer, { se.serialize_i64(value.naive_utc().timestamp()) } pub fn deserialize<'de, D>(de: D) -> Result, D::Error> - where - D: Deserializer<'de>, + where + D: Deserializer<'de>, { let ts: i64 = i64::deserialize(de)?; - Ok(DateTime::from_utc(NaiveDateTime::from_timestamp(ts, 0), Utc)) + Ok(DateTime::from_utc( + NaiveDateTime::from_timestamp(ts, 0), + Utc, + )) } } diff --git a/src/well_known_mime.rs b/src/well_known_mime.rs index 09cf482..e3c5f0c 100644 --- a/src/well_known_mime.rs +++ b/src/well_known_mime.rs @@ -1,18 +1,20 @@ -use serde::{Serialize, Serializer, Deserialize, Deserializer}; -use std::fmt::{Write, Display, Formatter}; use std::fmt; -use serde::de::Visitor; +use std::fmt::{Display, Formatter}; +use lazy_static::lazy_static; -#[derive(Serialize,Deserialize,Debug,PartialEq,Eq,Hash)] +/// Mime type +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash)] pub enum Mime { + /// Well-known mime WellKnown(usize), + /// Custom mime Custom(String), } impl From for Mime { fn from(s: String) -> Self { - if let Ok(index) = WELL_KNOWN.binary_search(&s.as_str()) { - Mime::WellKnown(index) + if let Some(index) = MIME_LOOKUP.get(&s.as_str()) { + Mime::WellKnown(*index) } else { Mime::Custom(s) } @@ -21,8 +23,8 @@ impl From for Mime { impl<'a> From<&'a str> for Mime { fn from(s: &'a str) -> Self { - if let Ok(index) = WELL_KNOWN.binary_search(&s) { - Mime::WellKnown(index) + if let Some(index) = MIME_LOOKUP.get(&s) { + Mime::WellKnown(*index) } else { Mime::Custom(s.to_string()) } @@ -38,15 +40,16 @@ impl Display for Mime { } else { f.write_str("application/octet-stream") } - }, - Mime::Custom(s) => { - f.write_str(s) - }, + } + Mime::Custom(s) => f.write_str(s), } } } -// CAUTION!!!!! This list must be alphabetically sorted! +// The positions in this list must be kept stable - otherwise the persistence file may +// deserialize to the wrong type. +// +// If a new type needs to be added, add it at the end. const WELL_KNOWN : &[&str] = &[ "application/andrew-inset", "application/applixware", @@ -739,4 +742,18 @@ const WELL_KNOWN : &[&str] = &[ "video/x-msvideo", "video/x-sgi-movie", "x-conference/x-cooltalk", + // Extras + "application/toml", + "application/json5", ]; + +use std::collections::HashMap; +lazy_static!{ + static ref MIME_LOOKUP : HashMap<&'static str, usize> = { + let mut map = HashMap::new(); + for (n, entry) in WELL_KNOWN.iter().enumerate() { + map.insert(*entry, n); + } + map + }; +}