add a readme, remove junk

.gitignore

@@ -1,3 +1,4 @@
 /target
 .idea/
 postit.json
+postit.db

Cargo.lock

@@ -576,52 +576,6 @@ dependencies = [
  "memchr 1.0.2",
 ]
 
-[[package]]
-name = "num"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
-dependencies = [
- "num-bigint",
- "num-complex",
- "num-integer",
- "num-iter",
- "num-rational",
- "num-traits",
-]
-
-[[package]]
-name = "num-bigint"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
-dependencies = [
- "autocfg 1.0.0",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-complex"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
-dependencies = [
- "autocfg 1.0.0",
- "num-traits",
-]
-
-[[package]]
-name = "num-derive"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c8b15b261814f992e33760b1fca9fe8b693d8a65299f20c9901688636cfb746"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "num-integer"
 version = "0.1.42"
@@ -632,29 +586,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "num-iter"
-version = "0.1.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfb0800a0291891dd9f4fe7bd9c19384f98f7fbe0cd0f39a2c6b88b9868bbc00"
-dependencies = [
- "autocfg 1.0.0",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-rational"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
-dependencies = [
- "autocfg 1.0.0",
- "num-bigint",
- "num-integer",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.11"
@@ -811,9 +742,8 @@ dependencies = [
  "chrono",
  "clappconfig",
  "flate2",
+ "lazy_static",
  "log 0.4.8",
- "num",
- "num-derive",
  "parking_lot",
  "rand 0.7.3",
  "rouille",

Cargo.toml

@@ -3,6 +3,7 @@ name = "postit"
 version = "0.1.0"
 authors = ["Ondřej Hruška <ondra@ondrovo.com>"]
 edition = "2018"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -21,5 +22,4 @@ bincode = "1.2.1"
 flate2 = "1.0.14"
 anyhow = "1.0.28"
 tree_magic = { version = "0.2.3", default_features = false, features = ["staticmime"] }
-num = "0.2.1"
+lazy_static = "1.4.0"
-num-derive = "0.3.0"

README.md

@@ -0,0 +1,79 @@
+# PostIt file sharing server
+
+PostIt is designed to work as a temporary public storage for text (and other)
+files uploaded to it by software that need a publicly reachable page without 
+hosting its own server or even having a public IP.
+
+The primary use case is to share diagnostic and contextual information
+produced by Fediverse bots (think an interactive game where the game board 
+is rendered to an image or text file on demand). There are sure to be many 
+other uses I didn't think of.
+
+The uploaded files have a lifetime of 10 minutes, which can be shortened or 
+extended up to 1 hour (or more, as configured).
+
+## Uploading a file
+
+To upload a file, send a POST request to the running PostIt server.
+
+```none
+$ curl -X POST --data-binary @RICKROLL.txt 0.0.0.0:7745 -i
+HTTP/1.1 200 OK
+X-Secret: 5273d775746e393b
+X-Expire: 599
+Content-Length: 16
+
+421d082ef85827ea
+```
+
+Take note of the `X-Secret` header, you will need it to update or delete the file. 
+
+If you only want to share the file, this is all you need. Grab the file ID from the response body
+and share it. The URL is `/<FILE_ID>`, e.g.
+
+```none
+$ curl -X GET 0.0.0.0:7745/421d082ef85827ea -i
+HTTP/1.1 200 OK
+Content-Type: text/plain; charset=utf8
+X-Expire: 459
+Content-Length: 688
+
+File content here...
+```
+
+### Content type
+
+The server attempts to auto-detect the file's `Content-Type`. The fallback is `text/plain`.
+If you wish to set a custom type, use the `Content-Type` header when uploading the file, e.g.
+
+```
+$ curl -X POST --data-binary @RICKROLL.txt 0.0.0.0:7745 -i -H 'Content-Type: application/json'
+```
+
+### Expiration time
+
+To customize the expiration time, use the header `X-Expire: <seconds>`, or a GET argument `?expire=<seconds>` e.g. 
+
+```
+$ curl -X POST --data-binary @RICKROLL.txt 0.0.0.0:7745 -i -H 'X-Expire: 60'
+```
+
+## Updating a file
+
+A file you uploaded can be deleted or modified using the **secret token** obtained in respose to its upload.
+Send the token as the `X-Secret` header, or GET argument `?secret=....`
+
+File is updated by sending a `PUT` request to the file's URL.
+
+The `PUT` request can change file expiration (`X-Expire: <secs>` or GET arg `expire`), 
+update its `Content-Type`, or replace its content.
+
+Note that sending `PUT` with empty body will *not* clear the file, in that case the file content is
+not changed at all. This can be used to extend file's expiration without changing it in any other way
+(by sending the `X-Expire` header). 
+
+## Deleting a file
+
+The `DELETE` verb, unsurprisingly, deletes a file. As with `PUT`, the secret token is required.
+
+.

src/config.rs

@@ -40,7 +40,7 @@ pub(crate) struct Config {
     pub(crate) compression: bool,
 
     /// Persistence file
-    pub(crate) persist_file : String,
+    pub(crate) persist_file: String,
 }
 
 impl Default for Config {

src/main.rs

@@ -2,36 +2,38 @@
 extern crate serde_derive;
 #[macro_use]
 extern crate log;
-#[macro_use]
-extern crate num_derive;
 
 use crate::config::Config;
+use crate::well_known_mime::Mime;
+use chrono::{DateTime, Utc};
 use clappconfig::{anyhow, AppConfig};
 use parking_lot::Mutex;
 use rand::rngs::OsRng;
 use rand::Rng;
 use rouille::{Request, Response, ResponseBody};
+
 use std::borrow::Cow;
 use std::collections::HashMap;
+
+use std::fs::OpenOptions;
 use std::hash::{Hash, Hasher};
 use std::io::Read;
 use std::time::Duration;
-use chrono::{Utc, DateTime};
+use siphasher::sip::SipHasher;
-use std::fs::OpenOptions;
-use std::fmt::Display;
-use serde::export::Formatter;
-use std::fmt;
-use serde::{Serialize, Serializer, Deserialize, Deserializer};
-use serde::de::{DeserializeOwned, Visitor};
-use crate::well_known_mime::Mime;
 
 mod config;
 mod well_known_mime;
 
-const HDR_EXPIRES : &str = "X-Expires";
+/// Header to set expiry (seconds)
-const HDR_SECRET : &str = "X-Secret";
+const HDR_EXPIRY: &str = "X-Expire";
+/// Header to pass secret token for update/delete
+const HDR_SECRET: &str = "X-Secret";
+/// GET param to pass secret token (as a substitute for header)
+const GET_EXPIRY: &str = "expire";
+/// GET param to pass secret token (as a substitute for header)
+const GET_SECRET: &str = "secret";
 
-const FAVICON : &[u8] = include_bytes!("favicon.ico");
+const FAVICON: &[u8] = include_bytes!("favicon.ico");
 
 /// Post ID (represented as a 16-digit hex string)
 type PostId = u64;
@@ -41,7 +43,7 @@ type Secret = u64;
 type DataHash = u64;
 
 /// Post stored in the repository
-#[derive(Debug,Serialize,Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 struct Post {
     /// Content-Type
     mime: Mime,
@@ -59,6 +61,17 @@ impl Post {
     pub fn is_expired(&self) -> bool {
         self.expires < Utc::now()
     }
+
+    /// Get remaining lifetime
+    pub fn time_remains(&self) -> Duration {
+        let seconds_remains = self.expires.signed_duration_since(Utc::now())
+            .num_seconds();
+        if seconds_remains < 0 {
+            Duration::from_secs(0)
+        } else {
+            Duration::from_secs(seconds_remains as u64)
+        }
+    }
 }
 
 fn main() -> anyhow::Result<()> {
@@ -72,6 +85,7 @@ fn main() -> anyhow::Result<()> {
                 error!("Load failed: {}", e);
             }
         }
+        store.gc_expired_posts();
         store
     });
 
@@ -82,7 +96,7 @@ fn main() -> anyhow::Result<()> {
         info!("{} {}", method, req.raw_url());
 
         if req.url() == "/favicon.ico" {
-            return Response::from_data("image/vnd.microsoft.icon", FAVICON);
+            return decorate_response(Response::from_data("image/vnd.microsoft.icon", FAVICON));
         }
 
         store_w.gc_expired_posts_if_needed();
@@ -104,14 +118,21 @@ fn main() -> anyhow::Result<()> {
             warn!("Error resp: {}", resp.status_code);
         }
 
-        resp
+        decorate_response(resp)
     });
 }
 
+fn decorate_response(resp : Response) -> Response {
+    resp.without_header("Server")
+        .with_additional_header("Server", "postit.rs")
+        .with_additional_header("Access-Control-Allow-Origin", "*")
+        .with_additional_header("X-Version", env!("CARGO_PKG_VERSION"))
+}
+
 type PostsMap = HashMap<PostId, Post>;
 type DataMap = HashMap<DataHash, (usize, Vec<u8>)>;
 
-#[derive(Debug,Serialize,Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 struct Repository {
     #[serde(skip)]
     config: Config,
@@ -175,7 +196,7 @@ impl Repository {
             .read(true)
             .open(&self.config.persist_file)?;
 
-        let result : Repository = if self.config.compression {
+        let result: Repository = if self.config.compression {
             let flate = flate2::read::DeflateDecoder::new(file);
             bincode::deserialize_from(flate)?
         } else {
@@ -193,7 +214,7 @@ impl Repository {
     fn serve_delete(&mut self, req: &Request) -> Response {
         let post_id = match self.request_to_post_id(req, true) {
             Ok(Some(pid)) => pid,
-            Ok(None) => return error_with_text(400, "Post ID required."),
+            Ok(None) => return error_with_text(400, "File ID required."),
             Err(resp) => return resp,
         };
 
@@ -207,13 +228,16 @@ impl Repository {
     /// POST inserts a new record
     /// PUT updates a record
     fn serve_post_put(&mut self, req: &Request) -> Response {
+        let is_post = req.method() == "POST";
+        let is_put = req.method() == "PUT";
+
         // Post ID is empty for POST, set for PUT
         let post_id = match self.request_to_post_id(req, true) {
             Ok(pid) => {
-                if req.method() == "PUT" && pid.is_none() {
+                if is_put && pid.is_none() {
                     warn!("PUT without ID!");
                     return error_with_text(400, "PUT requires a file ID!");
-                } else if req.method() == "POST" && pid.is_some() {
+                } else if is_post && pid.is_some() {
                     warn!("POST with ID!");
                     return error_with_text(400, "Use PUT to update a file!");
                 }
@@ -232,11 +256,17 @@ impl Repository {
             body.take(self.config.max_file_size as u64 + 1)
                 .read_to_end(&mut data)
                 .unwrap();
-            if data.len() > self.config.max_file_size {
+
+            if is_post && data.len() == 0 {
+                warn!("Empty body!");
+                return error_with_text(400, "Empty body!");
+            } else if data.len() > self.config.max_file_size {
+                warn!("Upload too large!");
                 return empty_error(413);
             }
         } else {
-            return error_with_text(400, "Empty body!");
+            // Should not be possible
+            panic!("Req data None!");
         }
 
         // Convert "application/x-www-form-urlencoded" to text/plain (CURL uses this)
@@ -247,7 +277,13 @@ impl Repository {
             Some(v) => Some(v),
         };
 
-        let expiry = match req.header(HDR_EXPIRES) {
+        let expiry = req.get_param(GET_EXPIRY);
+        let mut expiry_s = expiry.as_ref().map(|s| s.as_str());
+        if expiry_s.is_none() {
+            expiry_s = req.header(HDR_EXPIRY);
+        }
+
+        let expiry = match expiry_s {
             Some(text) => match text.parse() {
                 Ok(v) => {
                     let dur = Duration::from_secs(v);
@@ -266,31 +302,38 @@ impl Repository {
                 Err(_) => {
                     return error_with_text(
                         400,
-                        "Malformed \"X-Expires\", use relative time in seconds.",
+                        "Malformed expiration, use relative time in seconds.",
                     );
                 }
             },
             None => None,
         };
 
-        if let Some(id) = post_id {
+        let the_id;
+
+        let resp = if let Some(id) = post_id {
             // UPDATE
             self.update(id, data, mime, expiry);
+            the_id = id;
             Response::text("Updated OK.")
         } else {
             // INSERT
-            let (id, token) = self.insert(data, mime, expiry.unwrap_or(self.config.default_expiry));
+            the_id = self.insert(data, mime, expiry.unwrap_or(self.config.default_expiry));
+            Response::text(format!("{:016x}", the_id))
+        };
 
-            Response::text(format!("{:016x}", id))
+        let post = self.posts.get(&the_id).unwrap();
-                .with_additional_header("X-Secret", format!("{:016x}", token))
+
-        }
+        resp
+            .with_additional_header(HDR_SECRET, format!("{:016x}", post.secret))
+            .with_additional_header(HDR_EXPIRY, post.time_remains().as_secs().to_string())
     }
 
     /// Serve a GET or HEAD request
     fn serve_get_head(&mut self, req: &Request) -> Response {
         let post_id = match self.request_to_post_id(req, false) {
             Ok(Some(pid)) => pid,
-            Ok(None) => return error_with_text(400, "Post ID required."),
+            Ok(None) => return error_with_text(400, "File ID required."),
             Err(resp) => return resp,
         };
 
@@ -305,12 +348,25 @@ impl Repository {
                     return error_with_text(500, "File data lost.");
                 }
 
+                let seconds_remains = post.expires.signed_duration_since(Utc::now())
+                    .num_seconds();
+
                 Response {
                     status_code: 200,
-                    headers: vec![(
+                    headers: vec![
-                        "Content-Type".into(),
+                        (
-                        format!("{}; charset=utf8", post.mime).into(),
+                            "Content-Type".into(),
-                    )],
+                            format!("{}; charset=utf8", post.mime).into(),
+                        ),
+                        (
+                            "Cache-Control".into(),
+                            format!("public, max-age={}", seconds_remains).into()
+                        ),
+                        (
+                            HDR_EXPIRY.into(),
+                            seconds_remains.to_string().into()
+                        )
+                    ],
                     data: if req.method() == "HEAD" {
                         ResponseBody::empty()
                     } else {
@@ -361,30 +417,38 @@ impl Repository {
                 None => {
                     warn!("ID {} does not exist!", id);
                     return Err(error_with_text(404, "No file with this ID!"));
-                },
+                }
                 Some(post) => {
                     if post.is_expired() {
                         warn!("Access of expired file {}!", id);
                         return Err(error_with_text(404, "No file with this ID!"));
                     }
 
-                    let secret: u64 = match req.header(HDR_SECRET).map(|v| u64::from_str_radix(v, 16)) {
+                    let secret = req.get_param(GET_SECRET);
-                        Some(Ok(bytes)) => bytes,
+                    let mut secret_str = secret.as_ref().map(|s| s.as_str());
-                        None => {
+
-                            warn!("Missing secret token!");
+                    if secret_str.is_none() {
-                            return Err(error_with_text(400, "Secret token required!"));
+                        secret_str = req.header(HDR_SECRET);
-                        }
+                    }
-                        Some(Err(e)) => {
+
-                            warn!("Token parse error: {:?}", e);
+                    let secret: u64 =
-                            return Err(error_with_text(400, "Bad secret token format!"));
+                        match secret_str.map(|v| u64::from_str_radix(v, 16)) {
-                        },
+                            Some(Ok(bytes)) => bytes,
-                    };
+                            None => {
+                                warn!("Missing secret token!");
+                                return Err(error_with_text(400, "Secret token required!"));
+                            }
+                            Some(Err(e)) => {
+                                warn!("Token parse error: {:?}", e);
+                                return Err(error_with_text(400, "Bad secret token format!"));
+                            }
+                        };
 
                     if post.secret != secret {
                         warn!("Secret token mismatch");
                         return Err(error_with_text(401, "Invalid secret token!"));
                     }
-                },
+                }
             }
         }
 
@@ -394,7 +458,12 @@ impl Repository {
 
     /// Drop expired posts, if cleaning is due
     fn gc_expired_posts_if_needed(&mut self) {
-        if Utc::now().signed_duration_since(self.last_gc_time).to_std().unwrap_or_default() > self.config.expired_gc_interval {
+        if Utc::now()
+            .signed_duration_since(self.last_gc_time)
+            .to_std()
+            .unwrap_or_default()
+            > self.config.expired_gc_interval
+        {
             self.gc_expired_posts();
             self.last_gc_time = Utc::now();
         }
@@ -425,7 +494,7 @@ impl Repository {
 
     /// Get hash of a byte vector (for deduplication)
     fn hash_data(data: &Vec<u8>) -> DataHash {
-        let mut hasher = siphasher::sip::SipHasher::new();
+        let mut hasher = SipHasher::new();
         data.hash(&mut hasher);
         hasher.finish()
     }
@@ -462,7 +531,7 @@ impl Repository {
     }
 
     /// Insert a post
-    fn insert(&mut self, data: Vec<u8>, mime: Option<&str>, expires: Duration) -> (PostId, Secret) {
+    fn insert(&mut self, data: Vec<u8>, mime: Option<&str>, expires: Duration) -> PostId {
         info!(
             "Insert post with data of len {} bytes, mime {}, expiry {:?}",
             data.len(),
@@ -473,12 +542,8 @@ impl Repository {
         let hash = Self::hash_data(&data);
 
         let mime = match mime {
-            None => {
+            None => Mime::from(tree_magic::from_u8(&data)),
-                Mime::from(tree_magic::from_u8(&data))
+            Some(explicit) => Mime::from(explicit),
-            },
-            Some(explicit) => {
-                Mime::from(explicit)
-            },
         };
 
         Self::store_data_or_increment_rc(&mut self.data, hash, data);
@@ -492,7 +557,7 @@ impl Repository {
 
         let secret = OsRng.gen();
 
-        debug!("Post ID = #{:016x}", post_id);
+        debug!("File ID = #{:016x} (http://{}:{}/{:016x})", post_id, self.config.host, self.config.port, post_id);
         debug!("Data hash = #{:016x}, mime {}", hash, mime);
         debug!("Secret = #{:016x}", secret);
 
@@ -508,7 +573,7 @@ impl Repository {
 
         self.dirty = true;
 
-        (post_id, secret)
+        post_id
     }
 
     /// Update a post by ID
@@ -523,18 +588,21 @@ impl Repository {
                 .unwrap_or("unchanged".into())
         );
 
-        let hash = Self::hash_data(&data);
         let post = self.posts.get_mut(&id).unwrap(); // post existence was checked before
 
-        if hash != post.hash {
+        if !data.is_empty() {
-            debug!("Data hash = #{:016x} (content changed)", hash);
+            let hash = Self::hash_data(&data);
 
-            Self::drop_data_or_decrement_rc(&mut self.data, post.hash);
+            if hash != post.hash {
-            Self::store_data_or_increment_rc(&mut self.data, hash, data);
+                debug!("Data hash = #{:016x} (content changed)", hash);
-            post.hash = hash;
+
-            self.dirty = true;
+                Self::drop_data_or_decrement_rc(&mut self.data, post.hash);
-        } else {
+                Self::store_data_or_increment_rc(&mut self.data, hash, data);
-            debug!("Data hash = #{:016x} (no change)", hash);
+                post.hash = hash;
+                self.dirty = true;
+            } else {
+                debug!("Data hash = #{:016x} (no change)", hash);
+            }
         }
 
         if let Some(mime) = mime {
@@ -566,22 +634,25 @@ impl Repository {
 
 /// Serialize chrono unix timestamp as seconds
 mod serde_chrono_datetime_as_unix {
+    use chrono::{DateTime, NaiveDateTime, Utc};
     use serde::{self, Deserialize, Deserializer, Serializer};
-    use chrono::{DateTime, Utc, NaiveDateTime};
 
     pub fn serialize<S>(value: &DateTime<Utc>, se: S) -> Result<S::Ok, S::Error>
-        where
+    where
-            S: Serializer,
+        S: Serializer,
     {
         se.serialize_i64(value.naive_utc().timestamp())
     }
 
     pub fn deserialize<'de, D>(de: D) -> Result<DateTime<Utc>, D::Error>
-        where
+    where
-            D: Deserializer<'de>,
+        D: Deserializer<'de>,
     {
         let ts: i64 = i64::deserialize(de)?;
-        Ok(DateTime::from_utc(NaiveDateTime::from_timestamp(ts, 0), Utc))
+        Ok(DateTime::from_utc(
+            NaiveDateTime::from_timestamp(ts, 0),
+            Utc,
+        ))
     }
 }
 

src/well_known_mime.rs

@@ -1,18 +1,20 @@
-use serde::{Serialize, Serializer, Deserialize, Deserializer};
-use std::fmt::{Write, Display, Formatter};
 use std::fmt;
-use serde::de::Visitor;
+use std::fmt::{Display, Formatter};
+use lazy_static::lazy_static;
 
-#[derive(Serialize,Deserialize,Debug,PartialEq,Eq,Hash)]
+/// Mime type
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
 pub enum Mime {
+    /// Well-known mime
     WellKnown(usize),
+    /// Custom mime
     Custom(String),
 }
 
 impl From<String> for Mime {
     fn from(s: String) -> Self {
-        if let Ok(index) = WELL_KNOWN.binary_search(&s.as_str()) {
+        if let Some(index) = MIME_LOOKUP.get(&s.as_str()) {
-            Mime::WellKnown(index)
+            Mime::WellKnown(*index)
         } else {
             Mime::Custom(s)
         }
@@ -21,8 +23,8 @@ impl From<String> for Mime {
 
 impl<'a> From<&'a str> for Mime {
     fn from(s: &'a str) -> Self {
-        if let Ok(index) = WELL_KNOWN.binary_search(&s) {
+        if let Some(index) = MIME_LOOKUP.get(&s) {
-            Mime::WellKnown(index)
+            Mime::WellKnown(*index)
         } else {
             Mime::Custom(s.to_string())
         }
@@ -38,15 +40,16 @@ impl Display for Mime {
                 } else {
                     f.write_str("application/octet-stream")
                 }
-            },
+            }
-            Mime::Custom(s) => {
+            Mime::Custom(s) => f.write_str(s),
-                f.write_str(s)
-            },
         }
     }
 }
 
-// CAUTION!!!!! This list must be alphabetically sorted!
+// The positions in this list must be kept stable - otherwise the persistence file may
+// deserialize to the wrong type.
+//
+// If a new type needs to be added, add it at the end.
 const WELL_KNOWN : &[&str] = &[
     "application/andrew-inset",
     "application/applixware",
@@ -739,4 +742,18 @@ const WELL_KNOWN : &[&str] = &[
     "video/x-msvideo",
     "video/x-sgi-movie",
     "x-conference/x-cooltalk",
+    // Extras
+    "application/toml",
+    "application/json5",
 ];
+
+use std::collections::HashMap;
+lazy_static!{
+    static ref MIME_LOOKUP : HashMap<&'static str, usize> = {
+        let mut map = HashMap::new();
+        for (n, entry) in WELL_KNOWN.iter().enumerate() {
+            map.insert(*entry, n);
+        }
+        map
+    };
+}